h2m: add optional stats to header-to-metadata filter

agrawroh · agrawroh · commit c7cd3384fa48 · 2025-07-27T14:33:38.000-07:00
Signed-off-by: Rohit Agrawal &lt;rohit.agrawal@databricks.com&gt;
diff --git a/api/envoy/extensions/filters/http/header_to_metadata/v3/header_to_metadata.proto b/api/envoy/extensions/filters/http/header_to_metadata/v3/header_to_metadata.proto
@@ -138,4 +138,15 @@ message Config {
 
   // The list of rules to apply to responses.
   repeated Rule response_rules = 2;
+
+  // Optional prefix to use when emitting filter statistics. When configured, 
+  // statistics are emitted with the prefix ``http_filter_name.<stat_prefix>``.
+  //
+  // This emits statistics such as:
+  //
+  // - ``http_filter_name.my_header_converter.rules_processed``
+  // - ``http_filter_name.my_header_converter.metadata_added``
+  //
+  // If not configured, no statistics are emitted.
+  string stat_prefix = 3;
 }
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
@@ -64,5 +64,11 @@ new_features:
 - area: dns_filter, redis_proxy and prefix_matcher_map
   change: |
     Switch to using Radix Tree instead of Trie for performance improvements.
+- area: header_to_metadata
+  change: |
+    Added optional statistics collection for the Header-To-Metadata filter. When the :ref:`stat_prefix
+    <envoy_v3_api_field_extensions.filters.http.header_to_metadata.v3.Config.stat_prefix>` field is configured,
+    the filter emits detailed counters for rule processing, metadata operations, etc. See
+    :ref:`Header-To-Metadata filter statistics <config_http_filters_header_to_metadata>` for details.
 
 deprecated:
diff --git a/docs/root/configuration/http/http_filters/header_to_metadata_filter.rst b/docs/root/configuration/http/http_filters/header_to_metadata_filter.rst
@@ -20,6 +20,26 @@ A typical use case for this filter is to dynamically match requests with load ba
 subsets. For this, a given header's value would be extracted and attached to the request
 as dynamic metadata which would then be used to match a subset of endpoints.
 
+Statistics
+----------
+
+The filter can optionally emit statistics when the :ref:`stat_prefix <envoy_v3_api_field_extensions.filters.http.header_to_metadata.v3.Config.stat_prefix>` field is configured.
+These statistics are rooted at *http_filter_name.<stat_prefix>* with the following counters:
+
+.. csv-table::
+  :header: Name, Type, Description
+  :widths: 1, 1, 2
+
+  request_rules_processed, Counter, Total number of request rules processed
+  response_rules_processed, Counter, Total number of response rules processed
+  request_metadata_added, Counter, Total number of metadata entries successfully added from request headers
+  response_metadata_added, Counter, Total number of metadata entries successfully added from response headers
+  request_header_not_found, Counter, Total number of times expected request headers were missing
+  response_header_not_found, Counter, Total number of times expected response headers were missing
+  base64_decode_failed, Counter, Total number of times Base64 decoding failed
+  header_value_too_long, Counter, Total number of times header values exceeded the maximum length
+  regex_substitution_failed, Counter, Total number of times regex substitution resulted in empty values
+
 Example
 -------
 
@@ -79,7 +99,24 @@ Note that this filter also supports per route configuration:
 This can be used to either override the global configuration or if the global configuration
 is empty (no rules), it can be used to only enable the filter at a per route level.
 
-Statistics
-----------
+Configuration with Statistics
+-----------------------------
+
+To enable statistics collection, configure the ``stat_prefix`` field:
+
+.. literalinclude:: _include/header-to-metadata-filter-with-stats.yaml
+    :language: yaml
+    :lines: 25-40
+    :lineno-start: 25
+    :linenos:
+    :caption: :download:`header-to-metadata-filter-with-stats.yaml <_include/header-to-metadata-filter-with-stats.yaml>`
+
+This configuration would emit statistics such as:
+
+- ``http_filter_name.header_converter.request_rules_processed``
+- ``http_filter_name.header_converter.request_metadata_added``
+- ``http_filter_name.header_converter.response_rules_processed``
+- ``http_filter_name.header_converter.response_metadata_added``
+- ``http_filter_name.header_converter.request_header_not_found``
 
-Currently, this filter generates no statistics.
+When ``stat_prefix`` is not configured, no statistics are emitted.
diff --git a/source/extensions/filters/http/header_to_metadata/config.cc b/source/extensions/filters/http/header_to_metadata/config.cc
@@ -17,8 +17,8 @@ namespace HeaderToMetadataFilter {
 absl::StatusOr<Http::FilterFactoryCb> HeaderToMetadataConfig::createFilterFactoryFromProtoTyped(
     const envoy::extensions::filters::http::header_to_metadata::v3::Config& proto_config,
     const std::string&, Server::Configuration::FactoryContext& context) {
-  absl::StatusOr<ConfigSharedPtr> filter_config_or =
-      Config::create(proto_config, context.serverFactoryContext().regexEngine(), false);
+  absl::StatusOr<ConfigSharedPtr> filter_config_or = Config::create(
+      proto_config, context.serverFactoryContext().regexEngine(), context.scope(), false);
   RETURN_IF_ERROR(filter_config_or.status());
 
   return [filter_config = std::move(filter_config_or.value())](
@@ -32,7 +32,8 @@ absl::StatusOr<Router::RouteSpecificFilterConfigConstSharedPtr>
 HeaderToMetadataConfig::createRouteSpecificFilterConfigTyped(
     const envoy::extensions::filters::http::header_to_metadata::v3::Config& config,
     Server::Configuration::ServerFactoryContext& context, ProtobufMessage::ValidationVisitor&) {
-  absl::StatusOr<ConfigSharedPtr> config_or = Config::create(config, context.regexEngine(), true);
+  absl::StatusOr<ConfigSharedPtr> config_or =
+      Config::create(config, context.regexEngine(), context.scope(), true);
   RETURN_IF_ERROR(config_or.status());
   return std::move(config_or.value());
 }
diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc
@@ -111,15 +111,16 @@ Rule::Rule(const ProtoRule& rule, Regex::Engine& regex_engine, absl::Status& cre
 
 absl::StatusOr<ConfigSharedPtr>
 Config::create(const envoy::extensions::filters::http::header_to_metadata::v3::Config& config,
-               Regex::Engine& regex_engine, bool per_route) {
+               Regex::Engine& regex_engine, Stats::Scope& scope, bool per_route) {
   absl::Status creation_status = absl::OkStatus();
-  auto cfg = ConfigSharedPtr(new Config(config, regex_engine, per_route, creation_status));
+  auto cfg = ConfigSharedPtr(new Config(config, regex_engine, scope, per_route, creation_status));
   RETURN_IF_NOT_OK_REF(creation_status);
   return cfg;
 }
 
 Config::Config(const envoy::extensions::filters::http::header_to_metadata::v3::Config config,
-               Regex::Engine& regex_engine, const bool per_route, absl::Status& creation_status) {
+               Regex::Engine& regex_engine, Stats::Scope& scope, const bool per_route,
+               absl::Status& creation_status) {
   absl::StatusOr<bool> request_set_or =
       Config::configToVector(config.request_rules(), request_rules_, regex_engine);
   SET_AND_RETURN_IF_NOT_OK(request_set_or.status(), creation_status);
@@ -130,6 +131,11 @@ Config::Config(const envoy::extensions::filters::http::header_to_metadata::v3::C
   SET_AND_RETURN_IF_NOT_OK(response_set_or.status(), creation_status);
   response_set_ = response_set_or.value();
 
+  // Generate stats only if stat_prefix is configured (opt-in behavior).
+  if (!config.stat_prefix().empty()) {
+    stats_.emplace(generateStats(config.stat_prefix(), scope));
+  }
+
   // Note: empty configs are fine for the global config, which would be the case for enabling
   //       the filter globally without rules and then applying them at the virtual host or
   //       route level. At the virtual or route level, it makes no sense to have an empty
@@ -158,6 +164,12 @@ absl::StatusOr<bool> Config::configToVector(const ProtobufRepeatedRule& proto_ru
   return true;
 }
 
+HeaderToMetadataFilterStats Config::generateStats(const std::string& stat_prefix,
+                                                  Stats::Scope& scope) {
+  const std::string final_prefix = fmt::format("http_filter_name.{}", stat_prefix);
+  return {ALL_HEADER_TO_METADATA_FILTER_STATS(POOL_COUNTER_PREFIX(scope, final_prefix))};
+}
+
 HeaderToMetadataFilter::HeaderToMetadataFilter(const ConfigSharedPtr config) : config_(config) {}
 
 HeaderToMetadataFilter::~HeaderToMetadataFilter() = default;
@@ -166,7 +178,8 @@ Http::FilterHeadersStatus HeaderToMetadataFilter::decodeHeaders(Http::RequestHea
                                                                 bool) {
   const auto* config = getConfig();
   if (config->doRequest()) {
-    writeHeaderToMetadata(headers, config->requestRules(), *decoder_callbacks_);
+    writeHeaderToMetadata(headers, config->requestRules(), *decoder_callbacks_,
+                          HeaderDirection::Request);
   }
 
   return Http::FilterHeadersStatus::Continue;
@@ -181,7 +194,8 @@ Http::FilterHeadersStatus HeaderToMetadataFilter::encodeHeaders(Http::ResponseHe
                                                                 bool) {
   const auto* config = getConfig();
   if (config->doResponse()) {
-    writeHeaderToMetadata(headers, config->responseRules(), *encoder_callbacks_);
+    writeHeaderToMetadata(headers, config->responseRules(), *encoder_callbacks_,
+                          HeaderDirection::Response);
   }
   return Http::FilterHeadersStatus::Continue;
 }
@@ -193,21 +207,28 @@ void HeaderToMetadataFilter::setEncoderFilterCallbacks(
 
 bool HeaderToMetadataFilter::addMetadata(StructMap& struct_map, const std::string& meta_namespace,
                                          const std::string& key, std::string value, ValueType type,
-                                         ValueEncode encode) const {
+                                         ValueEncode encode, HeaderDirection direction) const {
   ProtobufWkt::Value val;
+  const auto* config = getConfig();
 
   ASSERT(!value.empty());
 
   if (value.size() >= MAX_HEADER_VALUE_LEN) {
     // Too long, go away.
     ENVOY_LOG(debug, "metadata value is too long");
+    if (config->stats().has_value()) {
+      config->stats().value().header_value_too_long_.inc();
+    }
     return false;
   }
 
   if (encode == envoy::extensions::filters::http::header_to_metadata::v3::Config::BASE64) {
     value = Base64::decodeWithoutPadding(value);
     if (value.empty()) {
       ENVOY_LOG(debug, "Base64 decode failed");
+      if (config->stats().has_value()) {
+        config->stats().value().base64_decode_failed_.inc();
+      }
       return false;
     }
   }
@@ -240,6 +261,15 @@ bool HeaderToMetadataFilter::addMetadata(StructMap& struct_map, const std::strin
   auto& keyval = struct_map[meta_namespace];
   (*keyval.mutable_fields())[key] = std::move(val);
 
+  // Increment metadata_added stat if stats are enabled.
+  if (config->stats().has_value()) {
+    if (direction == HeaderDirection::Request) {
+      config->stats().value().request_metadata_added_.inc();
+    } else {
+      config->stats().value().response_metadata_added_.inc();
+    }
+  }
+
   return true;
 }
 
@@ -249,38 +279,68 @@ const std::string& HeaderToMetadataFilter::decideNamespace(const std::string& ns
 
 // add metadata['key']= value depending on header present or missing case
 void HeaderToMetadataFilter::applyKeyValue(std::string&& value, const Rule& rule,
-                                           const KeyValuePair& keyval, StructMap& np) {
+                                           const KeyValuePair& keyval, StructMap& np,
+                                           HeaderDirection direction) {
+  const auto* config = getConfig();
+
   if (!keyval.value().empty()) {
     value = keyval.value();
   } else {
     const auto& matcher = rule.regexRewrite();
     if (matcher != nullptr) {
+      std::string original_value = value;
       value = matcher->replaceAll(value, rule.regexSubstitution());
+      // If we had a non-empty input but got an empty result from regex, it could indicate a
+      // failure.
+      if (!original_value.empty() && value.empty()) {
+        if (config->stats().has_value()) {
+          config->stats().value().regex_substitution_failed_.inc();
+        }
+      }
     }
   }
   if (!value.empty()) {
     const auto& nspace = decideNamespace(keyval.metadata_namespace());
-    addMetadata(np, nspace, keyval.key(), value, keyval.type(), keyval.encode());
+    addMetadata(np, nspace, keyval.key(), value, keyval.type(), keyval.encode(), direction);
   } else {
     ENVOY_LOG(debug, "value is empty, not adding metadata");
   }
 }
 
 void HeaderToMetadataFilter::writeHeaderToMetadata(Http::HeaderMap& headers,
                                                    const HeaderToMetadataRules& rules,
-                                                   Http::StreamFilterCallbacks& callbacks) {
+                                                   Http::StreamFilterCallbacks& callbacks,
+                                                   HeaderDirection direction) {
   StructMap structs_by_namespace;
+  const auto* config = getConfig();
 
   for (const auto& rule : rules) {
     const auto& proto_rule = rule.rule();
     absl::optional<std::string> value = rule.selector_->extract(headers);
 
+    // Increment rules_processed stat if stats are enabled.
+    if (config->stats().has_value()) {
+      if (direction == HeaderDirection::Request) {
+        config->stats().value().request_rules_processed_.inc();
+      } else {
+        config->stats().value().response_rules_processed_.inc();
+      }
+    }
+
     if (value && proto_rule.has_on_header_present()) {
       applyKeyValue(std::move(value).value_or(""), rule, proto_rule.on_header_present(),
-                    structs_by_namespace);
+                    structs_by_namespace, direction);
     } else if (!value && proto_rule.has_on_header_missing()) {
+      // Increment header_not_found stat if stats are enabled.
+      if (config->stats().has_value()) {
+        if (direction == HeaderDirection::Request) {
+          config->stats().value().request_header_not_found_.inc();
+        } else {
+          config->stats().value().response_header_not_found_.inc();
+        }
+      }
       applyKeyValue(std::move(value).value_or(""), rule, proto_rule.on_header_missing(),
-                    structs_by_namespace);
+                    structs_by_namespace, direction);
     }
   }
   // Any matching rules?
diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h
@@ -17,11 +17,37 @@ namespace Extensions {
 namespace HttpFilters {
 namespace HeaderToMetadataFilter {
 
+/**
+ * All stats for the Header-To-Metadata filter. @see stats_macros.h
+ */
+#define ALL_HEADER_TO_METADATA_FILTER_STATS(COUNTER)                                               \
+  COUNTER(request_rules_processed)                                                                 \
+  COUNTER(response_rules_processed)                                                                \
+  COUNTER(request_metadata_added)                                                                  \
+  COUNTER(response_metadata_added)                                                                 \
+  COUNTER(request_header_not_found)                                                                \
+  COUNTER(response_header_not_found)                                                               \
+  COUNTER(base64_decode_failed)                                                                    \
+  COUNTER(header_value_too_long)                                                                   \
+  COUNTER(regex_substitution_failed)
+
+/**
+ * Wrapper struct for header-to-metadata filter stats. @see stats_macros.h
+ */
+struct HeaderToMetadataFilterStats {
+  ALL_HEADER_TO_METADATA_FILTER_STATS(GENERATE_COUNTER_STRUCT)
+};
+
 using ProtoRule = envoy::extensions::filters::http::header_to_metadata::v3::Config::Rule;
 using ValueType = envoy::extensions::filters::http::header_to_metadata::v3::Config::ValueType;
 using ValueEncode = envoy::extensions::filters::http::header_to_metadata::v3::Config::ValueEncode;
 using KeyValuePair = envoy::extensions::filters::http::header_to_metadata::v3::Config::KeyValuePair;
 
+/**
+ * Enum to distinguish between request and response processing for stats collection.
+ */
+enum class HeaderDirection { Request, Response };
+
 // Interface for getting values from a cookie or a header.
 class ValueSelector {
 public:
@@ -98,18 +124,20 @@ class Config : public ::Envoy::Router::RouteSpecificFilterConfig,
 public:
   static absl::StatusOr<std::shared_ptr<Config>>
   create(const envoy::extensions::filters::http::header_to_metadata::v3::Config& config,
-         Regex::Engine& regex_engine, bool per_route = false);
+         Regex::Engine& regex_engine, Stats::Scope& scope, bool per_route = false);
 
   const HeaderToMetadataRules& requestRules() const { return request_rules_; }
   const HeaderToMetadataRules& responseRules() const { return response_rules_; }
   bool doResponse() const { return response_set_; }
   bool doRequest() const { return request_set_; }
+  const absl::optional<HeaderToMetadataFilterStats>& stats() const { return stats_; }
 
 private:
   using ProtobufRepeatedRule = Protobuf::RepeatedPtrField<ProtoRule>;
 
   Config(const envoy::extensions::filters::http::header_to_metadata::v3::Config config,
-         Regex::Engine& regex_engine, bool per_route, absl::Status& creation_status);
+         Regex::Engine& regex_engine, Stats::Scope& scope, bool per_route,
+         absl::Status& creation_status);
 
   /**
    *  configToVector is a helper function for converting from configuration (protobuf types) into
@@ -125,12 +153,22 @@ class Config : public ::Envoy::Router::RouteSpecificFilterConfig,
   static absl::StatusOr<bool> configToVector(const ProtobufRepeatedRule&, HeaderToMetadataRules&,
                                              Regex::Engine&);
 
+  /**
+   * Generate stats for the header-to-metadata filter.
+   * @param stat_prefix the prefix to use for stats.
+   * @param scope the stats scope.
+   * @return HeaderToMetadataFilterStats the generated stats.
+   */
+  static HeaderToMetadataFilterStats generateStats(const std::string& stat_prefix,
+                                                   Stats::Scope& scope);
+
   const std::string& decideNamespace(const std::string& nspace) const;
 
   HeaderToMetadataRules request_rules_;
   HeaderToMetadataRules response_rules_;
   bool response_set_;
   bool request_set_;
+  absl::optional<HeaderToMetadataFilterStats> stats_;
 };
 
 using ConfigSharedPtr = std::shared_ptr<Config>;
@@ -193,12 +231,14 @@ class HeaderToMetadataFilter : public Http::StreamFilter,
    *  @param rules the header-to-metadata mapping set in configuration.
    *  @param callbacks the callback used to fetch the StreamInfo (which is then used to get
    *                   metadata). Callable with both encoder_callbacks_ and decoder_callbacks_.
+   *  @param direction whether processing request or response headers for stats collection.
    */
   void writeHeaderToMetadata(Http::HeaderMap& headers, const HeaderToMetadataRules& rules,
-                             Http::StreamFilterCallbacks& callbacks);
+                             Http::StreamFilterCallbacks& callbacks, HeaderDirection direction);
   bool addMetadata(StructMap&, const std::string&, const std::string&, std::string, ValueType,
-                   ValueEncode) const;
-  void applyKeyValue(std::string&&, const Rule&, const KeyValuePair&, StructMap&);
+                   ValueEncode, HeaderDirection direction) const;
+  void applyKeyValue(std::string&&, const Rule&, const KeyValuePair&, StructMap&,
+                     HeaderDirection direction);
   const std::string& decideNamespace(const std::string& nspace) const;
   const Config* getConfig() const;
 };
diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc
diff --git a/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc b/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc
