Merge branch 'dev-v3.x' into zach/router-1910

Author: BobaFetters

.changesets/breaking_zach_router_1758_remove_connectors_subgraphs.md

@@ -0,0 +1,40 @@
+### Remove the deprecated `connectors.subgraphs` configuration field
+
+The deprecated `connectors.subgraphs` configuration field has been removed. The replacement `connectors.sources` field has been available since 2.x with a deprecation warning.
+
+Existing configurations are migrated automatically at startup: each `subgraphs.<subgraph_name>.sources.<source_name>` entry is collapsed into a single `sources` entry keyed by `<subgraph_name>.<source_name>`, and any `$config` block at the subgraph level is copied onto each source. The router logs a notice describing the rewrite.
+
+The migration preserves the deprecated runtime's precedence rules, including when a `connectors.sources` entry already exists for the same `<subgraph>.<source>` key: the deprecated `override_url` / `max_requests_per_operation` win over it when set, and a subgraph-level `$config` always overwrites any source-level `$config` — matching the order in which the old `apply_config` applied the two shapes.
+
+Two cases can't be fully replicated automatically, and the router logs a separate notice for them:
+
+- A subgraph entry that only declares `$config` (no `sources`) can't be expressed in the new shape, since `$config` is per-source there; its `$config` is dropped.
+- The deprecated runtime applied a subgraph's `$config` to *every* connector under that subgraph, including ones on `@source`s the deprecated config's own `sources` map didn't list. The migration can only copy `$config` onto the composite keys it can see, so if your schema declares additional sources for a migrated subgraph, copy `$config` onto their `connectors.sources` entries by hand.
+
+Before:
+
+```yaml
+connectors:
+  subgraphs:
+    my_subgraph:
+      $config:
+        api_key: "secret"
+      sources:
+        my_source:
+          override_url: https://example.com
+          max_requests_per_operation: 50
+```
+
+After (equivalent shape, produced automatically):
+
+```yaml
+connectors:
+  sources:
+    my_subgraph.my_source:
+      $config:
+        api_key: "secret"
+      override_url: https://example.com
+      max_requests_per_operation: 50
+```
+
+By [@BobaFetters](https://github.com/BobaFetters) in https://github.com/apollographql/router/pull/9520

.changesets/feat_reduce_exported_apollo_traces.md

@@ -0,0 +1,12 @@
+### Reduce the volume of traces sent to Apollo
+
+The Router now caps the volume of traces it exports to Apollo (GraphOS) with a throttle that runs after the configured trace samplers. This helps to avoid the processing and transfer of traces that would be dropped by GraphOS at ingestion time anyway. A new option, `telemetry.apollo.tracing.throttle`, selects the strategy:
+
+- `representative_traces` (default): Sends at most one representative trace per minute for each distinct combination of operation, client, latency bucket, error status, and operation type. Duplicate traces that share a combination already seen in the current minute are dropped. Error traces are retained at a higher rate. This mirrors the representative-trace filtering used by Apollo's engine reports pipeline.
+- `rate_limited`: Sends every trace but caps the export rate at a fixed maximum of 100 traces per second per Router instance. This is cheaper in CPU and memory than `representative_traces`. The rate is not configurable.
+
+This throttle applies only to the trace pipeline that exports to Apollo Studio, and only to whatever the existing head samplers (`telemetry.apollo.sampler` and `telemetry.exporters.tracing.common.sampler`) already let through — it can only further reduce that traffic. Traces exported to a customer's own OTLP collector or to Datadog are unaffected. The existing `telemetry.apollo.sampler` continues to work as before, and to disable Apollo trace export entirely, set it to `always_off`.
+
+Independently of the throttle, the Router now also always drops any individual trace larger than 10MB before exporting it to Apollo, since GraphOS rejects oversized traces at ingestion anyway. This limit is fixed and not configurable.
+
+By [@bonnici](https://github.com/bonnici) in https://github.com/apollographql/router/pull/9848

apollo-router/src/configuration/metrics.rs

@@ -423,6 +423,8 @@ impl InstrumentData {
             "$.tracing.batch_processor.max_export_timeout",
             opt.tracing.batch_processor.max_queue_size,
             "$.tracing.batch_processor.max_queue_size",
+            opt.tracing.throttle,
+            "$.tracing.throttle",
             opt.metrics.otlp.batch_processor.scheduled_delay,
             "$.metrics.otlp.batch_processor.scheduled_delay",
             opt.metrics.otlp.batch_processor.max_export_timeout,
@@ -447,12 +449,12 @@ impl InstrumentData {
             "$[?(@.expose_sources_in_context == true)]",
             opt.max_requests_per_operation_per_source,
             "$[?(@.max_requests_per_operation_per_source)]",
-            opt.subgraph.config,
-            "$[?(@.subgraphs..['$config'])]",
+            opt.source.config,
+            "$[?(@.sources..['$config'])]",
             opt.source.override_url,
-            "$[?(@.subgraphs..sources..override_url)]",
+            "$[?(@.sources..override_url)]",
             opt.source.max_requests_per_operation,
-            "$[?(@.subgraphs..sources..max_requests_per_operation)]"
+            "$[?(@.sources..max_requests_per_operation)]"
         );
 
         populate_config_instrument!(

apollo-router/src/configuration/migrations/0036-preview_connectors_subgraphs.yaml

@@ -21,6 +21,7 @@ expression: "& metrics.non_zero()"
           opt.tracing.batch_processor.max_export_timeout: 30s
           opt.tracing.batch_processor.max_queue_size: 2048
           opt.tracing.batch_processor.scheduled_delay: 5s
+          opt.tracing.throttle: rate_limited
 - name: apollo.router.config.experimental_local_field_metrics
   data:
     datapoints:

apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@apollo_telemetry.router.yaml.snap

@@ -1,6 +1,6 @@
 ---
 source: apollo-router/src/configuration/metrics.rs
-expression: "&metrics.non_zero()"
+expression: "& metrics.non_zero()"
 ---
 - name: apollo.router.config.connectors
   data:
@@ -10,6 +10,6 @@ expression: "&metrics.non_zero()"
           opt.debug_extensions: true
           opt.expose_sources_in_context: true
           opt.max_requests_per_operation_per_source: true
+          opt.source.config: true
           opt.source.max_requests_per_operation: true
           opt.source.override_url: true
-          opt.subgraph.config: true

apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@connectors.router.yaml.snap

@@ -21,6 +21,7 @@ expression: "& metrics.non_zero()"
           opt.tracing.batch_processor.max_export_timeout: false
           opt.tracing.batch_processor.max_queue_size: false
           opt.tracing.batch_processor.scheduled_delay: false
+          opt.tracing.throttle: false
 - name: apollo.router.config.experimental_chaos
   data:
     datapoints:

apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@experimental_features.router.yaml.snap

@@ -277,7 +277,7 @@ expression: "&schema"
               "type": "null"
             }
           ],
-          "description": "Per-exporter sampler for traces sent to Apollo Studio.\n\nUses the same trace-ID-based algorithm as `telemetry.exporters.tracing.common.sampler`.\nAccepts a decimal between 0.0 and 1.0, `always_on`, or `always_off`.\nShould be ≤ the common sampler; setting it higher has no effect.\n\nWhen `parent_based_sampler` is enabled (the default), traces arriving with a `traceparent`\nheader already marked as sampled by the calling service will be passed through to this\nexporter regardless of this sampler's value — including when set to `always_off`."
+          "description": "Per-exporter sampler for traces sent to Apollo Studio.\n\nUses the same trace-ID-based algorithm as `telemetry.exporters.tracing.common.sampler`.\nAccepts a decimal between 0.0 and 1.0, `always_on`, or `always_off`.\nShould be ≤ the common sampler; setting it higher has no effect.\n\nWhen `parent_based_sampler` is enabled (the default), traces arriving with a `traceparent`\nheader already marked as sampled by the calling service will be passed through to this\nexporter regardless of this sampler's value — including when set to `always_off`.\n\nWhatever this sampler lets through is then subject to `tracing.throttle`, which acts as a\nback-stop that further reduces trace volume sent to Apollo Studio."
         },
         "send_headers": {
           "allOf": [
@@ -320,6 +320,21 @@ expression: "&schema"
       },
       "type": "object"
     },
+    "ApolloTraceThrottleConfig": {
+      "description": "The back-stop strategy used to throttle the volume of traces exported to Apollo Studio, after\nhead sampling.",
+      "oneOf": [
+        {
+          "const": "representative_traces",
+          "description": "Send at most one representative trace per minute for each distinct combination of dimensions\n(operation, client, latency bucket, error status, operation type). Duplicate traces sharing\na combination already seen within the current minute are dropped. These traces would end up\nbeing dropped by Apollo at ingestion time anyway.",
+          "type": "string"
+        },
+        {
+          "const": "rate_limited",
+          "description": "Send every trace, but cap the export rate at a fixed maximum of 100 traces per second (per\nrouter instance). Cheaper than `representative_traces` in CPU and memory, at the cost of a\nless even sample. The rate is not configurable.",
+          "type": "string"
+        }
+      ]
+    },
     "ApolloUsageReportsBatchProcessorConfiguration": {
       "properties": {
         "max_export_timeout": {
@@ -2973,15 +2988,6 @@ expression: "&schema"
           "default": {},
           "description": "Map of subgraph_name.connector_source_name to source configuration",
           "type": "object"
-        },
-        "subgraphs": {
-          "additionalProperties": {
-            "$ref": "#/definitions/SubgraphConnectorConfiguration"
-          },
-          "default": {},
-          "deprecated": true,
-          "description": "A map of subgraph name to connectors config for that subgraph",
-          "type": "object"
         }
       },
       "type": "object"
@@ -10268,27 +10274,6 @@ expression: "&schema"
         }
       ]
     },
-    "SubgraphConnectorConfiguration": {
-      "additionalProperties": false,
-      "description": "Configuration for a connector subgraph",
-      "properties": {
-        "$config": {
-          "additionalProperties": true,
-          "default": {},
-          "description": "Other values that can be used by connectors via `{$config.<key>}`",
-          "type": "object"
-        },
-        "sources": {
-          "additionalProperties": {
-            "$ref": "#/definitions/SourceConfiguration"
-          },
-          "default": {},
-          "description": "A map of `@source(name:)` to configuration for that source",
-          "type": "object"
-        }
-      },
-      "type": "object"
-    },
     "SubgraphDeduplicationConfigConfiguration": {
       "description": "Configuration options pertaining to the subgraph server component.",
       "properties": {
@@ -12638,6 +12623,14 @@ expression: "&schema"
             }
           ],
           "description": "Configuration for tracing batch processor."
+        },
+        "throttle": {
+          "allOf": [
+            {
+              "$ref": "#/definitions/ApolloTraceThrottleConfig"
+            }
+          ],
+          "description": "Back-stop that throttles the volume of traces sent to Apollo Studio, applied *after* the\n`telemetry.apollo.sampler` and `telemetry.exporters.tracing.common.sampler` head samplers."
         }
       },
       "type": "object"

apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap

@@ -7,8 +7,9 @@ connectors:
   debug_extensions: true
   expose_sources_in_context: true
   max_requests_per_operation_per_source: 50
-  subgraphs:
-    subgraph_name:
-      sources:
-        source_name:
-          override_url: "http://localhost:5280"
+  sources:
+    subgraph_name.source_name:
+      override_url: "http://localhost:5280"
+      max_requests_per_operation: 50
+      $config:
+        api_key: secret

apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__upgrade_old_configuration@connectors_preview.router.yaml.snap

@@ -0,0 +1,23 @@
+---
+source: apollo-router/src/configuration/upgrade.rs
+expression: config
+---
+{
+  "connectors": {
+    "sources": {
+      "sub_a.src_1": {
+        "override_url": "http://one",
+        "$config": {
+          "api_key": "secret"
+        }
+      },
+      "sub_a.src_2": {
+        "override_url": "http://two",
+        "max_requests_per_operation": 5,
+        "$config": {
+          "api_key": "secret"
+        }
+      }
+    }
+  }
+}