GCP quota alerts via pulumi (#4296)

* refactor alertStrategy

* add quotas alert policy

* off by default in base.yaml

* remove 0 quotas

* work around google API limitation

googleapi: Error 400: Alert policies with "prometheus_query_language" condition type can only have a single condition.: provider=google-beta@9.10.0
* fix aligner

Field aggregation.perSeriesAligner had an invalid value of "ALIGN_SUM":
The aligner cannot be applied to metrics with kind GAUGE and value type
BOOL.

* threshold variable; add cluster label

* type-check baseArgs

---------

Signed-off-by: Stephen Compall <stephen.compall@digitalasset.com>

Author: stephencompall-DA

cluster/configs/shared/base.yaml

@@ -124,6 +124,8 @@ monitoring:
       sequencerRateLimits:
         rejectionRateThreshold: 0
         circuitBreakerStateThreshold: 0.5
+      gcpQuotas:
+        enabled: false
     # Alert on secrets appearing in logs (JWTs, Bearer tokens, passwords, etc.)
     # Key=value secrets (excluding masked values)
     # JWTs identified by base64 header prefix "eyJhbGc" (decodes to '{"alg')

cluster/deployment/scratchneta/config.resolved.yaml

@@ -52,6 +52,8 @@ monitoring:
         thresholdPerNamespace: 0.05
       deployment:
         pendingPeriodMinutes: 5
+      gcpQuotas:
+        enabled: false
       ingestion:
         thresholdEntriesPerBatch: 80
       loadTester:

cluster/deployment/scratchnetb/config.resolved.yaml

@@ -52,6 +52,8 @@ monitoring:
         thresholdPerNamespace: 0.05
       deployment:
         pendingPeriodMinutes: 5
+      gcpQuotas:
+        enabled: false
       ingestion:
         thresholdEntriesPerBatch: 80
       loadTester:

cluster/deployment/scratchnetc/config.resolved.yaml

@@ -52,6 +52,8 @@ monitoring:
         thresholdPerNamespace: 0.05
       deployment:
         pendingPeriodMinutes: 5
+      gcpQuotas:
+        enabled: false
       ingestion:
         thresholdEntriesPerBatch: 80
       loadTester:

cluster/deployment/scratchnetd/config.resolved.yaml

@@ -52,6 +52,8 @@ monitoring:
         thresholdPerNamespace: 0.05
       deployment:
         pendingPeriodMinutes: 5
+      gcpQuotas:
+        enabled: false
       ingestion:
         thresholdEntriesPerBatch: 80
       loadTester:

cluster/deployment/scratchnete/config.resolved.yaml

@@ -52,6 +52,8 @@ monitoring:
         thresholdPerNamespace: 0.05
       deployment:
         pendingPeriodMinutes: 5
+      gcpQuotas:
+        enabled: false
       ingestion:
         thresholdEntriesPerBatch: 80
       loadTester:

cluster/pulumi/infra/src/config.ts

@@ -76,6 +76,9 @@ const MonitoringConfigSchema = z
           rejectionRateThreshold: z.number(),
           circuitBreakerStateThreshold: z.number(),
         }),
+        gcpQuotas: z.object({
+          enabled: z.boolean(),
+        }),
       }),
       logAlerts: z.object({}).catchall(z.string()).default({}),
       loggedSecretsFilter: z.string().optional(),

cluster/pulumi/infra/src/gcpAlerts.ts

@@ -38,6 +38,18 @@ export function getNotificationChannel(
     : undefined;
 }
 
+function getAlertStrategy(notificationChannel: gcp.monitoring.NotificationChannel) {
+  return {
+    autoClose: '3600s',
+    notificationChannelStrategies: [
+      {
+        notificationChannelNames: [notificationChannel.name],
+        renotifyInterval: `${4 * 60 * 60}s`, // 4 hours
+      },
+    ],
+  };
+}
+
 export function installGcpLoggingAlerts(
   notificationChannel: gcp.monitoring.NotificationChannel
 ): void {
@@ -81,15 +93,7 @@ ${Object.keys(logAlerts)
   const alertCount = enableChaosMesh ? 50 : 1;
   const displayName = `Log warnings and errors > ${alertCount} ${CLUSTER_BASENAME}`;
   new gcp.monitoring.AlertPolicy('logsAlert', {
-    alertStrategy: {
-      autoClose: '3600s',
-      notificationChannelStrategies: [
-        {
-          notificationChannelNames: [notificationChannel.name],
-          renotifyInterval: `${4 * 60 * 60}s`, // 4 hours
-        },
-      ],
-    },
+    alertStrategy: getAlertStrategy(notificationChannel),
     combiner: 'OR',
     conditions: [
       {
@@ -156,15 +160,7 @@ ${ensureTrailingNewline(loggedSecretsFilter)}`;
 
   const displayName = `Logged secrets detected in ${CLUSTER_BASENAME}`;
   new gcp.monitoring.AlertPolicy('loggedSecretsAlert', {
-    alertStrategy: {
-      autoClose: '3600s',
-      notificationChannelStrategies: [
-        {
-          notificationChannelNames: [notificationChannel.name],
-          renotifyInterval: `${4 * 60 * 60}s`, // 4 hours
-        },
-      ],
-    },
+    alertStrategy: getAlertStrategy(notificationChannel),
     combiner: 'OR',
     conditions: [
       {
@@ -222,15 +218,7 @@ jsonPayload.state=~"STARTED"`,
 
   const displayName = `Cluster ${CLUSTER_BASENAME} is being updated`;
   new gcp.monitoring.AlertPolicy('updateClusterAlert', {
-    alertStrategy: {
-      autoClose: '3600s',
-      notificationChannelStrategies: [
-        {
-          notificationChannelNames: [notificationChannel.name],
-          renotifyInterval: `${4 * 60 * 60}s`, // 4 hours
-        },
-      ],
-    },
+    alertStrategy: getAlertStrategy(notificationChannel),
     combiner: 'OR',
     conditions: [
       {
@@ -273,15 +261,7 @@ resource.type="cloudsql_database"
 
   const displayName = `Possible CloudSQL maintenance going on in ${CLUSTER_BASENAME}`;
   new gcp.monitoring.AlertPolicy('updateCloudSQLAlert', {
-    alertStrategy: {
-      autoClose: '3600s',
-      notificationChannelStrategies: [
-        {
-          notificationChannelNames: [notificationChannel.name],
-          renotifyInterval: `${4 * 60 * 60}s`, // 4 hours
-        },
-      ],
-    },
+    alertStrategy: getAlertStrategy(notificationChannel),
     combiner: 'OR',
     conditions: [
       {
@@ -309,3 +289,93 @@ resource.type="cloudsql_database"
     notificationChannels: [notificationChannel.name],
   });
 }
+
+export function installGcpQuotaAlerts(
+  notificationChannel: gcp.monitoring.NotificationChannel
+): void {
+  const quotaUsageThreshold = 0.9;
+  const quotaUsageThresholdPercent = quotaUsageThreshold * 100;
+
+  const baseArgs: Pick<
+    gcp.monitoring.AlertPolicyArgs,
+    'alertStrategy' | 'combiner' | 'notificationChannels' | 'userLabels'
+  > = {
+    alertStrategy: getAlertStrategy(notificationChannel),
+    combiner: 'OR',
+    notificationChannels: [notificationChannel.name],
+    userLabels: { cluster: CLUSTER_BASENAME },
+  };
+
+  new gcp.monitoring.AlertPolicy('quotaExceededAlert', {
+    ...baseArgs,
+    displayName: `Quota Exceeded in ${CLUSTER_BASENAME}`,
+    conditions: [
+      {
+        // "Quota Full" (Exceeded right now)
+        displayName: `Quota Exceeded in ${CLUSTER_BASENAME}`,
+        conditionThreshold: {
+          aggregations: [
+            {
+              alignmentPeriod: '60s',
+              crossSeriesReducer: 'REDUCE_SUM',
+              groupByFields: ['metric.label.quota_metric'],
+              perSeriesAligner: 'ALIGN_COUNT_TRUE',
+            },
+          ],
+          comparison: 'COMPARISON_GT',
+          duration: '60s',
+          filter:
+            'resource.type="consumer_quota" AND metric.type="serviceruntime.googleapis.com/quota/exceeded"',
+          trigger: {
+            count: 1,
+          },
+        },
+      },
+    ],
+  });
+
+  // Allocation quotas track consumed capacity (for example CPUs, IPs, disk)
+  // against fixed limits, while rate quotas track request throughput over time
+  // windows (for example API calls per minute). These are tracked separately so
+  // we have separate alerts for them
+
+  new gcp.monitoring.AlertPolicy('quotaAllocationAlert', {
+    ...baseArgs,
+    displayName: `Allocation Quota approaching limit (>${quotaUsageThresholdPercent}%) in ${CLUSTER_BASENAME}`,
+    conditions: [
+      {
+        // Tracks resources like CPUs, Static IPs, Disk Space
+        displayName: `Allocation Quota approaching limit (>${quotaUsageThresholdPercent}%) in ${CLUSTER_BASENAME}`,
+        conditionPrometheusQueryLanguage: {
+          query: `
+            serviceruntime_googleapis_com:quota_allocation_usage{monitored_resource="consumer_quota"}
+            / ignoring(limit_name) group_right()
+            (serviceruntime_googleapis_com:quota_limit{monitored_resource="consumer_quota"} > 0)
+            > ${quotaUsageThreshold}
+          `,
+          duration: '300s',
+        },
+      },
+    ],
+  });
+
+  new gcp.monitoring.AlertPolicy('quotaRateAlert', {
+    ...baseArgs,
+    displayName: `Rate Quota approaching limit (>${quotaUsageThresholdPercent}%) in ${CLUSTER_BASENAME}`,
+    conditions: [
+      {
+        // Tracks API requests, HSM operations per minute, etc.
+        displayName: `Rate Quota approaching limit (>${quotaUsageThresholdPercent}%) in ${CLUSTER_BASENAME}`,
+        conditionPrometheusQueryLanguage: {
+          query: `
+            serviceruntime_googleapis_com:quota_rate_net_usage{monitored_resource="consumer_quota"}
+            / ignoring(limit_name) group_right()
+            (serviceruntime_googleapis_com:quota_limit{monitored_resource="consumer_quota"} > 0)
+            > ${quotaUsageThreshold}
+          `,
+          duration: '300s',
+        },
+      },
+    ],
+  });
+}

cluster/pulumi/infra/src/index.ts

@@ -22,6 +22,7 @@ import {
   installGcpLoggingAlerts,
   installClusterMaintenanceUpdateAlerts,
   installLoggedSecretsAlerts,
+  installGcpQuotaAlerts,
 } from './gcpAlerts';
 import { configureGKEL7Gateway } from './gcpLoadBalancer';
 import { configureIstio, istioMonitoring } from './istio';
@@ -82,6 +83,9 @@ if (enableAlerts && !clusterIsResetPeriodically) {
     if (monitoringConfig.alerting.loggedSecretsFilter) {
       installLoggedSecretsAlerts(notificationChannel);
     }
+    if (monitoringConfig.alerting.alerts.gcpQuotas.enabled) {
+      installGcpQuotaAlerts(notificationChannel);
+    }
   }
 }
 istioMonitoring(network.ingressNs, []);