fix report negative job counts to Sentry, don't report 0 job counts to Influx (#505)

* fix report negative job counts to Sentry, don't report 0 job counts to Influx

* fix lint

Author: juliangruber

lib/metrics.js

@@ -1,6 +1,7 @@
 import { writeClient } from './telemetry.js'
 import { Point } from '@influxdata/influxdb-client'
 import EventEmitter from 'node:events'
+import * as Sentry from '@sentry/node'
 
 export class MetricsEvent {
   /**
@@ -21,6 +22,7 @@ export class Metrics {
     this.mergedMetrics = null
     /** @type {Map<string, MetricsEvent>} */
     this.moduleMetrics = new Map()
+    this.lastErrorReportedAt = 0
   }
 
   /**
@@ -46,15 +48,19 @@ export class Metrics {
       typeof metrics.totalJobsCompleted === 'number' &&
       typeof this.moduleMetrics.get(moduleName)?.totalJobsCompleted === 'number'
     ) {
-      writeClient.writePoint(
-        new Point('jobs-completed')
-          .stringField('module', moduleName)
-          .intField(
-            'value',
-            metrics.totalJobsCompleted -
-              this.moduleMetrics.get(moduleName).totalJobsCompleted
-          )
-      )
+      const diff = metrics.totalJobsCompleted -
+        this.moduleMetrics.get(moduleName).totalJobsCompleted
+      if (diff < 0) {
+        this.maybeReportErrorToSentry(
+          new Error(`Negative jobs completed for ${moduleName}`)
+        )
+      } else if (diff > 0) {
+        writeClient.writePoint(
+          new Point('jobs-completed')
+            .stringField('module', moduleName)
+            .intField('value', diff)
+        )
+      }
     }
     this.moduleMetrics.set(moduleName, resolvedMetrics)
     const mergedMetrics = {
@@ -84,6 +90,18 @@ export class Metrics {
   onUpdate (fn) {
     this.#events.on('update', fn)
   }
+
+  /**
+   * @param {unknown} err
+   */
+  maybeReportErrorToSentry (err) {
+    const now = Date.now()
+    if (now - this.lastErrorReportedAt < 4 /* HOURS */ * 3600_000) return
+    this.lastErrorReportedAt = now
+
+    console.error('Reporting the problem to Sentry for inspection by the Station team.')
+    Sentry.captureException(err)
+  }
 }
 
 export const metrics = new Metrics()

lib/telemetry.js

@@ -21,8 +21,8 @@ const processUUID = randomUUID()
 const client = new InfluxDB({
   url: 'https://eu-central-1-1.aws.cloud2.influxdata.com',
   token:
-    // station-core-20-2-1
-    'r22Y7xk_Ojh2KIPZogK68XcFarymORALLDSSDNdflbsoLTIOJOOnQrdqLEhW9zLf3grYGaGKHevjlYgCGSvo_w=='
+    // station-core-21-0-2
+    'MveJoNJL5I_333ehxXCjaPvUBGN46SprEzC4GzSCIXQHmwdvTN3y6utV-UxmxugL6hSY7eejvgFY161FsrDycQ=='
 })
 
 export const writeClient = client.getWriteApi(