Completing reports migration

definitions/output/reports/reports_dynamic.js

@@ -1,107 +1,215 @@
-const configs = new reports.HTTPArchiveReports()
-const metrics = configs.listMetrics()
+/**
+ * Dynamic Reports Generator
+ *
+ * This file automatically generates Dataform operations for HTTP Archive reports.
+ * It creates operations for each combination of:
+ * - Date range (from startDate to endDate)
+ * - Metrics (defined in includes/reports.js)
+ * - SQL types (histogram, timeseries)
+ * - Lenses (data filters like all, top1k, wordpress, etc.)
+ *
+ * Each operation:
+ * 1. Calculates metrics from crawl data
+ * 2. Stores results in BigQuery tables
+ * 3. Exports data to Cloud Storage as JSON
+ */
 
-const bucket = 'httparchive'
-const storagePath = '/reports/dev/'
+// Initialize configurations
+const httpArchiveReports = new reports.HTTPArchiveReports()
+const availableMetrics = httpArchiveReports.listMetrics()
+const availableLenses = httpArchiveReports.lenses
+
+// Configuration constants
+const EXPORT_CONFIG = {
+  bucket: constants.bucket,
+  storagePath: constants.storagePath,
+  dataset: 'reports',
+  testSuffix: '.json'
+}
+
+// Date range for report generation
+// Adjust these dates to update reports retrospectively
+const DATE_RANGE = {
+  startDate: constants.currentMonth, // '2025-07-01'
+  endDate: constants.currentMonth    // '2025-07-01'
+}
+
+/**
+ * Generates the Cloud Storage export path for a report
+ * @param {Object} reportConfig - Report configuration object
+ * @returns {string} - Cloud Storage object path
+ */
+function buildExportPath(reportConfig) {
+  const { sql, date, metric } = reportConfig
+  let objectPath = EXPORT_CONFIG.storagePath
 
-function generateExportQuery (metric, sql, params, ctx) {
-  let query = ''
   if (sql.type === 'histogram') {
-    query = `
-SELECT
-  * EXCEPT(date)
-FROM ${ctx.self()}
-WHERE date = '${params.date}'
-`
+    // Histogram exports are organized by date folders
+    const dateFolder = date.replaceAll('-', '_')
+    objectPath += `${dateFolder}/${metric.id}`
   } else if (sql.type === 'timeseries') {
-    query = `
-SELECT
-  FORMAT_DATE('%Y_%m_%d', date) AS date,
-  * EXCEPT(date)
-FROM ${ctx.self()}
-`
+    // Timeseries exports are organized by metric
+    objectPath += metric.id
   } else {
-    throw new Error('Unknown SQL type')
+    throw new Error(`Unknown SQL type: ${sql.type}`)
   }
 
-  const queryOutput = query.replace(/[\r\n]+/g, ' ')
-  return queryOutput
+  return objectPath + EXPORT_CONFIG.testSuffix
 }
 
-function generateExportPath (metric, sql, params) {
+/**
+ * Generates the BigQuery export query for a report
+ * @param {Object} reportConfig - Report configuration object
+ * @returns {string} - SQL query for exporting data
+ */
+function buildExportQuery(reportConfig) {
+  const { sql, date, metric, lens, tableName } = reportConfig
+
+  let query
   if (sql.type === 'histogram') {
-    return `${storagePath}${params.date.replaceAll('-', '_')}/${metric.id}.json`
+    query = `
+      SELECT
+        * EXCEPT(date, metric, lens)
+      FROM \`${EXPORT_CONFIG.dataset}.${tableName}\`
+      WHERE date = '${date}'
+        AND metric = '${metric.id}'
+        AND lens = '${lens.name}'
+      ORDER BY bin ASC
+    `
   } else if (sql.type === 'timeseries') {
-    return `${storagePath}${metric.id}.json`
+    query = `
+      SELECT
+        FORMAT_DATE('%Y_%m_%d', date) AS date,
+        * EXCEPT(date, metric, lens)
+      FROM \`${EXPORT_CONFIG.dataset}.${tableName}\`
+      WHERE metric = '${metric.id}'
+        AND lens = '${lens.name}'
+      ORDER BY date DESC
+    `
   } else {
-    throw new Error('Unknown SQL type')
+    throw new Error(`Unknown SQL type: ${sql.type}`)
   }
+
+  // Convert to single line for JSON embedding
+  return query.replace(/[\r\n]+/g, ' ').trim()
 }
 
-const iterations = []
-for (
-  let date = constants.currentMonth; date >= constants.currentMonth; date = constants.fnPastMonth(date)) {
-  iterations.push({
+/**
+ * Creates a report configuration object
+ * @param {string} date - Report date (YYYY-MM-DD)
+ * @param {Object} metric - Metric configuration
+ * @param {Object} sql - SQL configuration (type and query)
+ * @param {string} lensName - Lens name
+ * @param {string} lensSQL - Lens SQL filter
+ * @returns {Object} - Complete report configuration
+ */
+function createReportConfig(date, metric, sql, lensName, lensSQL) {
+  return {
     date,
-    devRankFilter: constants.devRankFilter
-  })
+    metric,
+    sql,
+    lens: { name: lensName, sql: lensSQL },
+    devRankFilter: constants.devRankFilter,
+    tableName: `${metric.id}_${sql.type}`
+  }
 }
 
-if (iterations.length === 1) {
-  const params = iterations[0]
-  metrics.forEach(metric => {
-    metric.SQL.forEach(sql => {
-      publish(metric.id + '_' + sql.type, {
-        type: 'incremental',
-        protected: true,
-        bigquery: sql.type === 'histogram' ? { partitionBy: 'date', clusterBy: ['client'] } : {},
-        schema: 'reports'
-        // tags: ['crawl_complete', 'http_reports']
-      }).preOps(ctx => `
---DELETE FROM ${ctx.self()}
---WHERE date = '${params.date}';
-      `).query(
-        ctx => sql.query(ctx, params)
-      ).postOps(ctx => `
-SELECT
-  reports.run_export_job(
-    JSON '''{
-      "destination": "cloud_storage",
-      "config": {
-        "bucket": "${bucket}",
-        "name": "${generateExportPath(metric, sql, params)}"
-      },
-      "query": "${generateExportQuery(metric, sql, params, ctx)}"
-    }'''
-  );
-      `)
-    })
-  })
-} else {
-  iterations.forEach(params => {
-    metrics.forEach(metric => {
+/**
+ * Generates all report configurations for the specified date range
+ * @returns {Array} - Array of report configuration objects
+ */
+function generateReportConfigurations() {
+  const reportConfigs = []
+
+  // Generate configurations for each date in range
+  for (let date = DATE_RANGE.endDate;
+    date >= DATE_RANGE.startDate;
+    date = constants.fnPastMonth(date)) {
+
+    // For each available metric
+    availableMetrics.forEach(metric => {
+      // For each SQL type (histogram, timeseries)
       metric.SQL.forEach(sql => {
-        operate(metric.id + '_' + sql.type + '_' + params.date, {
-          // tags: ['crawl_complete', 'http_reports']
-        }).queries(ctx => `
-DELETE FROM reports.${metric.id}_${sql.type}
-WHERE date = '${params.date}';
-
-INSERT INTO reports.${metric.id}_${sql.type}` + sql.query(ctx, params)
-        ).postOps(ctx => `
-  SELECT
-  reports.run_export_job(
-    JSON '''{
-      "destination": "cloud_storage",
-      "config": {
-        "bucket": "${bucket}",
-        "name": "${generateExportPath(metric, sql, params)}"
-      },
-      "query": "${generateExportQuery(metric, sql, params, ctx)}"
-    }'''
-  );
-        `)
+        // For each available lens (all, top1k, wordpress, etc.)
+        Object.entries(availableLenses).forEach(([lensName, lensSQL]) => {
+          const config = createReportConfig(date, metric, sql, lensName, lensSQL)
+          reportConfigs.push(config)
+        })
       })
     })
-  })
+  }
+
+  return reportConfigs
+}
+
+/**
+ * Creates a Dataform operation name for a report configuration
+ * @param {Object} reportConfig - Report configuration object
+ * @returns {string} - Operation name
+ */
+function createOperationName(reportConfig) {
+  const { tableName, date, lens } = reportConfig
+  return `${tableName}_${date}_${lens.name}`
+}
+
+/**
+ * Generates the SQL for a Dataform operation
+ * @param {Object} ctx - Dataform context
+ * @param {Object} reportConfig - Report configuration object
+ * @returns {string} - Complete SQL for the operation
+ */
+function generateOperationSQL(ctx, reportConfig) {
+  const { date, metric, lens, sql, tableName } = reportConfig
+
+  return `
+DECLARE job_config JSON;
+
+/* First report run - uncomment to create table
+CREATE TABLE IF NOT EXISTS ${EXPORT_CONFIG.dataset}.${tableName}
+PARTITION BY date
+CLUSTER BY metric, lens, client
+AS
+*/
+
+--/* Subsequent report run
+DELETE FROM ${EXPORT_CONFIG.dataset}.${tableName}
+WHERE date = '${date}'
+  AND metric = '${metric.id}'
+  AND lens = '${lens.name}';
+INSERT INTO ${EXPORT_CONFIG.dataset}.${tableName}
+--*/
+
+SELECT
+  '${metric.id}' AS metric,
+  '${lens.name}' AS lens,
+  *
+FROM (
+  ${sql.query(ctx, reportConfig)}
+);
+
+SET job_config = TO_JSON(
+  STRUCT(
+    "cloud_storage" AS destination,
+    STRUCT(
+      "httparchive" AS bucket,
+      "${buildExportPath(reportConfig)}" AS name
+    ) AS config,
+    r"${buildExportQuery(reportConfig)}" AS query
+  )
+);
+
+SELECT reports.run_export_job(job_config);
+`
 }
+
+// Generate all report configurations
+const reportConfigurations = generateReportConfigurations()
+
+// Create Dataform operations for each report configuration
+reportConfigurations.forEach(reportConfig => {
+  const operationName = createOperationName(reportConfig)
+
+  operate(operationName)
+    .tags(['crawl_complete', 'crawl_reports'])
+    .queries(ctx => generateOperationSQL(ctx, reportConfig))
+})

includes/constants.js

@@ -49,6 +49,7 @@ class DataformTemplateBuilder {
         if (typeof value === 'string') return `'${value}'`
         if (typeof value === 'number') return value.toString()
         if (typeof value === 'boolean') return value.toString()
+        if (typeof value === 'function') return value.toString()
 
         // For objects or arrays, use JSON.stringify
         return JSON.stringify(value)