merge

Author: jcblemai

.github/workflows/cache-hubval-deps.yaml

@@ -29,6 +29,7 @@ jobs:
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
+          pak-version: "devel" #Delete this line or comment it out when pak is fixed; added 2025-05-12.
           packages: |
             any::hubValidations
             any::sessioninfo

.github/workflows/pull_baselines.yml

@@ -0,0 +1,69 @@
+name: Pull baselines
+on:
+  workflow_dispatch:
+    inputs:
+      name:
+        description: 'Pull Baselines'
+        required: false
+      publish:
+        description: 'Pull baselines forecasts'
+        type: boolean
+        required: false
+        default: true
+  schedule:
+    - cron: "20 19 * * 3"
+
+permissions:
+    contents: write
+    pull-requests: write
+
+jobs:
+  pull_baselines:
+    if: github.repository_owner == 'cdcepi'
+    runs-on: macOS-latest
+    steps:    
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with: 
+          fetch-depth: 0
+      - name: Setup R 
+        uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+      - name: Install dependencies
+        run: |
+          install.packages("pak")
+          pak::pkg_install(c(
+          "readr", "dplyr", "tidyr",
+          "lubridate", "fs",
+          "github::hubverse-org/hubValidations"))
+        shell: Rscript {0}
+      - name: Pull baseline csv files
+        run:  Rscript auxiliary-data/pull-flusight-baselines.R
+      - name: Show validation output in logs (for debugging)
+        run: cat validation_result.md || echo "No validation_result.md found"
+      - name: Read validation result
+        id: validation
+        run: |
+          {
+            echo 'body<<EOF'
+            cat validation_result.md
+            echo 'EOF'
+          } >> "$GITHUB_OUTPUT"
+      - name: Commit changes and create PR 🚀
+        if: ${{ inputs.publish || github.event_name == 'schedule' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_DATETIME=$(date +'%Y-%m-%d_%H-%M-%S')
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -b create/baselines/"$PR_DATETIME"
+          git add .
+          git commit -m "Pull baseline forecasts"
+          git push --set-upstream origin create/baselines/"$PR_DATETIME"
+          gh pr create --title "Add new baseline forecasts" \
+            --body "${{ steps.validation.outputs.body }}" \
+            --base main \
+            --head create/baselines/"$PR_DATETIME"
+        shell: bash

.github/workflows/update_target_data.yml

@@ -2,12 +2,18 @@ name: Update target data
 on:
   workflow_dispatch:
     inputs:
-      name:
-        description: 'Update target data'
+      publish:
+        description: 'Publish the target data'
+        type: boolean
         required: false
+        default: true
   schedule:
     - cron: "20 16 * * 3"
 
+permissions:
+    contents: write
+    pull-requests: write
+
 jobs:
   update_target_data:
     if: github.repository_owner == 'cdcepi'
@@ -24,15 +30,19 @@ jobs:
       - name: Install dependencies
         run: |
           install.packages("RSocrata", repos = c("https://chicago.r-universe.dev", "https://cloud.r-project.org"))
-          install.packages(c("dplyr", "lubridate"))
+          install.packages(c("here", "cli", "tidyr", "readr", "dplyr", "lubridate"))
         shell: Rscript {0}
       - name: Generate target csv files
         run:  Rscript target-data/get_target_data.R
-      - name: Commit changes
-        uses: EndBug/add-and-commit@v7
-        with: 
-          add: 'target-data'
-          message: "Update target data"
-          default_author: github_actions
-          push: true
-
+      - name: Generate target csv files (with new hubverse flavor!)
+        run:  Rscript target-data/get_target_data_hubverse.R
+      - name: Commit changes and create PR 🚀
+        if: ${{ inputs.publish || github.event_name == 'schedule' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add target-data/
+          git commit -m "Update target data" || echo nothing to commit
+          git push origin main

.github/workflows/validate-submission.yaml

@@ -22,18 +22,19 @@ jobs:
         with:
           install-r: false
           use-public-rspm: true
-          extra-repositories: 'https://hubverse-org.r-universe.dev'
+          #extra-repositories: 'https://hubverse-org.r-universe.dev'
 
       - name: Update R
         run: |
           sudo apt-get update
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
+          pak-version: "devel" #Delete this line or comment it out when pak is fixed; added 2025-05-08.
           packages: |
-            any::hubValidations
-            any::sessioninfo
-
+           any::hubValidations
+           any::sessioninfo
+           
       - name: Run validations
         env:
           PR_NUMBER: ${{ github.event.number }}

auxiliary-data/pull-flusight-baselines.R

@@ -0,0 +1,96 @@
+
+# Script that runs with the Pull baselines action to download and save the weekly baselines from the FluSight-baseline repository.
+# Retrieves specifically FluSight-baseline, FluSight-base_seasonal, and FluSight-equal_cat.
+# Runs Hub Validations on the files.
+
+# Load required libraries
+library(lubridate)
+library(hubValidations)
+library(fs)
+
+# Prepare output file for PR body
+result_file <- "validation_result.md"
+
+# Wrap the entire script in a top-level tryCatch
+tryCatch({
+
+  # Set up reference date and file names
+  current_ref_date <- ceiling_date(Sys.Date(), "week") - days(1)
+  date_str <- format(current_ref_date, "%Y-%m-%d")
+
+  baseline_types <- c("FluSight-baseline", "FluSight-base_seasonal", "FluSight-equal_cat")
+  baseline_folders <- c("Flusight-baseline", "Flusight-seasonal-baseline", "Flusight-equal_cat")
+
+  downloaded_files <- c()
+  validation_results <- list()
+
+  for (i in seq_along(baseline_types)) {
+    type <- baseline_types[i]
+    folder <- baseline_folders[i]
+    filename <- paste0(date_str, "-", type, ".csv")
+
+    file_url <- paste0(
+      "https://raw.githubusercontent.com/cdcepi/Flusight-baseline/main/weekly-submission/forecasts/",
+      folder, "/", filename
+    )
+
+    target_dir <- file.path("model-output", type)
+    dir_create(target_dir, recurse = TRUE)
+    destfile <- file.path(target_dir, filename)
+
+    # Attempt to download
+    download_success <- tryCatch({
+      download.file(url = file_url, destfile = destfile, method = "libcurl")
+      cat("✅ Downloaded and saved:", destfile, "\n")
+      downloaded_files <- c(downloaded_files, file.path(type, filename))
+      TRUE
+    }, error = function(e) {
+      msg <- paste("❌ Failed to download", filename, "Reason:", e$message)
+      cat(msg, "\n")
+      validation_results[[file.path(type, filename)]] <- list(status = "error", message = msg)
+      FALSE
+    })
+
+    # Only attempt validation if download succeeded
+    if (download_success) {
+      file_path <- file.path(type, filename)
+      result <- tryCatch({
+        v <- hubValidations::validate_submission(hub_path = ".", file_path = file_path)
+
+        # Try to check for validation errors
+        err_msg <- tryCatch({
+          hubValidations::check_for_errors(v, verbose = TRUE)
+          NULL  # Passed
+        }, error = function(e) {
+          e$message  # Return error message
+        })
+
+        list(status = if (is.null(err_msg)) "pass" else "fail", message = err_msg)
+
+      }, error = function(e) {
+        list(status = "error", message = e$message)
+      })
+
+      validation_results[[file_path]] <- result
+    }
+  }
+
+  # Compose validation_result.md content
+  messages <- c("### 🧪 Validation Results")
+
+  for (file in names(validation_results)) {
+    res <- validation_results[[file]]
+    if (res$status == "pass") {
+      messages <- c(messages, paste0("✅ **", file, "** passed validation."))
+    } else {
+      messages <- c(messages, paste0("❌ **", file, "**: ", res$message))
+    }
+  }
+
+  writeLines(messages, result_file)
+
+}, error = function(e) {
+  writeLines(c("### 🧪 Validation Results", "❌ Script failed unexpectedly:", e$message), result_file)
+})
+
+