Np update truth paths

.dockstore.yml

@@ -68,6 +68,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl
 
+  - name: rnaseq_aou
+    subclass: WDL
+    primaryDescriptorPath: /all_of_us/rna_seq/rnaseq_aou.wdl
+
   - name: RNAWithUMIsPipeline
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/rna_seq/RNAWithUMIsPipeline.wdl
@@ -90,7 +94,7 @@ workflows:
 
   - name: SlideTags
     subclass: WDL
-    primaryDescriptorPath: /beta-pipelines/skylab/slidetags/SlideTags.wdl
+    primaryDescriptorPath: /pipelines/skylab/slidetags/SlideTags.wdl
 
   - name: snm3C-seq
     subclass: WDL
@@ -144,6 +148,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestPairedTag.wdl
 
+  - name: TestPeakCalling
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestPeakCalling.wdl
+
   - name: TestReblockGVCF
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestReblockGVCF.wdl
@@ -156,6 +164,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestSlideSeq.wdl
 
+  - name: TestSlideTags
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestSlideTags.wdl
+
   - name: Testsnm3C
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/Testsnm3C.wdl
@@ -200,7 +212,7 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl
 
-  - name: vds_to_vcf.wdl
+  - name: vds_to_vcf
     subclass: WDL
     primaryDescriptorPath: /all_of_us/ancestry/vds_to_vcf.wdl
 

.github/workflows/test_peakcalling.yml

@@ -0,0 +1,65 @@
+name: Test PeakCalling
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      # anything in the pipelines folder
+      - 'pipelines/skylab/peak_calling/**'
+      # tasks from the pipeline WDL and their dependencies
+      - 'tasks/broad/Utilities.wdl'
+      # verification WDL and its dependencies
+      - 'verification/VerifyPeakCalling.wdl'
+      - 'verification/VerifyTasks.wdl'
+      # test WDL and its dependencies
+      - 'verification/test-wdls/TestPeakCalling.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      # this file, the subworkflow file, and the firecloud_api script
+      - '.github/workflows/test_peakcalling.yml'
+      - '.github/workflows/warp_test_workflow.yml'
+      - 'scripts/firecloud_api/firecloud_api.py'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+
+jobs:
+  TestPeakCalling:
+    uses: ./.github/workflows/warp_test_workflow.yml
+    with:
+      pipeline_name: TestPeakCalling
+      dockstore_pipeline_name: PeakCalling
+      pipeline_dir: pipelines/skylab/peak_calling
+      use_call_cache: ${{ github.event.inputs.useCallCache || 'true' }}
+      update_truth: ${{ github.event.inputs.updateTruth || 'false' }}
+      test_type: ${{ github.event.inputs.testType }}
+      truth_branch: ${{ github.event.inputs.truthBranch || 'master' }}
+    secrets:
+      PDT_TESTER_SA_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+      DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}

.github/workflows/test_slidetags.yml

@@ -0,0 +1,68 @@
+name: Test Slide Tags
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      # anything in the pipelines folder
+      - 'pipelines/skylab/slidetags/**'
+      # tasks from the pipeline WDL and their dependencies
+      - 'tasks/skylab/StarAlign.wdl'
+      - 'tasks/skylab/Metrics.wdl'
+      - 'tasks/skylab/H5adUtils.wdl'
+      - 'tasks/skylab/CheckInputs.wdl'
+      - 'tasks/skylab/MergeSortBam.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      # verification WDL and its dependencies
+      - 'verification/VerifySlideTags.wdl'
+      - 'verification/VerifyTasks.wdl'
+      # test WDL and its dependencies
+      - 'verification/test-wdls/TestSlideTags.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      # this file, the subworkflow file, and the firecloud_api script
+      - '.github/workflows/test_slidetags.yml'
+      - '.github/workflows/warp_test_workflow.yml'
+      - 'scripts/firecloud_api/firecloud_api.py'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+jobs:
+  TestSlideTags:
+    uses: ./.github/workflows/warp_test_workflow.yml
+    with:
+      pipeline_name: TestSlideTags
+      dockstore_pipeline_name: SlideTags
+      pipeline_dir: pipelines/skylab/slidetags
+      use_call_cache: ${{ github.event.inputs.useCallCache || 'true' }}
+      update_truth: ${{ github.event.inputs.updateTruth || 'false' }}
+      test_type: ${{ github.event.inputs.testType }}
+      truth_branch: ${{ github.event.inputs.truthBranch || 'master' }}
+    secrets:
+      PDT_TESTER_SA_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+      DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}

.github/workflows/warp_test_workflow.yml

@@ -211,6 +211,7 @@ jobs:
           done
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+
           exit 1
         env:
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
@@ -249,19 +250,18 @@ jobs:
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="${{ inputs.pipeline_dir }}/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-
-          TRUTH_PATH="gs://broad-gotc-test-storage/${{ inputs.dockstore_pipeline_name }}/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
-          echo "Truth path: $TRUTH_PATH"
 
           RESULTS_PATH="gs://pd-test-results/${{ inputs.dockstore_pipeline_name }}/results/$CURRENT_TIME"
 
+
           # Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
-                --branch_name "$BRANCH_NAME" )
+                --branch_name "$TRUTH_BRANCH" \
+                --dockstore_pipeline_name "${{ inputs.dockstore_pipeline_name }}" )
             echo "Uploading the test input file: $test_input_file"
 
             # Create the submission_data.json file for this input_file

README.md

@@ -1,7 +1,7 @@
 
-### WDL Analysis Research Pipelines
+### Warp Analysis Research Pipelines
 
-The WDL Analysis Research Pipelines (WARP) repository is a collection of cloud-optimized pipelines for processing biological data from the Broad Institute Data Sciences Platform and collaborators.
+The Warp Analysis Research Pipelines (WARP) repository is a collection of cloud-optimized pipelines for processing biological data from the Broad Institute Data Sciences Platform and collaborators.
 
 WARP provides robust, standardized data analysis for the Broad Institute Genomics Platform and large consortia like the Human Cell Atlas and the BRAIN Initiative. WARP pipelines are rigorously scientifically validated, high scale, reproducible and open source, released under the [BSD 3-Clause license](https://github.com/broadinstitute/warp/blob/master/LICENSE).
 

all_of_us/ancestry/README.md

@@ -26,7 +26,7 @@ Key characteristics:
 #### Step 3. Generate Index Files**
    - Tabix index files (`.tbi`) are created alongside each VCF output.
 
-### Step 4. Create Lists of Outputs (task: `create_fofn`)
+#### Step 4. Create Lists of Outputs (task: `create_fofn`)
    - Two flat text files (`.fofn1.txt`, `.fofn2.txt`) are generated listing all full VCFs and index files.
 
 #### Step 5. Output Final Files

all_of_us/ancestry/run_ancestry.changelog.md

@@ -1,3 +1,9 @@
+# aou-9.0.0
+2025-05-23 (Date of Last Commit)
+
+* Updated the gnomad metadata tsv from gnomad.genomes.v3.1.hgdp_1kg_subset.sample_meta.tsv.gz to gs://gcp-public-data--gnomad/release/3.1/secondary_analyses/hgdp_1kg_v2/metadata_and_qc/gnomad_meta_updated.tsv
+* Updated which column the workflow uses for population labels; it now applies project_meta.project_pop
+
 # aou-8.0.0
 2025-04-16 (Date of Last Commit)
 

all_of_us/ancestry/run_ancestry.wdl

@@ -34,9 +34,9 @@ workflow run_ancestry {
         Int num_pcs=16
     }
 
-    File hgdp_metadata_file = select_first([hgdp_metadata_file_in, "gs://gcp-public-data--gnomad/release/3.1/vcf/genomes/gnomad.genomes.v3.1.hgdp_1kg_subset.sample_meta.tsv.gz"])
+    File hgdp_metadata_file = select_first([hgdp_metadata_file_in, "gs://gcp-public-data--gnomad/release/3.1/secondary_analyses/hgdp_1kg_v2/metadata_and_qc/gnomad_meta_updated.tsv"])
     Float other_cutoff = select_first([other_cutoff_in, 0.75])
-    String pipeline_version = "aou_8.0.0"
+    String pipeline_version = "aou_9.0.0"
 
     # Train the model on the intersection sites (full version that includes the samples)
     call create_hw_pca_training {
@@ -119,8 +119,8 @@ task create_hw_pca_training {
         eigenvalues_training, scores_training, loadings_training = get_PCA_scores("~{full_bgz}")
 
         # Apply any custom processing to the population labels from the training data
-        pop_label_pd = metadata_pd[['s', 'population_inference.pop']]
-        pop_label_pd['pop_label'] = metadata_pd['population_inference.pop'].apply(collapse_fin_to_eur)
+        pop_label_pd = metadata_pd[['s', 'project_meta.project_pop']]
+        pop_label_pd['pop_label'] = metadata_pd['project_meta.project_pop'].apply(collapse_fin_to_eur)
 
         # Join the labels to the training PCA feature set.
         pop_label_ht = hl.Table.from_pandas(pop_label_pd).key_by('s')