flowey: vmm_tests: introduce step to verify all tests run on at least one runner

Cargo.lock

@@ -1949,6 +1949,7 @@ dependencies = [
  "fs-err",
  "igvmfilegen_config",
  "log",
+ "quick-xml",
  "serde",
  "serde_json",
  "target-lexicon",
@@ -5990,6 +5991,15 @@ dependencies = [
  "syn 2.0.106",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.38.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.40"

Cargo.toml

@@ -495,6 +495,7 @@ proc-macro2 = "1.0"
 prost = "0.11"
 prost-build = "0.11"
 prost-types = "0.11"
+quick-xml = "0.38.3"
 quote = "1.0"
 range_map_vec = "0.2.0"
 rayon = "1.5"

flowey/flowey_cli/src/pipeline_resolver/ado_yaml.rs

@@ -193,6 +193,7 @@ pub fn ado_yaml(
         for ResolvedJobArtifact {
             flowey_var: _,
             name,
+            ..
         } in artifacts_used
         {
             ado_steps.push({
@@ -310,7 +311,10 @@ EOF
 
         // next, emit ado steps to create dirs for artifacts which will be
         // published
-        for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
+        for ResolvedJobArtifact {
+            flowey_var, name, ..
+        } in artifacts_published
+        {
             writeln!(
                 flowey_bootstrap_bash,
                 r#"mkdir -p "$(AgentTempDirNormal)/publish_artifacts/{name}""#
@@ -324,7 +328,10 @@ EOF
 
         // lastly, emit ado steps that report the dirs for any artifacts which
         // are used by this job
-        for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
+        for ResolvedJobArtifact {
+            flowey_var, name, ..
+        } in artifacts_used
+        {
             // do NOT use ADO macro syntax $(...), since this is in the same
             // bootstrap block as where those ADO vars get defined, meaning it's
             // not available yet!
@@ -425,17 +432,24 @@ EOF
         for ResolvedJobArtifact {
             flowey_var: _,
             name,
+            force_upload,
         } in artifacts_published
         {
             ado_steps.push({
-                let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
+                let mut map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
                     r#"
                         publish: $(FLOWEY_TEMP_DIR)/publish_artifacts/{name}
                         displayName: '🌼📦 Publish {name}'
                         artifact: {name}
                     "#
                 ))
                 .unwrap();
+                if *force_upload {
+                    map.insert(
+                        "condition".into(),
+                        serde_yaml::Value::String("always()".into()),
+                    );
+                }
                 map.into()
             });
         }

flowey/flowey_cli/src/pipeline_resolver/direct_run.rs

@@ -275,7 +275,10 @@ fn direct_run_do_work(
             serde_json::to_string(&persist_dir).unwrap().into(),
         );
 
-        for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
+        for ResolvedJobArtifact {
+            flowey_var, name, ..
+        } in artifacts_published
+        {
             let path = out_dir.join("artifacts").join(name);
             fs_err::create_dir_all(&path)?;
 
@@ -291,7 +294,10 @@ fn direct_run_do_work(
         }
         fs_err::create_dir_all(out_dir.join(".job_artifacts"))?;
 
-        for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
+        for ResolvedJobArtifact {
+            flowey_var, name, ..
+        } in artifacts_used
+        {
             let path = out_dir.join(".job_artifacts").join(name);
             fs_err::create_dir_all(&path)?;
             copy_dir_all(out_dir.join("artifacts").join(name), &path)?;

flowey/flowey_cli/src/pipeline_resolver/generic.rs

@@ -50,6 +50,7 @@ pub struct ResolvedPipeline {
 pub struct ResolvedJobArtifact {
     pub flowey_var: String,
     pub name: String,
+    pub force_upload: bool,
 }
 
 #[derive(Debug, Clone)]
@@ -112,6 +113,7 @@ pub fn resolve_pipeline(pipeline: Pipeline) -> anyhow::Result<ResolvedPipeline>
             name,
             published_by_job,
             used_by_jobs,
+            force_published: _,
         } in &artifacts
         {
             let no_existing = m
@@ -183,11 +185,16 @@ pub fn resolve_pipeline(pipeline: Pipeline) -> anyhow::Result<ResolvedPipeline>
 
         let artifacts_published: Vec<_> = artifacts_published
             .into_iter()
-            .map(|a| ResolvedJobArtifact {
-                flowey_var: flowey_core::pipeline::internal::consistent_artifact_runtime_var_name(
-                    &a, false,
-                ),
-                name: a,
+            .map(|a| {
+                let artifact_meta = artifacts.iter().find(|meta| meta.name == a).unwrap();
+                ResolvedJobArtifact {
+                    flowey_var:
+                        flowey_core::pipeline::internal::consistent_artifact_runtime_var_name(
+                            &a, false,
+                        ),
+                    name: a,
+                    force_upload: artifact_meta.force_published,
+                }
             })
             .collect();
         let artifacts_used: Vec<_> = artifacts_used
@@ -197,6 +204,7 @@ pub fn resolve_pipeline(pipeline: Pipeline) -> anyhow::Result<ResolvedPipeline>
                     &a, true,
                 ),
                 name: a,
+                force_upload: false,
             })
             .collect();
         let parameters_used: Vec<_> = parameters_used
@@ -245,6 +253,7 @@ pub fn resolve_pipeline(pipeline: Pipeline) -> anyhow::Result<ResolvedPipeline>
         name: _,
         published_by_job,
         used_by_jobs,
+        force_published: _,
     } in artifacts
     {
         let published_idx = job_graph_idx[published_by_job.expect("checked in loop above")];

flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs

@@ -368,7 +368,10 @@ EOF
 
         // next, emit GitHub steps to create dirs for artifacts which will be
         // published
-        for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
+        for ResolvedJobArtifact {
+            flowey_var, name, ..
+        } in artifacts_published
+        {
             writeln!(
                 flowey_bootstrap_bash,
                 r#"mkdir -p "$AgentTempDirNormal/publish_artifacts/{name}""#
@@ -392,7 +395,10 @@ EOF
 
         // lastly, emit GitHub steps that report the dirs for any artifacts which
         // are used by this job
-        for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
+        for ResolvedJobArtifact {
+            flowey_var, name, ..
+        } in artifacts_used
+        {
             let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
             match platform.kind() {
                 FlowPlatformKind::Windows => {
@@ -430,10 +436,11 @@ EOF
         for ResolvedJobArtifact {
             flowey_var: _,
             name,
+            force_upload,
         } in artifacts_published
         {
             gh_steps.push({
-                let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
+                let mut map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
                     r#"
                         name: 🌼📦 Publish {name}
                         uses: actions/upload-artifact@v4
@@ -444,6 +451,9 @@ EOF
                     "#
                 ))
                 .unwrap();
+                if *force_upload {
+                    map.insert("if".into(), serde_yaml::Value::String("always()".into()));
+                }
                 map.into()
             });
         }

flowey/flowey_core/src/pipeline.rs

@@ -639,6 +639,7 @@ impl Pipeline {
             name: owned_name,
             published_by_job: None,
             used_by_jobs: BTreeSet::new(),
+            force_published: false,
         });
 
         (PublishArtifact { idx }, UseArtifact { idx })
@@ -658,6 +659,30 @@ impl Pipeline {
         )
     }
 
+    /// Mark an artifact to be force published, meaning it will be published
+    /// even if the job fails.
+    ///
+    /// This is useful for artifacts that contain diagnostic information or logs
+    /// that are needed to debug failures.
+    #[track_caller]
+    pub fn force_publish_artifact(&mut self, artifact: &PublishArtifact) -> &mut Self {
+        self.artifacts[artifact.idx].force_published = true;
+        self
+    }
+
+    /// Mark a typed artifact to be force published, meaning it will be published
+    /// even if the job fails.
+    ///
+    /// This is useful for artifacts that contain diagnostic information or logs
+    /// that are needed to debug failures.
+    #[track_caller]
+    pub fn force_publish_typed_artifact<T: Artifact>(
+        &mut self,
+        artifact: &PublishTypedArtifact<T>,
+    ) -> &mut Self {
+        self.force_publish_artifact(&artifact.0)
+    }
+
     /// (ADO only) Set the pipeline-level name.
     ///
     /// <https://learn.microsoft.com/en-us/azure/devops/pipelines/process/run-number?view=azure-devops&tabs=yaml>
@@ -1330,6 +1355,7 @@ pub mod internal {
         pub name: String,
         pub published_by_job: Option<usize>,
         pub used_by_jobs: BTreeSet<usize>,
+        pub force_published: bool,
     }
 
     #[derive(Debug)]

flowey/flowey_hvlite/src/pipelines/checkin_gates.rs

@@ -925,6 +925,8 @@ impl IntoPipeline for CheckinGatesCli {
             KnownTestArtifacts::Ubuntu2404ServerX64Vhd,
         ];
 
+        let mut vmm_tests_results_artifacts = vec![];
+
         for VmmTestJobParams {
             platform,
             arch,
@@ -1010,11 +1012,22 @@ impl IntoPipeline for CheckinGatesCli {
         ] {
             let test_label = format!("{label}-vmm-tests");
 
-            let pub_vmm_tests_results = if matches!(backend_hint, PipelineBackendHint::Local) {
-                Some(pipeline.new_artifact(&test_label).0)
-            } else {
-                None
-            };
+            let (pub_vmm_tests_results_full, _) =
+                pipeline.new_artifact(format!("{label}-vmm-tests-results"));
+
+            let (pub_vmm_tests_junit_xml, use_vmm_tests_junit_xml) =
+                pipeline.new_artifact(format!("{label}-vmm-tests-results-junit-xml"));
+            let (pub_vmm_tests_nextest_list_json, use_vmm_tests_nextest_list_json) =
+                pipeline.new_artifact(format!("{label}-vmm-tests-results-nextest-list-json"));
+
+            pipeline.force_publish_artifact(&pub_vmm_tests_results_full);
+            pipeline.force_publish_artifact(&pub_vmm_tests_junit_xml);
+            pipeline.force_publish_artifact(&pub_vmm_tests_nextest_list_json);
+
+            vmm_tests_results_artifacts.push((
+                label.to_string(),
+                (use_vmm_tests_junit_xml, use_vmm_tests_nextest_list_json),
+            ));
 
             let use_vmm_tests_archive = match target {
                 CommonTriple::X86_64_WINDOWS_MSVC => &use_vmm_tests_archive_windows_x86,
@@ -1037,7 +1050,16 @@ impl IntoPipeline for CheckinGatesCli {
                         dep_artifact_dirs: resolve_vmm_tests_artifacts(ctx),
                         test_artifacts,
                         fail_job_on_test_fail: true,
-                        artifact_dir: pub_vmm_tests_results.map(|x| ctx.publish_artifact(x)),
+                        artifact_dirs:
+                            flowey_lib_common::publish_test_results::VmmTestResultsArtifacts {
+                                junit_xml: Some(ctx.publish_artifact(pub_vmm_tests_junit_xml)),
+                                nextest_list_json: Some(
+                                    ctx.publish_artifact(pub_vmm_tests_nextest_list_json),
+                                ),
+                                test_results_full: Some(
+                                    ctx.publish_artifact(pub_vmm_tests_results_full),
+                                ),
+                            },
                         needs_prep_run,
                         done: ctx.new_done_handle(),
                     }
@@ -1059,6 +1081,37 @@ impl IntoPipeline for CheckinGatesCli {
             }
         }
 
+        {
+            let job = pipeline
+                .new_job(
+                    FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
+                    FlowArch::X86_64,
+                    "verify all tests run at least once",
+                )
+                .gh_set_pool(crate::pipelines_shared::gh_pools::default_x86_pool(
+                    FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
+                ))
+                .dep_on(
+                    |ctx| flowey_lib_hvlite::_jobs::verify_all_tests_run::Request {
+                        test_artifacts: vmm_tests_results_artifacts
+                            .iter()
+                            .map(|elem| {
+                                (
+                                    elem.0.clone(),
+                                    flowey_lib_hvlite::_jobs::verify_all_tests_run::VmmTestResultsArtifacts {
+                                        junit_xml: ctx.use_artifact(&elem.1.0),
+                                        nextest_list_json: ctx.use_artifact(&elem.1.1),
+                                    },
+                                )
+                            })
+                            .collect(),
+                        done: ctx.new_done_handle(),
+                    },
+                )
+                .finish();
+            all_jobs.push(job);
+        }
+
         // test the flowey local backend by running cargo xflowey build-igvm on x64
         {
             let job = pipeline