flowey: use release branches to download artifacts for servicing tests (#1897)

This PR revives: #941. It adds
a new node for downloading release artifacts and uses them in servicing
tests. Tests for the following servicing scenarios are added:

Upgrades:
2505->latest in main
Downgrades:
latest in main->2505

---------

Co-authored-by: Daman Mulye <[email protected]>

Author: damanm24

.github/workflows/openvmm-ci.yaml

@@ -119,7 +119,12 @@ impl GhContextVarReader<'_, state::Global> {
 
     /// `github.token`
     pub fn token(self) -> ReadVar<String> {
-        self.read_var("github.token", true, false)
+        // TODO: change is_secret parameter to true.
+        // N.B. Flowey core treats all variables as secrets after a secret variable is read from. Secrecy is viral this way.
+        // This causes unintended consequences in the job, as all subsequent variables are also treated as secrets.
+        // We don't have a good way of fixing this issue yet. Hence, the change in parameter value here.
+        // GitHub redacts access tokens from being printed to logs anyways, so by flipping the is_secret parameter to false, the token won't be leaked.
+        self.read_var("github.token", false, false)
     }
 }
 

.github/workflows/openvmm-pr-release.yaml

@@ -21,6 +21,8 @@ impl IntoPipeline for RestorePackagesCli {
         );
 
         let mut pipeline = Pipeline::new();
+        let (_pub_last_release_igvm_files, use_last_release_igvm_files) =
+            pipeline.new_artifact("last-release-igvm-files");
         let mut job = pipeline
             .new_job(
                 FlowPlatform::host(backend_hint),
@@ -54,14 +56,16 @@ impl IntoPipeline for RestorePackagesCli {
             }
         };
 
-        for arch in arches {
-            job = job.dep_on(
-                |ctx| flowey_lib_hvlite::_jobs::local_restore_packages::Request {
-                    arch: arch.into(),
-                    done: ctx.new_done_handle(),
-                },
-            );
-        }
+        let arches = arches.into_iter().map(|arch| arch.into()).collect();
+
+        job = job.dep_on(
+            |ctx| flowey_lib_hvlite::_jobs::local_restore_packages::Request {
+                arches,
+                done: ctx.new_done_handle(),
+                release_artifact: ctx.use_artifact(&use_last_release_igvm_files),
+            },
+        );
+
         job.finish();
         Ok(pipeline)
     }

.github/workflows/openvmm-pr.yaml

@@ -105,7 +105,7 @@ pub fn get_cfg_common_params(
     }
 }
 
-#[derive(clap::ValueEnum, Clone, Copy)]
+#[derive(clap::ValueEnum, Clone, Copy, PartialEq)]
 pub enum CommonArchCli {
     X86_64,
     Aarch64,

flowey/flowey_core/src/node/github_context.rs

@@ -21,8 +21,6 @@ flowey_request! {
         pub path: WriteVar<PathBuf>,
         /// The Github actions run id to download artifacts from
         pub run_id: ReadVar<String>,
-        /// Github token to authenticate with
-        pub gh_token: ReadVar<String>,
     }
 }
 
@@ -43,12 +41,8 @@ impl SimpleFlowNode for Node {
             file_name,
             path,
             run_id,
-            gh_token,
         } = request;
 
-        ctx.req(crate::use_gh_cli::Request::WithAuth(
-            crate::use_gh_cli::GhCliAuth::AuthToken(gh_token),
-        ));
         let gh_cli = ctx.reqv(crate::use_gh_cli::Request::Get);
 
         ctx.emit_rust_step("download artifacts from github actions run", |ctx| {

flowey/flowey_hvlite/src/pipelines/restore_packages.rs

@@ -0,0 +1,62 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Gets the latest completed Github workflow id for a pipeline and branch
+use flowey::node::prelude::*;
+
+flowey_request! {
+    pub struct Request {
+        pub repo: String,
+        pub pipeline_name: String,
+        pub branch: ReadVar<String>,
+        pub gh_workflow_id: WriteVar<String>,
+    }
+}
+new_simple_flow_node!(struct Node);
+
+impl SimpleFlowNode for Node {
+    type Request = Request;
+
+    fn imports(ctx: &mut ImportCtx<'_>) {
+        ctx.import::<crate::use_gh_cli::Node>();
+    }
+
+    fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> {
+        let Request {
+            repo,
+            gh_workflow_id,
+            pipeline_name,
+            branch,
+        } = request;
+
+        let pipeline_name = pipeline_name.clone();
+
+        let gh_cli = ctx.reqv(crate::use_gh_cli::Request::Get);
+
+        ctx.emit_rust_step("get latest completed action id", |ctx| {
+            let pipeline_name = pipeline_name.clone();
+            let gh_cli = gh_cli.claim(ctx);
+            let gh_workflow_id = gh_workflow_id.claim(ctx);
+            let branch = branch.claim(ctx);
+
+            move |rt| {
+                let sh = xshell::Shell::new()?;
+                let gh_cli = rt.read(gh_cli);
+                let branch = rt.read(branch);
+
+                let id = xshell::cmd!(
+                    sh,
+                    "{gh_cli} run list -R {repo} -b {branch} -w {pipeline_name} -s completed --limit 1 --json databaseId -q .[0].databaseId"
+                )
+                .read()?;
+
+                log::info!("Got action id {id}");
+                rt.write(gh_workflow_id, &id);
+
+                Ok(())
+            }
+        });
+
+        Ok(())
+    }
+}

flowey/flowey_hvlite/src/pipelines_shared/cfg_common_params.rs

@@ -10,7 +10,6 @@ flowey_request! {
         pub github_commit_hash: ReadVar<String>,
         pub repo_path: ReadVar<PathBuf>,
         pub pipeline_name: String,
-        pub gh_token: ReadVar<String>,
         pub gh_workflow: WriteVar<GithubWorkflow>,
     }
 }
@@ -36,14 +35,10 @@ impl SimpleFlowNode for Node {
             github_commit_hash,
             gh_workflow,
             pipeline_name,
-            gh_token,
         } = request;
 
         let pipeline_name = pipeline_name.clone();
 
-        ctx.req(crate::use_gh_cli::Request::WithAuth(
-            crate::use_gh_cli::GhCliAuth::AuthToken(gh_token.clone()),
-        ));
         let gh_cli = ctx.reqv(crate::use_gh_cli::Request::Get);
 
         ctx.emit_rust_step("get action id", |ctx| {

flowey/flowey_lib_common/src/download_gh_artifact.rs

@@ -33,6 +33,7 @@ pub mod download_nuget_exe;
 pub mod download_protoc;
 pub mod gen_cargo_nextest_run_cmd;
 pub mod gh_download_azure_key_vault_secret;
+pub mod gh_latest_completed_workflow_id;
 pub mod gh_task_azure_login;
 pub mod gh_workflow_id;
 pub mod git_checkout;