From 3aa23984de344bd4bd0709920f74b6eac17a24ec Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 3 Mar 2023 22:17:58 -0500 Subject: [PATCH 1/3] [helios-build] Add the ability to download and unpack prebuilts --- config/projects.toml | 12 +++ tools/helios-build/Cargo.lock | 43 +++++++++ tools/helios-build/Cargo.toml | 4 + tools/helios-build/src/main.rs | 157 ++++++++++++++++++++++++++++++++- 4 files changed, 215 insertions(+), 1 deletion(-) diff --git a/config/projects.toml b/config/projects.toml index 0c9172d..cacc44f 100644 --- a/config/projects.toml +++ b/config/projects.toml @@ -1,6 +1,18 @@ [project.illumos] github = "illumos/illumos-gate" +# TODO: Before merging, make these values real? +# For local testing, it's possible to fool the downloading +# mechanism by placing the tarball with the specified hash +# in tmp/omicron/global-zone-packages.tar.gz +[project.omicron] +github = "oxidecomputer/omicron" +commit = "508bbd8a0ec6461300bc625289c00d4b6c77f97f" +[[project.omicron.prebuilts]] +name = "global-zone-packages.tar.gz" +sha2 = "492360eef1cb52f07a3e4134dd3effb60f71c3cfe6144c393e3bcfb94f7225bd" +unpack = true + [project.omnios-build] github = "oxidecomputer/helios-omnios-build" use_ssh = true diff --git a/tools/helios-build/Cargo.lock b/tools/helios-build/Cargo.lock index c86e4bc..1167070 100644 --- a/tools/helios-build/Cargo.lock +++ b/tools/helios-build/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "0.7.19" @@ -137,6 +143,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -211,6 +226,16 @@ dependencies = [ "instant", ] +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -360,7 +385,9 @@ dependencies = [ "anyhow", "atty", "digest 0.8.1", + "flate2", "getopts", + "hex", "json5", "libc", "md-5", @@ -370,6 +397,7 @@ dependencies = [ "serde", "serde_json", "sha-1", + "sha2", "slog", "slog-term", "time", @@ -386,6 +414,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.8" @@ -562,6 +596,15 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.4" diff --git a/tools/helios-build/Cargo.toml b/tools/helios-build/Cargo.toml index 318b586..9c41cf1 100644 --- a/tools/helios-build/Cargo.toml +++ b/tools/helios-build/Cargo.toml @@ -23,11 +23,15 @@ serde_json = "1" slog = "2.5" slog-term = "2.5" atty = "0.2" +flate2 = "1.0" +hex = "0.4" libc = "0.2" reqwest = { version = "0.11", features = [ "blocking" ] } digest = "0.8" md-5 = "0.8" sha-1 = "0.8" +sha2 = "0.10" +tar = "0.4" toml = "0.5" walkdir = "2.3" regex = "1.4" diff --git a/tools/helios-build/src/main.rs b/tools/helios-build/src/main.rs index fbcb41a..8a19b8d 100644 --- a/tools/helios-build/src/main.rs +++ b/tools/helios-build/src/main.rs @@ -1,8 +1,9 @@ mod common; use common::*; -use anyhow::{Result, Context, bail}; +use anyhow::{Result, Context, anyhow, bail}; use serde::Deserialize; +use sha2::Digest; use std::collections::HashMap; use std::process::Command; use std::os::unix::process::CommandExt; @@ -135,6 +136,20 @@ struct Projects { project: HashMap, } +#[derive(Debug, Deserialize)] +struct Prebuilt { + // The name of the file being downloaded + name: String, + + // The SHA-256 hash of the downloaded file + #[serde(default)] + sha2: Option, + + // If set, attempts to unpack prebuilt as a gzip-compressed file + #[serde(default)] + unpack: bool, +} + #[derive(Debug, Deserialize)] struct Project { github: Option, @@ -171,6 +186,12 @@ struct Project { cargo_build: bool, #[serde(default)] use_debug: bool, + + /* + * Artifacts to be downloaded from buildomat + */ + #[serde(default)] + prebuilts: Vec, } impl Project { @@ -187,6 +208,85 @@ impl Project { bail!("need github or url?"); } } + + fn should_download_prebuilt(&self, tmp: &Path, prebuilt: &Prebuilt) -> Result { + let prebuilt_path = tmp.join(&prebuilt.name); + if !prebuilt_path.exists() { + return Ok(true); + } + // Observe the digest of the downloaded file + let digest = get_sha256_digest(&prebuilt_path)?; + let Some(prebuilt_sha2) = &prebuilt.sha2 else { + eprintln!( + "Missing expected sha2. {} has the SHA-2: {}", + prebuilt_path.display(), + hex::encode(digest), + ); + return Ok(true); + }; + + let expected_digest = hex::decode(&prebuilt_sha2)?; + if digest != expected_digest { + eprintln!( + "Warning: {} has the SHA-2: {}, but we expected {}", + prebuilt_path.display(), + hex::encode(digest), + prebuilt_sha2, + ); + return Ok(true); + } + Ok(false) + } + + fn download_prebuilt(&self, name: &str, tmp: &Path, prebuilt: &Prebuilt) -> Result<()> { + if self.should_download_prebuilt(&tmp, &prebuilt)? { + self.download_prebuilt_no_cache(name, &tmp, &prebuilt)?; + } + Ok(()) + } + + fn download_prebuilt_no_cache(&self, name: &str, tmp: &Path, prebuilt: &Prebuilt) -> Result<()> { + println!("downloading: {}", prebuilt.name); + let prebuilt_path = tmp.join(&prebuilt.name); + let Some(repo) = &self.github else { + bail!("Project '{name}' can only download prebuilts from github"); + }; + let Some(commit) = &self.commit else { + bail!("Project '{name}' can only download prebuilts from pinned commit"); + }; + let url = format!( + "https://buildomat.eng.oxide.computer/public/file/{}/image/{}/{}", + repo, + commit, + prebuilt.name, + ); + let mut response = reqwest::blocking::get(&url)?; + if !response.status().is_success() { + bail!( + "Failed to get '{pre}' for '{proj}' from '{url}': {s}", + pre = prebuilt.name, + proj = name, + s = response.status() + ); + } + let mut file = std::fs::File::create(&prebuilt_path)?; + response.copy_to(&mut file)?; + + if let Some(prebuilt_sha2) = &prebuilt.sha2 { + let expected = hex::decode(&prebuilt_sha2)?; + let observed = get_sha256_digest(&prebuilt_path)?; + if observed != expected { + bail!( + "{} has the SHA-2: {}, but we expected {}", + prebuilt_path.display(), + hex::encode(observed), + prebuilt_sha2, + ); + } + } + + Ok(()) + } } fn ensure_dir(components: &[&str]) -> Result { @@ -1184,6 +1284,7 @@ fn cmd_image(ca: &CommandArg) -> Result<()> { */ let templates = top_path(&["image", "templates"])?; let extras = rel_path(Some(&gate), &["etc-stlouis", "extras"])?; + let omicron = top_path(&["tmp", "omicron", "global-zone-packages"])?; let brand_extras = rel_path(Some(&tempdir), &["omicron1"])?; let projects_extras = top_path(&["projects"])?; std::fs::create_dir_all(&brand_extras)?; @@ -1199,6 +1300,7 @@ fn cmd_image(ca: &CommandArg) -> Result<()> { cmd.arg("-F").arg("stress"); cmd.arg("-E").arg(&cdock); } + cmd.arg("-E").arg(&omicron); cmd.arg("-E").arg(&extras); cmd.arg("-E").arg(&brand_extras); cmd.arg("-E").arg(&projects_extras); @@ -1531,6 +1633,26 @@ fn git_commit_count>(path: P) -> Result { Ok(res.trim().parse()?) } +// Calculates the SHA256 digest of a single file file. +fn get_sha256_digest>(path: P) -> Result> { + let mut reader = std::io::BufReader::new( + std::fs::File::open(&path)? + ); + + let mut hasher = sha2::Sha256::new(); + let mut buffer = [0; 1024]; + + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } else { + hasher.update(&buffer[..count]); + } + } + Ok(hasher.finalize().to_vec()) +} + fn cmd_setup(ca: &CommandArg) -> Result<()> { let opts = baseopts(); @@ -1564,6 +1686,7 @@ fn cmd_setup(ca: &CommandArg) -> Result<()> { let url = project.url(false)?; let tmp = ensure_dir(&["tmp", &name])?; + // Checkout the repository if exists_dir(&path)? { println!("clone {} exists already at {}", url, path.display()); if project.auto_update { @@ -1683,6 +1806,38 @@ fn cmd_setup(ca: &CommandArg) -> Result<()> { println!("clone ok!"); } + // Download prebuilts from the repository + for prebuilt in &project.prebuilts { + project.download_prebuilt(&name, &tmp, &prebuilt)?; + + if prebuilt.unpack { + println!("Unpacking prebuilt {}", prebuilt.name); + + // Open the prebuilt tarball + let prebuilt_path = tmp.join(&prebuilt.name); + let gzr = flate2::read::GzDecoder::new(std::fs::File::open(&prebuilt_path)?); + let mut archive = tar::Archive::new(gzr); + + // The unpacked path is the path to the tarball, but without + // any extensions. + // + // For example: + // - foo.tar.gz -> foo/ + let filename = prebuilt_path.file_name() + .ok_or_else(|| anyhow!("Invalid filename for {}", prebuilt.name))? + .to_str() + .ok_or_else(|| anyhow!("Invalid unicode for {}", prebuilt.name))?; + let prefix = filename.split_once('.') + .ok_or_else(|| anyhow!("No file extension for {filename}"))? + .0; + let unpacked_path = tmp.join(prefix); + + // Unpack the tarball + let _ = std::fs::remove_dir_all(&unpacked_path); + archive.unpack(unpacked_path)?; + } + } + if project.site_sh { let mut ssp = path.clone(); ssp.push("lib"); From 98d924b25a271b02aa057fdbf74d6cd801ef71c5 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 3 Mar 2023 22:21:10 -0500 Subject: [PATCH 2/3] no stutter --- tools/helios-build/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/helios-build/src/main.rs b/tools/helios-build/src/main.rs index 8a19b8d..16c7700 100644 --- a/tools/helios-build/src/main.rs +++ b/tools/helios-build/src/main.rs @@ -1633,7 +1633,7 @@ fn git_commit_count>(path: P) -> Result { Ok(res.trim().parse()?) } -// Calculates the SHA256 digest of a single file file. +// Calculates the SHA256 digest of a single file. fn get_sha256_digest>(path: P) -> Result> { let mut reader = std::io::BufReader::new( std::fs::File::open(&path)? From a678967d8e4eb365d8e6f2784562165b80548486 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 6 Mar 2023 12:03:58 -0500 Subject: [PATCH 3/3] passing -E not necessary anymore with new -P flag --- config/projects.toml | 2 +- tools/helios-build/src/main.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/config/projects.toml b/config/projects.toml index cacc44f..f24b39d 100644 --- a/config/projects.toml +++ b/config/projects.toml @@ -10,7 +10,7 @@ github = "oxidecomputer/omicron" commit = "508bbd8a0ec6461300bc625289c00d4b6c77f97f" [[project.omicron.prebuilts]] name = "global-zone-packages.tar.gz" -sha2 = "492360eef1cb52f07a3e4134dd3effb60f71c3cfe6144c393e3bcfb94f7225bd" +sha2 = "7f7e0ad1e6f20a7a5743ce5245abb8abf60e41bf483ac730382f7a5053275209" unpack = true [project.omnios-build] diff --git a/tools/helios-build/src/main.rs b/tools/helios-build/src/main.rs index 3a9ca6d..483fb65 100644 --- a/tools/helios-build/src/main.rs +++ b/tools/helios-build/src/main.rs @@ -1410,7 +1410,6 @@ fn cmd_image(ca: &CommandArg) -> Result<()> { * packages, plus other packages from the upstream helios-dev repository. */ let templates = top_path(&["image", "templates"])?; - let omicron = top_path(&["tmp", "omicron", "global-zone-packages"])?; let brand_extras = rel_path(Some(&tempdir), &["omicron1"])?; let projects_extras = top_path(&["projects"])?; std::fs::create_dir_all(&brand_extras)?; @@ -1431,7 +1430,6 @@ fn cmd_image(ca: &CommandArg) -> Result<()> { cmd.arg("-F").arg("stress"); cmd.arg("-E").arg(&cdock); } - cmd.arg("-E").arg(&omicron); cmd.arg("-E").arg(&brand_extras); cmd.arg("-E").arg(&projects_extras); cmd.arg("-F").arg(format!("repo_publisher={}", publisher));