diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..9c36259 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,31 @@ +name: Build protosol + +on: + push: + branches: + - "**" + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Update submodules + run: git submodule update --init --recursive + + - name: Run deps.sh + run: bash ./deps.sh + + - name: Build (release) + run: cargo build --release + + diff --git a/.gitignore b/.gitignore index 2cf7f7a..08f8c58 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ # Cargo / Rust target/ **/*.rs.bk +opt/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..9b5d6c9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,8 @@ +[submodule "shlr/flatbuffers"] + path = shlr/flatbuffers + url = https://github.com/firedancer-io/flatbuffers.git + branch = v25.9.23-patches +[submodule "shlr/protobuf"] + path = shlr/protobuf + url = https://github.com/protocolbuffers/protobuf + branch = v25.2 diff --git a/Cargo.lock b/Cargo.lock index d71cb1e..92b96bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,7 +54,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -69,6 +69,25 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flatbuffers" +version = "25.9.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flatc-rust" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e61227926ef5b237af48bee74394cc4a5a221ebd10c5147a98e612f207851d" +dependencies = [ + "log", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -93,15 +112,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "indexmap" version = "2.12.0" @@ -121,24 +131,12 @@ dependencies = [ "either", ] -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - [[package]] name = "libc" version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -181,9 +179,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.1.25" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", "syn", @@ -200,9 +198,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", "prost-derive", @@ -210,16 +208,15 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "bytes", "heck", "itertools", - "lazy_static", "log", "multimap", + "once_cell", "petgraph", "prettyplease", "prost", @@ -227,14 +224,13 @@ dependencies = [ "regex", "syn", "tempfile", - "which", ] [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", "itertools", @@ -245,9 +241,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ "prost", ] @@ -256,6 +252,8 @@ dependencies = [ name = "protosol" version = "2.0.0" dependencies = [ + "flatbuffers", + "flatc-rust", "prost", "prost-build", ] @@ -305,16 +303,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] -name = "rustix" -version = "0.38.44" +name = "rustc_version" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "semver", ] [[package]] @@ -326,15 +320,21 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "linux-raw-sys", + "windows-sys", ] +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + [[package]] name = "syn" -version = "1.0.109" +version = "2.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" dependencies = [ "proc-macro2", "quote", @@ -350,8 +350,8 @@ dependencies = [ "fastrand", "getrandom", "once_cell", - "rustix 1.1.2", - "windows-sys 0.61.2", + "rustix", + "windows-sys", ] [[package]] @@ -369,33 +369,12 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -405,70 +384,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "wit-bindgen" version = "0.46.0" diff --git a/Cargo.toml b/Cargo.toml index 987bdff..4039ca0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,9 @@ readme = "README.md" links = "protosol" [build-dependencies] -prost-build = "0.11.9" +prost-build = "0.13.1" +flatc-rust = "0.2.0" [dependencies] -prost = "0.11.9" +prost = "0.13.1" +flatbuffers = "25.9.23" diff --git a/README.md b/README.md index 22d2050..4ab5214 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Protosol -Protocol buffer definitions for the Solana Virtual Machine (SVM) testing harness in Agave. This crate provides Rust bindings for protobuf schemas used in fuzzing and testing the Solana blockchain's execution environment. +Protocol buffer + flatbuffer definitions for the Solana Virtual Machine (SVM) testing harness in Agave. This crate provides Rust bindings for protobuf schemas used in fuzzing and testing the Solana blockchain's execution environment. ## Overview @@ -121,6 +121,13 @@ This crate is specifically designed for use with the anza-xyz/agave repository's ## Development +### Installing dependencies (builds flatc) + +```bash +git submodule update --init --recursive +./deps.sh +``` + ### Building ```bash @@ -152,6 +159,12 @@ proto/ ├── pack.proto # Compute budget testing ├── metadata.proto # Test fixture metadata └── *.options # Nanopb configuration files + +flatbuffers/ +├── elf.fbs # ELF VM program and state structures +├── context.fbs # Account and execution context definitions +├── metadata.fbs # Test fixture metadata for Flatbuffers + ``` ## License diff --git a/build.rs b/build.rs index b3bb799..fccec64 100644 --- a/build.rs +++ b/build.rs @@ -1,4 +1,8 @@ -use std::{env, fs, path::{Path, PathBuf}}; +extern crate flatc_rust; +use std::{ + env, fs, + path::{Path, PathBuf}, +}; /// Normalize path only on Windows, else just return unchanged. fn maybe_normalize_windows_path(path: &Path) -> PathBuf { @@ -18,34 +22,82 @@ fn maybe_normalize_windows_path(path: &Path) -> PathBuf { } } -fn main() -> Result<(), Box> { - let proto_dir = PathBuf::from("proto"); +fn monitor_and_get_files( + dir: &PathBuf, + env_var: &str, + extension: &str, +) -> Result<(Vec, PathBuf), Box> { let abs = env::current_dir() .expect("cwd") - .join(&proto_dir) + .join(dir) .canonicalize() .map(|p| maybe_normalize_windows_path(&p)) - .expect("canonicalize proto dir"); + .expect("canonicalize dir"); println!("cargo:rerun-if-changed={}", abs.display()); - println!("cargo:PROTO_DIR={}", abs.display()); + println!("cargo:{}={}", env_var, abs.display()); - let mut proto_files = vec![]; + let mut files = vec![]; for entry in fs::read_dir(&abs)? { let path = entry?.path(); - if path.extension().and_then(|e| e.to_str()) == Some("proto") { + if path.extension().and_then(|e| e.to_str()) == Some(extension) { println!("cargo:rerun-if-changed={}", path.display()); - proto_files.push(path); + files.push(path); } } + Ok((files, abs)) +} + +fn compile_protos() -> Result<(), Box> { + let proto_dir = PathBuf::from("proto"); + let (proto_files, abs) = monitor_and_get_files(&proto_dir, "PROTO_DIR", "proto")?; + let out_dir = PathBuf::from(env::var("OUT_DIR")?); let mut config = prost_build::Config::new(); + config.protoc_executable(PathBuf::from("opt/bin/protoc")); config.out_dir(&out_dir); config.compile_protos( - &proto_files.iter().map(|p| p.display().to_string()).collect::>(), + &proto_files + .iter() + .map(|p| p.display().to_string()) + .collect::>(), &[abs], )?; Ok(()) } + +fn compile_flatbuffers() -> Result<(), Box> { + let flatbuffer_dir = PathBuf::from("flatbuffers"); + let (flatbuffer_files, _) = monitor_and_get_files(&flatbuffer_dir, "FLATBUFFERS_DIR", "fbs")?; + + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + + // Use custom flatc from ./opt/bin/flatc relative to this build.rs + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); + let flatc_path = manifest_dir.join("opt").join("bin").join("flatc"); + + let flatc = flatc_rust::Flatc::from_path(&flatc_path); + flatc.check()?; + flatc.run(flatc_rust::Args { + lang: "rust", + inputs: flatbuffer_files + .iter() + .map(|p| p.as_path()) + .collect::>() + .as_slice(), + out_dir: out_dir.as_path(), + includes: &[flatbuffer_dir.as_path()], + extra: &["--gen-object-api", "--gen-compare"], + ..Default::default() + })?; + + Ok(()) +} + +fn main() -> Result<(), Box> { + compile_protos()?; + compile_flatbuffers()?; + Ok(()) +} diff --git a/deps.sh b/deps.sh new file mode 100755 index 0000000..84b8a3e --- /dev/null +++ b/deps.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Install prefix +PREFIX="$(pwd)/opt" + +install_flatbuffers () { + mkdir -p "$PREFIX/build/flatbuffers" + echo "[+] Configuring flatbuffers (flatc only)" + cmake \ + -S shlr/flatbuffers \ + -B "$PREFIX/build/flatbuffers" \ + -G"Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH="$PREFIX" \ + -DFLATBUFFERS_BUILD_TESTS=OFF \ + -DFLATBUFFERS_BUILD_FLATC=ON \ + -DFLATBUFFERS_BUILD_FLATLIB=OFF + + echo "[+] Building flatc binary only" + make -C "$PREFIX/build/flatbuffers" flatc -j `nproc` + + echo "[+] Installing flatc via cmake" + cmake --install "$PREFIX/build/flatbuffers" --config Release + + echo "[+] Successfully installed flatc" +} + +install_protobuf () { + mkdir -p "$PREFIX/build/protobuf" + + # Note: insane CMAKE_CXX_LINK_EXECUTABLE required to link with custom libc++ + echo "[+] Configuring protobuf" + cmake \ + -S shlr/protobuf \ + -B "$PREFIX/build/protobuf" \ + -G"Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR="lib" \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DBUILD_TESTING=OFF \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_BUILD_CONFORMANCE=OFF \ + -Dprotobuf_BUILD_EXAMPLES=OFF \ + -Dprotobuf_BUILD_PROTOBUF_BINARIES=ON \ + -Dprotobuf_BUILD_PROTOC_BINARIES=ON \ + -Dprotobuf_BUILD_LIBPROTOC=ON \ + -Dprotobuf_BUILD_SHARED_LIBS=OFF \ + -Dprotobuf_INSTALL=ON + + echo "[+] Building protobuf" + make -C "$PREFIX/build/protobuf" -j `nproc` + + echo "[+] Installing protobuf" + make -C "$PREFIX/build/protobuf" install -j `nproc` + echo "[+] Successfully installed protobuf" +} + +mkdir -pv "$PREFIX" +install_flatbuffers +install_protobuf diff --git a/flatbuffers/context.fbs b/flatbuffers/context.fbs new file mode 100644 index 0000000..74d138d --- /dev/null +++ b/flatbuffers/context.fbs @@ -0,0 +1,40 @@ +// Generated from context.proto + +namespace org.solana.sealevel.v2; + +struct Pubkey { + address:[ubyte:32]; +} + +struct Signature { + signature:[ubyte:64]; +} + +struct XXHash { + hash:[ubyte:8]; +} + +struct Hash { + hash:[ubyte:32]; +} + +struct LtHash { + hash:[ushort:1024]; +} + +table FeatureSet { + features:[ulong]; +} + +table Account { + address:Pubkey (required); + lamports:ulong; + data:[ubyte] (required); + executable:bool; + owner:Pubkey (required); +} + +table VoteAccount { + vote_account:Account (required); + stake:ulong; +} diff --git a/flatbuffers/elf.fbs b/flatbuffers/elf.fbs new file mode 100644 index 0000000..e2e9f27 --- /dev/null +++ b/flatbuffers/elf.fbs @@ -0,0 +1,28 @@ +// Generated from elf.proto + +include "metadata.fbs"; +include "context.fbs"; + +namespace org.solana.sealevel.v2; + +table ELFLoaderCtx { + elf_data:[ubyte] (required); + features:FeatureSet (required); + deploy_checks:bool; +} + +table ELFLoaderEffects { + err_code:ubyte; + rodata_hash:XXHash; + text_cnt:ulong; + text_off:ulong; + entry_pc:ulong; + calldests_hash:XXHash; +} + +table ELFLoaderFixture { + metadata:FixtureMetadata (required); + input:ELFLoaderCtx (required); + output:ELFLoaderEffects (required); +} + diff --git a/flatbuffers/metadata.fbs b/flatbuffers/metadata.fbs new file mode 100644 index 0000000..92450aa --- /dev/null +++ b/flatbuffers/metadata.fbs @@ -0,0 +1,8 @@ +// Generated from metadata.proto + +namespace org.solana.sealevel.v2; + +table FixtureMetadata { + fn_entrypoint:string (required); +} + diff --git a/shlr/flatbuffers b/shlr/flatbuffers new file mode 160000 index 0000000..26970e8 --- /dev/null +++ b/shlr/flatbuffers @@ -0,0 +1 @@ +Subproject commit 26970e8640957f98094b86f6eb2d7676f98e85f4 diff --git a/shlr/protobuf b/shlr/protobuf new file mode 160000 index 0000000..a9b006b --- /dev/null +++ b/shlr/protobuf @@ -0,0 +1 @@ +Subproject commit a9b006bddd52e289029f16aa77b77e8e0033d9ee