From 4e3d54eaf8f27efbdc86e6f1fda1158a19b8f35b Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 14 Jul 2025 14:56:38 +0000 Subject: [PATCH 1/3] Simplify fuzzing coverage (CI) scripts somewhat We recently introduced uploading coverage from no-corpus fuzzing runs into codecov in CI. Here, we simplify some of the scripts that do so, especially removing the second `ci-fuzz.sh` file we added to the repo. --- .github/workflows/build.yml | 7 +++++- ci/ci-fuzz.sh | 24 --------------------- contrib/generate_fuzz_coverage.sh | 36 ++++++++----------------------- 3 files changed, 15 insertions(+), 52 deletions(-) delete mode 100755 ci/ci-fuzz.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 088844788ce..5cdb0cf293b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -129,9 +129,14 @@ jobs: # Will anyone be impressed by your amazing coverage? No # Maybe if codecov wasn't broken we wouldn't need to do this... bash <(curl -s https://codecov.io/bash) -f target/codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" + cargo clean - name: Run fuzz coverage generation run: | - ./ci/ci-fuzz.sh + ./contrib/generate_fuzz_coverage.sh --output-dir `pwd` --output-codecov-json + # Could you use this to fake the coverage report for your PR? Sure. + # Will anyone be impressed by your amazing coverage? No + # Maybe if codecov wasn't broken we wouldn't need to do this... + bash <(curl -s https://codecov.io/bash) -f fuzz-codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" benchmark: runs-on: ubuntu-latest diff --git a/ci/ci-fuzz.sh b/ci/ci-fuzz.sh deleted file mode 100755 index d75b1d84636..00000000000 --- a/ci/ci-fuzz.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -eox pipefail - -echo -e "\n\nGenerating fuzz coverage report" -# In CI, store coverage in target directory for consistency with other artifacts -COVERAGE_DIR="target/coverage-report" -echo "Installing cargo-llvm-cov..." -# Install cargo-llvm-cov if not already installed -cargo install cargo-llvm-cov --locked - -echo "Cleaning up to save disk space..." -rm -rf target/* -echo "Disk cleanup completed" - - -echo "Running fuzz coverage generation..." -./contrib/generate_fuzz_coverage.sh --output-dir "$COVERAGE_DIR" -echo "Coverage generation completed. Checking results..." - -# Upload fuzz coverage to codecov if the file exists (CI only) -if [ -f "target/fuzz-codecov.json" ]; then - echo "Uploading fuzz coverage to codecov..." - bash <(curl -s https://codecov.io/bash) -f "target/fuzz-codecov.json" -F fuzz -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" -fi diff --git a/contrib/generate_fuzz_coverage.sh b/contrib/generate_fuzz_coverage.sh index 55b9dd489a9..e7bf12ea20a 100755 --- a/contrib/generate_fuzz_coverage.sh +++ b/contrib/generate_fuzz_coverage.sh @@ -1,15 +1,19 @@ #!/bin/bash -set -e -set -x +set -ex # Parse command line arguments OUTPUT_DIR="coverage-report" +OUTPUT_CODECOV_JSON=0 while [[ $# -gt 0 ]]; do case $1 in --output-dir) OUTPUT_DIR="$2" shift 2 ;; + --output-codecov-json) + OUTPUT_CODECOV_JSON=1 + shift 1 + ;; *) echo "Unknown option: $1" echo "Usage: $0 [--output-dir OUTPUT_DIRECTORY]" @@ -54,34 +58,12 @@ mkdir -p "$OUTPUT_DIR" export RUSTFLAGS="--cfg=fuzzing --cfg=secp256k1_fuzz --cfg=hashes_fuzz" # dont run this command when running in CI -if [ "$CI" != "true" ] && [ "$GITHUB_ACTIONS" != "true" ]; then +if [ "$OUTPUT_CODECOV_JSON" = "0" ]; then cargo llvm-cov --html --ignore-filename-regex "fuzz/" --output-dir "$OUTPUT_DIR" - - # Check if coverage report was generated successfully - # The report is generated in $OUTPUT_DIR/html/index.html when using --html --output-dir - if [ ! -f "$OUTPUT_DIR/html/index.html" ]; then - echo "Error: Failed to generate coverage report at $OUTPUT_DIR/html/index.html" - echo "Contents of $OUTPUT_DIR:" - ls -la "$OUTPUT_DIR" || echo "Directory $OUTPUT_DIR does not exist" - if [ -d "$OUTPUT_DIR/html" ]; then - echo "Contents of $OUTPUT_DIR/html:" - ls -la "$OUTPUT_DIR/html" - fi - exit 1 - fi echo "Coverage report generated in $OUTPUT_DIR/html/index.html" -fi - -# Generate codecov JSON format if running in CI environment -if [ "$CI" = "true" ] || [ "$GITHUB_ACTIONS" = "true" ]; then - echo "CI environment detected, generating codecov JSON format..." +else cargo llvm-cov --codecov --ignore-filename-regex "fuzz/" --output-path "$OUTPUT_DIR/fuzz-codecov.json" - - if [ -f "$OUTPUT_DIR/fuzz-codecov.json" ] && [[ "$OUTPUT_DIR" == *"target/"* ]]; then - TARGET_DIR="../target" - cp "$OUTPUT_DIR/fuzz-codecov.json" "$TARGET_DIR/fuzz-codecov.json" - echo "Fuzz codecov report copied to $TARGET_DIR/fuzz-codecov.json" - fi + echo "Fuzz codecov report available at $OUTPUT_DIR/fuzz-codecov.json" fi From 08795f21413374e0ce670ec1e97f99b638e491a1 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 15 Jul 2025 12:14:15 +0000 Subject: [PATCH 2/3] Switch to codecov CLI codecov recommends using the new CLI uploader rather than their (deprecated) bash uploader (which we use). It also appears to have a fail-on-error mode which we'd prefer over the bash one which silently swallows errors. --- .github/workflows/build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5cdb0cf293b..75b49d69bec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -125,10 +125,12 @@ jobs: cargo install cargo-llvm-cov export RUSTFLAGS="-Coverflow-checks=off" cargo llvm-cov --features rest-client,rpc-client,tokio,serde --codecov --hide-instantiations --output-path=target/codecov.json + curl --verbose -O https://cli.codecov.io/latest/linux/codecov + chmod +x codecov # Could you use this to fake the coverage report for your PR? Sure. # Will anyone be impressed by your amazing coverage? No # Maybe if codecov wasn't broken we wouldn't need to do this... - bash <(curl -s https://codecov.io/bash) -f target/codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" + ./codecov --verbose upload-process --disable-search --fail-on-error -f target/codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" cargo clean - name: Run fuzz coverage generation run: | @@ -136,7 +138,7 @@ jobs: # Could you use this to fake the coverage report for your PR? Sure. # Will anyone be impressed by your amazing coverage? No # Maybe if codecov wasn't broken we wouldn't need to do this... - bash <(curl -s https://codecov.io/bash) -f fuzz-codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" + ./codecov --verbose upload-process --disable-search --fail-on-error -f fuzz-codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" benchmark: runs-on: ubuntu-latest From 37cd32d4a29754b615a93e96e9c22cb7ae715415 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Fri, 18 Jul 2025 19:16:56 +0000 Subject: [PATCH 3/3] Separate codecov reports out between testing and fuzzing Its useful to be able to identify what kind of coverage we have for a line - test coverage is very different in nature from fuzzing coverage. Here we pass separate "job names" to codecov uploads to do so. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75b49d69bec..b2e37105c73 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -130,7 +130,7 @@ jobs: # Could you use this to fake the coverage report for your PR? Sure. # Will anyone be impressed by your amazing coverage? No # Maybe if codecov wasn't broken we wouldn't need to do this... - ./codecov --verbose upload-process --disable-search --fail-on-error -f target/codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" + ./codecov --verbose upload-process --disable-search --fail-on-error -f target/codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" -F 'tests' cargo clean - name: Run fuzz coverage generation run: | @@ -138,7 +138,7 @@ jobs: # Could you use this to fake the coverage report for your PR? Sure. # Will anyone be impressed by your amazing coverage? No # Maybe if codecov wasn't broken we wouldn't need to do this... - ./codecov --verbose upload-process --disable-search --fail-on-error -f fuzz-codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" + ./codecov --verbose upload-process --disable-search --fail-on-error -f fuzz-codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" -F 'fuzzing' benchmark: runs-on: ubuntu-latest