Skip to content
Tests

fix(toolexec): scope implicit test coverage #2714

Sign in to view logs

fix(toolexec): scope implicit test coverage

fix(toolexec): scope implicit test coverage #2714

Workflow file for this run

.github/workflows/validate.yml at 33af41a
name: Tests
on:
pull_request:
branches: ["**"]
merge_group:
branches: [main]
concurrency:
group: ${{ github.workflow }}-${{ github.event.number || ((github.event_name == 'push' && github.sha) || github.ref) }}
cancel-in-progress: true
permissions: read-all
env:
# Make sure we're actually testing with the intended Go release (i.e, ensure
# no automatic toolchain download happens).
GOTOOLCHAIN: local
jobs:
##############################################################################
# Run all the code generators; and refresh the LICENSES-3rdparty.csv file
generate:
runs-on: ubuntu-latest
name: Run all generators
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Setup go
id: setup-go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
with:
go-version: oldstable
cache-dependency-path: "**/go.mod"
- name: Run 'go generate ./...'
run: |-
mkdir -p "${GOCOVERDIR}"
find . -name go.mod -execdir go generate ./... \;
env:
GOFLAGS: -cover -covermode=atomic -coverpkg=github.com/DataDog/orchestrion/...,./...
GOCOVERDIR: ${{ github.workspace }}/coverage
- name: Consolidate coverage report
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
run: go tool covdata textfmt -i ./coverage -o ./coverage/generator.out
- name: Determine simple go version
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
id: go
run: |-
set -euo pipefail
echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' | cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
- name: Upload coverage report
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: coverage-generators+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
path: ./coverage/generator.out
- name: Run 'go mod tidy'
# Don't run for push, it's not necessary
if: github.event_name != 'push'
run: |-
currentTag=$(go run . version -static | cut -d' ' -f 2)
find . -iname go.mod -not -path './_docs/themes/**' -print0 | while IFS= read -r -d '' gomod; do
dir="$(dirname "${gomod}")"
if [[ -n $(go -C="${dir}" mod edit --json | jq '(.Replace // []).[] | select(.Old.Path == "github.com/DataDog/orchestrion")') ]]; then
echo "Aligning orchestrion version in ${dir} to ${currentTag} if necessary..."
go -C="${dir}" mod edit -go="$(go -C="${dir}" mod edit -json | jq -r .Go)" -require="github.com/DataDog/orchestrion@${currentTag}"
fi
go -C="${dir}" mod tidy -go="$(go -C="${dir}" mod edit -json | jq -r .Go)"
go -C="${dir}" mod edit -toolchain=none
done
- name: Refresh LICENSE-3rdparty.csv
run: ./_tools/make-licenses.sh
env:
TMPDIR: ${{ runner.temp }}
- name: Update embedded documentation
run: make docs
- name: Check if working tree is dirty
# Don't run for push, it's not necessary
if: github.event_name != 'push'
id: is-tree-dirty
run: |-
set -euxo pipefail
git add .
git status
git diff --staged --patch --exit-code > .repo.patch || echo 'result=true' >> "${GITHUB_OUTPUT}"
- name: Upload patch
# CONTRACT: .github/workflows/self-mutation.yml watches this workflow via `workflow_run` and
# commits this artifact back onto the pull request's branch. Renaming the artifact, the file,
# or this workflow's `name:` breaks that silently -- the two files are not otherwise linked.
if: github.event_name != 'push' && steps.is-tree-dirty.outputs.result == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
if-no-files-found: error
include-hidden-files: true
name: repo.patch
path: .repo.patch
- name: Fail build if working tree is dirty
if: github.event_name != 'push' && steps.is-tree-dirty.outputs.result == 'true'
env:
IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
run: |-
echo "::error::Files have been modified by 'go generate ./...' (see logs)."
cat .repo.patch
if [ "${IS_FORK}" = 'true' ]; then
echo '::notice::Automatic fixups are not available for fork pull requests. Run the' \
'generators locally and commit the result: `find . -name go.mod -execdir go' \
'generate ./... \;`, `./_tools/make-licenses.sh`, then `make docs`.'
else
echo '::notice::The "Self Mutation" workflow will push these changes to this branch' \
'once this run completes; see the Actions tab.'
fi
exit 1
##############################################################################
# Run the various linters we have set up...
lint:
needs: generate
runs-on: ubuntu-latest
name: Linters
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Setup go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Verify license headers
run: go run ./_tools/headercheck/header_check.go
- name: Check documentation for misspellings
uses: crate-ci/typos@8a48f81b6c64dcfea44b3633223084c4be58ac5f # v1.49.0
lint-go:
needs: generate
name: GolangCI Lint (${{ matrix.runs-on }} | ${{ matrix.working-directory || 'main' }})
strategy:
fail-fast: false # Get as much feedback as possible in one go...
matrix:
runs-on: [macos-latest, ubuntu-latest, windows-latest]
working-directory: ["", samples]
runs-on: ${{ matrix.runs-on }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Setup go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Run go vet
run: go vet ./...
- name: Run golangci-lint
uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6
with:
version: v2.9.0
working-directory: ${{ matrix.working-directory }}
##############################################################################
# Verify all GitHub workflows have hash-pinned actions
lint-actions:
runs-on: ubuntu-latest
name: GitHub Workflow Linters
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Ensure SHA pinned actions
uses: zgosalvez/github-actions-ensure-sha-pinned-actions@c5fc58bd0be7a4b94b73ce40250322d5b838a108 # v3
with:
allowlist: DataDog/dd-trace-go # Trust actions/workflows in the dd-trace-go repository
- name: Verify actions are pinned with ratchet
uses: sethvargo/ratchet@9ca118155b0f79464185703a50da367e111107cd # ratchet:exclude
with:
files: ".github/workflows/*.yml .github/actions/**/action.yml"
- name: Run actionlint
uses: reviewdog/action-actionlint@d63ba7532e0942965320cd8d73cbae4c7b3c5283 # v1.73.1
with:
reporter: github-check
level: error
fail_level: error
##############################################################################
# Lint Makefiles
lint-makefile:
runs-on: ubuntu-latest
name: Makefile Linters
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Setup go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Install checkmake
run: go install github.com/checkmake/checkmake/cmd/checkmake@latest
- name: Lint Makefiles
run: make lint/makefile
##############################################################################
# Check code formatting
format:
needs: generate
runs-on: ubuntu-latest
name: Format Check
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Setup go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Run make format
run: make format
- name: Check for formatting changes
run: |-
if [ -n "$(git status --porcelain)" ]; then
echo "::error::Files need formatting. Run 'make format' locally and commit the changes."
git status
git diff
exit 1
fi
##############################################################################
# Run all unit tests with coverage enabled
unit-tests:
needs: generate
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: ${{ github.event_name == 'merge_group' }}
matrix:
# Not running unit tests on macOS, because it's UNIX-like and the only
# os-specific code paths are UNIX vs. Windows.
runs-on: [ubuntu-latest, windows-latest]
go-version: [oldstable, stable]
name: Unit tests (go ${{ matrix.go-version }}, ${{ matrix.runs-on }})
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Setup Go
id: setup-go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.mod"
- name: Run unit tests
shell: bash
run: |-
mkdir -p coverage
# The root package's tests each run several complete instrumented builds of a synthetic module,
# which takes them well past Go's default 10 minute per-package timeout on CI runners.
test_args=("-shuffle=on" "-race" "-timeout=30m")
if [ "${{ github.event_name }}" != "merge_group" ]; then
test_args+=("-cover" "-covermode=atomic" "-coverpkg=./...,github.com/DataDog/orchestrion/...")
fi
go test "${test_args[@]}" "-coverprofile=${{ github.workspace }}/coverage/unit.out" ./...
go -C samples test "${test_args[@]}" "-coverprofile=${{ github.workspace }}/coverage/samples.out" ./...
- name: Determine simple go version
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
id: go
shell: bash
run: |-
set -euo pipefail
echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' | cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
- name: Upload coverage report
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: coverage-unit+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
path: |-
./coverage/unit.out
./coverage/samples.out
##############################################################################
# Run all benchmarks and generate report
benchmark:
needs: generate
# Benchmarks execute repository-controlled Go code on a custom runner. Only
# run them for merge queues or PRs from this repository, where the author
# already has write access, and never for fork pull requests.
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: arm-8core-linux
name: Benchmarks (run ${{ matrix.run_number }})
strategy:
matrix:
# Run benchmarks 6 times on different runners to smooth out compute discrepancies...
run_number: [1, 2, 3, 4, 5, 6]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Setup go
id: setup-go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Run benchmarks
run: |-
set -euo pipefail
go test -bench=. -timeout=1h -run=^$ . | tee ${{ runner.temp }}/benchmarks-${{ matrix.run_number }}.txt
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Upload benchmark report (raw)
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
if-no-files-found: error
name: benchmarks-${{ matrix.run_number }}.txt
path: ${{ runner.temp }}/benchmarks-${{ matrix.run_number }}.txt
benchmark-report:
needs: benchmark
runs-on: ubuntu-latest
name: Benchmark Report
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Setup go
id: setup-go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Download benchmark reports (raw)
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: benchmarks-*.txt
path: ${{ runner.temp }}
merge-multiple: true
- name: Format report
run: |-
set -euo pipefail
go -C _tools run golang.org/x/perf/cmd/benchstat \
-table=.name -row=/repo@alpha -col=/variant \
${{ runner.temp }}/benchmarks-*.txt \
| tee ${{ runner.temp }}/benchmarks-formatted.txt
- name: Setting Job Summary
run: |-
{
echo "### Benchmark Report"
echo '```'
cat ${{ runner.temp }}/benchmarks-formatted.txt
echo '```'
} >> "${GITHUB_STEP_SUMMARY}"
##############################################################################
# Run all integration tests and gather extensive coverage
integration-tests:
name: Integration Tests
needs: generate
if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
uses: DataDog/dd-trace-go/.github/workflows/orchestrion.yml@03c4c2a4c771fb1a2d6fb2da2af32dee8a0f724c # ratchet:DataDog/dd-trace-go/.github/workflows/orchestrion.yml@main
permissions:
id-token: write
contents: read
with:
collect-coverage: ${{ github.event_name != 'merge_group' }}
orchestrion-version: ${{ github.sha }}
##############################################################################
# Run end-to-end tests (PGO, etc.)
e2e-tests:
name: E2E Tests (go ${{ matrix.go-version }}, ${{ matrix.runs-on }})
needs: generate
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false
matrix:
runs-on: [ubuntu-latest, macos-latest, windows-latest]
go-version: [oldstable, stable]
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.mod"
- name: Build orchestrion
run: go build -o bin/orchestrion .
- name: Run E2E tests
run: go test -tags=e2e -v -timeout=15m ./test/e2e/ 2>&1 | tee test-e2e.log
env:
# These tests pin Orchestrion into a synthetic module, which downloads the complete
# `dd-trace-go` integration set. Those modules are not part of any committed `go.mod` file, so
# they are never in the Go module cache restored by `actions/setup-go`, and downloading them
# alone can take several minutes on the slowest runners.
E2E_TEST_TIMEOUT: 12m
- name: Upload E2E artifacts on failure
if: failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: e2e-artifacts-${{ matrix.runs-on }}-go${{ matrix.go-version }}
path: |
test-e2e.log
test/e2e/**/*.log
test/e2e/**/*.pgo
test/e2e/**/*.prof
if-no-files-found: ignore
##############################################################################
# Assert everything is complete. This simplifies branch protection settings.
complete:
runs-on: ubuntu-latest
name: Complete
needs:
- generate
- lint
- lint-go
- lint-actions
- lint-makefile
- format
- unit-tests
- integration-tests
- e2e-tests
- benchmark
if: '!cancelled()'
steps:
- name: Success
if: needs.generate.result != 'failure' && needs.lint.result != 'failure' && needs.lint-go.result != 'failure' && needs.lint-actions.result != 'failure' && needs.lint-makefile.result != 'failure' && needs.format.result != 'failure' && needs.unit-tests.result != 'failure' && needs.integration-tests.result != 'failure' && needs.e2e-tests.result != 'failure'
run: echo "OK"
- name: Failed
if: needs.generate.result == 'failure' || needs.lint.result == 'failure' || needs.lint-go.result == 'failure' || needs.lint-actions.result == 'failure' || needs.lint-makefile.result == 'failure' || needs.format.result == 'failure' || needs.unit-tests.result == 'failure' || needs.integration-tests.result == 'failure' || needs.e2e-tests.result == 'failure'
run: |-
echo "Failed!"
exit 1
##############################################################################
# Collect coverage artifacts and upload to Datadog.
coverage-matrix:
runs-on: ubuntu-latest
name: Compute Coverage Matrix
needs:
- unit-tests
- integration-tests
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
outputs:
artifacts: ${{ steps.compute.outputs.artifacts }}
files: ${{ steps.compute.outputs.files }}
matrix: ${{ steps.compute.outputs.matrix }}
steps:
- name: Setup Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v4
with:
node-version: latest
- name: Download Artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: coverage-*
- name: Compute Matrix
id: compute
run: |-
node <<-EOF
const fs = require('node:fs');
const path = require('node:path');
const process = require('node:process');
const flags = [];
const flagFiles = {};
for (const dirname of fs.readdirSync(process.cwd())) {
const prefix = 'coverage-';
if (!dirname.startsWith(prefix)) {
continue;
}
const files = fs.globSync(path.join(process.cwd(), dirname, '**', '*.out'));
console.log('Found asset named ' + dirname + ' with ' + files.length + ' report files.');
if (files.length == 0) {
continue;
}
for (const flag of dirname.substring(prefix.length).split('+')) {
if (!flags.includes(flag)) {
flags.push(flag);
}
flagFiles[flag] ??= [];
flagFiles[flag].push(...files);
}
}
console.log('Flags:', flags);
console.log('Files:', flagFiles);
// Join the lists because the workflow subsequently expects a whitespace-separted list.
for (const [flag, list] of Object.entries(flagFiles)) {
flagFiles[flag] = list.join(' ');
}
fs.writeFileSync(
path.join(process.env.GITHUB_OUTPUT),
[
"matrix=" + JSON.stringify({ flag: flags }),
"files=" + JSON.stringify(flagFiles),
].join('\n'),
);
EOF
coverage-upload-datadog:
runs-on: ubuntu-latest
name: Upload coverage to Datadog (${{ matrix.flag }})
needs: [coverage-matrix]
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
strategy:
fail-fast: true
matrix: ${{ fromJson(needs.coverage-matrix.outputs.matrix) }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
- name: Download Artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: coverage-*
- name: Upload coverage to Datadog
if: always()
continue-on-error: true
uses: DataDog/coverage-upload-github-action@d9548b1c3c4ab639d5d3ab29a1508af188975f77 # v1
with:
api_key: ${{ secrets.DD_CI_API_KEY }}
files: ${{ fromJson(needs.coverage-matrix.outputs.files)[matrix.flag] }}
flags: ${{ matrix.flag }}