ci(macos): Harden DMG signing - #49268
Merged
gh-worker-dd-mergequeue-cf854d[bot] merged 5 commits intoApr 17, 2026
Merged
Conversation
Contributor
Gitlab CI Configuration ChangesModified Jobs.agent_dmg .agent_dmg:
after_script:
- - "if [ \"${SIGN:-false}\" = true ]; then\n security delete-keychain \"build.keychain\"\
- \ || true\nfi\n"
- sudo umount /Volumes/Agent || true
artifacts:
expire_in: 2 weeks
paths:
- omnibus/pkg/*.dmg
- omnibus/pkg/version-manifest.json
before_script:
- sudo umount /Volumes/Agent || true
- rm -rf "$OMNIBUS_GIT_CACHE_DIR" || true
cache:
- key:
files:
- omnibus/Gemfile
- release.json
prefix: omnibus-deps-$CI_JOB_IMAGE-$CI_JOB_NAME-$OMNIBUS_RUBY_VERSION
paths:
- omnibus/vendor/bundle
id_tokens:
CI_IDENTITIES_GITLAB_ID_TOKEN:
aud: ci-identities
needs:
- go_mod_tidy_check
rules:
- if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/
variables:
SIGN: true
- if: $CI_COMMIT_BRANCH =~ /notarization/
variables:
SIGN: true
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- comp/core/gui/guiimpl/systray/**/*
- '**/*.m'
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- omnibus/**/*
- .gitlab-ci.yml
- release.json
- .gitlab/build/package_build/**/*
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- '*.bazel*'
- deps/**/*
- bazel/**/*
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- if: $CI_COMMIT_BRANCH == "main" || $DEPLOY_AGENT == "true" || $RUN_ALL_BUILDS
== "true" || $DDR_WORKFLOW_ID != null
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- omnibus/**/*
- pkg/config/config_template.yaml
- pkg/config/system-probe_template.yaml
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- allow_failure: true
when: manual
script:
- set -eo pipefail
- export VAULT_ADDR=https://vault.us1.ddbuild.io
- vault login -method=aws -no-print
- "if [ -z \"$TMPDIR\" ]; then\n echo \"TMPDIR must be set\" >& 2\n exit 1\nfi\n"
- export DDA_DIR="$TMPDIR/dda-${CI_JOB_ID}"
- export PATH="$DDA_DIR:$PATH"
- export DDA_NO_DYNAMIC_DEPS=1
- "# Perform installation only if the directory does not exist\nif [ ! -d \"$DDA_DIR\"\
\ ]; then\n robust_curl=\"curl -fsSL --retry 4\" # recommended flags + resist\
\ transient errors like `Connection reset by peer`\n # Get the commit from the\
\ build image variable in the format `vPIPELINE_ID-COMMIT`\n export BUILDIMAGES_COMMIT=\"\
${CI_IMAGE_LINUX#*-}\"\n export DDA_VERSION=\"$($robust_curl https://raw.githubusercontent.com/DataDog/datadog-agent-buildimages/${BUILDIMAGES_COMMIT}/dda.env\
\ | awk -F= '/^DDA_VERSION=/ {print $2}')\"\n # Detect architecture and download\
\ appropriate binary\n if [ \"$(uname -m)\" = \"arm64\" ]; then\n dda_target_triple=\"\
aarch64-apple-darwin\"\n else\n dda_target_triple=\"x86_64-apple-darwin\"\n\
\ fi\n $robust_curl -o dda.tar.gz https://github.com/DataDog/datadog-agent-dev/releases/download/${DDA_VERSION}/dda-${dda_target_triple}.tar.gz\n\
\ tar -xzf dda.tar.gz\n mkdir -p \"$DDA_DIR\"\n sudo mv dda $DDA_DIR\n rm\
\ -f dda.tar.gz\n dda self dep sync -f legacy-tasks\n dda self pip install awscli==1.29.45\n\
fi\n"
- echo Setting up Go
- mkdir -p ~/go
- export GO_VERSION="$(cat .go-version)"
- eval "$(gimme $GO_VERSION)"
- export PATH="$PATH:$GOROOT/bin"
- echo Go version should be $GO_VERSION
- go version
- dda inv check-go-version
- DD_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_API_KEY_ORG2"
token)" || exit $?; export DD_API_KEY
- DD_APP_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_APP_KEY_ORG2"
token)" || exit $?; export DD_APP_KEY
- 'AWS_TOKEN="$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds:
21600")"
RUNNER_ID="$(curl -s http://169.254.169.254/latest/meta-data/instance-id -H "X-aws-ec2-metadata-token:
$AWS_TOKEN" || hostname)"
datadog-ci tag --level job --tags macos_runner:"$RUNNER_ID"
echo "Reported runner ID to Datadog: $RUNNER_ID"
'
- "if [ \"$CI_COMMIT_BRANCH\" = \"main\" ] || [[ \"$CI_COMMIT_BRANCH\" =~ ^[0-9]+\\\
.[0-9]+\\.(x|[0-9]+)$ ]]; then\n dda inv -- -e macos.report-versions -l all ||\
\ true\nfi\n"
- "if [ \"$((RANDOM%20))\" -eq 0 ]; then\n echo Trying to remove inactive versions\n\
\ dda inv -- -e macos.remove-inactive-versions -l python -t \"$PYTHON_VERSION\"\
\ || true\n dda inv -- -e macos.remove-inactive-versions -l go -t \"$(cat .go-version)\"\
\ || true\nfi\n"
- 'export TMPDIR=/tmp/gitlabci
NEWTMPDIR="$RUNNER_TEMP_PROJECT_DIR/gitlabci"
sudo rm -fr "$(realpath $TMPDIR)" "$NEWTMPDIR"
mkdir "$NEWTMPDIR"
sudo ln -fs "$NEWTMPDIR" $TMPDIR
echo "Temporary folder created, TMPDIR=$TMPDIR -> $NEWTMPDIR"
'
- sudo bash -c "rm -rf /var/cache/omnibus/src/*" || true
- pushd omnibus && bundle config set --local path 'vendor/bundle' && popd
- 'export GOMODCACHE=~/gomodcache
mkdir -p $GOMODCACHE
'
- bash .gitlab/build/package_build/build_agent_dmg.sh
- $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/version-manifest.json $S3_SBOM_STORAGE_URI/$CI_JOB_NAME/version-manifest.json
stage: package_build
timeout: 2h
variables:
AWS_SHARED_CREDENTIALS_FILE: ${CI_PROJECT_DIR}/.aws/credentials-by-job-id/${CI_JOB_ID}
BAZELISK_HOME: $XDG_CACHE_HOME/bazelisk
INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus
INTEGRATION_WHEELS_SKIP_CACHE_UPLOAD: true
KEYCHAIN_NAME: build.keychain
NOTARIZATION_ATTEMPTS: 3
NOTARIZATION_TIMEOUT: 15m
NOTARIZATION_WAIT_TIME: 15s
S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable
XDG_CACHE_HOME: $RUNNER_TEMP_PROJECT_DIRagent_dmg-arm64-a7 agent_dmg-arm64-a7:
after_script:
- - "if [ \"${SIGN:-false}\" = true ]; then\n security delete-keychain \"build.keychain\"\
- \ || true\nfi\n"
- sudo umount /Volumes/Agent || true
artifacts:
expire_in: 2 weeks
paths:
- omnibus/pkg/*.dmg
- omnibus/pkg/version-manifest.json
before_script:
- sudo umount /Volumes/Agent || true
- rm -rf "$OMNIBUS_GIT_CACHE_DIR" || true
cache:
- key:
files:
- omnibus/Gemfile
- release.json
prefix: omnibus-deps-$CI_JOB_IMAGE-$CI_JOB_NAME-$OMNIBUS_RUBY_VERSION
paths:
- omnibus/vendor/bundle
id_tokens:
CI_IDENTITIES_GITLAB_ID_TOKEN:
aud: ci-identities
needs:
- go_mod_tidy_check
rules:
- if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/
variables:
SIGN: true
- if: $CI_COMMIT_BRANCH =~ /notarization/
variables:
SIGN: true
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- comp/core/gui/guiimpl/systray/**/*
- '**/*.m'
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- omnibus/**/*
- .gitlab-ci.yml
- release.json
- .gitlab/build/package_build/**/*
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- '*.bazel*'
- deps/**/*
- bazel/**/*
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- if: $CI_COMMIT_BRANCH == "main" || $DEPLOY_AGENT == "true" || $RUN_ALL_BUILDS
== "true" || $DDR_WORKFLOW_ID != null
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- omnibus/**/*
- pkg/config/config_template.yaml
- pkg/config/system-probe_template.yaml
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- allow_failure: true
when: manual
script:
- set -eo pipefail
- export VAULT_ADDR=https://vault.us1.ddbuild.io
- vault login -method=aws -no-print
- "if [ -z \"$TMPDIR\" ]; then\n echo \"TMPDIR must be set\" >& 2\n exit 1\nfi\n"
- export DDA_DIR="$TMPDIR/dda-${CI_JOB_ID}"
- export PATH="$DDA_DIR:$PATH"
- export DDA_NO_DYNAMIC_DEPS=1
- "# Perform installation only if the directory does not exist\nif [ ! -d \"$DDA_DIR\"\
\ ]; then\n robust_curl=\"curl -fsSL --retry 4\" # recommended flags + resist\
\ transient errors like `Connection reset by peer`\n # Get the commit from the\
\ build image variable in the format `vPIPELINE_ID-COMMIT`\n export BUILDIMAGES_COMMIT=\"\
${CI_IMAGE_LINUX#*-}\"\n export DDA_VERSION=\"$($robust_curl https://raw.githubusercontent.com/DataDog/datadog-agent-buildimages/${BUILDIMAGES_COMMIT}/dda.env\
\ | awk -F= '/^DDA_VERSION=/ {print $2}')\"\n # Detect architecture and download\
\ appropriate binary\n if [ \"$(uname -m)\" = \"arm64\" ]; then\n dda_target_triple=\"\
aarch64-apple-darwin\"\n else\n dda_target_triple=\"x86_64-apple-darwin\"\n\
\ fi\n $robust_curl -o dda.tar.gz https://github.com/DataDog/datadog-agent-dev/releases/download/${DDA_VERSION}/dda-${dda_target_triple}.tar.gz\n\
\ tar -xzf dda.tar.gz\n mkdir -p \"$DDA_DIR\"\n sudo mv dda $DDA_DIR\n rm\
\ -f dda.tar.gz\n dda self dep sync -f legacy-tasks\n dda self pip install awscli==1.29.45\n\
fi\n"
- echo Setting up Go
- mkdir -p ~/go
- export GO_VERSION="$(cat .go-version)"
- eval "$(gimme $GO_VERSION)"
- export PATH="$PATH:$GOROOT/bin"
- echo Go version should be $GO_VERSION
- go version
- dda inv check-go-version
- DD_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_API_KEY_ORG2"
token)" || exit $?; export DD_API_KEY
- DD_APP_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_APP_KEY_ORG2"
token)" || exit $?; export DD_APP_KEY
- 'AWS_TOKEN="$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds:
21600")"
RUNNER_ID="$(curl -s http://169.254.169.254/latest/meta-data/instance-id -H "X-aws-ec2-metadata-token:
$AWS_TOKEN" || hostname)"
datadog-ci tag --level job --tags macos_runner:"$RUNNER_ID"
echo "Reported runner ID to Datadog: $RUNNER_ID"
'
- "if [ \"$CI_COMMIT_BRANCH\" = \"main\" ] || [[ \"$CI_COMMIT_BRANCH\" =~ ^[0-9]+\\\
.[0-9]+\\.(x|[0-9]+)$ ]]; then\n dda inv -- -e macos.report-versions -l all ||\
\ true\nfi\n"
- "if [ \"$((RANDOM%20))\" -eq 0 ]; then\n echo Trying to remove inactive versions\n\
\ dda inv -- -e macos.remove-inactive-versions -l python -t \"$PYTHON_VERSION\"\
\ || true\n dda inv -- -e macos.remove-inactive-versions -l go -t \"$(cat .go-version)\"\
\ || true\nfi\n"
- 'export TMPDIR=/tmp/gitlabci
NEWTMPDIR="$RUNNER_TEMP_PROJECT_DIR/gitlabci"
sudo rm -fr "$(realpath $TMPDIR)" "$NEWTMPDIR"
mkdir "$NEWTMPDIR"
sudo ln -fs "$NEWTMPDIR" $TMPDIR
echo "Temporary folder created, TMPDIR=$TMPDIR -> $NEWTMPDIR"
'
- sudo bash -c "rm -rf /var/cache/omnibus/src/*" || true
- pushd omnibus && bundle config set --local path 'vendor/bundle' && popd
- 'export GOMODCACHE=~/gomodcache
mkdir -p $GOMODCACHE
'
- bash .gitlab/build/package_build/build_agent_dmg.sh
- $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/version-manifest.json $S3_SBOM_STORAGE_URI/$CI_JOB_NAME/version-manifest.json
stage: package_build
tags:
- macos:sonoma-arm64
- specific:true
timeout: 2h
variables:
AWS_SHARED_CREDENTIALS_FILE: ${CI_PROJECT_DIR}/.aws/credentials-by-job-id/${CI_JOB_ID}
BAZELISK_HOME: $XDG_CACHE_HOME/bazelisk
INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus
INTEGRATION_WHEELS_SKIP_CACHE_UPLOAD: true
KEYCHAIN_NAME: build.keychain
NOTARIZATION_ATTEMPTS: 3
NOTARIZATION_TIMEOUT: 15m
NOTARIZATION_WAIT_TIME: 15s
S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable
XDG_CACHE_HOME: $RUNNER_TEMP_PROJECT_DIRagent_dmg-x64-a7 agent_dmg-x64-a7:
after_script:
- - "if [ \"${SIGN:-false}\" = true ]; then\n security delete-keychain \"build.keychain\"\
- \ || true\nfi\n"
- sudo umount /Volumes/Agent || true
artifacts:
expire_in: 2 weeks
paths:
- omnibus/pkg/*.dmg
- omnibus/pkg/version-manifest.json
before_script:
- sudo umount /Volumes/Agent || true
- rm -rf "$OMNIBUS_GIT_CACHE_DIR" || true
cache:
- key:
files:
- omnibus/Gemfile
- release.json
prefix: omnibus-deps-$CI_JOB_IMAGE-$CI_JOB_NAME-$OMNIBUS_RUBY_VERSION
paths:
- omnibus/vendor/bundle
id_tokens:
CI_IDENTITIES_GITLAB_ID_TOKEN:
aud: ci-identities
needs:
- go_mod_tidy_check
rules:
- if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/
variables:
SIGN: true
- if: $CI_COMMIT_BRANCH =~ /notarization/
variables:
SIGN: true
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- comp/core/gui/guiimpl/systray/**/*
- '**/*.m'
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- omnibus/**/*
- .gitlab-ci.yml
- release.json
- .gitlab/build/package_build/**/*
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- '*.bazel*'
- deps/**/*
- bazel/**/*
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- if: $CI_COMMIT_BRANCH == "main" || $DEPLOY_AGENT == "true" || $RUN_ALL_BUILDS
== "true" || $DDR_WORKFLOW_ID != null
- changes:
compare_to: $COMPARE_TO_BRANCH
paths:
- omnibus/**/*
- pkg/config/config_template.yaml
- pkg/config/system-probe_template.yaml
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- allow_failure: true
when: manual
script:
- set -eo pipefail
- export VAULT_ADDR=https://vault.us1.ddbuild.io
- vault login -method=aws -no-print
- "if [ -z \"$TMPDIR\" ]; then\n echo \"TMPDIR must be set\" >& 2\n exit 1\nfi\n"
- export DDA_DIR="$TMPDIR/dda-${CI_JOB_ID}"
- export PATH="$DDA_DIR:$PATH"
- export DDA_NO_DYNAMIC_DEPS=1
- "# Perform installation only if the directory does not exist\nif [ ! -d \"$DDA_DIR\"\
\ ]; then\n robust_curl=\"curl -fsSL --retry 4\" # recommended flags + resist\
\ transient errors like `Connection reset by peer`\n # Get the commit from the\
\ build image variable in the format `vPIPELINE_ID-COMMIT`\n export BUILDIMAGES_COMMIT=\"\
${CI_IMAGE_LINUX#*-}\"\n export DDA_VERSION=\"$($robust_curl https://raw.githubusercontent.com/DataDog/datadog-agent-buildimages/${BUILDIMAGES_COMMIT}/dda.env\
\ | awk -F= '/^DDA_VERSION=/ {print $2}')\"\n # Detect architecture and download\
\ appropriate binary\n if [ \"$(uname -m)\" = \"arm64\" ]; then\n dda_target_triple=\"\
aarch64-apple-darwin\"\n else\n dda_target_triple=\"x86_64-apple-darwin\"\n\
\ fi\n $robust_curl -o dda.tar.gz https://github.com/DataDog/datadog-agent-dev/releases/download/${DDA_VERSION}/dda-${dda_target_triple}.tar.gz\n\
\ tar -xzf dda.tar.gz\n mkdir -p \"$DDA_DIR\"\n sudo mv dda $DDA_DIR\n rm\
\ -f dda.tar.gz\n dda self dep sync -f legacy-tasks\n dda self pip install awscli==1.29.45\n\
fi\n"
- echo Setting up Go
- mkdir -p ~/go
- export GO_VERSION="$(cat .go-version)"
- eval "$(gimme $GO_VERSION)"
- export PATH="$PATH:$GOROOT/bin"
- echo Go version should be $GO_VERSION
- go version
- dda inv check-go-version
- DD_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_API_KEY_ORG2"
token)" || exit $?; export DD_API_KEY
- DD_APP_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_APP_KEY_ORG2"
token)" || exit $?; export DD_APP_KEY
- 'AWS_TOKEN="$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds:
21600")"
RUNNER_ID="$(curl -s http://169.254.169.254/latest/meta-data/instance-id -H "X-aws-ec2-metadata-token:
$AWS_TOKEN" || hostname)"
datadog-ci tag --level job --tags macos_runner:"$RUNNER_ID"
echo "Reported runner ID to Datadog: $RUNNER_ID"
'
- "if [ \"$CI_COMMIT_BRANCH\" = \"main\" ] || [[ \"$CI_COMMIT_BRANCH\" =~ ^[0-9]+\\\
.[0-9]+\\.(x|[0-9]+)$ ]]; then\n dda inv -- -e macos.report-versions -l all ||\
\ true\nfi\n"
- "if [ \"$((RANDOM%20))\" -eq 0 ]; then\n echo Trying to remove inactive versions\n\
\ dda inv -- -e macos.remove-inactive-versions -l python -t \"$PYTHON_VERSION\"\
\ || true\n dda inv -- -e macos.remove-inactive-versions -l go -t \"$(cat .go-version)\"\
\ || true\nfi\n"
- 'export TMPDIR=/tmp/gitlabci
NEWTMPDIR="$RUNNER_TEMP_PROJECT_DIR/gitlabci"
sudo rm -fr "$(realpath $TMPDIR)" "$NEWTMPDIR"
mkdir "$NEWTMPDIR"
sudo ln -fs "$NEWTMPDIR" $TMPDIR
echo "Temporary folder created, TMPDIR=$TMPDIR -> $NEWTMPDIR"
'
- sudo bash -c "rm -rf /var/cache/omnibus/src/*" || true
- pushd omnibus && bundle config set --local path 'vendor/bundle' && popd
- 'export GOMODCACHE=~/gomodcache
mkdir -p $GOMODCACHE
'
- bash .gitlab/build/package_build/build_agent_dmg.sh
- $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/version-manifest.json $S3_SBOM_STORAGE_URI/$CI_JOB_NAME/version-manifest.json
stage: package_build
tags:
- macos:sonoma-amd64
- specific:true
timeout: 2h
variables:
AWS_SHARED_CREDENTIALS_FILE: ${CI_PROJECT_DIR}/.aws/credentials-by-job-id/${CI_JOB_ID}
BAZELISK_HOME: $XDG_CACHE_HOME/bazelisk
INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus
INTEGRATION_WHEELS_SKIP_CACHE_UPLOAD: true
KEYCHAIN_NAME: build.keychain
NOTARIZATION_ATTEMPTS: 3
NOTARIZATION_TIMEOUT: 15m
NOTARIZATION_WAIT_TIME: 15s
S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable
XDG_CACHE_HOME: $RUNNER_TEMP_PROJECT_DIRChanges Summary
ℹ️ Diff available in the job log. |
Contributor
Files inventory check summaryFile checks results against ancestor a58d2448: Results for datadog-agent_7.80.0~devel.git.20.098a9e9.pipeline.108258667-1_amd64.deb:No change detected |
Contributor
Static quality checks✅ Please find below the results from static quality gates 31 successful checks with minimal change (< 2 KiB)
On-wire sizes (compressed)
|
Regression DetectorRegression Detector ResultsMetrics dashboard Baseline: c7952ec Optimization Goals: ✅ No significant changes detected
|
| perf | experiment | goal | Δ mean % | Δ mean % CI | trials | links |
|---|---|---|---|---|---|---|
| ➖ | docker_containers_cpu | % cpu utilization | +1.41 | [-1.62, +4.45] | 1 | Logs |
Fine details of change detection per experiment
| perf | experiment | goal | Δ mean % | Δ mean % CI | trials | links |
|---|---|---|---|---|---|---|
| ➖ | quality_gate_logs | % cpu utilization | +3.70 | [+2.06, +5.34] | 1 | Logs bounds checks dashboard |
| ➖ | docker_containers_cpu | % cpu utilization | +1.41 | [-1.62, +4.45] | 1 | Logs |
| ➖ | ddot_metrics | memory utilization | +0.48 | [+0.29, +0.66] | 1 | Logs |
| ➖ | ddot_metrics_sum_cumulative | memory utilization | +0.21 | [+0.06, +0.36] | 1 | Logs |
| ➖ | ddot_metrics_sum_delta | memory utilization | +0.20 | [+0.01, +0.39] | 1 | Logs |
| ➖ | docker_containers_memory | memory utilization | +0.05 | [-0.04, +0.13] | 1 | Logs |
| ➖ | file_to_blackhole_500ms_latency | egress throughput | +0.02 | [-0.37, +0.42] | 1 | Logs |
| ➖ | tcp_dd_logs_filter_exclude | ingress throughput | +0.01 | [-0.10, +0.12] | 1 | Logs |
| ➖ | uds_dogstatsd_to_api | ingress throughput | -0.00 | [-0.21, +0.21] | 1 | Logs |
| ➖ | uds_dogstatsd_to_api_v3 | ingress throughput | -0.01 | [-0.22, +0.19] | 1 | Logs |
| ➖ | uds_dogstatsd_20mb_12k_contexts_20_senders | memory utilization | -0.01 | [-0.08, +0.05] | 1 | Logs |
| ➖ | file_to_blackhole_1000ms_latency | egress throughput | -0.02 | [-0.44, +0.40] | 1 | Logs |
| ➖ | file_to_blackhole_0ms_latency | egress throughput | -0.03 | [-0.57, +0.52] | 1 | Logs |
| ➖ | file_to_blackhole_100ms_latency | egress throughput | -0.08 | [-0.19, +0.02] | 1 | Logs |
| ➖ | otlp_ingest_metrics | memory utilization | -0.15 | [-0.31, +0.00] | 1 | Logs |
| ➖ | ddot_metrics_sum_cumulativetodelta_exporter | memory utilization | -0.18 | [-0.40, +0.04] | 1 | Logs |
| ➖ | quality_gate_idle_all_features | memory utilization | -0.21 | [-0.25, -0.18] | 1 | Logs bounds checks dashboard |
| ➖ | tcp_syslog_to_blackhole | ingress throughput | -0.26 | [-0.46, -0.06] | 1 | Logs |
| ➖ | quality_gate_idle | memory utilization | -0.26 | [-0.31, -0.20] | 1 | Logs bounds checks dashboard |
| ➖ | file_tree | memory utilization | -0.47 | [-0.53, -0.42] | 1 | Logs |
| ➖ | otlp_ingest_logs | memory utilization | -0.56 | [-0.67, -0.46] | 1 | Logs |
| ➖ | ddot_logs | memory utilization | -0.68 | [-0.75, -0.62] | 1 | Logs |
| ➖ | quality_gate_metrics_logs | memory utilization | -0.71 | [-0.94, -0.47] | 1 | Logs bounds checks dashboard |
Bounds Checks: ✅ Passed
| perf | experiment | bounds_check_name | replicates_passed | observed_value | links |
|---|---|---|---|---|---|
| ✅ | docker_containers_cpu | simple_check_run | 10/10 | 557 ≥ 26 | |
| ✅ | docker_containers_memory | memory_usage | 10/10 | 279.30MiB ≤ 370MiB | |
| ✅ | docker_containers_memory | simple_check_run | 10/10 | 698 ≥ 26 | |
| ✅ | file_to_blackhole_0ms_latency | memory_usage | 10/10 | 0.19GiB ≤ 1.20GiB | |
| ✅ | file_to_blackhole_0ms_latency | missed_bytes | 10/10 | 0B = 0B | |
| ✅ | file_to_blackhole_1000ms_latency | memory_usage | 10/10 | 0.24GiB ≤ 1.20GiB | |
| ✅ | file_to_blackhole_1000ms_latency | missed_bytes | 10/10 | 0B = 0B | |
| ✅ | file_to_blackhole_100ms_latency | memory_usage | 10/10 | 0.20GiB ≤ 1.20GiB | |
| ✅ | file_to_blackhole_100ms_latency | missed_bytes | 10/10 | 0B = 0B | |
| ✅ | file_to_blackhole_500ms_latency | memory_usage | 10/10 | 0.22GiB ≤ 1.20GiB | |
| ✅ | file_to_blackhole_500ms_latency | missed_bytes | 10/10 | 0B = 0B | |
| ✅ | quality_gate_idle | intake_connections | 10/10 | 4 = 4 | bounds checks dashboard |
| ✅ | quality_gate_idle | memory_usage | 10/10 | 174.52MiB ≤ 181MiB | bounds checks dashboard |
| ✅ | quality_gate_idle_all_features | intake_connections | 10/10 | 4 = 4 | bounds checks dashboard |
| ✅ | quality_gate_idle_all_features | memory_usage | 10/10 | 503.81MiB ≤ 550MiB | bounds checks dashboard |
| ✅ | quality_gate_logs | intake_connections | 10/10 | 4 ≤ 6 | bounds checks dashboard |
| ✅ | quality_gate_logs | memory_usage | 10/10 | 210.06MiB ≤ 220MiB | bounds checks dashboard |
| ✅ | quality_gate_logs | missed_bytes | 10/10 | 0B = 0B | bounds checks dashboard |
| ✅ | quality_gate_metrics_logs | cpu_usage | 10/10 | 365.28 ≤ 2000 | bounds checks dashboard |
| ✅ | quality_gate_metrics_logs | intake_connections | 10/10 | 4 ≤ 6 | bounds checks dashboard |
| ✅ | quality_gate_metrics_logs | memory_usage | 10/10 | 419.73MiB ≤ 475MiB | bounds checks dashboard |
| ✅ | quality_gate_metrics_logs | missed_bytes | 10/10 | 0B = 0B | bounds checks dashboard |
Explanation
Confidence level: 90.00%
Effect size tolerance: |Δ mean %| ≥ 5.00%
Performance changes are noted in the perf column of each table:
- ✅ = significantly better comparison variant performance
- ❌ = significantly worse comparison variant performance
- ➖ = no significant change in performance
A regression test is an A/B test of target performance in a repeatable rig, where "performance" is measured as "comparison variant minus baseline variant" for an optimization goal (e.g., ingress throughput). Due to intrinsic variability in measuring that goal, we can only estimate its mean value for each experiment; we report uncertainty in that value as a 90.00% confidence interval denoted "Δ mean % CI".
For each experiment, we decide whether a change in performance is a "regression" -- a change worth investigating further -- if all of the following criteria are true:
-
Its estimated |Δ mean %| ≥ 5.00%, indicating the change is big enough to merit a closer look.
-
Its 90.00% confidence interval "Δ mean % CI" does not contain zero, indicating that if our statistical model is accurate, there is at least a 90.00% chance there is a difference in performance between baseline and comparison variants.
-
Its configuration does not mark it "erratic".
CI Pass/Fail Decision
✅ Passed. All Quality Gates passed.
- quality_gate_idle_all_features, bounds check memory_usage: 10/10 replicas passed. Gate passed.
- quality_gate_idle_all_features, bounds check intake_connections: 10/10 replicas passed. Gate passed.
- quality_gate_idle, bounds check intake_connections: 10/10 replicas passed. Gate passed.
- quality_gate_idle, bounds check memory_usage: 10/10 replicas passed. Gate passed.
- quality_gate_metrics_logs, bounds check memory_usage: 10/10 replicas passed. Gate passed.
- quality_gate_metrics_logs, bounds check cpu_usage: 10/10 replicas passed. Gate passed.
- quality_gate_metrics_logs, bounds check intake_connections: 10/10 replicas passed. Gate passed.
- quality_gate_metrics_logs, bounds check missed_bytes: 10/10 replicas passed. Gate passed.
- quality_gate_logs, bounds check missed_bytes: 10/10 replicas passed. Gate passed.
- quality_gate_logs, bounds check intake_connections: 10/10 replicas passed. Gate passed.
- quality_gate_logs, bounds check memory_usage: 10/10 replicas passed. Gate passed.
rdesgroppes
reviewed
Apr 13, 2026
chouetz
added
changelog/no-changelog
rdesgroppes
changed the title
The agent_dmg-x64-a7 and agent_dmg-arm64-a7 jobs on main have been failing intermittently (~6% failure rate) with errSecInternalComponent from codesign. When securityd is in a bad state every retry fails, wasting ~90 minutes before the job gives up. This commit adds four layers of defence: 1. **Stale keychain cleanup (dmg.yml before_script)** — delete any leftover keychain from a previous job that was killed by the 2-hour timeout (after_script does not run on timeout kills). 2. **Belt-and-suspenders cleanup (build_agent_dmg.sh)** — also delete before creating, in case before_script and the build script run in different shell contexts. 3. **Disable keychain auto-lock** — the previous 1-hour timeout (`set-keychain-settings -lut 3600`) risked locking the keychain mid-build. Since the keychain is ephemeral and destroyed in after_script, auto-locking adds no security value. 4. **Keychain unlock + signing healthcheck (datadog-agent-finalize.rb)** — re-unlock the keychain right before signing, then attempt a single codesign operation before launching the parallel batch of ~480 files. If securityd is broken this fails in seconds, not 90 minutes. To support the unlock inside omnibus, KEYCHAIN_NAME and KEYCHAIN_PWD are added to the Darwin env passthrough and excluded from the cache key (same pattern as Linux DEB/RPM signing secrets).
Codesigning and notarization are only required for artifacts that ship to customers, which come from release branches (N.N.x). On main the DMG build still runs (to catch build regressions) but without signing. This eliminates the ~6% errSecInternalComponent failure rate on main entirely: unsigned builds bypass both the codesign retry loop and the Apple notarization round-trips, cutting build time from ~40 min to ~25 min and removing the dominant source of flakiness. Signing remains enabled for: - Release branches (N.N.x) — the only path to shipped artifacts - Branches matching "notarization" — for testing notarization changes
This reverts commit 96343b6a1087896689039b5a7cbe70640a9c1e6d.
chouetz
force-pushed
the
nschweitzer/harden-macos-signing
branch
from
April 15, 2026 07:29
56cd80c to
503578c
Compare
chouetz
marked this pull request as ready for review
April 15, 2026 07:40
![chatgpt-codex-connector[bot]](https://avatars.githubusercontent.com/in/1144995?s=60&v=4){kind=small}
There was a problem hiding this comment.
💡 Codex Review
Here are some automated review suggestions for this pull request.
Reviewed commit: 503578cd01
ℹ️ About Codex in GitHub
Your team has set up Codex to review pull requests in this repo. Reviews are triggered when you
- Open a pull request for review
- Mark a draft as ready
- Comment "@codex review".
If Codex has suggestions, it will comment; otherwise it will react with 👍.
Codex can also answer questions or update the PR. Try commenting "@codex address that feedback".
Comment on lines
+220
to
+221
| set -euo pipefail | ||
| test_file=$(find #{install_dir} -type f -perm +111 ! -path '*/Datadog Agent.app/*' -print | head -1) |
There was a problem hiding this comment.
find | head in healthcheck to avoid SIGPIPE exit
With set -euo pipefail enabled in this shell block, test_file=$(find ... -print | head -1) can fail with exit code 141 because head exits after one line and find receives SIGPIPE. On normal macOS package builds (where #{install_dir} contains many executable files), this aborts before the healthcheck/signing step and makes signed DMG jobs fail consistently. Use a non-pipeline selector (for example find ... -print -quit) so pipefail cannot trip here.
Useful? React with 👍 / 👎.
rdesgroppes
approved these changes
Apr 15, 2026
rdesgroppes
left a comment
rdesgroppes
left a comment
Contributor
There was a problem hiding this comment.
LG. Only question: did you try the changes on a temporary branch with notarization in its name?
aiuto
requested changes
Apr 15, 2026
aiuto
left a comment
aiuto
left a comment
Collaborator
There was a problem hiding this comment.
We will soon drop the omnibus scripts and call the install.sh
directly from a bazel target, so as much as possible should
be in the script.
Address review feedback: use a shell trap in build_agent_dmg.sh to guarantee keychain cleanup on exit/signals, instead of relying on CI before_script/after_script. This makes the script self-contained for the upcoming Bazel migration.
aiuto
approved these changes
Apr 17, 2026
Member
Author
|
/merge |
|
View all feedbacks in Devflow UI.
The expected merge time in
|
gh-worker-dd-mergequeue-cf854d
Bot
deleted the
nschweitzer/harden-macos-signing
branch
April 17, 2026 21:02
datadog-datadog-prod-us1-2 Bot
added a commit
that referenced
this pull request
Apr 17, 2026
This reverts commit d2c2f4e.
gh-worker-dd-mergequeue-cf854d Bot
pushed a commit
that referenced
this pull request
Apr 22, 2026
…thcheck) (#49634) ### What does this PR do? Reapplies #49268 (reverted by #49561) with a one-line fix to the signing healthcheck. ### Why the original was reverted The healthcheck introduced in #49268 used: ```bash set -euo pipefail test_file=\$(find \$install_dir -type f -perm +111 ! -path '*/Datadog Agent.app/*' -print | head -1) ``` Under `pipefail`, with a ~28000-file install tree, `find` keeps writing after `head -1` closes its stdin. `find` then dies with SIGPIPE → exit 141, which `pipefail` propagates. Every signing healthcheck therefore failed with exit 141 even though `codesign` itself succeeded — breaking every `agent_dmg-*` job on `main`. Log confirmation from the broken job (`1607692016`): ``` Signing healthcheck: codesigning ... codesign -o runtime --entitlements ... -s '...' "\$test_file" Signing healthcheck passed The following shell command exited with status 141: ERROR: Job failed: exit status 1 ``` ### The fix Replace `-print | head -1` with `-print -quit`. `find` prints the first match and exits 0 cleanly — no pipe, no SIGPIPE. macOS `find` has supported `-quit` since Leopard. Verified locally: ``` bash -c 'set -euo pipefail; x=\$(find / -type f -perm +111 -print | head -1)' # → exit 141 bash -c 'set -euo pipefail; x=\$(find / -type f -perm +111 -print -quit)' # → exit 0 ``` ### Why the first PR slipped through The healthcheck is gated by `if code_signing_identity`, which is only set when omnibus runs with signing enabled. On regular PR branches `agent_dmg-*` runs with `SIGN=false`, so the healthcheck code never executes pre-merge. The original PR description flagged *"Full CI run on a notarization branch is recommended"* — that validation wasn't done. ### How this PR is validated This branch is named `nschweitzer/notarization-reapply-harden-macos-signing` so the `.agent_dmg:rules` entry `\$CI_COMMIT_BRANCH =~ /notarization/` triggers `SIGN=true` (see `.gitlab/build/package_build/dmg.yml:35-37`). Both `agent_dmg-x64-a7` and `agent_dmg-arm64-a7` will exercise the full keychain + healthcheck + codesign + notarization path end-to-end. **Do not mark ready until both jobs pass on the final commit.** Sorry @rdesgroppes that I overlooked the tests on the first try ### Additional notes - Commits are split for reviewability: the first is the literal revert-of-the-revert of #49561; the second is the SIGPIPE fix with the rationale in the commit message and as a comment above the heredoc. - Motivation / original context: https://datadoghq.atlassian.net/browse/ACIX-1440 Co-authored-by: nicolas.schweitzer <nicolas.schweitzer@datadoghq.com>
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
3 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
What does this PR do?
To reduce flakiness of
agent_dmg-x64-a7andagent_dmg-arm64-a7onmain:before_script(handles jobs killed by the 2h timeout whereafter_scriptnever runs)build_agent_dmg.shbefore keychain creationset-keychain-settings -lut 3600risked locking the keychain mid-build; the keychain is ephemeral so auto-lock adds no security value)KEYCHAIN_NAME/KEYCHAIN_PWDthrough the omnibus env, excluded from cache key)Motivation
https://datadoghq.atlassian.net/browse/ACIX-1440
Over the last 14 days, both DMG jobs have had a ~6% failure rate on
maincaused byerrSecInternalComponentfromcodesign. Investigation showed:x64runner20862611andarm64runner20862607are affected)Describe how you validated your changes
python3 -m pytest tasks/unit_tests/omnibus_tests.py(12 passed)before_script/after_scriptcleanup andset-keychain-settingschanges are safe no-ops when signing is disableddatadog-agent-finalize.rbare guarded byif code_signing_identity(only runs when signing is enabled) andif keychain_name && keychain_pwd(gracefully skips if env vars aren't set)Additional Notes
KEYCHAIN_PWDandKEYCHAIN_NAMEpassthrough follows the same pattern as Linux signing secrets (DEB_SIGNING_PASSPHRASE,RPM_SIGNING_PASSPHRASE) already inOS_SPECIFIC_ENV_PASSTHROUGHretry.sh, so asecuritydfailure is caught immediately. The batch codesign step still usesretry.shfor transient Apple timestamp server errorsnotarizationbranch is recommended to validate the keychain hardening under real signing conditions