Skip to content
E2E / Main and Manual Suite

E2E PR #10695 (1f3f0687-2082-4dab-8b7f-3b7ed725347c) #8711

Sign in to view logs

E2E PR #10695 (1f3f0687-2082-4dab-8b7f-3b7ed725347c)

E2E PR #10695 (1f3f0687-2082-4dab-8b7f-3b7ed725347c) #8711

Workflow file for this run

.github/workflows/e2e.yaml at 860962d
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Manual PR runs bind the selected head or base SHA independently from reusable image publication.
name: E2E / Main and Manual Suite
run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id) || inputs.correlation_id != '' && inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && !inputs.allow_jetson_dispatch && !inputs.allow_dgx_spark_runner_queue && format('E2E full {0} ({1})', github.ref_name, inputs.correlation_id) || inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && !inputs.allow_jetson_dispatch && !inputs.allow_dgx_spark_runner_queue && format('E2E full {0}', github.ref_name) || inputs.correlation_id != '' && format('E2E {0} ({1})', github.ref_name, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}"
on:
push:
branches: [main]
workflow_dispatch:
inputs:
targets:
description: "Optional comma-separated typed target IDs. On trusted main runs, empty jobs and targets select all supported registry E2E tests. PR revision runs use the trusted controller target matrix."
required: false
default: ""
type: string
jobs:
description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for full Launchable E2E, or select staging-brev-launchable-identity for the explicit identity smoke run. Jetson dispatch and DGX Spark require their opt-in flags. PR revisions use the trusted controller matrix."
required: false
default: ""
type: string
include_staging_brev_launchable:
description: "Include staging Brev Launchable in a full E2E run when jobs and targets are empty."
required: false
default: false
type: boolean
inference_mode:
description: "Inference adapter mode for compatible Vitest E2E jobs: mock, internal-nvidia, or public-nvidia."
required: false
default: "mock"
type: choice
options:
- mock
- internal-nvidia
- public-nvidia
gateway_runtime:
description: "Compatibility input for a single gateway runtime. Prefer gateway_runtimes for parallel runtime coverage."
required: false
default: "docker"
type: choice
options:
- docker
- podman
gateway_runtimes:
description: "Comma-separated managed gateway runtimes to execute in parallel, for example docker,podman."
required: false
default: ""
type: string
allow_jetson_dispatch:
description: "Set true for a manual jetson-nvmap-gpu run. Main pushes dispatch it automatically. The operator-owned dispatch backend must be available, and JETSON_DISPATCH_URL must contain its verified HTTPS origin. Refer to test/e2e/docs/jetson-dispatch.md."
required: false
default: false
type: boolean
allow_dgx_spark_runner_queue:
description: "Before setting true for llama-cpp-dgx-spark-qualification, obtain repository administrator confirmation of an online DGX Spark runner in the authoritative NVIDIA/NemoClaw Settings -> Actions -> Runners inventory; queued jobs do not honor timeout-minutes before assignment."
required: false
default: false
type: boolean
pr_number:
description: Optional current PR number for manual exact-revision E2E.
required: false
type: string
default: ""
post_to_slack:
description: Post a selective-dispatch scorecard to the preview Slack route.
required: false
default: false
type: boolean
checkout_sha:
description: Optional lowercase 40-character PR head or exact PR base SHA for manual E2E.
required: false
default: ""
type: string
checkout_repository:
description: Optional repository containing the selected PR head or base SHA for manual E2E.
required: false
default: ""
type: string
base_sha:
description: Optional lowercase PR base SHA for manual exact-revision E2E.
required: false
default: ""
type: string
workflow_sha:
description: Optional trusted main workflow SHA for manual E2E.
required: false
default: ""
type: string
managed_image_revision:
description: Optional ancestor revision whose successful immutable PR managed-image contracts should be reused.
required: false
default: ""
type: string
correlation_id:
description: Run correlation ID (UUIDv4).
required: false
default: ""
type: string
permissions:
checks: read
contents: read
pull-requests: read
concurrency:
group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && format('full-{0}', github.run_id)) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'manual-pr' || inputs.jobs || 'all-jobs' }}
cancel-in-progress: ${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable,') && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable-identity,') && !inputs.include_staging_brev_launchable }}
env:
NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
NEMOCLAW_E2E_CORRELATION_ID: ${{ inputs.correlation_id }}
NEMOCLAW_E2E_SHARD: default
NEMOCLAW_GATEWAY_RUNTIMES: ${{ inputs.gateway_runtimes || inputs.gateway_runtime || 'docker' }}
jobs:
package-openshell-sdk:
if: ${{ github.event_name == 'workflow_dispatch' && contains(format(',{0},', inputs.jobs), ',external-gateway-health,') }}
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
packages: read
outputs:
artifact_name: ${{ steps.identity.outputs.artifact_name }}
steps:
- name: Check out trusted OpenShell SDK package verifier
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: |
ci/reviewed-npm-audit.json
scripts/audit-reviewed-npm-graph.mts
scripts/checks/package-openshell-sdk-for-pr.mts
scripts/lib/openclaw-npm-remediation.mts
scripts/lib/reviewed-npm-archive.mts
scripts/lib/reviewed-npm-audit.mts
sparse-checkout-cone-mode: false
- name: Set up Node for reviewed package download
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22"
registry-url: https://npm.pkg.github.com
scope: "@nvidia"
- id: package
name: Download and verify exact OpenShell SDK package
env:
NEMOCLAW_OPEN_SHELL_SDK_OUTPUT_DIRECTORY: ${{ runner.temp }}/openshell-sdk
NODE_AUTH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
artifact_path="$(node --experimental-strip-types scripts/checks/package-openshell-sdk-for-pr.mts)"
test -n "$artifact_path"
printf 'artifact_path=%s\n' "$artifact_path" >> "$GITHUB_OUTPUT"
- id: identity
name: Record reviewed OpenShell SDK artifact identity
env:
RUN_ATTEMPT: ${{ github.run_attempt }}
RUN_ID: ${{ github.run_id }}
run: |
set -euo pipefail
artifact_name="openshell-sdk-e2e-${RUN_ID}-${RUN_ATTEMPT}"
printf 'artifact_name=%s\n' "$artifact_name" >> "$GITHUB_OUTPUT"
- name: Upload reviewed OpenShell SDK archive
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ steps.identity.outputs.artifact_name }}
path: ${{ steps.package.outputs.artifact_path }}
if-no-files-found: error
retention-days: 1
base-image-publication:
runs-on: ubuntu-latest
timeout-minutes: 55
outputs:
dcode_base_contract: ${{ steps.validate_dcode_base.outputs.contract }}
dcode_base_ref: ${{ steps.validate_dcode_base.outputs.base_ref }}
managed_image_receipt: ${{ steps.validate_managed_cohort.outputs.receipt }}
managed_image_revision: ${{ steps.validate_managed_cohort.outputs.revision }}
managed_image_catalog: ${{ steps.select_pr_source.outputs.catalog }}
workload_source: managed-image
permissions:
actions: read
contents: read
pull-requests: read
steps:
- id: publication_mode
name: Classify base-image publication requirement
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
BASE_SHA: ${{ inputs.base_sha }}
EVENT_NAME: ${{ github.event_name }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
case "${REPOSITORY}:${REF}:${EVENT_NAME}:${CHECKOUT_SHA:+controller}" in
NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:)
expected_sha="$WORKFLOW_SHA"
allow_non_head=0
select_nearest_successful=0
;;
NVIDIA/NemoClaw:refs/heads/*:workflow_dispatch:controller)
[[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::manual PR publication selection requires an exact base SHA" >&2
exit 1
}
expected_sha="$BASE_SHA"
allow_non_head=1
select_nearest_successful=1
;;
*)
echo "::error::base-image publication mode is not trusted" >&2
exit 1
;;
esac
printf 'allow_non_head=%s\n' "${allow_non_head}" >> "${GITHUB_OUTPUT}"
printf 'expected_sha=%s\n' "${expected_sha}" >> "${GITHUB_OUTPUT}"
printf 'select_nearest_successful=%s\n' "${select_nearest_successful}" >> "${GITHUB_OUTPUT}"
- name: Check out trusted E2E workflow
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
fetch-depth: 0
persist-credentials: false
- name: Set up Node for publication verification
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- id: select_pr_source
name: Resolve exact PR managed-image publication
if: ${{ inputs.pr_number != '' }}
env:
BASE_SHA: ${{ inputs.base_sha }}
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }}
CANDIDATE_SHA: ${{ inputs.checkout_sha }}
GITHUB_TOKEN: ${{ github.token }}
MANAGED_IMAGE_SHA: ${{ inputs.managed_image_revision }}
PR_NUMBER: ${{ inputs.pr_number }}
shell: bash
run: |
set -euo pipefail
catalog_path="${RUNNER_TEMP}/pr-managed-image-catalog.json"
rm -f -- "$catalog_path"
if [[ -n "$MANAGED_IMAGE_SHA" ]]; then
[[ "$MANAGED_IMAGE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::managed_image_revision must be a lowercase 40-character SHA" >&2
exit 1
}
git merge-base --is-ancestor "$MANAGED_IMAGE_SHA" "$CANDIDATE_SHA" || {
echo "::error::managed_image_revision must be an ancestor of checkout_sha" >&2
exit 1
}
fi
selection="$(node --experimental-strip-types --no-warnings tools/e2e/pr-managed-image-publication.mts "$catalog_path")"
case "$selection" in
base-cohort)
[[ ! -e "$catalog_path" && ! -L "$catalog_path" ]] || {
echo "::error::base-cohort selection produced a candidate catalog" >&2
exit 1
}
;;
candidate-catalog)
[[ -f "$catalog_path" && ! -L "$catalog_path" && -s "$catalog_path" ]] || {
echo "::error::exact PR managed-image catalog is invalid" >&2
exit 1
}
catalog="$(jq -ce . "$catalog_path")"
(( ${#catalog} <= 65536 )) || {
echo "::error::exact PR managed-image catalog exceeds the output limit" >&2
exit 1
}
printf 'catalog=%s\n' "$catalog" >>"$GITHUB_OUTPUT"
echo "::notice::Jetson dispatch is unavailable because the exact PR managed-image catalog qualifies linux/amd64 only."
;;
*) echo "::error::PR managed-image selection is invalid" >&2; exit 1 ;;
esac
printf 'selection=%s\n' "$selection" >>"$GITHUB_OUTPUT"
- id: publication
name: Select base and optional managed-image publication
env:
EXPECTED_SHA: ${{ steps.publication_mode.outputs.expected_sha }}
GITHUB_TOKEN: ${{ github.token }}
PUBLICATION_HISTORY_ALLOW_NON_HEAD: ${{ steps.publication_mode.outputs.allow_non_head }}
REQUIRE_MANAGED_IMAGE_PUBLICATION: ${{ steps.select_pr_source.outputs.selection == 'candidate-catalog' && '0' || '1' }}
SELECT_NEAREST_SUCCESSFUL_PUBLICATION: ${{ steps.publication_mode.outputs.select_nearest_successful }}
shell: bash
run: |
set -euo pipefail
export GITHUB_REF=refs/heads/main
export GITHUB_SHA="$EXPECTED_SHA"
wait_seconds=3000
if [[ "$SELECT_NEAREST_SUCCESSFUL_PUBLICATION" == "1" ]]; then
wait_seconds=300
fi
node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds "$wait_seconds" --poll-seconds 30
- name: Download immutable Deep Agents Code base contract
env:
GITHUB_TOKEN: ${{ github.token }}
PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }}
PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }}
PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }}
run: node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/dcode-base-contract"
- id: validate_dcode_base
name: Validate immutable Deep Agents Code base
env:
PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }}
PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }}
PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }}
run: node --experimental-strip-types --no-warnings tools/e2e/dcode-base-image-contract.mts "${RUNNER_TEMP}/dcode-base-contract/contract.json"
- id: download_managed_cohort
name: Download immutable managed-image cohort contract
if: ${{ inputs.pr_number == '' || steps.select_pr_source.outputs.selection == 'base-cohort' }}
env:
GITHUB_TOKEN: ${{ github.token }}
PUBLICATION_ARTIFACT_KIND: managed-image-cohort
PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }}
PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }}
PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }}
run: node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/managed-image-cohort"
- id: validate_managed_cohort
name: Validate immutable managed-image cohort contract
if: ${{ inputs.pr_number == '' || steps.select_pr_source.outputs.selection == 'base-cohort' }}
env:
PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }}
PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }}
PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }}
run: node --experimental-strip-types --no-warnings tools/e2e/managed-image-cohort-contract.mts "${RUNNER_TEMP}/managed-image-cohort/cohort.json"
generate-matrix:
needs: base-image-publication
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
actions: read
contents: read
pull-requests: read
outputs:
cli_artifact_provenance: ${{ steps.record_cli_artifact.outputs.provenance }}
workload_source: managed-image
e2e_credentials_allowed: ${{ steps.e2e_credentials.outputs.allowed }}
matrix: ${{ steps.matrix.outputs.matrix }}
test_matrix: ${{ steps.matrix.outputs.test_matrix }}
hermes_selected: ${{ steps.matrix.outputs.hermes_selected }}
explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }}
release_required_jobs: ${{ steps.matrix.outputs.release_required_jobs }}
selected_jobs: ${{ steps.matrix.outputs.selected_jobs }}
selected_workflow_jobs: ${{ steps.matrix.outputs.selected_workflow_jobs }}
catalogue_standard_matrix: ${{ steps.matrix.outputs.catalogue_standard_matrix }}
catalogue_nvidia_api_matrix: ${{ steps.matrix.outputs.catalogue_nvidia_api_matrix }}
catalogue_nvidia_inference_matrix: ${{ steps.matrix.outputs.catalogue_nvidia_inference_matrix }}
catalogue_github_read_matrix: ${{ steps.matrix.outputs.catalogue_github_read_matrix }}
catalogue_brave_nvidia_inference_matrix: ${{ steps.matrix.outputs.catalogue_brave_nvidia_inference_matrix }}
gateway_runtimes: ${{ steps.matrix.outputs.gateway_runtimes }}
runtime_providers_by_job: ${{ steps.matrix.outputs.runtime_providers_by_job }}
runner_routing: ${{ steps.runner_routing.outputs.runner_routing }}
steps:
- id: runner_routing
name: Build trusted larger-runner routing
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
LARGER_RUNNER_LABEL: ${{ vars.E2E_LARGER_RUNNER_LABEL }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
larger_runner="ubuntu-latest"
if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -z "${CHECKOUT_SHA}" && -n "${LARGER_RUNNER_LABEL}" ]]; then
if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then
echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2
exit 1
fi
larger_runner="${LARGER_RUNNER_LABEL}"
fi
runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"common-egress-agent":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger}')"
printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}"
- id: candidate_authorization
name: Authenticate manual PR dispatch
if: ${{ inputs.pr_number != '' || inputs.checkout_sha != '' || inputs.checkout_repository != '' || inputs.base_sha != '' || inputs.workflow_sha != '' }}
env:
ALLOW_DGX_SPARK_RUNNER_QUEUE: ${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }}
ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}
BASE_SHA: ${{ inputs.base_sha }}
CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }}
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
INCLUDE_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable && 'true' || 'false' }}
JOBS: ${{ inputs.jobs }}
PR_NUMBER: ${{ inputs.pr_number }}
TARGETS: ${{ inputs.targets }}
WORKFLOW_EVENT: ${{ github.event_name }}
WORKFLOW_REF: ${{ github.ref }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == refs/heads/* ]] || {
echo "::error::Manual PR E2E must be dispatched from this repository branch" >&2
exit 1
}
[[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::pr_number must be a positive integer" >&2; exit 1; }
[[ "$CHECKOUT_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { echo "::error::checkout_repository must be an owner/repository name" >&2; exit 1; }
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA" >&2; exit 1; }
[[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::base_sha must be a lowercase 40-character SHA" >&2; exit 1; }
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::workflow_sha must match the trusted main workflow SHA" >&2; exit 1; }
pull_json="$(curl --fail --silent --show-error --proto '=https' \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")"
[[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must be open" >&2; exit 1; }
[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository must be NVIDIA/NemoClaw" >&2; exit 1; }
[[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch must be main" >&2; exit 1; }
[[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; }
if [[ "$CHECKOUT_SHA" == "$BASE_SHA" ]]; then
[[ "$ALLOW_JETSON_DISPATCH" != "true" && "$ALLOW_DGX_SPARK_RUNNER_QUEUE" != "true" ]] || { echo "::error::exact-base E2E cannot launch dedicated hardware dispatches" >&2; exit 1; }
[[ ",${TARGETS}," != *",jetson-nvmap-gpu,"* &&
",${JOBS}," != *",jetson-nvmap-gpu,"* &&
",${TARGETS}," != *",llama-cpp-dgx-spark-qualification,"* &&
",${JOBS}," != *",llama-cpp-dgx-spark-qualification,"* ]] || { echo "::error::exact-base E2E cannot select dedicated hardware jobs" >&2; exit 1; }
[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR base repository" >&2; exit 1; }
else
[[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; }
[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; }
fi
nvidia_owned=false
if [[ "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" &&
"$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]]; then
nvidia_owned=true
fi
if [[ ",${JOBS}," == *",staging-brev-launchable-identity,"* ]]; then
echo "::error::Launchable identity smoke runs only against trusted main" >&2
exit 1
fi
if [[ "$INCLUDE_LAUNCHABLE" == "true" || ",${JOBS}," == *",staging-brev-launchable,"* ]]; then
[[ "$nvidia_owned" == "true" ]] || { echo "::error::Launchable PR E2E requires an NVIDIA-owned source repository" >&2; exit 1; }
[[ "$CHECKOUT_REPOSITORY" == "NVIDIA/NemoClaw" ]] || { echo "::error::Launchable PR E2E requires a branch in NVIDIA/NemoClaw" >&2; exit 1; }
fi
printf 'nvidia_owned=%s\n' "$nvidia_owned" >> "$GITHUB_OUTPUT"
- id: controller_matrix
name: Build trusted controller target matrix
if: ${{ inputs.checkout_sha != '' && steps.candidate_authorization.outputs.nvidia_owned != 'true' }}
env:
JOBS: ${{ inputs.jobs }}
TARGETS: ${{ inputs.targets }}
shell: bash
run: |
set -euo pipefail
test_matrix='[]'
case "${JOBS}:${TARGETS}" in
:)
matrix='[{"id":"ubuntu-policy-custom-missing-presets-negative","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-openclaw","runner":"ubuntu-latest"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest"}]'
test_matrix='[{"id":"onboard-managed-image-buildless-e2e","file":"test/onboarding/onboard-managed-image-buildless-e2e.test.ts","project":"integration"},{"id":"vllm-docker-storage","file":"test/platform/images/vllm-docker-storage.test.ts","project":"integration"}]'
;;
inference-routing: | managed-image-protected-runtime: | native-runtime-qualification-producer: | :jetson-nvmap-gpu)
matrix='[]'
;;
:ubuntu-repo-cloud-langchain-deepagents-code)
matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"}]'
;;
:ubuntu-repo-docker-post-reboot-recovery)
matrix='[{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]'
;;
:ubuntu-repo-cloud-langchain-deepagents-code,ubuntu-repo-docker-post-reboot-recovery)
matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]'
;;
*)
echo "::error::PR E2E target is not approved by the trusted controller" >&2
exit 1
;;
esac
printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}"
printf 'test_matrix=%s\n' "${test_matrix}" >> "${GITHUB_OUTPUT}"
- name: Record trusted E2E dispatch receipt
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
ACTOR: ${{ github.actor }}
ALLOW_DGX_SPARK_RUNNER_QUEUE: ${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }}
ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}
ALLOW_JETSON_RUNNER_QUEUE: "false"
BASE_SHA: ${{ inputs.checkout_sha != '' && inputs.base_sha || github.sha }}
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }}
CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
DISPATCH_JOBS: ${{ inputs.jobs }}
DISPATCH_RECEIPT_DIR: ${{ runner.temp }}/nemoclaw-e2e-dispatch
DISPATCH_TARGETS: ${{ inputs.targets }}
EVENT_NAME: ${{ github.event_name }}
INCLUDE_STAGING_BREV_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable && 'true' || 'false' }}
PR_NUMBER: ${{ inputs.checkout_sha != '' && inputs.pr_number || '' }}
REPOSITORY: ${{ github.repository }}
RUN_ATTEMPT: ${{ github.run_attempt }}
RUN_ID: ${{ github.run_id }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
install -d -m 0700 "$DISPATCH_RECEIPT_DIR"
jq -n \
--arg actor "$ACTOR" \
--arg baseSha "$BASE_SHA" \
--arg candidateRepository "$CANDIDATE_REPOSITORY" \
--arg candidateSha "$CANDIDATE_SHA" \
--arg eventName "$EVENT_NAME" \
--arg jobs "$DISPATCH_JOBS" \
--arg prNumber "$PR_NUMBER" \
--arg repository "$REPOSITORY" \
--arg targets "$DISPATCH_TARGETS" \
--arg triggeringActor "$TRIGGERING_ACTOR" \
--arg workflowRunId "$RUN_ID" \
--arg workflowSha "$WORKFLOW_SHA" \
--argjson allowDgxSparkRunnerQueue "$ALLOW_DGX_SPARK_RUNNER_QUEUE" \
--argjson allowJetsonDispatch "$ALLOW_JETSON_DISPATCH" \
--argjson allowJetsonRunnerQueue "$ALLOW_JETSON_RUNNER_QUEUE" \
--argjson includeStagingBrevLaunchable "$INCLUDE_STAGING_BREV_LAUNCHABLE" \
--argjson workflowRunAttempt "$RUN_ATTEMPT" \
'{
kind: "nemoclaw-e2e-dispatch-v2",
actor: $actor,
repository: $repository,
prNumber: (if $prNumber == "" then null else ($prNumber | tonumber) end),
candidateRepository: $candidateRepository,
candidateSha: $candidateSha,
baseSha: $baseSha,
workflowSha: $workflowSha,
workflowRunId: $workflowRunId,
workflowRunAttempt: $workflowRunAttempt,
eventName: $eventName,
jobs: $jobs,
targets: $targets,
allowDgxSparkRunnerQueue: $allowDgxSparkRunnerQueue,
allowJetsonDispatch: $allowJetsonDispatch,
allowJetsonRunnerQueue: $allowJetsonRunnerQueue,
includeStagingBrevLaunchable: $includeStagingBrevLaunchable,
triggeringActor: $triggeringActor,
emptySelectors: ($jobs == "" and $targets == "")
}' >"$DISPATCH_RECEIPT_DIR/dispatch.json"
- name: Upload trusted E2E dispatch receipt
if: ${{ github.event_name == 'workflow_dispatch' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-dispatch-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/nemoclaw-e2e-dispatch/dispatch.json
- name: Authorize Launchable E2E maintainer dispatch
if: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.jobs == 'staging-brev-launchable-identity' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}
env:
ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ github.token }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
shell: bash
run: |
set -euo pipefail
read_collaborator_permission() {
local maintainer="$1"
local attempt curl_exit failure http_status permission_file
permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")"
for attempt in 1 2 3; do
: >"$permission_file"
if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \
--output "$permission_file" --write-out "%{http_code}" \
--header "Authorization: Bearer ${GITHUB_TOKEN}" \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission" \
2>/dev/null)"; then
if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then
if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then
if (( attempt > 1 )); then
echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2
fi
cat "$permission_file"
rm -f "$permission_file"
return 0
fi
echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2
rm -f "$permission_file"
return 1
fi
if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then
failure="HTTP ${http_status}"
case "$http_status" in
408 | 429 | 5??) ;;
*) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;;
esac
else
echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2
rm -f "$permission_file"
return 1
fi
else
curl_exit=$?
case "$curl_exit" in
5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;;
*) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;;
esac
fi
if (( attempt == 3 )); then
echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2
rm -f "$permission_file"
return 1
fi
echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2
sleep "$attempt"
done
}
require_maintainer() {
local maintainer="$1"
if [[ ! "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ || "$maintainer" == -* || "$maintainer" == *- ]]; then
echo "::error::Launchable E2E actor is invalid" >&2
exit 1
fi
local permission_json
permission_json="$(read_collaborator_permission "$maintainer")"
if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then
echo "::error::Launchable E2E permission response did not match the actor" >&2
exit 1
fi
case "$(jq -r '.role_name // ""' <<< "$permission_json")" in
maintain | admin) ;;
*)
echo "::error::Launchable E2E requires a repository maintainer or administrator" >&2
exit 1
;;
esac
}
require_maintainer "$ACTOR"
if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then
require_maintainer "$TRIGGERING_ACTOR"
fi
- name: Check out trusted E2E planner
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ github.workflow_sha }}
fetch-depth: 0
persist-credentials: false
- name: Set up Node for trusted E2E planning
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- name: Install trusted E2E planner dependencies
run: npm ci --ignore-scripts --no-audit --no-fund
- id: matrix
name: Generate E2E target matrix
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
CONTROLLER_MATRIX: ${{ steps.controller_matrix.outputs.matrix }}
CONTROLLER_TEST_MATRIX: ${{ steps.controller_matrix.outputs.test_matrix }}
INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }}
NEMOCLAW_GATEWAY_RUNTIMES: ${{ inputs.gateway_runtimes || inputs.gateway_runtime || 'docker' }}
JOBS: ${{ inputs.jobs }}
TARGETS: ${{ inputs.targets }}
EVENT_NAME: ${{ github.event_name }}
BEFORE_SHA: ${{ github.event.before }}
CANDIDATE_SHA: ${{ github.sha }}
NEMOCLAW_E2E_CREDENTIALS_ALLOWED: ${{ (inputs.checkout_sha == '' || steps.candidate_authorization.outputs.nvidia_owned == 'true') && 'true' || 'false' }}
NEMOCLAW_E2E_BRAVE_API_KEY_AVAILABLE: ${{ secrets.BRAVE_API_KEY != '' && 'true' || 'false' }}
NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }}
run: |
set -euo pipefail
if [[ "${JOBS:-}" == "native-runtime-qualification-producer" && -z "${TARGETS:-}" ]]; then
{
echo 'matrix=[]'
echo 'test_matrix=[]'
echo 'catalogue_standard_matrix=[]'
echo 'catalogue_nvidia_api_matrix=[]'
echo 'catalogue_nvidia_inference_matrix=[]'
echo 'catalogue_github_read_matrix=[]'
echo 'catalogue_brave_nvidia_inference_matrix=[]'
echo 'gateway_runtimes=["docker"]'
echo 'runtime_providers_by_job={"native-runtime-qualification-producer":["docker"]}'
echo 'selected_jobs=["native-runtime-qualification-producer"]'
echo 'selected_workflow_jobs=["native-runtime-qualification-producer"]'
echo 'hermes_selected=false'
echo 'explicit_only_jobs=native-runtime-qualification-producer'
echo 'release_required_jobs=[]'
} >>"$GITHUB_OUTPUT"
exit 0
fi
if [ "${EVENT_NAME:-}" = "push" ]; then
[[ "${BEFORE_SHA}" =~ ^[a-f0-9]{40}$ && "${CANDIDATE_SHA}" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Main push E2E requires exact before and candidate commit SHAs" >&2
exit 1
}
CHANGED_FILES="$(git diff --name-only --diff-filter=ACMRD "${BEFORE_SHA}" "${CANDIDATE_SHA}")"
export CHANGED_FILES
fi
npx --no-install tsx tools/e2e/workflow-plan.mts --ci-output
if [ -n "${CHECKOUT_SHA}" ] && [ "${NVIDIA_OWNED}" != "true" ]; then
matrix="$(sed -n 's/^matrix=//p' "${GITHUB_OUTPUT}")"
expected_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${CONTROLLER_MATRIX}")"
actual_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${matrix}")"
test_matrix="$(sed -n 's/^test_matrix=//p' "${GITHUB_OUTPUT}")"
expected_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${CONTROLLER_TEST_MATRIX}")"
actual_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${test_matrix}")"
if [ "${actual_controller_matrix}" != "${expected_controller_matrix}" ] ||
[ "${actual_controller_test_matrix}" != "${expected_controller_test_matrix}" ]; then
: > "${GITHUB_OUTPUT}"
echo "::error::E2E planner matrix does not match controller-selected targets or shared tests" >&2
exit 1
fi
fi
# Publish immutable source-run toolchains before candidate checkout or
# candidate-controlled workspace preparation can execute on this runner.
- name: Stage immutable native Podman E2E toolchains
uses: NVIDIA/NemoClaw/.github/actions/stage-native-podman-e2e-toolchains@1a0f53d5d7e5420556be72b50d79ed5a333d637d
with:
enabled: ${{ contains(format(',{0},', inputs.gateway_runtimes || inputs.gateway_runtime || 'docker'), ',podman,') && 'true' || 'false' }}
github-token: ${{ github.token }}
- name: Check out E2E candidate
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Validate manual PR checkout
if: ${{ inputs.checkout_sha != '' && (inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '') }}
env:
BASE_SHA: ${{ inputs.base_sha }}
CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }}
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
GITHUB_TOKEN: ${{ steps.candidate_authorization.outputs.nvidia_owned == 'true' && github.token || '' }}
NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }}
PR_NUMBER: ${{ inputs.pr_number }}
shell: bash
run: |
set -euo pipefail
[[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out commit does not match checkout_sha" >&2; exit 1; }
auth_args=()
if [[ "$NVIDIA_OWNED" == "true" ]]; then
[[ -n "$GITHUB_TOKEN" ]] || { echo "::error::authenticated NVIDIA-owned checkout validation requires a GitHub token" >&2; exit 1; }
auth_args=(--header "Authorization: Bearer ${GITHUB_TOKEN}")
fi
pull_json="$(curl --fail --silent --show-error --proto '=https' \
"${auth_args[@]}" \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")"
[[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open" >&2; exit 1; }
[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository changed before execution" >&2; exit 1; }
[[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch changed before execution" >&2; exit 1; }
[[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha changed before execution" >&2; exit 1; }
if [[ "$CHECKOUT_SHA" == "$BASE_SHA" ]]; then
[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before base execution" >&2; exit 1; }
else
[[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before head execution" >&2; exit 1; }
[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha changed before head execution" >&2; exit 1; }
fi
if [[ "$NVIDIA_OWNED" == "true" ]]; then
[[ "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" &&
"$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]] || {
echo "::error::PR source repository ownership changed before execution" >&2
exit 1
}
fi
- id: e2e_credentials
name: Authorize E2E credentials
if: ${{ inputs.checkout_sha != '' && (inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '') }}
env:
CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }}
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }}
REF: ${{ github.ref }}
WORKFLOW_REPOSITORY: ${{ github.repository }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
credentials_allowed=false
if [[ "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw" &&
"$NVIDIA_OWNED" == "true" &&
"$EVENT_NAME" == "workflow_dispatch" &&
"$REF" == refs/heads/* &&
"$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ &&
"$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ &&
"$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" &&
"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]]; then
credentials_allowed=true
fi
printf 'allowed=%s\n' "$credentials_allowed" >> "$GITHUB_OUTPUT"
- name: Prepare E2E workspace
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
- id: package_cli_artifact
name: Package exact-commit CLI
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
env:
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }}
CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
RUN_ATTEMPT: ${{ github.run_attempt }}
RUN_ID: ${{ github.run_id }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: scripts/e2e/package-cli-artifact.sh
- id: upload_cli_artifact
name: Publish content-addressed CLI artifact
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ steps.package_cli_artifact.outputs.artifact_name }}
path: ${{ runner.temp }}/nemoclaw-cli-artifact/
if-no-files-found: error
retention-days: 3
compression-level: 0
- id: record_cli_artifact
name: Record CLI artifact provenance
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
env:
ARTIFACT_DIGEST: ${{ steps.upload_cli_artifact.outputs.artifact-digest }}
ARTIFACT_ID: ${{ steps.upload_cli_artifact.outputs.artifact-id }}
ARTIFACT_NAME: ${{ steps.package_cli_artifact.outputs.artifact_name }}
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }}
CANDIDATE_SHA: ${{ steps.package_cli_artifact.outputs.candidate_sha }}
PAYLOAD_SHA256: ${{ steps.package_cli_artifact.outputs.payload_sha256 }}
RUN_ATTEMPT: ${{ github.run_attempt }}
RUN_ID: ${{ github.run_id }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]] ||
{ echo "::error::artifact upload did not return an immutable artifact ID"; exit 1; }
[[ "$ARTIFACT_DIGEST" =~ ^[a-f0-9]{64}$ ]] ||
{ echo "::error::artifact upload did not return a SHA-256 digest"; exit 1; }
provenance="$(jq -cn \
--arg artifactDigest "$ARTIFACT_DIGEST" \
--arg artifactId "$ARTIFACT_ID" \
--arg artifactName "$ARTIFACT_NAME" \
--arg candidateRepository "$CANDIDATE_REPOSITORY" \
--arg candidateSha "$CANDIDATE_SHA" \
--arg payloadSha256 "$PAYLOAD_SHA256" \
--arg runAttempt "$RUN_ATTEMPT" \
--arg runId "$RUN_ID" \
--arg workflowSha "$WORKFLOW_SHA" \
'{
kind: "nemoclaw-e2e-cli-provenance-v1",
artifactDigest: $artifactDigest,
artifactId: $artifactId,
artifactName: $artifactName,
candidateRepository: $candidateRepository,
candidateSha: $candidateSha,
payloadSha256: $payloadSha256,
workflowSha: $workflowSha,
runId: $runId,
runAttempt: $runAttempt
}')"
printf 'provenance=%s\n' "$provenance" >>"$GITHUB_OUTPUT"
{
echo "## Exact-commit CLI artifact"
echo
echo "- Candidate: \`${CANDIDATE_SHA}\`"
echo "- Artifact: \`${ARTIFACT_NAME}\` (ID \`${ARTIFACT_ID}\`)"
echo "- GitHub archive digest: \`${ARTIFACT_DIGEST}\`"
echo "- Payload digest: \`${PAYLOAD_SHA256}\`"
} >>"$GITHUB_STEP_SUMMARY"
native-runtime-qualification-producer-plan:
needs: generate-matrix
if: ${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }}
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
actions: read
contents: read
pull-requests: read
outputs:
matrix: ${{ steps.plan.outputs.matrix }}
steps:
- name: Check out the trusted qualification producer
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: |
src/lib/onboard/runtime-provider/native-qualification-authority.ts
test/e2e/registry/native-runtime-qualification.ts
tools/e2e/native-runtime-qualification-producer-plan.mts
sparse-checkout-cone-mode: false
- name: Check out the candidate for source authentication
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
path: .candidate-source
fetch-depth: 1
persist-credentials: false
- name: Set up Node for trusted qualification planning
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- id: source
name: Authenticate the candidate and dispatch artifact
env:
BASE_SHA: ${{ inputs.base_sha }}
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }}
CANDIDATE_SHA: ${{ inputs.checkout_sha }}
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ inputs.pr_number }}
PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }}
PRODUCER_RUN_ID: ${{ github.run_id }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$CANDIDATE_REPOSITORY" == "NVIDIA/NemoClaw" ]] || {
echo "::error::Native runtime qualification accepts only a branch in NVIDIA/NemoClaw" >&2
exit 1
}
[[ "$PRODUCER_RUN_ATTEMPT" == "1" ]] || {
echo "::error::Native runtime qualification producer runs cannot be rerun" >&2
exit 1
}
[[ "$BASE_SHA" == "$WORKFLOW_SHA" && "$CANDIDATE_SHA" != "$WORKFLOW_SHA" ]] || {
echo "::error::Native runtime qualification requires base_sha to equal workflow_sha and checkout_sha to name a different commit" >&2
exit 1
}
[[ "$(git -C .candidate-source rev-parse --verify 'HEAD^{commit}')" == "$CANDIDATE_SHA" ]] || {
echo "::error::Candidate checkout does not match the authenticated candidate commit" >&2
exit 1
}
pull="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")"
jq -e \
--arg baseSha "$BASE_SHA" \
--arg candidateRepository "$CANDIDATE_REPOSITORY" \
--arg candidateSha "$CANDIDATE_SHA" '
.state == "open" and
.head.repo.full_name == $candidateRepository and
.head.sha == $candidateSha and
.base.repo.full_name == "NVIDIA/NemoClaw" and
.base.ref == "main" and
.base.sha == $baseSha
' <<<"$pull" >/dev/null || {
echo "::error::Pull request identity changed before qualification planning" >&2
exit 1
}
artifact_name="e2e-dispatch-${PRODUCER_RUN_ID}-${PRODUCER_RUN_ATTEMPT}"
artifacts="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/artifacts" \
-f name="$artifact_name" \
-f per_page=100)"
artifact="$(jq -ce \
--arg name "$artifact_name" \
--arg runId "$PRODUCER_RUN_ID" \
--arg workflowSha "$WORKFLOW_SHA" '
select(.total_count == 1 and (.artifacts | length) == 1) |
.artifacts[0] |
select(.name == $name and .expired == false) |
select(.size_in_bytes > 0 and .size_in_bytes <= 1048576) |
select((.id | tostring | test("^[1-9][0-9]{0,19}$"))) |
select((.digest // "") | test("^sha256:[a-f0-9]{64}$")) |
select((.workflow_run.id | tostring) == $runId) |
select(.workflow_run.head_sha == $workflowSha)
' <<<"$artifacts")" || {
echo "::error::Trusted dispatch artifact is missing or ambiguous" >&2
exit 1
}
installer_sha256="$(sha256sum .candidate-source/scripts/install.sh | awk '{print $1}')"
[[ "$installer_sha256" =~ ^[a-f0-9]{64}$ ]] || {
echo "::error::Candidate installer SHA-256 is invalid" >&2
exit 1
}
printf 'artifact_id=%s\n' "$(jq -r '.id | tostring' <<<"$artifact")" >>"$GITHUB_OUTPUT"
printf 'artifact_name=%s\n' "$artifact_name" >>"$GITHUB_OUTPUT"
printf 'artifact_digest=%s\n' "$(jq -r '.digest' <<<"$artifact")" >>"$GITHUB_OUTPUT"
printf 'artifact_size=%s\n' "$(jq -r '.size_in_bytes' <<<"$artifact")" >>"$GITHUB_OUTPUT"
printf 'installer_sha256=%s\n' "$installer_sha256" >>"$GITHUB_OUTPUT"
- id: plan
name: Compile the trusted qualification producer matrix
env:
BASE_SHA: ${{ inputs.base_sha }}
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }}
CANDIDATE_SHA: ${{ inputs.checkout_sha }}
DISPATCH_ARTIFACT_DIGEST: ${{ steps.source.outputs.artifact_digest }}
DISPATCH_ARTIFACT_ID: ${{ steps.source.outputs.artifact_id }}
DISPATCH_ARTIFACT_NAME: ${{ steps.source.outputs.artifact_name }}
DISPATCH_ARTIFACT_SIZE: ${{ steps.source.outputs.artifact_size }}
INSTALLER_SHA256: ${{ steps.source.outputs.installer_sha256 }}
NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL: ${{ vars.NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL }}
PR_NUMBER: ${{ inputs.pr_number }}
PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }}
PRODUCER_RUN_ID: ${{ github.run_id }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: node --experimental-strip-types --no-warnings tools/e2e/native-runtime-qualification-producer-plan.mts --ci-output
native-runtime-qualification-podman-toolchain:
name: Build pinned native Podman toolchain / ${{ matrix.architecture }}
needs: [generate-matrix, native-runtime-qualification-producer-plan]
if: ${{ needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }}
strategy:
fail-fast: false
matrix:
include:
- architecture: amd64
runner: ubuntu-24.04
- architecture: arm64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
timeout-minutes: 45
permissions:
contents: read
steps:
- name: Check out the pinned Podman source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: podman-container-tools/podman
ref: cade97a52ebdf9dbf9e81de8009015776837a074 # v6.1.0
path: .podman-source
fetch-depth: 1
persist-credentials: false
- name: Check out the pinned Netavark source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: containers/netavark
ref: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab # v2.1.0
path: .netavark-source
fetch-depth: 1
persist-credentials: false
- name: Check out the pinned Aardvark DNS source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: containers/aardvark-dns
ref: cd7417681229219059939bdd9f0b3bd9ac9abb08 # v2.1.0
path: .aardvark-source
fetch-depth: 1
persist-credentials: false
- name: Set up pinned Go for the Podman build
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version: 1.25.9
cache: false
- name: Set up pinned Rust for the network helper builds
uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
with:
toolchain: 1.88.0
cache: false
rustflags: ""
- name: Install build dependencies from the signed runner OS repository
shell: bash
run: |
set -euo pipefail
sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update
sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \
--yes --no-install-recommends \
curl gcc git libapparmor-dev libbtrfs-dev libc6-dev \
libdevmapper-dev libglib2.0-dev \
libprotobuf-c-dev libprotobuf-dev libseccomp-dev libselinux1-dev \
libsqlite3-dev libsystemd-dev make pkg-config protobuf-compiler
- name: Build and package the pinned native toolchain
env:
AARDVARK_SOURCE_SHA: cd7417681229219059939bdd9f0b3bd9ac9abb08
EXPECTED_ARCHITECTURE: ${{ matrix.architecture }}
NETAVARK_SOURCE_SHA: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab
PASTA_SOURCE_ARCHIVE_SHA256: 54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958
PASTA_SOURCE_SHA: f8df3f1b228fe19a74a269334fdfe6cc7d0605ce
PASTA_VERSION: 2026_07_28.f8df3f1
PODMAN_SOURCE_SHA: cade97a52ebdf9dbf9e81de8009015776837a074
TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain
shell: bash
run: |
set -euo pipefail
[[ "$(dpkg --print-architecture)" == "$EXPECTED_ARCHITECTURE" ]]
[[ "$(git -C .podman-source rev-parse --verify 'HEAD^{commit}')" == "$PODMAN_SOURCE_SHA" ]]
[[ "$(git -C .netavark-source rev-parse --verify 'HEAD^{commit}')" == "$NETAVARK_SOURCE_SHA" ]]
[[ "$(git -C .aardvark-source rev-parse --verify 'HEAD^{commit}')" == "$AARDVARK_SOURCE_SHA" ]]
[[ ! -e .passt-source && ! -L .passt-source ]]
pasta_source_archive="${RUNNER_TEMP}/passt-${PASTA_SOURCE_SHA}.tar.gz"
[[ ! -e "$pasta_source_archive" && ! -L "$pasta_source_archive" ]]
/usr/bin/curl \
--fail --location --proto '=https' --proto-redir '=https' \
--retry 3 --show-error --silent --tlsv1.2 \
--output "$pasta_source_archive" \
"https://passt.top/passt/snapshot/passt-${PASTA_SOURCE_SHA}.tar.gz"
[[ -f "$pasta_source_archive" && ! -L "$pasta_source_archive" ]]
[[ "$(sha256sum "$pasta_source_archive" | cut -d' ' -f1)" == "$PASTA_SOURCE_ARCHIVE_SHA256" ]]
mkdir .passt-source
tar \
--extract --gzip --file="$pasta_source_archive" \
--directory=.passt-source --strip-components=1 \
--no-same-owner --no-same-permissions
[[ -f .passt-source/Makefile && ! -L .passt-source/Makefile ]]
[[ -f .passt-source/passt.c && ! -L .passt-source/passt.c ]]
[[ ! -e .passt-source/passt && ! -L .passt-source/passt ]]
for source in .podman-source .netavark-source .aardvark-source; do
[[ -z "$(git -C "$source" status --porcelain --untracked-files=no)" ]]
done
[[ "$(go version)" == go\ version\ go1.25.9\ * ]]
[[ "$(rustc --version)" == rustc\ 1.88.0\ * ]]
SOURCE_DATE_EPOCH=1786554266 \
BUILD_ORIGIN="NVIDIA/NemoClaw native runtime qualification" \
EXTRA_BUILDTAGS=containers_image_openpgp \
make --directory=.podman-source --jobs=2 \
podman rootlessport PREFIX=/usr/local
SOURCE_DATE_EPOCH=1785940686 CI=1 \
make --directory=.netavark-source --jobs=2 build
SOURCE_DATE_EPOCH=1785940850 CI=1 \
make --directory=.aardvark-source --jobs=2 build
SOURCE_DATE_EPOCH=1785255008 \
make --directory=.passt-source --jobs=2 VERSION="$PASTA_VERSION" passt
podman_dependencies="$(ldd .podman-source/bin/podman)"
printf '%s\n' "$podman_dependencies"
if grep -F "not found" <<<"$podman_dependencies"; then
echo "::error::Pinned Podman build has an unresolved runtime dependency" >&2
exit 1
fi
if grep -E "libgpgme|libsubid" <<<"$podman_dependencies"; then
echo "::error::Pinned Podman build must not require an optional host ABI" >&2
exit 1
fi
install -D -m 0755 .podman-source/bin/podman "$TOOLCHAIN_DIRECTORY/bin/podman"
[[ -f .passt-source/passt && ! -L .passt-source/passt ]]
install -D -m 0755 .passt-source/passt "$TOOLCHAIN_DIRECTORY/bin/pasta"
install -D -m 0755 .podman-source/bin/rootlessport \
"$TOOLCHAIN_DIRECTORY/libexec/podman/rootlessport"
install -D -m 0755 .netavark-source/bin/netavark \
"$TOOLCHAIN_DIRECTORY/libexec/podman/netavark"
install -D -m 0755 .aardvark-source/bin/aardvark-dns \
"$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns"
install -D -m 0644 \
.podman-source/vendor/go.podman.io/common/pkg/config/containers.conf \
"$TOOLCHAIN_DIRECTORY/share/containers/containers.conf"
[[ "$("$TOOLCHAIN_DIRECTORY/bin/podman" --version)" == "podman version 6.1.0" ]]
pasta_version_output="$("$TOOLCHAIN_DIRECTORY/bin/pasta" --version)"
[[ "${pasta_version_output%%$'\n'*}" == "pasta $PASTA_VERSION" ]] || {
echo "::error::Pinned qualification pasta version is invalid" >&2
exit 1
}
[[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" --version)" == "netavark 2.1.0" ]]
[[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" --version)" == "aardvark-dns 2.1.0" ]]
jq -n \
--arg architecture "$EXPECTED_ARCHITECTURE" \
--arg aardvarkDnsSourceSha "$AARDVARK_SOURCE_SHA" \
--arg netavarkSourceSha "$NETAVARK_SOURCE_SHA" \
--arg pastaSourceArchiveSha256 "$PASTA_SOURCE_ARCHIVE_SHA256" \
--arg pastaSourceSha "$PASTA_SOURCE_SHA" \
--arg pastaVersion "$PASTA_VERSION" \
--arg podmanSourceSha "$PODMAN_SOURCE_SHA" '
{
schemaVersion: 1,
kind: "nemoclaw-native-podman-toolchain-v1",
architecture: $architecture,
podmanVersion: "6.1.0",
podmanSourceSha: $podmanSourceSha,
netavarkVersion: "2.1.0",
netavarkSourceSha: $netavarkSourceSha,
aardvarkDnsVersion: "2.1.0",
aardvarkDnsSourceSha: $aardvarkDnsSourceSha,
pastaVersion: $pastaVersion,
pastaSourceArchiveSha256: $pastaSourceArchiveSha256,
pastaSourceSha: $pastaSourceSha,
goVersion: "1.25.9",
rustVersion: "1.88.0"
}
' >"$TOOLCHAIN_DIRECTORY/manifest.json"
(
cd "$TOOLCHAIN_DIRECTORY"
sha256sum \
bin/pasta \
bin/podman \
libexec/podman/aardvark-dns \
libexec/podman/netavark \
libexec/podman/rootlessport \
manifest.json \
share/containers/containers.conf >SHA256SUMS
)
- name: Upload the pinned native Podman toolchain
if: success()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: native-runtime-podman-toolchain-${{ matrix.architecture }}
path: ${{ runner.temp }}/native-runtime-podman-toolchain/
native-runtime-qualification-producer:
name: ${{ matrix.jobName }}
needs:
- generate-matrix
- native-runtime-qualification-podman-toolchain
- native-runtime-qualification-producer-plan
if: ${{ needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.native-runtime-qualification-producer-plan.outputs.matrix) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 120
permissions:
contents: read
env:
NEMOCLAW_RUN_LIVE_E2E: "1"
QUALIFICATION_ROW: ${{ toJSON(matrix) }}
steps:
- name: Require an ephemeral qualification runner
env:
REVIEWED_EPHEMERAL_POOL: ${{ vars.NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL }}
RUNNER_ENVIRONMENT: ${{ runner.environment }}
shell: bash
run: |
set -euo pipefail
if [[ "$RUNNER_ENVIRONMENT" != "github-hosted" && "$REVIEWED_EPHEMERAL_POOL" != "enabled" ]]; then
echo "::error::Self-hosted native runtime qualification requires the reviewed ephemeral runner pool" >&2
exit 1
fi
- name: Check out the trusted qualification harness
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ matrix.source.workflowSha }}
path: .qualification-workflow
persist-credentials: false
sparse-checkout: |
src/lib/onboard/runtime-provider/native-qualification-authority.ts
scripts/checks/run-native-runtime-installer-qualification.sh
test/e2e/registry/native-runtime-qualification.ts
tools/e2e/native-runtime-qualification-producer-plan.mts
tools/e2e/native-runtime-qualification-producer-evidence.mts
sparse-checkout-cone-mode: false
- name: Check out the candidate commit
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ matrix.source.candidateRepository }}
ref: ${{ matrix.source.candidateSha }}
path: .candidate-runtime
fetch-depth: 1
persist-credentials: false
- name: Set up Node for candidate execution
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.19.0
- name: Require a reviewed Ubuntu runtime host
shell: bash
run: |
set -euo pipefail
[[ -r /etc/os-release ]] || {
echo "::error::Protected runner does not expose an OS release identity" >&2
exit 1
}
# shellcheck disable=SC1091
source /etc/os-release
[[ "${ID:-}" == "ubuntu" ]] || {
echo "::error::Protected runner must use a reviewed Ubuntu image" >&2
exit 1
}
[[ "${VERSION_ID:-}" == "24.04" || "${VERSION_ID:-}" == "26.04" ]] || {
echo "::error::Protected runner Ubuntu release is not reviewed for Podman qualification" >&2
exit 1
}
- name: Download the pinned native Podman toolchain
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: native-runtime-podman-toolchain-${{ matrix.case.architecture }}
path: ${{ runner.temp }}/native-runtime-podman-toolchain
- name: Install the pinned native Podman toolchain and rootless prerequisites
env:
EXPECTED_ARCHITECTURE: ${{ matrix.case.architecture }}
TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain
shell: bash
run: |
set -euo pipefail
[[ -x /usr/bin/apt-get ]] || {
echo "::error::Protected runner cannot install rootless Podman prerequisites from its signed OS repository" >&2
exit 1
}
sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update
sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \
--yes --no-install-recommends \
acl apparmor btrfs-progs conmon \
golang-github-containers-common iptables nftables runc slirp4netns uidmap
[[ -d "$TOOLCHAIN_DIRECTORY" && ! -L "$TOOLCHAIN_DIRECTORY" ]]
[[ -z "$(find -P "$TOOLCHAIN_DIRECTORY" -type l -print -quit)" ]]
mapfile -t actual_files < <(
cd "$TOOLCHAIN_DIRECTORY"
find . -type f -print | LC_ALL=C sort
)
expected_files=(
./SHA256SUMS
./bin/pasta
./bin/podman
./libexec/podman/aardvark-dns
./libexec/podman/netavark
./libexec/podman/rootlessport
./manifest.json
./share/containers/containers.conf
)
[[ "${actual_files[*]}" == "${expected_files[*]}" ]] || {
echo "::error::Downloaded native Podman toolchain contains unexpected files" >&2
exit 1
}
(
cd "$TOOLCHAIN_DIRECTORY"
sha256sum --check --strict SHA256SUMS
)
jq -e \
--arg architecture "$EXPECTED_ARCHITECTURE" '
type == "object" and
keys == [
"aardvarkDnsSourceSha",
"aardvarkDnsVersion",
"architecture",
"goVersion",
"kind",
"netavarkSourceSha",
"netavarkVersion",
"pastaSourceArchiveSha256",
"pastaSourceSha",
"pastaVersion",
"podmanSourceSha",
"podmanVersion",
"rustVersion",
"schemaVersion"
] and
.schemaVersion == 1 and
.kind == "nemoclaw-native-podman-toolchain-v1" and
.architecture == $architecture and
.podmanVersion == "6.1.0" and
.podmanSourceSha == "cade97a52ebdf9dbf9e81de8009015776837a074" and
.netavarkVersion == "2.1.0" and
.netavarkSourceSha == "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" and
.aardvarkDnsVersion == "2.1.0" and
.aardvarkDnsSourceSha == "cd7417681229219059939bdd9f0b3bd9ac9abb08" and
.pastaVersion == "2026_07_28.f8df3f1" and
.pastaSourceArchiveSha256 == "54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958" and
.pastaSourceSha == "f8df3f1b228fe19a74a269334fdfe6cc7d0605ce" and
.goVersion == "1.25.9" and
.rustVersion == "1.88.0"
' "$TOOLCHAIN_DIRECTORY/manifest.json" >/dev/null
for target in \
/usr/local/bin/podman \
/usr/local/libexec/podman/aardvark-dns \
/usr/local/libexec/podman/netavark \
/usr/local/libexec/podman/rootlessport \
/usr/share/containers/containers.conf; do
[[ ! -L "$target" ]] || {
echo "::error::Native Podman toolchain target must not be a symlink: $target" >&2
exit 1
}
done
sudo install --owner=root --group=root --mode=0755 \
"$TOOLCHAIN_DIRECTORY/bin/podman" /usr/local/bin/podman
for helper in aardvark-dns netavark rootlessport; do
sudo install -D --owner=root --group=root --mode=0755 \
"$TOOLCHAIN_DIRECTORY/libexec/podman/$helper" \
"/usr/local/libexec/podman/$helper"
done
sudo install --owner=root --group=root --mode=0644 \
"$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" \
/usr/share/containers/containers.conf
[[ "$(command -v podman)" == "/usr/local/bin/podman" ]]
conmon_version="$(conmon --version | awk 'NR == 1 { print $NF }')"
runc_version="$(runc --version | awk 'NR == 1 { print $NF }')"
dpkg --compare-versions "$conmon_version" ge 2.1.7
dpkg --compare-versions "$runc_version" ge 1.1.11
[[ "$(/usr/local/libexec/podman/netavark --version)" == "netavark 2.1.0" ]]
[[ "$(/usr/local/libexec/podman/aardvark-dns --version)" == "aardvark-dns 2.1.0" ]]
version="$(podman --version)"
[[ "$version" == "podman version 6.1.0" ]]
- name: Prepare the credential-free execution account and disable Docker
id: boundary
env:
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain
shell: bash
run: |
set -euo pipefail
for command in apparmor_parser awk cat curl getent git grep groupdel id jq node npm pgrep podman setfacl sha256sum stat systemctl tee unlink useradd userdel usermod; do
command -v "$command" >/dev/null || {
echo "::error::Protected runner is missing required command: $command" >&2
exit 1
}
done
[[ "$(podman --version)" == "podman version 6.1.0" ]] || {
echo "::error::Protected runner must provide Podman 6.1.0" >&2
exit 1
}
sudo systemctl stop docker.service docker.socket 2>/dev/null || true
sudo systemctl mask --runtime docker.service docker.socket 2>/dev/null || true
sudo pkill -TERM -x dockerd 2>/dev/null || true
sudo rm -f /var/run/docker.sock /run/docker.sock
! systemctl is-active --quiet docker.service
! systemctl is-active --quiet docker.socket
! pgrep -x dockerd >/dev/null
[[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]]
account="nemoclawq"
ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then
echo "::error::Qualification account, group, or subordinate-ID authorization already exists" >&2
exit 1
fi
[[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || {
echo "::error::Qualification account ownership marker already exists" >&2
exit 1
}
account_created_without_marker=0
rollback_unmarked_account() {
local result="$?"
trap - EXIT
if ((result != 0 && account_created_without_marker == 1)); then
sudo userdel --remove "$account" 2>/dev/null || true
sudo groupdel "$account" 2>/dev/null || true
sudo rm -f -- "$ownership_marker"
if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then
echo "::error::Partially created qualification account could not be rolled back" >&2
exit 1
fi
fi
exit "$result"
}
trap rollback_unmarked_account EXIT
sudo useradd --create-home --shell /usr/sbin/nologin --user-group "$account"
account_created_without_marker=1
uid="$(id -u "$account")"
gid="$(id -g "$account")"
home="$(getent passwd "$account" | cut -d: -f6)"
group_entry="$(getent group "$account")"
[[ "$uid" =~ ^[0-9]+$ && "$gid" =~ ^[0-9]+$ && "$home" == "/home/${account}" && -d "$home" && ! -L "$home" ]] || {
echo "::error::Qualification account identity is missing or invalid" >&2
exit 1
}
[[ "$group_entry" == "${account}:x:${gid}:" ]] || {
echo "::error::Qualification private group identity is missing or invalid" >&2
exit 1
}
printf '%s:%s:%s\n' "$account" "$uid" "$gid" | sudo tee "$ownership_marker" >/dev/null
sudo chown root:root "$ownership_marker"
sudo chmod 0400 "$ownership_marker"
[[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || {
echo "::error::Qualification account ownership marker is invalid" >&2
exit 1
}
account_created_without_marker=0
trap - EXIT
ensure_subordinate_range() {
local file="$1"
local option="$2"
local range_start=100000
local range_end
local conflict_end
[[ -f "$file" && ! -L "$file" ]] || {
echo "::error::Rootless Podman subordinate-ID file is missing or invalid: $file" >&2
exit 1
}
if awk -F: -v account="$account" '
$1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 }
END { exit found ? 0 : 1 }
' "$file"; then
return
fi
while :; do
((range_start <= 4294901760)) || {
echo "::error::Protected runner has no free subordinate-ID range for rootless Podman" >&2
exit 1
}
range_end=$((range_start + 65535))
conflict_end="$(awk -F: -v start="$range_start" -v end="$range_end" '
$2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ {
current_end = $2 + $3 - 1
if ($2 <= end && current_end >= start && current_end > maximum) maximum = current_end
}
END { if (maximum != "") print maximum }
' "$file")"
[[ -n "$conflict_end" ]] || break
range_start=$((conflict_end + 1))
done
range_end=$((range_start + 65535))
sudo usermod "$option" "${range_start}-${range_end}" "$account"
awk -F: -v account="$account" '
$1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 }
END { exit found ? 0 : 1 }
' "$file" || {
echo "::error::Protected runner did not provision rootless Podman subordinate IDs" >&2
exit 1
}
}
ensure_subordinate_range /etc/subuid --add-subuids
ensure_subordinate_range /etc/subgid --add-subgids
printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT"
printf 'account_created=true\n' >>"$GITHUB_OUTPUT"
printf 'uid=%s\n' "$uid" >>"$GITHUB_OUTPUT"
printf 'gid=%s\n' "$gid" >>"$GITHUB_OUTPUT"
runtime_dir="/run/user/${uid}"
runtime_directory_unit="user-runtime-dir@${uid}.service"
user_manager_unit="user@${uid}.service"
user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d"
user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf"
trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user"
[[ ! -e "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" ]] || {
echo "::error::Qualification systemd user-manager drop-in directory already exists" >&2
exit 1
}
sudo install -d --owner=root --group=root --mode=0755 "$user_manager_dropin_directory"
printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"\n' "$trusted_user_unit_path" |
sudo tee "$user_manager_dropin" >/dev/null
sudo chown root:root "$user_manager_dropin"
sudo chmod 0444 "$user_manager_dropin"
[[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || {
echo "::error::Qualification systemd user-manager drop-in is invalid" >&2
exit 1
}
sudo systemctl daemon-reload
verify_user_manager_unit_path() {
local environment
environment="$(sudo -u "$1" env -i \
HOME="$2" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$3" \
/usr/bin/systemctl --user show-environment)"
tr ' ' '\n' <<<"$environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$trusted_user_unit_path" >/dev/null || {
echo "::error::Qualification systemd user manager did not inherit the trusted unit path" >&2
return 1
}
}
verify_user_bus() {
local execution_account="$1"
local expected_uid="$2"
local bus="$3"
local context="$4"
sudo /usr/bin/test -S "$bus" &&
sudo /usr/bin/test ! -L "$bus" &&
[[ "$(sudo stat -c '%u' -- "$bus")" == "$expected_uid" ]] &&
sudo -u "$execution_account" /usr/bin/test -S "$bus" || {
echo "::error::Qualification systemd user bus $context" >&2
return 1
}
}
sudo systemctl start "$user_manager_unit"
systemctl is-active --quiet "$runtime_directory_unit" || {
echo "::error::Qualification systemd runtime-directory unit is not active" >&2
exit 1
}
[[ -d "$runtime_dir" && ! -L "$runtime_dir" && "$(stat -c '%u:%g:%a' "$runtime_dir")" == "${uid}:${gid}:700" ]] || {
echo "::error::Qualification runtime directory is missing or invalid" >&2
exit 1
}
systemctl is-active --quiet "$user_manager_unit" || {
echo "::error::Qualification systemd user manager is not active" >&2
exit 1
}
sudo -u "$account" env -i \
HOME="$home" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$runtime_dir" \
/usr/bin/systemctl --user start dbus.socket
sudo -u "$account" env -i \
HOME="$home" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$runtime_dir" \
/usr/bin/systemctl --user is-active --quiet dbus.socket || {
echo "::error::Qualification systemd user bus socket unit is not active" >&2
exit 1
}
verify_user_bus "$account" "$uid" "$runtime_dir/bus" \
"is invalid or inaccessible to the execution account"
verify_user_manager_unit_path "$account" "$home" "$runtime_dir"
storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
storage_config="${storage_config_directory}/storage.conf"
containers_config="${storage_config_directory}/containers.conf"
podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
apparmor_profile="${storage_config_directory}/podman.apparmor"
apparmor_profile_name="nemoclaw-native-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
pasta_executable="${helper_directory}/pasta"
pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor"
pasta_apparmor_profile_name="nemoclaw-native-pasta-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
[[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || {
echo "::error::Qualification storage configuration directory already exists" >&2
exit 1
}
[[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || {
echo "::error::Qualification helper directory already exists" >&2
exit 1
}
sudo install -d -o root -g root -m 0755 "$storage_config_directory"
[[ -f "$TOOLCHAIN_DIRECTORY/bin/podman" && ! -L "$TOOLCHAIN_DIRECTORY/bin/podman" ]] || {
echo "::error::Pinned qualification Podman executable source is missing or invalid" >&2
exit 1
}
[[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || {
echo "::error::Run-owned qualification Podman executable already exists" >&2
exit 1
}
sudo install --owner=root --group=root --mode=0555 \
"$TOOLCHAIN_DIRECTORY/bin/podman" "$podman_executable"
[[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || {
echo "::error::Run-owned qualification Podman executable is invalid" >&2
exit 1
}
[[ "$(sha256sum "$podman_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/podman" | cut -d' ' -f1)" ]] || {
echo "::error::Run-owned qualification Podman executable digest changed during installation" >&2
exit 1
}
[[ -f "$TOOLCHAIN_DIRECTORY/bin/pasta" && ! -L "$TOOLCHAIN_DIRECTORY/bin/pasta" ]] || {
echo "::error::Pinned qualification pasta executable source is missing or invalid" >&2
exit 1
}
sudo install -d --owner=root --group=root --mode=0555 "$helper_directory"
sudo install --owner=root --group=root --mode=0555 \
"$TOOLCHAIN_DIRECTORY/bin/pasta" "$pasta_executable"
[[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || {
echo "::error::Run-owned qualification helper directory is invalid" >&2
exit 1
}
[[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || {
echo "::error::Run-owned qualification pasta executable is invalid" >&2
exit 1
}
[[ "$(sha256sum "$pasta_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/pasta" | cut -d' ' -f1)" ]] || {
echo "::error::Run-owned qualification pasta executable digest changed during installation" >&2
exit 1
}
printf '%s\n' \
'[storage]' \
'driver = "overlay"' \
"runroot = \"${home}/.local/share/containers/runroot\"" \
"graphroot = \"${home}/.local/share/containers/storage\"" \
"rootless_storage_path = \"${home}/.local/share/containers/storage\"" | sudo tee "$storage_config" >/dev/null
sudo chown root:root "$storage_config"
sudo chmod 0444 "$storage_config"
[[ -f "$storage_config" && ! -L "$storage_config" && "$(stat -c '%u:%g:%a' "$storage_config")" == "0:0:444" ]] || {
echo "::error::Qualification storage configuration is not root-owned and read-only" >&2
exit 1
}
printf '%s\n' \
'[network]' \
'firewall_driver = "nftables"' | sudo tee "$containers_config" >/dev/null
sudo chown root:root "$containers_config"
sudo chmod 0444 "$containers_config"
[[ -f "$containers_config" && ! -L "$containers_config" && "$(stat -c '%u:%g:%a' "$containers_config")" == "0:0:444" ]] || {
echo "::error::Qualification containers configuration is not root-owned and read-only" >&2
exit 1
}
if [[ -r /sys/module/apparmor/parameters/enabled ]] && grep -q '^Y' /sys/module/apparmor/parameters/enabled; then
printf '%s\n' \
'# This ephemeral profile grants user namespaces only to the pinned qualification Podman binary.' \
'' \
'abi <abi/4.0>,' \
'include <tunables/global>' \
'' \
"profile ${apparmor_profile_name} ${podman_executable} flags=(unconfined) {" \
' userns,' \
'}' | sudo tee "$apparmor_profile" >/dev/null
sudo chown root:root "$apparmor_profile"
sudo chmod 0444 "$apparmor_profile"
[[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || {
echo "::error::Qualification AppArmor profile is not root-owned and read-only" >&2
exit 1
}
sudo apparmor_parser -r "$apparmor_profile"
printf '%s\n' \
'# This ephemeral profile is limited to the immutable run-owned pasta helper.' \
'' \
'abi <abi/4.0>,' \
'include <tunables/global>' \
'' \
"profile ${pasta_apparmor_profile_name} ${pasta_executable} flags=(unconfined) {" \
' userns,' \
'}' | sudo tee "$pasta_apparmor_profile" >/dev/null
sudo chown root:root "$pasta_apparmor_profile"
sudo chmod 0444 "$pasta_apparmor_profile"
[[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || {
echo "::error::Qualification pasta AppArmor profile is not root-owned and read-only" >&2
exit 1
}
sudo apparmor_parser -r "$pasta_apparmor_profile"
fi
ancestor="$(dirname "$CANDIDATE_DIRECTORY")"
while [[ "$ancestor" != "/home" ]]; do
[[ ("$ancestor" == "/home/runner" || "$ancestor" == /home/runner/*) && -d "$ancestor" && ! -L "$ancestor" ]] || {
echo "::error::Candidate checkout ancestor is outside the reviewed runner workspace" >&2
exit 1
}
sudo setfacl --modify "u:${account}:--x" "$ancestor"
ancestor="$(dirname "$ancestor")"
done
sudo chown -R "$uid:$gid" "$CANDIDATE_DIRECTORY"
node_directory="$(dirname "$(command -v node)")"
[[ "$node_directory" == /* && -x "$node_directory/node" && -x "$node_directory/npm" ]] || {
echo "::error::Pinned Node toolchain path is invalid" >&2
exit 1
}
guard_dir="${RUNNER_TEMP}/native-runtime-docker-guard"
install -d -m 0755 "$guard_dir"
printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker"
chmod 0555 "$guard_dir/docker"
podman_info="$(sudo -u "$account" env -i \
CONTAINERS_CONF="$containers_config" \
CONTAINERS_STORAGE_CONF="$storage_config" \
HOME="$home" \
LANG=C.UTF-8 \
PATH="$guard_dir:$helper_directory:/usr/local/bin:/usr/bin:/bin" \
XDG_RUNTIME_DIR="$runtime_dir" \
"$podman_executable" info --format json)" || {
echo "::error::Credential-free rootless Podman readiness failed" >&2
exit 1
}
jq -e '
.host.security.rootless == true and
.store.graphDriverName == "overlay" and
(((.store.graphOptions // {})["overlay.mount_program"].Executable? // "") == "")
' <<<"$podman_info" >/dev/null || {
echo "::error::Qualification requires native rootless overlay storage" >&2
exit 1
}
printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT"
printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT"
printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT"
printf 'helper_dir=%s\n' "$helper_directory" >>"$GITHUB_OUTPUT"
printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT"
printf 'podman_executable=%s\n' "$podman_executable" >>"$GITHUB_OUTPUT"
printf 'containers_config=%s\n' "$containers_config" >>"$GITHUB_OUTPUT"
printf 'runtime_directory_unit=%s\n' "$runtime_directory_unit" >>"$GITHUB_OUTPUT"
printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT"
printf 'user_manager_unit=%s\n' "$user_manager_unit" >>"$GITHUB_OUTPUT"
- name: Prepare GPU resources with the NVIDIA API key
id: gpu_resources
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
ACCOUNT_GID: ${{ steps.boundary.outputs.gid }}
ACCOUNT_UID: ${{ steps.boundary.outputs.uid }}
ACCELERATION: ${{ matrix.case.acceleration }}
ARCHITECTURE: ${{ matrix.case.architecture }}
CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }}
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }}
INFERENCE: ${{ matrix.case.inference }}
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }}
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }}
STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }}
shell: bash
run: |
set -euo pipefail
umask 077
if [[ "$ACCELERATION" != "nvidia-gpu" ]]; then
printf 'runner_contract=\n' >>"$GITHUB_OUTPUT"
exit 0
fi
[[ -n "$NVIDIA_API_KEY" ]] || {
echo "::error::Native runtime GPU preparation requires the NVIDIA_API_KEY repository secret" >&2
exit 1
}
uid="$(id -u "$ACCOUNT")"
gid="$(id -g "$ACCOUNT")"
[[ "$uid" == "$ACCOUNT_UID" && "$gid" == "$ACCOUNT_GID" ]] || {
echo "::error::Qualification account identity changed before GPU resource preparation" >&2
exit 1
}
storage_config_directory="$(dirname "$STORAGE_CONFIG")"
runner_contract="${storage_config_directory}/runner-contract.json"
registry_auth_directory="${storage_config_directory}/registry-auth"
registry_auth_file="${registry_auth_directory}/auth.json"
resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
model_directory="${resource_directory}/model"
model_revision="7ae557604adf67be50417f59c2c2f167def9a775"
model_name="Qwen/Qwen2.5-0.5B-Instruct"
case "$ARCHITECTURE" in
amd64)
probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:9855f4c8500addf185360474184b9efdaf4384284779aa9173dcd70164a4ae6f"
nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:a0fdbecdf51792dadc48d284fde3199a58d5a2067007ad5b80319975fe81ce93"
vllm_image="nvcr.io/nvidia/vllm@sha256:7be6c2f676c36059a494fe17254e69ae5c677535ba6191044e5fc8e42a91c773"
;;
arm64)
probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:a54fdceac3bc2a8d177f07db942defc2f7237e18d07fca2ff00718ba5aee4940"
nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:8342257b9744e9bc23a02e0f45badad6b88474727473965b4dc83e8fee45956a"
vllm_image="nvcr.io/nvidia/vllm@sha256:447995cbb57e6c7cf792cab95e9852e5f62b5fb6d2f39e030fa4eda9a54eadb4"
;;
*)
echo "::error::GPU resource architecture is unsupported" >&2
exit 1
;;
esac
[[ ! -e "$runner_contract" && ! -L "$runner_contract" ]] || {
echo "::error::Run-owned GPU runner contract already exists" >&2
exit 1
}
[[ ! -e "$registry_auth_directory" && ! -L "$registry_auth_directory" ]] || {
echo "::error::Run-owned registry authentication directory already exists" >&2
exit 1
}
sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$registry_auth_directory"
cleanup_registry_auth() {
local result="$?"
trap - EXIT
if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then
sudo test -f "$registry_auth_file" &&
sudo test ! -L "$registry_auth_file" &&
[[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || {
echo "::error::Run-owned registry authentication file cleanup target is invalid" >&2
return 1
}
sudo unlink "$registry_auth_file"
fi
sudo rmdir "$registry_auth_directory" || {
echo "::error::Run-owned registry authentication directory was not empty" >&2
return 1
}
return "$result"
}
trap cleanup_registry_auth EXIT
printf '%s' "$NVIDIA_API_KEY" | sudo -u "$ACCOUNT" env -i \
CONTAINERS_CONF="$CONTAINERS_CONFIG" \
CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH="$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
REGISTRY_AUTH_FILE="$registry_auth_file" \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
"$PODMAN_EXECUTABLE" login nvcr.io --username '$oauthtoken' --password-stdin >/dev/null
sudo test -f "$registry_auth_file" &&
sudo test ! -L "$registry_auth_file" &&
[[ "$(sudo stat -c '%u:%g:%h' -- "$registry_auth_file")" == "${uid}:${gid}:1" ]] || {
echo "::error::Run-owned registry authentication file is invalid" >&2
exit 1
}
sudo chmod 0600 -- "$registry_auth_file"
[[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || {
echo "::error::Run-owned registry authentication file permissions are invalid" >&2
exit 1
}
image_to_pull="$probe_image"
if [[ "$INFERENCE" == "nim" ]]; then
image_to_pull="$nim_image"
elif [[ "$INFERENCE" == "vllm" ]]; then
image_to_pull="$vllm_image"
fi
images=("$probe_image")
[[ "$image_to_pull" == "$probe_image" ]] || images+=("$image_to_pull")
for image in "${images[@]}"; do
sudo -u "$ACCOUNT" env -i \
CONTAINERS_CONF="$CONTAINERS_CONFIG" \
CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
REGISTRY_AUTH_FILE="$registry_auth_file" \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
"$PODMAN_EXECUTABLE" pull "$image"
done
cleanup_registry_auth
trap - EXIT
unset NVIDIA_API_KEY
if [[ "$INFERENCE" == "nim" || "$INFERENCE" == "vllm" ]]; then
[[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || {
echo "::error::Run-owned GPU model resource already exists" >&2
exit 1
}
sudo install -d --owner=root --group=root --mode=0711 "$resource_directory"
sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$model_directory"
download_model_file() {
local file="$1"
local size="$2"
local algorithm="$3"
local digest="$4"
local target="${model_directory}/${file}"
sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
/usr/bin/curl \
--fail --location --proto '=https' --retry 3 --show-error --silent --tlsv1.2 \
--output "$target" \
"https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/${model_revision}/${file}?download=true"
sudo test -f "$target" &&
sudo test ! -L "$target" &&
[[ "$(sudo stat -c '%u:%g:%h:%s' -- "$target")" == "${uid}:${gid}:1:${size}" ]] || {
echo "::error::Downloaded GPU model file metadata is invalid: $file" >&2
exit 1
}
sudo chmod 0600 -- "$target"
[[ "$(sudo stat -c '%u:%g:%a:%h:%s' -- "$target")" == "${uid}:${gid}:600:1:${size}" ]] || {
echo "::error::Downloaded GPU model file permissions are invalid: $file" >&2
exit 1
}
if [[ "$algorithm" == "sha256" ]]; then
[[ "$(sudo sha256sum -- "$target" | cut -d' ' -f1)" == "$digest" ]]
else
[[ "$(sudo git hash-object --no-filters -- "$target")" == "$digest" ]]
fi || {
echo "::error::Downloaded GPU model file digest is invalid: $file" >&2
exit 1
}
}
download_model_file config.json 659 sha1 0dbb161213629a23f0fc00ef286e6b1e366d180f
download_model_file generation_config.json 242 sha1 dfc11073787daf1b0f9c0f1499487ab5f4c93738
download_model_file merges.txt 1671839 sha1 20024bfe7c83998e9aeaf98a0cd6a2ce6306c2f0
download_model_file model.safetensors 988097824 sha256 fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe
download_model_file tokenizer.json 7031645 sha1 443909a61d429dff23010e5bddd28ff530edda00
download_model_file tokenizer_config.json 7305 sha1 07bfe0640cb5a0037f9322287fbfc682806cf672
download_model_file vocab.json 2776833 sha1 4783fe10ac3adce15ac8f358ef5462739852c569
for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do
sudo chown root:root "${model_directory}/${file}"
sudo chmod 0444 "${model_directory}/${file}"
done
sudo chown root:root "$model_directory"
sudo chmod 0555 "$model_directory"
sudo chmod 0555 "$resource_directory"
fi
jq -n \
--arg architecture "$ARCHITECTURE" \
--arg gpuProbeImageRef "$probe_image" \
--arg model "$model_name" \
--arg modelPath "$model_directory" \
--arg modelRevision "$model_revision" \
--arg nimImageRef "$nim_image" \
--arg vllmImageRef "$vllm_image" '
{
schemaVersion: 1,
kind: "nemoclaw-native-runtime-qualification-runner-v1",
architecture: $architecture,
gpuProbeImageRef: $gpuProbeImageRef,
nim: {
imageRef: $nimImageRef,
model: $model,
modelPath: $modelPath,
modelRevision: $modelRevision
},
vllm: {
imageRef: $vllmImageRef,
model: $model,
modelPath: $modelPath,
modelRevision: $modelRevision
}
}
' | sudo tee "$runner_contract" >/dev/null
sudo chown root:root "$runner_contract"
sudo chmod 0444 "$runner_contract"
[[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || {
echo "::error::Run-owned GPU runner contract is invalid" >&2
exit 1
}
printf 'runner_contract=%s\n' "$runner_contract" >>"$GITHUB_OUTPUT"
- name: Install locked candidate test dependencies without scripts
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }}
NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }}
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
shell: bash
run: |
set -euo pipefail
sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
/bin/bash --noprofile --norc -c '
set -euo pipefail
cd "$1"
for file in package.json package-lock.json; do
[[ -f "$file" && ! -L "$file" && -O "$file" ]] || {
echo "Candidate dependency manifest is missing or invalid: $file" >&2
exit 1
}
done
exec npm --prefix "$1" ci --ignore-scripts
' bash "$CANDIDATE_DIRECTORY"
- name: Run the authenticated installer qualification
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
ACCOUNT_GID: ${{ steps.boundary.outputs.gid }}
ACCOUNT_UID: ${{ steps.boundary.outputs.uid }}
ARCHITECTURE: ${{ matrix.case.architecture }}
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
CANDIDATE_SHA: ${{ matrix.source.candidateSha }}
CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }}
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }}
INSTALLER_RECEIPT_PARENT: ${{ runner.temp }}/native-runtime-installer
INSTALLER_SHA256: ${{ matrix.installerSha256 }}
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }}
RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }}
STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }}
TRUSTED_USER_UNIT_PATH: /usr/lib/systemd/user:/lib/systemd/user
USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }}
shell: bash
run: |
set -euo pipefail
install -d -m 0700 "$INSTALLER_RECEIPT_PARENT"
sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$INSTALLER_RECEIPT_PARENT"
sudo -u "$ACCOUNT" env -i \
CONTAINERS_CONF="$CONTAINERS_CONFIG" \
CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
bash .qualification-workflow/scripts/checks/run-native-runtime-installer-qualification.sh \
--candidate-checkout "$CANDIDATE_DIRECTORY" \
--candidate-sha "$CANDIDATE_SHA" \
--installer-sha256 "$INSTALLER_SHA256" \
--architecture "$ARCHITECTURE" \
--artifact-dir "$INSTALLER_RECEIPT_PARENT/receipts"
sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT"
! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || {
echo "::error::Qualification systemd user lifecycle remained active after installer isolation" >&2
exit 1
}
sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true
sudo systemctl start "$USER_MANAGER_UNIT"
systemctl is-active --quiet "$USER_MANAGER_UNIT" && systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || {
echo "::error::Qualification systemd user lifecycle did not restart after installer isolation" >&2
exit 1
}
sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
/usr/bin/systemctl --user start dbus.socket
sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
/usr/bin/systemctl --user is-active --quiet dbus.socket || {
echo "::error::Qualification systemd user bus socket unit did not restart after installer isolation" >&2
exit 1
}
sudo /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" &&
sudo /usr/bin/test ! -L "$RUNTIME_DIRECTORY/bus" &&
[[ "$(sudo stat -c '%u' -- "$RUNTIME_DIRECTORY/bus")" == "$ACCOUNT_UID" ]] &&
sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" || {
echo "::error::Qualification systemd user bus is invalid or inaccessible after installer isolation" >&2
exit 1
}
manager_environment="$(sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
/usr/bin/systemctl --user show-environment)"
tr ' ' '\n' <<<"$manager_environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$TRUSTED_USER_UNIT_PATH" >/dev/null || {
echo "::error::Qualification systemd user manager lost the trusted unit path after installer isolation" >&2
exit 1
}
sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts" || {
echo "::error::Installer receipt directory is missing or invalid" >&2
exit 1
}
sudo test ! -L "$INSTALLER_RECEIPT_PARENT/receipts" || {
echo "::error::Installer receipt directory must not be a symlink" >&2
exit 1
}
- name: Execute the candidate qualification case without credentials
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
ACCOUNT_GID: ${{ steps.boundary.outputs.gid }}
ACCOUNT_UID: ${{ steps.boundary.outputs.uid }}
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }}
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }}
NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }}
PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }}
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
RUNNER_CONTRACT: ${{ steps.gpu_resources.outputs.runner_contract }}
RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }}
RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }}
STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }}
USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }}
shell: bash
run: |
set -euo pipefail
live_test="test/e2e/live/native-runtime-qualification-case.test.ts"
[[ -f "$CANDIDATE_DIRECTORY/$live_test" && ! -L "$CANDIDATE_DIRECTORY/$live_test" ]] || {
echo "::error::Candidate commit does not provide the native runtime qualification case executor" >&2
exit 1
}
receipt_directory="${RUNNER_TEMP}/native-runtime-case"
install -d -m 0700 "$receipt_directory"
sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$receipt_directory"
cd "$CANDIDATE_DIRECTORY"
sudo -u "$ACCOUNT" env -i \
CI=true \
CONTAINERS_CONF="$CONTAINERS_CONFIG" \
CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \
E2E_DEFAULT_ENABLED=0 \
E2E_JOB=1 \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT="$receipt_directory/execution.json" \
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW="$QUALIFICATION_ROW" \
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE" \
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT="$RUNNER_CONTRACT" \
NEMOCLAW_RUN_LIVE_E2E=1 \
PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
"$CANDIDATE_DIRECTORY/node_modules/.bin/vitest" run \
--config "$CANDIDATE_DIRECTORY/vitest.config.ts" \
--project e2e-live \
"$live_test"
sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT"
! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || {
echo "::error::Qualification systemd user lifecycle remained active after candidate execution" >&2
exit 1
}
sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true
- name: Verify Docker stayed unavailable
shell: bash
run: |
set -euo pipefail
! systemctl is-active --quiet docker.service
! systemctl is-active --quiet docker.socket
! pgrep -x dockerd >/dev/null
[[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]]
- name: Validate receipts and emit bounded evidence
env:
EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-evidence
EXECUTION_RECEIPT_PATH: ${{ runner.temp }}/native-runtime-case/execution.json
INSTALLER_RECEIPT_DIRECTORY: ${{ runner.temp }}/native-runtime-installer/receipts
NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }}
shell: bash
run: |
set -euo pipefail
sudo --preserve-env=EVIDENCE_DIRECTORY,EXECUTION_RECEIPT_PATH,INSTALLER_RECEIPT_DIRECTORY,QUALIFICATION_ROW \
"$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \
.qualification-workflow/tools/e2e/native-runtime-qualification-producer-evidence.mts
sudo chown -R "$(id -u):$(id -g)" "$EVIDENCE_DIRECTORY"
- name: Remove qualification resources
if: always()
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
ACCOUNT_CREATED: ${{ steps.boundary.outputs.account_created }}
shell: bash
run: |
set -euo pipefail
reported_account="${ACCOUNT:-}"
reported_created="${ACCOUNT_CREATED:-}"
ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
account=""
uid=""
gid=""
if [[ -e "$ownership_marker" || -L "$ownership_marker" ]]; then
[[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || {
echo "::error::Qualification account ownership marker cleanup target is invalid" >&2
exit 1
}
ownership="$(sudo cat "$ownership_marker")"
[[ "$ownership" =~ ^nemoclawq:([0-9]+):([0-9]+)$ ]] || {
echo "::error::Qualification account ownership marker content is invalid" >&2
exit 1
}
account="nemoclawq"
uid="${BASH_REMATCH[1]}"
gid="${BASH_REMATCH[2]}"
[[ -z "$reported_account" || "$reported_account" == "$account" ]] || {
echo "::error::Qualification account output does not match its ownership marker" >&2
exit 1
}
if getent passwd "$uid" >/dev/null && ! getent passwd "$account" >/dev/null; then
echo "::error::Qualification account UID belongs to a different host account" >&2
exit 1
fi
if getent passwd "$account" >/dev/null; then
[[ "$(id -u "$account")" == "$uid" && "$(id -g "$account")" == "$gid" ]] || {
echo "::error::Qualification account identity changed before cleanup" >&2
exit 1
}
fi
if getent group "$account" >/dev/null; then
[[ "$(getent group "$account")" == "${account}:x:${gid}:" ]] || {
echo "::error::Qualification private group identity changed before cleanup" >&2
exit 1
}
elif getent passwd "$account" >/dev/null; then
echo "::error::Qualification private group disappeared before cleanup" >&2
exit 1
fi
runtime_dir="/run/user/${uid}"
runtime_directory_unit="user-runtime-dir@${uid}.service"
user_manager_unit="user@${uid}.service"
user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d"
user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf"
trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user"
storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
apparmor_profile="${storage_config_directory}/podman.apparmor"
pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor"
helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
pasta_executable="${helper_directory}/pasta"
registry_auth_directory="${storage_config_directory}/registry-auth"
registry_auth_file="${registry_auth_directory}/auth.json"
runner_contract="${storage_config_directory}/runner-contract.json"
resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
model_directory="${resource_directory}/model"
sudo systemctl stop "$user_manager_unit" "$runtime_directory_unit" 2>/dev/null || true
! systemctl is-active --quiet "$user_manager_unit" && ! systemctl is-active --quiet "$runtime_directory_unit" || {
echo "::error::Qualification systemd user lifecycle remained active during cleanup" >&2
exit 1
}
if getent passwd "$account" >/dev/null; then
sudo pkill -KILL -u "$uid" 2>/dev/null || true
fi
if [[ -e "$user_manager_dropin_directory" || -L "$user_manager_dropin_directory" ]]; then
[[ -d "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" && "$(stat -c '%u:%g:%a' "$user_manager_dropin_directory")" == "0:0:755" ]] || {
echo "::error::Qualification systemd user-manager drop-in directory cleanup target is invalid" >&2
exit 1
}
[[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || {
echo "::error::Qualification systemd user-manager drop-in cleanup target is invalid" >&2
exit 1
}
expected_user_manager_dropin="$(printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"' "$trusted_user_unit_path")"
[[ "$(cat "$user_manager_dropin")" == "$expected_user_manager_dropin" ]] || {
echo "::error::Qualification systemd user-manager drop-in content changed before cleanup" >&2
exit 1
}
sudo unlink "$user_manager_dropin"
sudo rmdir "$user_manager_dropin_directory"
sudo systemctl daemon-reload
fi
if [[ -e "$storage_config_directory" || -L "$storage_config_directory" ]]; then
[[ -d "$storage_config_directory" && ! -L "$storage_config_directory" ]] || {
echo "::error::Qualification storage configuration cleanup target is invalid" >&2
exit 1
}
if [[ -e "$apparmor_profile" || -L "$apparmor_profile" ]]; then
[[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || {
echo "::error::Qualification AppArmor profile cleanup target is invalid" >&2
exit 1
}
sudo apparmor_parser -R "$apparmor_profile"
sudo rm -f -- "$apparmor_profile"
fi
if [[ -e "$pasta_apparmor_profile" || -L "$pasta_apparmor_profile" ]]; then
[[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || {
echo "::error::Qualification pasta AppArmor profile cleanup target is invalid" >&2
exit 1
}
sudo apparmor_parser -R "$pasta_apparmor_profile"
sudo rm -f -- "$pasta_apparmor_profile"
fi
if [[ -e "$registry_auth_directory" || -L "$registry_auth_directory" ]]; then
[[ -d "$registry_auth_directory" && ! -L "$registry_auth_directory" && "$(stat -c '%u:%g:%a' "$registry_auth_directory")" == "${uid}:${gid}:700" ]] || {
echo "::error::Qualification registry authentication directory cleanup target is invalid" >&2
exit 1
}
if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then
sudo test -f "$registry_auth_file" &&
sudo test ! -L "$registry_auth_file" &&
[[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || {
echo "::error::Qualification registry authentication file cleanup target is invalid" >&2
exit 1
}
sudo unlink "$registry_auth_file"
fi
sudo rmdir "$registry_auth_directory"
fi
if [[ -e "$runner_contract" || -L "$runner_contract" ]]; then
[[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || {
echo "::error::Qualification GPU runner contract cleanup target is invalid" >&2
exit 1
}
sudo unlink "$runner_contract"
fi
if [[ -e "$storage_config_directory/containers.conf" || -L "$storage_config_directory/containers.conf" ]]; then
[[ -f "$storage_config_directory/containers.conf" && ! -L "$storage_config_directory/containers.conf" && "$(stat -c '%u:%g:%a' "$storage_config_directory/containers.conf")" == "0:0:444" ]] || {
echo "::error::Qualification containers configuration cleanup target is invalid" >&2
exit 1
}
sudo unlink "$storage_config_directory/containers.conf"
fi
sudo rm -f -- "$storage_config_directory/storage.conf"
sudo rmdir "$storage_config_directory"
fi
if [[ -e "$resource_directory" || -L "$resource_directory" ]]; then
[[ -d "$resource_directory" && ! -L "$resource_directory" && ("$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:711" || "$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:555") ]] || {
echo "::error::Qualification GPU resource directory cleanup target is invalid" >&2
exit 1
}
if sudo test -e "$model_directory" || sudo test -L "$model_directory"; then
model_mode="$(sudo stat -c '%u:%g:%a' -- "$model_directory")"
sudo test -d "$model_directory" &&
sudo test ! -L "$model_directory" &&
[[ "$model_mode" == "${uid}:${gid}:700" || "$model_mode" == "0:0:555" ]] || {
echo "::error::Qualification GPU model directory cleanup target is invalid" >&2
exit 1
}
for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do
target="${model_directory}/${file}"
if sudo test -e "$target" || sudo test -L "$target"; then
file_mode="$(sudo stat -c '%u:%g:%a:%h' -- "$target")"
sudo test -f "$target" &&
sudo test ! -L "$target" &&
[[ "$file_mode" == "${uid}:${gid}:600:1" || "$file_mode" == "0:0:444:1" ]] || {
echo "::error::Qualification GPU model file cleanup target is invalid: $file" >&2
exit 1
}
sudo unlink "$target"
fi
done
sudo rmdir "$model_directory" || {
echo "::error::Qualification GPU model directory contains unexpected entries" >&2
exit 1
}
fi
sudo rmdir "$resource_directory" || {
echo "::error::Qualification GPU resource directory contains unexpected entries" >&2
exit 1
}
fi
if [[ -e "$podman_executable" || -L "$podman_executable" ]]; then
[[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || {
echo "::error::Qualification Podman executable cleanup target is invalid" >&2
exit 1
}
sudo rm -f -- "$podman_executable"
fi
if [[ -e "$helper_directory" || -L "$helper_directory" ]]; then
[[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || {
echo "::error::Qualification helper directory cleanup target is invalid" >&2
exit 1
}
[[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || {
echo "::error::Qualification pasta executable cleanup target is invalid" >&2
exit 1
}
sudo rm -f -- "$pasta_executable"
sudo rmdir "$helper_directory"
fi
if [[ -e "$runtime_dir" || -L "$runtime_dir" ]]; then
echo "::error::Qualification runtime directory remains after its systemd cleanup" >&2
exit 1
fi
if getent passwd "$account" >/dev/null; then
sudo userdel --remove "$account"
fi
if getent group "$account" >/dev/null; then
sudo groupdel "$account"
fi
if getent passwd "$account" >/dev/null; then
echo "::error::Qualification account still exists after cleanup" >&2
exit 1
fi
if getent group "$account" >/dev/null; then
echo "::error::Qualification private group still exists after cleanup" >&2
exit 1
fi
if grep -q "^${account}:" /etc/subuid /etc/subgid; then
echo "::error::Qualification subordinate-ID authorization remains after cleanup" >&2
exit 1
fi
[[ ! -e "/run/user/${uid}" && ! -L "/run/user/${uid}" ]] || {
echo "::error::Qualification runtime directory remains after cleanup" >&2
exit 1
}
storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
[[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || {
echo "::error::Qualification storage configuration remains after cleanup" >&2
exit 1
}
[[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || {
echo "::error::Qualification Podman executable remains after cleanup" >&2
exit 1
}
[[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || {
echo "::error::Qualification helper directory remains after cleanup" >&2
exit 1
}
[[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || {
echo "::error::Qualification GPU resource directory remains after cleanup" >&2
exit 1
}
sudo rm -f -- "$ownership_marker"
elif [[ -n "$reported_account" || "$reported_created" == "true" ]]; then
echo "::error::Qualification account output exists without its ownership marker" >&2
exit 1
fi
[[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || {
echo "::error::Qualification account ownership marker remains after cleanup" >&2
exit 1
}
- name: Upload the qualification case evidence
if: success()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: ${{ matrix.artifactName }}
path: ${{ runner.temp }}/native-runtime-evidence/
native-runtime-qualification-producer-aggregate:
name: Aggregate native runtime qualification evidence
needs:
[
generate-matrix,
native-runtime-qualification-producer-plan,
native-runtime-qualification-producer,
]
if: ${{ always() && needs.native-runtime-qualification-producer-plan.result == 'success' && needs.native-runtime-qualification-producer.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }}
runs-on: ubuntu-24.04
timeout-minutes: 15
permissions:
actions: read
contents: read
pull-requests: read
steps:
- name: Check out the qualification aggregator
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ github.workflow_sha }}
path: .qualification-aggregate
persist-credentials: false
sparse-checkout: |
src/lib/onboard/runtime-provider/native-qualification-authority.ts
test/e2e/registry/native-runtime-qualification.ts
tools/e2e/native-runtime-qualification-producer-plan.mts
tools/e2e/native-runtime-qualification-producer-aggregate.mts
sparse-checkout-cone-mode: false
- name: Download the exact case evidence cohort
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: native-runtime-qualification-evidence-${{ inputs.checkout_sha }}-*
path: ${{ runner.temp }}/native-runtime-case-artifacts
merge-multiple: false
- name: Resolve this aggregate job identity
id: aggregate-job
env:
GH_TOKEN: ${{ github.token }}
PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }}
PRODUCER_RUN_ID: ${{ github.run_id }}
shell: bash
run: |
set -euo pipefail
jobs="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/attempts/${PRODUCER_RUN_ATTEMPT}/jobs" \
-f per_page=100)"
total_count="$(jq -er '.total_count | select(type == "number" and . >= 1)' <<<"$jobs")" || {
echo "::error::Aggregate job lookup returned an invalid job count" >&2
exit 1
}
(( total_count <= 100 )) || {
echo "::error::Aggregate job lookup exceeds the bounded 100-job page" >&2
exit 1
}
job_id="$(jq -er \
--arg name 'Aggregate native runtime qualification evidence' \
--argjson runId "$PRODUCER_RUN_ID" \
--argjson attempt "$PRODUCER_RUN_ATTEMPT" '
[.jobs[] | select(
.name == $name and
.run_id == $runId and
.run_attempt == $attempt and
.status == "in_progress"
)] |
select(length == 1) |
.[0].id
' <<<"$jobs")" || {
echo "::error::Could not resolve one in-progress aggregate job identity" >&2
exit 1
}
[[ "$job_id" =~ ^[1-9][0-9]{0,19}$ ]] || {
echo "::error::Aggregate job identity is invalid" >&2
exit 1
}
printf 'job_id=%s\n' "$job_id" >>"$GITHUB_OUTPUT"
- name: Set up Node for qualification aggregation
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.19.0
- name: Validate and aggregate all 24 case receipts
working-directory: .qualification-aggregate
env:
AGGREGATE_JOB_ID: ${{ steps.aggregate-job.outputs.job_id }}
CASE_ARTIFACT_ROOT: ${{ runner.temp }}/native-runtime-case-artifacts
EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-aggregate
QUALIFICATION_PLAN: ${{ needs.native-runtime-qualification-producer-plan.outputs.matrix }}
run: >-
node --experimental-strip-types --no-warnings tools/e2e/native-runtime-qualification-producer-aggregate.mts
- name: Upload aggregate evidence
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: native-runtime-qualification-${{ inputs.checkout_sha }}
path: ${{ runner.temp }}/native-runtime-aggregate/
if-no-files-found: error
retention-days: 30
compression-level: 9
retired-selector-compatibility:
needs: generate-matrix
if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,') || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,') || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,') || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,')) }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/retired-selector-compatibility
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Verify retired selector replacements
env:
JOBS: ${{ inputs.jobs }}
TARGETS: ${{ inputs.targets }}
run: npx tsx tools/e2e/retired-selector-compatibility.mts
- name: Upload retired selector compatibility evidence
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-retired-selector-compatibility
path: e2e-artifacts/live/retired-selector-compatibility/
staging-brev-launchable:
name: Exact staging Brev Launchable
needs: generate-matrix
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}
runs-on: ubuntu-latest
timeout-minutes: 180
permissions:
contents: read
concurrency:
group: staging-brev-launchable-cpu
cancel-in-progress: false
env:
CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker
INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }}
E2E_AGENT_RUNTIME: "openclaw"
E2E_OBSERVABLE_OUTCOME: "The staging image boots and completes the full E2E scenario"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Brev Launchable Docker host; NVIDIA hosted inference"
steps:
- name: Checkout trusted Launchable lane
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: |
tools/e2e/brev-launchable-e2e.sh
sparse-checkout-cone-mode: false
- id: workspace
name: Prepare the trusted lane
env:
BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_API_KEY || '' }}
BREV_CLI_SHA256: d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32
BREV_CLI_VERSION: 0.6.334
BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_ORG_ID || '' }}
run: |
set -euo pipefail
work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-e2e.XXXXXX")"
chmod 700 "$work_dir"
archive="${RUNNER_TEMP}/brev-cli.tar.gz"
curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz"
printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c -
tar -xzf "$archive" -C "${RUNNER_TEMP}" brev && sudo install -m 0755 "${RUNNER_TEMP}/brev" /usr/local/bin/brev
brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID"
printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT"
- name: Build, deploy, verify, test, and clean up
env:
BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }}
GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }}
NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
WORK_DIR: ${{ steps.workspace.outputs.work_dir }}
run: tools/e2e/brev-launchable-e2e.sh
- name: Upload Launchable evidence
if: ${{ always() && steps.workspace.outputs.work_dir != '' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: staging-brev-launchable-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ steps.workspace.outputs.work_dir }}/lane.log
${{ steps.workspace.outputs.work_dir }}/launchable-e2e.json
${{ steps.workspace.outputs.work_dir }}/full-e2e.log
${{ steps.workspace.outputs.work_dir }}/cleanup.json
staging-brev-launchable-identity:
name: Exact staging Brev Launchable identity
needs: generate-matrix
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && inputs.jobs == 'staging-brev-launchable-identity' && inputs.targets == '' }}
runs-on: ubuntu-latest
timeout-minutes: 180
permissions:
contents: read
concurrency:
group: staging-brev-launchable-cpu
cancel-in-progress: false
env:
CANDIDATE_SHA: ${{ github.sha }}
E2E_DEFAULT_ENABLED: "0"
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: agnostic
INSTANCE_NAME: nclaw-identity-${{ github.run_id }}-${{ github.run_attempt }}
E2E_AGENT_RUNTIME: "none"
E2E_OBSERVABLE_OUTCOME: "The staging image boots, passes the SSH access probe, and matches the baked runtime identity"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Brev Launchable host; no inference endpoint"
steps:
- name: Checkout trusted Launchable identity lane
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: |
tools/e2e/brev-launchable-e2e.sh
sparse-checkout-cone-mode: false
- id: workspace
name: Prepare the trusted identity lane
env:
BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && secrets.BREV_API_KEY || '' }}
BREV_CLI_SHA256: d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32
BREV_CLI_VERSION: 0.6.334
BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && secrets.BREV_ORG_ID || '' }}
run: |
set -euo pipefail
install -d -m 0700 "$HOME/.brev"
work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-identity.XXXXXX")"
chmod 700 "$work_dir"
archive="${RUNNER_TEMP}/brev-cli.tar.gz"
curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz"
printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c -
tar -xzf "$archive" -C "${RUNNER_TEMP}" brev && sudo install -m 0755 "${RUNNER_TEMP}/brev" /usr/local/bin/brev
brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID"
printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT"
- name: Build, boot, and verify identity
timeout-minutes: 150
env:
BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }}
GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }}
NEMOCLAW_BREV_DEFER_CLEANUP: "1"
NEMOCLAW_BREV_LAUNCHABLE_IDENTITY_ONLY: "1"
WORK_DIR: ${{ steps.workspace.outputs.work_dir }}
run: tools/e2e/brev-launchable-e2e.sh
- name: Verify identity workspace cleanup
if: ${{ always() && steps.workspace.outputs.work_dir != '' }}
timeout-minutes: 15
env:
BREV_CREATE_RECONCILE_SECONDS: "120"
BREV_DELETE_TIMEOUT_SECONDS: "600"
POLL_SECONDS: "15"
WORK_DIR: ${{ steps.workspace.outputs.work_dir }}
run: tools/e2e/brev-launchable-e2e.sh cleanup-owned-workspace
- name: Remove Brev API credentials
if: always()
run: |
set -euo pipefail
credentials="$HOME/.brev/credentials.json"
rm -f -- "$credentials"
test ! -e "$credentials"
- name: Upload Launchable identity evidence
if: ${{ always() && steps.workspace.outputs.work_dir != '' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: staging-brev-launchable-identity-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ steps.workspace.outputs.work_dir }}/lane.log
${{ steps.workspace.outputs.work_dir }}/launchable-identity.json
${{ steps.workspace.outputs.work_dir }}/cleanup.json
live:
name: ${{ matrix.label }}
needs: [base-image-publication, generate-matrix]
if: ${{ needs.generate-matrix.outputs.matrix != '[]' }}
runs-on: ${{ matrix.runner }}
timeout-minutes: ${{ matrix.timeout_minutes }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live
E2E_EXECUTION_ID: ${{ matrix.execution_id }}
E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }}
E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }}
NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF: ${{ needs.generate-matrix.outputs.workload_source == 'managed-image' && needs.base-image-publication.outputs.dcode_base_ref || '' }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
# Keep only the credential-bearing step anchored. Cleanup mappings stay
# explicit because strict YAML decoders reject 100 or more aliases here.
- &dockerhub-auth
name: Authenticate to Docker Hub
uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f
with:
auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && '1' || '0' }}
username: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }}
token: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }}
- name: Configure live E2E trace directory
env:
TARGET_ID: ${{ matrix.id }}
shell: bash
run: |
set -euo pipefail
printf 'NEMOCLAW_TRACE_DIR=%s\n' "${RUNNER_TEMP}/nemoclaw-e2e-traces/${TARGET_ID}" >> "${GITHUB_ENV}"
# invalidState: the selected PR-modifiable TUI check needs a PTY driver,
# but the fixed GitHub-hosted runner image does not provide expect.
# sourceBoundary: privileged host setup runs from the first-party
# host-dependency-setup action pinned to an immutable full SHA, never the
# PR-controlled target ref; the check only consumes expect without privilege.
# whyNotSourceFix: GitHub-hosted jobs cannot use a repository-owned host
# image, and caching privileged dpkg state between clean runners is not
# supported.
# regressionTest: the workflow-boundary suite pins this target, condition,
# ordering, action provenance, and package mapping.
# removalCondition: remove the install when the hosted runner supplies
# expect or the acceptance check no longer requires a PTY.
- name: Install Deep Agents Code TUI host dependencies
if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}
uses: NVIDIA/NemoClaw/.github/actions/host-dependency-setup@4def1501b34ce586f83b91af50a66b5d22b31d75
with:
packages: expect
# Configure NEMOCLAW_TRACE_DIR before workspace prep so every child
# command writes raw traces under runner temp, never under upload roots.
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Record immutable Deep Agents Code base evidence
if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' && needs.generate-matrix.outputs.workload_source == 'managed-image' }}
env:
BASE_CONTRACT: ${{ needs.base-image-publication.outputs.dcode_base_contract }}
CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
TARGET_ID: ${{ matrix.id }}
TARGET_LABEL: ${{ matrix.label }}
shell: bash
run: |
set -euo pipefail
evidence_dir="${E2E_ARTIFACT_DIR}/${TARGET_ID}"
install -d -m 0700 "${evidence_dir}"
jq -e -n \
--arg candidateSha "${CANDIDATE_SHA}" \
--argjson baseContract "${BASE_CONTRACT}" \
'{contractVersion: 1, candidateSha: $candidateSha, base: $baseContract}
| select(
(.candidateSha | test("^[0-9a-f]{40}$"))
and (.base.reference | test("@sha256:[0-9a-f]{64}$"))
)' >"${evidence_dir}/dcode-base-image.json"
test_artifact_name="$(node -e '
const slug = process.argv[1]
.trim()
.toLowerCase()
.replace(/[^a-z0-9._-]+/g, "-")
.replace(/^-+|-+$/g, "");
process.stdout.write(slug || "unnamed-test");
' "${TARGET_LABEL}")"
test_evidence_dir="${E2E_ARTIFACT_DIR}/${test_artifact_name}"
install -d -m 0700 "${test_evidence_dir}"
install -m 0600 \
"${evidence_dir}/dcode-base-image.json" \
"${test_evidence_dir}/dcode-base-image.json"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Prepare native Podman E2E runtime
uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de
with:
enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}
# invalidState: a profile plugin installed with --no-deps can import even
# when an incomplete base image omitted its required upstream packages.
# sourceBoundary: this trusted workflow scopes the repo-owned stripped-base
# build to the exact DCode target; the production Dockerfile must reject it
# at the isolated import gate before its later dependency-consistency check.
# whyNotSourceFix: dependency completeness belongs to the hash-locked base;
# resolving dependencies during local plugin install would duplicate that
# trust boundary, so the regression constructs the invalid input instead.
# regressionTest: workflow-boundary tests pin the target, script, and
# ordering; build-gate tests pin the base build and failure contract.
# removalCondition: remove only if package installation no longer uses
# --no-deps or an equivalent earlier build gate proves both imports.
- name: Verify DCode profile import gate rejects missing base dependencies
if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}
shell: bash
run: bash scripts/check-dcode-profile-import-gate.sh
- name: Run live E2E tests
env:
E2E_TARGET_ID: ${{ matrix.id }}
NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
TARGET_ID: ${{ matrix.id }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/registry-targets.test.ts --selector "^${TARGET_ID}:"
# The sanitizer reads raw traces only after checking the workflow-owned
# runner-temp path, then writes the timing-only file into upload roots.
- name: Build trusted live E2E timing summary
if: always()
env:
TARGET_ID: ${{ matrix.id }}
shell: bash
run: |
set -euo pipefail
NEMOCLAW_TRACE_DIR="${NEMOCLAW_TRACE_DIR:-}"
expected_trace_dir="${RUNNER_TEMP}/nemoclaw-e2e-traces/${TARGET_ID}"
if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then
echo "::error title=E2E trace sanitization refused::NEMOCLAW_TRACE_DIR does not match its workflow-owned RUNNER_TEMP path. No raw traces were read or uploaded. Correct the trace path configuration before rerunning." >&2
printf 'Expected trace path: %s\n' "${expected_trace_dir}" >&2
exit 1
fi
python3 scripts/e2e/sanitize-trace-timing.py \
"${NEMOCLAW_TRACE_DIR}" \
"${E2E_ARTIFACT_DIR}/${TARGET_ID}"
# Cleanup intentionally runs after sanitization and before upload so raw
# trace JSON never becomes part of the uploaded artifact surface.
- name: Delete raw live E2E traces
if: always()
env:
TARGET_ID: ${{ matrix.id }}
shell: bash
run: |
set -euo pipefail
NEMOCLAW_TRACE_DIR="${NEMOCLAW_TRACE_DIR:-}"
expected_trace_dir="${RUNNER_TEMP}/nemoclaw-e2e-traces/${TARGET_ID}"
if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then
echo "::error title=E2E trace cleanup refused::NEMOCLAW_TRACE_DIR does not match its workflow-owned RUNNER_TEMP path. No files were deleted. Correct the trace path configuration before rerunning." >&2
printf 'Expected trace path: %s\n' "${expected_trace_dir}" >&2
exit 1
fi
rm -rf -- "${NEMOCLAW_TRACE_DIR}"
- name: Summarize artifacts
if: always()
env:
TARGET_ID: ${{ matrix.id }}
TARGET_LABEL: ${{ matrix.label }}
run: |
python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY"
import json
import os
from pathlib import Path
root = Path(os.environ["E2E_ARTIFACT_DIR"]) / os.environ["TARGET_ID"]
plan_path = root / "run-plan.json"
print("## E2E Targets")
print()
print("- Project: `e2e-live`")
print(f"- Target: `{os.environ['TARGET_ID']}`")
print(f"- Label: `{os.environ['TARGET_LABEL']}`")
print(f"- Artifact root: `{root}`")
print()
print("| Target | Manifest | Expected state | Suites | Phases |")
print("| --- | --- | --- | --- | --- |")
if plan_path.exists():
plan = json.loads(plan_path.read_text(encoding="utf-8"))
suites = ", ".join(plan.get("suiteIds") or []) or "(none)"
phases = ", ".join(plan.get("phases") or []) or "(none)"
print(
"| "
f"`{plan.get('targetId') or os.environ['TARGET_ID']}` | "
f"`{plan.get('manifestPath') or 'not-yet-defined'}` | "
f"`{plan.get('expectedStateId') or 'not-yet-defined'}` | "
f"{suites} | {phases} |"
)
else:
print(
"| "
f"`{os.environ['TARGET_ID']}` | `(missing run-plan.json)` | "
"`(missing run-plan.json)` | `(missing)` | `(missing)` |"
)
PY
- name: Upload E2E artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-${{ matrix.execution_id }}
path: |
e2e-artifacts/live/${{ matrix.id }}/run-plan.json
e2e-artifacts/live/${{ matrix.id }}/target.json
e2e-artifacts/live/${{ matrix.id }}/target-result.json
e2e-artifacts/live/${{ matrix.id }}/test-progress.json
e2e-artifacts/live/${{ matrix.id }}/environment.result.json
e2e-artifacts/live/${{ matrix.id }}/onboarding.result.json
e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json
e2e-artifacts/live/${{ matrix.id }}/dcode-base-image.json
e2e-artifacts/live/${{ matrix.id }}/cloud-onboard-trace-timing-summary.json
e2e-artifacts/live/${{ matrix.id }}/onboard-progress-budget.json
e2e-artifacts/live/risk-signal.json
e2e-artifacts/live/${{ matrix.id }}/actions/
e2e-artifacts/live/${{ matrix.id }}/logs/
e2e-artifacts/live/${{ matrix.id }}/shell/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# Credential-free tests opt in with a tag beside the test. Discovery supplies
# only a validated test ID, file, and Vitest project; this E2E workflow owns
# the shared job's runner, setup, timeout, permissions, and artifact policy.
shared-e2e:
name: Shared E2E (${{ matrix.execution_id }})
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.test_matrix != '[]' }}
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.test_matrix) }}
env:
CHECK_DOC_LINKS_REMOTE: "0"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/${{ matrix.execution_id }}
E2E_EXECUTION_ID: ${{ matrix.execution_id }}
E2E_TARGET_ID: ${{ matrix.id }}
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Prepare native Podman E2E runtime
uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de
with:
enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}
- name: Run tagged credential-free test
env:
TEST_FILE: ${{ matrix.file }}
TEST_PROJECT: ${{ matrix.project }}
run: |
set -euo pipefail
npx vitest run --project "${TEST_PROJECT}" "${TEST_FILE}" \
--tags-filter=e2e/credential-free \
--silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts
- name: Upload test artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-${{ matrix.execution_id }}
path: e2e-artifacts/live/${{ matrix.execution_id }}/
catalogue-standard:
name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ needs.generate-matrix.outputs.catalogue_standard_matrix != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_standard_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
runtime_provider: ${{ matrix.runtime_provider }}
execution_id: ${{ matrix.execution_id }}
coverage_variant: ${{ matrix.coverage_variant }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }}
managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
workload_source: ${{ needs.generate-matrix.outputs.workload_source }}
credential_boundary: no provider credential
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: false
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }}
DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }}
catalogue-nvidia-api:
name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_api_matrix != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_nvidia_api_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
runtime_provider: ${{ matrix.runtime_provider }}
execution_id: ${{ matrix.execution_id }}
coverage_variant: ${{ matrix.coverage_variant }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }}
managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
workload_source: ${{ needs.generate-matrix.outputs.workload_source }}
credential_boundary: NVIDIA API key
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: false
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }}
DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }}
NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }}
catalogue-nvidia-inference:
name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_inference_matrix != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_nvidia_inference_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
runtime_provider: ${{ matrix.runtime_provider }}
execution_id: ${{ matrix.execution_id }}
coverage_variant: ${{ matrix.coverage_variant }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }}
managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
workload_source: ${{ needs.generate-matrix.outputs.workload_source }}
credential_boundary: NVIDIA inference API key
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: false
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }}
DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }}
NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
catalogue-github-read:
name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ needs.generate-matrix.outputs.catalogue_github_read_matrix != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_github_read_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
runtime_provider: ${{ matrix.runtime_provider }}
execution_id: ${{ matrix.execution_id }}
coverage_variant: ${{ matrix.coverage_variant }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }}
managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
workload_source: ${{ needs.generate-matrix.outputs.workload_source }}
credential_boundary: GitHub read token
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: true
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }}
DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }}
catalogue-brave-nvidia-inference:
name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ needs.generate-matrix.outputs.catalogue_brave_nvidia_inference_matrix != '[]' }}
strategy:
fail-fast: false
max-parallel: 2
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_brave_nvidia_inference_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
runtime_provider: ${{ matrix.runtime_provider }}
execution_id: ${{ matrix.execution_id }}
coverage_variant: ${{ matrix.coverage_variant }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }}
managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
workload_source: ${{ needs.generate-matrix.outputs.workload_source }}
credential_boundary: Brave and NVIDIA inference API keys
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: false
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }}
DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }}
BRAVE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.BRAVE_API_KEY || '' }}
NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
openshell-gateway-auth-contract:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-gateway-auth-contract') }}
runs-on: ubuntu-latest
timeout-minutes: 20
env:
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker
E2E_TARGET_ID: "openshell-gateway-auth-contract"
E2E_AGENT_RUNTIME: "none"
E2E_OBSERVABLE_OUTCOME: "Gateway mTLS and sandbox JWT authentication boundaries hold"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; no inference endpoint"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-gateway-auth-contract
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_CANDIDATE_VERSION: "0.0.106"
NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.106"
DOCKER_GRPC_PROBE_IMAGE: "node:22-trixie-slim@sha256:db8a96a63e5264607ada2d206758876ebbed6a12be2ada7517793cbfb0c2a29c"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Install OpenShell CLI
run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh
- name: Pre-pull pinned gateway auth probe image
run: docker pull "$DOCKER_GRPC_PROBE_IMAGE"
- name: Run OpenShell gateway auth contract live test
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell-gateway >/dev/null 2>&1; then
OPENSHELL_GATEWAY_BIN="$(command -v openshell-gateway)"
elif [ -x "$HOME/.local/bin/openshell-gateway" ]; then
OPENSHELL_GATEWAY_BIN="$HOME/.local/bin/openshell-gateway"
else
echo "::error::OpenShell gateway binary not found after install"
ls -la /usr/local/bin/openshell-gateway "$HOME/.local/bin/openshell-gateway" 2>&1 || true
exit 1
fi
export OPENSHELL_GATEWAY_BIN
echo "Using OPENSHELL_GATEWAY_BIN=$OPENSHELL_GATEWAY_BIN"
"$OPENSHELL_GATEWAY_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openshell-gateway-auth-source-contract.test.ts
- id: artifact_safety
name: Validate final OpenShell gateway auth contract artifacts
if: always()
run: node --experimental-strip-types --no-warnings tools/e2e/openshell-gateway-auth-artifact-safety.mts "$E2E_ARTIFACT_DIR"
- name: Upload OpenShell gateway auth contract artifacts
if: ${{ always() && steps.artifact_safety.outcome == 'success' && steps.artifact_safety.outputs.approved_path != '' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-openshell-gateway-auth-contract
path: ${{ steps.artifact_safety.outputs.approved_path }}
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
external-gateway-health:
needs: [generate-matrix, package-openshell-sdk]
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'external-gateway-health') }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
E2E_JOB: "1"
E2E_DEFAULT_ENABLED: "0"
E2E_TARGET_ID: "external-gateway-health"
E2E_AGENT_RUNTIME: "none"
E2E_OBSERVABLE_OUTCOME: "The exact Blueprint Runner observes public gateway health over explicit HTTPS and CA"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu host with OpenShell 0.0.106; no inference endpoint"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/external-gateway-health
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.106"
NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST: "1"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Download reviewed OpenShell SDK archive
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ needs.package-openshell-sdk.outputs.artifact_name }}
path: ${{ runner.temp }}/openshell-sdk
- name: Install reviewed OpenShell SDK archive without package credentials
run: |
set -euo pipefail
mapfile -t archives < <(find "$RUNNER_TEMP/openshell-sdk" -maxdepth 1 -type f -name '*.tgz' -print)
test "${#archives[@]}" -eq 1
env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN \
npm install --no-save --package-lock=false --ignore-scripts "${archives[0]}"
- name: Install OpenShell CLI
run: env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN bash scripts/install-openshell.sh
- name: Run external gateway health live test
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
npx tsx tools/e2e/live-vitest-invocation.mts run \
--test-path test/e2e/live/external-gateway-health.test.ts
- name: Upload external gateway health artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-external-gateway-health
path: e2e-artifacts/live/external-gateway-health/
mcp-bridge:
name: MCP bridge (${{ matrix.agent }}, ${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge') }}
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }}
permissions:
contents: read
# Keep each destructive agent lifecycle on a fresh runner. This bounds the
# blast radius of runner loss and gives every adapter independent evidence.
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
agent: [openclaw, hermes, deepagents]
runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['mcp-bridge'] }}
include:
- agent: openclaw
agent_runtime: openclaw
coverage_variant: openclaw
- agent: hermes
agent_runtime: hermes
coverage_variant: hermes
- agent: deepagents
agent_runtime: langchain-deepagents-code
coverage_variant: deepagents
env:
E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }}
E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }}
NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker,podman
E2E_TARGET_ID: "mcp-bridge"
E2E_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu managed runtime host; local compatible inference and MCP endpoint"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_E2E_SHARD: ${{ matrix.agent }}
NEMOCLAW_MCP_BRIDGE_AGENT: ${{ matrix.agent }}
NEMOCLAW_OPENSHELL_CHANNEL: stable
NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: ${{ matrix.runtime_provider == 'docker' && '1' || '0' }}
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }}
OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ghcr.io/nvidia/openshell/supervisor@sha256:722f44669722961b7f432b0b81de25b91a58f34a61d6403bef967acaf2b3af01
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && matrix.agent == 'hermes' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
DISPATCH_SHA: ${{ github.sha }}
ENV: /dev/null
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
LC_ALL: C
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }}
RUNNER_OS_KIND: ${{ runner.os }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: &trusted-hermes-e2e-swap |
set -euo pipefail
readonly swap_dir="/mnt/nemoclaw-hermes-e2e-swap"
readonly required_swap_bytes=34359738368
readonly swap_file_bytes=34359742464
readonly reserve_bytes=17179869184
readonly activation_observation_attempts=5
readonly activation_observation_delay_seconds=1
swap_file=""
swap_activation_succeeded=0
fail() {
printf 'Trusted Hermes E2E swap setup failed: %s\n' "$1" >&2
exit 1
}
if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" ]]; then
fail "workflow must run from NVIDIA/NemoClaw"
fi
if [[ "${EVENT_NAME}" != "push" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then
fail "workflow event must be push or workflow_dispatch"
fi
if [[ "${EVENT_NAME}" == "push" && "${REF}" != "refs/heads/main" ]]; then
fail "push workflow must run from NVIDIA/NemoClaw main"
fi
if [[ "${EVENT_NAME}" == "workflow_dispatch" && "${REF}" != refs/heads/* ]]; then
fail "manual workflow must run from an NVIDIA/NemoClaw branch"
fi
# PR E2E mode: maintainer-dispatched PR commit.
if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then
if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then
fail "checkout SHA must be lowercase 40-hex"
fi
if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then
fail "workflow source must match the trusted dispatch revision"
fi
else
# Direct-main mode: push or manual trigger on main.
if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then
fail "direct main runs must not request an alternate checkout or workflow revision"
fi
if [[ ! "${WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then
fail "direct main workflow source must match the run revision"
fi
fi
if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then
fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner"
fi
mnt_metadata="$(/usr/bin/stat -c "%F:%u:%g" -- /mnt)"
if [[ "${mnt_metadata}" != "directory:0:0" ]]; then
fail "/mnt must be a root-owned directory"
fi
read_active_swap_bytes() {
/usr/bin/sudo -n /usr/sbin/swapon --show=SIZE --bytes --noheadings |
/usr/bin/awk '{ total += $1 } END { printf "%.0f", total }'
}
active_swap_bytes="$(read_active_swap_bytes)"
active_swap_bytes="${active_swap_bytes:-0}"
if [[ ! "${active_swap_bytes}" =~ ^[0-9]+$ ]]; then
fail "unable to determine active swap capacity"
fi
if (( active_swap_bytes >= required_swap_bytes )); then
printf 'Hermes E2E swap is already sufficient: %s bytes active\n' "${active_swap_bytes}"
exit 0
fi
available_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")"
if [[ ! "${available_bytes}" =~ ^[0-9]+$ ]]; then
fail "unable to determine available disk capacity under /mnt"
fi
required_disk_bytes=$((swap_file_bytes + reserve_bytes))
if (( available_bytes < required_disk_bytes )); then
fail "insufficient disk capacity: ${available_bytes} bytes available, ${required_disk_bytes} required"
fi
if /usr/bin/sudo -n /usr/bin/test -e "${swap_dir}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_dir}"; then
fail "refusing unexpected pre-existing swap path"
fi
directory_created=0
cleanup_partial_swap() {
status="$?"
if (( status != 0 && directory_created == 1 )); then
if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then
fixed_swap_active=0
while IFS= read -r active_swap_name; do
if [[ -n "${swap_file}" && "${active_swap_name}" == "${swap_file}" ]]; then
fixed_swap_active=1
break
fi
done <<< "${active_swap_names}"
if (( fixed_swap_active == 1 || swap_activation_succeeded == 1 )); then
if /usr/bin/sudo -n /usr/sbin/swapoff "${swap_file}" 2>/dev/null; then
/usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true
/usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true
else
printf 'Preserving active Hermes E2E swap after setup failure: %s\n' "${swap_file}" >&2
fi
else
if [[ -n "${swap_file}" ]]; then
/usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true
fi
/usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true
fi
else
printf 'Preserving Hermes E2E swap because active swap could not be queried: %s\n' "${swap_file}" >&2
fi
fi
trap - EXIT
exit "${status}"
}
trap cleanup_partial_swap EXIT
/usr/bin/sudo -n /usr/bin/mkdir -m 0700 -- "${swap_dir}"
directory_created=1
directory_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%F:%u:%g:%a" -- "${swap_dir}")"
if [[ "${directory_metadata}" != "directory:0:0:700" ]]; then
fail "swap directory must be a root-owned mode-0700 directory"
fi
swap_file="$(/usr/bin/sudo -n /usr/bin/mktemp --tmpdir="${swap_dir}" nemoclaw-hermes.XXXXXXXX.swap)"
if ! /usr/bin/sudo -n /usr/bin/test -f "${swap_file}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_file}"; then
fail "swap file must be a regular non-symlink"
fi
file_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%u:%g:%a" -- "${swap_file}")"
if [[ "${file_metadata}" != "0:0:600" ]]; then
fail "swap file must be root-owned mode 0600"
fi
/usr/bin/sudo -n /usr/bin/fallocate -l "${swap_file_bytes}" "${swap_file}"
file_size_bytes="$(/usr/bin/sudo -n /usr/bin/stat -c "%s" -- "${swap_file}")"
if [[ ! "${file_size_bytes}" =~ ^[0-9]+$ || "${file_size_bytes}" -ne "${swap_file_bytes}" ]]; then
fail "swap file size does not match the fixed backing allocation"
fi
remaining_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")"
if [[ ! "${remaining_bytes}" =~ ^[0-9]+$ || "${remaining_bytes}" -lt "${reserve_bytes}" ]]; then
fail "swap allocation did not preserve the required disk reserve"
fi
/usr/bin/sudo -n /usr/sbin/mkswap --quiet "${swap_file}"
/usr/bin/sudo -n /usr/sbin/swapon "${swap_file}"
swap_activation_succeeded=1
observe_provisioned_swap() {
activation_observation_attempt=1
while (( activation_observation_attempt <= activation_observation_attempts )); do
provisioned_swap_active=0
if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then
while IFS= read -r active_swap_name; do
if [[ "${active_swap_name}" == "${swap_file}" ]]; then
provisioned_swap_active=1
break
fi
done <<< "${active_swap_names}"
fi
if observed_swap_bytes="$(read_active_swap_bytes 2>/dev/null)"; then
observed_swap_bytes="${observed_swap_bytes:-0}"
if [[ "${observed_swap_bytes}" =~ ^[0-9]+$ ]] &&
(( provisioned_swap_active == 1 && observed_swap_bytes >= required_swap_bytes )); then
active_swap_bytes="${observed_swap_bytes}"
return 0
fi
fi
if (( activation_observation_attempt < activation_observation_attempts )); then
/usr/bin/sleep "${activation_observation_delay_seconds}"
fi
activation_observation_attempt=$((activation_observation_attempt + 1))
done
return 1
}
if ! observe_provisioned_swap; then
fail "unable to verify the required active swap capacity after bounded observation"
fi
trap - EXIT
printf 'Hermes E2E swap ready: %s bytes active\n' "${active_swap_bytes}"
/usr/bin/sudo -n /usr/sbin/swapon --show
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Initialize runner comparison telemetry
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts initialize
- name: Prepare native Podman E2E runtime
uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de
with:
enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}
- name: Install and verify cloudflared prerequisite
# Update posture: maintainers review upstream cloudflared releases and
# update the version and reviewed SHA256 together in both explicit MCP
# lanes; mutable package repositories and unreviewed latest releases
# are intentionally rejected by the workflow-contract tests.
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"
- name: Generate MCP test TLS
run: bash test/e2e/setup-mcp-test-tls.sh
- name: Install OpenShell CLI
env:
NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1"
run: |
set -euo pipefail
bash scripts/install-openshell.sh
if [[ "$NEMOCLAW_MCP_BRIDGE_AGENT" == "deepagents" ]]; then
mkdir -p "$E2E_ARTIFACT_DIR/mcp-bridge-deepagents"
jq -n '{
schemaVersion: 1,
sourceRepository: "NVIDIA/OpenShell",
releaseTag: "v0.0.106",
sourceSha: "c4b500a7de64d0b66e3ee8098f58d14299092162",
artifacts: {
cli: {binarySha256: "98ecf95113fea999e94a928043e57b04cf58a45a1b66ae8bffc73d1bc8bb1d59"},
gateway: {binarySha256: "e6cde8a54568aa1926ff6584ffd6984314c68dad64d2722509618a74094c622c"},
standaloneSandbox: {binarySha256: "019301ec8618abbed8135e8d39dde7bea47e5e92813bbc17768550de34db59f8"}
}
}' > "$E2E_ARTIFACT_DIR/mcp-bridge-deepagents/openshell-exact-main-provenance.json"
fi
- name: Run MCP OpenShell provider live test
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts
- name: Finalize runner comparison telemetry
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts finalize
- id: mcp_artifact_secret_scan
name: Scan MCP artifacts for fixture credentials
if: always()
run: >-
npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }}
- name: Upload MCP server artifacts
if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-mcp-bridge-${{ matrix.agent }}-${{ matrix.runtime_provider }}
path: e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
openshell-credential-generation-window:
name: OpenShell credential generation window (${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-credential-generation-window') }}
runs-on: ubuntu-latest
permissions:
contents: read
# Keep the credential-generation lifecycle on a fresh runner so it can
# execute in parallel with, and fail independently from, the Deep Agents
# MCP lifecycle without sharing destructive sandbox state.
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['openshell-credential-generation-window'] }}
env:
E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }}
E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }}
NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker,podman
E2E_TARGET_ID: "openshell-credential-generation-window"
E2E_AGENT_RUNTIME: "openclaw"
E2E_OBSERVABLE_OUTCOME: "Credential expiry rotation detach and rebuild preserve the intended access window"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu managed runtime host; local compatible inference and MCP endpoint"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_OPENSHELL_CHANNEL: stable
NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1"
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }}
OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ghcr.io/nvidia/openshell/supervisor@sha256:722f44669722961b7f432b0b81de25b91a58f34a61d6403bef967acaf2b3af01
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Prepare native Podman E2E runtime
uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de
with:
enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}
- name: Install and verify cloudflared prerequisite
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"
- name: Generate MCP test TLS
run: bash test/e2e/setup-mcp-test-tls.sh
- name: Install OpenShell CLI
env:
NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1"
run: |
set -euo pipefail
bash scripts/install-openshell.sh
mkdir -p "$E2E_ARTIFACT_DIR/openshell-credential-generation-window"
jq -n '{
schemaVersion: 1,
sourceRepository: "NVIDIA/OpenShell",
releaseTag: "v0.0.106",
sourceSha: "c4b500a7de64d0b66e3ee8098f58d14299092162",
artifacts: {
cli: {binarySha256: "98ecf95113fea999e94a928043e57b04cf58a45a1b66ae8bffc73d1bc8bb1d59"},
gateway: {binarySha256: "e6cde8a54568aa1926ff6584ffd6984314c68dad64d2722509618a74094c622c"},
standaloneSandbox: {binarySha256: "019301ec8618abbed8135e8d39dde7bea47e5e92813bbc17768550de34db59f8"}
}
}' > "$E2E_ARTIFACT_DIR/openshell-credential-generation-window/openshell-exact-main-provenance.json"
- name: Run OpenShell credential generation-window live test
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx vitest run --project e2e-live \
test/e2e/live/openshell-credential-generation-window.test.ts \
-t '^openshell-credential-generation-window$' \
--no-file-parallelism \
--silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts
- id: credential_window_artifact_secret_scan
name: Scan credential-window artifacts for fixture credentials
if: always()
run: >-
npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }}
- name: Upload credential-window artifacts
if: ${{ always() && steps.credential_window_artifact_secret_scan.outcome == 'success' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-openshell-credential-generation-window-${{ matrix.runtime_provider }}
path: e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
openshell-dev-artifact:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge-dev') }}
runs-on: ubuntu-latest
permissions:
contents: read
timeout-minutes: 15
outputs:
artifact_name: ${{ steps.resolve_openshell_dev_artifact.outputs.artifact_name }}
source_commit: ${{ steps.resolve_openshell_dev_artifact.outputs.source_commit }}
manifest_sha256: ${{ steps.resolve_openshell_dev_artifact.outputs.manifest_sha256 }}
steps:
- name: Checkout trusted OpenShell dev tooling
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
path: .trusted-openshell-dev-artifact
persist-credentials: false
sparse-checkout: |
scripts/install-openshell.sh
tools/e2e/openshell-dev-artifact.mts
- name: Set up Node for OpenShell dev artifact resolution
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- id: resolve_openshell_dev_artifact
name: Resolve immutable OpenShell dev artifact
run: >-
node --experimental-strip-types --no-warnings "${{ github.workspace }}/.trusted-openshell-dev-artifact/tools/e2e/openshell-dev-artifact.mts" resolve "${{ runner.temp }}/openshell-dev-artifact"
- name: Upload OpenShell dev artifact resolution
if: ${{ always() }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: ${{ steps.resolve_openshell_dev_artifact.outputs.artifact_name || format('openshell-dev-infrastructure-failure-{0}-{1}', github.run_id, github.run_attempt) }}
path: ${{ runner.temp }}/openshell-dev-artifact/
mcp-bridge-dev:
name: MCP bridge dev (${{ matrix.agent }}, ${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix, openshell-dev-artifact]
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge-dev') }}
runs-on: ubuntu-latest
permissions:
contents: read
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
agent: [openclaw, hermes, deepagents]
runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['mcp-bridge-dev'] }}
include:
- agent: openclaw
agent_runtime: openclaw
coverage_variant: openclaw
- agent: hermes
agent_runtime: hermes
coverage_variant: hermes
- agent: deepagents
agent_runtime: langchain-deepagents-code
coverage_variant: deepagents
env:
E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }}
E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }}
NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker,podman
E2E_TARGET_ID: "mcp-bridge-dev"
E2E_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu managed runtime host; local compatible inference and MCP endpoint"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_E2E_SHARD: ${{ matrix.agent }}
NEMOCLAW_MCP_BRIDGE_AGENT: ${{ matrix.agent }}
NEMOCLAW_OPENSHELL_CHANNEL: dev
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }}
steps:
# setup-node probes package managers in the workspace.
# Run it before candidate checkout with automatic caching disabled.
- name: Set up Node.js for trusted OpenShell verification
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
package-manager-cache: false
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Checkout trusted OpenShell dev tooling
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
path: .trusted-openshell-dev-artifact
persist-credentials: false
sparse-checkout: |
.github/scripts/copy-openshell-dev-asset.sh
.github/scripts/docker-auth-cleanup.sh
scripts/install-openshell.sh
tools/e2e/openshell-dev-artifact.mts
- name: Restore immutable OpenShell dev artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ needs.openshell-dev-artifact.outputs.artifact_name }}
path: ${{ runner.temp }}/openshell-dev-artifact
digest-mismatch: error
- name: Verify immutable OpenShell dev artifact
env:
OPENSHELL_DEV_ARTIFACT_DIR: ${{ runner.temp }}/openshell-dev-artifact
OPENSHELL_DEV_EXPECTED_MANIFEST_SHA256: ${{ needs.openshell-dev-artifact.outputs.manifest_sha256 }}
OPENSHELL_DEV_EXPECTED_SOURCE_COMMIT: ${{ needs.openshell-dev-artifact.outputs.source_commit }}
run: >-
node --experimental-strip-types --no-warnings "${{ github.workspace }}/.trusted-openshell-dev-artifact/tools/e2e/openshell-dev-artifact.mts" verify "$OPENSHELL_DEV_ARTIFACT_DIR" "$OPENSHELL_DEV_EXPECTED_SOURCE_COMMIT" "$OPENSHELL_DEV_EXPECTED_MANIFEST_SHA256"
- name: Revoke Docker auth before OpenShell development tooling
shell: bash
run: >-
bash "${{ github.workspace }}/.trusted-openshell-dev-artifact/.github/scripts/docker-auth-cleanup.sh"
- name: Install immutable OpenShell dev artifact
env:
NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1"
NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1"
OPENSHELL_DEV_ASSET_DIR: ${{ runner.temp }}/openshell-dev-artifact/assets
run: |
set -euo pipefail
shim_dir="$(mktemp -d)"
trap 'rm -rf "$shim_dir"' EXIT
cat >"$shim_dir/gh" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
if [[ "$#" -ne 10 || "$1" != "release" || "$2" != "download" || "$3" != "dev" || "$4" != "--repo" || "$5" != "NVIDIA/OpenShell" || "$6" != "--pattern" || "$8" != "--dir" || "${10}" != "--clobber" ]]; then
printf 'Unsupported gh invocation for retained OpenShell assets.\n' >&2
exit 64
fi
asset="$7"
destination="$9"
case "$asset" in
openshell-x86_64-unknown-linux-musl.tar.gz | openshell-checksums-sha256.txt | openshell-gateway-x86_64-unknown-linux-gnu.tar.gz | openshell-gateway-checksums-sha256.txt | openshell-sandbox-x86_64-unknown-linux-musl.tar.gz | openshell-sandbox-checksums-sha256.txt) ;;
*)
printf 'Unsupported retained OpenShell asset: %s\n' "$asset" >&2
exit 64
;;
esac
bash "${{ github.workspace }}/.trusted-openshell-dev-artifact/.github/scripts/copy-openshell-dev-asset.sh" \
"$OPENSHELL_DEV_ASSET_DIR" "$asset" "$destination"
EOF
cat >"$shim_dir/curl" <<'EOF'
#!/usr/bin/env bash
printf 'Network fallback is disabled for retained OpenShell assets.\n' >&2
exit 1
EOF
chmod 700 "$shim_dir/gh" "$shim_dir/curl"
PATH="$shim_dir:$PATH" \
bash "${{ github.workspace }}/.trusted-openshell-dev-artifact/scripts/install-openshell.sh"
# Dependency preparation can read candidate project configuration. Keep
# it after the trusted installer has consumed the verified OpenShell
# assets and Docker credentials have been revoked.
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
# The restore action executes the candidate CLI for its final identity
# check. Candidate-controlled state starts with dependency preparation.
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Prepare native Podman E2E runtime
uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de
with:
enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}
- name: Install and verify cloudflared prerequisite
# Update posture: keep this dev compatibility lane on the same reviewed
# version/SHA256 pair as the stable lane; workflow-contract tests fail
# if the pins diverge or installation becomes mutable.
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"
- name: Generate MCP test TLS
run: bash test/e2e/setup-mcp-test-tls.sh
- id: mcp_runtime_compatibility
name: Classify OpenShell credential-boundary compatibility
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
npx tsx tools/e2e/mcp-bridge-runtime-compatibility.mts
- name: Run MCP OpenShell provider live test
if: ${{ steps.mcp_runtime_compatibility.outputs.mode == 'full-lifecycle' }}
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts
- id: mcp_artifact_secret_scan
name: Scan MCP artifacts for fixture credentials
if: always()
run: >-
npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }}
- name: Upload MCP server artifacts
if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-mcp-bridge-dev-${{ matrix.agent }}-${{ matrix.runtime_provider }}
path: e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# Manual PR qualification also requires the exact candidate activation contract.
managed-image-multiarch-startup:
name: Protected managed-image startup (${{ matrix.platform }})
needs: [base-image-publication, generate-matrix]
if: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-multiarch-startup') || contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-protected-runtime')) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 210
permissions:
contents: read
strategy:
fail-fast: false
matrix:
include:
- platform: linux/amd64
runner: ubuntu-24.04
shard: linux-amd64
environment_or_inference_endpoint: "AMD64 Ubuntu; exact managed image startup"
coverage_variant: linux-amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
shard: linux-arm64
environment_or_inference_endpoint: "Arm64 Ubuntu; exact managed image startup"
coverage_variant: linux-arm64
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker
E2E_TARGET_ID: "managed-image-multiarch-startup"
E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code"
E2E_OBSERVABLE_OUTCOME: "Exact managed images start directly on the native architecture"
RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-multiarch-activation-v1.json
NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_HEAD_SHA: ${{ inputs.checkout_sha || github.sha }}
NEMOCLAW_E2E_SHARD: ${{ matrix.shard }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE: ${{ github.workspace }}/.protected-managed-image-build-cache/${{ matrix.shard }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT: protected-managed-image-build-cache-${{ github.run_id }}-${{ inputs.checkout_sha || github.sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/contracts.json
NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/evidence.json
NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: ${{ matrix.platform }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
NEMOCLAW_PROTECTED_REGISTRY_NAME: nemoclaw-managed-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_RUN_LIVE_E2E: "1"
steps:
- name: Validate protected exact-head dispatch
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
PLATFORM: ${{ matrix.platform }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$REPOSITORY" == "NVIDIA/NemoClaw" && ( ( "$EVENT_NAME" == "push" && "$REF" == "refs/heads/main" ) || ( "$EVENT_NAME" == "workflow_dispatch" && "$REF" == refs/heads/* ) ) ]] || {
echo "::error::Protected managed-image startup must run from NVIDIA/NemoClaw" >&2
exit 1
}
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Protected managed-image startup requires candidate and base SHAs" >&2
exit 1
}
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || {
echo "::error::Protected managed-image startup requires the exact trusted workflow SHA" >&2
exit 1
}
case "${PLATFORM}:${RUNNER_ARCH_KIND}" in
linux/amd64:X64 | linux/arm64:ARM64) ;;
*)
echo "::error::Protected managed-image startup requires a native ${PLATFORM} runner" >&2
exit 1
;;
esac
- name: Checkout protected managed-image candidate source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Validate trusted Hermes resolver checkout path
shell: bash
run: |
set -euo pipefail
trusted_resolver_root="$GITHUB_WORKSPACE/.trusted-hermes-resolver"
[[ ! -e "$trusted_resolver_root" && ! -L "$trusted_resolver_root" ]] || {
echo "::error::Trusted Hermes resolver checkout path already exists" >&2
exit 1
}
- name: Checkout trusted Hermes resolver
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
path: .trusted-hermes-resolver
sparse-checkout: .github/actions/resolve-reviewed-hermes-platform
fetch-depth: 1
persist-credentials: false
- *dockerhub-auth
- name: Set up protected managed-image Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
with:
driver-opts: network=host
buildkitd-config-inline: |
[registry."localhost:5000"]
http = true
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Validate candidate activation contract
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
activation="ci/protected-managed-image-multiarch-activation-v1.json"
[[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || {
echo "::error::Protected managed-image checkout does not match the candidate SHA" >&2
exit 1
}
[[ -f "$activation" && ! -L "$activation" ]] || {
echo "::error::Protected managed-image activation contract is absent" >&2
exit 1
}
jq -e '
(keys | sort) == ["agents", "contractVersion", "jobId", "platforms"] and
.contractVersion == 1 and
.jobId == "managed-image-multiarch-startup" and
.agents == ["openclaw", "hermes", "langchain-deepagents-code"] and
.platforms == ["linux/amd64", "linux/arm64"]
' "$activation" >/dev/null || {
echo "::error::Protected managed-image activation contract is invalid" >&2
exit 1
}
install -d -m 0700 "$E2E_ARTIFACT_DIR"
- id: hermes-base
name: Resolve reviewed Hermes platform base image
uses: ./.trusted-hermes-resolver/.github/actions/resolve-reviewed-hermes-platform
with:
dockerfile-path: agents/hermes/Dockerfile
platform: ${{ matrix.platform }}
- name: Remove trusted Hermes resolver checkout
if: always()
shell: bash
run: |
set -euo pipefail
trusted_resolver_root="$GITHUB_WORKSPACE/.trusted-hermes-resolver"
if [[ -e "$trusted_resolver_root" || -L "$trusted_resolver_root" ]]; then
[[ -d "$trusted_resolver_root" && ! -L "$trusted_resolver_root" ]] || {
echo "::error::Trusted Hermes resolver checkout path changed type" >&2
exit 1
}
rm -rf -- "$trusted_resolver_root"
fi
[[ ! -e "$trusted_resolver_root" && ! -L "$trusted_resolver_root" ]] || {
echo "::error::Trusted Hermes resolver checkout remained after cleanup" >&2
exit 1
}
- id: bases
name: Resolve digest-pinned platform base images
env:
DCODE_BASE_CONTRACT: ${{ needs.base-image-publication.outputs.dcode_base_contract }}
PLATFORM: ${{ matrix.platform }}
shell: bash
run: |
set -euo pipefail
arch="${PLATFORM#linux/}"
work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-protected-bases.XXXXXX")"
trap 'rm -rf -- "$work_dir"' EXIT
resolve_base() {
local output_name="$1"
local alias="$2"
local repository="$3"
local alias_raw="$work_dir/${output_name}-alias.raw"
local exact_raw="$work_dir/${output_name}-exact.raw"
docker buildx imagetools inspect "$alias" --raw > "$alias_raw"
local digest
digest="$(
jq -er --arg arch "$arch" '
if (
.mediaType == "application/vnd.oci.image.index.v1+json" or
.mediaType == "application/vnd.docker.distribution.manifest.list.v2+json"
) then
[.manifests[] | select(.platform.os == "linux" and .platform.architecture == $arch)]
| if length == 1 then .[0].digest else error("not one exact platform descriptor") end
else
error("base alias is not a platform index")
end
' "$alias_raw"
)"
[[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || {
echo "::error::${output_name} base alias returned an invalid digest" >&2
exit 1
}
local reference="${repository}@${digest}"
docker buildx imagetools inspect "$reference" --raw > "$exact_raw"
[[ "sha256:$(sha256sum "$exact_raw" | awk '{print $1}')" == "$digest" ]] || {
echo "::error::${output_name} exact base bytes do not match the selected digest" >&2
exit 1
}
printf '%s=%s\n' "$output_name" "$reference" >> "$GITHUB_OUTPUT"
}
resolve_base openclaw \
ghcr.io/nvidia/nemoclaw/sandbox-base:latest \
ghcr.io/nvidia/nemoclaw/sandbox-base
dcode_reference="$(
jq -er --arg platform "$PLATFORM" \
'.platformReferences[$platform]' <<< "$DCODE_BASE_CONTRACT"
)"
[[ "$dcode_reference" =~ ^ghcr[.]io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base@sha256:[a-f0-9]{64}$ ]] || {
echo "::error::dcode base publication returned an invalid exact platform reference" >&2
exit 1
}
dcode_digest="${dcode_reference##*@}"
docker buildx imagetools inspect "$dcode_reference" --raw > "$work_dir/dcode-exact.raw"
[[ "sha256:$(sha256sum "$work_dir/dcode-exact.raw" | awk '{print $1}')" == "$dcode_digest" ]] || {
echo "::error::dcode exact base bytes do not match the published platform digest" >&2
exit 1
}
printf 'dcode=%s\n' "$dcode_reference" >> "$GITHUB_OUTPUT"
- name: Start isolated protected managed-image registry
shell: bash
run: |
set -euo pipefail
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
echo "::error::Protected registry name already exists" >&2
exit 1
fi
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then
echo "::error::Refusing to reuse an existing localhost:5000 registry" >&2
exit 1
fi
docker run --detach \
--name "$NEMOCLAW_PROTECTED_REGISTRY_NAME" \
--label "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" \
--label "io.nvidia.nemoclaw.e2e-platform=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM}" \
--publish 127.0.0.1:5000:5000 \
docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373
for _ in $(seq 1 30); do
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null; then
exit 0
fi
sleep 1
done
docker logs "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >&2
exit 1
- name: Build exact all-agent protected managed images
env:
BASE_DCODE: ${{ steps.bases.outputs.dcode }}
BASE_HERMES: ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@${{ steps.hermes-base.outputs.digest }}
BASE_OPENCLAW: ${{ steps.bases.outputs.openclaw }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
cache_args=()
if [[ "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" == "linux/amd64" ]]; then
cache_args=(--cache-to "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE")
fi
scripts/checks/build-protected-managed-images.sh \
--output "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \
--revision "$CHECKOUT_SHA" \
--cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \
--platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" \
--openclaw-base "$BASE_OPENCLAW" \
--hermes-base "$BASE_HERMES" \
--dcode-base "$BASE_DCODE" \
"${cache_args[@]}"
- name: Run every exact managed-image contract directly
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
direct_runs="${RUNNER_TEMP}/protected-managed-image-direct-runs.jsonl"
: > "$direct_runs"
while IFS= read -r entry; do
agent="$(jq -er '.agent' <<< "$entry")"
digest="$(jq -er '.digest' <<< "$entry")"
platform="$(jq -er '.platform' <<< "$entry")"
reference="$(jq -er '.reference' <<< "$entry")"
[[ "$platform" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" ]] || {
echo "::error::Protected managed-image contract changed platform" >&2
exit 1
}
npx --no-install tsx scripts/checks/run-managed-image-direct-e2e.ts \
--agent "$agent" \
--image "$reference" \
--platform "$platform"
jq -nc \
--arg agent "$agent" \
--arg digest "$digest" \
--arg platform "$platform" \
--arg reference "$reference" \
'{agent: $agent, digest: $digest, platform: $platform, reference: $reference}' \
>> "$direct_runs"
done < <(jq -c '.[]' "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT")
contract_sha="sha256:$(sha256sum "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" | awk '{print $1}')"
jq -n \
--arg baseSha "$BASE_SHA" \
--arg cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \
--arg contractSha256 "$contract_sha" \
--arg headSha "$CHECKOUT_SHA" \
--arg platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" \
--arg workflowSha "$WORKFLOW_SHA" \
--argjson runAttempt "$GITHUB_RUN_ATTEMPT" \
--argjson runId "$GITHUB_RUN_ID" \
--slurpfile contracts "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \
--slurpfile directRuns "$direct_runs" \
'{
kind: "nemoclaw-protected-managed-image-multiarch-v1",
headSha: $headSha,
baseSha: $baseSha,
workflowSha: $workflowSha,
platform: $platform,
cohort: $cohort,
contractSha256: $contractSha256,
contracts: $contracts[0],
directRuns: $directRuns,
run: {id: $runId, attempt: $runAttempt}
}' > "${NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE}.tmp"
mv \
"${NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE}.tmp" \
"$NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE"
- name: Remove isolated protected managed-image registry
if: always()
shell: bash
run: |
set -euo pipefail
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
owner="$(
docker container inspect \
--format '{{index .Config.Labels "io.nvidia.nemoclaw.e2e-owner"}}' \
"$NEMOCLAW_PROTECTED_REGISTRY_NAME"
)"
[[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]] || {
echo "::error::Refusing to remove a registry not owned by this protected shard" >&2
exit 1
}
docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null
fi
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
echo "::error::Protected managed-image registry container remained after cleanup" >&2
exit 1
fi
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then
echo "::error::Protected managed-image registry listener remained after cleanup" >&2
exit 1
fi
- name: Validate protected managed-image evidence
shell: bash
run: >-
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/managed-image-multiarch-startup.test.ts
- name: Publish exact amd64 protected runtime build cache
if: ${{ matrix.platform == 'linux/amd64' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT }}
path: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE }}/
if-no-files-found: error
retention-days: 1
compression-level: 0
overwrite: true
- name: Upload protected managed-image evidence
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-managed-image-multiarch-startup-${{ matrix.shard }}
path: e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# The trusted plan compiler runs on a standard runner before the protected
# job can be assigned. Candidate workflow code is never evaluated; trusted
# main compiles the exact candidate's declarative YAML into bounded outputs.
llama-cpp-dgx-spark-plan:
name: Compile protected llama.cpp DGX Spark plan
needs: generate-matrix
if: ${{ inputs.allow_dgx_spark_runner_queue && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'llama-cpp-dgx-spark-qualification') }}
runs-on: ubuntu-24.04
timeout-minutes: 15
permissions:
contents: read
outputs:
agent_qualification_execution: ${{ steps.plan.outputs.agent_qualification_execution }}
environment: ${{ steps.plan.outputs.environment }}
execution: ${{ steps.plan.outputs.execution }}
model_host_path: ${{ steps.plan.outputs.model_host_path }}
plan: ${{ steps.plan.outputs.plan }}
plan_sha256: ${{ steps.plan.outputs.plan_sha256 }}
qualification: ${{ steps.plan.outputs.qualification }}
runner: ${{ steps.plan.outputs.runner }}
steps:
- name: Validate trusted llama.cpp plan dispatch
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$REPOSITORY" == "NVIDIA/NemoClaw" && "$REF" == "refs/heads/main" && ( "$EVENT_NAME" == "push" || "$EVENT_NAME" == "workflow_dispatch" ) ]] || {
echo "::error::Protected llama.cpp planning must run from trusted NVIDIA/NemoClaw main" >&2
exit 1
}
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Protected llama.cpp planning requires candidate and base SHAs" >&2
exit 1
}
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || {
echo "::error::Protected llama.cpp planning requires the exact trusted workflow SHA" >&2
exit 1
}
- name: Checkout trusted llama.cpp plan compiler
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
fetch-depth: 0
persist-credentials: false
- name: Checkout exact llama.cpp candidate configuration
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
path: .candidate-llama-cpp
fetch-depth: 0
persist-credentials: false
- name: Set up Node for trusted llama.cpp plan compilation
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: npm
- name: Install trusted llama.cpp plan dependencies
run: npm ci --ignore-scripts
- id: plan
name: Compile exact candidate llama.cpp qualification plan
env:
CANDIDATE_ROOT: ${{ github.workspace }}/.candidate-llama-cpp
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
[[ "$(git -C "$CANDIDATE_ROOT" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || {
echo "::error::Protected llama.cpp candidate checkout does not match the candidate SHA" >&2
exit 1
}
node --experimental-strip-types --no-warnings \
scripts/checks/export-llama-cpp-dgx-spark-qualification-plan.mts \
--source-root "$CANDIDATE_ROOT"
# The opt-in flag prevents assignment to an unavailable DGX Spark runner.
# Qualification runs only when the trusted plan reports execution=enabled.
llama-cpp-dgx-spark-qualification:
name: Protected llama.cpp on NVIDIA DGX Spark
needs: [generate-matrix, llama-cpp-dgx-spark-plan]
if: ${{ inputs.allow_dgx_spark_runner_queue && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'llama-cpp-dgx-spark-qualification') && needs.llama-cpp-dgx-spark-plan.outputs.execution == 'enabled' }}
runs-on: ${{ needs.llama-cpp-dgx-spark-plan.outputs.runner }}
environment:
name: ${{ needs.llama-cpp-dgx-spark-plan.outputs.environment }}
timeout-minutes: 300
permissions:
contents: read
env:
E2E_DEFAULT_ENABLED: "0"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-dgx-spark-qualification
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker
E2E_TARGET_ID: "llama-cpp-dgx-spark-qualification"
E2E_AGENT_RUNTIME: "unresolved"
E2E_OBSERVABLE_OUTCOME: "Exact NemoClaw-built llama.cpp image produces protected DGX Spark evidence"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA DGX Spark GB10; local llama.cpp inference"
E2E_UNRESOLVED_REASON: "The protected plan can enable or skip its OpenClaw subqualification"
NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
NEMOCLAW_E2E_SHARD: linux-arm64-gpu-dgx-spark-gb10
NEMOCLAW_LLAMA_CPP_QUALIFICATION_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
NEMOCLAW_LLAMA_CPP_QUALIFICATION_HEAD_SHA: ${{ inputs.checkout_sha || github.sha }}
NEMOCLAW_LLAMA_CPP_QUALIFICATION_CANDIDATE_ROOT: ${{ github.workspace }}/.candidate-llama-cpp
NEMOCLAW_LLAMA_CPP_QUALIFICATION_EVIDENCE: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-dgx-spark-qualification/evidence.json
NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN: ${{ github.workspace }}/.llama-cpp-qualification/plan.json
NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN_SHA256: ${{ needs.llama-cpp-dgx-spark-plan.outputs.plan_sha256 }}
NEMOCLAW_LLAMA_CPP_QUALIFICATION_REGISTRY: nemoclaw-llama-cpp-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_LLAMA_CPP_QUALIFICATION_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
NEMOCLAW_RUN_LIVE_E2E: "1"
RELEASE_E2E_ACTIVATION_PATH: ci/llama-cpp-dgx-spark-qualification-v1.yaml
steps:
- name: Validate protected llama.cpp exact-head dispatch
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$REPOSITORY" == "NVIDIA/NemoClaw" && "$REF" == "refs/heads/main" && ( "$EVENT_NAME" == "push" || "$EVENT_NAME" == "workflow_dispatch" ) ]] || {
echo "::error::Protected llama.cpp qualification must run from trusted NVIDIA/NemoClaw main" >&2
exit 1
}
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Protected llama.cpp qualification requires candidate and base SHAs" >&2
exit 1
}
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || {
echo "::error::Protected llama.cpp qualification requires the exact trusted workflow SHA" >&2
exit 1
}
[[ "$RUNNER_ARCH_KIND" == "ARM64" ]] || {
echo "::error::Protected llama.cpp qualification requires native Linux ARM64" >&2
exit 1
}
- name: Checkout trusted llama.cpp qualification
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
fetch-depth: 0
persist-credentials: false
- name: Checkout exact llama.cpp qualification candidate
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
path: .candidate-llama-cpp
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Set up protected llama.cpp Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
with:
driver: docker
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Install OpenShell CLI for declarative OpenClaw qualification
if: ${{ needs.llama-cpp-dgx-spark-plan.outputs.agent_qualification_execution == 'enabled' }}
run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh
- name: Materialize trusted llama.cpp qualification plan
env:
CANDIDATE_ROOT: ${{ github.workspace }}/.candidate-llama-cpp
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
PLAN: ${{ needs.llama-cpp-dgx-spark-plan.outputs.plan }}
PLAN_SHA256: ${{ needs.llama-cpp-dgx-spark-plan.outputs.plan_sha256 }}
shell: bash
run: |
set -euo pipefail
[[ "$(git -C "$CANDIDATE_ROOT" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || {
echo "::error::Protected llama.cpp candidate checkout does not match the candidate SHA" >&2
exit 1
}
install -d -m 0700 "$E2E_ARTIFACT_DIR"
install -d -m 0700 "$(dirname "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN")"
umask 077
printf '%s' "$PLAN" > "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN"
[[ "sha256:$(sha256sum "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN" | awk '{print $1}')" == "$PLAN_SHA256" ]] || {
echo "::error::Protected llama.cpp qualification plan digest changed" >&2
exit 1
}
- id: qualify
name: Build and qualify exact llama.cpp candidate
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CANDIDATE_ROOT: ${{ github.workspace }}/.candidate-llama-cpp
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
MODEL_HOST_PATH: ${{ needs.llama-cpp-dgx-spark-plan.outputs.model_host_path }}
WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$(git rev-parse --verify HEAD)" == "$WORKFLOW_SHA" ]] || {
echo "::error::Protected llama.cpp qualification must execute trusted workflow code" >&2
exit 1
}
npx --no-install tsx scripts/checks/run-llama-cpp-dgx-spark-qualification.mts \
--base-sha "$BASE_SHA" \
--candidate-root "$CANDIDATE_ROOT" \
--head-sha "$CHECKOUT_SHA" \
--model-host-path "$MODEL_HOST_PATH" \
--output "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_EVIDENCE" \
--plan "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN" \
--plan-sha256 "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN_SHA256" \
--registry-name "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_REGISTRY" \
--run-attempt "$GITHUB_RUN_ATTEMPT" \
--run-id "$GITHUB_RUN_ID" \
--workflow-sha "$WORKFLOW_SHA"
- name: Remove protected llama.cpp qualification resources
if: always()
shell: bash
run: >-
npx --no-install tsx scripts/checks/run-llama-cpp-dgx-spark-qualification.mts --cleanup-only --registry-name "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_REGISTRY" --run-attempt "$GITHUB_RUN_ATTEMPT" --run-id "$GITHUB_RUN_ID"
- name: Validate protected llama.cpp evidence
shell: bash
run: >-
npx --no-install tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/llama-cpp-dgx-spark-qualification.test.ts
- name: Upload protected llama.cpp evidence
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-llama-cpp-dgx-spark-qualification
path: e2e-artifacts/live/llama-cpp-dgx-spark-qualification/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# This lane runs after managed-image startup succeeds. Manual PR qualification
# requires the exact candidate activation contract. Its
# hosted amd64 prerequisite exports the exact candidate build cache.
# This GPU lane imports the exact candidate build cache and its complete
# locked npm cache seed, then rebuilds with BuildKit networking disabled.
# invalidState: Cold managed-image builds on the protected GPU runner can
# stall in npm and terminate with "Exit handler never called!" before any
# GPU, Ollama, NIM, or vLLM assertion executes.
# sourceBoundary: The hosted amd64 lane builds the exact candidate. It adds
# every reachable Linux x64 glibc registry archive from the candidate
# lockfile to the same-run cache artifact. The GPU lane verifies the manifest,
# lock digest, archive set, and
# each SHA-512 integrity before the candidate build can consume the seed.
# BuildKit records can accelerate the rebuild, while the seed makes a missing
# npm layer reproducible without weakening the fail-closed network boundary.
# whyNotBuildCacheOnly: Protected runs 31156103003, 31158136234, and
# 31159822518 proved that network-key drift and incomplete local cache exports
# can miss the npm layer before runtime qualification.
# regressionTest: The materializer and build-script behavior tests reject an
# incomplete, substituted, unexpected, or symlinked archive handoff and pin
# exact --cache-from/--network none arguments. Workflow boundary tests pin the
# same-run cache artifact.
# lifecycleDecision: PR #8261 owns removal after three consecutive protected
# GPU cold builds of all three images reach GPU, Ollama, NIM, and vLLM
# assertions without the hosted cache.
managed-image-protected-runtime:
name: Protected managed-image GPU and local inference
needs: [base-image-publication, generate-matrix, managed-image-multiarch-startup]
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && needs['base-image-publication'].result == 'success' && needs['generate-matrix'].result == 'success' && needs['managed-image-multiarch-startup'].result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-protected-runtime') }}
runs-on: linux-amd64-gpu-rtxpro6000-latest-1
timeout-minutes: 300
permissions:
contents: read
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker
E2E_TARGET_ID: "managed-image-protected-runtime"
E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code"
E2E_OBSERVABLE_OUTCOME: "Protected GPU runtime supports Ollama vLLM NIM rollback and cleanup"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local and hosted inference services"
RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
NEMOCLAW_E2E_SHARD: linux-amd64-gpu
NEMOCLAW_E2E_TESTED_ROOT: ${{ github.workspace }}/.candidate-runtime
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE: ${{ github.workspace }}/.protected-managed-image-build-cache/linux-amd64
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT: protected-managed-image-build-cache-${{ github.run_id }}-${{ inputs.checkout_sha || github.sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime/contracts.json
NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: linux/amd64
NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
NEMOCLAW_PROTECTED_REGISTRY_NAME: nemoclaw-managed-runtime-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_RUN_LIVE_E2E: "1"
OPENSHELL_GATEWAY: nemoclaw
steps:
- name: Validate protected runtime exact-head dispatch
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$REPOSITORY" == "NVIDIA/NemoClaw" && ( ( "$EVENT_NAME" == "push" && "$REF" == "refs/heads/main" ) || ( "$EVENT_NAME" == "workflow_dispatch" && "$REF" == refs/heads/* ) ) ]] || {
echo "::error::Protected managed-image runtime must run from NVIDIA/NemoClaw" >&2
exit 1
}
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Protected managed-image runtime requires candidate and base SHAs" >&2
exit 1
}
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || {
echo "::error::Protected managed-image runtime requires the exact trusted workflow SHA" >&2
exit 1
}
[[ "$RUNNER_ARCH_KIND" == "X64" ]] || {
echo "::error::Protected managed-image runtime requires a native linux/amd64 GPU runner" >&2
exit 1
}
- name: Checkout trusted protected runtime qualification
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
fetch-depth: 0
persist-credentials: false
- name: Checkout exact protected runtime candidate source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
path: .candidate-runtime
fetch-depth: 0
persist-credentials: false
- name: Download exact protected runtime build cache
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT }}
path: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE }}
- *dockerhub-auth
- name: Set up protected runtime Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
with:
driver-opts: network=host
buildkitd-config-inline: |
[registry."localhost:5000"]
http = true
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
- name: Validate protected runtime activation contract
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
candidate_root=".candidate-runtime"
activation="$candidate_root/ci/protected-managed-image-runtime-activation-v1.json"
[[ "$(git -C "$candidate_root" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || {
echo "::error::Protected managed-image runtime checkout does not match the candidate SHA" >&2
exit 1
}
[[ -f "$activation" && ! -L "$activation" ]] || {
echo "::error::Protected managed-image runtime activation contract is absent" >&2
exit 1
}
jq -e '
(keys | sort) == ["agents", "contractVersion", "jobId", "platform", "providers"] and
.contractVersion == 1 and
.jobId == "managed-image-protected-runtime" and
.agents == ["openclaw", "hermes", "langchain-deepagents-code"] and
.platform == "linux/amd64" and
.providers == ["ollama", "nim", "vllm"]
' "$activation" >/dev/null || {
echo "::error::Protected managed-image runtime activation contract is invalid" >&2
exit 1
}
install -d -m 0700 "$E2E_ARTIFACT_DIR"
- id: runtime-hermes-base
name: Resolve reviewed Hermes runtime base image
uses: ./.github/actions/resolve-reviewed-hermes-platform
with:
dockerfile-path: .candidate-runtime/agents/hermes/Dockerfile
platform: linux/amd64
- id: runtime-bases
name: Resolve digest-pinned amd64 runtime base images
env:
DCODE_BASE_REF: ${{ needs.base-image-publication.outputs.dcode_base_ref }}
shell: bash
run: |
set -euo pipefail
work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-runtime-bases.XXXXXX")"
trap 'rm -rf -- "$work_dir"' EXIT
resolve_base() {
local output_name="$1"
local alias="$2"
local repository="$3"
local alias_raw="$work_dir/${output_name}-alias.raw"
local exact_raw="$work_dir/${output_name}-exact.raw"
docker buildx imagetools inspect "$alias" --raw > "$alias_raw"
local digest
digest="$(
jq -er '
if (
.mediaType == "application/vnd.oci.image.index.v1+json" or
.mediaType == "application/vnd.docker.distribution.manifest.list.v2+json"
) then
[.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64")]
| if length == 1 then .[0].digest else error("not one exact amd64 descriptor") end
else
error("base alias is not a platform index")
end
' "$alias_raw"
)"
[[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || {
echo "::error::${output_name} base alias returned an invalid digest" >&2
exit 1
}
local reference="${repository}@${digest}"
docker buildx imagetools inspect "$reference" --raw > "$exact_raw"
[[ "sha256:$(sha256sum "$exact_raw" | awk '{print $1}')" == "$digest" ]] || {
echo "::error::${output_name} exact base bytes do not match the selected digest" >&2
exit 1
}
printf '%s=%s\n' "$output_name" "$reference" >> "$GITHUB_OUTPUT"
}
resolve_base openclaw \
ghcr.io/nvidia/nemoclaw/sandbox-base:latest \
ghcr.io/nvidia/nemoclaw/sandbox-base
[[ "$DCODE_BASE_REF" =~ ^ghcr[.]io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base@sha256:[a-f0-9]{64}$ ]] || {
echo "::error::dcode base publication returned an invalid exact reference" >&2
exit 1
}
dcode_digest="${DCODE_BASE_REF##*@}"
docker buildx imagetools inspect "$DCODE_BASE_REF" --raw > "$work_dir/dcode-exact.raw"
[[ "sha256:$(sha256sum "$work_dir/dcode-exact.raw" | awk '{print $1}')" == "$dcode_digest" ]] || {
echo "::error::dcode exact base bytes do not match the published digest" >&2
exit 1
}
printf 'dcode=%s\n' "$DCODE_BASE_REF" >> "$GITHUB_OUTPUT"
- name: Start isolated protected runtime registry
shell: bash
run: |
set -euo pipefail
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
echo "::error::Protected runtime registry name already exists" >&2
exit 1
fi
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then
echo "::error::Refusing to reuse an existing localhost:5000 registry" >&2
exit 1
fi
docker run --detach \
--name "$NEMOCLAW_PROTECTED_REGISTRY_NAME" \
--label "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" \
--label "io.nvidia.nemoclaw.e2e-platform=linux/amd64" \
--publish 127.0.0.1:5000:5000 \
docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373
for _ in $(seq 1 30); do
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null; then
exit 0
fi
sleep 1
done
docker logs "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >&2
exit 1
- name: Build exact all-agent protected runtime images
env:
BASE_DCODE: ${{ steps.runtime-bases.outputs.dcode }}
BASE_HERMES: ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@${{ steps.runtime-hermes-base.outputs.digest }}
BASE_OPENCLAW: ${{ steps.runtime-bases.outputs.openclaw }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
scripts/checks/build-protected-managed-images.sh \
--output "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \
--revision "$CHECKOUT_SHA" \
--cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \
--platform linux/amd64 \
--source-root "$GITHUB_WORKSPACE/.candidate-runtime" \
--cache-from "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE" \
--openclaw-base "$BASE_OPENCLAW" \
--hermes-base "$BASE_HERMES" \
--dcode-base "$BASE_DCODE"
- name: Install OpenShell CLI
shell: bash
run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh
- name: Run all-agent GPU, local inference, rollback, and cleanup qualification
env:
NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }}
shell: bash
run: |
set -euo pipefail
[[ "$(git rev-parse --verify HEAD)" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA" ]] || {
echo "::error::Protected NIM qualification must execute trusted workflow code" >&2
exit 1
}
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
export OPENSHELL_BIN="$(command -v openshell)"
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/managed-image-protected-runtime.test.ts
- name: Remove isolated protected runtime registry
if: always()
shell: bash
run: |
set -euo pipefail
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
owner="$(
docker container inspect \
--format '{{index .Config.Labels "io.nvidia.nemoclaw.e2e-owner"}}' \
"$NEMOCLAW_PROTECTED_REGISTRY_NAME"
)"
[[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]] || {
echo "::error::Refusing to remove a runtime registry not owned by this protected job" >&2
exit 1
}
docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null
fi
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
echo "::error::Protected managed-image runtime registry remained after cleanup" >&2
exit 1
fi
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then
echo "::error::Protected managed-image runtime registry listener remained after cleanup" >&2
exit 1
fi
- name: Upload protected managed-image runtime artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-managed-image-protected-runtime
path: e2e-artifacts/live/managed-image-protected-runtime/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
hermes-e2e:
name: Hermes E2E (${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }}
timeout-minutes: 85
strategy:
fail-fast: false
matrix:
runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['hermes-e2e'] }}
env:
E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }}
E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }}
NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker,podman
E2E_TARGET_ID: "hermes-e2e"
E2E_AGENT_RUNTIME: "hermes"
E2E_OBSERVABLE_OUTCOME: "Install onboarding health inference lifecycle dashboard and security succeed"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; mock or NVIDIA hosted inference"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-e2e/${{ matrix.runtime_provider }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }}
NEMOCLAW_E2E_INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }}
NEMOCLAW_E2E_HERMES_DASHBOARD: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_AGENT: hermes
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_RECREATE_SANDBOX: "1"
NEMOCLAW_SANDBOX_NAME: e2e-hermes
NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60"
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (github.event_name == 'push' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,') || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,')))) }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
DISPATCH_SHA: ${{ github.sha }}
ENV: /dev/null
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
LC_ALL: C
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }}
RUNNER_OS_KIND: ${{ runner.os }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: *trusted-hermes-e2e-swap
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Initialize runner comparison telemetry
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts initialize
- name: Prepare native Podman E2E runtime
uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de
with:
enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}
- name: Run Hermes live Vitest test
env:
NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-e2e.test.ts
- name: Finalize runner comparison telemetry
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts finalize
- name: Upload Hermes live Vitest artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-hermes-e2e-${{ matrix.runtime_provider }}
path: e2e-artifacts/live/hermes-e2e/${{ matrix.runtime_provider }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
hermes-gpu-startup:
name: Hermes GPU startup (${{ matrix.scenario }}, ${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'hermes-gpu-startup') }}
runs-on: linux-amd64-gpu-rtxpro6000-latest-1
timeout-minutes: 90
strategy:
fail-fast: false
max-parallel: 2
matrix:
scenario: [native, fallback, compatibility-only]
runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['hermes-gpu-startup'] }}
exclude:
- scenario: fallback
runtime_provider: podman
- scenario: compatibility-only
runtime_provider: podman
env:
E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }}
E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }}
NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker,podman
E2E_TARGET_ID: "hermes-gpu-startup"
E2E_AGENT_RUNTIME: "hermes"
E2E_OBSERVABLE_OUTCOME: "Hermes GPU startup reaches the stable Ready route"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local GPU inference"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/${{ matrix.runtime_provider }}
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
NEMOCLAW_E2E_SHARD: ${{ matrix.scenario }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }}
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_AGENT: hermes
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_RECREATE_SANDBOX: "1"
NEMOCLAW_SANDBOX_GPU: "1"
NEMOCLAW_SANDBOX_NAME: ${{ matrix.scenario }}
NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Checkout trusted Hermes GPU runtime fixture
if: ${{ matrix.scenario == 'fallback' }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: NVIDIA/NemoClaw
ref: ${{ github.workflow_sha }}
path: .trusted-hermes-gpu-fixture-${{ github.run_id }}-${{ github.run_attempt }}
sparse-checkout: tools/e2e/hermes-gpu-docker-runtime-fixture.sh
sparse-checkout-cone-mode: false
persist-credentials: false
- name: Install trusted Hermes GPU runtime fixture
if: ${{ matrix.scenario == 'fallback' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
TRUSTED_DISPATCH_SHA: ${{ github.sha }}
TRUSTED_FIXTURE_SHA256: e273c4baa7fe89546d64517cf56eafec30aeda7b355971263605ab1327fade02
TRUSTED_WORKFLOW_SHA: ${{ github.workflow_sha }}
run: |
set -euo pipefail
trusted_checkout="$GITHUB_WORKSPACE/.trusted-hermes-gpu-fixture-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
trusted_source="$trusted_checkout/tools/e2e/hermes-gpu-docker-runtime-fixture.sh"
trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}"
trusted_state_root=/var/lib/nemoclaw-e2e
run_trusted_fixture() {
/usr/bin/sudo -n /usr/bin/env -i \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
/bin/bash "$trusted_fixture" "$@"
}
[[ "$TRUSTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] \
|| { echo "Trusted workflow SHA must be an immutable commit" >&2; exit 1; }
[[ "$TRUSTED_DISPATCH_SHA" = "$TRUSTED_WORKFLOW_SHA" ]] \
|| { echo "Trusted fixture must match the dispatched main workflow" >&2; exit 1; }
[[ "$TRUSTED_FIXTURE_SHA256" =~ ^[a-f0-9]{64}$ ]] \
|| { echo "Trusted fixture SHA-256 must be pinned" >&2; exit 1; }
[ "$(/usr/bin/git -C "$trusted_checkout" rev-parse HEAD)" = "$TRUSTED_WORKFLOW_SHA" ] \
|| { echo "Trusted fixture checkout does not match the workflow SHA" >&2; exit 1; }
[ -f "$trusted_source" ] && [ ! -L "$trusted_source" ] \
|| { echo "Trusted Docker fixture must be a regular non-symlink file" >&2; exit 1; }
/usr/bin/sudo /usr/bin/install -d -o root -g root -m 0755 /usr/local/libexec/nemoclaw
/usr/bin/sudo /usr/bin/install -o root -g root -m 0500 \
"$trusted_source" "$trusted_fixture"
[ "$(/usr/bin/sudo /usr/bin/stat -c '%a %u %g' "$trusted_fixture")" = "500 0 0" ] \
|| { echo "Trusted Docker fixture ownership or mode is invalid" >&2; exit 1; }
printf '%s %s\n' "$TRUSTED_FIXTURE_SHA256" "$trusted_fixture" \
| /usr/bin/sudo /usr/bin/sha256sum -c -
/usr/bin/sudo /usr/bin/cmp -s "$trusted_source" "$trusted_fixture" \
|| { echo "Installed Docker fixture does not match trusted workflow code" >&2; exit 1; }
/usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$trusted_state_root"
# Recover any root-owned snapshot left by a hard-cancelled earlier run before
# PR-controlled build or test code executes on this persistent GPU runner.
if ! /usr/bin/sudo /usr/bin/find "$trusted_state_root" -mindepth 1 -maxdepth 1 \
-type d -name 'hermes-gpu-fallback-docker-runtime.*' -print0 \
| while IFS= read -r -d '' stale_state_dir; do
if ! run_trusted_fixture restore \
"$stale_state_dir" /etc/docker/daemon.json >/dev/null; then
exit 1
fi
done; then
echo "Could not recover stale Docker fallback state" >&2
exit 1
fi
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Reassert trusted Node runtime
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
NODE_OPTIONS: ""
with:
node-version: "22"
- name: Prepare native Podman E2E runtime
uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de
with:
enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}
- name: Run Hermes GPU startup live Vitest test
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
run: |
set -euo pipefail
if [ "$E2E_HERMES_GPU_STARTUP_SCENARIO" = fallback ]; then
umask 077
daemon_json=/etc/docker/daemon.json
trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}"
trusted_state_root=/var/lib/nemoclaw-e2e
run_trusted_fixture() {
/usr/bin/sudo -n /usr/bin/env -i \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
/bin/bash "$trusted_fixture" "$@"
}
mkdir -p "$E2E_ARTIFACT_DIR"
[ "$(/usr/bin/sudo /usr/bin/stat -c '%a %u %g' "$trusted_fixture")" = "500 0 0" ] \
|| { echo "Trusted Docker fixture ownership or mode changed" >&2; exit 1; }
/usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$trusted_state_root"
state_dir="$(/usr/bin/sudo /usr/bin/mktemp -d \
"$trusted_state_root/hermes-gpu-fallback-docker-runtime.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.fallback.XXXXXX")"
/usr/bin/sudo /usr/bin/chown root:root "$state_dir"
/usr/bin/sudo /usr/bin/chmod 0700 "$state_dir"
# SOURCE_OF_TRUTH_REVIEW (Hermes GPU fallback fixture; #6110):
# invalidState: a cancelled fallback test leaves the runner-global Docker default
# runtime changed, contaminating later OpenShell jobs on this self-hosted runner.
# sourceBoundary: daemon mutation uses only immutable root-owned workflow_sha
# code; the PR test cannot replace the helper or its root-owned snapshot state.
# whyNotSourceFix: the compatibility retry must prove its explicit GPU envelope
# without inheriting an NVIDIA daemon default from this managed runner.
# regressionTest: hermes-gpu-startup-workflow-boundary requires this same-step trap.
# removalCondition: remove the daemon fixture when the GPU runner supplies an
# isolated runc-default Docker daemon for this compatibility qualification.
restore_docker_default_runtime() {
local command_status=$?
local restore_status=0
local restored_runtime=""
trap - EXIT INT TERM
set +e
restored_runtime="$(run_trusted_fixture restore "$state_dir" "$daemon_json")"
restore_status=$?
if [ "$restore_status" -eq 0 ] && [ -n "$restored_runtime" ]; then
printf '%s\n' "$restored_runtime" \
>"$E2E_ARTIFACT_DIR/docker-default-runtime-restored.txt" \
|| restore_status=1
fi
if [ "$restore_status" -ne 0 ]; then
exit 1
fi
exit "$command_status"
}
# Force runc only within this trap-guarded process. The wrapper rejects native
# create before progress, and the compatibility retry must add GPU access
# explicitly instead of inheriting the managed runner's NVIDIA daemon default.
trap restore_docker_default_runtime EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
original_runtime="$(run_trusted_fixture capture "$state_dir" "$daemon_json")"
printf '%s\n' "$original_runtime" \
>"$E2E_ARTIFACT_DIR/docker-default-runtime-before.txt"
selected_runtime="$(run_trusted_fixture select-runc "$state_dir" "$daemon_json")"
printf '%s\n' "$selected_runtime" \
>"$E2E_ARTIFACT_DIR/docker-default-runtime-during.txt"
fi
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-gpu-startup.test.ts
- name: Recover Docker daemon after Hermes GPU fallback fixture
if: always()
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
run: |
set -euo pipefail
if [ "$E2E_HERMES_GPU_STARTUP_SCENARIO" != fallback ]; then
exit 0
fi
recovery_failed=0
restored_runtime=""
trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}"
trusted_state_root=/var/lib/nemoclaw-e2e
state_prefix="hermes-gpu-fallback-docker-runtime.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.fallback."
run_trusted_fixture() {
/usr/bin/sudo -n /usr/bin/env -i \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
/bin/bash "$trusted_fixture" "$@"
}
if ! /usr/bin/sudo /usr/bin/find "$trusted_state_root" -mindepth 1 -maxdepth 1 \
-type d -name "${state_prefix}*" -print0 \
| while IFS= read -r -d '' state_dir; do
if ! restored_runtime="$(run_trusted_fixture restore \
"$state_dir" /etc/docker/daemon.json)"; then
exit 1
fi
if [ -n "$restored_runtime" ]; then
printf '%s\n' "$restored_runtime" \
>"$E2E_ARTIFACT_DIR/docker-default-runtime-restored.txt"
fi
done; then
recovery_failed=1
fi
if [ "$recovery_failed" -ne 0 ]; then
echo "Independent Docker daemon recovery could not prove restoration" >&2
exit 1
fi
- name: Remove trusted Hermes GPU runtime fixture
if: ${{ always() && matrix.scenario == 'fallback' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
run: |
set -euo pipefail
trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}"
/usr/bin/sudo /usr/bin/rm -f -- "$trusted_fixture"
- name: Upload Hermes GPU startup artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-hermes-gpu-startup-${{ matrix.scenario }}-${{ matrix.runtime_provider }}
path: e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/${{ matrix.runtime_provider }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
jetson-nvmap-gpu:
needs: [base-image-publication, generate-matrix]
if: ${{ always() && needs['base-image-publication'].result == 'success' && needs['base-image-publication'].outputs.managed_image_revision != '' && needs['generate-matrix'].result == 'success' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.allow_jetson_dispatch && (inputs.checkout_repository == '' || inputs.checkout_repository == github.repository) && ((inputs.jobs == '' && inputs.targets == '') || contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'jetson-nvmap-gpu')))) }}
concurrency:
group: jetson-nvmap-gpu-dispatch
cancel-in-progress: false
runs-on: ubuntu-latest
timeout-minutes: 60
permissions:
contents: read
id-token: write
steps:
- name: Check out trusted Jetson controller
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: NVIDIA/NemoClaw
ref: ${{ github.workflow_sha }}
persist-credentials: false
- name: Set up Node for Jetson controller
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- name: Dispatch exact commit to Jetson through operator backend
env:
E2E_ARTIFACT_DIR: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu
JETSON_DISPATCH_CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
JETSON_DISPATCH_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }}
JETSON_DISPATCH_URL: ${{ vars.JETSON_DISPATCH_URL }}
run: node --experimental-strip-types --no-warnings tools/e2e/jetson-dispatch-client.mts
- name: Upload Jetson nvmap GPU artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-jetson-nvmap-gpu
path: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu/
cloud-onboard:
name: Cloud onboard (${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'cloud-onboard') }}
runs-on: ubuntu-latest
timeout-minutes: 70
strategy:
fail-fast: false
matrix:
runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['cloud-onboard'] }}
env:
E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }}
E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }}
NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker,podman
E2E_TARGET_ID: "cloud-onboard"
E2E_AGENT_RUNTIME: "openclaw"
E2E_OBSERVABLE_OUTCOME: "Public install onboarding hosted inference and security checks succeed"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/cloud-onboard/${{ matrix.runtime_provider }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }}
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-cloud-onboard"
NEMOCLAW_PUBLIC_INSTALL_REF: "${{ inputs.checkout_sha || github.sha }}"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Configure cloud-onboard trace directory
shell: bash
run: |
set -euo pipefail
printf 'NEMOCLAW_TRACE_DIR=%s\n' "${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces" >> "${GITHUB_ENV}"
# invalidState: the cloud-onboard DCode TUI check requires a PTY driver,
# but the fixed GitHub-hosted runner image does not provide expect.
# sourceBoundary: privileged host setup runs from the first-party
# host-dependency-setup action pinned to an immutable full SHA, never the
# repository target ref; the check only consumes expect without privilege.
# whyNotSourceFix: GitHub-hosted jobs cannot use a repository-owned host
# image, and caching privileged dpkg state between clean runners is not
# supported.
# regressionTest: workflow-boundary tests pin the ordering, action
# provenance, and package mapping.
# removalCondition: remove when the hosted runner supplies expect or the
# cloud-onboard acceptance check no longer requires a PTY.
- name: Install cloud-onboard DCode TUI host dependencies
uses: NVIDIA/NemoClaw/.github/actions/host-dependency-setup@4def1501b34ce586f83b91af50a66b5d22b31d75
with:
packages: expect
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Prepare native Podman E2E runtime
uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de
with:
enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}
- name: Install OpenShell CLI
run: bash scripts/install-openshell.sh
- name: Run cloud-onboard live Vitest test
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/cloud-onboard.test.ts
- name: Build trusted cloud-onboard timing summary
if: always()
shell: bash
run: |
set -euo pipefail
NEMOCLAW_TRACE_DIR="${NEMOCLAW_TRACE_DIR:-}"
expected_trace_dir="${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces"
if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then
echo "::error title=E2E trace sanitization refused::NEMOCLAW_TRACE_DIR does not match its workflow-owned RUNNER_TEMP path. No raw traces were read or uploaded. Correct the trace path configuration before rerunning." >&2
printf 'Expected trace path: %s\n' "${expected_trace_dir}" >&2
exit 1
fi
python3 scripts/e2e/sanitize-trace-timing.py \
"${NEMOCLAW_TRACE_DIR}" \
"${E2E_ARTIFACT_DIR}"
# The target process must emit full local traces for diagnosis, but those
# traces may contain prompts, environment data, and credential material.
# Keep cleanup as a separate always() step so a sanitizer failure cannot
# bypass it. A runner-level termination is contained by the ephemeral
# GitHub-hosted runner.temp boundary. Remove this step only when the trace
# producer itself emits the allowlisted timing-only schema.
- name: Delete raw cloud-onboard traces
if: always()
shell: bash
run: |
set -euo pipefail
NEMOCLAW_TRACE_DIR="${NEMOCLAW_TRACE_DIR:-}"
expected_trace_dir="${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces"
if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then
echo "::error title=E2E trace cleanup refused::NEMOCLAW_TRACE_DIR does not match its workflow-owned RUNNER_TEMP path. No files were deleted. Correct the trace path configuration before rerunning." >&2
printf 'Expected trace path: %s\n' "${expected_trace_dir}" >&2
exit 1
fi
rm -rf -- "${NEMOCLAW_TRACE_DIR}"
- name: Upload cloud-onboard artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-cloud-onboard-${{ matrix.runtime_provider }}
path: e2e-artifacts/live/cloud-onboard/${{ matrix.runtime_provider }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
messaging-providers:
name: Messaging providers (${{ matrix.runtime_provider }})
needs: [base-image-publication, generate-matrix]
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'messaging-providers') }}
runs-on: ubuntu-latest
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['messaging-providers'] }}
env:
E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }}
E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }}
E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }}
NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }}
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker,podman
E2E_TARGET_ID: "messaging-providers"
E2E_AGENT_RUNTIME: "openclaw"
E2E_OBSERVABLE_OUTCOME: "Provider configuration redaction and optional real sends succeed"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference and messaging providers"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/messaging-providers/${{ matrix.runtime_provider }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }}
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Prepare native Podman E2E runtime
uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de
with:
enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}
- name: Run messaging providers live Vitest test
# The test keeps
# the legacy fake-token defaults, optional _REAL secret overrides,
# provider placeholder redaction checks, and WhatsApp QR-only parity.
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
TELEGRAM_BOT_TOKEN_REAL: ${{ secrets.TELEGRAM_BOT_TOKEN_REAL }}
DISCORD_BOT_TOKEN_REAL: ${{ secrets.DISCORD_BOT_TOKEN_REAL }}
SLACK_BOT_TOKEN_REAL: ${{ secrets.SLACK_BOT_TOKEN_REAL }}
SLACK_APP_TOKEN_REAL: ${{ secrets.SLACK_APP_TOKEN_REAL }}
TELEGRAM_ALLOWED_IDS: ${{ secrets.TELEGRAM_ALLOWED_IDS }}
TELEGRAM_AUTHORIZED_CHAT_IDS: ${{ secrets.TELEGRAM_AUTHORIZED_CHAT_IDS }}
TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
TELEGRAM_CHAT_ID_E2E: ${{ secrets.TELEGRAM_CHAT_ID_E2E }}
DISCORD_CHANNEL_ID_E2E: ${{ secrets.DISCORD_CHANNEL_ID_E2E }}
SLACK_CHANNEL_ID_E2E: ${{ secrets.SLACK_CHANNEL_ID_E2E }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/messaging-providers.test.ts
- name: Upload messaging providers artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-messaging-providers-${{ matrix.runtime_provider }}
path: e2e-artifacts/live/messaging-providers/${{ matrix.runtime_provider }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# Main-push current-lifecycle coverage for #6108. This proves a custom plugin
# installs across devices and remains usable after restart and recreation.
openclaw-plugin-runtime-exdev:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openclaw-plugin-runtime-exdev') }}
runs-on: ubuntu-latest
permissions:
contents: read
# Two bounded 25-minute onboarding operations plus the 15-minute Vitest
# buffer need 65 minutes; allow 20 more for setup and teardown.
timeout-minutes: 85
env:
E2E_JOB: "1"
E2E_GATEWAY_RUNTIMES: docker
E2E_TARGET_ID: "openclaw-plugin-runtime-exdev"
E2E_AGENT_RUNTIME: "openclaw"
E2E_OBSERVABLE_OUTCOME: "OpenClaw installs the custom plugin across devices; plugin behavior survives restart and recreation"
E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; current package; no inference endpoint"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-oc-exdev"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Pre-pull current-checkout Docker Hub builder image
shell: bash
run: |
set -euo pipefail
docker pull node:22-trixie-slim@sha256:db8a96a63e5264607ada2d206758876ebbed6a12be2ada7517793cbfb0c2a29c
- name: Remove Docker auth before current-checkout fixture
if: always()
uses: NVIDIA/NemoClaw/.github/actions/docker-auth-cleanup@d5f37099766ca82a4516e7d8f0de117cda197fe3
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Run OpenClaw cross-device plugin lifecycle live test
run: |
set -euo pipefail
test -n "${DOCKER_CONFIG:-}"
test ! -e "${DOCKER_CONFIG}"
test -z "${DOCKERHUB_USERNAME:-}"
test -z "${DOCKERHUB_TOKEN:-}"
env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \
npx tsx tools/e2e/live-vitest-invocation.mts run \
--test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \
--selector current-lifecycle
- name: Upload OpenClaw cross-device plugin lifecycle artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
report-to-pr:
runs-on: ubuntu-latest
timeout-minutes: 15
# This entire workflow is dispatch-only. Keeping selective jobs in `needs`
# makes the report wait for and record any requested job without adding
# skipped checks to the normal pull_request workflow.
needs:
&e2e-result-jobs [
base-image-publication,
generate-matrix,
retired-selector-compatibility,
staging-brev-launchable,
staging-brev-launchable-identity,
live,
shared-e2e,
catalogue-standard,
catalogue-nvidia-api,
catalogue-nvidia-inference,
catalogue-github-read,
catalogue-brave-nvidia-inference,
openshell-gateway-auth-contract,
external-gateway-health,
mcp-bridge,
openshell-credential-generation-window,
openshell-dev-artifact,
mcp-bridge-dev,
managed-image-multiarch-startup,
llama-cpp-dgx-spark-plan,
llama-cpp-dgx-spark-qualification,
managed-image-protected-runtime,
hermes-e2e,
hermes-gpu-startup,
messaging-providers,
jetson-nvmap-gpu,
cloud-onboard,
openclaw-plugin-runtime-exdev,
native-runtime-qualification-producer,
native-runtime-qualification-producer-aggregate,
]
if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }}
permissions:
actions: read
contents: read
# The issue-comment endpoint accepts pull request write permission for PR comments.
# Keep issues: write absent so this job cannot restore general issue routing.
pull-requests: write
steps:
- name: Check out the trusted E2E reporting helper
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: |
tools/e2e/report-e2e-results.mts
tools/e2e/selector-aliases.mts
sparse-checkout-cone-mode: false
- name: Post E2E target results to PR
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }}
TEST_MATRIX: ${{ needs.generate-matrix.outputs.test_matrix }}
JOB_PR_NUMBER: ${{ inputs.pr_number }}
JOB_TARGETS: ${{ inputs.targets }}
JOBS: ${{ inputs.jobs }}
NEEDS_JSON: ${{ toJSON(needs) }}
with:
script: |
const path = require('node:path');
const { pathToFileURL } = require('node:url');
const { resolveReportPr, loadReportJobs, renderE2eReport } = await import(
pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'tools/e2e/report-e2e-results.mts')).href
);
const needs = JSON.parse(process.env.NEEDS_JSON || '{}');
const prNumber = await resolveReportPr({ github, context, core, env: process.env });
if (prNumber === undefined) return;
const { apiJobs, loaded } = await loadReportJobs({ github, context, core });
const report = renderE2eReport({
needs,
env: process.env,
apiJobs,
apiJobsLoaded: loaded,
context,
});
if (report.fatal) {
core.setFailed(report.fatal);
return;
}
for (const warning of report.warnings) core.warning(warning);
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: report.body,
});
relevant-e2e:
name: Relevant E2E
runs-on: ubuntu-latest
timeout-minutes: 5
needs: *e2e-result-jobs
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'push' }}
permissions:
contents: read
steps:
- name: Check out the E2E result evaluator
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: tools/e2e/release-qualification.mts
sparse-checkout-cone-mode: false
- name: Require every selected E2E result
env:
NEEDS_JSON: ${{ toJSON(needs) }}
RELEASE_REQUIRED_JOBS: ${{ needs.generate-matrix.outputs.selected_workflow_jobs }}
run: node --experimental-strip-types --no-warnings tools/e2e/release-qualification.mts
- name: Summarize selected E2E
env:
SELECTED_WORKFLOW_JOBS: ${{ needs.generate-matrix.outputs.selected_workflow_jobs }}
run: |
set -euo pipefail
if [ "${SELECTED_WORKFLOW_JOBS}" = "[]" ]; then
echo "No E2E target owns a file changed by this push." >> "${GITHUB_STEP_SUMMARY}"
else
{
echo "The selected E2E workflow jobs passed:"
echo
jq -r '.[] | "- `\(.)`"' <<< "${SELECTED_WORKFLOW_JOBS}"
} >> "${GITHUB_STEP_SUMMARY}"
fi
release-qualification:
name: Release qualification
runs-on: ubuntu-latest
timeout-minutes: 5
needs: *e2e-result-jobs
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && inputs.jobs == '' && inputs.targets == '' && inputs.include_staging_brev_launchable && !inputs.allow_jetson_dispatch && !inputs.allow_dgx_spark_runner_queue }}
permissions:
contents: read
steps:
- name: Check out the qualification evaluator
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: tools/e2e/release-qualification.mts
sparse-checkout-cone-mode: false
- name: Require every release E2E result
env:
NEEDS_JSON: ${{ toJSON(needs) }}
RELEASE_REQUIRED_JOBS: ${{ needs.generate-matrix.outputs.release_required_jobs }}
run: node --experimental-strip-types --no-warnings tools/e2e/release-qualification.mts
# ── Push/manual scorecard ─────────────────────────────────────────────────
scorecard:
runs-on: ubuntu-latest
timeout-minutes: 15
needs: *e2e-result-jobs
if: ${{ always() && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '')) }}
permissions:
actions: read
contents: read
steps:
- name: Checkout scorecard builders
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
sparse-checkout: |
ci/onboard-performance-budget.json
scripts/audit-test-runtime.mts
scripts/lib/read-artifact-zip.mts
scripts/scorecard
sparse-checkout-cone-mode: false
- name: Download E2E progress artifacts
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ${{ runner.temp }}/e2e-runtime-audit
pattern: e2e-*
- name: Generate E2E scorecard
id: scorecard
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }}
JOBS: ${{ inputs.jobs }}
RUNTIME_ARTIFACTS: ${{ runner.temp }}/e2e-runtime-audit
RUNTIME_SUMMARY_FILE: ${{ runner.temp }}/e2e-runtime-summary.json
TARGETS: ${{ inputs.targets }}
NEEDS_JSON: ${{ toJSON(needs) }}
with:
script: |
const path = require('path');
const { buildScorecard } = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/coordinate-scorecard.mts'),
);
const scorecardJobs = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/summarize-jobs.mts'),
);
const traceTiming = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-trace-timing.mts'),
);
const runtimeAudit = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'),
);
const runtimeHistory = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-runtime-history.mts'),
);
const firstTurnLatency = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-first-turn-latency.mts'),
);
const sandboxPhaseTail = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-sandbox-phase-tail.mts'),
);
const needs = JSON.parse(process.env.NEEDS_JSON || '{}');
// GitHub's jobs API is the canonical source because `needs.live`
// collapses every matrix target into one result and has no job URL.
// The typed helper owns and tests the degraded `needs` fallback.
const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core });
let runtimeSummaryMarkdown;
let runtimeHistoryRows = null;
try {
const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]);
runtimeSummaryMarkdown = runtimeAudit.formatRuntimeAuditSummary(runtimeRows);
runtimeHistoryRows = runtimeAudit.collectRuntimeHistorySamples([
process.env.RUNTIME_ARTIFACTS,
]);
} catch {
core.warning('E2E test phase runtime summary unavailable: invalid progress artifact');
runtimeSummaryMarkdown = [
'## E2E Test Phase Runtime',
'',
'The summary is unavailable because a `test-progress.json` artifact was invalid.',
'',
].join('\n');
}
const runtimeHistoryMarkdown = runtimeHistoryRows === null
? [
'## E2E Push Runtime Trend',
'',
'The trend is unavailable because a `test-progress.json` artifact was invalid.',
'',
].join('\n')
: await runtimeHistory.buildRuntimeHistory(
{ github, context, core },
runtimeHistoryRows,
process.env.RUNTIME_SUMMARY_FILE,
{
currentFirstTurnLatency:
firstTurnLatency.readCurrentFirstTurnLatencySample(
process.env.RUNTIME_ARTIFACTS,
),
currentSandboxPhaseTail:
sandboxPhaseTail.readCurrentSandboxPhaseTailSample(
process.env.RUNTIME_ARTIFACTS,
),
loadPriorPushHistory: runtimeHistory.loadPriorPushHistory,
},
);
const trace = await traceTiming.buildTraceTimingResult({ github, context, core });
if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage);
const { summaryMarkdown, scorecardData, slackData } = buildScorecard({
eventName: context.eventName,
actor: context.actor || '',
serverUrl: context.serverUrl,
repo: context.repo,
runId: context.runId,
rawJobs: process.env.JOBS || '',
rawTargets: process.env.TARGETS || '',
rawExplicitOnly: process.env.EXPLICIT_ONLY_JOBS || '',
needs,
apiJobs,
trace,
today: new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
});
await core.summary
.addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}\n${runtimeHistoryMarkdown}`)
.write();
core.setOutput('scorecardData', JSON.stringify(scorecardData));
core.setOutput('slackData', JSON.stringify(slackData));
- name: Post scorecard to Slack
# Webhook secrets never enter branch-dispatched runs. The payload is
# computed in the preceding no-secret step; this fixed publisher does
# not load code from the checked-out workflow ref.
if: ${{ steps.scorecard.outputs.slackData != '' && github.ref == 'refs/heads/main' }}
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
SLACK_WEBHOOK_URL_DAILY: ${{ secrets.SLACK_WEBHOOK_URL_DAILY }}
SLACK_WEBHOOK_URL_FULLRUN: ${{ secrets.SLACK_WEBHOOK_URL_FULLRUN }}
SLACK_WEBHOOK_URL_PREVIEW: ${{ secrets.SLACK_WEBHOOK_URL_PREVIEW }}
SLACK_DATA: ${{ steps.scorecard.outputs.slackData }}
POST_TO_SLACK: ${{ inputs.post_to_slack }}
with:
script: |
const data = JSON.parse(process.env.SLACK_DATA);
const envByChannel = {
daily: 'SLACK_WEBHOOK_URL_DAILY',
fullrun: 'SLACK_WEBHOOK_URL_FULLRUN',
preview: 'SLACK_WEBHOOK_URL_PREVIEW',
};
const channel = data?.channel;
const payload = data?.payload;
const attachments = payload?.attachments;
const validPayload =
Object.hasOwn(envByChannel, channel) &&
payload !== null &&
typeof payload === 'object' &&
typeof payload.text === 'string' &&
Array.isArray(attachments) &&
attachments.every(
(attachment) =>
attachment !== null &&
typeof attachment === 'object' &&
typeof attachment.color === 'string' &&
Array.isArray(attachment.blocks),
);
if (!validPayload) {
core.setFailed('Invalid precomputed Slack payload');
return;
}
if (channel === 'preview' && process.env.POST_TO_SLACK !== 'true') {
core.info('Selective dispatch without post_to_slack — skipping');
return;
}
const webhookUrl = process.env[envByChannel[channel]];
if (!webhookUrl) {
core.info(`Slack webhook for "${channel}" not configured — skipping`);
return;
}
const response = await fetch(webhookUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(data.payload),
});
if (!response.ok) {
core.setFailed(`Slack webhook returned ${response.status}`);
}
- name: Upload E2E runtime summary
if: ${{ always() && github.event_name == 'push' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-runtime-summary
path: ${{ runner.temp }}/e2e-runtime-summary.json