diff --git a/.github/actions/extract-version-from-tag/action.yaml b/.github/actions/extract-version-from-tag/action.yaml index 98d8c74..4b7ab3c 100644 --- a/.github/actions/extract-version-from-tag/action.yaml +++ b/.github/actions/extract-version-from-tag/action.yaml @@ -36,15 +36,14 @@ runs: TAG=${GITHUB_REF##*/} SOURCE="tag" else - echo "No version sources found, exiting" - exit 1 + echo "No version sources found. Using fallback generated from latest tag." + TAG=$(git describe --tags) + VERSION=$TAG + SOURCE="fallback" fi - # Strip the prefix if it exists - if [[ "$TAG" == "${{ inputs.tag-prefix }}"* ]]; then - VERSION=${TAG#${{ inputs.tag-prefix }}} - else - VERSION=$TAG + if [[ "$VERSION" == "${{ inputs.tag-prefix }}"* ]]; then + VERSION=${VERSION#${{ inputs.tag-prefix }}} fi echo "version=$VERSION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/VERSION.example b/.github/workflows/VERSION.example index 0495c4a..c813fe1 100644 --- a/.github/workflows/VERSION.example +++ b/.github/workflows/VERSION.example @@ -1 +1 @@ -1.2.3 +1.2.5 diff --git a/.github/workflows/deploy-artifacts/README.md b/.github/workflows/deploy-artifacts/README.md index 3bbbac8..79ef7e5 100644 --- a/.github/workflows/deploy-artifacts/README.md +++ b/.github/workflows/deploy-artifacts/README.md @@ -17,17 +17,25 @@ The workflow processes artifacts from a `build-artifacts` directory and creates | Input | Description | Required | Default | | -------------------------------- | ---------------------------------------------------------- | -------- | ------------------------------- | | `project` | JFrog Artifactory project name | Yes | - | -| `build-name` | Name for the unified build | Yes | - | +| `build-name` | JFrog build name | Yes | - | +| `build-id` | JFrog build ID for the overall build info | Yes | - | +| `metadata-build-id` | JFrog build ID for the build metadata | Yes | - | | `version` | Version string for build info | Yes | - | | `artifactory-url` | JFrog Artifactory URL | No | `https://artifact.aerospike.io` | | `artifactory-oidc-provider-name` | OIDC provider name for authentication | No | `gh-citrusleaf` | | `artifactory-oidc-audience` | OIDC audience for authentication | No | `citrusleaf` | -| `artifact-name` | Name of the artifacts to download | No | `build-artifacts` | +| `artifact-name` | Name of the artifacts to download | No | `signed-artifacts` | | `retention-days` | Retention days for the artifacts | No | `1` | | `runs-on` | The runner to use for the build | No | `ubuntu-22.04` | | `checkout-path` | Directory to checkout the shared-workflows repository into | No | `shared-workflows` | | `dry-run` | Whether to run in dry-run mode | No | `false` | +## Outputs + +| Output | Description | +| ---------- | ----------------- | +| `build-id` | The build ID used | + ## Structure ### Debian/Ubuntu @@ -79,13 +87,15 @@ jobs: upload: uses: aerospike/shared-workflows/.github/workflows/reusable_deploy-artifacts.yaml@CURRENTGITSHA # vn.n.n with: - project: database - build-name: database - version: ${{ github.ref_name }} + project: database + build-name: database + build-id: 1234567890 + metadata-build-id: 1234567890-metadata + version: ${{ github.ref_name }} artifactory-url: https://artifact.aerospike.io artifactory-oidc-provider-name: gh-citrusleaf artifactory-oidc-audience: citrusleaf - artifact-name: build-artifacts + artifact-name: signed-artifacts retention-days: 1 dry-run: false ``` diff --git a/.github/workflows/deploy-artifacts/entrypoint.sh b/.github/workflows/deploy-artifacts/entrypoint.sh index 380e90e..919db0d 100755 --- a/.github/workflows/deploy-artifacts/entrypoint.sh +++ b/.github/workflows/deploy-artifacts/entrypoint.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +set -x export PS4='+($LINENO): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' trap 'handle_error ${LINENO}' ERR # shellcheck disable=SC2317 @@ -36,12 +37,13 @@ while [[ $# -gt 0 ]]; do echo "Uploads artifacts to JFrog Artifactory" >&2 echo "" >&2 echo "Options:" >&2 + echo " --metadata-build-number JFrog build ID for the build metadata" >&2 echo " --dry-run Show what would be uploaded without actually uploading" >&2 echo " --help, -h Show this help message" >&2 echo "" >&2 echo "Examples:" >&2 - echo " $0 database my-app v1.0.0 1754566442238" >&2 - echo " $0 database my-app v1.0.0 1754566442238 --dry-run" >&2 + echo " $0 database my-app v1.0.0 1754566442238 1754566442238-metadata" >&2 + echo " $0 database my-app v1.0.0 1754566442238 1754566442238-metadata --dry-run" >&2 exit 0 ;; -*) @@ -59,6 +61,8 @@ while [[ $# -gt 0 ]]; do VERSION="$1" elif [[ -z "${BUILD_NUMBER:-}" ]]; then BUILD_NUMBER="$1" + elif [[ -z "${METADATA_BUILD_NUMBER:-}" ]]; then + METADATA_BUILD_NUMBER="$1" else echo "Use --help for usage information" >&2 exit 1 @@ -68,7 +72,6 @@ while [[ $# -gt 0 ]]; do esac done - if [[ -z "${PROJECT:-}" ]]; then error "project is required Use --help for usage information" @@ -89,6 +92,12 @@ if [[ -z "${BUILD_NUMBER:-}" ]]; then Use --help for usage information" fi +if [[ -z "${METADATA_BUILD_NUMBER:-}" ]]; then + error "metadata-build-number is required +Use --help for usage information" +fi +ARTIFACT_BUILD_NUMBER="$BUILD_NUMBER-artifacts" + # Source the package utilities SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 @@ -109,7 +118,7 @@ run() { } run_optional() { - "$@" || echo "Warning: $*" >&2 + run "$@" || echo "Warning: $*" >&2 } structure_build_artifacts() { @@ -157,7 +166,7 @@ upload_deb_packages() { run jf rt upload "$deb" "$PROJECT-deb-dev-local" --flat=false \ --build-name="$BUILD_NAME" \ - --build-number="$BUILD_NUMBER" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ --project="$PROJECT" \ --target-props "version=$VERSION;deb.distribution=$codename;deb.component=main;deb.architecture=$arch" \ --deb "$codename/main/$arch" @@ -167,25 +176,10 @@ upload_deb_packages() { echo " Uploading signature: $deb.asc" >&2 run jf rt upload "$deb.asc" "$PROJECT-deb-dev-local" --flat=false \ --build-name="$BUILD_NAME" \ - --build-number="$BUILD_NUMBER" \ - --project="$PROJECT" - fi - - if [[ -f "$deb.sha256" ]]; then - echo " Uploading checksum: $deb.sha256" >&2 - run jf rt upload "$deb.sha256" "$PROJECT-deb-dev-local" --flat=false \ - --build-name="$BUILD_NAME" \ - --build-number="$BUILD_NUMBER" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ --project="$PROJECT" fi - if [[ -f "$deb.asc.sha256" ]]; then - echo " Uploading signature checksum: $deb.asc.sha256" >&2 - run jf rt upload "$deb.asc.sha256" "$PROJECT-deb-dev-local" --flat=false \ - --build-name="$BUILD_NAME" \ - --build-number="$BUILD_NUMBER" \ - --project="$PROJECT" - fi done < <(find . -name "*.deb" -print0) } @@ -214,7 +208,7 @@ upload_rpm_packages() { # Upload the RPM run jf rt upload "$rpm" "$PROJECT-rpm-dev-local" --flat=false \ --build-name="$BUILD_NAME" \ - --build-number="$BUILD_NUMBER" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ --project="$PROJECT" \ --target-props "version=$VERSION;rpm.distribution=$dist;rpm.component=main;rpm.architecture=$arch" @@ -223,25 +217,10 @@ upload_rpm_packages() { echo " Uploading signature: $rpm.asc" >&2 run jf rt upload "$rpm.asc" "$PROJECT-rpm-dev-local" --flat=false \ --build-name="$BUILD_NAME" \ - --build-number="$BUILD_NUMBER" \ - --project="$PROJECT" - fi - - if [[ -f "$rpm.sha256" ]]; then - echo " Uploading checksum: $rpm.sha256" >&2 - run jf rt upload "$rpm.sha256" "$PROJECT-rpm-dev-local" --flat=false \ - --build-name="$BUILD_NAME" \ - --build-number="$BUILD_NUMBER" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ --project="$PROJECT" fi - if [[ -f "$rpm.asc.sha256" ]]; then - echo " Uploading signature checksum: $rpm.asc.sha256" >&2 - run jf rt upload "$rpm.asc.sha256" "$PROJECT-rpm-dev-local" --flat=false \ - --build-name="$BUILD_NAME" \ - --build-number="$BUILD_NUMBER" \ - --project="$PROJECT" - fi done < <(find . -name "*.rpm" -print0) } @@ -253,45 +232,89 @@ upload_generic_files() { fi while IFS= read -r -d '' file; do + echo "Processing generic file: $file" >&2 if [[ -f "$file" ]]; then echo "Uploading generic file: $file" >&2 run jf rt upload "$file" "$PROJECT-generic-dev-local" --flat=false \ --build-name="$BUILD_NAME" \ - --build-number="$BUILD_NUMBER" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ --project="$PROJECT" fi - done < <(find . -type f \( -not -name "*.deb" -not -name "*.rpm" -not -name "*.asc" -not -name "*.sha256" \) -print0) + done < <(find . \( -not -name "*.deb" -not -name "*.rpm" -not -name "*.asc" \) -print0) } -# Note for future: This helper that checks if the build exists is needed -# because this functionality is not in the jf command line. There are ways to do this with the rest API but for the specific -# purpose of checking if there is already a build with the same name and number, this looks like the cleanest solution. -check_build_exists() { - local build_name="$1" - local build_number="$2" - local project="$3" - - local count - count=$(jf rt search "${project}-build-info/$build_name/$build_number-*" --count 2>/dev/null | tail -n 1) - - if [[ "$count" =~ ^[0-9]+$ ]] && [[ "$count" -gt 0 ]]; then - return 0 # Build exists + +# Collects build-info metadata by querying Artifactory for JSON files. +# JFrog API doesn't support wildcarding build IDs, so we fetch the JSON +# files to extract the collection of build IDs for this parent build. +discover_build_infos() { + local project="$1" + local parent_build_name="$2" + local build_info_search_pattern="$3" + local build_info_repo="${4:-${project}-build-info}" + + echo "Discovering build-infos for project: $project" >&2 + echo "Parent build name: $parent_build_name" >&2 + echo "Search pattern: $build_info_search_pattern" >&2 + echo "Build-info repo: $build_info_repo" >&2 + + read -r -d '' AQL <&2 + # Query for build-info JSON files + local resp + resp=$(run jf rt curl -XPOST api/search/aql \ + -H 'Content-Type: text/plain' \ + -d "$AQL") + echo "search response: $resp" >&2 + # Extract child build IDs from JSON file names + if echo "$resp" | jq -e '.results' > /dev/null 2>&1; then + mapfile -t CHILD_BUILD_IDS < <(echo "$resp" | jq -r '.results[] | .name' | sed 's/-[0-9]*\.json$//' | sort -u) + + echo "Found ${#CHILD_BUILD_IDS[@]} child builds" >&2 + + CHILD_BUILD_IDS_RESULT=("${CHILD_BUILD_IDS[@]}") + return 0 else - return 1 # Build does not exist + echo "No build-info files found" >&2 + CHILD_BUILD_IDS_RESULT=() + return 1 fi } + +# This function handles publishing the tree of build info to jfrog +# 1. publish the signed artifacts +# 2. append metadata and artifact builld infos to new build info +# 3. publish the new build info publish_build_info() { - if [[ "$DRY_RUN" == "true" ]]; then - echo "Would publish build info..." >&2 - else - echo "Publishing build info..." >&2 - fi - echo "about to add optional build info and publishing" - run_optional jf rt build-collect-env "$BUILD_NAME" "$BUILD_NUMBER" --project="$PROJECT" - run_optional jf rt build-add-git "$BUILD_NAME" "$BUILD_NUMBER" --project="$PROJECT" - run_optional jf rt build-add-dependencies "$BUILD_NAME" "$BUILD_NUMBER" . --project="$PROJECT" - run jf rt build-publish "$BUILD_NAME" "$BUILD_NUMBER" --project="$PROJECT" + run jf rt build-publish "$BUILD_NAME" "$ARTIFACT_BUILD_NUMBER" --project="$PROJECT" + + discover_build_infos "$PROJECT" "$BUILD_NAME" "$METADATA_BUILD_NUMBER*" "$PROJECT-build-info" + + # Access the results + if [[ ${#CHILD_BUILD_IDS_RESULT[@]} -gt 0 ]]; then + echo "Found ${#CHILD_BUILD_IDS_RESULT[@]} child builds:" >&2 + for build_id in "${CHILD_BUILD_IDS_RESULT[@]}"; do + echo " $BUILD_NAME/$build_id" + run jf rt build-append "$BUILD_NAME" "$BUILD_NUMBER" \ + "$BUILD_NAME" "$build_id" --project="$PROJECT" + done + fi + + run jf rt build-append "$BUILD_NAME" "$BUILD_NUMBER" \ + "$BUILD_NAME" "$ARTIFACT_BUILD_NUMBER" --project="$PROJECT" + run jf rt build-publish "$BUILD_NAME" "$BUILD_NUMBER" --project="$PROJECT" } main() { @@ -305,12 +328,14 @@ main() { echo "Version: $VERSION" >&2 echo "Dry run: $DRY_RUN" >&2 echo "Build number: $BUILD_NUMBER" >&2 + echo "Metadata build number: $METADATA_BUILD_NUMBER" >&2 mkdir -p structured_build_artifacts shopt -s globstar nullglob structure_build_artifacts cd structured_build_artifacts - + echo "Structured build artifacts:" >&2 + find . >&2 # Upload all packages upload_rpm_packages upload_deb_packages diff --git a/.github/workflows/deploy-artifacts/test-entrypoint.sh b/.github/workflows/deploy-artifacts/test-entrypoint.sh index 214e728..8c7a8ab 100755 --- a/.github/workflows/deploy-artifacts/test-entrypoint.sh +++ b/.github/workflows/deploy-artifacts/test-entrypoint.sh @@ -96,6 +96,12 @@ capture_dry_run_output() { # Run the dry-run and capture output (both stdout and stderr) local dry_run_output dry_run_output=$(eval "$test_args" 2>&1) + exit_code=$? + if [[ $exit_code -ne 0 ]]; then + #if error print output and move on + echo " Error received during dry run" + echo "$dry_run_output" + fi echo "Save the full output to $output_file" echo "$dry_run_output" > "$output_file" @@ -181,7 +187,7 @@ cd "$TEST_DIR" || error "Failed to cd to $TEST_DIR" capture_dry_run_output \ "DEB and RPM upload test" \ - "$SCRIPT_DIR/entrypoint.sh test-project test-build v1.0.0 12345 --dry-run" \ + "$SCRIPT_DIR/entrypoint.sh test-project test-build v1.0.0 12345 12345-metadata --dry-run" \ "$TEST_DIR/test1_output.txt" echo "" @@ -198,23 +204,26 @@ target-props.*deb.distribution=bookworm target-props.*rpm.distribution= "; then test1_success=false + cat "$TEST_DIR/test1_output.txt.upload_commands" fi if ! verify_commands "Build info commands" "$TEST_DIR/test1_output.txt.build_commands" " -jf rt build-collect-env.*test-build.*12345 -jf rt build-add-git.*test-build.*12345 -jf rt build-add-dependencies.*test-build.*12345 jf rt build-publish.*test-build.*12345 "; then test1_success=false + cat "$TEST_DIR/test1_output.txt.build_commands" fi if ! verify_command_count "Upload commands" "$TEST_DIR/test1_output.txt.upload_commands" 4; then test1_success=false + echo "Wrong number of commands" + cat "$TEST_DIR/test1_output.txt.upload_commands" fi -if ! verify_command_count "Build commands" "$TEST_DIR/test1_output.txt.build_commands" 4; then +if ! verify_command_count "Build commands" "$TEST_DIR/test1_output.txt.build_commands" 3; then test1_success=false + echo "Wrong number of commands" + cat "$TEST_DIR/test1_output.txt.build_commands" fi record_test_result "Test 1: DEB and RPM upload" "$test1_success" @@ -224,7 +233,7 @@ echo "" echo " Test 2: Uploading nested files" capture_dry_run_output \ "Nested files upload test" \ - "$SCRIPT_DIR/entrypoint.sh test-project test-build v1.0.0 12345 --dry-run" \ + "$SCRIPT_DIR/entrypoint.sh test-project test-build v1.0.0 12345 12345-metadata --dry-run" \ "$TEST_DIR/test2_output.txt" echo "" @@ -249,7 +258,7 @@ echo "" echo " Test 3: Uploading all files" capture_dry_run_output \ "All files upload test" \ - "$SCRIPT_DIR/entrypoint.sh test-project test-build v1.0.0 12345 --dry-run" \ + "$SCRIPT_DIR/entrypoint.sh test-project test-build v1.0.0 12345 12345-metadata --dry-run" \ "$TEST_DIR/test3_output.txt" echo "" @@ -260,9 +269,6 @@ test3_success=true # Generic files are not copied to structured_build_artifacts, so they won't be uploaded if ! verify_commands "All build info commands" "$TEST_DIR/test3_output.txt.build_commands" " -jf rt build-collect-env.*test-build.*12345 -jf rt build-add-git.*test-build.*12345 -jf rt build-add-dependencies.*test-build.*12345 jf rt build-publish.*test-build.*12345 "; then test3_success=false @@ -272,7 +278,7 @@ if ! verify_command_count "All upload commands" "$TEST_DIR/test3_output.txt.uplo test3_success=false fi -if ! verify_command_count "All build commands" "$TEST_DIR/test3_output.txt.build_commands" 4; then +if ! verify_command_count "All build commands" "$TEST_DIR/test3_output.txt.build_commands" 3; then test3_success=false fi @@ -290,6 +296,7 @@ if echo "$missing_project_output" | grep -q "Error: project is required"; then echo " Missing project error handled correctly" else echo " Missing project error not handled correctly" + echo "$missing_project_output" test4_success=false fi @@ -300,6 +307,7 @@ if echo "$missing_build_name_output" | grep -q "Error: build-name is required"; echo " Missing build-name error handled correctly" else echo " Missing build-name error not handled correctly" + echo "$missing_build_name_output" test4_success=false fi @@ -310,6 +318,7 @@ if echo "$missing_version_output" | grep -q "Error: version is required"; then echo " Missing version error handled correctly" else echo " Missing version error not handled correctly" + echo "$missing_version_output" test4_success=false fi @@ -320,16 +329,18 @@ if echo "$missing_build_number_output" | grep -q "Error: build-number is require echo " Missing build-number error handled correctly" else echo " Missing build-number error not handled correctly" + echo "$missing_build_number_output" test4_success=false fi echo " Testing invalid option..." # shellcheck disable=SC2015 -invalid_option_output=$(cd "$TEST_DIR" && "$SCRIPT_DIR/entrypoint.sh" --invalid-option test-project test-build v1.0.0 12345 2>&1 || true) +invalid_option_output=$(cd "$TEST_DIR" && "$SCRIPT_DIR/entrypoint.sh" --invalid-option test-project test-build v1.0.0 12345 12345-metadata 2>&1 || true) if echo "$invalid_option_output" | grep -q "Unknown option"; then echo " Invalid option error handled correctly" else echo " Invalid option error not handled correctly" + echo "$invalid_option_output" test4_success=false fi @@ -342,11 +353,12 @@ test5_success=true echo " Testing structured build artifacts creation..." # shellcheck disable=SC2015 -structured_output=$(cd "$TEST_DIR" && "$SCRIPT_DIR/entrypoint.sh" test-project test-build v1.0.0 12345 --dry-run 2>&1 || true) +structured_output=$(cd "$TEST_DIR" && "$SCRIPT_DIR/entrypoint.sh" test-project test-build v1.0.0 12345 12345-metadata --dry-run 2>&1 || true) if echo "$structured_output" | grep -q "Processing DEB:"; then echo " Structured build artifacts processing works correctly" else echo " Structured build artifacts processing not working correctly" + echo "$structured_output" test5_success=false fi @@ -356,6 +368,7 @@ if echo "$structured_output" | grep -q "Processing RPM:"; then echo " RPM processing works correctly" else echo " RPM processing not working correctly" + echo "$structured_output" test5_success=false fi diff --git a/.github/workflows/example_reusable-integration.yaml b/.github/workflows/example_reusable-integration.yaml index 8f1b513..d7bbd3e 100644 --- a/.github/workflows/example_reusable-integration.yaml +++ b/.github/workflows/example_reusable-integration.yaml @@ -6,13 +6,12 @@ permissions: id-token: write contents: read -# This workflow demonstrates the complete CI/CD pipeline using all four reusable +# This workflow demonstrates the complete CI/CD pipeline using all reusable # workflows from shared-workflows. It shows how to chain workflows together to -# create a production-ready artifact pipeline. -# See the complementary docs/CICD-with-shared-actions.md for more details. +# create a production-ready artifact pipeline with build-info aggregation. # # WORKFLOW FLOW: -# 1. extract-version: The examples need a version to work with and this provides one from a test file. +# 1. extract-version: Extract version from test file # 2. build-packages: Build packages across multiple platforms using matrix strategy (uses reusable_execute-build.yaml) # 3. package-built-artifacts: Package built artifacts into distributable formats (handles packaging in this example but could be done in the build step) # 4. sign-artifacts: GPG sign the packaged artifacts for security (uses reusable_sign-artifacts.yaml) @@ -21,9 +20,8 @@ permissions: # # # USAGE: -# This is an example workflow that consumers can copy and adapt for their own -# projects. It demonstrates best practices for integrating shared-workflows -# components into a complete CI/CD pipeline. +# This example demonstrates best practices for integrating shared-workflows +# components into a complete CI/CD pipeline with build-info aggregation. # ============================================================================= jobs: @@ -49,7 +47,7 @@ jobs: # a single workflow that can be instrumented in the future. # This is also an example of using a matrix strategy to build in GHA build-packages: - uses: ./.github/workflows/reusable_execute-build.yaml + uses: aerospike/shared-workflows/.github/workflows/reusable_execute-build.yaml@feature/INFRA-189-build-info-in-build-step needs: extract-version # Matrix strategy: Multi-platform builds across Ubuntu/Debian/RPM families (x86_64 native, ARM64 emulated) strategy: @@ -128,24 +126,27 @@ jobs: runs-on: ${{ matrix.runs-on }} project: test build-name: test-build - build-version: v1.0.0 + build-id: ${{ github.run_number }}-buildinfo-${{ matrix.distro }}-${{ matrix.arch }} checkout-path: project-location # In the case of running in shared_workflows checkout-path and source-path will have the same content but we put them here by way of example. source-path: repo-source + working-directory: repo-source/.github/workflows/execute-build/test_apps/hi build-script: | - cd repo-source/.github/workflows/execute-build/test_apps/hi uname -m uname -a which gcc + pwd + find . DISTRO=${{ matrix.distro }} \ ARCH=${{ matrix.arch }} \ EMULATED=${{ matrix.emulated }} \ make docker-build - artifact-directory: repo-source/.github/workflows/execute-build/test_apps/hi/build + artifact-directory: build artifact-name: build-artifacts-${{ matrix.distro }}-${{ matrix.arch }} retention-days: 1 # default artifactory-url: https://artifact.aerospike.io # default artifactory-oidc-provider-name: gh-dev-test artifactory-oidc-audience: aerospike/testing + publish-build-info: true dry-run: false # default # This job demonstrates custom packaging logic using standard GitHub Actions (actions/checkout, actions/download-artifact, actions/upload-artifact) # No reusable actions from shared-workflows are used here - this is custom packaging logic that could be done in the build step @@ -163,7 +164,6 @@ jobs: - name: Download All Matrix Artifacts uses: actions/download-artifact@v4 with: - pattern: build-artifacts-* path: build-artifacts merge-multiple: true - name: Package Artifacts @@ -179,11 +179,12 @@ jobs: retention-days: 1 sign-artifacts: needs: package-built-artifacts - uses: ./.github/workflows/reusable_sign-artifacts.yaml + uses: aerospike/shared-workflows/.github/workflows/reusable_sign-artifacts.yaml@feature/INFRA-189-build-info-in-build-step with: retention-days: 1 artifactory-oidc-provider-name: gh-dev-test artifactory-oidc-audience: aerospike/testing + artifact-name: packaged-artifacts secrets: gpg-private-key: ${{ secrets.GPG_SECRET_KEY }} gpg-public-key: ${{ secrets.GPG_PUBLIC_KEY }} @@ -191,10 +192,13 @@ jobs: deploy-artifacts: needs: [sign-artifacts, extract-version] - uses: ./.github/workflows/reusable_deploy-artifacts.yaml + uses: aerospike/shared-workflows/.github/workflows/reusable_deploy-artifacts.yaml@feature/INFRA-189-build-info-in-build-step with: project: test build-name: test-build + metadata-build-id: ${{ github.run_number }}-buildinfo + build-id: ${{ github.run_number }} + artifact-name: ${{ needs.sign-artifacts.outputs.artifact-name }} version: ${{ needs.extract-version.outputs.version }} retention-days: 1 artifactory-oidc-provider-name: gh-dev-test @@ -203,10 +207,10 @@ jobs: # create-release-bundle - Create a release bundle from deployed artifacts for distribution create-release-bundle: needs: [deploy-artifacts, extract-version] - uses: ./.github/workflows/reusable_create-release-bundle.yaml + uses: aerospike/shared-workflows/.github/workflows/reusable_create-release-bundle.yaml@feature/INFRA-189-build-info-in-build-step with: project: test - build-names: test-build:${{ needs.deploy-artifacts.outputs.build-id }} # for multiple builds it would be test-build-1:build_id1,test-build-2:build_id2" + build-names: test-build:${{ github.run_number }} # for multiple builds it would be test-build-1:build_id1,test-build-2:build_id2" bundle-name: test-release version: ${{ needs.extract-version.outputs.version }} artifactory-oidc-provider-name: gh-dev-test diff --git a/.github/workflows/execute-build/README.md b/.github/workflows/execute-build/README.md index 735bfc7..cc52ce2 100644 --- a/.github/workflows/execute-build/README.md +++ b/.github/workflows/execute-build/README.md @@ -12,12 +12,13 @@ This workflow executes a custom build script and uploads the resulting artifacts | -------------------------------- | -------------------------------------------------------------------- | -------- | ------------------------------- | | `project` | JFrog Artifactory project name | Yes | - | | `build-name` | Name for this build in artifactory | Yes | - | -| `build-version` | Version for this build in artifactory | Yes | - | +| `build-id` | Build ID for this build in artifactory | Yes | - | | `build-script` | Inline bash commands to execute | No\* | - | | `build-script-path` | Path to the build script file to execute | No\* | - | | `artifact-directory` | Directory that will contain all artifacts from this build | Yes | - | | `artifact-name` | Name for the uploaded artifacts | No | `build-artifacts` | | `retention-days` | Retention days for the artifacts | No | `1` | +| `working-directory` | Working directory for build script execution | No | - | | `artifactory-url` | JFrog Artifactory URL | No | `https://artifact.aerospike.io` | | `artifactory-oidc-provider-name` | OIDC provider name | No | `gh-aerospike` | | `artifactory-oidc-audience` | OIDC audience | No | `aerospike` | @@ -47,7 +48,7 @@ jobs: with: project: my-project build-name: my-app - build-version: v1.0.0 + build-id: 1234567890 build-script: make clean && make all && cp build/* dist/ artifact-directory: dist artifact-name: my-build-artifacts @@ -64,7 +65,7 @@ jobs: with: project: my-project build-name: my-app - build-version: v1.0.0 + build-id: 1234567890 build-script-path: ./scripts/build.sh artifact-directory: dist artifact-name: my-build-artifacts diff --git a/.github/workflows/execute-build/entrypoint.sh b/.github/workflows/execute-build/entrypoint.sh index 00dea87..d239812 100755 --- a/.github/workflows/execute-build/entrypoint.sh +++ b/.github/workflows/execute-build/entrypoint.sh @@ -25,7 +25,9 @@ error() { # Default values DRY_RUN="false" BUILD_NAME="" -BUILD_VERSION="" +BUILD_ID="" +PROJECT="" +PUBLISH_BUILD_INFO="false" show_help() { echo "Usage: $0 --artifact-directory (--build-script | --build-script-path ) [OPTIONS]" >&2 @@ -40,14 +42,16 @@ show_help() { echo " --build-script-path Path to build script file to execute" >&2 echo "" >&2 echo "Options:" >&2 - echo " --build-name Build name for JFrog upload" >&2 - echo " --build-version Build version for JFrog upload" >&2 + echo " --build-name Build name for JFrog build-info" >&2 + echo " --build-id Build ID for JFrog build-info" >&2 + echo " --project JFrog project for build-info" >&2 + echo " --publish-build-info Publish build-info to JFrog (without uploading artifacts)" >&2 echo " --dry-run Show what would be done without actually doing it" >&2 echo " --help, -h Show this help message" >&2 echo "" >&2 echo "Examples:" >&2 - echo " $0 --build-script-path ./scripts/build.sh --artifact-directory build-output --build-name my-app --build-version v1.0.0" >&2 - echo " $0 --build-script 'make all && cp build/* artifacts/' --artifact-directory artifacts --build-name my-app --build-version v1.0.0" >&2 + echo " $0 --build-script-path ./scripts/build.sh --artifact-directory build-output --build-name my-app --build-id 123" >&2 + echo " $0 --build-script 'make all && cp build/* artifacts/' --artifact-directory artifacts --build-name my-app --build-id 123" >&2 echo " $0 --build-script-path make.sh --artifact-directory artifacts --dry-run" >&2 } @@ -78,10 +82,18 @@ while [[ $# -gt 0 ]]; do BUILD_NAME="$2" shift 2 ;; - --build-version) - BUILD_VERSION="$2" + --build-id) + BUILD_ID="$2" shift 2 ;; + --project) + PROJECT="$2" + shift 2 + ;; + --publish-build-info) + PUBLISH_BUILD_INFO="true" + shift + ;; --dry-run) DRY_RUN="true" shift @@ -112,6 +124,19 @@ if [[ -z "${ARTIFACT_DIRECTORY:-}" ]]; then error "--artifact-directory is required. Use --help for usage information" fi +# Validate build-info parameters if publishing +if [[ "$PUBLISH_BUILD_INFO" == "true" ]]; then + if [[ -z "${BUILD_NAME:-}" ]]; then + error "--build-name is required when --publish-build-info is specified" + fi + if [[ -z "${BUILD_ID:-}" ]]; then + error "--build-id is required when --publish-build-info is specified" + fi + if [[ -z "${PROJECT:-}" ]]; then + error "--project is required when --publish-build-info is specified" + fi +fi + # Wrapper function that either executes or echoes commands run() { if [[ "$DRY_RUN" == "true" ]]; then @@ -122,6 +147,9 @@ run() { "$@" fi } +run_optional() { + run "$@" || echo "Warning: $*" >&2 +} main() { if [[ "$DRY_RUN" == "true" ]]; then @@ -134,23 +162,19 @@ main() { echo "Build script: $BUILD_SCRIPT" >&2 echo "Artifact directory: $ARTIFACT_DIRECTORY" >&2 echo "Build name: $BUILD_NAME" >&2 - echo "Build version: $BUILD_VERSION" >&2 + echo "Build ID: $BUILD_ID" >&2 echo "Dry run: $DRY_RUN" >&2 # Handle build script based on type local resolved_build_script if [[ "$BUILD_SCRIPT_TYPE" == "inline" ]]; then - # Create a temporary script file from inline commands - # is_inline_script=true - local temp_script="/tmp/build-script-$$.sh" echo "#!/bin/bash" > "$temp_script" echo "set -euo pipefail" >> "$temp_script" echo "$BUILD_SCRIPT" >> "$temp_script" chmod +x "$temp_script" echo "temp_script: $temp_script" - cat "$temp_script" resolved_build_script="$temp_script" echo "Created temporary script from inline commands: $temp_script" >&2 @@ -183,6 +207,18 @@ main() { echo " Would verify artifacts in: $ARTIFACT_DIRECTORY" >&2 fi + # Collect and publish build-info if requested + if [[ "$PUBLISH_BUILD_INFO" == "true" ]]; then + echo "Collecting build-info for $BUILD_NAME/$BUILD_ID..." >&2 + echo "Publishing from working directory: $(pwd)" >&2 + + run_optional jf rt build-collect-env "$BUILD_NAME" "$BUILD_ID" --project="$PROJECT" + run_optional jf rt build-add-git "$BUILD_NAME" "$BUILD_ID" --project="$PROJECT" + run jf rt build-publish "$BUILD_NAME" "$BUILD_ID" --project="$PROJECT" + + echo "Published build-info: $BUILD_NAME/$BUILD_ID" >&2 + fi + echo "Build-artifacts workflow completed successfully!" >&2 } diff --git a/.github/workflows/reusable_create-release-bundle.yaml b/.github/workflows/reusable_create-release-bundle.yaml index f8732c8..53f2d08 100644 --- a/.github/workflows/reusable_create-release-bundle.yaml +++ b/.github/workflows/reusable_create-release-bundle.yaml @@ -61,7 +61,7 @@ jobs: create-release-bundle: runs-on: ${{ inputs.runs-on }} env: - CHECKOUT_REF: 8fa7b46c484ec4330adc5c0b5916d1dfadda83de # Latest commit hash for development + CHECKOUT_REF: feature/INFRA-189-build-info-in-build-step # Latest commit hash for latest release steps: - name: Checkout shared-workflows repository uses: actions/checkout@v5 diff --git a/.github/workflows/reusable_deploy-artifacts.yaml b/.github/workflows/reusable_deploy-artifacts.yaml index 3198601..04c9c92 100644 --- a/.github/workflows/reusable_deploy-artifacts.yaml +++ b/.github/workflows/reusable_deploy-artifacts.yaml @@ -11,6 +11,16 @@ on: description: JFrog build name required: true type: string + build-id: + description: | + JFrog build ID for the overall build info. This will be the parent build info with sub build infos for build metadata and artifacts. + This should be a unique build ID for the overall build info and will be uploaded to JFrog as the parent build info. + required: true + type: string + metadata-build-id: + description: JFrog build ID for the build metadata + required: true + type: string version: description: Version of the artifact to upload required: true @@ -55,11 +65,6 @@ on: required: false type: boolean default: false - outputs: - build-id: - description: The generated build ID - value: ${{ jobs.deploy.outputs.build-id }} - permissions: contents: read id-token: write @@ -70,7 +75,7 @@ jobs: outputs: build-id: ${{ steps.deploy.outputs.build-id }} env: - CHECKOUT_REF: 8fa7b46c484ec4330adc5c0b5916d1dfadda83de # Latest commit hash for development + CHECKOUT_REF: feature/INFRA-189-build-info-in-build-step # Latest commit hash for latest release steps: - name: Checkout shared-workflows repository uses: actions/checkout@v5 @@ -108,11 +113,12 @@ jobs: if [ "${{ inputs.dry-run }}" = "true" ]; then dry_run_arg="--dry-run" fi - BUILD_ID=$(date +%s%3N) # timestamp in milliseconds (hex) + ${{ inputs.checkout-path }}/.github/workflows/deploy-artifacts/entrypoint.sh \ "${{ inputs.project }}" \ "${{ inputs.build-name }}" \ "${{ inputs.version }}" \ - "$BUILD_ID" \ + "${{ inputs.build-id }}" \ + "${{ inputs.metadata-build-id }}" \ $dry_run_arg - echo "build-id=$BUILD_ID" >> $GITHUB_OUTPUT + echo "build-id=${{ inputs.build-id }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/reusable_execute-build.yaml b/.github/workflows/reusable_execute-build.yaml index 0d72309..61b9490 100644 --- a/.github/workflows/reusable_execute-build.yaml +++ b/.github/workflows/reusable_execute-build.yaml @@ -11,8 +11,8 @@ on: description: Name for this build in artifactory required: true type: string - build-version: - description: Version for this build in artifactory + build-id: + description: Build ID for this build in artifactory required: true type: string build-script: @@ -84,6 +84,28 @@ on: required: false type: boolean default: false + publish-build-info: + description: | + Whether to publish build-info to JFrog. If true will publish build-info to JFrog without uploading artifacts. + required: false + type: boolean + default: false + working-directory: + description: Working directory for build script execution + required: false + type: string + default: . + + outputs: + build-name: + description: The build name used for JFrog build-info + value: ${{ jobs.execute-build.outputs.build-name }} + build-id: + description: The build ID used for JFrog build-info + value: ${{ jobs.execute-build.outputs.build-id }} + artifact-name: + description: The name of the uploaded artifacts on github + value: ${{ jobs.execute-build.outputs.artifact-name }} permissions: contents: read @@ -92,8 +114,12 @@ permissions: jobs: execute-build: runs-on: ${{ inputs.runs-on }} + outputs: + build-name: ${{ steps.build-info.outputs.build-name }} + build-id: ${{ steps.build-info.outputs.build-id }} + artifact-name: ${{ steps.build-info.outputs.artifact-name }} env: - CHECKOUT_REF: 8fa7b46c484ec4330adc5c0b5916d1dfadda83de # Latest commit hash for development + CHECKOUT_REF: feature/INFRA-189-build-info-in-build-step # Latest commit hash for latest release steps: - name: Checkout shared-workflows repository uses: actions/checkout@v5 @@ -110,6 +136,8 @@ jobs: repository: ${{ inputs.source-repository }} ref: ${{ inputs.source-ref }} path: ${{ inputs.source-path }} + fetch-tags: true + submodules: recursive fetch-depth: 1 - name: Set up JFrog CLI @@ -123,38 +151,46 @@ jobs: - name: Execute Build Script shell: bash - working-directory: ${{ github.workspace }} run: | - chmod +x ${{ inputs.checkout-path }}/.github/workflows/execute-build/entrypoint.sh - - dry_run_arg="" - if [ "${{ inputs.dry-run }}" = "true" ]; then - dry_run_arg="--dry-run" + entrypoint=$(realpath ${{ inputs.checkout-path }}/.github/workflows/execute-build/entrypoint.sh) + chmod +x "$entrypoint" + if [[ -n "${{ inputs.working-directory }}" ]]; then + cd "${{ inputs.working-directory }}" fi + echo "Working directory: $(pwd)" + echo "Entrypoint script: $entrypoint" - if ${{ inputs.build-script != '' }}; then - ${{ inputs.checkout-path }}/.github/workflows/execute-build/entrypoint.sh \ - --build-script '${{ inputs.build-script }}' \ - --artifact-directory "${{ inputs.artifact-directory }}" \ - --build-name "${{ inputs.build-name }}" \ - --build-version "${{ inputs.build-version }}" \ - $dry_run_arg - elif ${{ inputs.build-script-path != '' }}; then - ${{ inputs.checkout-path }}/.github/workflows/execute-build/entrypoint.sh \ - --build-script-path "${{ inputs.build-script-path }}" \ - --artifact-directory "${{ inputs.artifact-directory }}" \ - --build-name "${{ inputs.build-name }}" \ - --build-version "${{ inputs.build-version }}" \ - $dry_run_arg + if ${{ inputs.build-script != '' }}; then # Inline build script + script_arg="--build-script" + script_value='${{ inputs.build-script }}' + elif ${{ inputs.build-script-path != '' }}; then # Build script file + script_arg="--build-script-path" + script_value="${{ inputs.build-script-path }}" else echo "Error: Either build-script or build-script-path must be given" exit 1 fi + "$entrypoint" \ + "$script_arg" "$script_value" \ + --artifact-directory "${{ inputs.artifact-directory }}" \ + --build-name "${{ inputs.build-name }}" \ + --build-id "${{ inputs.build-id }}" \ + --project "${{ inputs.project }}" \ + ${{ inputs.publish-build-info && '--publish-build-info' || '' }} \ + ${{ inputs.dry-run && '--dry-run' || '' }} - name: Upload Artifacts if: inputs.dry-run != true uses: actions/upload-artifact@v4 with: name: ${{ inputs.artifact-name }} - path: ${{ inputs.artifact-directory }} + path: ${{ inputs.working-directory != '' && format('{0}/{1}', inputs.working-directory, inputs.artifact-directory) || inputs.artifact-directory }} retention-days: ${{ inputs.retention-days }} + + - name: Set Build Info Outputs + if: inputs.publish-build-info == true + id: build-info + run: | + echo "build-name=${{ inputs.build-name }}" >> $GITHUB_OUTPUT + echo "build-id=${{ inputs.build-id }}" >> $GITHUB_OUTPUT + echo "artifact-name=${{ inputs.artifact-name }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/reusable_sign-artifacts.yaml b/.github/workflows/reusable_sign-artifacts.yaml index 7840459..0e2b10e 100644 --- a/.github/workflows/reusable_sign-artifacts.yaml +++ b/.github/workflows/reusable_sign-artifacts.yaml @@ -52,14 +52,20 @@ on: required: true gpg-key-pass: required: true + outputs: + artifact-name: + description: The name of the uploaded signed artifacts on github + value: ${{ jobs.sign.outputs.artifact-name }} permissions: contents: read id-token: write jobs: sign: runs-on: ${{ inputs.runs-on }} + outputs: + artifact-name: ${{ steps.sign-outputs.outputs.artifact-name }} env: - CHECKOUT_REF: 8fa7b46c484ec4330adc5c0b5916d1dfadda83de # Latest commit hash for development + CHECKOUT_REF: feature/INFRA-189-build-info-in-build-step # Latest commit hash for latest release steps: - name: Download Unsigned Artifacts uses: actions/download-artifact@v4 @@ -96,3 +102,7 @@ jobs: path: ${{ inputs.artifact-name }} overwrite: true retention-days: ${{ inputs.retention-days }} + - name: Set Outputs + id: sign-outputs + run: | + echo "artifact-name=${{ inputs.artifact-name }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test_deploy-artifacts-workflow.yaml b/.github/workflows/test_deploy-artifacts-workflow.yaml index ef9d1de..6ccbfd4 100644 --- a/.github/workflows/test_deploy-artifacts-workflow.yaml +++ b/.github/workflows/test_deploy-artifacts-workflow.yaml @@ -29,6 +29,8 @@ jobs: project: test build-name: test-upload-workflow version: v1.0.0 + build-id: ${{ github.run_number }} + metadata-build-id: ${{ github.run_number }}-metadata artifactory-url: https://artifact.aerospike.io artifactory-oidc-provider-name: gh-aerospike artifactory-oidc-audience: aerospike diff --git a/.github/workflows/test_execute-build-workflow.yaml b/.github/workflows/test_execute-build-workflow.yaml index 89f9b63..5318968 100644 --- a/.github/workflows/test_execute-build-workflow.yaml +++ b/.github/workflows/test_execute-build-workflow.yaml @@ -14,9 +14,9 @@ jobs: with: project: test build-name: test-build - build-version: v1.0.0 + build-id: v1.0.0 build-script: | - echo "checking that current-source is correctly present and building" + echo "Need to ensure current-source is correctly present and building" cat current-source/README.md &&\ cd project-location/.github/workflows/execute-build/test_apps/hi && make checkout-path: project-location