diff --git a/.github/workflows/deploy-artifacts/create-test-fixtures.sh b/.github/workflows/deploy-artifacts/create-test-fixtures.sh index fd49973..5e954fb 100755 --- a/.github/workflows/deploy-artifacts/create-test-fixtures.sh +++ b/.github/workflows/deploy-artifacts/create-test-fixtures.sh @@ -18,24 +18,36 @@ if [[ -f "tests/nano-tiny_8.4-1_arm64.deb" ]]; then cp "tests/nano-tiny_8.4-1_arm64.deb" "$BUILD_ARTIFACTS_DIR/test-ubuntu22.04.deb" echo " Copied nano-tiny_8.4-1_arm64.deb as test-ubuntu22.04.deb" else - echo " nano-tiny_8.4-1_arm64.deb not found, creating mock" - echo "test-deb-content" > "$BUILD_ARTIFACTS_DIR/test-ubuntu22.04.deb" + echo "Error: tests/nano-tiny_8.4-1_arm64.deb not found. Cannot create mock DEB file." >&2 + exit 1 fi if [[ -f "tests/test-1.0-2.noarch.rpm" ]]; then cp "tests/test-1.0-2.noarch.rpm" "$BUILD_ARTIFACTS_DIR/" echo " Copied test-1.0-2.noarch.rpm" else - echo " test-1.0-2.noarch.rpm not found, creating mock" - echo "test-rpm-content" > "$BUILD_ARTIFACTS_DIR/test-1.0-2.noarch.rpm" + echo "Error: tests/test-1.0-2.noarch.rpm not found. Cannot create mock RPM file." >&2 + exit 1 fi cp -v tests/some/structure/Aerospike.Client.8.0.2.nupkg "$BUILD_ARTIFACTS_DIR/Aerospike.Client.8.0.2.nupkg" -# Create some additional test files -echo "test jar content" > "$BUILD_ARTIFACTS_DIR/test.jar" -echo "test zip content" > "$BUILD_ARTIFACTS_DIR/test.zip" -echo "test tar content" > "$BUILD_ARTIFACTS_DIR/test.tar.gz" +# Create some additional test files with valid formats +# Create a valid JAR file (JAR is a ZIP with META-INF/MANIFEST.MF) +mkdir -p "$BUILD_ARTIFACTS_DIR/temp-jar/META-INF" +echo "Manifest-Version: 1.0" > "$BUILD_ARTIFACTS_DIR/temp-jar/META-INF/MANIFEST.MF" +cd "$BUILD_ARTIFACTS_DIR/temp-jar" && zip -q -r "../test.jar" . && cd - > /dev/null +rm -rf "$BUILD_ARTIFACTS_DIR/temp-jar" + +# Create a valid ZIP file +echo "test zip content" > "$BUILD_ARTIFACTS_DIR/temp-zip-content.txt" +cd "$BUILD_ARTIFACTS_DIR" && zip -q "test.zip" "temp-zip-content.txt" && cd - > /dev/null +rm -f "$BUILD_ARTIFACTS_DIR/temp-zip-content.txt" + +# Create a valid TAR.GZ file +echo "test tar content" > "$BUILD_ARTIFACTS_DIR/temp-tar-content.txt" +cd "$BUILD_ARTIFACTS_DIR" && tar -czf "test.tar.gz" "temp-tar-content.txt" && cd - > /dev/null +rm -f "$BUILD_ARTIFACTS_DIR/temp-tar-content.txt" mkdir -p "$BUILD_ARTIFACTS_DIR/nested/dir" echo "test-nested-dir-content" > "$BUILD_ARTIFACTS_DIR/nested/dir/test-nested-dir.txt" @@ -45,13 +57,15 @@ if [[ -f "tests/nano-tiny_8.4-1_arm64.deb" ]]; then cp "tests/nano-tiny_8.4-1_arm64.deb" "$BUILD_ARTIFACTS_DIR/nested/dir/test-debian12.deb" echo " Copied nano-tiny_8.4-1_arm64.deb as nested test-debian12.deb" else - echo "test-nested-deb-content" > "$BUILD_ARTIFACTS_DIR/nested/dir/test-debian12.deb" + echo "Error: tests/nano-tiny_8.4-1_arm64.deb not found. This file is required for nested test fixtures." >&2 + exit 1 fi if [[ -f "tests/test-1.0-2.noarch.rpm" ]]; then cp "tests/test-1.0-2.noarch.rpm" "$BUILD_ARTIFACTS_DIR/nested/dir/nested.rpm" echo " Copied test-1.0-2.noarch.rpm as nested.rpm" else - echo "test-nested-rpm-content" > "$BUILD_ARTIFACTS_DIR/nested/dir/nested.rpm" + echo "Error: tests/test-1.0-2.noarch.rpm not found. This file is required for nested test fixtures." >&2 + exit 1 fi echo "Test files created:" diff --git a/.github/workflows/deploy-artifacts/entrypoint.sh b/.github/workflows/deploy-artifacts/entrypoint.sh index 07b3628..79f3dda 100755 --- a/.github/workflows/deploy-artifacts/entrypoint.sh +++ b/.github/workflows/deploy-artifacts/entrypoint.sh @@ -4,19 +4,19 @@ export PS4='+($LINENO): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' trap 'handle_error ${LINENO}' ERR # shellcheck disable=SC2317 handle_error() { - local exit_code=$? - local line_number=$1 - echo "Error: Command failed with exit code $exit_code at line $line_number" >&2 - exit 1 + local exit_code=$? + local line_number=$1 + echo "Error: Command failed with exit code $exit_code at line $line_number" >&2 + exit 1 } error() { - local reason="${1:-}" - if [[ -n "$reason" ]]; then - echo "Error: $reason" >&2 - else - echo "Error" >&2 - fi - exit 1 + local reason="${1-}" + if [[ -n $reason ]]; then + echo "Error: $reason" >&2 + else + echo "Error" >&2 + fi + exit 1 } # Default values @@ -25,74 +25,80 @@ DRY_RUN="false" echo "Command line: $0 $*" >&2 # Parse command line arguments while [[ $# -gt 0 ]]; do - case $1 in - --dry-run) - DRY_RUN="true" - shift - ;; - --help|-h) - echo "Usage: $0 [OPTIONS]" >&2 - echo "" >&2 - echo "Uploads artifacts to JFrog Artifactory" >&2 - echo "" >&2 - echo "Options:" >&2 - echo " --metadata-build-number Build ID prefix used to discover related metadata builds (searches for *.json)" >&2 - echo " --dry-run Show what would be uploaded without actually uploading" >&2 - echo " --help, -h Show this help message" >&2 - echo "" >&2 - echo "Examples:" >&2 - echo " $0 database my-app v1.0.0 1754566442238 1754566442238-metadata" >&2 - echo " $0 database my-app v1.0.0 1754566442238 1754566442238-metadata --dry-run" >&2 - exit 0 - ;; - -*) - echo "Unknown option: $1" >&2 - echo "Use --help for usage information" >&2 - exit 1 - ;; - *) - # Positional arguments - if [[ -z "${PROJECT:-}" ]]; then - PROJECT="$1" - elif [[ -z "${BUILD_NAME:-}" ]]; then - BUILD_NAME="$1" - elif [[ -z "${VERSION:-}" ]]; then - VERSION="$1" - elif [[ -z "${BUILD_NUMBER:-}" ]]; then - BUILD_NUMBER="$1" - elif [[ -z "${METADATA_BUILD_NUMBER:-}" ]]; then - METADATA_BUILD_NUMBER="$1" - else - echo "Use --help for usage information" >&2 - exit 1 - fi - shift - ;; - esac + case $1 in + --dry-run) + DRY_RUN="true" + shift + ;; + --jar-group-id) + JAR_GROUP_ID="$2" + shift 2 + ;; + --help | -h) + echo "Usage: $0 [OPTIONS]" >&2 + echo "" >&2 + echo "Uploads artifacts to JFrog Artifactory" >&2 + echo "" >&2 + echo "Options:" >&2 + echo " --metadata-build-number Build ID prefix used to discover related metadata builds (searches for *.json)" >&2 + echo " --jar-group-id Maven group ID for JAR artifacts" >&2 + echo " --dry-run Show what would be uploaded without actually uploading" >&2 + echo " --help, -h Show this help message" >&2 + echo "" >&2 + echo "Examples:" >&2 + echo " $0 database my-app v1.0.0 1754566442238 1754566442238-metadata" >&2 + echo " $0 database my-app v1.0.0 1754566442238 1754566442238-metadata --dry-run" >&2 + echo " $0 database my-app v1.0.0 1754566442238 1754566442238-metadata --jar-group-id com.aerospike.test" >&2 + exit 0 + ;; + -*) + echo "Unknown option: $1" >&2 + echo "Use --help for usage information" >&2 + exit 1 + ;; + *) + # Positional arguments + if [[ -z ${PROJECT-} ]]; then + PROJECT="$1" + elif [[ -z ${BUILD_NAME-} ]]; then + BUILD_NAME="$1" + elif [[ -z ${VERSION-} ]]; then + VERSION="$1" + elif [[ -z ${BUILD_NUMBER-} ]]; then + BUILD_NUMBER="$1" + elif [[ -z ${METADATA_BUILD_NUMBER-} ]]; then + METADATA_BUILD_NUMBER="$1" + else + echo "Use --help for usage information" >&2 + exit 1 + fi + shift + ;; + esac done -if [[ -z "${PROJECT:-}" ]]; then - error "project is required +if [[ -z ${PROJECT-} ]]; then + error "project is required Use --help for usage information" fi -if [[ -z "${BUILD_NAME:-}" ]]; then - error "build-name is required +if [[ -z ${BUILD_NAME-} ]]; then + error "build-name is required Use --help for usage information" fi -if [[ -z "${VERSION:-}" ]]; then - error "version is required +if [[ -z ${VERSION-} ]]; then + error "version is required Use --help for usage information" fi -if [[ -z "${BUILD_NUMBER:-}" ]]; then - error "build-number is required +if [[ -z ${BUILD_NUMBER-} ]]; then + error "build-number is required Use --help for usage information" fi -if [[ -z "${METADATA_BUILD_NUMBER:-}" ]]; then - error "metadata-build-number is required +if [[ -z ${METADATA_BUILD_NUMBER-} ]]; then + error "metadata-build-number is required Use --help for usage information" fi ARTIFACT_BUILD_NUMBER="$BUILD_NUMBER-artifacts" @@ -102,183 +108,288 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/package_utils.sh" - # Wrapper function that either executes or echoes commands run() { - if [[ "$DRY_RUN" == "true" ]]; then - # Define color variables - local green='\033[0;32m' - local reset='\033[0m' - - echo -e "${green} $*${reset}" >&2 - else - "$@" - fi + if [[ $DRY_RUN == "true" ]]; then + # Define color variables + local green='\033[0;32m' + local reset='\033[0m' + + echo -e "${green} $*${reset}" >&2 + else + "$@" + fi } run_optional() { - run "$@" || echo "Warning: $*" >&2 + run "$@" || echo "Warning: $*" >&2 } structure_build_artifacts() { - echo "Structuring build artifacts..." >&2 - echo "current files: $(ls -la build-artifacts)" >&2 - mkdir -p structured_build_artifacts/deb - mkdir -p structured_build_artifacts/rpm - mkdir -p structured_build_artifacts/nupkg - mkdir -p structured_build_artifacts/generic - while IFS= read -r -d '' deb; do - if [[ ! -f "$deb" ]]; then - continue - fi - - echo "Processing DEB: $deb" >&2 - process_deb "$deb" "./structured_build_artifacts/deb" - done < <(find build-artifacts -name "*.deb" -print0) - - while IFS= read -r -d '' rpm; do - if [[ ! -f "$rpm" ]]; then - continue - fi - - echo "Processing RPM: $rpm" >&2 - process_rpm "$rpm" "./structured_build_artifacts/rpm" - done < <(find build-artifacts -name "*.rpm" -print0) - echo "current files: $(ls -la build-artifacts)" >&2 - - while IFS= read -r -d '' nupkg; do - if [[ ! -f "$nupkg" ]]; then - continue - fi - - echo "Processing NUPKG: $nupkg" >&2 - process_nupkg "$nupkg" "./structured_build_artifacts/nupkg" - done < <(find build-artifacts -name "*.nupkg" -print0) - echo "current files: $(ls -la build-artifacts)" >&2 - - while IFS= read -r -d '' generic; do - if [[ ! -f "$generic" ]]; then - echo "Skipping non-file: $generic" >&2 - continue - fi - echo "Processing generic file: $generic" >&2 - process_generic "$generic" "./structured_build_artifacts/generic" - done < <(find build-artifacts \( -not -name "*.deb" -not -name "*.rpm" -not -name "*.asc" \) -type f -print0) + echo "Structuring build artifacts..." >&2 + mkdir -p structured_build_artifacts/deb + mkdir -p structured_build_artifacts/rpm + mkdir -p structured_build_artifacts/jar + mkdir -p structured_build_artifacts/nupkg + mkdir -p structured_build_artifacts/generic + while IFS= read -r -d '' deb; do + if [[ ! -f $deb ]]; then + continue + fi + + echo "Processing DEB: $deb" >&2 + process_deb "$deb" "./structured_build_artifacts/deb" + done < <(find build-artifacts -name "*.deb" -print0) + + while IFS= read -r -d '' rpm; do + if [[ ! -f $rpm ]]; then + continue + fi + + echo "Processing RPM: $rpm" >&2 + process_rpm "$rpm" "./structured_build_artifacts/rpm" + done < <(find build-artifacts -name "*.rpm" -print0) + + while IFS= read -r -d '' jar; do + if [[ ! -f $jar ]]; then + continue + fi + + echo "Processing JAR: $jar" >&2 + process_jar "$jar" "./structured_build_artifacts/jar" + done < <(find build-artifacts -name "*.jar" -print0) + + while IFS= read -r -d '' nupkg; do + if [[ ! -f $nupkg ]]; then + continue + fi + + echo "Processing NUPKG: $nupkg" >&2 + process_nupkg "$nupkg" "./structured_build_artifacts/nupkg" + done < <(find build-artifacts -name "*.nupkg" -print0) + + while IFS= read -r -d '' generic; do + if [[ ! -f $generic ]]; then + echo "Skipping non-file: $generic" >&2 + continue + fi + echo "Processing generic file: $generic" >&2 + process_generic "$generic" "./structured_build_artifacts/generic" + done < <(find build-artifacts \( -not -name "*.deb" -not -name "*.rpm" -not -name "*.asc" -not -name "*.jar" -not -name "*.pom" -not -name "*.nupkg" \) -type f -print0) } upload_deb_packages() { - if [[ "$DRY_RUN" == "true" ]]; then - echo "Would upload DEB packages to JFrog..." >&2 - else - echo "Uploading DEB packages to JFrog..." >&2 - fi - while IFS= read -r -d '' deb; do - if [[ ! -f "$deb" ]]; then - continue - fi - # Get package metadata - pkgname=$(dpkg-deb -f "$deb" Package) - arch=$(dpkg-deb -f "$deb" Architecture) - if ! codename=$(get_codename_for_deb "$deb"); then - error "Failed to get codename for $deb" - fi - - echo " Package: $pkgname, Arch: $arch, Codename: $codename" >&2 - # Upload the DEB - - run jf rt upload "$deb" "$PROJECT-deb-dev-local" --flat=false \ - --build-name="$BUILD_NAME" \ - --build-number="$ARTIFACT_BUILD_NUMBER" \ - --project="$PROJECT" \ - --target-props "version=$VERSION;deb.distribution=$codename;deb.component=main;deb.architecture=$arch" \ - --deb "$codename/main/$arch" - - # Upload signature and checksum if they exist - if [[ -f "$deb.asc" ]]; then - echo " Uploading signature: $deb.asc" >&2 - run jf rt upload "$deb.asc" "$PROJECT-deb-dev-local" --flat=false \ - --build-name="$BUILD_NAME" \ - --build-number="$ARTIFACT_BUILD_NUMBER" \ - --project="$PROJECT" - fi - done < <(find . -name "*.deb" -print0) + if [[ $DRY_RUN == "true" ]]; then + echo "Would upload DEB packages to JFrog..." >&2 + else + echo "Uploading DEB packages to JFrog..." >&2 + fi + while IFS= read -r -d '' deb; do + if [[ ! -f $deb ]]; then + continue + fi + # Get package metadata + pkgname=$(dpkg-deb -f "$deb" Package) + arch=$(dpkg-deb -f "$deb" Architecture) + if ! codename=$(get_codename_for_deb "$deb"); then + error "Failed to get codename for $deb" + fi + + echo " Package: $pkgname, Arch: $arch, Codename: $codename" >&2 + # Upload the DEB + + run jf rt upload "$deb" "$PROJECT-deb-dev-local" --flat=false \ + --build-name="$BUILD_NAME" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ + --project="$PROJECT" \ + --target-props "version=$VERSION;deb.distribution=$codename;deb.component=main;deb.architecture=$arch" \ + --deb "$codename/main/$arch" + + # Upload signature and checksum if they exist + if [[ -f "$deb.asc" ]]; then + echo " Uploading signature: $deb.asc" >&2 + run jf rt upload "$deb.asc" "$PROJECT-deb-dev-local" --flat=false \ + --build-name="$BUILD_NAME" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ + --project="$PROJECT" + fi + done < <(find . -name "*.deb" -print0) } upload_rpm_packages() { - - if [[ "$DRY_RUN" == "true" ]]; then - echo "Would upload RPM packages to JFrog..." >&2 - else - echo "Uploading RPM packages to JFrog..." >&2 - fi - while IFS= read -r -d '' rpm; do - if [[ ! -f "$rpm" ]]; then - continue - fi - - # Get metadata using the shared function - read -r -a metadata < <(get_rpm_metadata "$rpm") - pkgname="${metadata[0]}" - version="${metadata[1]}" - arch="${metadata[2]}" - dist="${metadata[3]}" - - echo " Package: $pkgname, Version: $version, Arch: $arch, Dist: $dist" >&2 - - # Upload the RPM - run jf rt upload "$rpm" "$PROJECT-rpm-dev-local" --flat=false \ - --build-name="$BUILD_NAME" \ - --build-number="$ARTIFACT_BUILD_NUMBER" \ - --project="$PROJECT" \ - --target-props "version=$VERSION;rpm.distribution=$dist;rpm.component=main;rpm.architecture=$arch" - - # Upload signature and checksums if they exist - if [[ -f "$rpm.asc" ]]; then - echo " Uploading signature: $rpm.asc" >&2 - run jf rt upload "$rpm.asc" "$PROJECT-rpm-dev-local" --flat=false \ - --build-name="$BUILD_NAME" \ - --build-number="$ARTIFACT_BUILD_NUMBER" \ - --project="$PROJECT" - fi - done < <(find . -name "*.rpm" -print0) + if [[ $DRY_RUN == "true" ]]; then + echo "Would upload RPM packages to JFrog..." >&2 + else + echo "Uploading RPM packages to JFrog..." >&2 + fi + while IFS= read -r -d '' rpm; do + if [[ ! -f $rpm ]]; then + continue + fi + + # Get metadata using the shared function + read -r -a metadata < <(get_rpm_metadata "$rpm") + pkgname="${metadata[0]}" + version="${metadata[1]}" + arch="${metadata[2]}" + dist="${metadata[3]}" + + echo " Package: $pkgname, Version: $version, Arch: $arch, Dist: $dist" >&2 + + # Upload the RPM + run jf rt upload "$rpm" "$PROJECT-rpm-dev-local" --flat=false \ + --build-name="$BUILD_NAME" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ + --project="$PROJECT" \ + --target-props "version=$VERSION;rpm.distribution=$dist;rpm.component=main;rpm.architecture=$arch" + + # Upload signature and checksums if they exist + if [[ -f "$rpm.asc" ]]; then + echo " Uploading signature: $rpm.asc" >&2 + run jf rt upload "$rpm.asc" "$PROJECT-rpm-dev-local" --flat=false \ + --build-name="$BUILD_NAME" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ + --project="$PROJECT" + fi + done < <(find . -name "*.rpm" -print0) } -upload_generic_files() { - if [[ "$DRY_RUN" == "true" ]]; then - echo "Would upload generic files..." >&2 - else - echo "Uploading generic files..." >&2 - fi - echo "Finding files..." >&2 - find . >&2 - while IFS= read -r -d '' file; do - echo "Processing generic file: $file" >&2 - if [[ -f "$file" ]]; then - echo "Uploading generic file: $file" >&2 - run jf rt upload "$file" "$PROJECT-generic-dev-local" --flat=false \ - --build-name="$BUILD_NAME" \ - --build-number="$ARTIFACT_BUILD_NUMBER" \ - --project="$PROJECT" - fi - done < <(find . \( -not -name "*.deb" -not -name "*.rpm" -not -name "*.asc" \) -print0) +upload_jar_packages() { + if [[ $DRY_RUN == "true" ]]; then + echo "Would upload JAR/POM files to JFrog..." >&2 + else + echo "Uploading JAR/POM files to JFrog..." >&2 + fi + + # Find all JAR and POM files, then process unique base names + declare -A processed_artifacts + + while IFS= read -r -d '' artifact; do + if [[ ! -f $artifact ]]; then + continue + fi + + # Get the directory and base name + artifact_dir=$(dirname "$artifact") + artifact_name=$(basename "$artifact") + base_name="${artifact_name%.jar}" + base_name="${base_name%.pom}" + + # Skip if we've already processed this base artifact + artifact_key="$artifact_dir/$base_name" + if [[ -n ${processed_artifacts[$artifact_key]-} ]]; then + continue + fi + processed_artifacts[$artifact_key]=1 + + # Determine which file to use for metadata extraction (prefer POM as it's the source of truth) + local pkgname version group_id + + # Fallback: Extract metadata from JAR if no POM exists + read -r -a metadata < <(get_jar_metadata "$artifact_dir/${base_name}.jar") + pkgname="${metadata[0]}" + version="${metadata[1]}" + + # Group ID priority: metadata > flag > empty + # If metadata has group_id, use it; otherwise use JAR_GROUP_ID if provided via flag + if [[ -n ${metadata[2]-} ]]; then + group_id="${metadata[2]}" + else + group_id="${JAR_GROUP_ID-}" + fi + + # If no group_id is available, move to generic directory for generic upload + if [[ -z $group_id ]]; then + echo " Moving JAR without group_id to generic directory: $artifact" >&2 + for ext in jar pom jar.asc pom.asc; do + local artifact_file="$artifact_dir/${base_name}.${ext}" + if [[ -f $artifact_file ]]; then + mv "$artifact_file" "../generic/" + fi + done + continue + fi + + echo " Package: $pkgname, Version: $version, Group ID: $group_id" >&2 + + # Upload all related Maven artifact files (jar, pom, signatures) + for ext in jar pom jar.asc pom.asc; do + local artifact_file="$artifact_dir/${base_name}.${ext}" + if [[ -f $artifact_file ]]; then + echo " Uploading $ext: $artifact_file" >&2 + run jf rt upload "$artifact_file" "$PROJECT-maven-dev-local" --flat=false \ + --build-name="$BUILD_NAME" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ + --project="$PROJECT" \ + --target-props "group_id=$group_id;package_name=$pkgname;version=$version" + fi + done + done < <(find . \( -name "*.jar" -o -name "*.pom" \) -print0) +} + +upload_nupkg_packages() { + if [[ $DRY_RUN == "true" ]]; then + echo "Would upload NuGet packages to JFrog..." >&2 + else + echo "Uploading NuGet packages to JFrog..." >&2 + fi + while IFS= read -r -d '' nupkg; do + if [[ ! -f $nupkg ]]; then + continue + fi + + echo " Uploading NuGet package: $nupkg" >&2 + run jf rt upload "$nupkg" "$PROJECT-nuget-dev-local" --flat=false \ + --build-name="$BUILD_NAME" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ + --project="$PROJECT" + + # Upload signature if it exists + if [[ -f "$nupkg.asc" ]]; then + echo " Uploading signature: $nupkg.asc" >&2 + run jf rt upload "$nupkg.asc" "$PROJECT-nuget-dev-local" --flat=false \ + --build-name="$BUILD_NAME" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ + --project="$PROJECT" + fi + done < <(find . -name "*.nupkg" -print0) } +upload_generic_files() { + if [[ $DRY_RUN == "true" ]]; then + echo "Would upload generic files..." >&2 + else + echo "Uploading generic files..." >&2 + fi + echo "Finding files..." >&2 + find . >&2 + while IFS= read -r -d '' file; do + echo "Processing generic file: $file" >&2 + if [[ -f $file ]]; then + echo "Uploading generic file: $file" >&2 + run jf rt upload "$file" "$PROJECT-generic-dev-local" --flat=false \ + --build-name="$BUILD_NAME" \ + --build-number="$ARTIFACT_BUILD_NUMBER" \ + --project="$PROJECT" + fi + done < <(find . \( -not -name "*.deb" -not -name "*.rpm" -not -name "*.asc" -not -name "*.nupkg" \) -print0) +} # Collects build-info metadata by querying Artifactory for JSON files. # JFrog API doesn't support wildcarding build IDs, so we fetch the JSON # files to extract the collection of build IDs for this parent build. discover_build_infos() { - local project="$1" - local parent_build_name="$2" - local build_info_search_pattern="$3" - local build_info_repo="${4:-${project}-build-info}" + local project="$1" + local parent_build_name="$2" + local build_info_search_pattern="$3" + local build_info_repo="${4:-${project}-build-info}" - echo "Discovering build-infos for project: $project" >&2 - echo "Parent build name: $parent_build_name" >&2 - echo "Search pattern: $build_info_search_pattern" >&2 - echo "Build-info repo: $build_info_repo" >&2 + echo "Discovering build-infos for project: $project" >&2 + echo "Parent build name: $parent_build_name" >&2 + echo "Search pattern: $build_info_search_pattern" >&2 + echo "Build-info repo: $build_info_repo" >&2 - read -r -d '' AQL <&2 - # Query for build-info JSON files - local resp - resp=$(run jf rt curl -XPOST api/search/aql \ - -H 'Content-Type: text/plain' \ - -d "$AQL") - echo "search response: $resp" >&2 - # Extract child build IDs from JSON file names - if echo "$resp" | jq -e '.results' > /dev/null 2>&1; then - mapfile -t CHILD_BUILD_IDS < <(echo "$resp" | jq -r '.results[] | .name' | sed 's/-[0-9]*\.json$//' | sort -u) - - echo "Found ${#CHILD_BUILD_IDS[@]} child builds" >&2 - - CHILD_BUILD_IDS_RESULT=("${CHILD_BUILD_IDS[@]}") - return 0 - else - echo "No build-info files found" >&2 - CHILD_BUILD_IDS_RESULT=() - return 1 - fi + echo run jf rt curl -XPOST api/search/aql \ + -H 'Content-Type: text/plain' \ + -d "$AQL" >&2 + # Query for build-info JSON files + local resp + resp=$(run jf rt curl -XPOST api/search/aql \ + -H 'Content-Type: text/plain' \ + -d "$AQL") + echo "search response: $resp" >&2 + # Extract child build IDs from JSON file names + if echo "$resp" | jq -e '.results' >/dev/null 2>&1; then + mapfile -t CHILD_BUILD_IDS < <(echo "$resp" | jq -r '.results[] | .name' | sed 's/-[0-9]*\.json$//' | sort -u) + + echo "Found ${#CHILD_BUILD_IDS[@]} child builds" >&2 + + CHILD_BUILD_IDS_RESULT=("${CHILD_BUILD_IDS[@]}") + return 0 + else + echo "No build-info files found" >&2 + CHILD_BUILD_IDS_RESULT=() + return 1 + fi } - # This function handles publishing the tree of build info to jfrog # 1. publish the signed artifacts # 2. append metadata and artifact builld infos to new build info # 3. publish the new build info publish_build_info() { - run jf rt build-publish "$BUILD_NAME" "$ARTIFACT_BUILD_NUMBER" --project="$PROJECT" + run jf rt build-publish "$BUILD_NAME" "$ARTIFACT_BUILD_NUMBER" --project="$PROJECT" - discover_build_infos "$PROJECT" "$BUILD_NAME" "$METADATA_BUILD_NUMBER*" "$PROJECT-build-info" || true + discover_build_infos "$PROJECT" "$BUILD_NAME" "$METADATA_BUILD_NUMBER*" "$PROJECT-build-info" || true - # Access the results - if [[ ${#CHILD_BUILD_IDS_RESULT[@]} -gt 0 ]]; then - echo "Found ${#CHILD_BUILD_IDS_RESULT[@]} child builds:" >&2 - for build_id in "${CHILD_BUILD_IDS_RESULT[@]}"; do - echo " $BUILD_NAME/$build_id" - run jf rt build-append "$BUILD_NAME" "$BUILD_NUMBER" \ - "$BUILD_NAME" "$build_id" --project="$PROJECT" - done - fi + # Access the results + if [[ ${#CHILD_BUILD_IDS_RESULT[@]} -gt 0 ]]; then + echo "Found ${#CHILD_BUILD_IDS_RESULT[@]} child builds:" >&2 + for build_id in "${CHILD_BUILD_IDS_RESULT[@]}"; do + echo " $BUILD_NAME/$build_id" + run jf rt build-append "$BUILD_NAME" "$BUILD_NUMBER" \ + "$BUILD_NAME" "$build_id" --project="$PROJECT" + done + fi - run jf rt build-append "$BUILD_NAME" "$BUILD_NUMBER" \ - "$BUILD_NAME" "$ARTIFACT_BUILD_NUMBER" --project="$PROJECT" - run jf rt build-publish "$BUILD_NAME" "$BUILD_NUMBER" --project="$PROJECT" + run jf rt build-append "$BUILD_NAME" "$BUILD_NUMBER" \ + "$BUILD_NAME" "$ARTIFACT_BUILD_NUMBER" --project="$PROJECT" + run jf rt build-publish "$BUILD_NAME" "$BUILD_NUMBER" --project="$PROJECT" } main() { - if [[ "$DRY_RUN" == "true" ]]; then - echo "Would deploy artifacts to JFrog Artifactory" >&2 - else - echo "Deploying artifacts to JFrog Artifactory" >&2 - fi - echo "Project: $PROJECT" >&2 - echo "Build name: $BUILD_NAME" >&2 - echo "Version: $VERSION" >&2 - echo "Dry run: $DRY_RUN" >&2 - echo "Build number: $BUILD_NUMBER" >&2 - echo "Metadata build number: $METADATA_BUILD_NUMBER" >&2 - echo "current files: $(ls -la build-artifacts)" >&2 - mkdir -p structured_build_artifacts - shopt -s globstar nullglob - - structure_build_artifacts - cd structured_build_artifacts - # Upload all packages - cd rpm - upload_rpm_packages - cd .. - cd deb - upload_deb_packages - cd .. - cd generic - upload_generic_files - cd .. - # Publish build info once for the unified build - publish_build_info - - echo "Deploy complete!" >&2 - echo "Build name: $BUILD_NAME" >&2 - echo "Build number: $BUILD_NUMBER" >&2 + if [[ $DRY_RUN == "true" ]]; then + echo "Would deploy artifacts to JFrog Artifactory" >&2 + else + echo "Deploying artifacts to JFrog Artifactory" >&2 + fi + echo "Project: $PROJECT" >&2 + echo "Build name: $BUILD_NAME" >&2 + echo "Version: $VERSION" >&2 + echo "Dry run: $DRY_RUN" >&2 + echo "Build number: $BUILD_NUMBER" >&2 + echo "Metadata build number: $METADATA_BUILD_NUMBER" >&2 + + if [[ ! -d build-artifacts ]]; then + error "build-artifacts directory does not exist. Artifacts must be downloaded before running this script." + fi + + mkdir -p structured_build_artifacts + shopt -s globstar nullglob + + structure_build_artifacts + cd structured_build_artifacts + # Upload all packages + cd rpm + upload_rpm_packages + cd .. + cd deb + upload_deb_packages + cd .. + cd jar + upload_jar_packages + cd .. + cd nupkg + upload_nupkg_packages + cd .. + cd generic + upload_generic_files + cd .. + # Publish build info once for the unified build + publish_build_info + + echo "Deploy complete!" >&2 + echo "Build name: $BUILD_NAME" >&2 + echo "Build number: $BUILD_NUMBER" >&2 } main "$@" diff --git a/.github/workflows/deploy-artifacts/package_utils.sh b/.github/workflows/deploy-artifacts/package_utils.sh index 571b130..fdaec93 100755 --- a/.github/workflows/deploy-artifacts/package_utils.sh +++ b/.github/workflows/deploy-artifacts/package_utils.sh @@ -10,6 +10,33 @@ handle_error() { exit 1 } +# Function to extract JAR metadata +get_jar_metadata() { + local jar="$1" + local filename="${jar##*/}" # Get just the filename without path + local pkgname version group_id + local pom_props + + # Initial parsing - handle versions with SNAPSHOT, SNAPSHOT_, etc. + pkgname=$(echo "$filename" | sed -E 's/-[0-9][0-9A-Za-z._\-]*\.jar$//') + version=$(echo "$filename" | sed -E 's/^[^-]+-([0-9][0-9A-Za-z._\-]*)\.jar$/\1/') + + pom_props=$(unzip -Z1 "$jar" | awk '/pom\.properties$/ {print; exit}') + if [[ -n "$pom_props" ]]; then + pkgname=$(unzip -p "$jar" "$pom_props" | grep '^artifactId=' | cut -d= -f2) + version=$(unzip -p "$jar" "$pom_props" | grep '^version=' | cut -d= -f2) + group_id=$(unzip -p "$jar" "$pom_props" | grep '^groupId=' | cut -d= -f2) + else + # Fallback to filename parsing if pom.properties is not found + pkgname=$(echo "$filename" | sed -E 's/-[0-9][0-9A-Za-z._\-]*\.jar$//') + version=$(echo "$filename" | sed -E 's/^[^-]+-([0-9][0-9A-Za-z._\-]*)\.jar$/\1/') + group_id="" + fi + + # Return package name, version, and group_id + echo "$pkgname $version $group_id" +} + # Function to extract RPM metadata and distribution # Unlike for debs this requires parsing the name (because the distro name is not standard) get_rpm_metadata() { @@ -59,6 +86,46 @@ process_rpm() { cp -v "$rpm" "$target/$rpm_name" >&2 } +process_jar() { + local jar="$1" + local dest_dir="$2" + local -a metadata + local pkgname version group_id group_path + + # Get metadata using the new function + read -r -a metadata < <(get_jar_metadata "$jar") + pkgname="${metadata[0]}" + version="${metadata[1]}" + # Use :- to handle empty group_id (when array may only have 2 elements due to trailing space trimming) + group_id="${metadata[2]:-}" + group_path="${group_id:+${group_id//./\/}}" + + local target="$dest_dir/$group_path/$pkgname/$version" + echo "DEBUG: Creating directory structure:" >&2 + echo " Group id: $group_id" >&2 + echo " Package name: $pkgname" >&2 + echo " Version: $version" >&2 + mkdir -p "$target" + + # Get the directory and base name of the jar file + local jar_dir + local jar_name + local base_name + jar_dir=$(dirname "$jar") + jar_name=$(basename "$jar") + base_name="${jar_name%.jar}" # Remove .jar extension + + echo "Copying Maven artifacts to: $target" >&2 + + # Copy jar, pom, and asc files + for ext in jar pom jar.asc pom.asc; do + local file="$jar_dir/${base_name}.${ext}" + if [[ -f "$file" ]]; then + cp -v "$file" "$target/" >&2 + fi + done +} + get_codename_for_deb() { case "$1" in *ubuntu20.04*) echo "focal" ;; diff --git a/.github/workflows/deploy-artifacts/tests/bats/test_all_files_upload.bats b/.github/workflows/deploy-artifacts/tests/bats/test_all_files_upload.bats index 1ba50e1..3c1dcea 100755 --- a/.github/workflows/deploy-artifacts/tests/bats/test_all_files_upload.bats +++ b/.github/workflows/deploy-artifacts/tests/bats/test_all_files_upload.bats @@ -12,7 +12,7 @@ load "$HELPERS_DIR/assertions.bash" setup_file() { setup_test_artifacts - # Verify expected generic file fixtures exist + # Verify expected file fixtures exist (JAR as Maven artifact, ZIP and TAR.GZ as generic) local -a expected_files=( "$BUILD_ARTIFACTS_DIR/test.jar" "$BUILD_ARTIFACTS_DIR/test.zip" @@ -56,16 +56,17 @@ teardown_file() { # Parse commands into arrays mapfile -t upload_cmd_array < <(echo "$upload_commands") - # Validate generic file uploads (JAR, ZIP, TAR.GZ) + # Validate file uploads (JAR as Maven, ZIP and TAR.GZ as generic) local jar_found=false local zip_found=false local tar_found=false for cmd in "${upload_cmd_array[@]}"; do - # Check for JAR file + # Check for JAR file (should be uploaded as Maven artifact) if [[ $cmd =~ test\.jar ]]; then jar_found=true - assert_upload_command_valid "$cmd" "test.jar" "test-project-generic-dev-local" "" \ + assert_upload_command_valid "$cmd" "test.jar" "test-project-maven-dev-local" \ + "group_id=com.example.test;package_name=test.jar;version=test.jar" \ "test-build" "12345-artifacts" "test-project" fi @@ -84,7 +85,7 @@ teardown_file() { fi done - # Verify generic files were found + # Verify files were found [[ $jar_found == true ]] || (echo "JAR file upload not found" >&2 && return 1) [[ $zip_found == true ]] || (echo "ZIP file upload not found" >&2 && return 1) [[ $tar_found == true ]] || (echo "TAR.GZ file upload not found" >&2 && return 1) diff --git a/.github/workflows/deploy-artifacts/tests/bats/test_java_upload.bats b/.github/workflows/deploy-artifacts/tests/bats/test_java_upload.bats index a621d37..e977420 100755 --- a/.github/workflows/deploy-artifacts/tests/bats/test_java_upload.bats +++ b/.github/workflows/deploy-artifacts/tests/bats/test_java_upload.bats @@ -64,8 +64,8 @@ teardown_file() { if [[ $cmd =~ \.jar ]]; then jar_found=true - # Java artifacts go to generic repo - local expected_repo="test-project-generic-dev-local" + # Java artifacts go to Maven repo + local expected_repo="test-project-maven-dev-local" # Extract filename from command local filename @@ -73,8 +73,9 @@ teardown_file() { filename="${BASH_REMATCH[1]}" fi - # Validate command structure - assert_upload_command_valid "$cmd" "$filename" "$expected_repo" "" \ + # Validate command structure with Maven target-props + assert_upload_command_valid "$cmd" "$filename" "$expected_repo" \ + "group_id=com.example.test;package_name=test.jar;version=test.jar" \ "test-build" "12345-artifacts" "test-project" fi done @@ -106,8 +107,8 @@ teardown_file() { local output output=$(run_entrypoint_dry_run "test-project" "test-build" "v1.0.0" "12345" "12345-metadata") - # Verify "Processing generic file:" messages appear for JAR files - [[ $output == *"Processing generic file:"* ]] || (echo "Generic file processing messages not found" >&2 && return 1) + # Verify "Processing JAR:" messages appear for JAR files + [[ $output == *"Processing JAR:"* ]] || (echo "JAR processing messages not found" >&2 && return 1) # Extract upload commands and verify JAR is present local upload_commands diff --git a/.github/workflows/deploy-artifacts/tests/bats/test_nupkg_upload.bats b/.github/workflows/deploy-artifacts/tests/bats/test_nupkg_upload.bats index 3ef46ff..3f020a1 100755 --- a/.github/workflows/deploy-artifacts/tests/bats/test_nupkg_upload.bats +++ b/.github/workflows/deploy-artifacts/tests/bats/test_nupkg_upload.bats @@ -65,9 +65,8 @@ teardown_file() { if [[ $cmd =~ \.nupkg ]]; then nupkg_found=true - # Determine expected repository (check if nupkg has separate repo or uses generic) - # Based on entrypoint.sh, nupkg files go to generic repo - local expected_repo="test-project-generic-dev-local" + # NuGet packages go to NuGet-specific repository + local expected_repo="test-project-nuget-dev-local" # Extract filename from command local filename diff --git a/.github/workflows/deploy-artifacts/tests/helpers/setup.bash b/.github/workflows/deploy-artifacts/tests/helpers/setup.bash index d011d9b..7962596 100755 --- a/.github/workflows/deploy-artifacts/tests/helpers/setup.bash +++ b/.github/workflows/deploy-artifacts/tests/helpers/setup.bash @@ -11,60 +11,60 @@ BUILD_ARTIFACTS_DIR="$TEST_DIR/build-artifacts" # Setup function called before each test file setup_test_artifacts() { - # Create test directory and fixtures - rm -rf "$TEST_DIR" - mkdir -p "$BUILD_ARTIFACTS_DIR" - - # Run create-test-fixtures.sh to set up test artifacts - # Must run from git root so it can find tests/ directory - cd "$GIT_ROOT" || exit 1 - "$DEPLOY_ARTIFACTS_DIR/create-test-fixtures.sh" - cd "$DEPLOY_ARTIFACTS_DIR/test-artifacts" || exit 1 - - # Verify test files exist - declare -a TEST_FILES=( - "$BUILD_ARTIFACTS_DIR/test-ubuntu22.04.deb" - "$BUILD_ARTIFACTS_DIR/test-1.0-2.noarch.rpm" - "$BUILD_ARTIFACTS_DIR/test.jar" - "$BUILD_ARTIFACTS_DIR/test.zip" - "$BUILD_ARTIFACTS_DIR/test.tar.gz" - "$BUILD_ARTIFACTS_DIR/nested/dir/test-debian12.deb" - "$BUILD_ARTIFACTS_DIR/nested/dir/nested.rpm" - ) - - for file in "${TEST_FILES[@]}"; do - if [[ ! -f "$file" ]]; then - echo "Error: Missing expected test file: $file" >&2 - exit 1 - fi - done - - # Change to test directory for consistent context - cd "$TEST_DIR" || exit 1 + # Create test directory and fixtures + rm -rf "$TEST_DIR" + mkdir -p "$BUILD_ARTIFACTS_DIR" + + # Run create-test-fixtures.sh to set up test artifacts + # Must run from git root so it can find tests/ directory + cd "$GIT_ROOT" || exit 1 + "$DEPLOY_ARTIFACTS_DIR/create-test-fixtures.sh" + cd "$DEPLOY_ARTIFACTS_DIR/test-artifacts" || exit 1 + + # Verify test files exist + declare -a TEST_FILES=( + "$BUILD_ARTIFACTS_DIR/test-ubuntu22.04.deb" + "$BUILD_ARTIFACTS_DIR/test-1.0-2.noarch.rpm" + "$BUILD_ARTIFACTS_DIR/test.jar" + "$BUILD_ARTIFACTS_DIR/test.zip" + "$BUILD_ARTIFACTS_DIR/test.tar.gz" + "$BUILD_ARTIFACTS_DIR/nested/dir/test-debian12.deb" + "$BUILD_ARTIFACTS_DIR/nested/dir/nested.rpm" + ) + + for file in "${TEST_FILES[@]}"; do + if [[ ! -f $file ]]; then + echo "Error: Missing expected test file: $file" >&2 + exit 1 + fi + done + + # Change to test directory for consistent context + cd "$TEST_DIR" || exit 1 } teardown_test_artifacts() { - # Cleanup test artifacts (optional - comment out for debugging) - # rm -rf "$TEST_DIR" - : + # Cleanup test artifacts (optional - comment out for debugging) + # rm -rf "$TEST_DIR" + : } # Run entrypoint.sh with dry-run and capture output run_entrypoint_dry_run() { - local project="${1:-test-project}" - local build_name="${2:-test-build}" - local version="${3:-v1.0.0}" - local build_number="${4:-12345}" - local metadata_build_number="${5:-12345-metadata}" - - cd "$TEST_DIR" || exit 1 - "$DEPLOY_ARTIFACTS_DIR/entrypoint.sh" \ - "$project" \ - "$build_name" \ - "$version" \ - "$build_number" \ - "$metadata_build_number" \ - --dry-run \ - 2>&1 -} + local project="${1:-test-project}" + local build_name="${2:-test-build}" + local version="${3:-v1.0.0}" + local build_number="${4:-12345}" + local metadata_build_number="${5:-12345-metadata}" + cd "$TEST_DIR" || exit 1 + "$DEPLOY_ARTIFACTS_DIR/entrypoint.sh" \ + "$project" \ + "$build_name" \ + "$version" \ + "$build_number" \ + "$metadata_build_number" \ + --dry-run \ + --jar-group-id "com.example.test" \ + 2>&1 +} diff --git a/.github/workflows/reusable_deploy-artifacts.yaml b/.github/workflows/reusable_deploy-artifacts.yaml index 1d0e08d..aca0792 100644 --- a/.github/workflows/reusable_deploy-artifacts.yaml +++ b/.github/workflows/reusable_deploy-artifacts.yaml @@ -33,6 +33,10 @@ on: required: false type: boolean default: false + jar-group-id: + description: Maven group ID for JAR artifacts (used as fallback when metadata doesn't provide one) + required: false + type: string gh-artifact-name: description: Name of signed artifacts on github required: false @@ -112,6 +116,7 @@ jobs: with: name: ${{ inputs.gh-artifact-name }} path: ./build-artifacts + - name: Verify artifacts shell: bash run: | @@ -127,11 +132,17 @@ jobs: dry_run_arg="--dry-run" fi + jar_group_id_arg="" + if [ -n "${{ inputs.jar-group-id }}" ]; then + jar_group_id_arg="--jar-group-id ${{ inputs.jar-group-id }}" + fi + ${{ inputs.gh-checkout-path }}/.github/workflows/deploy-artifacts/entrypoint.sh \ "${{ inputs.jf-project }}" \ "${{ inputs.jf-build-name }}" \ "${{ inputs.version }}" \ "${{ inputs.jf-build-id }}" \ "${{ inputs.jf-metadata-build-id }}" \ - $dry_run_arg + $dry_run_arg \ + $jar_group_id_arg echo "jf-build-id=${{ inputs.jf-build-id }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/reusable_execute-build.yaml b/.github/workflows/reusable_execute-build.yaml index ff91225..e9e8ebb 100644 --- a/.github/workflows/reusable_execute-build.yaml +++ b/.github/workflows/reusable_execute-build.yaml @@ -102,6 +102,28 @@ on: type: string default: . + # Java/Maven setup + setup-java: + description: Whether to set up Java environment + required: false + type: boolean + default: false + java-version: + description: Java version to set up (e.g., "17", "21") + required: false + type: string + default: "21" + java-distribution: + description: Java distribution (temurin, zulu, adopt, corretto, etc.) + required: false + type: string + default: temurin + java-cache: + description: Package manager to cache (maven, gradle, sbt). + required: false + type: string + default: maven + outputs: gh-artifact-name: description: The name of the uploaded artifacts on github @@ -146,6 +168,14 @@ jobs: submodules: recursive fetch-depth: 1 + - name: Set up Java + if: inputs.setup-java + uses: actions/setup-java@v4 + with: + distribution: ${{ inputs.java-distribution }} + java-version: ${{ inputs.java-version }} + cache: ${{ inputs.java-cache }} + - name: Set up JFrog CLI uses: jfrog/setup-jfrog-cli@5b06f730cc5a6f55d78b30753f8583454b08c0aa # v4.8.1 env: