Refactor CD scripts to eliminate details.txt dependency and support m…

tools/cd_scripts/e2e_test.sh

@@ -100,11 +100,20 @@ if [[ "$RUN_LIGHT_TEST" == "true" ]]; then
 	echo "Running light tests only..."
 fi
 
-#details.txt file contains the release version and commit hash of the current release.
-# Using dynamic bucket.
-gcloud storage cp gs://${BUCKET_NAME_TO_USE}/version-detail/details.txt .
-# Writing VM instance name to details.txt (Format: release-test-<os-name>)
-curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google" >>details.txt
+# Fetch parameters from VM metadata attributes or environment variables first.
+META_VERSION=$(curl -sfS -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/RELEASE_VERSION" 2>/dev/null || echo "${RELEASE_VERSION:-}")
+META_COMMIT=$(curl -sfS -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/COMMIT_HASH" 2>/dev/null || echo "${COMMIT_HASH:-}")
+VM_NAME=$(curl -sfS -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/name" 2>/dev/null || hostname)
+
+if [[ -n "$META_VERSION" && -n "$META_COMMIT" ]]; then
+  echo "$META_VERSION" > details.txt
+  echo "$META_COMMIT" >> details.txt
+  echo "$VM_NAME" >> details.txt
+else
+  # Fallback to fetching details.txt from GCS if metadata is not provided
+  gcloud storage cp gs://${BUCKET_NAME_TO_USE}/version-detail/details.txt . || true
+  echo "$VM_NAME" >> details.txt
+fi
 
 # Function to create the local user
 create_user() {

tools/cd_scripts/improved_e2e_test.sh

@@ -66,12 +66,8 @@ while (( $# >= 1 )); do
         --release-version)
             RELEASE_VERSION="$2"
             shift 2
-            # Regex breakdown:
-            # ^      : Start of string
-            # [0-9]+ : One or more digits
-            #  \.     : A literal dot
-            # $      : End of string
-            RE="^[0-9]+\.[0-9]+\.[0-9]+$"
+            # Allow SemVer (X.Y.Z), YYYY.MM.DD-nightly, YYYY.MM.DD-dev, or general version strings
+            RE="^[a-zA-Z0-9_.-]+$"
             if [[ ! $RELEASE_VERSION =~ $RE ]]; then
                 log_error "--release-version value '$RELEASE_VERSION' is incorrectly formatted."
                 usage 1

tools/cd_scripts/install_test.sh

@@ -68,50 +68,67 @@ sudo /usr/local/google-cloud-sdk/install.sh --quiet
 export PATH=/usr/local/google-cloud-sdk/bin:$PATH
 gcloud version && rm gcloud.tar.gz
 
-#details.txt file contains the release version and commit hash of the current release.
-gcloud storage cp  gs://gcsfuse-release-packages/version-detail/details.txt .
-# Writing VM instance name to details.txt (Format: release-test-<os-name>)
-vm_instance_name=$(curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google")
-# first line of details.txt contains the release version in the format MAJOR.MINOR.PATCH
-to_release_version=$(sed '1q' details.txt | tr -d '\n')
-echo $vm_instance_name >> details.txt
+# Helper function to fetch metadata value from GCE metadata server
+function fetch_meta_data_value() {
+  local metadata_key=$1
+  curl -sfS -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/$metadata_key" 2>/dev/null || true
+}
+
+# Fetch configuration directly from instance metadata or environment
+vm_instance_name=$(curl -sfS -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/name" 2>/dev/null || hostname)
+to_release_version=$(fetch_meta_data_value "RELEASE_VERSION")
+upload_bucket=$(fetch_meta_data_value "UPLOAD_BUCKET")
+upload_bucket=${upload_bucket:-"gcsfuse-release-packages"}
+
+# Backward compatibility fallback if RELEASE_VERSION is not present in metadata
+if [[ -z "$to_release_version" ]]; then
+  if gcloud storage cp "gs://${upload_bucket}/version-detail/details.txt" ./details.txt 2>/dev/null; then
+    to_release_version=$(sed -n 1p details.txt | tr -d '\r\n')
+  fi
+fi
+
+# Detect OS distribution via /etc/os-release
+IS_DEBIAN_OR_UBUNTU=false
+if [ -f /etc/os-release ]; then
+  # shellcheck disable=SC1091
+  . /etc/os-release
+  distro_id="${ID:-}"
+  distro_like="${ID_LIKE:-}"
+  if [[ "$distro_id" =~ ^(ubuntu|debian)$ ]] || [[ "$distro_like" =~ (ubuntu|debian) ]]; then
+    IS_DEBIAN_OR_UBUNTU=true
+  fi
+fi
+
 touch ~/logs.txt
 
-# Based on the os type(from vm instance name) in detail.txt, run the following
-# commands to install gcsfuse.
-if grep -q ubuntu details.txt || grep -q debian details.txt;
-then
-    if grep -q "~beta" details.txt;
-    then
+# Based on the OS type, run the installation commands
+if [[ "$IS_DEBIAN_OR_UBUNTU" == true ]]; then
+    if [[ "$to_release_version" == *"~beta"* ]]; then
          export GCSFUSE_REPO=gcsfuse-beta
     else
-         export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
+         export GCSFUSE_REPO="gcsfuse-$(lsb_release -c -s 2>/dev/null || echo 'bullseye')"
     fi
-    #  For ubuntu and debian os
 
     # Don't use apt-key for Debian 11+ and Ubuntu 21+
-    if { [[ $vm_instance_name == *"debian"*  &&  !( "$vm_instance_name" < "release-test-debian-11") ]]; } || { [[ $vm_instance_name == *"ubuntu"*  && !("$vm_instance_name" < "release-test-ubuntu-21") ]]; }
-    then
+    if { [[ $vm_instance_name == *"debian"* && !( "$vm_instance_name" < "release-test-debian-11") ]]; } || { [[ $vm_instance_name == *"ubuntu"* && !("$vm_instance_name" < "release-test-ubuntu-21") ]]; }; then
       echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
       curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo tee /usr/share/keyrings/cloud.google.asc >> ~/apt_key_logs.txt
     else
       echo "deb https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
       curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - >> ~/apt_key_logs.txt
     fi
 
-    if grep -q -i warning ~/apt_key_logs.txt;
-    then
+    if grep -q -i warning ~/apt_key_logs.txt; then
       echo "Failure: Got warning while using apt-key" >> ~/logs.txt
     fi
 
     sudo apt-get update
     # Install to be released gcsfuse version (It can be a patch to older version so allow downgrades)
     sudo apt-get install -y --allow-downgrades gcsfuse="$to_release_version" >> ~/logs.txt
 else
-#  For rhel and centos
-    sudo yum install fuse
-    if grep -q "~beta" details.txt;
-    then
+    # For RHEL and CentOS
+    sudo yum install -y fuse
+    if [[ "$to_release_version" == *"~beta"* ]]; then
         YUM_REPO_NAME=gcsfuse-el7-x86_64-beta
     else
         YUM_REPO_NAME=gcsfuse-el7-x86_64
@@ -126,66 +143,58 @@ repo_gpgcheck=0
 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 EOF
-# Attempt a install first, falling back to a standard downgrade if to be released is older version than already installed (patch releases).
-sudo yum install -y gcsfuse-"$to_release_version" || sudo yum downgrade -y gcsfuse-"$to_release_version" >> ~/logs.txt
+    # Attempt an install first, falling back to a standard downgrade if to-be-released is older version than already installed (patch releases).
+    sudo yum install -y gcsfuse-"$to_release_version" || sudo yum downgrade -y gcsfuse-"$to_release_version" >> ~/logs.txt
 fi
 
 # Verify gcsfuse version (successful installation)
 gcsfuse --version |& tee version.txt
 installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3)
-if grep -q $installed_version details.txt; then
+if [[ "$installed_version" == "$to_release_version"* ]] || [[ "$to_release_version" == *"$installed_version"* ]]; then
     echo "GCSFuse to be released version installed correctly." &>> ~/logs.txt
 else
-    echo "Failure detected in to be released gcsfuse version installation." &>> ~/logs.txt
+    echo "Failure detected in to be released gcsfuse version installation: expected $to_release_version, got $installed_version" &>> ~/logs.txt
 fi
 
 # Uninstall gcsfuse and install old version.
-if grep -q ubuntu details.txt || grep -q debian details.txt;
-then
+if [[ "$IS_DEBIAN_OR_UBUNTU" == true ]]; then
   sudo apt-get remove -y gcsfuse |& tee -a ~/logs.txt
   sudo apt-get install -y gcsfuse=1.2.0 |& tee -a ~/logs.txt
 else
   sudo yum -y remove gcsfuse |& tee -a ~/logs.txt
   sudo yum install -y gcsfuse-1.2.0 |& tee -a ~/logs.txt
 fi
 
-# verify old version installation
+# Verify old version installation
 gcsfuse --version |& tee version.txt
 installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3)
-if [ $installed_version == "1.2.0" ]; then
+if [ "$installed_version" == "1.2.0" ]; then
   echo "GCSFuse old version (1.2.0) installed successfully" &>> ~/logs.txt
 else
   echo "Failure detected in GCSFuse old version installation." &>> ~/logs.txt
 fi
 
 # Upgrade gcsfuse to latest version.
-if grep -q ubuntu details.txt || grep -q debian details.txt;
-then
+if [[ "$IS_DEBIAN_OR_UBUNTU" == true ]]; then
     sudo apt-get install --only-upgrade gcsfuse |& tee -a ~/logs.txt
 else
     sudo yum -y upgrade gcsfuse |& tee -a ~/logs.txt
 fi
 
 # Verify that gcsfuse has been upgraded to the to_be_released version using version comparison.
-# This is to ensure that the correct version is installed after the upgrade.
 gcsfuse --version |& tee version.txt
 installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3)
-# The following command compares the two versions:
-# 1. `printf` outputs to_release_version and installed_version on a new line.
-# 2. `sort -V` sorts them naturally (version sort).
-# 3. `tail -n 1` gets the last line, which is the highest version.
-# The condition is true if installed_version is greater than or equal to to_release_version.
 if [[ "$(printf '%s\n%s\n' "$to_release_version" "$installed_version" | sort -V | tail -n 1)" == "$installed_version" ]]; then
   echo "GCSFuse successfully upgraded to latest version: installed_version ($installed_version), to_release_version: ($to_release_version)" &>> ~/logs.txt
 else
   echo "Failure detected in upgrading to latest gcsfuse version: installed_version ($installed_version), to_release_version: ($to_release_version)" &>> ~/logs.txt
 fi
 
 if grep -q Failure ~/logs.txt; then
-    echo "Test failed" &>> ~/logs.txt ;
+    echo "Test failed" &>> ~/logs.txt
 else
     touch success.txt
-    gcloud storage cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/   ;
+    gcloud storage cp success.txt "gs://${upload_bucket}/v${to_release_version}/installation-test/${vm_instance_name}/"
 fi
 
-gcloud storage cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/
+gcloud storage cp ~/logs.txt "gs://${upload_bucket}/v${to_release_version}/installation-test/${vm_instance_name}/"

tools/cd_scripts/package_gcsfuse.sh

@@ -65,10 +65,22 @@ sudo apt-get install qemu-user-static binfmt-support
 git clone https://github.com/GoogleCloudPlatform/gcsfuse.git
 cd gcsfuse/tools/package_gcsfuse_docker/
 git checkout "$COMMIT_HASH"
+if [[ "$RELEASE_VERSION" == v* ]]; then
+  REL_DIR="$RELEASE_VERSION"
+else
+  REL_DIR="v$RELEASE_VERSION"
+fi
+
+# Debian packaging (dpkg-deb) requires Version strings to start with a digit (0-9).
+PKG_VERSION="${RELEASE_VERSION}"
+if [[ "${PKG_VERSION}" =~ ^[^0-9] ]]; then
+  PKG_VERSION="0.0.0-${PKG_VERSION}"
+fi
+
 echo "Building docker for ${architecture} ..."
-sudo docker buildx build --load . -t gcsfuse-release-${architecture}:"$RELEASE_VERSION_TAG" --build-arg GCSFUSE_VERSION="$RELEASE_VERSION" --build-arg ARCHITECTURE=${architecture} --build-arg BRANCH_NAME="$COMMIT_HASH" &> docker_${architecture}.log
-gcloud storage cp docker_${architecture}.log gs://"$UPLOAD_BUCKET"/v"$RELEASE_VERSION"/
-sudo docker run  -v $HOME/gcsfuse/release:/release gcsfuse-release-${architecture}:"$RELEASE_VERSION_TAG" cp -r /packages/. /release/v"$RELEASE_VERSION"
+sudo docker buildx build --load . -t gcsfuse-release-${architecture}:"$RELEASE_VERSION_TAG" --build-arg GCSFUSE_VERSION="$PKG_VERSION" --build-arg ARCHITECTURE=${architecture} --build-arg BRANCH_NAME="$COMMIT_HASH" &> docker_${architecture}.log
+gcloud storage cp docker_${architecture}.log gs://"$UPLOAD_BUCKET"/"$REL_DIR"/
+sudo docker run  -v $HOME/gcsfuse/release:/release gcsfuse-release-${architecture}:"$RELEASE_VERSION_TAG" cp -r /packages/. /release/"$REL_DIR"
 
 echo "Upload files in the bucket ..."
-gcloud storage cp --recursive $HOME/gcsfuse/release/v"$RELEASE_VERSION" gs://"$UPLOAD_BUCKET"/
+gcloud storage cp --recursive $HOME/gcsfuse/release/"$REL_DIR" gs://"$UPLOAD_BUCKET"/