From bf0e6abdaa997b3e34290d1f79be81f038afd654 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Thu, 25 Dec 2025 14:49:51 +0000 Subject: [PATCH 1/4] Migrate gsutil usage to gcloud storage --- gcs-fetcher/cloudbuild.yaml | 9 +++++---- gke-deploy/cmd/run/run.go | 2 +- gke-deploy/doc/deploying-with-cloud-build.md | 2 +- gke-deploy/doc/trigger.yaml | 8 ++++---- .../test/cloudbuild_gcb_prepare_apply_gcs.yaml | 6 +++--- gke-deploy/testservices/gsutil.go | 2 +- gsutil/README.md | 16 +++++++--------- gsutil/cloudbuild.yaml | 8 ++++---- gsutil/notice.sh | 2 +- 9 files changed, 27 insertions(+), 28 deletions(-) diff --git a/gcs-fetcher/cloudbuild.yaml b/gcs-fetcher/cloudbuild.yaml index 877494867..95fa87987 100644 --- a/gcs-fetcher/cloudbuild.yaml +++ b/gcs-fetcher/cloudbuild.yaml @@ -29,8 +29,9 @@ steps: args: - '--location=gs://${PROJECT_ID}_cloudbuild/manifest-${BUILD_ID}.json' # Print the manifest. -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cat', 'gs://${PROJECT_ID}_cloudbuild/manifest-${BUILD_ID}.json'] +- name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'gcloud' + args: ['storage', 'cat', 'gs://${PROJECT_ID}_cloudbuild/manifest-${BUILD_ID}.json'] # Fetch the manifest. - name: 'gcr.io/$PROJECT_ID/gcs-fetcher' args: @@ -48,7 +49,7 @@ steps: - -c - | tar czvf /tmp/out.tar.gz . - gsutil cp /tmp/out.tar.gz gs://${PROJECT_ID}_cloudbuild/ + gcloud storage cp /tmp/out.tar.gz gs://${PROJECT_ID}_cloudbuild/ - name: gcr.io/$PROJECT_ID/gcs-fetcher args: - --type=TarGzArchive @@ -67,7 +68,7 @@ steps: - | apt-get install -y zip zip -r /tmp/out.zip . - gsutil cp /tmp/out.zip gs://${PROJECT_ID}_cloudbuild/ + gcloud storage cp /tmp/out.zip gs://${PROJECT_ID}_cloudbuild/ - name: gcr.io/$PROJECT_ID/gcs-fetcher args: - --type=ZipArchive diff --git a/gke-deploy/cmd/run/run.go b/gke-deploy/cmd/run/run.go index d6dec94c5..3feef6deb 100644 --- a/gke-deploy/cmd/run/run.go +++ b/gke-deploy/cmd/run/run.go @@ -162,7 +162,7 @@ func run(_ *cobra.Command, options *options) error { } applyConfig := expandedOutput if strings.HasPrefix(options.output, "gs://") { - // Without this, gsutil copies the entire expanded output directory, rather than just the files in the directory, which fails applying the deployment if the --recursive flag isn't set. + // Without this, gcloud storage copies the entire expanded output directory, rather than just the files in the directory, which fails applying the deployment if the --recursive flag isn't set. applyConfig = applyConfig + "/*" } if err := d.Apply(ctx, options.clusterName, options.clusterLocation, options.clusterProject, applyConfig, options.namespace, options.waitTimeout, options.recursive); err != nil { diff --git a/gke-deploy/doc/deploying-with-cloud-build.md b/gke-deploy/doc/deploying-with-cloud-build.md index b7aff2793..fb5eaa65a 100644 --- a/gke-deploy/doc/deploying-with-cloud-build.md +++ b/gke-deploy/doc/deploying-with-cloud-build.md @@ -35,7 +35,7 @@ gcloud projects add-iam-policy-binding $PROJECT --member=serviceAccount:$SERVICE # Create a bucket that will be used to store configs suggested and expanded by gke-deploy. BUCKET=my-bucket -gsutil mb -p $PROJECT gs://$BUCKET +gcloud storage buckets create gs://$BUCKET --project=$PROJECT ``` ## Examples diff --git a/gke-deploy/doc/trigger.yaml b/gke-deploy/doc/trigger.yaml index 1e4b43a69..df60523c5 100644 --- a/gke-deploy/doc/trigger.yaml +++ b/gke-deploy/doc/trigger.yaml @@ -32,7 +32,7 @@ build: - --output=output - --annotation=gcb-build-id=$BUILD_ID - id: Save configs - name: gcr.io/cloud-builders/gsutil + name: gcr.io/cloud-builders/gcloud entrypoint: sh args: - -c @@ -40,9 +40,9 @@ build: set -e if [ $_OUTPUT_BUCKET_PATH ]; then - gsutil cp -r output/suggested gs://$_OUTPUT_BUCKET_PATH/config/$BUILD_ID/suggested + gcloud storage cp --recursive output/suggested gs://$_OUTPUT_BUCKET_PATH/config/$BUILD_ID/suggested echo "Copied suggested base configs to gs://$_OUTPUT_BUCKET_PATH/config/$BUILD_ID/suggested" - gsutil cp -r output/expanded gs://$_OUTPUT_BUCKET_PATH/config/$BUILD_ID/expanded + gcloud storage cp --recursive output/expanded gs://$_OUTPUT_BUCKET_PATH/config/$BUILD_ID/expanded echo "Copied expanded configs to gs://$_OUTPUT_BUCKET_PATH/config/$BUILD_ID/expanded" fi - id: Apply deploy @@ -75,4 +75,4 @@ substitutions: _K8S_YAML_PATH: @CONFIGS@ _K8S_APP_NAME: @APP_NAME@ _K8S_NAMESPACE: @NAMESPACE@ - _OUTPUT_BUCKET_PATH: @OUTPUT_BUCKET_PATH@ + _OUTPUT_BUCKET_PATH: @OUTPUT_BUCKET_PATH@ \ No newline at end of file diff --git a/gke-deploy/test/cloudbuild_gcb_prepare_apply_gcs.yaml b/gke-deploy/test/cloudbuild_gcb_prepare_apply_gcs.yaml index c44f2b258..252164862 100644 --- a/gke-deploy/test/cloudbuild_gcb_prepare_apply_gcs.yaml +++ b/gke-deploy/test/cloudbuild_gcb_prepare_apply_gcs.yaml @@ -10,15 +10,15 @@ steps: - '--label=foo=bar' - '--annotation=hi=bye' - '--output=$_OUTPUT' -- name: 'gcr.io/cloud-builders/gsutil' +- name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'sh' args: - '-c' - | set -x # Print commands set -e # Fail if any command below fails - gsutil -q stat $_OUTPUT/expanded/expanded-resources.yaml - gsutil -q stat $_OUTPUT/suggested/suggested-resources.yaml + gcloud -q storage objects list --stat --fetch-encrypted-object-hashes $_OUTPUT/expanded/expanded-resources.yaml + gcloud -q storage objects list --stat --fetch-encrypted-object-hashes $_OUTPUT/suggested/suggested-resources.yaml - name: 'gcr.io/$PROJECT_ID/gke-deploy' args: - 'apply' diff --git a/gke-deploy/testservices/gsutil.go b/gke-deploy/testservices/gsutil.go index bfa378197..84be0d8b5 100644 --- a/gke-deploy/testservices/gsutil.go +++ b/gke-deploy/testservices/gsutil.go @@ -81,7 +81,7 @@ func copyDir(srcdir, destdir string, info os.FileInfo) error { return nil } -// Copy simulates the gsutil copy. +// Copy simulates the gcloud storage cp. func Copy(src, dest string) error { info, err := os.Stat(src) if err != nil { diff --git a/gsutil/README.md b/gsutil/README.md index 0b19aa5d6..df6683cd1 100644 --- a/gsutil/README.md +++ b/gsutil/README.md @@ -20,49 +20,47 @@ Suggested alternative images include: google/cloud-sdk:debian_component_based google/cloud-sdk:slim -Please note that the `gsutil` entrypoint must be specified to use these images. +Please note that the `gcloud` entrypoint is the default for these images and should be used for `gcloud storage` commands. When executed in the Cloud Build environment, commands are executed with credentials of the [builder service account](https://cloud.google.com/cloud-build/docs/permissions) for the build project. -To migrate to the Cloud SDK team's official image, make the following changes -to your `cloudbuild.yaml`: +To migrate to the Cloud SDK team's official image for use with `gcloud storage`, make the following changes to your `cloudbuild.yaml` and update the `args` as shown in the examples below: ``` - name: 'gcr.io/cloud-builders/gsutil' + name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' -+ entrypoint: 'gsutil' ``` ------- ## Examples -The following examples demonstrate build requests that use `gsutil`. +The following examples demonstrate build requests that use `gcloud storage`. For these to work, the builder service account must have permission to access the necessary buckets and objects. ### Copy an object from Google Cloud Storage -This `cloudbuild.yaml` invokes `gsutil cp` to copy an object to the build's +This `cloudbuild.yaml` invokes `gcloud storage cp` to copy an object to the build's workspace. ``` steps: - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' - args: ['gsutil', 'cp', 'gs://mybucket/remotefile.zip', 'localfile.zip'] + args: ['storage', 'cp', 'gs://mybucket/remotefile.zip', 'localfile.zip'] ``` ### Copy a local file to Google Cloud Storage -This `cloudbuild.yaml` invokes `gsutil cp` to copy a local file to Google Cloud +This `cloudbuild.yaml` invokes `gcloud storage cp` to copy a local file to Google Cloud Storage. ``` steps: - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' - args: ['gsutil', 'cp', 'localfile.zip', 'gs://mybucket/remotefile.zip'] + args: ['storage', 'cp', 'localfile.zip', 'gs://mybucket/remotefile.zip'] ``` diff --git a/gsutil/cloudbuild.yaml b/gsutil/cloudbuild.yaml index c6eaf1a8b..4e09df6b9 100644 --- a/gsutil/cloudbuild.yaml +++ b/gsutil/cloudbuild.yaml @@ -8,13 +8,13 @@ steps: - '--pull' - '--tag=gcr.io/$PROJECT_ID/gsutil' - '.' -- name: 'gcr.io/$PROJECT_ID/gsutil' - args: ['version'] +- name: 'gcr.io/cloud-builders/gcloud' + args: ['-v'] # Invoke a command that requires auth, to check that it gets piped through # correctly. -- name: 'gcr.io/$PROJECT_ID/gsutil' - args: ['ls'] +- name: 'gcr.io/cloud-builders/gcloud' + args: ['storage', 'ls'] images: - 'gcr.io/$PROJECT_ID/gsutil' diff --git a/gsutil/notice.sh b/gsutil/notice.sh index 3703ff168..0628f7034 100755 --- a/gsutil/notice.sh +++ b/gsutil/notice.sh @@ -20,4 +20,4 @@ images. ***** END OF NOTICE ***** ' fi -/builder/google-cloud-sdk/bin/gsutil "$@" +/builder/google-cloud-sdk/bin/gcloud storage "$@" From cd0628ba0bff2f1eb01b6811735f01651994a0f0 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 26 Dec 2025 13:59:18 +0000 Subject: [PATCH 2/4] Reverted gsutil files to original state --- gsutil/README.md | 15 ++++++++------- gsutil/cloudbuild.yaml | 8 ++++---- gsutil/notice.sh | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/gsutil/README.md b/gsutil/README.md index df6683cd1..f1a1622d7 100644 --- a/gsutil/README.md +++ b/gsutil/README.md @@ -20,14 +20,15 @@ Suggested alternative images include: google/cloud-sdk:debian_component_based google/cloud-sdk:slim -Please note that the `gcloud` entrypoint is the default for these images and should be used for `gcloud storage` commands. +Please note that the `gsutil` entrypoint must be specified to use these images. When executed in the Cloud Build environment, commands are executed with credentials of the [builder service account](https://cloud.google.com/cloud-build/docs/permissions) for the build project. -To migrate to the Cloud SDK team's official image for use with `gcloud storage`, make the following changes to your `cloudbuild.yaml` and update the `args` as shown in the examples below: +TTo migrate to the Cloud SDK team's official image, make the following changes +to your `cloudbuild.yaml`: ``` - name: 'gcr.io/cloud-builders/gsutil' @@ -38,29 +39,29 @@ To migrate to the Cloud SDK team's official image for use with `gcloud storage`, ## Examples -The following examples demonstrate build requests that use `gcloud storage`. +The following examples demonstrate build requests that use `gsutil`. For these to work, the builder service account must have permission to access the necessary buckets and objects. ### Copy an object from Google Cloud Storage -This `cloudbuild.yaml` invokes `gcloud storage cp` to copy an object to the build's +This `cloudbuild.yaml` invokes `gsutil cp` to copy an object to the build's workspace. ``` steps: - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' - args: ['storage', 'cp', 'gs://mybucket/remotefile.zip', 'localfile.zip'] + args: ['gsutil', 'cp', 'gs://mybucket/remotefile.zip', 'localfile.zip'] ``` ### Copy a local file to Google Cloud Storage -This `cloudbuild.yaml` invokes `gcloud storage cp` to copy a local file to Google Cloud +This `cloudbuild.yaml` invokes `gsutil cp` to copy a local file to Google Cloud Storage. ``` steps: - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' - args: ['storage', 'cp', 'localfile.zip', 'gs://mybucket/remotefile.zip'] + args: ['gsutil', 'cp', 'localfile.zip', 'gs://mybucket/remotefile.zip'] ``` diff --git a/gsutil/cloudbuild.yaml b/gsutil/cloudbuild.yaml index 4e09df6b9..c6eaf1a8b 100644 --- a/gsutil/cloudbuild.yaml +++ b/gsutil/cloudbuild.yaml @@ -8,13 +8,13 @@ steps: - '--pull' - '--tag=gcr.io/$PROJECT_ID/gsutil' - '.' -- name: 'gcr.io/cloud-builders/gcloud' - args: ['-v'] +- name: 'gcr.io/$PROJECT_ID/gsutil' + args: ['version'] # Invoke a command that requires auth, to check that it gets piped through # correctly. -- name: 'gcr.io/cloud-builders/gcloud' - args: ['storage', 'ls'] +- name: 'gcr.io/$PROJECT_ID/gsutil' + args: ['ls'] images: - 'gcr.io/$PROJECT_ID/gsutil' diff --git a/gsutil/notice.sh b/gsutil/notice.sh index 0628f7034..3703ff168 100755 --- a/gsutil/notice.sh +++ b/gsutil/notice.sh @@ -20,4 +20,4 @@ images. ***** END OF NOTICE ***** ' fi -/builder/google-cloud-sdk/bin/gcloud storage "$@" +/builder/google-cloud-sdk/bin/gsutil "$@" From 763839dc112fc4cb63b658191c7ce822f75c010e Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 26 Dec 2025 14:01:19 +0000 Subject: [PATCH 3/4] Reverted gsutil files to original state --- gsutil/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gsutil/README.md b/gsutil/README.md index f1a1622d7..3562027be 100644 --- a/gsutil/README.md +++ b/gsutil/README.md @@ -27,7 +27,7 @@ credentials of the [builder service account](https://cloud.google.com/cloud-build/docs/permissions) for the build project. -TTo migrate to the Cloud SDK team's official image, make the following changes +To migrate to the Cloud SDK team's official image, make the following changes to your `cloudbuild.yaml`: ``` From 532e084147216914f35585ce5cf8bef830eb98fa Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 26 Dec 2025 14:07:04 +0000 Subject: [PATCH 4/4] Reverted gsutil files to original state --- gsutil/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/gsutil/README.md b/gsutil/README.md index 3562027be..0b19aa5d6 100644 --- a/gsutil/README.md +++ b/gsutil/README.md @@ -33,6 +33,7 @@ to your `cloudbuild.yaml`: ``` - name: 'gcr.io/cloud-builders/gsutil' + name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' ++ entrypoint: 'gsutil' ``` -------