From 696db7a2a0d0a4a2cdcfecead428d50128a7c796 Mon Sep 17 00:00:00 2001 From: Travis Thompson Date: Fri, 24 Oct 2025 11:19:00 -0700 Subject: [PATCH] feat: add support for gcs backend --- README.md | 34 +++++ backends/cache_gcs | 142 +++++++++++++++++++++ tests/cache_gcs.bats | 291 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 467 insertions(+) create mode 100755 backends/cache_gcs create mode 100644 tests/cache_gcs.bats diff --git a/README.md b/README.md index 2345478..70ced78 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ The plugin supports various backends and compression algorithms, and some enviro - `tar` for tar compression - `zip/unzip` for zip compression - `aws` AWS CLI for reading and writing to an AWS S3 backend +- `gsutil` or `gcloud storage` Google Cloud SDK CLI for reading and writing to a GCS backend ## Mandatory parameters @@ -62,6 +63,7 @@ You can specify multiple levels in an array to save the same artifact as a cache Defines how the cache is stored and restored. Can be any string (see [Customizable Backends](#customizable-backends)), but the plugin natively supports the following: * `fs` (default) * `s3` +* `gcs` #### `fs` @@ -124,6 +126,38 @@ steps: compression: zstd ``` +#### `gcs` + +Store things in a Google Cloud Storage (GCS) bucket. The backend automatically detects and uses either `gcloud storage` (preferred) or `gsutil` CLI tools, whichever is available. You need to make sure at least one of these commands is available and appropriately configured with the necessary credentials and access permissions. + +You also need the agent to have access to the following defined environment variables: +* `BUILDKITE_PLUGIN_GCS_CACHE_BUCKET`: the bucket to use (backend will fail if not defined) +* `BUILDKITE_PLUGIN_GCS_CACHE_PREFIX`: optional prefix to use for the cache within the bucket +* `BUILDKITE_PLUGIN_GCS_CACHE_CLI`: optional CLI preference, either `gcloud` or `gsutil` (auto-detects if not set, preferring `gcloud storage`) + +Setting the `BUILDKITE_PLUGIN_GCS_CACHE_QUIET` environment variable will reduce logging of file operations to GCS. + +#### Example + +```yaml +env: + BUILDKITE_PLUGIN_GCS_CACHE_BUCKET: "my-cache-bucket" # Required: GCS bucket to store cache objects + BUILDKITE_PLUGIN_GCS_CACHE_PREFIX: "buildkite/cache" + BUILDKITE_PLUGIN_GCS_CACHE_QUIET: "true" + +steps: + - label: ':nodejs: Install dependencies' + command: npm ci + plugins: + - cache#v1.7.0: + backend: gcs + path: node_modules + manifest: package-lock.json + restore: file + save: file + compression: zstd +``` + ### `compression` (string) Allows for the cached file/folder to be saved/restored as a single file. You will need to make sure to use the same compression when saving and restoring or it will cause a cache miss. diff --git a/backends/cache_gcs b/backends/cache_gcs new file mode 100755 index 0000000..1de5449 --- /dev/null +++ b/backends/cache_gcs @@ -0,0 +1,142 @@ +#!/bin/bash + +if [ -z "${BUILDKITE_PLUGIN_GCS_CACHE_BUCKET}" ]; then + echo '+++ 🚨 Missing GCS bucket configuration' + exit 1 +fi + +# Detect which GCS CLI to use +detect_gcs_cli() { + # Check for explicit preference + if [ -n "${BUILDKITE_PLUGIN_GCS_CACHE_CLI}" ]; then + echo "${BUILDKITE_PLUGIN_GCS_CACHE_CLI}" + return + fi + + # Auto-detect: prefer gcloud storage if available + if command -v gcloud &>/dev/null && gcloud storage --help &>/dev/null 2>&1; then + echo "gcloud" + elif command -v gsutil &>/dev/null; then + echo "gsutil" + else + echo '+++ 🚨 Neither gcloud storage nor gsutil found' >&2 + exit 1 + fi +} + +# Lazy load CLI selection +get_gcs_cli() { + if [ -z "${GCS_CLI}" ]; then + GCS_CLI=$(detect_gcs_cli) + fi + echo "${GCS_CLI}" +} + +build_key() { + if [ -n "${BUILDKITE_PLUGIN_GCS_CACHE_PREFIX}" ]; then + echo "${BUILDKITE_PLUGIN_GCS_CACHE_PREFIX}/${1}" + else + echo "$1" + fi +} + +gcs_cmd() { + local cmd_args=() + local cli + cli=$(get_gcs_cli) + + if [ "${cli}" = "gcloud" ]; then + cmd_args=(gcloud storage) + if [ -n "${BUILDKITE_PLUGIN_GCS_CACHE_QUIET}" ]; then + cmd_args+=(--verbosity=none) + fi + else + cmd_args=(gsutil) + if [ -n "${BUILDKITE_PLUGIN_GCS_CACHE_QUIET}" ]; then + cmd_args+=(-q) + fi + fi + + "${cmd_args[@]}" "$@" +} + +gcs_copy() { + local from="$1" + local to="$2" + local use_rsync="${3:-true}" + local cli + cli=$(get_gcs_cli) + + if [ "${use_rsync}" = 'true' ]; then + # Use rsync for directories + if [ "${cli}" = "gcloud" ]; then + gcs_cmd rsync -r -d "${from}" "${to}" + else + gcs_cmd -m rsync -r -d "${from}" "${to}" + fi + else + # Use cp for single files + gcs_cmd cp "${from}" "${to}" + fi +} + +gcs_exists() { + local key="$1" + local full_path="gs://${BUILDKITE_PLUGIN_GCS_CACHE_BUCKET}/$(build_key "${key}")" + + # Check if the object exists using ls + # For directories, ls will list contents + # For files, it will return the file path + if gcs_cmd ls "${full_path}*" &>/dev/null; then + return 0 + else + return 1 + fi +} + +restore_cache() { + local from="$1" + local to="$2" + local use_rsync='false' + local key="$(build_key "${from}")" + local full_path="gs://${BUILDKITE_PLUGIN_GCS_CACHE_BUCKET}/${key}" + + # Check if it's a directory by trying to list it as a prefix + if gcs_cmd ls "${full_path}/" &>/dev/null; then + use_rsync='true' + fi + + gcs_copy "${full_path}" "${to}" "${use_rsync}" +} + +save_cache() { + local to="$1" + local from="$2" + local use_rsync='true' + local key="$(build_key "${to}")" + local full_path="gs://${BUILDKITE_PLUGIN_GCS_CACHE_BUCKET}/${key}" + + if [ -f "${from}" ]; then + use_rsync='false' + fi + + gcs_copy "${from}" "${full_path}" "${use_rsync}" +} + +exists_cache() { + if [ -z "$1" ]; then exit 1; fi + gcs_exists "$1" +} + +OPCODE="$1" +shift + +if [ "$OPCODE" = 'exists' ]; then + exists_cache "$@" +elif [ "$OPCODE" = 'get' ]; then + restore_cache "$@" +elif [ "$OPCODE" = 'save' ]; then + save_cache "$@" +else + exit 255 +fi \ No newline at end of file diff --git a/tests/cache_gcs.bats b/tests/cache_gcs.bats new file mode 100644 index 0000000..b9f1880 --- /dev/null +++ b/tests/cache_gcs.bats @@ -0,0 +1,291 @@ +#!/usr/bin/env bats + +# To debug stubs, uncomment these lines: +# export GSUTIL_STUB_DEBUG=/dev/tty +# export GCLOUD_STUB_DEBUG=/dev/tty + +setup() { + load "${BATS_PLUGIN_PATH}/load.bash" + + export BUILDKITE_PLUGIN_GCS_CACHE_BUCKET=my-bucket + # Force gsutil for testing unless explicitly testing gcloud + export BUILDKITE_PLUGIN_GCS_CACHE_CLI=gsutil +} + +# teardown() { +# rm -rf "${BUILDKITE_PLUGIN_GCS_CACHE_FOLDER}" +# } + +@test 'Missing bucket configuration makes plugin fail' { + unset BUILDKITE_PLUGIN_GCS_CACHE_BUCKET + + run "${PWD}/backends/cache_gcs" + + assert_failure + assert_output --partial 'Missing GCS bucket configuration' +} + +@test 'Invalid operation fails silently with 255' { + run "${PWD}/backends/cache_gcs" invalid + + assert_failure 255 + assert_output '' +} + +@test 'Exists on empty file fails' { + run "${PWD}/backends/cache_gcs" exists "" + + assert_failure + assert_output '' +} + +@test 'Exists on non-existing file fails' { + stub gsutil 'ls \* : exit 1' + + run "${PWD}/backends/cache_gcs" exists PATH/THAT/DOES/NOT/EXIST + + assert_failure + assert_output '' + + unstub gsutil +} + +@test 'Exists on existing file/folder works' { + stub gsutil 'ls \* : echo "gs://my-bucket/existing"' + + run "${PWD}/backends/cache_gcs" exists existing + + assert_success + assert_output '' + + unstub gsutil +} + +@test 'Quiet flag passed when environment is set' { + export BUILDKITE_PLUGIN_GCS_CACHE_QUIET=1 + stub gsutil \ + '-q -m rsync -r -d \* \* : true ' \ + '-q ls \* : exit 1 ' \ + '-q -m rsync -r -d \* \* : true ' \ + '-m rsync -r -d \* \* : true ' \ + 'ls \* : exit 1 ' \ + '-m rsync -r -d \* \* : true ' + + run "${PWD}/backends/cache_gcs" save from to + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" get from to + + assert_success + assert_output '' + + unset BUILDKITE_PLUGIN_GCS_CACHE_QUIET + + run "${PWD}/backends/cache_gcs" save from to + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" get from to + + assert_success + assert_output '' + + unstub gsutil +} + +@test 'File exists and can be restored after save' { + touch "${BATS_TEST_TMPDIR}/new-file" + mkdir "${BATS_TEST_TMPDIR}/gcs-cache" + stub gsutil \ + "ls \* : exit 1" \ + "cp \* \* : ln -s \$2 $BATS_TEST_TMPDIR/gcs-cache/\$(echo \$3 | md5sum | cut -c-32)" \ + "ls \* : echo 'gs://my-bucket/new-file'" \ + 'ls \* : exit 1 ' \ + "cp \* \* : cp -r $BATS_TEST_TMPDIR/gcs-cache/\$(echo \$2 | md5sum | cut -c-32) \$3" + + run "${PWD}/backends/cache_gcs" exists new-file + + assert_failure + assert_output '' + + run "${PWD}/backends/cache_gcs" save new-file "${BATS_TEST_TMPDIR}/new-file" + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" exists new-file + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" get new-file "${BATS_TEST_TMPDIR}/other-file" + + assert_success + assert_output '' + + diff "${BATS_TEST_TMPDIR}/new-file" "${BATS_TEST_TMPDIR}/other-file" + + unstub gsutil + rm -rf "${BATS_TEST_TMPDIR}/gcs-cache" + rm -rf "${BATS_TEST_TMPDIR}/new-file" +} + +@test 'Folder exists and can be restored after save' { + mkdir "${BATS_TEST_TMPDIR}/gcs-cache" + mkdir "${BATS_TEST_TMPDIR}/new-folder" + echo 'random content' > "${BATS_TEST_TMPDIR}/new-folder/new-file" + + stub gsutil \ + "ls \* : exit 1" \ + "-m rsync -r -d \* \* : ln -s \$4 $BATS_TEST_TMPDIR/gcs-cache/\$(echo \$5 | md5sum | cut -c-32)" \ + "ls \* : echo 'gs://my-bucket/new-folder'" \ + 'ls \* : echo "gs://my-bucket/new-folder/"' \ + "-m rsync -r -d \* \* : cp -r $BATS_TEST_TMPDIR/gcs-cache/\$(echo \$4 | md5sum | cut -c-32) \$5" + + run "${PWD}/backends/cache_gcs" exists new-folder + + assert_failure + assert_output '' + + run "${PWD}/backends/cache_gcs" save new-folder "${BATS_TEST_TMPDIR}/new-folder" + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" exists new-folder + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" get new-folder "${BATS_TEST_TMPDIR}/other-folder" + + assert_success + assert_output '' + + find "${BATS_TEST_TMPDIR}/new-folder" + + find "${BATS_TEST_TMPDIR}/other-folder" + diff -r "${BATS_TEST_TMPDIR}/new-folder" "${BATS_TEST_TMPDIR}/other-folder" + + rm -rf "${BATS_TEST_TMPDIR}/gcs-cache" + rm -rf "${BATS_TEST_TMPDIR}/new-folder" + rm -rf "${BATS_TEST_TMPDIR}/other-folder" +} + +@test 'Prefix is used when environment is set' { + export BUILDKITE_PLUGIN_GCS_CACHE_PREFIX=my-prefix + + stub gsutil \ + 'ls gs://my-bucket/my-prefix/test-key\* : echo "gs://my-bucket/my-prefix/test-key"' \ + 'ls gs://my-bucket/test-key\* : exit 1' + + run "${PWD}/backends/cache_gcs" exists test-key + + assert_success + assert_output '' + + unset BUILDKITE_PLUGIN_GCS_CACHE_PREFIX + + run "${PWD}/backends/cache_gcs" exists test-key + + assert_failure + assert_output '' + + unstub gsutil +} + +@test 'gcloud storage CLI works when selected' { + export BUILDKITE_PLUGIN_GCS_CACHE_CLI=gcloud + + stub gcloud \ + 'storage --help : true' \ + 'storage ls \* : echo "gs://my-bucket/existing"' + + run "${PWD}/backends/cache_gcs" exists existing + + assert_success + assert_output '' + + unstub gcloud +} + +@test 'gcloud storage quiet flag passed when environment is set' { + export BUILDKITE_PLUGIN_GCS_CACHE_CLI=gcloud + export BUILDKITE_PLUGIN_GCS_CACHE_QUIET=1 + + stub gcloud \ + 'storage --help : true' \ + 'storage --verbosity=none rsync -r -d \* \* : echo ' \ + 'storage --verbosity=none ls \* : exit 1 ' \ + 'storage --verbosity=none rsync -r -d \* \* : echo ' \ + 'storage rsync -r -d \* \* : echo ' \ + 'storage ls \* : exit 1 ' \ + 'storage rsync -r -d \* \* : echo ' + + run "${PWD}/backends/cache_gcs" save from to + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" get from to + + assert_success + assert_output '' + + unset BUILDKITE_PLUGIN_GCS_CACHE_QUIET + + run "${PWD}/backends/cache_gcs" save from to + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" get from to + + assert_success + assert_output '' + + unstub gcloud +} + +@test 'gcloud storage file operations work' { + export BUILDKITE_PLUGIN_GCS_CACHE_CLI=gcloud + touch "${BATS_TEST_TMPDIR}/new-file" + mkdir "${BATS_TEST_TMPDIR}/gcs-cache" + + stub gcloud \ + 'storage --help : true' \ + "storage ls \* : exit 1" \ + "storage cp \* \* : ln -s \$3 $BATS_TEST_TMPDIR/gcs-cache/\$(echo \$4 | md5sum | cut -c-32)" \ + "storage ls \* : echo 'gs://my-bucket/new-file'" \ + 'storage ls \* : exit 1 ' \ + "storage cp \* \* : cp -r $BATS_TEST_TMPDIR/gcs-cache/\$(echo \$3 | md5sum | cut -c-32) \$4" + + run "${PWD}/backends/cache_gcs" exists new-file + + assert_failure + assert_output '' + + run "${PWD}/backends/cache_gcs" save new-file "${BATS_TEST_TMPDIR}/new-file" + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" exists new-file + + assert_success + assert_output '' + + run "${PWD}/backends/cache_gcs" get new-file "${BATS_TEST_TMPDIR}/other-file" + + assert_success + assert_output '' + + diff "${BATS_TEST_TMPDIR}/new-file" "${BATS_TEST_TMPDIR}/other-file" + + unstub gcloud + rm -rf "${BATS_TEST_TMPDIR}/gcs-cache" + rm -rf "${BATS_TEST_TMPDIR}/new-file" +} \ No newline at end of file