Merge pull request #75 from buildkite-plugins/toote_multiple_save_levels

Multiple save levels

Author: pzeballos

README.md

@@ -32,9 +32,11 @@ The file or folder to cache.
 
 The maximum caching level to restore, if available. See [the available caching levels](#caching-levels)
 
-#### `save` (string, specific values)
+#### `save` (string or array of strings, specific values)
 
-The level to use for saving the cache. See [the available caching levels](#caching-levels)
+The level(s) to use for saving the cache. See [the available caching levels](#caching-levels).
+
+You can specify multiple levels in an array to save the same artifact as a cache for all those levels.
 
 ## Options
 
@@ -118,7 +120,7 @@ You can always have more complicated logic by using the plugin multiple times wi
 * second step:
   - will restore the file-level cache of the `node_modules` folder saved by the first step and run `npm test`
 * third step (that will only run on the `master` branch):
-  - will restore the file-level cache saved by the first step, run `npm run deploy` and finally save the contents of the `node_modules` folder as a pipeline-level cache for usage as a basis even when the lockfile changes (in the first step)
+  - will restore the file-level cache saved by the first step, run `npm run deploy` and finally save the contents of the `node_modules` folder as both a pipeline-level and global (all-level) cache for usage as a basis even when the lockfile changes (in the first step)
 
 ```yaml
 steps:
@@ -147,7 +149,9 @@ steps:
           manifest: package-lock.json
           path: node_modules
           restore: file
-          save: pipeline
+          save:
+            - pipeline
+            - all
 
 ```
 

hooks/post-command

@@ -15,22 +15,29 @@ if [ -z "${CACHE_PATH}" ] ; then
   exit 1
 fi
 
-LEVEL=$(plugin_read_config SAVE 'no')
-if [ "${LEVEL}" = 'no' ]; then
-  echo 'Cache not setup for saving'
-  exit 0
-elif [ "${LEVEL}" = 'file' ] && [ -z "$(plugin_read_config MANIFEST)" ]; then
-  echo "+++ 🚨 Missing manifest option in the cache plugin for file-level saving"
-  exit 1
-fi
-
 COMPRESS=$(plugin_read_config COMPRESSION 'none')
 if ! validate_compression "${COMPRESS}"; then
   echo "+++ 🚨 Invalid value for compression option"
   exit 1
 fi
 
-KEY=$(build_key "${LEVEL}" "${CACHE_PATH}" "${COMPRESS}")
+SAVE_LEVELS=()
+if plugin_read_list_into_result SAVE; then
+  for LEVEL in "${result[@]}"; do
+    SAVE_LEVELS+=("${LEVEL}")
+
+    # this validates the level as well
+    KEY=$(build_key "${LEVEL}" "${CACHE_PATH}" "${COMPRESS}")
+
+    if [ "${LEVEL}" = 'file' ] && [ -z "$(plugin_read_config MANIFEST)" ]; then
+      echo "+++ 🚨 Missing manifest option in the cache plugin for file-level saving"
+      exit 1
+    fi
+  done
+else
+  echo 'Cache not setup for saving'
+  exit 0
+fi
 
 if compression_active; then
   ACTUAL_PATH=$(mktemp)
@@ -39,5 +46,9 @@ else
   ACTUAL_PATH="${CACHE_PATH}"
 fi
 
-echo "Saving ${LEVEL}-level cache of ${CACHE_PATH}"
-backend_exec save "${KEY}" "${ACTUAL_PATH}"
+for LEVEL in "${SAVE_LEVELS[@]}"; do
+  KEY=$(build_key "${LEVEL}" "${CACHE_PATH}" "${COMPRESS}")
+
+  echo "Saving ${LEVEL}-level cache of ${CACHE_PATH}"
+  backend_exec save "${KEY}" "${ACTUAL_PATH}"
+done

plugin.yml

@@ -3,6 +3,15 @@ description: Persist cache in between build steps
 author: https://github.com/buildkite-plugins
 requirements: []
 configuration:
+  $defs:
+    valid_levels:
+      type: string
+      enum:
+        - file
+        - step
+        - branch
+        - pipeline
+        - all
   properties:
     backend:
       type: string
@@ -17,21 +26,15 @@ configuration:
     path:
       type: string
     restore:
-      type: string
-      enum:
-        - file
-        - step
-        - branch
-        - pipeline
-        - all
+      $ref: '#/$defs/valid_levels'
     save:
-      type: string
-      enum:
-        - file
-        - step
-        - branch
-        - pipeline
-        - all
+      oneOf:
+        - $ref: '#/$defs/valid_levels'
+        - type: array
+          items:
+            $ref: '#/$defs/valid_levels'
   additionalProperties: false
-  required:
-    - path
+  anyOf:
+    - required: [ path, save ]
+    - required: [ path, restore ]
+

tests/post-command-tgz.bats

@@ -85,4 +85,19 @@ teardown() {
   assert_success
   assert_output --partial 'Saving all-level cache'
   assert_output --partial 'Compressing tests/data/my_files with tgz'
-}
+}
+
+@test "Multiple level saving" {
+  export BUILDKITE_PLUGIN_CACHE_SAVE_0=all
+  export BUILDKITE_PLUGIN_CACHE_SAVE_1=pipeline
+
+  # add an extra save, but tar should still be called only once
+  stub cache_dummy \
+    "save \* \* : echo saving \$3 in \$2"
+
+  run "$PWD/hooks/post-command"
+
+  assert_success
+  assert_output --partial 'Saving all-level cache'
+  assert_output --partial 'Saving pipeline-level cache'
+}

tests/post-command-zip.bats

@@ -87,3 +87,18 @@ teardown() {
   assert_output --partial 'Saving all-level cache'
   assert_output --partial 'Compressing tests/data/my_files with zip'
 }
+
+@test "Multiple level saving" {
+  export BUILDKITE_PLUGIN_CACHE_SAVE_0=all
+  export BUILDKITE_PLUGIN_CACHE_SAVE_1=pipeline
+
+  # add an extra save, but zip should still be called only once
+  stub cache_dummy \
+    "save \* \* : echo saving \$3 in \$2"
+
+  run "$PWD/hooks/post-command"
+
+  assert_success
+  assert_output --partial 'Saving all-level cache'
+  assert_output --partial 'Saving pipeline-level cache'
+}

tests/post-command.bats

@@ -137,4 +137,45 @@ teardown() {
   assert_output --partial 'Saving all-level cache'
 
   unstub cache_dummy
-}
+}
+
+@test "Multiple level saving" {
+  export BUILDKITE_PLUGIN_CACHE_SAVE_0=all
+  export BUILDKITE_PLUGIN_CACHE_SAVE_1=pipeline
+
+  stub cache_dummy \
+    "save \* \* : echo saving \$3 in \$2" \
+    "save \* \* : echo saving \$3 in \$2"
+
+  run "$PWD/hooks/post-command"
+
+  assert_success
+  assert_output --partial 'Saving all-level cache'
+  assert_output --partial 'Saving pipeline-level cache'
+
+  unstub cache_dummy
+}
+
+@test "Multiple level file without manifest fails" {
+  export BUILDKITE_PLUGIN_CACHE_SAVE_0=all
+  export BUILDKITE_PLUGIN_CACHE_SAVE_1=file
+
+  run "$PWD/hooks/post-command"
+
+  assert_failure
+
+  assert_output --partial 'Missing manifest option'
+  refute_output --partial 'Saving file-level cache'
+}
+
+@test "Multiple level containing invalid one fails" {
+  export BUILDKITE_PLUGIN_CACHE_SAVE_0=pipeline
+  export BUILDKITE_PLUGIN_CACHE_SAVE_1=unreal
+
+  run "$PWD/hooks/post-command"
+
+  assert_failure
+
+  assert_output --partial 'Invalid cache level unreal'
+  refute_output --partial 'Saving pipeline-level cache'
+}