diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ae7db338588..3a4c2c042f4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,3 +9,4 @@ * @googleapis/cloud-sdk-nodejs-team /handwritten/bigquery @googleapis/bigquery-team /handwritten/cloud-profiler @googleapis/cloud-profiler-team +/handwritten/storage @googleapis/gcs-team diff --git a/.github/workflows/conformance-test.yaml b/.github/workflows/conformance-test.yaml new file mode 100644 index 00000000000..845d280462d --- /dev/null +++ b/.github/workflows/conformance-test.yaml @@ -0,0 +1,21 @@ +on: + push: + branches: + - main + paths: + - 'handwritten/storage/**' + pull_request: + paths: + - 'handwritten/storage/**' +name: conformance +jobs: + conformance-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 14 + - run: node --version + - run: cd handwritten/storage && npm install + - run: cd handwritten/storage && npm run conformance-test diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 9afa1ee8000..ce0a192bec3 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,9 +1,10 @@ { - "handwritten/datastore": "10.1.0", - "handwritten/logging-winston": "6.0.1", "handwritten/bigquery": "8.2.0", "handwritten/cloud-profiler": "6.0.4", + "handwritten/datastore": "10.1.0", "handwritten/logging-bunyan": "5.1.1", + "handwritten/logging-winston": "6.0.1", + "handwritten/storage": "7.19.0", "packages/gapic-node-processing": "0.1.6", "packages/google-ads-admanager": "0.5.0", "packages/google-ads-datamanager": "0.1.0", diff --git a/handwritten/storage/.OwlBot.yaml b/handwritten/storage/.OwlBot.yaml new file mode 100644 index 00000000000..5781d72d218 --- /dev/null +++ b/handwritten/storage/.OwlBot.yaml @@ -0,0 +1,19 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +deep-preserve-regex: + - /.kokoro/samples-test.sh + + +begin-after-commit-hash: 674a41e0de2869f44f45eb7b1a605852a5394bba diff --git a/handwritten/storage/.eslintignore b/handwritten/storage/.eslintignore new file mode 100644 index 00000000000..ea5b04aebe6 --- /dev/null +++ b/handwritten/storage/.eslintignore @@ -0,0 +1,7 @@ +**/node_modules +**/coverage +test/fixtures +build/ +docs/ +protos/ +samples/generated/ diff --git a/handwritten/storage/.eslintrc.json b/handwritten/storage/.eslintrc.json new file mode 100644 index 00000000000..78215349546 --- /dev/null +++ b/handwritten/storage/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "./node_modules/gts" +} diff --git a/handwritten/storage/.gitattributes b/handwritten/storage/.gitattributes new file mode 100644 index 00000000000..33739cb74e4 --- /dev/null +++ b/handwritten/storage/.gitattributes @@ -0,0 +1,4 @@ +*.ts text eol=lf +*.js text eol=lf +protos/* linguist-generated +**/api-extractor.json linguist-language=JSON-with-Comments diff --git a/handwritten/storage/.gitignore b/handwritten/storage/.gitignore new file mode 100644 index 00000000000..6ff88ca912c --- /dev/null +++ b/handwritten/storage/.gitignore @@ -0,0 +1,16 @@ +**/*.log +**/node_modules +.coverage +.nyc_output +docs/ +out/ +build/ +system-test/secrets.js +system-test/*key.json +*.lock +.DS_Store +google-cloud-logging-winston-*.tgz +google-cloud-logging-bunyan-*.tgz +.vscode +package-lock.json +__pycache__ diff --git a/handwritten/storage/.jsdoc.json b/handwritten/storage/.jsdoc.json new file mode 100644 index 00000000000..1873939a0ae --- /dev/null +++ b/handwritten/storage/.jsdoc.json @@ -0,0 +1,29 @@ +{ + "opts": { + "readme": "./README.md", + "package": "./package.json", + "template": "./node_modules/jsdoc-fresh", + "recurse": true, + "verbose": true, + "destination": "./docs/" + }, + "plugins": ["plugins/markdown", "jsdoc-region-tag"], + "source": { + "excludePattern": "(^|\\/|\\\\)[._]", + "include": ["build/cjs/src"], + "includePattern": "\\.js$" + }, + "templates": { + "copyright": "Copyright 2019 Google, LLC.", + "includeDate": false, + "sourceFiles": false, + "systemName": "@google-cloud/storage", + "theme": "lumen", + "default": { + "outputSourceFiles": false + } + }, + "markdown": { + "idInHeadings": true + } +} diff --git a/handwritten/storage/.kokoro/.gitattributes b/handwritten/storage/.kokoro/.gitattributes new file mode 100644 index 00000000000..87acd4f484e --- /dev/null +++ b/handwritten/storage/.kokoro/.gitattributes @@ -0,0 +1 @@ +* linguist-generated=true diff --git a/handwritten/storage/.kokoro/common.cfg b/handwritten/storage/.kokoro/common.cfg new file mode 100644 index 00000000000..f34b972eed8 --- /dev/null +++ b/handwritten/storage/.kokoro/common.cfg @@ -0,0 +1,24 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-storage/handwritten/storage/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/test.sh" +} diff --git a/handwritten/storage/.kokoro/conformance-test.sh b/handwritten/storage/.kokoro/conformance-test.sh new file mode 100644 index 00000000000..190f6dfc0e8 --- /dev/null +++ b/handwritten/storage/.kokoro/conformance-test.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +cd $(dirname $0)/.. + +npm install +# If tests are running against main branch, configure flakybot +# to open issues on failures: +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]] || [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"nightly"* ]]; then + export MOCHA_REPORTER_OUTPUT=test_output_sponge_log.xml + export MOCHA_REPORTER=xunit + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP +fi + +npm run conformance-test diff --git a/handwritten/storage/.kokoro/continuous/node14/common.cfg b/handwritten/storage/.kokoro/continuous/node14/common.cfg new file mode 100644 index 00000000000..9df80119791 --- /dev/null +++ b/handwritten/storage/.kokoro/continuous/node14/common.cfg @@ -0,0 +1,24 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-storage/handwritten/storage/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:14-user" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/test.sh" +} diff --git a/handwritten/storage/.kokoro/continuous/node14/lint.cfg b/handwritten/storage/.kokoro/continuous/node14/lint.cfg new file mode 100644 index 00000000000..7a49e5d4776 --- /dev/null +++ b/handwritten/storage/.kokoro/continuous/node14/lint.cfg @@ -0,0 +1,4 @@ +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/lint.sh" +} diff --git a/handwritten/storage/.kokoro/continuous/node14/samples-test.cfg b/handwritten/storage/.kokoro/continuous/node14/samples-test.cfg new file mode 100644 index 00000000000..5e9326097d0 --- /dev/null +++ b/handwritten/storage/.kokoro/continuous/node14/samples-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/samples-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/handwritten/storage/.kokoro/continuous/node14/system-test.cfg b/handwritten/storage/.kokoro/continuous/node14/system-test.cfg new file mode 100644 index 00000000000..240576acdaf --- /dev/null +++ b/handwritten/storage/.kokoro/continuous/node14/system-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/system-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account,client-library-test-universe-domain-credential" +} \ No newline at end of file diff --git a/handwritten/storage/.kokoro/continuous/node14/test.cfg b/handwritten/storage/.kokoro/continuous/node14/test.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/handwritten/storage/.kokoro/continuous/node18/common.cfg b/handwritten/storage/.kokoro/continuous/node18/common.cfg new file mode 100644 index 00000000000..f34b972eed8 --- /dev/null +++ b/handwritten/storage/.kokoro/continuous/node18/common.cfg @@ -0,0 +1,24 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-storage/handwritten/storage/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/test.sh" +} diff --git a/handwritten/storage/.kokoro/continuous/node18/lint.cfg b/handwritten/storage/.kokoro/continuous/node18/lint.cfg new file mode 100644 index 00000000000..7a49e5d4776 --- /dev/null +++ b/handwritten/storage/.kokoro/continuous/node18/lint.cfg @@ -0,0 +1,4 @@ +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/lint.sh" +} diff --git a/handwritten/storage/.kokoro/continuous/node18/samples-test.cfg b/handwritten/storage/.kokoro/continuous/node18/samples-test.cfg new file mode 100644 index 00000000000..5e9326097d0 --- /dev/null +++ b/handwritten/storage/.kokoro/continuous/node18/samples-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/samples-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/handwritten/storage/.kokoro/continuous/node18/system-test.cfg b/handwritten/storage/.kokoro/continuous/node18/system-test.cfg new file mode 100644 index 00000000000..e1fcec9e9ad --- /dev/null +++ b/handwritten/storage/.kokoro/continuous/node18/system-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/system-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/handwritten/storage/.kokoro/continuous/node18/test.cfg b/handwritten/storage/.kokoro/continuous/node18/test.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/handwritten/storage/.kokoro/docs.sh b/handwritten/storage/.kokoro/docs.sh new file mode 100755 index 00000000000..85901242b5e --- /dev/null +++ b/handwritten/storage/.kokoro/docs.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +cd $(dirname $0)/.. + +npm install + +npm run docs-test diff --git a/handwritten/storage/.kokoro/lint.sh b/handwritten/storage/.kokoro/lint.sh new file mode 100755 index 00000000000..c7ffa6438b0 --- /dev/null +++ b/handwritten/storage/.kokoro/lint.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global +export PATH="${NPM_CONFIG_PREFIX}/bin:${PATH}" + +# Ensure the npm global directory is writable, otherwise rebuild `npm` +mkdir -p ${NPM_CONFIG_PREFIX}/lib +npm config -g ls || npm i -g npm@`npm --version` + +cd $(dirname $0)/.. + +npm install + +# Install and link samples +if [ -f samples/package.json ]; then + cd samples/ + npm link ../ + npm install + cd .. +fi + +npm run lint diff --git a/handwritten/storage/.kokoro/populate-secrets.sh b/handwritten/storage/.kokoro/populate-secrets.sh new file mode 100755 index 00000000000..deb2b199eb4 --- /dev/null +++ b/handwritten/storage/.kokoro/populate-secrets.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Copyright 2020 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is called in the early stage of `trampoline_v2.sh` to +# populate secrets needed for the CI builds. + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + +# Populates requested secrets set in SECRET_MANAGER_KEYS + +# In Kokoro CI builds, we use the service account attached to the +# Kokoro VM. This means we need to setup auth on other CI systems. +# For local run, we just use the gcloud command for retrieving the +# secrets. + +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + GCLOUD_COMMANDS=( + "docker" + "run" + "--entrypoint=gcloud" + "--volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR}" + "gcr.io/google.com/cloudsdktool/cloud-sdk" + ) + if [[ "${TRAMPOLINE_CI:-}" == "kokoro" ]]; then + SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" + else + echo "Authentication for this CI system is not implemented yet." + exit 2 + # TODO: Determine appropriate SECRET_LOCATION and the GCLOUD_COMMANDS. + fi +else + # For local run, use /dev/shm or temporary directory for + # KOKORO_GFILE_DIR. + if [[ -d "/dev/shm" ]]; then + export KOKORO_GFILE_DIR=/dev/shm + else + export KOKORO_GFILE_DIR=$(mktemp -d -t ci-XXXXXXXX) + fi + SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" + GCLOUD_COMMANDS=("gcloud") +fi + +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} + +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + "${GCLOUD_COMMANDS[@]}" \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret $key > \ + "$SECRET_LOCATION/$key" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + exit 2 + fi +done diff --git a/handwritten/storage/.kokoro/pre-samples-test.sh b/handwritten/storage/.kokoro/pre-samples-test.sh new file mode 100644 index 00000000000..a1ffa0cea3b --- /dev/null +++ b/handwritten/storage/.kokoro/pre-samples-test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. .kokoro/setup-vars.sh diff --git a/handwritten/storage/.kokoro/pre-system-test.sh b/handwritten/storage/.kokoro/pre-system-test.sh new file mode 100755 index 00000000000..abc85f9d4b8 --- /dev/null +++ b/handwritten/storage/.kokoro/pre-system-test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. .kokoro/setup-vars.sh + +# Lease a second service account for testing listing with multiple service accounts +export HMAC_KEY_TEST_SECOND_SERVICE_ACCOUNT=$(./gimmeproj -project=$HMAC_PROJECT lease 15m) +# Add to the list of leased service account for clean up after tests +export LEASED_SERVICE_ACCOUNTS="$LEASED_SERVICE_ACCOUNTS $HMAC_KEY_TEST_SECOND_SERVICE_ACCOUNT" diff --git a/handwritten/storage/.kokoro/presubmit/node14/common.cfg b/handwritten/storage/.kokoro/presubmit/node14/common.cfg new file mode 100644 index 00000000000..9df80119791 --- /dev/null +++ b/handwritten/storage/.kokoro/presubmit/node14/common.cfg @@ -0,0 +1,24 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-storage/handwritten/storage/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:14-user" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/test.sh" +} diff --git a/handwritten/storage/.kokoro/presubmit/node14/samples-test.cfg b/handwritten/storage/.kokoro/presubmit/node14/samples-test.cfg new file mode 100644 index 00000000000..5e9326097d0 --- /dev/null +++ b/handwritten/storage/.kokoro/presubmit/node14/samples-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/samples-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/handwritten/storage/.kokoro/presubmit/node14/system-test.cfg b/handwritten/storage/.kokoro/presubmit/node14/system-test.cfg new file mode 100644 index 00000000000..240576acdaf --- /dev/null +++ b/handwritten/storage/.kokoro/presubmit/node14/system-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/system-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account,client-library-test-universe-domain-credential" +} \ No newline at end of file diff --git a/handwritten/storage/.kokoro/presubmit/node14/test.cfg b/handwritten/storage/.kokoro/presubmit/node14/test.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/handwritten/storage/.kokoro/presubmit/node18/common.cfg b/handwritten/storage/.kokoro/presubmit/node18/common.cfg new file mode 100644 index 00000000000..f34b972eed8 --- /dev/null +++ b/handwritten/storage/.kokoro/presubmit/node18/common.cfg @@ -0,0 +1,24 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-storage/handwritten/storage/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/test.sh" +} diff --git a/handwritten/storage/.kokoro/presubmit/node18/samples-test.cfg b/handwritten/storage/.kokoro/presubmit/node18/samples-test.cfg new file mode 100644 index 00000000000..5e9326097d0 --- /dev/null +++ b/handwritten/storage/.kokoro/presubmit/node18/samples-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/samples-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/handwritten/storage/.kokoro/presubmit/node18/system-test.cfg b/handwritten/storage/.kokoro/presubmit/node18/system-test.cfg new file mode 100644 index 00000000000..e1fcec9e9ad --- /dev/null +++ b/handwritten/storage/.kokoro/presubmit/node18/system-test.cfg @@ -0,0 +1,12 @@ +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-nodejs" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/system-test.sh" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "long-door-651-kokoro-system-test-service-account" +} \ No newline at end of file diff --git a/handwritten/storage/.kokoro/presubmit/node18/test.cfg b/handwritten/storage/.kokoro/presubmit/node18/test.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/handwritten/storage/.kokoro/presubmit/windows/common.cfg b/handwritten/storage/.kokoro/presubmit/windows/common.cfg new file mode 100644 index 00000000000..d6e25e0b1b8 --- /dev/null +++ b/handwritten/storage/.kokoro/presubmit/windows/common.cfg @@ -0,0 +1,2 @@ +# Format: //devtools/kokoro/config/proto/build.proto + diff --git a/handwritten/storage/.kokoro/presubmit/windows/test.cfg b/handwritten/storage/.kokoro/presubmit/windows/test.cfg new file mode 100644 index 00000000000..eb8febf58d9 --- /dev/null +++ b/handwritten/storage/.kokoro/presubmit/windows/test.cfg @@ -0,0 +1,2 @@ +# Use the test file directly +build_file: "nodejs-storage/handwritten/storage/.kokoro/test.bat" diff --git a/handwritten/storage/.kokoro/publish.sh b/handwritten/storage/.kokoro/publish.sh new file mode 100755 index 00000000000..ca1d47af347 --- /dev/null +++ b/handwritten/storage/.kokoro/publish.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +# Start the releasetool reporter +python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script + +cd $(dirname $0)/.. + +NPM_TOKEN=$(cat $KOKORO_KEYSTORE_DIR/73713_google-cloud-npm-token-1) +echo "//wombat-dressing-room.appspot.com/:_authToken=${NPM_TOKEN}" > ~/.npmrc + +npm install +npm pack . +# npm provides no way to specify, observe, or predict the name of the tarball +# file it generates. We have to look in the current directory for the freshest +# .tgz file. +TARBALL=$(ls -1 -t *.tgz | head -1) + +npm publish --access=public --registry=https://wombat-dressing-room.appspot.com "$TARBALL" + +# Kokoro collects *.tgz and package-lock.json files and stores them in Placer +# so we can generate SBOMs and attestations. +# However, we *don't* want Kokoro to collect package-lock.json and *.tgz files +# that happened to be installed with dependencies. +find node_modules -name package-lock.json -o -name "*.tgz" | xargs rm -f \ No newline at end of file diff --git a/handwritten/storage/.kokoro/release/common.cfg b/handwritten/storage/.kokoro/release/common.cfg new file mode 100644 index 00000000000..3ba2eb095fe --- /dev/null +++ b/handwritten/storage/.kokoro/release/common.cfg @@ -0,0 +1,8 @@ +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} diff --git a/handwritten/storage/.kokoro/release/docs-devsite.cfg b/handwritten/storage/.kokoro/release/docs-devsite.cfg new file mode 100644 index 00000000000..aed93dc0989 --- /dev/null +++ b/handwritten/storage/.kokoro/release/docs-devsite.cfg @@ -0,0 +1,26 @@ +# service account used to publish up-to-date docs. +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} + +# doc publications use a Python image. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-storage/handwritten/storage/.kokoro/trampoline_v2.sh" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/release/docs-devsite.sh" +} diff --git a/handwritten/storage/.kokoro/release/docs-devsite.sh b/handwritten/storage/.kokoro/release/docs-devsite.sh new file mode 100755 index 00000000000..81a89f6c172 --- /dev/null +++ b/handwritten/storage/.kokoro/release/docs-devsite.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +if [[ -z "$CREDENTIALS" ]]; then + # if CREDENTIALS are explicitly set, assume we're testing locally + # and don't set NPM_CONFIG_PREFIX. + export NPM_CONFIG_PREFIX=${HOME}/.npm-global + export PATH="$PATH:${NPM_CONFIG_PREFIX}/bin" + cd $(dirname $0)/../.. +fi + +npm install +npm install --no-save @google-cloud/cloud-rad@^0.4.0 +# publish docs to devsite +npx @google-cloud/cloud-rad . cloud-rad diff --git a/handwritten/storage/.kokoro/release/docs.cfg b/handwritten/storage/.kokoro/release/docs.cfg new file mode 100644 index 00000000000..dde75bb7342 --- /dev/null +++ b/handwritten/storage/.kokoro/release/docs.cfg @@ -0,0 +1,26 @@ +# service account used to publish up-to-date docs. +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} + +# doc publications use a Python image. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-storage/handwritten/storage/.kokoro/trampoline_v2.sh" + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/release/docs.sh" +} diff --git a/handwritten/storage/.kokoro/release/docs.sh b/handwritten/storage/.kokoro/release/docs.sh new file mode 100755 index 00000000000..e9079a60530 --- /dev/null +++ b/handwritten/storage/.kokoro/release/docs.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# build jsdocs (Python is installed on the Node 18 docker image). +if [[ -z "$CREDENTIALS" ]]; then + # if CREDENTIALS are explicitly set, assume we're testing locally + # and don't set NPM_CONFIG_PREFIX. + export NPM_CONFIG_PREFIX=${HOME}/.npm-global + export PATH="$PATH:${NPM_CONFIG_PREFIX}/bin" + cd $(dirname $0)/../.. +fi +npm install +npm run docs + +# create docs.metadata, based on package.json and .repo-metadata.json. +npm i json@9.0.6 -g +python3 -m docuploader create-metadata \ + --name=$(cat .repo-metadata.json | json name) \ + --version=$(cat package.json | json version) \ + --language=$(cat .repo-metadata.json | json language) \ + --distribution-name=$(cat .repo-metadata.json | json distribution_name) \ + --product-page=$(cat .repo-metadata.json | json product_documentation) \ + --github-repository=$(cat .repo-metadata.json | json repo) \ + --issue-tracker=$(cat .repo-metadata.json | json issue_tracker) +cp docs.metadata ./docs/docs.metadata + +# deploy the docs. +if [[ -z "$CREDENTIALS" ]]; then + CREDENTIALS=${KOKORO_KEYSTORE_DIR}/73713_docuploader_service_account +fi +if [[ -z "$BUCKET" ]]; then + BUCKET=docs-staging +fi +python3 -m docuploader upload ./docs --credentials $CREDENTIALS --staging-bucket $BUCKET diff --git a/handwritten/storage/.kokoro/release/publish.cfg b/handwritten/storage/.kokoro/release/publish.cfg new file mode 100644 index 00000000000..f7e75eae268 --- /dev/null +++ b/handwritten/storage/.kokoro/release/publish.cfg @@ -0,0 +1,51 @@ +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-npm-token-1" + } + } +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,client-library-test-universe-domain-credential" +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "nodejs-storage/handwritten/storage/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/node:18-user" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/nodejs-storage/handwritten/storage/.kokoro/publish.sh" +} + +# Store the packages we uploaded to npmjs.org and their corresponding +# package-lock.jsons in Placer. That way, we have a record of exactly +# what we published, and which version of which tools we used to publish +# it, which we can use to generate SBOMs and attestations. +action { + define_artifacts { + regex: "github/**/*.tgz" + regex: "github/**/package-lock.json" + strip_prefix: "github" + } +} diff --git a/handwritten/storage/.kokoro/samples-test.sh b/handwritten/storage/.kokoro/samples-test.sh new file mode 100755 index 00000000000..d195fbf7cf3 --- /dev/null +++ b/handwritten/storage/.kokoro/samples-test.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +# Ensure the npm global directory is writable, otherwise rebuild `npm` +mkdir -p $NPM_CONFIG_PREFIX +npm config -g ls || npm i -g npm@`npm --version` + +# Setup service account credentials. +export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/secret_manager/long-door-651-kokoro-system-test-service-account +export GCLOUD_PROJECT=long-door-651 + +cd $(dirname $0)/.. + +# Run a pre-test hook, if a pre-samples-test.sh is in the project +if [ -f .kokoro/pre-samples-test.sh ]; then + set +x + . .kokoro/pre-samples-test.sh + set -x +fi + +if [ -f samples/package.json ]; then + npm install + + # Install and link samples + cd samples/ + npm link ../ + npm install + cd .. + # If tests are running against main branch, configure flakybot + # to open issues on failures: + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]] || [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"nightly"* ]]; then + export MOCHA_REPORTER_OUTPUT=test_output_sponge_log.xml + export MOCHA_REPORTER=xunit + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP + fi + + npm run samples-test +fi + +# codecov combines coverage across integration and unit tests. Include +# the logic below for any environment you wish to collect coverage for: +COVERAGE_NODE=18 +if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then + NYC_BIN=./node_modules/nyc/bin/nyc.js + if [ -f "$NYC_BIN" ]; then + $NYC_BIN report || true + fi + bash $KOKORO_GFILE_DIR/codecov.sh +else + echo "coverage is only reported for Node $COVERAGE_NODE" +fi diff --git a/handwritten/storage/.kokoro/setup-vars.sh b/handwritten/storage/.kokoro/setup-vars.sh new file mode 100644 index 00000000000..b6ce41d93c9 --- /dev/null +++ b/handwritten/storage/.kokoro/setup-vars.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# nodejs-storage/handwritten/storage's system tests require additional project and +# system test key +export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/storage-key.json +export GCN_STORAGE_2ND_PROJECT_ID=gcloud-node-whitelist-ci-tests +export GCN_STORAGE_2ND_PROJECT_KEY=${KOKORO_GFILE_DIR}/no-whitelist-key.json + +export GOOGLE_CLOUD_KMS_KEY_ASIA="projects/long-door-651/locations/asia/keyRings/test-key-asia/cryptoKeys/test-key-asia" +export GOOGLE_CLOUD_KMS_KEY_US="projects/long-door-651/locations/us/keyRings/test-key-us/cryptoKeys/test-key-us" + +# For testing SA HMAC +export HMAC_PROJECT=gimme-acc +curl https://storage.googleapis.com/gimme-proj/linux_amd64/gimmeproj > gimmeproj +chmod +x gimmeproj +./gimmeproj version + +export HMAC_KEY_TEST_SERVICE_ACCOUNT=$(./gimmeproj -project=$HMAC_PROJECT lease 15m) +echo Leased service account: $HMAC_KEY_TEST_SERVICE_ACCOUNT +export LEASED_SERVICE_ACCOUNTS=$HMAC_KEY_TEST_SERVICE_ACCOUNT + +cleanup_service_accounts () { + for i in $LEASED_SERVICE_ACCOUNTS; do + ./gimmeproj -project=$HMAC_PROJECT "done" $i + done +} + +trap cleanup_service_accounts EXIT diff --git a/handwritten/storage/.kokoro/system-test.sh b/handwritten/storage/.kokoro/system-test.sh new file mode 100755 index 00000000000..6b69337ef07 --- /dev/null +++ b/handwritten/storage/.kokoro/system-test.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +# Setup service account credentials. +export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/secret_manager/long-door-651-kokoro-system-test-service-account +export GCLOUD_PROJECT=long-door-651 + +# For universe domain testing +export TEST_UNIVERSE_DOMAIN_CREDENTIAL=${KOKORO_GFILE_DIR}/secret_manager/client-library-test-universe-domain-credential +export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-domain) +export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id) +export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location) + +cd $(dirname $0)/.. + +# Run a pre-test hook, if a pre-system-test.sh is in the project +if [ -f .kokoro/pre-system-test.sh ]; then + set +x + . .kokoro/pre-system-test.sh + set -x +fi + +npm install + +# If tests are running against main branch, configure flakybot +# to open issues on failures: +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]] || [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"nightly"* ]]; then + export MOCHA_REPORTER_OUTPUT=test_output_sponge_log.xml + export MOCHA_REPORTER=xunit + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP +fi + +npm run system-test + +# codecov combines coverage across integration and unit tests. Include +# the logic below for any environment you wish to collect coverage for: +COVERAGE_NODE=18 +if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then + NYC_BIN=./node_modules/nyc/bin/nyc.js + if [ -f "$NYC_BIN" ]; then + $NYC_BIN report || true + fi + bash $KOKORO_GFILE_DIR/codecov.sh +else + echo "coverage is only reported for Node $COVERAGE_NODE" +fi diff --git a/handwritten/storage/.kokoro/test.bat b/handwritten/storage/.kokoro/test.bat new file mode 100644 index 00000000000..caf825656c2 --- /dev/null +++ b/handwritten/storage/.kokoro/test.bat @@ -0,0 +1,33 @@ +@rem Copyright 2018 Google LLC. All rights reserved. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@echo "Starting Windows build" + +cd /d %~dp0 +cd .. + +@rem npm path is not currently set in our image, we should fix this next time +@rem we upgrade Node.js in the image: +SET PATH=%PATH%;/cygdrive/c/Program Files/nodejs/npm + +call nvm use 18 +call which node + +call npm install || goto :error +call npm run test || goto :error + +goto :EOF + +:error +exit /b 1 diff --git a/handwritten/storage/.kokoro/test.sh b/handwritten/storage/.kokoro/test.sh new file mode 100755 index 00000000000..0d9f6392a75 --- /dev/null +++ b/handwritten/storage/.kokoro/test.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +export NPM_CONFIG_PREFIX=${HOME}/.npm-global + +cd $(dirname $0)/.. + +npm install +# If tests are running against main branch, configure flakybot +# to open issues on failures: +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]] || [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"nightly"* ]]; then + export MOCHA_REPORTER_OUTPUT=test_output_sponge_log.xml + export MOCHA_REPORTER=xunit + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP +fi +# Unit tests exercise the entire API surface, which may include +# deprecation warnings: +export MOCHA_THROW_DEPRECATION=false +npm test + +# codecov combines coverage across integration and unit tests. Include +# the logic below for any environment you wish to collect coverage for: +COVERAGE_NODE=18 +if npx check-node-version@3.3.0 --silent --node $COVERAGE_NODE; then + NYC_BIN=./node_modules/nyc/bin/nyc.js + if [ -f "$NYC_BIN" ]; then + $NYC_BIN report || true + fi + bash $KOKORO_GFILE_DIR/codecov.sh +else + echo "coverage is only reported for Node $COVERAGE_NODE" +fi diff --git a/handwritten/storage/.kokoro/trampoline.sh b/handwritten/storage/.kokoro/trampoline.sh new file mode 100755 index 00000000000..f693a1ce7aa --- /dev/null +++ b/handwritten/storage/.kokoro/trampoline.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is not used any more, but we keep this file for making it +# easy to roll back. +# TODO: Remove this file from the template. + +set -eo pipefail + +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT + +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" diff --git a/handwritten/storage/.kokoro/trampoline_v2.sh b/handwritten/storage/.kokoro/trampoline_v2.sh new file mode 100755 index 00000000000..e1adcd24e9e --- /dev/null +++ b/handwritten/storage/.kokoro/trampoline_v2.sh @@ -0,0 +1,510 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# If you want to make a change to this file, consider doing so at: +# https://github.com/googlecloudplatform/docker-ci-helper +# +# This script is for running CI builds. For Kokoro builds, we +# set this script to `build_file` field in the Kokoro configuration. + +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. +# +# Here is an example for running this script. +# TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:18-user \ +# TRAMPOLINE_BUILD_FILE=.kokoro/system-test.sh \ +# .kokoro/trampoline_v2.sh + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.7" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" + # Contains path to build artifacts being executed. + "KOKORO_BUILD_ARTIFACTS_SUBDIR" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For flakybot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")/handwritten/storage" +else + PROJECT_ROOT="$(repo_root $(pwd))/handwritten/storage" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# Auto-injected conditional check +# Check if the package directory has changes. If not, skip tests. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # The package path is hardcoded during migration + RELATIVE_PKG_PATH="handwritten/storage" + + echo "Checking for changes in ${RELATIVE_PKG_PATH}..." + + # Determine the diff range based on the CI system/event + # Safe default: HEAD~1..HEAD + DIFF_RANGE="HEAD~1..HEAD" + + if git diff --quiet "${DIFF_RANGE}" -- "${RELATIVE_PKG_PATH}"; then + echo "No changes detected in ${RELATIVE_PKG_PATH}. Skipping tests." + exit 0 + else + echo "Changes detected in ${RELATIVE_PKG_PATH}. Proceeding with tests." + fi +fi + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}" | grep "${TRAMPOLINE_IMAGE%:*}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/handwritten/storage/.mocharc.cjs b/handwritten/storage/.mocharc.cjs new file mode 100644 index 00000000000..0b600509bed --- /dev/null +++ b/handwritten/storage/.mocharc.cjs @@ -0,0 +1,29 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +const config = { + "enable-source-maps": true, + "throw-deprecation": true, + "timeout": 10000, + "recursive": true +} +if (process.env.MOCHA_THROW_DEPRECATION === 'false') { + delete config['throw-deprecation']; +} +if (process.env.MOCHA_REPORTER) { + config.reporter = process.env.MOCHA_REPORTER; +} +if (process.env.MOCHA_REPORTER_OUTPUT) { + config['reporter-option'] = `output=${process.env.MOCHA_REPORTER_OUTPUT}`; +} +module.exports = config diff --git a/handwritten/storage/.nycrc b/handwritten/storage/.nycrc new file mode 100644 index 00000000000..b18d5472b62 --- /dev/null +++ b/handwritten/storage/.nycrc @@ -0,0 +1,24 @@ +{ + "report-dir": "./.coverage", + "reporter": ["text", "lcov"], + "exclude": [ + "**/*-test", + "**/.coverage", + "**/apis", + "**/benchmark", + "**/conformance", + "**/docs", + "**/samples", + "**/scripts", + "**/protos", + "**/test", + "**/*.d.ts", + ".jsdoc.js", + "**/.jsdoc.js", + "karma.conf.js", + "webpack-tests.config.js", + "webpack.config.js" + ], + "exclude-after-remap": false, + "all": true +} diff --git a/handwritten/storage/.prettierignore b/handwritten/storage/.prettierignore new file mode 100644 index 00000000000..9340ad9b86d --- /dev/null +++ b/handwritten/storage/.prettierignore @@ -0,0 +1,6 @@ +**/node_modules +**/coverage +test/fixtures +build/ +docs/ +protos/ diff --git a/handwritten/storage/.prettierrc.cjs b/handwritten/storage/.prettierrc.cjs new file mode 100644 index 00000000000..d1b95106f4c --- /dev/null +++ b/handwritten/storage/.prettierrc.cjs @@ -0,0 +1,17 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module.exports = { + ...require('gts/.prettierrc.json') +} diff --git a/handwritten/storage/.readme-partials.yaml b/handwritten/storage/.readme-partials.yaml new file mode 100644 index 00000000000..f764f6b95a3 --- /dev/null +++ b/handwritten/storage/.readme-partials.yaml @@ -0,0 +1,8 @@ +introduction: |- + > Node.js idiomatic client for [Cloud Storage][product-docs]. + + [Cloud Storage](https://cloud.google.com/storage/docs) allows world-wide + storage and retrieval of any amount of data at any time. You can use Google + Cloud Storage for a range of scenarios including serving website content, + storing data for archival and disaster recovery, or distributing large data + objects to users via direct download. diff --git a/handwritten/storage/.repo-metadata.json b/handwritten/storage/.repo-metadata.json new file mode 100644 index 00000000000..20b7ff6ce50 --- /dev/null +++ b/handwritten/storage/.repo-metadata.json @@ -0,0 +1,16 @@ +{ + "name": "storage", + "name_pretty": "Google Cloud Storage", + "product_documentation": "https://cloud.google.com/storage", + "client_documentation": "https://cloud.google.com/nodejs/docs/reference/storage/latest", + "issue_tracker": "https://issuetracker.google.com/savedsearches/559782", + "release_level": "stable", + "language": "nodejs", + "repo": "googleapis/google-cloud-node", + "distribution_name": "@google-cloud/storage", + "api_id": "storage-api.googleapis.com", + "requires_billing": true, + "codeowner_team": "@googleapis/gcs-team", + "api_shortname": "storage", + "library_type": "GAPIC_MANUAL" +} diff --git a/handwritten/storage/.trampolinerc b/handwritten/storage/.trampolinerc new file mode 100644 index 00000000000..2feed5b5844 --- /dev/null +++ b/handwritten/storage/.trampolinerc @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Template for .trampolinerc + +# Add required env vars here. +required_envvars+=( +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + "AUTORELEASE_PR" + "VERSION" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE="" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi + +# Secret Manager secrets. +source ${PROJECT_ROOT}/handwritten/storage/.kokoro/populate-secrets.sh diff --git a/handwritten/storage/CHANGELOG.md b/handwritten/storage/CHANGELOG.md new file mode 100644 index 00000000000..e6b755e5812 --- /dev/null +++ b/handwritten/storage/CHANGELOG.md @@ -0,0 +1,1824 @@ +# Changelog + +[npm history][1] + +[1]: https://www.npmjs.com/package/@google-cloud/storage?activeTab=versions + +## [7.19.0](https://github.com/googleapis/nodejs-storage/compare/v7.18.0...v7.19.0) (2026-02-05) + + +### Features + +* Enable full object checksum validation on JSON path ([#2687](https://github.com/googleapis/nodejs-storage/issues/2687)) ([08a8962](https://github.com/googleapis/nodejs-storage/commit/08a896240e2ee110c13a5803ce5a2fbf38fbedc4)) + + +### Bug Fixes + +* **deps:** Update dependency fast-xml-parser to v5 [security] ([#2713](https://github.com/googleapis/nodejs-storage/issues/2713)) ([420935a](https://github.com/googleapis/nodejs-storage/commit/420935a0334b0723ec052bb4d6b8527dd2acffea)) + +## [7.18.0](https://github.com/googleapis/nodejs-storage/compare/v7.17.3...v7.18.0) (2025-11-28) + + +### Features + +* **listBuckets:** Add support for returning partial success ([#2678](https://github.com/googleapis/nodejs-storage/issues/2678)) ([c7004da](https://github.com/googleapis/nodejs-storage/commit/c7004daae0b506eeefc5f4f241d9e3df3589b0e8)) + +## [7.17.3](https://github.com/googleapis/nodejs-storage/compare/v7.17.2...v7.17.3) (2025-11-03) + + +### Bug Fixes + +* :bug: fix the issue 2667, do not mutate object given to options … ([#2668](https://github.com/googleapis/nodejs-storage/issues/2668)) ([8a9f259](https://github.com/googleapis/nodejs-storage/commit/8a9f25968a08dc5cdaeb056054104db738fe651b)) +* Revert implement path containment to prevent traversal attacks ([254b6b2](https://github.com/googleapis/nodejs-storage/commit/254b6b2f9e79ce2e572f87694998538c9b1371c1)) + +## [7.17.2](https://github.com/googleapis/nodejs-storage/compare/v7.17.1...v7.17.2) (2025-10-06) + + +### Bug Fixes + +* Common Service: should retry a request failed ([#2652](https://github.com/googleapis/nodejs-storage/issues/2652)) ([b38b5d2](https://github.com/googleapis/nodejs-storage/commit/b38b5d221f2cb72658c1eb4a726315ab395a542c)) +* Implement path containment to prevent traversal attacks ([#2654](https://github.com/googleapis/nodejs-storage/issues/2654)) ([08d7abf](https://github.com/googleapis/nodejs-storage/commit/08d7abf32dd365b24ce34c66174be06c30bfce8f)) + +## [7.17.1](https://github.com/googleapis/nodejs-storage/compare/v7.17.0...v7.17.1) (2025-08-27) + + +### Bug Fixes + +* Respect useAuthWithCustomEndpoint flag for resumable uploads ([#2637](https://github.com/googleapis/nodejs-storage/issues/2637)) ([707b4f2](https://github.com/googleapis/nodejs-storage/commit/707b4f2fe1d67878bcd8f1434e4cbb57c951994e)) + +## [7.17.0](https://github.com/googleapis/nodejs-storage/compare/v7.16.0...v7.17.0) (2025-08-18) + + +### Features + +* Add CSEK to download ([#2604](https://github.com/googleapis/nodejs-storage/issues/2604)) ([cacc0be](https://github.com/googleapis/nodejs-storage/commit/cacc0be49a4fe81c384180bdfd77820b6b3f3001)) + + +### Bug Fixes + +* Propagate errors when using pipelines ([#2560](https://github.com/googleapis/nodejs-storage/issues/2560)) ([#2624](https://github.com/googleapis/nodejs-storage/issues/2624)) ([a43b490](https://github.com/googleapis/nodejs-storage/commit/a43b4904ecf2ebacde22bc6efbdcf97ac886e28d)) +* Typo correction ([#2610](https://github.com/googleapis/nodejs-storage/issues/2610)) ([9cae69c](https://github.com/googleapis/nodejs-storage/commit/9cae69cc280227737b5a1a1476eae1b2612b162b)) + +## [7.16.0](https://github.com/googleapis/nodejs-storage/compare/v7.15.2...v7.16.0) (2025-03-31) + + +### Features + +* Add moveFileAtomic method ([#2586](https://github.com/googleapis/nodejs-storage/issues/2586)) ([e25fb8c](https://github.com/googleapis/nodejs-storage/commit/e25fb8c659dbdf4a08c87fe2a929c8d62e6162e1)) + +## [7.15.2](https://github.com/googleapis/nodejs-storage/compare/v7.15.1...v7.15.2) (2025-02-20) + + +### Bug Fixes + +* Export SaveData type from index.ts ([#2580](https://github.com/googleapis/nodejs-storage/issues/2580)) ([dbf510c](https://github.com/googleapis/nodejs-storage/commit/dbf510c08f28c720ba66268eea6a62e139179ab4)) + +## [7.15.1](https://github.com/googleapis/nodejs-storage/compare/v7.15.0...v7.15.1) (2025-02-11) + + +### Bug Fixes + +* **getFiles:** Add nextPageToken to fields for autoPaginate ([#2570](https://github.com/googleapis/nodejs-storage/issues/2570)) ([75c309c](https://github.com/googleapis/nodejs-storage/commit/75c309c0761e4029dcd13024e748d8957052f766)) + +## [7.15.0](https://github.com/googleapis/nodejs-storage/compare/v7.14.0...v7.15.0) (2024-12-20) + + +### Features + +* Add ability to configure and utilize soft-delete and restore buckets ([#2566](https://github.com/googleapis/nodejs-storage/issues/2566)) ([25cdbb9](https://github.com/googleapis/nodejs-storage/commit/25cdbb918645362ce6994679e8a1c5e7cc666c87)) + +## [7.14.0](https://github.com/googleapis/nodejs-storage/compare/v7.13.0...v7.14.0) (2024-10-29) + + +### Features + +* Add support for restore token ([#2548](https://github.com/googleapis/nodejs-storage/issues/2548)) ([8241e91](https://github.com/googleapis/nodejs-storage/commit/8241e91e78d47b4cdaea2d941f75fd6a4fa29230)) +* Adds integration tests for Universe Domain configuration ([#2538](https://github.com/googleapis/nodejs-storage/issues/2538)) ([53db6ba](https://github.com/googleapis/nodejs-storage/commit/53db6ba7406b99e507cacfa6195cb5a7d308914b)) +* Adds integration tests for Universe Domain configuration with ([53db6ba](https://github.com/googleapis/nodejs-storage/commit/53db6ba7406b99e507cacfa6195cb5a7d308914b)) +* **storage:** Add support for 'skipIfExists' option for downloadMany ([#2526](https://github.com/googleapis/nodejs-storage/issues/2526)) ([729efb2](https://github.com/googleapis/nodejs-storage/commit/729efb213f96b1a406a1caa54870f50e96796639)) + +## [7.13.0](https://github.com/googleapis/nodejs-storage/compare/v7.12.1...v7.13.0) (2024-09-17) + + +### Features + +* **storage:** Add support for 'fields' query parameter to getFiles ([#2521](https://github.com/googleapis/nodejs-storage/issues/2521)) ([f78fe92](https://github.com/googleapis/nodejs-storage/commit/f78fe92348a0b383314b4fbfb55638d47af052ff)) + + +### Bug Fixes + +* **retry:** Export RETRYABLE_ERR_FN_DEFAULT ([#2517](https://github.com/googleapis/nodejs-storage/issues/2517)) ([db890fd](https://github.com/googleapis/nodejs-storage/commit/db890fd1b25d5cd94e3bcd97ec21fa8f77b1b724)) + +## [7.12.1](https://github.com/googleapis/nodejs-storage/compare/v7.12.0...v7.12.1) (2024-08-07) + + +### Bug Fixes + +* **deps:** Update fast-xml-parser to 4.4.1 due to security vulnerability ([#2505](https://github.com/googleapis/nodejs-storage/issues/2505)) ([b97d474](https://github.com/googleapis/nodejs-storage/commit/b97d474445efbcde91e690f6ea4160cfc9fd1ed4)) + +## [7.12.0](https://github.com/googleapis/nodejs-storage/compare/v7.11.3...v7.12.0) (2024-07-15) + + +### Features + +* Add function to allow user to set destination in transfer manager ([#2497](https://github.com/googleapis/nodejs-storage/issues/2497)) ([dc1e488](https://github.com/googleapis/nodejs-storage/commit/dc1e488b50dd7e2deab3e8b28c7d6ece36b90b0e)) + +## [7.11.3](https://github.com/googleapis/nodejs-storage/compare/v7.11.2...v7.11.3) (2024-07-09) + + +### Bug Fixes + +* Error serialization in resumable-upload.ts ([#2493](https://github.com/googleapis/nodejs-storage/issues/2493)) ([c2e555c](https://github.com/googleapis/nodejs-storage/commit/c2e555c95f7a8e3e231e57e2fa3967caeb860772)) +* Handle unhandled error in `startResumableUpload_` ([#2495](https://github.com/googleapis/nodejs-storage/issues/2495)) ([d5257ba](https://github.com/googleapis/nodejs-storage/commit/d5257ba4aa9efc1dd70c303286782d23a95e8568)) +* Make CreateBucketRequest extend from BucketMetadata to allow all… ([#2489](https://github.com/googleapis/nodejs-storage/issues/2489)) ([013a5a4](https://github.com/googleapis/nodejs-storage/commit/013a5a45aed8734f797837deb0e805f0ca43a9a6)) + +## [7.11.2](https://github.com/googleapis/nodejs-storage/compare/v7.11.1...v7.11.2) (2024-06-07) + + +### Bug Fixes + +* Support uint8array in file.save ([#2480](https://github.com/googleapis/nodejs-storage/issues/2480)) ([1477fe1](https://github.com/googleapis/nodejs-storage/commit/1477fe15e5b6dae7cdfb2a1d00121e5d674c8f7c)) + +## [7.11.1](https://github.com/googleapis/nodejs-storage/compare/v7.11.0...v7.11.1) (2024-05-21) + + +### Bug Fixes + +* Add missing projectIdentifier to GetServiceAccountOptions ([#2468](https://github.com/googleapis/nodejs-storage/issues/2468)) ([d49e9d2](https://github.com/googleapis/nodejs-storage/commit/d49e9d2cbab42eb2390eeeccc562e3283df6384c)) +* Allow files in directories to be downloaded onto local machine ([#2199](https://github.com/googleapis/nodejs-storage/issues/2199)) ([9f62429](https://github.com/googleapis/nodejs-storage/commit/9f62429dad234167dc6f0969b40c7942bab83aee)) +* Do not set `customEndpoint` if `apiEndpoint === default` ([#2460](https://github.com/googleapis/nodejs-storage/issues/2460)) ([b4dbd73](https://github.com/googleapis/nodejs-storage/commit/b4dbd73189b9fae4c23d614753670ee562bc717b)) +* Improve GetFilesResponse interface ([#2466](https://github.com/googleapis/nodejs-storage/issues/2466)) ([918db28](https://github.com/googleapis/nodejs-storage/commit/918db2818395488ff925324a9962879bb56368d7)) + +## [7.11.0](https://github.com/googleapis/nodejs-storage/compare/v7.10.2...v7.11.0) (2024-05-03) + + +### Features + +* Add ability to enable hierarchical namespace on buckets ([#2453](https://github.com/googleapis/nodejs-storage/issues/2453)) ([4e5726f](https://github.com/googleapis/nodejs-storage/commit/4e5726fe60ecede6bec6508e4b4ec214f2e173dd)) + +## [7.10.2](https://github.com/googleapis/nodejs-storage/compare/v7.10.1...v7.10.2) (2024-04-26) + + +### Bug Fixes + +* Use correct indices for file.from and fix tests to verify names ([#2449](https://github.com/googleapis/nodejs-storage/issues/2449)) ([d4240fa](https://github.com/googleapis/nodejs-storage/commit/d4240fa5c8c0353de81cc8c052eea2915c3e383c)) + +## [7.10.1](https://github.com/googleapis/nodejs-storage/compare/v7.10.0...v7.10.1) (2024-04-22) + + +### Bug Fixes + +* Change copyoptions type ([#2439](https://github.com/googleapis/nodejs-storage/issues/2439)) ([2ebd7ac](https://github.com/googleapis/nodejs-storage/commit/2ebd7aca6c474147e5a1d1fb2a96b7d052a08a21)) +* Expand types of custom metadata within FileMetadata ([#2442](https://github.com/googleapis/nodejs-storage/issues/2442)) ([1d434a9](https://github.com/googleapis/nodejs-storage/commit/1d434a905392b00bb48ebbb812034e062ed27dd2)) + +## [7.10.0](https://github.com/googleapis/nodejs-storage/compare/v7.9.0...v7.10.0) (2024-04-15) + + +### Features + +* Add ability to create a File object from URL ([#2432](https://github.com/googleapis/nodejs-storage/issues/2432)) ([1b71fcc](https://github.com/googleapis/nodejs-storage/commit/1b71fcc7687fb8d66e04fb92b15671729f1407e7)) +* Allow setting contentEncoding during compose ([#2431](https://github.com/googleapis/nodejs-storage/issues/2431)) ([6e81e05](https://github.com/googleapis/nodejs-storage/commit/6e81e05b2615f1b8307fcca9a147362677e95e7f)) + + +### Bug Fixes + +* Destroy pipeline streams when returned stream errors ([#2437](https://github.com/googleapis/nodejs-storage/issues/2437)) ([fe1ac65](https://github.com/googleapis/nodejs-storage/commit/fe1ac655a8d321e225f4828c7adf57342c4a8455)) +* Remove extraneous mime-types package in favor of mime ([#2435](https://github.com/googleapis/nodejs-storage/issues/2435)) ([63a71f2](https://github.com/googleapis/nodejs-storage/commit/63a71f2e81181ad976e982ef79d6148913a87c1f)) + +## [7.9.0](https://github.com/googleapis/nodejs-storage/compare/v7.8.0...v7.9.0) (2024-03-18) + + +### Features + +* Add ability to configure and utilize soft-delete and restore ([#2425](https://github.com/googleapis/nodejs-storage/issues/2425)) ([7da5a7d](https://github.com/googleapis/nodejs-storage/commit/7da5a7da86ad649a8132e3183f4b3e3f9bb2eace)) + +## [7.8.0](https://github.com/googleapis/nodejs-storage/compare/v7.7.0...v7.8.0) (2024-03-07) + + +### Features + +* Add includeFoldersAsPrefixes for managed folders ([#2413](https://github.com/googleapis/nodejs-storage/issues/2413)) ([3044d3c](https://github.com/googleapis/nodejs-storage/commit/3044d3cfb1b4a24f07fd6ec29e3d20d5818c4ca3)) +* Base TPC Support ([#2397](https://github.com/googleapis/nodejs-storage/issues/2397)) ([a3f4891](https://github.com/googleapis/nodejs-storage/commit/a3f4891ee60e57cc19929489cae6110b07955216)) + + +### Bug Fixes + +* Cannot read properties of null (reading length) in stream-shift ([#2422](https://github.com/googleapis/nodejs-storage/issues/2422)) ([11ebe2b](https://github.com/googleapis/nodejs-storage/commit/11ebe2bf905f8c15101446ecfe5a2d7c6005d0c3)) +* Do not automatically set overrideUnlockedRetention ([#2421](https://github.com/googleapis/nodejs-storage/issues/2421)) ([c781bdc](https://github.com/googleapis/nodejs-storage/commit/c781bdcd89f63b22af1c491a6e517e110331d4ca)) + +## [7.7.0](https://github.com/googleapis/nodejs-storage/compare/v7.6.0...v7.7.0) (2023-11-29) + + +### Features + +* Implement object retention lock for bucket / files ([#2365](https://github.com/googleapis/nodejs-storage/issues/2365)) ([c140868](https://github.com/googleapis/nodejs-storage/commit/c140868bb5c7d1f0edd586fb2ca55bc613caf5d4)) + + +### Bug Fixes + +* TransferManager#downloadFileInChunks issues ([#2373](https://github.com/googleapis/nodejs-storage/issues/2373)) ([65950f3](https://github.com/googleapis/nodejs-storage/commit/65950f3d5c2ed73c56afdf579d7a949b4505e649)) + +## [7.6.0](https://github.com/googleapis/nodejs-storage/compare/v7.5.0...v7.6.0) (2023-11-08) + + +### Features + +* Resume Resumable Uploads ([#2333](https://github.com/googleapis/nodejs-storage/issues/2333)) ([2ba4009](https://github.com/googleapis/nodejs-storage/commit/2ba4009e599c5690c51849f307199be3452d3b07)) + +## [7.5.0](https://github.com/googleapis/nodejs-storage/compare/v7.4.0...v7.5.0) (2023-10-30) + + +### Features + +* Support autoclass v2.1 ([#2325](https://github.com/googleapis/nodejs-storage/issues/2325)) ([6572ce9](https://github.com/googleapis/nodejs-storage/commit/6572ce9dc146a0d1e13418a43204058d5909510e)) + +## [7.4.0](https://github.com/googleapis/nodejs-storage/compare/v7.3.2...v7.4.0) (2023-10-24) + + +### Features + +* Support building in CJS and ESM formats ([#2296](https://github.com/googleapis/nodejs-storage/issues/2296)) ([c848076](https://github.com/googleapis/nodejs-storage/commit/c84807662839b3671230a50b2a0c6f7a6efef528)) + +## [7.3.2](https://github.com/googleapis/nodejs-storage/compare/v7.3.1...v7.3.2) (2023-10-24) + + +### Bug Fixes + +* Close Open Handle for Empty Objects ([#2338](https://github.com/googleapis/nodejs-storage/issues/2338)) ([c51cd94](https://github.com/googleapis/nodejs-storage/commit/c51cd946171e8749453eef080d2853d31a6e72c8)) + +## [7.3.1](https://github.com/googleapis/nodejs-storage/compare/v7.3.0...v7.3.1) (2023-10-19) + + +### Bug Fixes + +* Add user-agent header to transfer manager and resumable upload ([#2334](https://github.com/googleapis/nodejs-storage/issues/2334)) ([0520867](https://github.com/googleapis/nodejs-storage/commit/0520867e51a2758ddf2773c0d910c937d55e21b1)) +* **deps:** Update dependency retry-request to v7 ([#2327](https://github.com/googleapis/nodejs-storage/issues/2327)) ([f20ef3c](https://github.com/googleapis/nodejs-storage/commit/f20ef3cb7bf8cbdda988e792edc7abd9a1b516f1)) +* Destroy sockets to stop memory leaking when stream errors ([#2336](https://github.com/googleapis/nodejs-storage/issues/2336)) ([04939ee](https://github.com/googleapis/nodejs-storage/commit/04939ee4976581cc4168943f0c578bc49458cff7)) + +## [7.3.0](https://github.com/googleapis/nodejs-storage/compare/v7.2.0...v7.3.0) (2023-10-11) + + +### Features + +* Add transfer manager MPU sample, adjust defaults, prepare for GA ([#2318](https://github.com/googleapis/nodejs-storage/issues/2318)) ([a7d09c1](https://github.com/googleapis/nodejs-storage/commit/a7d09c16aa7f732db089c55d2692ce5fd88c52f3)) + + +### Bug Fixes + +* Simplify the code for downloadInChunks ([#2323](https://github.com/googleapis/nodejs-storage/issues/2323)) ([6519929](https://github.com/googleapis/nodejs-storage/commit/6519929153e0006b61d205860de93c9e64b99d81)) + +## [7.2.0](https://github.com/googleapis/nodejs-storage/compare/v7.1.0...v7.2.0) (2023-10-05) + + +### Features + +* Add headers option to MPU ([#2303](https://github.com/googleapis/nodejs-storage/issues/2303)) ([7f58f30](https://github.com/googleapis/nodejs-storage/commit/7f58f30588735d129fda0503e1daec5f605f8447)) +* Transfer Manager Metrics ([#2305](https://github.com/googleapis/nodejs-storage/issues/2305)) ([9be3b6a](https://github.com/googleapis/nodejs-storage/commit/9be3b6a97d9c4685d3c01a6d44e618087be54ea1)) + + +### Bug Fixes + +* Make sure destination uses posix separator instead of win ([#2304](https://github.com/googleapis/nodejs-storage/issues/2304)) ([1d4ea74](https://github.com/googleapis/nodejs-storage/commit/1d4ea74a3cc441dfccc47893f0318234f213921b)) + +## [7.1.0](https://github.com/googleapis/nodejs-storage/compare/v7.0.1...v7.1.0) (2023-09-07) + + +### Features + +* Export `ApiError` ([#2291](https://github.com/googleapis/nodejs-storage/issues/2291)) ([c1d1b35](https://github.com/googleapis/nodejs-storage/commit/c1d1b3505b1c6b0306f632a48cbd7d774b2b94d0)) +* Support iterables in file@save ([4356dd0](https://github.com/googleapis/nodejs-storage/commit/4356dd0e6bea5241c4cacd1a58697a332ccf4784)) +* Support iterables in file@save ([49327ff](https://github.com/googleapis/nodejs-storage/commit/49327ff576b2367d9efdff2f82a515b0538ee471)) +* Support iterables in file@save ([c0d9d58](https://github.com/googleapis/nodejs-storage/commit/c0d9d58b56a9a3485b6c0e5eb92411bb094f7bcb)) + + +### Bug Fixes + +* `File#save` iterable fixes ([#2293](https://github.com/googleapis/nodejs-storage/issues/2293)) ([87c3f41](https://github.com/googleapis/nodejs-storage/commit/87c3f419e2a5a3a30ea581aaa6127dfac261be17)) +* **deps:** Update dependency @google-cloud/paginator to v5 ([#2263](https://github.com/googleapis/nodejs-storage/issues/2263)) ([0c9b342](https://github.com/googleapis/nodejs-storage/commit/0c9b3425b47c3031ec4bac6d45d8cdca48b2f1a6)) +* **deps:** Update dependency @google-cloud/projectify to v4 ([#2264](https://github.com/googleapis/nodejs-storage/issues/2264)) ([c881bae](https://github.com/googleapis/nodejs-storage/commit/c881bae96b40f609d2b7a8d7388c6a76d34faab1)) +* **deps:** Update dependency @google-cloud/promisify to v4 ([#2262](https://github.com/googleapis/nodejs-storage/issues/2262)) ([9d46ff3](https://github.com/googleapis/nodejs-storage/commit/9d46ff3c02315c5a3516aa5f2755ee0471ba036b)) +* **deps:** Update dependency @google-cloud/pubsub to v4 ([#2256](https://github.com/googleapis/nodejs-storage/issues/2256)) ([18282bb](https://github.com/googleapis/nodejs-storage/commit/18282bbefe1201e51867c676a86301f8086aaf1e)) + + +### Miscellaneous Chores + +* Release 7.1.0 ([#2274](https://github.com/googleapis/nodejs-storage/issues/2274)) ([e0f45c2](https://github.com/googleapis/nodejs-storage/commit/e0f45c24b44d11c72ff40cc534be11cb2a65192f)) + +## [7.0.1](https://github.com/googleapis/nodejs-storage/compare/v7.0.0...v7.0.1) (2023-08-07) + + +### Bug Fixes + +* Export new types from index.ts ([#2258](https://github.com/googleapis/nodejs-storage/issues/2258)) ([93d2a0f](https://github.com/googleapis/nodejs-storage/commit/93d2a0f9f6d6a3de652aaeb3dd914bbc0bb593e0)) + +## [7.0.0](https://github.com/googleapis/nodejs-storage/compare/v6.12.0...v7.0.0) (2023-08-03) + + +### ⚠ BREAKING CHANGES + +* Make node 14 the minimum supported version ([#2244](https://github.com/googleapis/nodejs-storage/issues/2244)) +* Add stronger typings to metadata ([#2234](https://github.com/googleapis/nodejs-storage/issues/2234)) +* Remove extend and Treat Provided Options as Mutable ([#2204](https://github.com/googleapis/nodejs-storage/issues/2204)) +* Stronger typing for lifecycle rules ([#2215](https://github.com/googleapis/nodejs-storage/issues/2215)) +* Do not return responsebody in delete, only raw response ([#2236](https://github.com/googleapis/nodejs-storage/issues/2236)) +* Remove extraneous validation in favor of server errors ([#2237](https://github.com/googleapis/nodejs-storage/issues/2237)) +* Mark bucket.setLabels and associated interfaces as deprecated ([#2214](https://github.com/googleapis/nodejs-storage/issues/2214)) +* Disable source maps for smaller bundle size ([#2240](https://github.com/googleapis/nodejs-storage/issues/2240)) + + + +### Features + +* Make node 14 the minimum supported version ([#2244](https://github.com/googleapis/nodejs-storage/issues/2244)) ([f48dcd2](https://github.com/googleapis/nodejs-storage/commit/f48dcd2d00081aea8990f35b68a91248f3862abe)) + +## [6.12.0](https://github.com/googleapis/nodejs-storage/compare/v6.11.0...v6.12.0) (2023-07-13) + + +### Features + +* Add header for deno runtime for metrics tracking ([#2220](https://github.com/googleapis/nodejs-storage/issues/2220)) ([5083920](https://github.com/googleapis/nodejs-storage/commit/50839209063e75996b2a57bd7664760e0e5331ca)) +* MPU for transfer manager ([#2192](https://github.com/googleapis/nodejs-storage/issues/2192)) ([ae83421](https://github.com/googleapis/nodejs-storage/commit/ae83421e617a5761c75a7c8a15eaa1ea7c7fb1de)) + +## [6.11.0](https://github.com/googleapis/nodejs-storage/compare/v6.10.1...v6.11.0) (2023-06-02) + + +### Features + +* Add support for matchGlob list option ([#2206](https://github.com/googleapis/nodejs-storage/issues/2206)) ([79dd839](https://github.com/googleapis/nodejs-storage/commit/79dd8394fbbc0c97aa3acb86ad2248fd58b243b4)) + +## [6.10.1](https://github.com/googleapis/nodejs-storage/compare/v6.10.0...v6.10.1) (2023-05-10) + + +### Performance Improvements + +* Improve Multiple Chunk Upload Performance ([#2185](https://github.com/googleapis/nodejs-storage/issues/2185)) ([3b2b877](https://github.com/googleapis/nodejs-storage/commit/3b2b87707072e5dc9221a5ba3c727c70db13a593)) + +## [6.10.0](https://github.com/googleapis/nodejs-storage/compare/v6.9.5...v6.10.0) (2023-05-02) + + +### Features + +* Retry Socket Connection Timeouts on Node 20+ ([#2187](https://github.com/googleapis/nodejs-storage/issues/2187)) ([733b560](https://github.com/googleapis/nodejs-storage/commit/733b560c2f634884dd31c916c208ee6395d4fbe1)) + +## [6.9.5](https://github.com/googleapis/nodejs-storage/compare/v6.9.4...v6.9.5) (2023-03-30) + + +### Bug Fixes + +* Check that err.message is a string before attempting indexOf ([#2173](https://github.com/googleapis/nodejs-storage/issues/2173)) ([130818d](https://github.com/googleapis/nodejs-storage/commit/130818d291a0004e5d36e5ee72a8c0687b9db181)) +* V4 Signing Errors with exactly 7 day expiry ([#2170](https://github.com/googleapis/nodejs-storage/issues/2170)) ([f930998](https://github.com/googleapis/nodejs-storage/commit/f9309985d130a574dd23ecf7b6fb5b58b01d42a0)) + +## [6.9.4](https://github.com/googleapis/nodejs-storage/compare/v6.9.3...v6.9.4) (2023-03-02) + + +### Bug Fixes + +* Refactor createReadStream to remove unnecessary stream ([#2153](https://github.com/googleapis/nodejs-storage/issues/2153)) ([2c97310](https://github.com/googleapis/nodejs-storage/commit/2c97310da9edd1afbf61711631979433f10249a5)) + +## [6.9.3](https://github.com/googleapis/nodejs-storage/compare/v6.9.2...v6.9.3) (2023-02-15) + + +### Bug Fixes + +* Reduce memory footprint of deleteFiles by utilizing getFilesStream and using smaller queue of promises ([#2147](https://github.com/googleapis/nodejs-storage/issues/2147)) ([f792f25](https://github.com/googleapis/nodejs-storage/commit/f792f25e3fd38003056f649eaa638a782290cbac)) + +## [6.9.2](https://github.com/googleapis/nodejs-storage/compare/v6.9.1...v6.9.2) (2023-02-06) + + +### Bug Fixes + +* Correctly handle if a user has no permissions when calling iam.testPermissions ([#2140](https://github.com/googleapis/nodejs-storage/issues/2140)) ([cce902d](https://github.com/googleapis/nodejs-storage/commit/cce902d27cf3c4a23730550b72d10dc76425c974)) + +## [6.9.1](https://github.com/googleapis/nodejs-storage/compare/v6.9.0...v6.9.1) (2023-01-24) + + +### Bug Fixes + +* Setting file metadata is conditionally idempotent on ifmetagenerationmatch not ifgenerationmatch ([#2131](https://github.com/googleapis/nodejs-storage/issues/2131)) ([f20c28c](https://github.com/googleapis/nodejs-storage/commit/f20c28c5875c9a7095b028912550512459fbf844)) + +## [6.9.0](https://github.com/googleapis/nodejs-storage/compare/v6.8.0...v6.9.0) (2023-01-04) + + +### Features + +* Add ability to upload entire directory, add samples for upload … ([#2118](https://github.com/googleapis/nodejs-storage/issues/2118)) ([b0f32ce](https://github.com/googleapis/nodejs-storage/commit/b0f32ced04eae1b8ad88a0939fa763cb16b08df9)) + +## [6.8.0](https://github.com/googleapis/nodejs-storage/compare/v6.7.0...v6.8.0) (2022-12-07) + + +### Features + +* Implement parallel operations ([#2067](https://github.com/googleapis/nodejs-storage/issues/2067)) ([#2109](https://github.com/googleapis/nodejs-storage/issues/2109)) ([ce15b5e](https://github.com/googleapis/nodejs-storage/commit/ce15b5ef68353efbc005cb3a1a780e064ea04deb)) + +## [6.7.0](https://github.com/googleapis/nodejs-storage/compare/v6.6.0...v6.7.0) (2022-11-03) + + +### Features + +* Support autoclass ([#2078](https://github.com/googleapis/nodejs-storage/issues/2078)) ([7e83580](https://github.com/googleapis/nodejs-storage/commit/7e8358008467dd2d77702734e05f54bc06c9ca5b)) + +## [6.6.0](https://github.com/googleapis/nodejs-storage/compare/v6.5.4...v6.6.0) (2022-10-25) + + +### Features + +* **crc32c:** Convenient Method For Reading Files ([#2095](https://github.com/googleapis/nodejs-storage/issues/2095)) ([2145c81](https://github.com/googleapis/nodejs-storage/commit/2145c8177f3659fb5f193045866fbcd1220aaeaf)) + + +### Bug Fixes + +* Remove async from final function which causes double invocation … ([#2094](https://github.com/googleapis/nodejs-storage/issues/2094)) ([1a3df98](https://github.com/googleapis/nodejs-storage/commit/1a3df985e9096229bc2909921b4974680e12be2a)) + +## [6.5.4](https://github.com/googleapis/nodejs-storage/compare/v6.5.3...v6.5.4) (2022-10-20) + + +### Bug Fixes + +* Revert STORAGE_EMULATOR_HOST handling ([#2089](https://github.com/googleapis/nodejs-storage/issues/2089)) ([48dce65](https://github.com/googleapis/nodejs-storage/commit/48dce654064470f7496d160d87b696ab5cfd65d4)) + +## [6.5.3](https://github.com/googleapis/nodejs-storage/compare/v6.5.2...v6.5.3) (2022-10-18) + + +### Bug Fixes + +* Correct STORAGE_EMULATOR_HOST handling ([#2069](https://github.com/googleapis/nodejs-storage/issues/2069), [#1314](https://github.com/googleapis/nodejs-storage/issues/1314)) ([#2070](https://github.com/googleapis/nodejs-storage/issues/2070)) ([c75b8b8](https://github.com/googleapis/nodejs-storage/commit/c75b8b82262dddb794304a71f648bd6e03cafb30)) + +## [6.5.2](https://github.com/googleapis/nodejs-storage/compare/v6.5.1...v6.5.2) (2022-09-23) + + +### Bug Fixes + +* Determine `Content-Length` Before Attempting Multi-chunk Upload ([#2074](https://github.com/googleapis/nodejs-storage/issues/2074)) ([666402a](https://github.com/googleapis/nodejs-storage/commit/666402a6a65c2ee8e91bc4fe072e91c0b893864e)) + +## [6.5.1](https://github.com/googleapis/nodejs-storage/compare/v6.5.0...v6.5.1) (2022-09-20) + + +### Bug Fixes + +* Revert skip validation ([#2023](https://github.com/googleapis/nodejs-storage/issues/2023)) ([70ab224](https://github.com/googleapis/nodejs-storage/commit/70ab22459b51b9781e40b0cc86663c1658e43520)) + +## [6.5.0](https://github.com/googleapis/nodejs-storage/compare/v6.4.2...v6.5.0) (2022-09-15) + + +### Features + +* Add multiple lifecycle rules ([#2062](https://github.com/googleapis/nodejs-storage/issues/2062)) ([fbe2deb](https://github.com/googleapis/nodejs-storage/commit/fbe2deb72bd98db54a03cf228a360a871ba1915b)) + +## [6.4.2](https://github.com/googleapis/nodejs-storage/compare/v6.4.1...v6.4.2) (2022-09-01) + + +### Bug Fixes + +* remove pip install statements ([#1546](https://github.com/googleapis/nodejs-storage/issues/1546)) ([#2049](https://github.com/googleapis/nodejs-storage/issues/2049)) ([c90ba0f](https://github.com/googleapis/nodejs-storage/commit/c90ba0feecca9d39de3c52f12cc9423b7a2d4d47)) +* truncated `createReadStream` through early `end` event ([#2056](https://github.com/googleapis/nodejs-storage/issues/2056)) ([a4716a4](https://github.com/googleapis/nodejs-storage/commit/a4716a4ed1053560f2692647c8a90131763c1e72)) + +## [6.4.1](https://github.com/googleapis/nodejs-storage/compare/v6.4.0...v6.4.1) (2022-08-12) + + +### Bug Fixes + +* Remove `pumpify` ([#2029](https://github.com/googleapis/nodejs-storage/issues/2029)) ([edc1d64](https://github.com/googleapis/nodejs-storage/commit/edc1d64069a6038c301c3b775f116fbf69b10b28)) +* Retry `EPIPE` Connection Errors + Attempt Retries in Resumable Upload Connection Errors ([#2040](https://github.com/googleapis/nodejs-storage/issues/2040)) ([ba35321](https://github.com/googleapis/nodejs-storage/commit/ba35321c3fef9a1874ff8fbea43885e2e7746b4f)) + +## [6.4.0](https://github.com/googleapis/nodejs-storage/compare/v6.3.0...v6.4.0) (2022-08-10) + + +### Features + +* add functionality for passing preconditions at the function level ([#1993](https://github.com/googleapis/nodejs-storage/issues/1993)) ([21f173e](https://github.com/googleapis/nodejs-storage/commit/21f173eb17d4216e2b42ffdd1ed0104aeda7cf13)) + +## [6.3.0](https://github.com/googleapis/nodejs-storage/compare/v6.2.3...v6.3.0) (2022-08-01) + + +### Features + +* custom dual regions refactor implementation ([#2012](https://github.com/googleapis/nodejs-storage/issues/2012)) ([9a614fb](https://github.com/googleapis/nodejs-storage/commit/9a614fb2b624f8234fa9d40f352dae177b2b0374)) + +## [6.2.3](https://github.com/googleapis/nodejs-storage/compare/v6.2.2...v6.2.3) (2022-07-13) + + +### Bug Fixes + +* force setMetadata calls to use promise version, invoke callbacks manually ([#2000](https://github.com/googleapis/nodejs-storage/issues/2000)) ([f488647](https://github.com/googleapis/nodejs-storage/commit/f488647d5ac8ccfdb479d7ea24a377d10573b8c4)) + +## [6.2.2](https://github.com/googleapis/nodejs-storage/compare/v6.2.1...v6.2.2) (2022-06-29) + + +### Bug Fixes + +* correctly handle an empty file in download function ([#1995](https://github.com/googleapis/nodejs-storage/issues/1995)) ([f1a5a0b](https://github.com/googleapis/nodejs-storage/commit/f1a5a0bd121b84c3d56a9eddbd31593089634574)) + +## [6.2.1](https://github.com/googleapis/nodejs-storage/compare/v6.2.0...v6.2.1) (2022-06-27) + + +### Bug Fixes + +* explicitly import URL so that .d.ts file is correctly generated ([#1992](https://github.com/googleapis/nodejs-storage/issues/1992)) ([a968f50](https://github.com/googleapis/nodejs-storage/commit/a968f508e7c98e160b658ffc3ab568997548172f)) + +## [6.2.0](https://github.com/googleapis/nodejs-storage/compare/v6.1.0...v6.2.0) (2022-06-22) + + +### Features + +* Convenient `gs://` URI retrieval ([#1987](https://github.com/googleapis/nodejs-storage/issues/1987)) ([58fad6d](https://github.com/googleapis/nodejs-storage/commit/58fad6d9a3bd92966306e98fd7dedd3992995cb7)) + + +### Bug Fixes + +* **deps:** update dependency @google-cloud/projectify to v3 ([#1986](https://github.com/googleapis/nodejs-storage/issues/1986)) ([71a61ec](https://github.com/googleapis/nodejs-storage/commit/71a61ec4ef2436bc091f390af14a02b6920b2b87)) +* **deps:** update dependency @google-cloud/pubsub to v3 ([#1972](https://github.com/googleapis/nodejs-storage/issues/1972)) ([004b97e](https://github.com/googleapis/nodejs-storage/commit/004b97ec0ae866997c84ee2327db87e59f510f00)) +* Requests Respect `config.projectIdRequired` === `false` ([#1988](https://github.com/googleapis/nodejs-storage/issues/1988)) ([8813369](https://github.com/googleapis/nodejs-storage/commit/881336944be37e15c45b12973064245adc519860)) + +## [6.1.0](https://github.com/googleapis/nodejs-storage/compare/v6.0.1...v6.1.0) (2022-06-08) + + +### Features + +* support OLM Prefix/Suffix ([#1847](https://github.com/googleapis/nodejs-storage/issues/1847)) ([c22984c](https://github.com/googleapis/nodejs-storage/commit/c22984caa8e8ae09a61d308876b2b3d97503777b)) + +### [6.0.1](https://github.com/googleapis/nodejs-storage/compare/v6.0.0...v6.0.1) (2022-05-25) + + +### Bug Fixes + +* capture and throw on non-existent files ([#1969](https://github.com/googleapis/nodejs-storage/issues/1969)) ([52d81c0](https://github.com/googleapis/nodejs-storage/commit/52d81c026f30aef0902ea7173dfa6da2e7f97d50)) + +## [6.0.0](https://github.com/googleapis/nodejs-storage/compare/v5.20.5...v6.0.0) (2022-05-24) + + +### ⚠ BREAKING CHANGES + +* update TypeScript +* remove deprecated fields +* remove configstore +* align resumable upload behavior +* utilize internalized CRC32C utilities +* drop node 10 support + +### Build System + +* drop node 10 support ([77fa8d9](https://github.com/googleapis/nodejs-storage/commit/77fa8d9f95afbc830b57188ce0d2dfac46476d0b)) + + +### Code Refactoring + +* align resumable upload behavior ([77fa8d9](https://github.com/googleapis/nodejs-storage/commit/77fa8d9f95afbc830b57188ce0d2dfac46476d0b)) +* remove configstore ([77fa8d9](https://github.com/googleapis/nodejs-storage/commit/77fa8d9f95afbc830b57188ce0d2dfac46476d0b)) +* remove deprecated fields ([77fa8d9](https://github.com/googleapis/nodejs-storage/commit/77fa8d9f95afbc830b57188ce0d2dfac46476d0b)) +* utilize internalized CRC32C utilities ([77fa8d9](https://github.com/googleapis/nodejs-storage/commit/77fa8d9f95afbc830b57188ce0d2dfac46476d0b)) + + +### deps + +* update TypeScript ([77fa8d9](https://github.com/googleapis/nodejs-storage/commit/77fa8d9f95afbc830b57188ce0d2dfac46476d0b)) + +### [5.20.5](https://github.com/googleapis/nodejs-storage/compare/v5.20.4...v5.20.5) (2022-05-19) + + +### Bug Fixes + +* **chore:** move uuid package to dependencies ([#1952](https://github.com/googleapis/nodejs-storage/issues/1952)) ([0ff5aa3](https://github.com/googleapis/nodejs-storage/commit/0ff5aa3e9ff8b4dcc536b9494e44b9a2fb217727)) + +### [5.20.4](https://github.com/googleapis/nodejs-storage/compare/v5.20.3...v5.20.4) (2022-05-18) + + +### Bug Fixes + +* revert native typescript mocha tests ([#1947](https://github.com/googleapis/nodejs-storage/issues/1947)) ([1d0ea7d](https://github.com/googleapis/nodejs-storage/commit/1d0ea7d2281a049bc99c6bd810dd24ffc83c6a09)) +* support empty object uploads for resumable upload ([#1949](https://github.com/googleapis/nodejs-storage/issues/1949)) ([da6016e](https://github.com/googleapis/nodejs-storage/commit/da6016e20b681d6a75ed1a5459cfd333b58c70a9)) + +### [5.20.3](https://github.com/googleapis/nodejs-storage/compare/v5.20.2...v5.20.3) (2022-05-17) + + +### Bug Fixes + +* move retrieval of package.json to utility function ([#1941](https://github.com/googleapis/nodejs-storage/issues/1941)) ([ac5cbdf](https://github.com/googleapis/nodejs-storage/commit/ac5cbdf0d3c363e7dab43e9002b01a0dae877642)) + +### [5.20.2](https://github.com/googleapis/nodejs-storage/compare/v5.20.1...v5.20.2) (2022-05-17) + + +### Bug Fixes + +* use path.join and __dirname for require package.json ([#1936](https://github.com/googleapis/nodejs-storage/issues/1936)) ([b868762](https://github.com/googleapis/nodejs-storage/commit/b86876201ec7d4ce58ae5c1d9635dad82a1fdc4b)) + +### [5.20.1](https://github.com/googleapis/nodejs-storage/compare/v5.20.0...v5.20.1) (2022-05-16) + + +### Bug Fixes + +* do not use import on package.json ([#1932](https://github.com/googleapis/nodejs-storage/issues/1932)) ([d0f0494](https://github.com/googleapis/nodejs-storage/commit/d0f04941f1cabf7c153f7e5abb91c358f12ef83e)) + +## [5.20.0](https://github.com/googleapis/nodejs-storage/compare/v5.19.4...v5.20.0) (2022-05-16) + + +### Features + +* add x-goog-api-client headers for retry metrics ([#1920](https://github.com/googleapis/nodejs-storage/issues/1920)) ([0c7e4f6](https://github.com/googleapis/nodejs-storage/commit/0c7e4f6ade4cee1715e14b3cd9abf2aa2a56c0b9)) + +### [5.19.4](https://github.com/googleapis/nodejs-storage/compare/v5.19.3...v5.19.4) (2022-04-28) + + +### Bug Fixes + +* don't modify passed in options ([#1895](https://github.com/googleapis/nodejs-storage/issues/1895)) ([cd80ca3](https://github.com/googleapis/nodejs-storage/commit/cd80ca318a2b10379b8b166a59f3943b97576475)) + +### [5.19.3](https://github.com/googleapis/nodejs-storage/compare/v5.19.2...v5.19.3) (2022-04-20) + + +### Bug Fixes + +* export idempotencystrategy and preconditionoptions from index ([#1880](https://github.com/googleapis/nodejs-storage/issues/1880)) ([8aafe04](https://github.com/googleapis/nodejs-storage/commit/8aafe0453a8e2dc41f848dd5165d3e86d6a160ed)) + +### [5.19.2](https://github.com/googleapis/nodejs-storage/compare/v5.19.1...v5.19.2) (2022-04-14) + + +### Bug Fixes + +* deleting, getting, and getting metadata for notifications ([#1872](https://github.com/googleapis/nodejs-storage/issues/1872)) ([451570e](https://github.com/googleapis/nodejs-storage/commit/451570e6038a1b91b5723db9b941cd916fd76348)) + +### [5.19.1](https://github.com/googleapis/nodejs-storage/compare/v5.19.0...v5.19.1) (2022-04-08) + + +### Bug Fixes + +* prevent retrying 200 response ([#1857](https://github.com/googleapis/nodejs-storage/issues/1857)) ([638a47b](https://github.com/googleapis/nodejs-storage/commit/638a47b4e7ecc6e94b3b11d1ccc7c52afdeaafe1)) + +## [5.19.0](https://github.com/googleapis/nodejs-storage/compare/v5.18.3...v5.19.0) (2022-04-06) + + +### Features + +* Dual Region Support ([#1814](https://github.com/googleapis/nodejs-storage/issues/1814)) ([caf7ee5](https://github.com/googleapis/nodejs-storage/commit/caf7ee561fd640b0daea92c7837c47e66070c30c)) + +### [5.18.3](https://github.com/googleapis/nodejs-storage/compare/v5.18.2...v5.18.3) (2022-03-28) + + +### Bug Fixes + +* encode name portion when calling publicUrl function ([#1828](https://github.com/googleapis/nodejs-storage/issues/1828)) ([5522b35](https://github.com/googleapis/nodejs-storage/commit/5522b35e83857421a00e71c4e93ba6ae0ffccb90)) +* fixed typo ([#1803](https://github.com/googleapis/nodejs-storage/issues/1803)) ([be70dae](https://github.com/googleapis/nodejs-storage/commit/be70dae33751ddc3e0ae5a55b5cdbf2002a42932)) + +### [5.18.2](https://github.com/googleapis/nodejs-storage/compare/v5.18.1...v5.18.2) (2022-02-16) + + +### Bug Fixes + +* resumable uploads should respect autoRetry & multipart uploads should correctly use preconditions ([#1779](https://github.com/googleapis/nodejs-storage/issues/1779)) ([1e72586](https://github.com/googleapis/nodejs-storage/commit/1e725867dce8f78070435b96b65f97f2253c0e80)) + +### [5.18.1](https://github.com/googleapis/nodejs-storage/compare/v5.18.0...v5.18.1) (2022-01-26) + + +### Bug Fixes + +* **gcs-resumable-upload:** Stop Duplicate Response Handlers on Retries ([#1764](https://github.com/googleapis/nodejs-storage/issues/1764)) ([fe44871](https://github.com/googleapis/nodejs-storage/commit/fe4487187aa405e7d7f8e0bec485bbddb76ea050)) + +## [5.18.0](https://github.com/googleapis/nodejs-storage/compare/v5.17.0...v5.18.0) (2022-01-18) + + +### Features + +* Expose `chunkSize` param for `CreateResumableUploadOptions` ([#1754](https://github.com/googleapis/nodejs-storage/issues/1754)) ([3acfd5b](https://github.com/googleapis/nodejs-storage/commit/3acfd5b2412d046c471d8d707023e034dc1a167a)) + +## [5.17.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.16.1...v5.17.0) (2022-01-10) + + +### Features + +* add support for rpo (turbo replication) metadata field when cre… ([#1648](https://www.github.com/googleapis/nodejs-storage/issues/1648)) ([291e6ef](https://www.github.com/googleapis/nodejs-storage/commit/291e6ef48efcfca55b4a7dca8868a57c0eeec89b)) + + +### Bug Fixes + +* remove compodoc dev dependency ([#1745](https://www.github.com/googleapis/nodejs-storage/issues/1745)) ([809bf11](https://www.github.com/googleapis/nodejs-storage/commit/809bf11b8a2a2203db82aec38b6a6023a805bd62)) + +### [5.16.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.16.0...v5.16.1) (2021-11-29) + + +### Bug Fixes + +* change properties with function value to methods ([#1715](https://www.github.com/googleapis/nodejs-storage/issues/1715)) ([c365254](https://www.github.com/googleapis/nodejs-storage/commit/c36525402da8e748971473b1cdd2423e8fd953e1)) +* revert skip validation ([#1718](https://www.github.com/googleapis/nodejs-storage/issues/1718)) ([0c75e33](https://www.github.com/googleapis/nodejs-storage/commit/0c75e33eb0291aa7dfc704c86733f4c0dc78d322)) +* stop File.download from truncating output file on failure ([#1720](https://www.github.com/googleapis/nodejs-storage/issues/1720)) ([d77979b](https://www.github.com/googleapis/nodejs-storage/commit/d77979b1003dbb89cd9d4725330de50b1f8d9262)) + +## [5.16.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.15.5...v5.16.0) (2021-11-09) + + +### Features + +* improved error messages for resumable uploads ([#1708](https://www.github.com/googleapis/nodejs-storage/issues/1708)) ([50cdbb6](https://www.github.com/googleapis/nodejs-storage/commit/50cdbb6f730ee7f96cb598c4cda412fc4bcc8807)) + + +### Bug Fixes + +* add scenario 3 conformance tests ([#1702](https://www.github.com/googleapis/nodejs-storage/issues/1702)) ([e16a3a5](https://www.github.com/googleapis/nodejs-storage/commit/e16a3a5eb09a388743259d54c31e62d7fc220bf0)) +* retry uri creation dep update & conformance tests ([#1700](https://www.github.com/googleapis/nodejs-storage/issues/1700)) ([d265f8c](https://www.github.com/googleapis/nodejs-storage/commit/d265f8c5e4e6a8c8239e959dfb4d0acbf4cdfe0a)) + +### [5.15.6](https://www.github.com/googleapis/nodejs-storage/compare/v5.15.5...v5.15.6) (2021-11-05) + + +### Bug Fixes + +* add scenario 3 conformance tests ([#1702](https://www.github.com/googleapis/nodejs-storage/issues/1702)) ([e16a3a5](https://www.github.com/googleapis/nodejs-storage/commit/e16a3a5eb09a388743259d54c31e62d7fc220bf0)) +* retry uri creation dep update & conformance tests ([#1700](https://www.github.com/googleapis/nodejs-storage/issues/1700)) ([d265f8c](https://www.github.com/googleapis/nodejs-storage/commit/d265f8c5e4e6a8c8239e959dfb4d0acbf4cdfe0a)) + +### [5.15.5](https://www.github.com/googleapis/nodejs-storage/compare/v5.15.4...v5.15.5) (2021-11-03) + + +### Bug Fixes + +* **deps:** update dependency mime to v3 ([#1696](https://www.github.com/googleapis/nodejs-storage/issues/1696)) ([f337208](https://www.github.com/googleapis/nodejs-storage/commit/f33720883bb6d797d2fb89d5e6ff9584d216be74)) +* explicitly define function type of getFilesStream ([#1697](https://www.github.com/googleapis/nodejs-storage/issues/1697)) ([c950c23](https://www.github.com/googleapis/nodejs-storage/commit/c950c23742bb9291a3e15b95ae0ee4a13466c361)) + +### [5.15.4](https://www.github.com/googleapis/nodejs-storage/compare/v5.15.3...v5.15.4) (2021-11-01) + + +### Bug Fixes + +* check e is not null ([#1692](https://www.github.com/googleapis/nodejs-storage/issues/1692)) ([56ff485](https://www.github.com/googleapis/nodejs-storage/commit/56ff485cbe28ba048c9a689711b034c416853b1f)) + +### [5.15.3](https://www.github.com/googleapis/nodejs-storage/compare/v5.15.2...v5.15.3) (2021-10-14) + + +### Bug Fixes + +* do not use src precondition options in copy. ([#1666](https://www.github.com/googleapis/nodejs-storage/issues/1666)) ([678ae77](https://www.github.com/googleapis/nodejs-storage/commit/678ae77dfb2c1eb2272734f34315b4d0ec726076)) + +### [5.15.2](https://www.github.com/googleapis/nodejs-storage/compare/v5.15.1...v5.15.2) (2021-10-13) + + +### Bug Fixes + +* remove bucket preconditions from deleteFiles, it is a file operation not bucket ([#1661](https://www.github.com/googleapis/nodejs-storage/issues/1661)) ([6b7a06d](https://www.github.com/googleapis/nodejs-storage/commit/6b7a06defe1a3cadc6fad9258ff3fb01a2ecce0a)) + +### [5.15.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.15.0...v5.15.1) (2021-10-12) + + +### Bug Fixes + +* check generation on source files not metageneration on bucket ([#1654](https://www.github.com/googleapis/nodejs-storage/issues/1654)) ([760231c](https://www.github.com/googleapis/nodejs-storage/commit/760231ca520f4eedf878c245489cb07f95e153af)) + +## [5.15.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.14.7...v5.15.0) (2021-10-07) + + +### Features + +* add support for useAuthWithCustomEndpoint option ([#1645](https://www.github.com/googleapis/nodejs-storage/issues/1645)) ([d11d6b4](https://www.github.com/googleapis/nodejs-storage/commit/d11d6b4b2678cb240928e2dfe20f983d2ae896f6)) + + +### Bug Fixes + +* update common dep ([#1644](https://www.github.com/googleapis/nodejs-storage/issues/1644)) ([793467f](https://www.github.com/googleapis/nodejs-storage/commit/793467f25abf46eb7ba5e6cd1b80f735faa035dc)) + +### [5.14.8](https://www.github.com/googleapis/nodejs-storage/compare/v5.14.7...v5.14.8) (2021-10-06) + + +### Bug Fixes + +* update common dep ([#1644](https://www.github.com/googleapis/nodejs-storage/issues/1644)) ([793467f](https://www.github.com/googleapis/nodejs-storage/commit/793467f25abf46eb7ba5e6cd1b80f735faa035dc)) + +### [5.14.7](https://www.github.com/googleapis/nodejs-storage/compare/v5.14.6...v5.14.7) (2021-10-06) + + +### Bug Fixes + +* file preconditions should respect ifGenerationMatch not ifMetagenerationMatch ([#1642](https://www.github.com/googleapis/nodejs-storage/issues/1642)) ([eb6f31f](https://www.github.com/googleapis/nodejs-storage/commit/eb6f31f3f6c439e21bbc0cd46f32a7327e15f65e)) + +### [5.14.6](https://www.github.com/googleapis/nodejs-storage/compare/v5.14.5...v5.14.6) (2021-10-06) + + +### Bug Fixes + +* pass precondition opts to new file ([#1638](https://www.github.com/googleapis/nodejs-storage/issues/1638)) ([1523ca9](https://www.github.com/googleapis/nodejs-storage/commit/1523ca962fda070cf60e5b81d062e3a291461c83)) + +### [5.14.5](https://www.github.com/googleapis/nodejs-storage/compare/v5.14.4...v5.14.5) (2021-10-01) + + +### Bug Fixes + +* fix logic for buckets that do not have a generation ([#1634](https://www.github.com/googleapis/nodejs-storage/issues/1634)) ([9069bdc](https://www.github.com/googleapis/nodejs-storage/commit/9069bdc9a0ab495ab62033f432ea0e180e3b182e)) +* updated connection reset string ([#1632](https://www.github.com/googleapis/nodejs-storage/issues/1632)) ([b841d5b](https://www.github.com/googleapis/nodejs-storage/commit/b841d5b98c3d98b5f1dc7776a887159294eb0b36)) + +### [5.14.4](https://www.github.com/googleapis/nodejs-storage/compare/v5.14.3...v5.14.4) (2021-09-27) + + +### Bug Fixes + +* respect precondition settings from constructors ([#1617](https://www.github.com/googleapis/nodejs-storage/issues/1617)) ([6a48942](https://www.github.com/googleapis/nodejs-storage/commit/6a48942e540e3d96e2a5b396b8e74cbe732178be)) + +### [5.14.3](https://www.github.com/googleapis/nodejs-storage/compare/v5.14.2...v5.14.3) (2021-09-22) + + +### Bug Fixes + +* set autoretry back to instance value at end of file functions ([#1604](https://www.github.com/googleapis/nodejs-storage/issues/1604)) ([db3b59d](https://www.github.com/googleapis/nodejs-storage/commit/db3b59d731c9760d55bf112c211bdd644da911c4)) + +### [5.14.2](https://www.github.com/googleapis/nodejs-storage/compare/v5.14.1...v5.14.2) (2021-09-13) + + +### Bug Fixes + +* **build:** set default branch to main ([#1587](https://www.github.com/googleapis/nodejs-storage/issues/1587)) ([b39ce95](https://www.github.com/googleapis/nodejs-storage/commit/b39ce95a2ec9d8dd2114863898181ea10670d962)) + +### [5.14.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.14.0...v5.14.1) (2021-09-08) + + +### Bug Fixes + +* **types:** remove duplicated definition of BucketOptions and make sure proper version is exported ([#1583](https://www.github.com/googleapis/nodejs-storage/issues/1583)) ([d8f4bc5](https://www.github.com/googleapis/nodejs-storage/commit/d8f4bc59bd3e2cebfe6494842414cd9f7e32018f)) + +## [5.14.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.13.2...v5.14.0) (2021-08-26) + + +### Features + +* retries for conditionally idempotent operations ([#1561](https://www.github.com/googleapis/nodejs-storage/issues/1561)) ([653f4b4](https://www.github.com/googleapis/nodejs-storage/commit/653f4b488e8603e4008e51b45920fb7de7138eab)) + + +### Bug Fixes + +* allow retries of metadatata operations in system tests ([#1568](https://www.github.com/googleapis/nodejs-storage/issues/1568)) ([9398566](https://www.github.com/googleapis/nodejs-storage/commit/939856680d279dcd9587328d0cc58f789b022f5a)) + +### [5.13.2](https://www.github.com/googleapis/nodejs-storage/compare/v5.13.1...v5.13.2) (2021-08-26) + + +### Bug Fixes + +* update getLabels definition to actually allow no arguments when used in typescript ([#1559](https://www.github.com/googleapis/nodejs-storage/issues/1559)) ([176dbb5](https://www.github.com/googleapis/nodejs-storage/commit/176dbb5223f4442d10fd098ffa2cda5cf12144f2)) + +### [5.13.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.13.0...v5.13.1) (2021-08-18) + + +### Bug Fixes + +* **deps:** update dependency date-and-time to v2 ([#1537](https://www.github.com/googleapis/nodejs-storage/issues/1537)) ([9d0d69e](https://www.github.com/googleapis/nodejs-storage/commit/9d0d69eaf908817dec274abe915bd96bb22c663a)) + +## [5.13.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.12.0...v5.13.0) (2021-08-09) + + +### Features + +* Precondition checks ([#1523](https://www.github.com/googleapis/nodejs-storage/issues/1523)) ([7c24417](https://www.github.com/googleapis/nodejs-storage/commit/7c244178649f120cfefe58994b515da7ca6b7ffb)) + +## [5.12.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.11.1...v5.12.0) (2021-08-03) + + +### Features + +* pass precondition parameters to gcs-resumable-upload ([#1516](https://www.github.com/googleapis/nodejs-storage/issues/1516)) ([65211dd](https://www.github.com/googleapis/nodejs-storage/commit/65211ddb8ae19229154b4aca3d5ff97f2aaa9f56)) + +### [5.11.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.11.0...v5.11.1) (2021-08-02) + + +### Bug Fixes + +* don't retry non-idempotent functions ([#1517](https://www.github.com/googleapis/nodejs-storage/issues/1517)) ([c938795](https://www.github.com/googleapis/nodejs-storage/commit/c938795e8a5f14ba7724f2d0e334310dd8e8f207)) + +## [5.11.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.10.0...v5.11.0) (2021-07-23) + + +### Features + +* retries for resumable bucket.upload and file.save ([#1511](https://www.github.com/googleapis/nodejs-storage/issues/1511)) ([9bf163c](https://www.github.com/googleapis/nodejs-storage/commit/9bf163c3e3569bad1440940ff1a6bfd42404bb32)) + +## [5.10.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.9.0...v5.10.0) (2021-07-22) + + +### Features + +* retry multipart Bucket.upload ([#1509](https://www.github.com/googleapis/nodejs-storage/issues/1509)) ([730d0a0](https://www.github.com/googleapis/nodejs-storage/commit/730d0a0d4a6aa5192d998c54292d3423d3ddeaaa)) + +## [5.9.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.8.5...v5.9.0) (2021-07-21) + + +### Features + +* customize retry behavior implementation ([#1474](https://www.github.com/googleapis/nodejs-storage/issues/1474)) ([#1493](https://www.github.com/googleapis/nodejs-storage/issues/1493)) ([49008e3](https://www.github.com/googleapis/nodejs-storage/commit/49008e313b89ce6035543bf2cf1e60e253404520)) + +### [5.8.5](https://www.github.com/googleapis/nodejs-storage/compare/v5.8.4...v5.8.5) (2021-05-04) + + +### Bug Fixes + +* **deps:** updated gcs-resumable-upload dependency ([#1459](https://www.github.com/googleapis/nodejs-storage/issues/1459)) ([afaccc7](https://www.github.com/googleapis/nodejs-storage/commit/afaccc70375a2c778e4306a59bf8a86736c17f6c)) + +### [5.8.4](https://www.github.com/googleapis/nodejs-storage/compare/v5.8.3...v5.8.4) (2021-04-19) + + +### Bug Fixes + +* **deps:** update dependency date-and-time to v1 ([#1434](https://www.github.com/googleapis/nodejs-storage/issues/1434)) ([91ee6ca](https://www.github.com/googleapis/nodejs-storage/commit/91ee6cab38769d36b00d702e17b222518ad4e752)) +* don't fail if ~/.config doesn't exist ([#1428](https://www.github.com/googleapis/nodejs-storage/issues/1428)) ([3cfaba1](https://www.github.com/googleapis/nodejs-storage/commit/3cfaba19d4223c68f4382c06674f135838d32eb8)) + +### [5.8.3](https://www.github.com/googleapis/nodejs-storage/compare/v5.8.2...v5.8.3) (2021-03-29) + + +### Bug Fixes + +* update CopyOptions type to include cacheControl, contentType and contentEncoding ([#1426](https://www.github.com/googleapis/nodejs-storage/issues/1426)) ([efa5bb8](https://www.github.com/googleapis/nodejs-storage/commit/efa5bb8a22ab68c0c3e8549850e5db4d57ff29bb)) + +### [5.8.2](https://www.github.com/googleapis/nodejs-storage/compare/v5.8.1...v5.8.2) (2021-03-23) + + +### Bug Fixes + +* **perf:** pull hashes without refreshing metadata ([#1419](https://www.github.com/googleapis/nodejs-storage/issues/1419)) ([f3ec627](https://www.github.com/googleapis/nodejs-storage/commit/f3ec6278df3c3df4d4cddf5293be4dda95f0cbf7)) + +### [5.8.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.8.0...v5.8.1) (2021-03-02) + + +### Bug Fixes + +* deprecate `options.promise` and sync options with Service ([#1391](https://www.github.com/googleapis/nodejs-storage/issues/1391)) ([59cfe27](https://www.github.com/googleapis/nodejs-storage/commit/59cfe272de16c41e9c768953a25677294f520cc7)) +* **types:** support metadata override in file.copy() ([#1406](https://www.github.com/googleapis/nodejs-storage/issues/1406)) ([dda6d30](https://www.github.com/googleapis/nodejs-storage/commit/dda6d305638a07cbca188e459544393d8624f4f0)) + +## [5.8.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.7.4...v5.8.0) (2021-02-18) + + +### Features + +* adds support workload identity federation ([#1404](https://www.github.com/googleapis/nodejs-storage/issues/1404)) ([7d3a3f1](https://www.github.com/googleapis/nodejs-storage/commit/7d3a3f148361e56cbcd4dc4c8fb178b75f9208bf)) + +### [5.7.4](https://www.github.com/googleapis/nodejs-storage/compare/v5.7.3...v5.7.4) (2021-02-01) + + +### Bug Fixes + +* specified acceptable types for File.save data parameter ([#1388](https://www.github.com/googleapis/nodejs-storage/issues/1388)) ([607f6c1](https://www.github.com/googleapis/nodejs-storage/commit/607f6c1c8b9ae5414513957f54a5de2490c454b1)) + +### [5.7.3](https://www.github.com/googleapis/nodejs-storage/compare/v5.7.2...v5.7.3) (2021-01-22) + + +### Bug Fixes + +* retry multipart uploads on File.save() ([#1385](https://www.github.com/googleapis/nodejs-storage/issues/1385)) ([4dec8c1](https://www.github.com/googleapis/nodejs-storage/commit/4dec8c1a362e3f80cbbf49f8bf7e7eaa2e2ce3bc)) + +### [5.7.2](https://www.github.com/googleapis/nodejs-storage/compare/v5.7.1...v5.7.2) (2021-01-11) + + +### Bug Fixes + +* deprecated directory. prefix should be used instead ([#1370](https://www.github.com/googleapis/nodejs-storage/issues/1370)) ([fae4c06](https://www.github.com/googleapis/nodejs-storage/commit/fae4c0635909a831d65bd3111b27250795d4735b)) + +### [5.7.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.7.0...v5.7.1) (2021-01-07) + + +### Bug Fixes + +* bump dependencies See [#1372](https://www.github.com/googleapis/nodejs-storage/issues/1372) for details ([#1375](https://www.github.com/googleapis/nodejs-storage/issues/1375)) ([7cf0264](https://www.github.com/googleapis/nodejs-storage/commit/7cf0264c0cea2e0d668818924aaa737381c07bfa)) + +## [5.7.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.6.0...v5.7.0) (2020-12-10) + + +### Features + +* support metadata updates from makePrivate() methods ([#1355](https://www.github.com/googleapis/nodejs-storage/issues/1355)) ([3db1e83](https://www.github.com/googleapis/nodejs-storage/commit/3db1e832af1cd3a394305b0a1120953d85f86249)) + + +### Bug Fixes + +* capitalize action in Bucket#addLifecycleRule ([#1358](https://www.github.com/googleapis/nodejs-storage/issues/1358)) ([205f39f](https://www.github.com/googleapis/nodejs-storage/commit/205f39f970639c06258fb1e0cd18a4d963729262)) +* jsdoc for bucket ([#1359](https://www.github.com/googleapis/nodejs-storage/issues/1359)) ([5fa530a](https://www.github.com/googleapis/nodejs-storage/commit/5fa530ab909e837558deecd7e1aa50a7cc4f5830)) + +## [5.6.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.5.1...v5.6.0) (2020-12-02) + + +### Features + +* allow ignoring 404 errors during delete() operations ([#1347](https://www.github.com/googleapis/nodejs-storage/issues/1347)) ([dab0e7d](https://www.github.com/googleapis/nodejs-storage/commit/dab0e7d9345499411cec57186ef7404026f041eb)) + +### [5.5.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.5.0...v5.5.1) (2020-12-01) + + +### Bug Fixes + +* add publicUrl to exclude promisifyAll ([#1339](https://www.github.com/googleapis/nodejs-storage/issues/1339)) ([ea2c2c9](https://www.github.com/googleapis/nodejs-storage/commit/ea2c2c9d670b8a8eefffa3e67dd2599135c0d933)) +* error if both `storageClass` and specific storage class name are provided ([#1323](https://www.github.com/googleapis/nodejs-storage/issues/1323)) ([91a65f8](https://www.github.com/googleapis/nodejs-storage/commit/91a65f86fe5f9608437b91d6e67192c78b0e8d7b)) +* only throw if `storageClass` and specific storage class name provide different values ([#1346](https://www.github.com/googleapis/nodejs-storage/issues/1346)) ([1765608](https://www.github.com/googleapis/nodejs-storage/commit/1765608430d98c555e4a7431c28cd5878e65c7df)) +* **docs:** explain manual pagination and add usage sample ([#1317](https://www.github.com/googleapis/nodejs-storage/issues/1317)) ([16b779d](https://www.github.com/googleapis/nodejs-storage/commit/16b779de912f6ac082ec5ee3d904b125a5526485)) + +## [5.5.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.4.0...v5.5.0) (2020-11-03) + + +### Features + +* accessibleAt v4 signed urls ([#1328](https://www.github.com/googleapis/nodejs-storage/issues/1328)) ([1e0295e](https://www.github.com/googleapis/nodejs-storage/commit/1e0295eebd1120ce6cbcabee4cf1aaa825455d4b)) + +## [5.4.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.3.0...v5.4.0) (2020-10-29) + + +### Features + +* **userAgent:** allow for optional user agent to be provided ([#1313](https://www.github.com/googleapis/nodejs-storage/issues/1313)) ([13a064f](https://www.github.com/googleapis/nodejs-storage/commit/13a064f3e7342640c5c6e64e060e8558062a31c7)) +* add custom time field in metadata sample ([#1285](https://www.github.com/googleapis/nodejs-storage/issues/1285)) ([9e3474f](https://www.github.com/googleapis/nodejs-storage/commit/9e3474f87dabd300d914c0f44f49a13dae34227d)) +* add File#rename ([#1311](https://www.github.com/googleapis/nodejs-storage/issues/1311)) ([c77eaa3](https://www.github.com/googleapis/nodejs-storage/commit/c77eaa363be18b6b9ddd1f7b505e23be8cd5bf98)) +* public url of a file ([#1324](https://www.github.com/googleapis/nodejs-storage/issues/1324)) ([5ec256e](https://www.github.com/googleapis/nodejs-storage/commit/5ec256e58d04dbcb8dee8cde7460b306da5ff880)) + + +### Bug Fixes + +* set customEndpoint for custom environments ([#1316](https://www.github.com/googleapis/nodejs-storage/issues/1316)) ([60910e1](https://www.github.com/googleapis/nodejs-storage/commit/60910e1903f95a0abca6b9855d11833b32666e2c)) +* **deps:** update dependency gaxios to v4 ([#1322](https://www.github.com/googleapis/nodejs-storage/issues/1322)) ([f9b16d8](https://www.github.com/googleapis/nodejs-storage/commit/f9b16d82e5c6a26d76f721a277cf07b518b63a42)) +* moved publicUrl comment ([#1327](https://www.github.com/googleapis/nodejs-storage/issues/1327)) ([249ed2c](https://www.github.com/googleapis/nodejs-storage/commit/249ed2c4e5fe9cfb029ca8e60c1bafab5c370a48)) +* self-upload files for Unicode system tests ([#1318](https://www.github.com/googleapis/nodejs-storage/issues/1318)) ([e826a52](https://www.github.com/googleapis/nodejs-storage/commit/e826a526c3a5a8515cc7a07c15f86fcfb0fb2f93)) +* **deps:** update dependency duplexify to v4 ([#1302](https://www.github.com/googleapis/nodejs-storage/issues/1302)) ([1ce3d89](https://www.github.com/googleapis/nodejs-storage/commit/1ce3d895b3f5d12776f0634b41b1baef7aa4491c)) +* **deps:** update dependency yargs to v16 ([#1289](https://www.github.com/googleapis/nodejs-storage/issues/1289)) ([12f2133](https://www.github.com/googleapis/nodejs-storage/commit/12f2133b7e3ec9612b3694d93b9ff756cb1bbd66)) +* **types:** add `CreateBucketRequest.storageClass` ([#1299](https://www.github.com/googleapis/nodejs-storage/issues/1299)) ([d854c26](https://www.github.com/googleapis/nodejs-storage/commit/d854c2609eb87886a010e726daa5f1339ee9ea38)), closes [#1248](https://www.github.com/googleapis/nodejs-storage/issues/1248) + +## [5.3.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.2.0...v5.3.0) (2020-08-24) + + +### Features + +* support noncurrent time object lifecycle condition ([#1216](https://www.github.com/googleapis/nodejs-storage/issues/1216)) ([d198aff](https://www.github.com/googleapis/nodejs-storage/commit/d198affa98b0dd027d6628eaff9fcd2dca5c7b47)) + +## [5.2.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.1.2...v5.2.0) (2020-08-10) + + +### Features + +* remove through2 dependency ([#1240](https://www.github.com/googleapis/nodejs-storage/issues/1240)) ([97c73dd](https://www.github.com/googleapis/nodejs-storage/commit/97c73ddec4dbb7cbf7a1486c16ed24dbe9be6d83)) + + +### Bug Fixes + +* support request interceptors for file uploads ([#1225](https://www.github.com/googleapis/nodejs-storage/issues/1225)) ([aa4e4ec](https://www.github.com/googleapis/nodejs-storage/commit/aa4e4ecf49c2e67fdeb90d54e06335fe8671c185)) +* **deps:** update dependency date-and-time to ^0.14.0 ([#1263](https://www.github.com/googleapis/nodejs-storage/issues/1263)) ([408aff9](https://www.github.com/googleapis/nodejs-storage/commit/408aff9234b30f4d68bc9027fbc3088ab059578a)) + +### [5.1.2](https://www.github.com/googleapis/nodejs-storage/compare/v5.1.1...v5.1.2) (2020-07-06) + + +### Bug Fixes + +* **deps:** update dependency through2 to v4 ([#1226](https://www.github.com/googleapis/nodejs-storage/issues/1226)) ([2be25e8](https://www.github.com/googleapis/nodejs-storage/commit/2be25e8fbd4bc60f8e702c2120cf99510508cd15)) + +### [5.1.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.1.0...v5.1.1) (2020-06-19) + + +### Bug Fixes + +* **deps:** update dependency p-limit to v3 ([#1210](https://www.github.com/googleapis/nodejs-storage/issues/1210)) ([7da0379](https://www.github.com/googleapis/nodejs-storage/commit/7da03797791ed20ffbbc6456c7cfad1653c59e76)) +* update node issue template ([#1220](https://www.github.com/googleapis/nodejs-storage/issues/1220)) ([eaac92b](https://www.github.com/googleapis/nodejs-storage/commit/eaac92bb789aea32718280fb67c6496022fa56d3)) + +## [5.1.0](https://www.github.com/googleapis/nodejs-storage/compare/v5.0.1...v5.1.0) (2020-06-11) + + +### Features + +* **secrets:** begin migration to secret manager from keystore ([#1211](https://www.github.com/googleapis/nodejs-storage/issues/1211)) ([11b16ae](https://www.github.com/googleapis/nodejs-storage/commit/11b16ae1b06273ab169d7b1f65a54196956ef4ea)) +* allow setting timeouts on uploads ([#1208](https://www.github.com/googleapis/nodejs-storage/issues/1208)) ([01b3691](https://www.github.com/googleapis/nodejs-storage/commit/01b3691bbfc4a93e2229ea07184637479c515183)) + + +### Bug Fixes + +* **deps:** update dependency @google-cloud/pubsub to v2 ([#1201](https://www.github.com/googleapis/nodejs-storage/issues/1201)) ([c684b2a](https://www.github.com/googleapis/nodejs-storage/commit/c684b2abe99169a8801d76fda38a4cbfba91f417)) + +### [5.0.1](https://www.github.com/googleapis/nodejs-storage/compare/v5.0.0...v5.0.1) (2020-05-20) + + +### Bug Fixes + +* StorageOptions.apiEndpoint overrides simple and resumable uploads ([#1161](https://www.github.com/googleapis/nodejs-storage/issues/1161)) ([cf8ea5c](https://www.github.com/googleapis/nodejs-storage/commit/cf8ea5cbbecea50233b1681c7f8aba121ec37a1a)) +* **types:** assert in typescript 3.7 ([#1198](https://www.github.com/googleapis/nodejs-storage/issues/1198)) ([d5aa8b7](https://www.github.com/googleapis/nodejs-storage/commit/d5aa8b7892366c9fbbdc33dd001f644da61bb22e)) + +## [5.0.0](https://www.github.com/googleapis/nodejs-storage/compare/v4.7.0...v5.0.0) (2020-05-13) + + +### ⚠ BREAKING CHANGES + +* automatically detect contentType if not provided (#1190) +* drop keepAcl parameter in file copy (#1166) +* drop support for node.js 8.x + +### Features + +* automatically detect contentType if not provided ([#1190](https://www.github.com/googleapis/nodejs-storage/issues/1190)) ([b31ba4a](https://www.github.com/googleapis/nodejs-storage/commit/b31ba4a11399b57538ddf0d6ca2e10b2aa3fbc3a)) +* enable bytes read tracking ([#1074](https://www.github.com/googleapis/nodejs-storage/issues/1074)) ([0776a04](https://www.github.com/googleapis/nodejs-storage/commit/0776a044f3b2149b485e114369e952688df75645)) + + +### Bug Fixes + +* **bucket:** Only disable resumable uploads for bucket.upload (fixes [#1133](https://www.github.com/googleapis/nodejs-storage/issues/1133)) ([#1135](https://www.github.com/googleapis/nodejs-storage/issues/1135)) ([2c20148](https://www.github.com/googleapis/nodejs-storage/commit/2c201486b7b2d3146846ac96c877a904c4a674b0)), closes [/github.com/googleapis/nodejs-storage/pull/1135#issuecomment-620070038](https://www.github.com/googleapis//github.com/googleapis/nodejs-storage/pull/1135/issues/issuecomment-620070038) +* add whitespace to generateV4SignedPolicy ([#1136](https://www.github.com/googleapis/nodejs-storage/issues/1136)) ([dcee78b](https://www.github.com/googleapis/nodejs-storage/commit/dcee78b98da23b02fe7d2f13a9270546bc07bba8)) +* apache license URL ([#468](https://www.github.com/googleapis/nodejs-storage/issues/468)) ([#1151](https://www.github.com/googleapis/nodejs-storage/issues/1151)) ([e8116d3](https://www.github.com/googleapis/nodejs-storage/commit/e8116d3c6fa7412858692e67745b514eef78850e)) +* Point to team in correct org ([#1185](https://www.github.com/googleapis/nodejs-storage/issues/1185)) ([0bb1909](https://www.github.com/googleapis/nodejs-storage/commit/0bb19098013acf71cc3842f78ff333a8e356331a)) +* **deps:** update dependency @google-cloud/common to v3 ([#1134](https://www.github.com/googleapis/nodejs-storage/issues/1134)) ([774ac5c](https://www.github.com/googleapis/nodejs-storage/commit/774ac5c75f02238418cc8ed7242297ea573ca9cb)) +* **deps:** update dependency @google-cloud/paginator to v3 ([#1131](https://www.github.com/googleapis/nodejs-storage/issues/1131)) ([c1614d9](https://www.github.com/googleapis/nodejs-storage/commit/c1614d98e3047db379e09299b1014e80d73ed52f)) +* **deps:** update dependency @google-cloud/promisify to v2 ([#1127](https://www.github.com/googleapis/nodejs-storage/issues/1127)) ([06624a5](https://www.github.com/googleapis/nodejs-storage/commit/06624a534cd1fdbc38455eee8d89f9f60ba75758)) +* **deps:** update dependency uuid to v8 ([#1170](https://www.github.com/googleapis/nodejs-storage/issues/1170)) ([6a98d64](https://www.github.com/googleapis/nodejs-storage/commit/6a98d64831baf1ca1ec2f03ecc4914745cba1c86)) +* **deps:** update gcs-resumable-upload, remove gitnpm usage ([#1186](https://www.github.com/googleapis/nodejs-storage/issues/1186)) ([c78c9cd](https://www.github.com/googleapis/nodejs-storage/commit/c78c9cde49dccb2fcd4ce10e4e9f8299d65f6838)) +* **v4-policy:** encode special characters ([#1169](https://www.github.com/googleapis/nodejs-storage/issues/1169)) ([6e48539](https://www.github.com/googleapis/nodejs-storage/commit/6e48539d76ca27e6f4c6cf2ac0872970f7391fed)) +* sync to googleapis/conformance-tests@fa559a1 ([#1167](https://www.github.com/googleapis/nodejs-storage/issues/1167)) ([5500446](https://www.github.com/googleapis/nodejs-storage/commit/550044619d2f17a1977c83bce5df915c6dc9578c)), closes [#1168](https://www.github.com/googleapis/nodejs-storage/issues/1168) + + +### Miscellaneous Chores + +* drop keepAcl parameter in file copy ([#1166](https://www.github.com/googleapis/nodejs-storage/issues/1166)) ([5a4044a](https://www.github.com/googleapis/nodejs-storage/commit/5a4044a8ba13f248fc4f791248f797eb0f1f3c16)) + + +### Build System + +* drop support for node.js 8.x ([b80c025](https://www.github.com/googleapis/nodejs-storage/commit/b80c025f106052fd25554c64314b3b3520e829b5)) + +## [4.7.0](https://www.github.com/googleapis/nodejs-storage/compare/v4.6.0...v4.7.0) (2020-03-26) + + +### Features + +* add remove conditional binding ([#1107](https://www.github.com/googleapis/nodejs-storage/issues/1107)) ([2143705](https://www.github.com/googleapis/nodejs-storage/commit/21437053e9497aa95ef37a865ffbdbaf4134138f)) +* v4 POST with signed policy ([#1102](https://www.github.com/googleapis/nodejs-storage/issues/1102)) ([a3d5b88](https://www.github.com/googleapis/nodejs-storage/commit/a3d5b88b8d3d25b6e16808eb5c1425aa0a8c5ecc)), closes [#1125](https://www.github.com/googleapis/nodejs-storage/issues/1125) + + +### Bug Fixes + +* **deps:** update dependency date-and-time to ^0.13.0 ([#1106](https://www.github.com/googleapis/nodejs-storage/issues/1106)) ([b759605](https://www.github.com/googleapis/nodejs-storage/commit/b7596058e130ee2d82dc2221f24220b83c04fdae)) +* **deps:** update dependency gaxios to v3 ([#1129](https://www.github.com/googleapis/nodejs-storage/issues/1129)) ([5561452](https://www.github.com/googleapis/nodejs-storage/commit/5561452cb0b6e5a1dcabea6973db57799422abb7)) +* **types:** wrap GetSignedUrlResponse ([#1119](https://www.github.com/googleapis/nodejs-storage/issues/1119)) ([0c7ac16](https://www.github.com/googleapis/nodejs-storage/commit/0c7ac161f808201562f60710b9ec7bce4fbf819f)) + +## [4.6.0](https://www.github.com/googleapis/nodejs-storage/compare/v4.5.0...v4.6.0) (2020-03-13) + + +### Features + +* **storage:** Add versioning as optional metadata param of createBucket ([#1090](https://www.github.com/googleapis/nodejs-storage/issues/1090)) ([39869e3](https://www.github.com/googleapis/nodejs-storage/commit/39869e3c6c62eabe1840f0fd884b361265e2cb76)) + +## [4.5.0](https://www.github.com/googleapis/nodejs-storage/compare/v4.4.0...v4.5.0) (2020-03-06) + + +### Features + +* **v4:** support v4 signed URL for GA ([#1072](https://www.github.com/googleapis/nodejs-storage/issues/1072)) ([79d7b8f](https://www.github.com/googleapis/nodejs-storage/commit/79d7b8f5a187f23e58bf84db3f4b8c4b1a921978)), closes [#1051](https://www.github.com/googleapis/nodejs-storage/issues/1051) [#1056](https://www.github.com/googleapis/nodejs-storage/issues/1056) [#1066](https://www.github.com/googleapis/nodejs-storage/issues/1066) [#1068](https://www.github.com/googleapis/nodejs-storage/issues/1068) [#1069](https://www.github.com/googleapis/nodejs-storage/issues/1069) [#1067](https://www.github.com/googleapis/nodejs-storage/issues/1067) [#1070](https://www.github.com/googleapis/nodejs-storage/issues/1070) + +## [4.4.0](https://www.github.com/googleapis/nodejs-storage/compare/v4.3.2...v4.4.0) (2020-03-03) + + +### Features + +* **bucket:** support single source in combine ([#1076](https://www.github.com/googleapis/nodejs-storage/issues/1076)) ([#1085](https://www.github.com/googleapis/nodejs-storage/issues/1085)) ([251a617](https://www.github.com/googleapis/nodejs-storage/commit/251a617791ca4b0e148b741d1931013150becc02)) + +### [4.3.2](https://www.github.com/googleapis/nodejs-storage/compare/v4.3.1...v4.3.2) (2020-03-02) + + +### Bug Fixes + +* **deps:** update dependency uuid to v7 ([#1081](https://www.github.com/googleapis/nodejs-storage/issues/1081)) ([c72d57f](https://www.github.com/googleapis/nodejs-storage/commit/c72d57f6f2982dad512d425ee3f041b43a87c278)) +* **tests:** flaky getSignedPolicy tests ([#1093](https://www.github.com/googleapis/nodejs-storage/issues/1093)) ([531050a](https://www.github.com/googleapis/nodejs-storage/commit/531050a05e5b1eeedb25647417a8ae9df8d76f29)) + +### [4.3.1](https://www.github.com/googleapis/nodejs-storage/compare/v4.3.0...v4.3.1) (2020-02-06) + + +### Bug Fixes + +* **samples:** fix overwritten IAM conditions sample ([#1042](https://www.github.com/googleapis/nodejs-storage/issues/1042)) ([25d839c](https://www.github.com/googleapis/nodejs-storage/commit/25d839ccf421276d8a4c18b2be95004ca832d84d)) +* skip validation for server decompressed objects ([#1063](https://www.github.com/googleapis/nodejs-storage/issues/1063)) ([d6e3738](https://www.github.com/googleapis/nodejs-storage/commit/d6e37382da1ed3b72771770cb9447c62c91f26a5)) +* unhandled promise rejection warning in samples ([#1056](https://www.github.com/googleapis/nodejs-storage/issues/1056)) ([e95ec19](https://www.github.com/googleapis/nodejs-storage/commit/e95ec19756388e6fc4fc8e0d49a40c613ed006c6)) + +## [4.3.0](https://www.github.com/googleapis/nodejs-storage/compare/v4.2.0...v4.3.0) (2020-01-21) + + +### Features + +* add support for CORS as a first class citizen ([#1020](https://www.github.com/googleapis/nodejs-storage/issues/1020)) ([9fee6d9](https://www.github.com/googleapis/nodejs-storage/commit/9fee6d969a1311a3db18bf523d3614adef0526ce)) + + +### Bug Fixes + +* **deps:** update dependency date-and-time to ^0.12.0 ([#1032](https://www.github.com/googleapis/nodejs-storage/issues/1032)) ([a324564](https://www.github.com/googleapis/nodejs-storage/commit/a324564e8f29345412047b7f6296098211e0e831)) +* a couple of typos from the refactorNodeSampleStorage branch ([#1038](https://www.github.com/googleapis/nodejs-storage/issues/1038)) ([8faa6c6](https://www.github.com/googleapis/nodejs-storage/commit/8faa6c6698b3b7ef4857d978482209bc5077e08e)) +* do not modify constructor options ([#974](https://www.github.com/googleapis/nodejs-storage/issues/974)) ([4969148](https://www.github.com/googleapis/nodejs-storage/commit/4969148f5e114d86aa4928109f099ec15d56fda5)) +* refactor getMetadata sample into its own file ([#1008](https://www.github.com/googleapis/nodejs-storage/issues/1008)) ([6ed1af8](https://www.github.com/googleapis/nodejs-storage/commit/6ed1af8aadd6f72cf0957d02e403f7c551166a5c)) +* refactor getRetentionPolicy sample into its own file ([#993](https://www.github.com/googleapis/nodejs-storage/issues/993)) ([47e4ad8](https://www.github.com/googleapis/nodejs-storage/commit/47e4ad8c8a4fd16f4f1d9d6d9bfdc9e30b1ab999)) +* refactor getUniformBucketLevelAccess into its own file ([#981](https://www.github.com/googleapis/nodejs-storage/issues/981)) ([0ba69f1](https://www.github.com/googleapis/nodejs-storage/commit/0ba69f1b3d6093701dac927fa4543d2d911ce8b0)) +* refactor rotateEncryptionKey sample into its own file ([#1030](https://www.github.com/googleapis/nodejs-storage/issues/1030)) ([afdf0fe](https://www.github.com/googleapis/nodejs-storage/commit/afdf0febe8760c0819736961065d134e231a0afa)) +* refactor uploadEncryptedFile sample into its own file ([#1028](https://www.github.com/googleapis/nodejs-storage/issues/1028)) ([ba4520b](https://www.github.com/googleapis/nodejs-storage/commit/ba4520b5f925968717222ffe5d2b1dbcfbea4334)) +* refactoring disableUniformBucketLevelAccess into its own file ([#980](https://www.github.com/googleapis/nodejs-storage/issues/980)) ([1481e20](https://www.github.com/googleapis/nodejs-storage/commit/1481e20d8332ee2806116166fb16028506487d2d)) + +## [4.2.0](https://www.github.com/googleapis/nodejs-storage/compare/v4.1.3...v4.2.0) (2020-01-02) + + +### Features + +* **iam:** getIamPolicy to support requestedPolicyVersion ([#960](https://www.github.com/googleapis/nodejs-storage/issues/960)) ([0f38f75](https://www.github.com/googleapis/nodejs-storage/commit/0f38f7597f5e671b4c08830f8d457f9ca2c03cd1)) +* add support for Archive storage class ([#908](https://www.github.com/googleapis/nodejs-storage/issues/908)) ([63f63f4](https://www.github.com/googleapis/nodejs-storage/commit/63f63f448d2d220b1e77a7dcd379f7ac7fc22af3)) + + +### Bug Fixes + +* **deps:** pin TypeScript below 3.7.0 ([#952](https://www.github.com/googleapis/nodejs-storage/issues/952)) ([1fea60c](https://www.github.com/googleapis/nodejs-storage/commit/1fea60c04fd9762c5506c22df0992cdb8fce4ea0)) +* add createBucket file and updated relevant test ([#940](https://www.github.com/googleapis/nodejs-storage/issues/940)) ([913b43e](https://www.github.com/googleapis/nodejs-storage/commit/913b43e66bb10bd5dbf6b0bca9e65edd48b54234)) +* update PolicyDocument types ([#944](https://www.github.com/googleapis/nodejs-storage/issues/944)) ([b1c05b2](https://www.github.com/googleapis/nodejs-storage/commit/b1c05b27029215c8bc313a8f4f16ff13d5af2391)) + +### [4.1.3](https://www.github.com/googleapis/nodejs-storage/compare/v4.1.2...v4.1.3) (2019-11-18) + + +### Bug Fixes + +* **deps:** update dependency date-and-time to ^0.11.0 ([#934](https://www.github.com/googleapis/nodejs-storage/issues/934)) ([c4bf1c6](https://www.github.com/googleapis/nodejs-storage/commit/c4bf1c616d0a9402237708bddac1341c620f0542)) +* **deps:** update dependency yargs to v15 ([#933](https://www.github.com/googleapis/nodejs-storage/issues/933)) ([f40fe0c](https://www.github.com/googleapis/nodejs-storage/commit/f40fe0c5bd4e9b89ebe007e59986580fae4d7e09)) + +### [4.1.2](https://www.github.com/googleapis/nodejs-storage/compare/v4.1.1...v4.1.2) (2019-11-12) + + +### Bug Fixes + +* do not check access of configPath ([#915](https://www.github.com/googleapis/nodejs-storage/issues/915)) ([a21a644](https://www.github.com/googleapis/nodejs-storage/commit/a21a6443346f91f275233a9a07fb79550035e157)) +* missing snippets with jsdoc-region-tag ([#924](https://www.github.com/googleapis/nodejs-storage/issues/924)) ([310ba90](https://www.github.com/googleapis/nodejs-storage/commit/310ba90a48c6f02a31c1254037dfcdb4da4b7150)) +* **docs:** add jsdoc-region-tag plugin ([#929](https://www.github.com/googleapis/nodejs-storage/issues/929)) ([74526e7](https://www.github.com/googleapis/nodejs-storage/commit/74526e7f42cfa92c18ff332d0b9e10ea3b1324cf)) + +### [4.1.1](https://www.github.com/googleapis/nodejs-storage/compare/v4.1.0...v4.1.1) (2019-11-07) + + +### Bug Fixes + +* update format for upload sample ([#918](https://www.github.com/googleapis/nodejs-storage/issues/918)) ([d77208b](https://www.github.com/googleapis/nodejs-storage/commit/d77208bee82dc2d76c72fbe8d5db1bfeb8e392ff)) + +## [4.1.0](https://www.github.com/googleapis/nodejs-storage/compare/v4.0.1...v4.1.0) (2019-10-31) + + +### Features + +* [Storage] Support UniformBucketLevelAccess ([#903](https://www.github.com/googleapis/nodejs-storage/issues/903)) ([35b1bd9](https://www.github.com/googleapis/nodejs-storage/commit/35b1bd9f6db351ad1e51b135a3f25cc5e1566600)) + +### [4.0.1](https://www.github.com/googleapis/nodejs-storage/compare/v4.0.0...v4.0.1) (2019-10-31) + + +### Bug Fixes + +* **docs:** missing quote in signed url jsdoc ([#896](https://www.github.com/googleapis/nodejs-storage/issues/896)) ([f2d567f](https://www.github.com/googleapis/nodejs-storage/commit/f2d567f279fee0f48c2d6a607f4066c9da372549)) +* use storage.googleapis.com for api endpoint ([862fb16](https://www.github.com/googleapis/nodejs-storage/commit/862fb16db2990c958c0097252a44c775431a7b3f)) +* **signed-url:** replace encodeURIComponent with custom encoding function ([#905](https://www.github.com/googleapis/nodejs-storage/issues/905)) ([ba41517](https://www.github.com/googleapis/nodejs-storage/commit/ba415179f1d5951742c1b239e0500bbd2a815ddd)) + +## [4.0.0](https://www.github.com/googleapis/nodejs-storage/compare/v3.5.0...v4.0.0) (2019-10-17) + + +### ⚠ BREAKING CHANGES + +* allow leading slashes in file name (#820) + +### Bug Fixes + +* **deps:** update hash-stream-validation ([#884](https://www.github.com/googleapis/nodejs-storage/issues/884)) ([96a7fc2](https://www.github.com/googleapis/nodejs-storage/commit/96a7fc297a563819b09727990eb9ee15a421310b)) +* allow leading slashes in file name ([#820](https://www.github.com/googleapis/nodejs-storage/issues/820)) ([92e115d](https://www.github.com/googleapis/nodejs-storage/commit/92e115dca81604909fc34e983abcf47409d3f417)) + +## [3.5.0](https://www.github.com/googleapis/nodejs-storage/compare/v3.4.0...v3.5.0) (2019-10-14) + + +### Features + +* **bucket:** add enableLogging method ([#876](https://www.github.com/googleapis/nodejs-storage/issues/876)) ([b09ecac](https://www.github.com/googleapis/nodejs-storage/commit/b09ecac79b70bf99e19f0f23ffcecd17e34516bb)) + +## [3.4.0](https://www.github.com/googleapis/nodejs-storage/compare/v3.3.1...v3.4.0) (2019-10-10) + + +### Bug Fixes + +* file#move do not delete origin file if same as destination ([#874](https://www.github.com/googleapis/nodejs-storage/issues/874)) ([dcaba8a](https://www.github.com/googleapis/nodejs-storage/commit/dcaba8a)) +* pass predefined acl as destinationPredefinedAcl to qs ([#872](https://www.github.com/googleapis/nodejs-storage/issues/872)) ([09b8fa4](https://www.github.com/googleapis/nodejs-storage/commit/09b8fa4)) + + +### Features + +* add flag to allow disabling auto decompression by client ([#850](https://www.github.com/googleapis/nodejs-storage/issues/850)) ([9ebface](https://www.github.com/googleapis/nodejs-storage/commit/9ebface)) +* allow setting standard Bucket storage class ([#873](https://www.github.com/googleapis/nodejs-storage/issues/873)) ([12a99e9](https://www.github.com/googleapis/nodejs-storage/commit/12a99e9)) + +### [3.3.1](https://www.github.com/googleapis/nodejs-storage/compare/v3.3.0...v3.3.1) (2019-09-30) + + +### Bug Fixes + +* create correct v4 signed url with cname ([#868](https://www.github.com/googleapis/nodejs-storage/issues/868)) ([ace3b5e](https://www.github.com/googleapis/nodejs-storage/commit/ace3b5e)) + +## [3.3.0](https://www.github.com/googleapis/nodejs-storage/compare/v3.2.1...v3.3.0) (2019-09-19) + + +### Bug Fixes + +* **deps:** update dependency @google-cloud/pubsub to ^0.32.0 ([#849](https://www.github.com/googleapis/nodejs-storage/issues/849)) ([fdf70bb](https://www.github.com/googleapis/nodejs-storage/commit/fdf70bb)) +* add warning for unsupported keepAcl param in file#copy ([#841](https://www.github.com/googleapis/nodejs-storage/issues/841)) ([473bda0](https://www.github.com/googleapis/nodejs-storage/commit/473bda0)) +* remove unsupported keepAcl param ([#837](https://www.github.com/googleapis/nodejs-storage/issues/837)) ([5f69a3d](https://www.github.com/googleapis/nodejs-storage/commit/5f69a3d)) +* use storage.googleapis.com for api endpoint ([#854](https://www.github.com/googleapis/nodejs-storage/issues/854)) ([27fa02f](https://www.github.com/googleapis/nodejs-storage/commit/27fa02f)) +* **deps:** update dependency @google-cloud/pubsub to v1 ([#858](https://www.github.com/googleapis/nodejs-storage/issues/858)) ([31466f4](https://www.github.com/googleapis/nodejs-storage/commit/31466f4)) +* **deps:** update dependency date-and-time to ^0.10.0 ([#857](https://www.github.com/googleapis/nodejs-storage/issues/857)) ([e9ec9cf](https://www.github.com/googleapis/nodejs-storage/commit/e9ec9cf)) + + +### Features + +* adds support for asyncIterators (via readable-stream@3 dependency) ([dd5ae7f](https://www.github.com/googleapis/nodejs-storage/commit/dd5ae7f)) +* allow removal of resumable upload cache ([#773](https://www.github.com/googleapis/nodejs-storage/issues/773)) ([da943db](https://www.github.com/googleapis/nodejs-storage/commit/da943db)), closes [#217](https://www.github.com/googleapis/nodejs-storage/issues/217) + +### [3.2.1](https://www.github.com/googleapis/nodejs-storage/compare/v3.2.0...v3.2.1) (2019-08-28) + + +### Bug Fixes + +* **docs:** stop redirecting reference docs to anchor, add new sample to README ([bbb5537](https://www.github.com/googleapis/nodejs-storage/commit/bbb5537)) +* **samples:** fix failing sample view IAM member-role groups ([1c4f21f](https://www.github.com/googleapis/nodejs-storage/commit/1c4f21f)) + +## [3.2.0](https://www.github.com/googleapis/nodejs-storage/compare/v3.1.0...v3.2.0) (2019-08-22) + + +### Bug Fixes + +* **deps:** update @google-cloud/common with fixes for http ([#809](https://www.github.com/googleapis/nodejs-storage/issues/809)) ([8598631](https://www.github.com/googleapis/nodejs-storage/commit/8598631)) +* **deps:** update dependency @google-cloud/pubsub to ^0.31.0 ([#814](https://www.github.com/googleapis/nodejs-storage/issues/814)) ([604e564](https://www.github.com/googleapis/nodejs-storage/commit/604e564)) +* **deps:** update dependency date-and-time to ^0.9.0 ([#805](https://www.github.com/googleapis/nodejs-storage/issues/805)) ([8739a7d](https://www.github.com/googleapis/nodejs-storage/commit/8739a7d)) +* **ts:** fix nock @~11.0.0 ([#819](https://www.github.com/googleapis/nodejs-storage/issues/819)) ([48f9b44](https://www.github.com/googleapis/nodejs-storage/commit/48f9b44)) + + +### Features + +* hmac service account ([#751](https://www.github.com/googleapis/nodejs-storage/issues/751)) ([ed1ec7b](https://www.github.com/googleapis/nodejs-storage/commit/ed1ec7b)) + +## [3.1.0](https://www.github.com/googleapis/nodejs-storage/compare/v3.0.4...v3.1.0) (2019-08-09) + + +### Bug Fixes + +* **deps:** update dependency @google-cloud/paginator to v2 ([#781](https://www.github.com/googleapis/nodejs-storage/issues/781)) ([23244e9](https://www.github.com/googleapis/nodejs-storage/commit/23244e9)) +* **deps:** update dependency @google-cloud/pubsub to ^0.30.0 ([#778](https://www.github.com/googleapis/nodejs-storage/issues/778)) ([7256650](https://www.github.com/googleapis/nodejs-storage/commit/7256650)) +* allow calls with no request, add JSON proto ([30fff15](https://www.github.com/googleapis/nodejs-storage/commit/30fff15)) +* **deps:** update dependency date-and-time to ^0.8.0 ([#779](https://www.github.com/googleapis/nodejs-storage/issues/779)) ([ab2734d](https://www.github.com/googleapis/nodejs-storage/commit/ab2734d)) +* **deps:** upgrade @google-cloud/common version to show original… ([#795](https://www.github.com/googleapis/nodejs-storage/issues/795)) ([ea63cbe](https://www.github.com/googleapis/nodejs-storage/commit/ea63cbe)) +* **deps:** use the latest extend ([#800](https://www.github.com/googleapis/nodejs-storage/issues/800)) ([a7f0172](https://www.github.com/googleapis/nodejs-storage/commit/a7f0172)) + + +### Features + +* **file:** allow setting configPath of resumable upload ([#642](https://www.github.com/googleapis/nodejs-storage/issues/642)) ([a8ceb78](https://www.github.com/googleapis/nodejs-storage/commit/a8ceb78)) + +### [3.0.4](https://www.github.com/googleapis/nodejs-storage/compare/v3.0.3...v3.0.4) (2019-07-25) + + +### Bug Fixes + +* **deps:** update dependency pumpify to v2 ([#740](https://www.github.com/googleapis/nodejs-storage/issues/740)) ([71a4f59](https://www.github.com/googleapis/nodejs-storage/commit/71a4f59)) + +### [3.0.3](https://www.github.com/googleapis/nodejs-storage/compare/v3.0.2...v3.0.3) (2019-07-16) + + +### Bug Fixes + +* **typescript:** make SetLabelOptions optional ([#766](https://www.github.com/googleapis/nodejs-storage/issues/766)) ([4336882](https://www.github.com/googleapis/nodejs-storage/commit/4336882)) + +### [3.0.2](https://www.github.com/googleapis/nodejs-storage/compare/v3.0.1...v3.0.2) (2019-07-01) + + +### Bug Fixes + +* **docs:** fix sample code in docs ([#759](https://www.github.com/googleapis/nodejs-storage/issues/759)) ([f9e5fd8](https://www.github.com/googleapis/nodejs-storage/commit/f9e5fd8)) +* **docs:** link to reference docs section on googleapis.dev ([#753](https://www.github.com/googleapis/nodejs-storage/issues/753)) ([5e3a96b](https://www.github.com/googleapis/nodejs-storage/commit/5e3a96b)) + +### [3.0.1](https://www.github.com/googleapis/nodejs-storage/compare/v3.0.0...v3.0.1) (2019-06-14) + + +### Bug Fixes + +* async should be dependency ([#743](https://www.github.com/googleapis/nodejs-storage/issues/743)) ([e542b8b](https://www.github.com/googleapis/nodejs-storage/commit/e542b8b)) + +## [3.0.0](https://www.github.com/googleapis/nodejs-storage/compare/v2.5.0...v3.0.0) (2019-06-14) + + +### ⚠ BREAKING CHANGES + +* upgrade engines field to >=8.10.0 (#688) + +### Bug Fixes + +* **deps:** update dependency @google-cloud/common to v1 ([#705](https://www.github.com/googleapis/nodejs-storage/issues/705)) ([72a9f51](https://www.github.com/googleapis/nodejs-storage/commit/72a9f51)) +* **deps:** update dependency @google-cloud/paginator to v1 ([#695](https://www.github.com/googleapis/nodejs-storage/issues/695)) ([ada995e](https://www.github.com/googleapis/nodejs-storage/commit/ada995e)) +* **deps:** update dependency @google-cloud/promisify to v1 ([#693](https://www.github.com/googleapis/nodejs-storage/issues/693)) ([5df2f83](https://www.github.com/googleapis/nodejs-storage/commit/5df2f83)) +* **deps:** update dependency @google-cloud/pubsub to ^0.29.0 ([#714](https://www.github.com/googleapis/nodejs-storage/issues/714)) ([3ee1a2c](https://www.github.com/googleapis/nodejs-storage/commit/3ee1a2c)) +* **deps:** update dependency arrify to v2 ([#667](https://www.github.com/googleapis/nodejs-storage/issues/667)) ([ce02c27](https://www.github.com/googleapis/nodejs-storage/commit/ce02c27)) +* validate action of getSignedUrl() function ([#684](https://www.github.com/googleapis/nodejs-storage/issues/684)) ([1b09d24](https://www.github.com/googleapis/nodejs-storage/commit/1b09d24)) +* **deps:** update dependency date-and-time to ^0.7.0 ([#736](https://www.github.com/googleapis/nodejs-storage/issues/736)) ([7071f26](https://www.github.com/googleapis/nodejs-storage/commit/7071f26)) +* **deps:** update dependency xdg-basedir to v4 ([#681](https://www.github.com/googleapis/nodejs-storage/issues/681)) ([8b40e6a](https://www.github.com/googleapis/nodejs-storage/commit/8b40e6a)) +* **docs:** move to new client docs URL ([#738](https://www.github.com/googleapis/nodejs-storage/issues/738)) ([a637f99](https://www.github.com/googleapis/nodejs-storage/commit/a637f99)) +* **ts:** improve return types for response metadata ([#666](https://www.github.com/googleapis/nodejs-storage/issues/666)) ([da42bed](https://www.github.com/googleapis/nodejs-storage/commit/da42bed)) +* **types:** fix signatures of listing methods ([#703](https://www.github.com/googleapis/nodejs-storage/issues/703)) ([42937a8](https://www.github.com/googleapis/nodejs-storage/commit/42937a8)) + + +### Build System + +* upgrade engines field to >=8.10.0 ([#688](https://www.github.com/googleapis/nodejs-storage/issues/688)) ([6a1fa0f](https://www.github.com/googleapis/nodejs-storage/commit/6a1fa0f)) + + +### Features + +* add file.isPublic() function ([#708](https://www.github.com/googleapis/nodejs-storage/issues/708)) ([f86cadb](https://www.github.com/googleapis/nodejs-storage/commit/f86cadb)) +* support apiEndpoint override ([#728](https://www.github.com/googleapis/nodejs-storage/issues/728)) ([61eeb64](https://www.github.com/googleapis/nodejs-storage/commit/61eeb64)) + +## v2.5.0 + +04-04-2019 12:27 PDT + +This release brings an option to file#getSignedURL to create a version 4 Signed URL. + +```javascript +file.getSignedUrl({ + version: 'v4', // optional, defaults to v2 (existing version) + action: 'read', + expires: FUTURE_DATE, +}) +``` + +### New Features +- feat: introduce v4 signed url ([#637](https://github.com/googleapis/nodejs-storage/pull/637)) + +### Dependencies +- chore(deps): update dependency @types/node to v11.13.0 ([#662](https://github.com/googleapis/nodejs-storage/pull/662)) +- chore(deps): update dependency @types/tmp to v0.1.0 +- chore(deps): upgrade to newest version of @google-cloud/common ([#657](https://github.com/googleapis/nodejs-storage/pull/657)) +- chore(deps): update dependency typescript to ~3.4.0 +- chore(deps): update dependency tmp to ^0.1.0 ([#641](https://github.com/googleapis/nodejs-storage/pull/641)) + +### Documentation +- docs: regenerate the samples/README.md ([#649](https://github.com/googleapis/nodejs-storage/pull/649)) +- docs: slight difference in how nightly synthtool run generated README ([#650](https://github.com/googleapis/nodejs-storage/pull/650)) +- docs: new synthtool generated README ([#645](https://github.com/googleapis/nodejs-storage/pull/645)) +- docs(samples): refactor the quickstart to match the new rubric ([#647](https://github.com/googleapis/nodejs-storage/pull/647)) +- docs: update README format +- docs: add requires_billing, retire .cloud-repo-tools.json ([#644](https://github.com/googleapis/nodejs-storage/pull/644)) +- docs: add additional api_id field ([#640](https://github.com/googleapis/nodejs-storage/pull/640)) +- docs: document destination option ([#633](https://github.com/googleapis/nodejs-storage/pull/633)) +- docs: clarify in docs, the meaning of ASIA and coldline ([#632](https://github.com/googleapis/nodejs-storage/pull/632)) +- docs: add a .repo-metadata.json ([#639](https://github.com/googleapis/nodejs-storage/pull/639)) + +### Internal / Testing Changes +- test(v2-sign): add multi-valued headers system-test ([#646](https://github.com/googleapis/nodejs-storage/pull/646)) +- refactor: replace once with onetime ([#660](https://github.com/googleapis/nodejs-storage/pull/660)) +- fix: do not download cached files ([#643](https://github.com/googleapis/nodejs-storage/pull/643)) +- chore: publish to npm using wombat ([#634](https://github.com/googleapis/nodejs-storage/pull/634)) +- build: use per-repo npm publish token ([#630](https://github.com/googleapis/nodejs-storage/pull/630)) + +## v2.4.3 + +03-13-2019 17:10 PDT + +### Bug Fixes / Implementation Changes +- fix: getSigned(Policy|Url) throws if expiration is invalid Date ([#614](https://github.com/googleapis/nodejs-storage/pull/614)) +- fix: handle errors from file#createReadStream ([#615](https://github.com/googleapis/nodejs-storage/pull/615)) + +### Dependencies +- fix(deps): update dependency @google-cloud/paginator to ^0.2.0 +- fix(deps): update dependency gcs-resumable-upload to v1 ([#619](https://github.com/googleapis/nodejs-storage/pull/619)) +- fix(deps): update dependency @google-cloud/pubsub to ^0.27.0 ([#620](https://github.com/googleapis/nodejs-storage/pull/620)) +- fix(deps): update dependency @google-cloud/pubsub to ^0.26.0 ([#618](https://github.com/googleapis/nodejs-storage/pull/618)) +- fix(deps): update dependency @google-cloud/pubsub to ^0.25.0 ([#616](https://github.com/googleapis/nodejs-storage/pull/616)) +- chore(deps): update dependency mocha to v6 ([#611](https://github.com/googleapis/nodejs-storage/pull/611)) +- fix(deps): update dependency @google-cloud/promisify to ^0.4.0 ([#609](https://github.com/googleapis/nodejs-storage/pull/609)) +- chore(deps): update dependency @types/tmp to v0.0.34 ([#608](https://github.com/googleapis/nodejs-storage/pull/608)) +- fix(deps): update dependency yargs to v13 ([#606](https://github.com/googleapis/nodejs-storage/pull/606)) + +### Documentation +- docs: update links in contrib guide ([#610](https://github.com/googleapis/nodejs-storage/pull/610)) +- docs: update contributing path in README ([#603](https://github.com/googleapis/nodejs-storage/pull/603)) +- chore: move CONTRIBUTING.md to root ([#601](https://github.com/googleapis/nodejs-storage/pull/601)) + +### Internal / Testing Changes +- build: Add docuploader credentials to node publish jobs ([#624](https://github.com/googleapis/nodejs-storage/pull/624)) +- build: use node10 to run samples-test, system-test etc ([#623](https://github.com/googleapis/nodejs-storage/pull/623)) +- build: update release configuration +- build: use linkinator for docs test ([#607](https://github.com/googleapis/nodejs-storage/pull/607)) +- build: create docs test npm scripts ([#605](https://github.com/googleapis/nodejs-storage/pull/605)) +- build: test using @grpc/grpc-js in CI ([#604](https://github.com/googleapis/nodejs-storage/pull/604)) +- chore: remove console.log in system test ([#599](https://github.com/googleapis/nodejs-storage/pull/599)) + +## v2.4.2 + +02-05-2019 16:55 PST + +### Dependencies + +- deps: update @google-cloud/common ([#596](https://github.com/googleapis/nodejs-storage/pull/596)) +- chore(deps): update dependency typescript to ~3.3.0 ([#591](https://github.com/googleapis/nodejs-storage/pull/591)) + +### Documentation + +- docs: add lint/fix example to contributing guide ([#594](https://github.com/googleapis/nodejs-storage/pull/594)) + +### Internal / Testing Changes + +- test: skip public bucket system tests running under VPCSC ([#595](https://github.com/googleapis/nodejs-storage/pull/595)) + +## v2.4.1 + +01-29-2019 13:05 PST + +### Implementation Changes +- fix(ts): fix Storage.createBucket overloaded signature ([#589](https://github.com/googleapis/nodejs-storage/pull/589)) + +### Dependencies +- fix(deps): update dependency @google-cloud/pubsub to ^0.24.0 ([#588](https://github.com/googleapis/nodejs-storage/pull/588)) + +## v2.4.0 + +01-28-2019 12:13 PST + +### New Features +- fix: `expires` can be a Date, string, or number ([#548](https://github.com/googleapis/nodejs-storage/pull/548)) + +### Dependencies +- deps: upgrade nodejs-common ([#582](https://github.com/googleapis/nodejs-storage/pull/582)) +- chore(deps): update dependency eslint-config-prettier to v4 ([#586](https://github.com/googleapis/nodejs-storage/pull/586)) +- fix(deps): update dependency @google-cloud/pubsub to ^0.23.0 ([#583](https://github.com/googleapis/nodejs-storage/pull/583)) +- fix(deps): update dependency concat-stream to v2 ([#563](https://github.com/googleapis/nodejs-storage/pull/563)) + +### Documentation +- docs(samples): Bucket Policy Only Samples ([#557](https://github.com/googleapis/nodejs-storage/pull/557)) +- fix(docs): move jsdoc away from interface ([#565](https://github.com/googleapis/nodejs-storage/pull/565)) + +### Internal / Testing Changes +- test: Bucket Policy Only related system test ([#579](https://github.com/googleapis/nodejs-storage/pull/579)) +- build: check broken links in generated docs ([#567](https://github.com/googleapis/nodejs-storage/pull/567)) +- build: include only build/src in compiled source ([#572](https://github.com/googleapis/nodejs-storage/pull/572)) + +## v2.3.4 + +12-19-2018 14:21 PST + +### Implementation Changes +- fix(types): file.getMetadata should resolves to Metadata, not File ([#560](https://github.com/googleapis/nodejs-storage/pull/560)) + +### Internal / Testing Changes +- refactor: modernize the sample tests ([#558](https://github.com/googleapis/nodejs-storage/pull/558)) +- chore(build): inject yoshi automation key ([#555](https://github.com/googleapis/nodejs-storage/pull/555)) +- chore: update nyc and eslint configs ([#554](https://github.com/googleapis/nodejs-storage/pull/554)) +- chore: fix publish.sh permission +x ([#552](https://github.com/googleapis/nodejs-storage/pull/552)) +- fix(build): fix Kokoro release script ([#551](https://github.com/googleapis/nodejs-storage/pull/551)) +- build: add Kokoro configs for autorelease ([#550](https://github.com/googleapis/nodejs-storage/pull/550)) + +## v2.3.3 + +12-06-2018 17:09 PST + +### Dependencies +- chore(deps): update dependency @types/configstore to v4 ([#537](https://github.com/googleapis/nodejs-storage/pull/537)) +- chore(deps): update dependency @google-cloud/pubsub to ^0.22.0 ([#535](https://github.com/googleapis/nodejs-storage/pull/535)) + +### Documentation +- fix(docs): place doc comment above the last overload ([#544](https://github.com/googleapis/nodejs-storage/pull/544)) +- docs: update readme badges ([#536](https://github.com/googleapis/nodejs-storage/pull/536)) + +### Internal / Testing Changes +- chore: always nyc report before calling codecov ([#543](https://github.com/googleapis/nodejs-storage/pull/543)) +- chore: nyc ignore build/test by default ([#542](https://github.com/googleapis/nodejs-storage/pull/542)) +- chore: update license file ([#539](https://github.com/googleapis/nodejs-storage/pull/539)) + +## v2.3.2 + +This patch release fixed an issue affecting reading from a file on GCS ([#528](https://github.com/googleapis/nodejs-storage/issues/528)). + +### Dependencies +- fix(dep): upgrade teeny-request to v3.11.3 ([#529](https://github.com/googleapis/nodejs-storage/pull/529)) +- fix(deps): update dependency @google-cloud/common to ^0.27.0 ([#525](https://github.com/googleapis/nodejs-storage/pull/525)) + +### Internal / Testing Changes +- refactor: convert sample tests from ava to mocha ([#523](https://github.com/googleapis/nodejs-storage/pull/523)) +- docs(samples): updated samples code to use async await ([#521](https://github.com/googleapis/nodejs-storage/pull/521)) +- chore: add synth.metadata +- fix(ts): Update bucket options types ([#518](https://github.com/googleapis/nodejs-storage/pull/518)) + +## v2.3.1 + +11-14-2018 22:15 PST + +### Bug fixes +- fix: fix TypeScript and system tests ([#515](https://github.com/googleapis/nodejs-storage/pull/515)) +- fix(deps): update dependency through2 to v3 ([#507](https://github.com/googleapis/nodejs-storage/pull/507)) +- docs: File#setMetadata in parallel results in unpredictable state ([#504](https://github.com/googleapis/nodejs-storage/pull/504)) + +### Internal / Testing Changes +- chore(deps): update dependency gts to ^0.9.0 ([#514](https://github.com/googleapis/nodejs-storage/pull/514)) +- chore: update eslintignore config ([#513](https://github.com/googleapis/nodejs-storage/pull/513)) +- chore(deps): update dependency @google-cloud/nodejs-repo-tools to v3 ([#512](https://github.com/googleapis/nodejs-storage/pull/512)) +- refactor: use object.assign where possible ([#510](https://github.com/googleapis/nodejs-storage/pull/510)) +- chore: drop contributors from multiple places ([#511](https://github.com/googleapis/nodejs-storage/pull/511)) +- chore: use latest npm on Windows ([#509](https://github.com/googleapis/nodejs-storage/pull/509)) + +## v2.3.0 + +### Implementation Changes +- fix(types): Fixes getSignedUrl Return Type ([#496](https://github.com/googleapis/nodejs-storage/pull/496)) +- +### New Features +- Introduce Object Lifecycle Management ([#471](https://github.com/googleapis/nodejs-storage/pull/471)) + +### Dependencies +- chore(deps): update dependency eslint-plugin-node to v8 ([#490](https://github.com/googleapis/nodejs-storage/pull/490)) + +### Internal / Testing Changes +- chore: update issue templates ([#488](https://github.com/googleapis/nodejs-storage/pull/488)) + +## v2.2.0 + +### Bug Fixes +- fix: re-enable typescript types ([#484](https://github.com/googleapis/nodejs-storage/pull/484)) + +### Dependencies +- fix(deps): update dependency @google-cloud/common to ^0.26.0 (edited) ([#480](https://github.com/googleapis/nodejs-storage/pull/480)) +- chore: Remove 'is' dependency ([#462](https://github.com/googleapis/nodejs-storage/pull/462)) +- chore: upgrade teeny-request to 3.11.0 with type definitions ([#457](https://github.com/googleapis/nodejs-storage/pull/457)) +- feat: use small HTTP dependency ([#416](https://github.com/googleapis/nodejs-storage/pull/416)) + +### Documentation +- docs: Minor docs correction ([#465](https://github.com/googleapis/nodejs-storage/pull/465)) + +### Internal / Testing Changes +- chore: remove old issue template ([#485](https://github.com/googleapis/nodejs-storage/pull/485)) +- chore(typescript): enable noImplicitAny ([#483](https://github.com/googleapis/nodejs-storage/pull/483)) +- chore(typescript): improve typescript types and update tests ([#482](https://github.com/googleapis/nodejs-storage/pull/482)) +- build: run tests on node11 ([#481](https://github.com/googleapis/nodejs-storage/pull/481)) +- chores(build): do not collect sponge.xml from windows builds ([#478](https://github.com/googleapis/nodejs-storage/pull/478)) +- chores(build): run codecov on continuous builds ([#476](https://github.com/googleapis/nodejs-storage/pull/476)) +- chore: update new issue template ([#475](https://github.com/googleapis/nodejs-storage/pull/475)) +- fix: enable noImplicitAny for src/bucket.ts ([#472](https://github.com/googleapis/nodejs-storage/pull/472)) +- fix(tests): use unique prefix for system tests to avoid collision with another run ([#468](https://github.com/googleapis/nodejs-storage/pull/468)) +- fix: improve the types ([#467](https://github.com/googleapis/nodejs-storage/pull/467)) +- chore: move class Storage to storage.ts, create index.ts that contains all exports ([#464](https://github.com/googleapis/nodejs-storage/pull/464)) +- chore: add types to many unit tests ([#463](https://github.com/googleapis/nodejs-storage/pull/463)) +- fix: Annotate Iam types ([#461](https://github.com/googleapis/nodejs-storage/pull/461)) +- fix: complete bucket.ts noImplicitAny ([#460](https://github.com/googleapis/nodejs-storage/pull/460)) +- fix: improve the types on acl.ts ([#459](https://github.com/googleapis/nodejs-storage/pull/459)) +- fix: improve types (7) ([#458](https://github.com/googleapis/nodejs-storage/pull/458)) +- fix: improve the types ([#453](https://github.com/googleapis/nodejs-storage/pull/453)) +- chore: update build config ([#455](https://github.com/googleapis/nodejs-storage/pull/455)) +- fix: improve typescript types in src/file.ts ([#450](https://github.com/googleapis/nodejs-storage/pull/450)) +- build: fix codecov uploading on Kokoro ([#451](https://github.com/googleapis/nodejs-storage/pull/451)) +- test: Attempt to re-enable iam#testPermissions ([#429](https://github.com/googleapis/nodejs-storage/pull/429)) +- chore(deps): update dependency sinon to v7 ([#449](https://github.com/googleapis/nodejs-storage/pull/449)) +- Re-generate library using /synth.py ([#448](https://github.com/googleapis/nodejs-storage/pull/448)) +- Correct parameter name. ([#446](https://github.com/googleapis/nodejs-storage/pull/446)) + +## v2.1.0 + +This release brings support for Bucket/Object lock operations, as well as disable TypeScript as we continue to annotate the project with types. + +### New Features +- feat: Support Bucket/Object lock operations ([#374](https://github.com/googleapis/nodejs-storage/pull/374)) + +### Implementation Changes +- disable types for now ([#392](https://github.com/googleapis/nodejs-storage/pull/392)) +- Don't publish sourcemaps ([#412](https://github.com/googleapis/nodejs-storage/pull/412)) +#### TypeScript support (in progress) +- fix: add better types for file.ts ([#436](https://github.com/googleapis/nodejs-storage/pull/436)) +- fix: use ~ for typescript (and fix compile errors) ([#426](https://github.com/googleapis/nodejs-storage/pull/426)) +- fix: Add typing for File#download() ([#409](https://github.com/googleapis/nodejs-storage/pull/409)) +- chore: convert system tests to typescript ([#424](https://github.com/googleapis/nodejs-storage/pull/424)) +- Improve TypeScript types (part 4) ([#402](https://github.com/googleapis/nodejs-storage/pull/402)) +- ts: convert jsdoc types to typescript interfaces (1) ([#383](https://github.com/googleapis/nodejs-storage/pull/383)) +- fix: TS definition ([#387](https://github.com/googleapis/nodejs-storage/pull/387)) +- Annotate types [#3](https://github.com/googleapis/nodejs-storage/pull/3) ([#391](https://github.com/googleapis/nodejs-storage/pull/391)) +- Annotate types (2) ([#388](https://github.com/googleapis/nodejs-storage/pull/388)) + +### Dependencies +- chore(deps): update dependency eslint-plugin-prettier to v3 ([#419](https://github.com/googleapis/nodejs-storage/pull/419)) + +### Documentation +- docs: Modify source location for templates ([#410](https://github.com/googleapis/nodejs-storage/pull/410)) +- docs: Explain `Bucket#upload()` still exists ([#421](https://github.com/googleapis/nodejs-storage/pull/421)) + +### Internal / Testing Changes +- fix(tests): fix system tests on CircleCI ([#431](https://github.com/googleapis/nodejs-storage/pull/431)) +- fix(tests): system-test compiles to ./build, fix relative path ([#428](https://github.com/googleapis/nodejs-storage/pull/428)) +- Update kokoro config ([#425](https://github.com/googleapis/nodejs-storage/pull/425)) +- chore(samples): convert samples to async/await ([#422](https://github.com/googleapis/nodejs-storage/pull/422)) +- build: samples test by adding approprate test variables ([#423](https://github.com/googleapis/nodejs-storage/pull/423)) +- build: bring in latest kokoro cfgs to run System tests on PRs ([#413](https://github.com/googleapis/nodejs-storage/pull/413)) +- test: remove appveyor config ([#411](https://github.com/googleapis/nodejs-storage/pull/411)) +- Enable prefer-const in the eslint config ([#404](https://github.com/googleapis/nodejs-storage/pull/404)) +- fix(test): instantiate PubSub using new ([#403](https://github.com/googleapis/nodejs-storage/pull/403)) +- fix: optionsOrCallback could be undefined if not given, check before assign ([#401](https://github.com/googleapis/nodejs-storage/pull/401)) +- Fix the requesterPays methods ([#400](https://github.com/googleapis/nodejs-storage/pull/400)) +- Enable no-var in eslint ([#398](https://github.com/googleapis/nodejs-storage/pull/398)) +- samples: don't use USA formatted dates for expiry ([#396](https://github.com/googleapis/nodejs-storage/pull/396)) +- fix: copy(): Use correct destination file name in URI ([#389](https://github.com/googleapis/nodejs-storage/pull/389)) + +## v2.0.3 + +### Implementation Changes +- Improve TypeScript types ([#381](https://github.com/googleapis/nodejs-storage/pull/381)) +- Make some parameters optional ([#380](https://github.com/googleapis/nodejs-storage/pull/380)) + +## v2.0.2 + +### Implementation Changes +- Improve the types (#377) + +## v2.0.1 + +**This fixes types declaration issues with projects using TypeScript.** + +### Implementation Changes +- Enable noImplicitThis in the tsconfig ([#370](https://github.com/googleapis/nodejs-storage/pull/370)) +- Fix the path to the d.ts ([#364](https://github.com/googleapis/nodejs-storage/pull/364)) +- fix: make dependency on request explicit ([#361](https://github.com/googleapis/nodejs-storage/pull/361)) +- fix: remove trailing slashes from bucket name. ([#266](https://github.com/googleapis/nodejs-storage/pull/266)) + +### Dependencies +- fix(deps): update dependency @google-cloud/common to ^0.24.0 ([#367](https://github.com/googleapis/nodejs-storage/pull/367)) +- fix(deps): update dependency gcs-resumable-upload to ^0.13.0 ([#368](https://github.com/googleapis/nodejs-storage/pull/368)) +- Remove unused dependencies ([#363](https://github.com/googleapis/nodejs-storage/pull/363)) +- Remove safe-buffer ([#359](https://github.com/googleapis/nodejs-storage/pull/359)) +- samples: update dependency @google-cloud/storage to v2 ([#350](https://github.com/googleapis/nodejs-storage/pull/350)) + +### Internal / Testing Changes +- Update CI config ([#371](https://github.com/googleapis/nodejs-storage/pull/371)) +- build(kokoro): run docker as user node ([#358](https://github.com/googleapis/nodejs-storage/pull/358)) +- build: fix multiline in circle.yml ([#357](https://github.com/googleapis/nodejs-storage/pull/357)) +- fix executable modes on .sh's; add pre-system-test.sh hook ([#356](https://github.com/googleapis/nodejs-storage/pull/356)) +- decrypt both service account keys ([#353](https://github.com/googleapis/nodejs-storage/pull/353)) +- Retry npm install in CI ([#352](https://github.com/googleapis/nodejs-storage/pull/352)) +- Add synth script and run it ([#351](https://github.com/googleapis/nodejs-storage/pull/351)) + +## v2.0.0 + +**This release has breaking changes**. This release has a few notable breaking changes. Please take care when upgrading! + +### require syntax changes +The import style of this library has been changed to support [es module](https://nodejs.org/api/esm.html) syntax. This provides both forward compatibility with es modules, and better supports the TypeScript and Babel ecosystems. As a result, the import syntax has changed: + +#### Old Code +```js +const storage = require('@google-cloud/storage')(); +// or... +const Storage = require('@google-cloud/storage'); +const storage = new Storage({ + // config... +}); +``` + +#### New Code +```js +const {Storage} = require('@google-cloud/storage'); +const storage = new Storage({ + // config... +}); +``` + +### `bucket.upload` no longer accepts URLs +To better support a variety of HTTP clients, the remote fetching functionality of `bucket.upload` has been removed. It can be replaced with your favorite HTTP client. + +#### Old Code +```js +bucket.upload('https://example.com/images/image.png', function(err, file, res) { + // handle upload... +}); +``` + +#### New Code + +```js +const request = require('request'); +const file = bucket.file(name); +const writeStream = file.createWriteStream(); +request(url).pipe(writeStream); +``` + +### Breaking changes +- semver: do not support upload() from url (#337) +- fix: drop support for node.js 4.x and 9.x (#282) + +### Features +- refactor(ts): merge initial TypeScript conversion (#334) +- feat: Add Storage#getServiceAccount(). (#331) +- Kms sample (#209) + +### Bug fixes +- fix: gzip and Cache-Control headers in upload sample (#225) +- fix: move this.[ROLE]s initialization from Acl to AclAccessorRoleMethods (#252) +- fix: signedURL cname (#210) (#234) + +### Internal / Testing Changes +- chore(deps): update dependency nyc to v13 (#341) +- fix(deps): update dependency @google-cloud/common to ^0.23.0 (#340) +- test: throw on deprecation (#319) +- chore(deps): update dependency eslint-config-prettier to v3 (#336) +- fix(deps): update dependency gcs-resumable-upload to ^0.12.0 (#317) +- Fix system tests for string comparisons (#328) +- chore: ignore package-lock.json (#326) +- chore: update renovate config (#322) +- chore: regen lock files (#318) +- chore(deps): lock file maintenance (#313) +- chore: move mocha options to mocha.opts (#311) +- chore(deps): lock file maintenance (#309) +- test: use strictEqual in tests (#306) +- chore(deps): update dependency eslint-plugin-node to v7 (#305) +- chore(deps): lock file maintenance (#303) +- chore(deps): lock file maintenance (#285) +- fix: test meant to assert err msg exists (#280) +- fix(deps): update dependency yargs to v12 (#270) +- fix(deps): update dependency uuid to v3.3.2 (#269) +- chore: update gcs-resumable-upload to 0.11.1 (#265) +- fix(deps): update dependency uuid to v3.3.0 (#262) +- chore(deps): update dependency sinon to v6 (#263) +- Configure Renovate (#250) +- refactor: drop repo-tool as an exec wrapper (#258) +- chore: update sample lockfiles (#256) +- fix: update linking for samples (#254) +- chore(package): update eslint to version 5.0.0 (#253) +- refactor(es6): Refactor constructor pattern as ES6 class (#246) +- Update @google-cloud/common to the latest version 🚀 (#226) +- system-tests: fix channel test. (#243) +- refactor: Update to the latest version of nodejs-common and gcs-resumable-upload (#202) +- Fix permission of bash script for Kokoro (#223) +- chore(package): update nyc to version 12.0.2 (#216) +- chore: fix prettier incompatibility (#211) diff --git a/handwritten/storage/CODE_OF_CONDUCT.md b/handwritten/storage/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..2add2547a81 --- /dev/null +++ b/handwritten/storage/CODE_OF_CONDUCT.md @@ -0,0 +1,94 @@ + +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual's behavior may have a +negative impact on the project or its community. + +## Conflict Resolution + +We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct. + +If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe. + +Reports should be directed to *googleapis-stewards@google.com*, the +Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out to the Project Steward, please email +opensource@google.com. + +We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone's safety, we may take action without notice. + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/handwritten/storage/CONTRIBUTING.md b/handwritten/storage/CONTRIBUTING.md new file mode 100644 index 00000000000..72c44cada5e --- /dev/null +++ b/handwritten/storage/CONTRIBUTING.md @@ -0,0 +1,74 @@ +# How to become a contributor and submit your own code + +**Table of contents** + +* [Contributor License Agreements](#contributor-license-agreements) +* [Contributing a patch](#contributing-a-patch) +* [Running the tests](#running-the-tests) +* [Releasing the library](#releasing-the-library) + +## Contributor License Agreements + +We'd love to accept your sample apps and patches! Before we can take them, we +have to jump a couple of legal hurdles. + +Please fill out either the individual or corporate Contributor License Agreement +(CLA). + + * If you are an individual writing original source code and you're sure you + own the intellectual property, then you'll need to sign an [individual CLA](https://developers.google.com/open-source/cla/individual). + * If you work for a company that wants to allow you to contribute your work, + then you'll need to sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). + +Follow either of the two links above to access the appropriate CLA and +instructions for how to sign and return it. Once we receive it, we'll be able to +accept your pull requests. + +## Contributing A Patch + +1. Submit an issue describing your proposed change to the repo in question. +1. The repo owner will respond to your issue promptly. +1. If your proposed change is accepted, and you haven't already done so, sign a + Contributor License Agreement (see details above). +1. Fork the desired repo, develop and test your code changes. +1. Ensure that your code adheres to the existing style in the code to which + you are contributing. +1. Ensure that your code has an appropriate set of tests which all pass. +1. Title your pull request following [Conventional Commits](https://www.conventionalcommits.org/) styling. +1. Submit a pull request. + +### Before you begin + +1. [Select or create a Cloud Platform project][projects]. +1. [Set up authentication with a service account][auth] so you can access the + API from your local workstation. + + +## Running the tests + +1. [Prepare your environment for Node.js setup][setup]. + +1. Install dependencies: + + npm install + +1. Run the tests: + + # Run unit tests. + npm test + + # Run sample integration tests. + npm run samples-test + + # Run all system tests. + npm run system-test + +1. Lint (and maybe fix) any changes: + + npm run fix + +[setup]: https://cloud.google.com/nodejs/docs/setup +[projects]: https://console.cloud.google.com/project +[billing]: https://support.google.com/cloud/answer/6293499#enable-billing + +[auth]: https://cloud.google.com/docs/authentication/getting-started \ No newline at end of file diff --git a/handwritten/storage/LICENSE b/handwritten/storage/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/handwritten/storage/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/handwritten/storage/README.md b/handwritten/storage/README.md new file mode 100644 index 00000000000..ea2c8dc0d8a --- /dev/null +++ b/handwritten/storage/README.md @@ -0,0 +1,302 @@ +[//]: # "This README.md file is auto-generated, all changes to this file will be lost." +[//]: # "To regenerate it, use `python -m synthtool`." +Google Cloud Platform logo + +# [Google Cloud Storage: Node.js Client](https://github.com/googleapis/nodejs-storage) + +[![release level](https://img.shields.io/badge/release%20level-stable-brightgreen.svg?style=flat)](https://cloud.google.com/terms/launch-stages) +[![npm version](https://img.shields.io/npm/v/@google-cloud/storage.svg)](https://www.npmjs.com/package/@google-cloud/storage) + + + + +> Node.js idiomatic client for [Cloud Storage][product-docs]. + +[Cloud Storage](https://cloud.google.com/storage/docs) allows world-wide +storage and retrieval of any amount of data at any time. You can use Google +Cloud Storage for a range of scenarios including serving website content, +storing data for archival and disaster recovery, or distributing large data +objects to users via direct download. + + +A comprehensive list of changes in each version may be found in +[the CHANGELOG](https://github.com/googleapis/nodejs-storage/blob/main/CHANGELOG.md). + +* [Google Cloud Storage Node.js Client API Reference][client-docs] +* [Google Cloud Storage Documentation][product-docs] +* [github.com/googleapis/nodejs-storage](https://github.com/googleapis/nodejs-storage) + +Read more about the client libraries for Cloud APIs, including the older +Google APIs Client Libraries, in [Client Libraries Explained][explained]. + +[explained]: https://cloud.google.com/apis/docs/client-libraries-explained + +**Table of contents:** + + +* [Quickstart](#quickstart) + * [Before you begin](#before-you-begin) + * [Installing the client library](#installing-the-client-library) + * [Using the client library](#using-the-client-library) +* [Samples](#samples) +* [Versioning](#versioning) +* [Contributing](#contributing) +* [License](#license) + +## Quickstart + +### Before you begin + +1. [Select or create a Cloud Platform project][projects]. +1. [Enable billing for your project][billing]. +1. [Enable the Google Cloud Storage API][enable_api]. +1. [Set up authentication][auth] so you can access the + API from your local workstation. + +### Installing the client library + +```bash +npm install @google-cloud/storage +``` + + +### Using the client library + +```javascript +// Imports the Google Cloud client library +const {Storage} = require('@google-cloud/storage'); + +// For more information on ways to initialize Storage, please see +// https://googleapis.dev/nodejs/storage/latest/Storage.html + +// Creates a client using Application Default Credentials +const storage = new Storage(); + +// Creates a client from a Google service account key +// const storage = new Storage({keyFilename: 'key.json'}); + +/** + * TODO(developer): Uncomment these variables before running the sample. + */ +// The ID of your GCS bucket +// const bucketName = 'your-unique-bucket-name'; + +async function createBucket() { + // Creates the new bucket + await storage.createBucket(bucketName); + console.log(`Bucket ${bucketName} created.`); +} + +createBucket().catch(console.error); + +``` + + + +## Samples + +Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage/tree/main/samples) directory. Each sample's `README.md` has instructions for running its sample. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +| Add Bucket Conditional Binding | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/addBucketConditionalBinding.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/addBucketConditionalBinding.js,samples/README.md) | +| Add Bucket Default Owner Acl | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/addBucketDefaultOwnerAcl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/addBucketDefaultOwnerAcl.js,samples/README.md) | +| Add Bucket Iam Member | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/addBucketIamMember.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/addBucketIamMember.js,samples/README.md) | +| Storage Add Bucket Label. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/addBucketLabel.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/addBucketLabel.js,samples/README.md) | +| Add Bucket Owner Acl | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/addBucketOwnerAcl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/addBucketOwnerAcl.js,samples/README.md) | +| Bucket Website Configuration. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/addBucketWebsiteConfiguration.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/addBucketWebsiteConfiguration.js,samples/README.md) | +| Add File Owner Acl | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/addFileOwnerAcl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/addFileOwnerAcl.js,samples/README.md) | +| Storage Get Bucket Metadata. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/bucketMetadata.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/bucketMetadata.js,samples/README.md) | +| Change Bucket's Default Storage Class. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/changeDefaultStorageClass.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/changeDefaultStorageClass.js,samples/README.md) | +| Storage File Convert CSEK to CMEK. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/changeFileCSEKToCMEK.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/changeFileCSEKToCMEK.js,samples/README.md) | +| Storage Combine files. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/composeFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/composeFile.js,samples/README.md) | +| Storage Configure Bucket Cors. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/configureBucketCors.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/configureBucketCors.js,samples/README.md) | +| Configure Retries | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/configureRetries.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/configureRetries.js,samples/README.md) | +| Copy File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/copyFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/copyFile.js,samples/README.md) | +| Copy Old Version Of File. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/copyOldVersionOfFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/copyOldVersionOfFile.js,samples/README.md) | +| Create a Dual-Region Bucket | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithDualRegion.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithDualRegion.js,samples/README.md) | +| Create a hierarchical namespace enabled bucket | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithHierarchicalNamespace.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithHierarchicalNamespace.js,samples/README.md) | +| Create a Bucket with object retention enabled. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithObjectRetention.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithObjectRetention.js,samples/README.md) | +| Create Bucket With Storage Class and Location. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithStorageClassAndLocation.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithStorageClassAndLocation.js,samples/README.md) | +| Create Bucket With Turbo Replication | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithTurboReplication.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithTurboReplication.js,samples/README.md) | +| Create New Bucket | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createNewBucket.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createNewBucket.js,samples/README.md) | +| Create Notification | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createNotification.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createNotification.js,samples/README.md) | +| Delete Bucket | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/deleteBucket.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/deleteBucket.js,samples/README.md) | +| Delete File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/deleteFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/deleteFile.js,samples/README.md) | +| Delete Notification | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/deleteNotification.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/deleteNotification.js,samples/README.md) | +| Delete Old Version Of File. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/deleteOldVersionOfFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/deleteOldVersionOfFile.js,samples/README.md) | +| Disable Bucket Lifecycle Management | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/disableBucketLifecycleManagement.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/disableBucketLifecycleManagement.js,samples/README.md) | +| Storage Disable Bucket Versioning. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/disableBucketVersioning.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/disableBucketVersioning.js,samples/README.md) | +| Disable Default Event Based Hold | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/disableDefaultEventBasedHold.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/disableDefaultEventBasedHold.js,samples/README.md) | +| Disable Requester Pays | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/disableRequesterPays.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/disableRequesterPays.js,samples/README.md) | +| Disable Soft Delete | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/disableSoftDelete.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/disableSoftDelete.js,samples/README.md) | +| Disable Uniform Bucket Level Access | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/disableUniformBucketLevelAccess.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/disableUniformBucketLevelAccess.js,samples/README.md) | +| Download Byte Range | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadByteRange.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadByteRange.js,samples/README.md) | +| Download Encrypted File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadEncryptedFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadEncryptedFile.js,samples/README.md) | +| Download File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFile.js,samples/README.md) | +| Download a File in Chunks With Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFileInChunksWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFileInChunksWithTransferManager.js,samples/README.md) | +| Download File Using Requester Pays | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFileUsingRequesterPays.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFileUsingRequesterPays.js,samples/README.md) | +| Download Folder With Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadFolderWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadFolderWithTransferManager.js,samples/README.md) | +| Download Into Memory | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadIntoMemory.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadIntoMemory.js,samples/README.md) | +| Download Many Files With Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadManyFilesWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadManyFilesWithTransferManager.js,samples/README.md) | +| Storage Download Public File. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/downloadPublicFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/downloadPublicFile.js,samples/README.md) | +| Enable Bucket Lifecycle Management | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/enableBucketLifecycleManagement.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/enableBucketLifecycleManagement.js,samples/README.md) | +| Storage Enable Bucket Versioning. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/enableBucketVersioning.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/enableBucketVersioning.js,samples/README.md) | +| Enable Default Event Based Hold | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/enableDefaultEventBasedHold.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/enableDefaultEventBasedHold.js,samples/README.md) | +| Enable Default KMS Key | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/enableDefaultKMSKey.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/enableDefaultKMSKey.js,samples/README.md) | +| Enable Requester Pays | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/enableRequesterPays.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/enableRequesterPays.js,samples/README.md) | +| Enable Uniform Bucket Level Access | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/enableUniformBucketLevelAccess.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/enableUniformBucketLevelAccess.js,samples/README.md) | +| Change File's Storage Class. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/fileChangeStorageClass.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/fileChangeStorageClass.js,samples/README.md) | +| Storage Set File Metadata. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/fileSetMetadata.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/fileSetMetadata.js,samples/README.md) | +| Generate Encryption Key | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/generateEncryptionKey.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/generateEncryptionKey.js,samples/README.md) | +| Generate Signed Url | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/generateSignedUrl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/generateSignedUrl.js,samples/README.md) | +| Generate V4 Read Signed Url | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/generateV4ReadSignedUrl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/generateV4ReadSignedUrl.js,samples/README.md) | +| Generate V4 Signed Policy | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/generateV4SignedPolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/generateV4SignedPolicy.js,samples/README.md) | +| Generate V4 Upload Signed Url | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/generateV4UploadSignedUrl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/generateV4UploadSignedUrl.js,samples/README.md) | +| Get Autoclass | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getAutoclass.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getAutoclass.js,samples/README.md) | +| Get Default Event Based Hold | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getDefaultEventBasedHold.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getDefaultEventBasedHold.js,samples/README.md) | +| Get Metadata | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getMetadata.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getMetadata.js,samples/README.md) | +| Get Metadata Notifications | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getMetadataNotifications.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getMetadataNotifications.js,samples/README.md) | +| Get Public Access Prevention | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getPublicAccessPrevention.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getPublicAccessPrevention.js,samples/README.md) | +| Get RPO | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getRPO.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getRPO.js,samples/README.md) | +| Get Requester Pays Status | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getRequesterPaysStatus.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getRequesterPaysStatus.js,samples/README.md) | +| Get Retention Policy | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getRetentionPolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getRetentionPolicy.js,samples/README.md) | +| Storage Get Service Account. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getServiceAccount.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getServiceAccount.js,samples/README.md) | +| Get Soft Delete Policy | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getSoftDeletePolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getSoftDeletePolicy.js,samples/README.md) | +| Get Soft Deleted Bucket | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getSoftDeletedBucket.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getSoftDeletedBucket.js,samples/README.md) | +| Get Uniform Bucket Level Access | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/getUniformBucketLevelAccess.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/getUniformBucketLevelAccess.js,samples/README.md) | +| Activate HMAC SA Key. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/hmacKeyActivate.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/hmacKeyActivate.js,samples/README.md) | +| Create HMAC SA Key. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/hmacKeyCreate.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/hmacKeyCreate.js,samples/README.md) | +| Deactivate HMAC SA Key. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/hmacKeyDeactivate.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/hmacKeyDeactivate.js,samples/README.md) | +| Delete HMAC SA Key. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/hmacKeyDelete.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/hmacKeyDelete.js,samples/README.md) | +| Get HMAC SA Key Metadata. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/hmacKeyGet.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/hmacKeyGet.js,samples/README.md) | +| List HMAC SA Keys Metadata. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/hmacKeysList.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/hmacKeysList.js,samples/README.md) | +| List Buckets | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listBuckets.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listBuckets.js,samples/README.md) | +| List Buckets Partial Success | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listBucketsPartialSuccess.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listBucketsPartialSuccess.js,samples/README.md) | +| List Files | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listFiles.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listFiles.js,samples/README.md) | +| List Files By Prefix | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listFilesByPrefix.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listFilesByPrefix.js,samples/README.md) | +| List Files Paginate | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listFilesPaginate.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listFilesPaginate.js,samples/README.md) | +| List Files with Old Versions. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listFilesWithOldVersions.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listFilesWithOldVersions.js,samples/README.md) | +| List Notifications | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listNotifications.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listNotifications.js,samples/README.md) | +| List Soft Deleted Bucket | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listSoftDeletedBucket.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listSoftDeletedBucket.js,samples/README.md) | +| List Soft Deleted Object Versions | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listSoftDeletedObjectVersions.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listSoftDeletedObjectVersions.js,samples/README.md) | +| List Soft Deleted Objects | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/listSoftDeletedObjects.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/listSoftDeletedObjects.js,samples/README.md) | +| Lock Retention Policy | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/lockRetentionPolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/lockRetentionPolicy.js,samples/README.md) | +| Storage Make Bucket Public. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/makeBucketPublic.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/makeBucketPublic.js,samples/README.md) | +| Make Public | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/makePublic.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/makePublic.js,samples/README.md) | +| Move File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/moveFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/moveFile.js,samples/README.md) | +| Move File Atomic | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/moveFileAtomic.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/moveFileAtomic.js,samples/README.md) | +| Print Bucket Acl | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/printBucketAcl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/printBucketAcl.js,samples/README.md) | +| Print Bucket Acl For User | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/printBucketAclForUser.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/printBucketAclForUser.js,samples/README.md) | +| Print File Acl | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/printFileAcl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/printFileAcl.js,samples/README.md) | +| Print File Acl For User | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/printFileAclForUser.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/printFileAclForUser.js,samples/README.md) | +| Quickstart | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/quickstart.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/quickstart.js,samples/README.md) | +| Release Event Based Hold | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/releaseEventBasedHold.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/releaseEventBasedHold.js,samples/README.md) | +| Release Temporary Hold | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/releaseTemporaryHold.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/releaseTemporaryHold.js,samples/README.md) | +| Remove Bucket Conditional Binding | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/removeBucketConditionalBinding.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/removeBucketConditionalBinding.js,samples/README.md) | +| Storage Remove Bucket Cors Configuration. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/removeBucketCors.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/removeBucketCors.js,samples/README.md) | +| Remove Bucket Default Owner | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/removeBucketDefaultOwner.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/removeBucketDefaultOwner.js,samples/README.md) | +| Remove Bucket Iam Member | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/removeBucketIamMember.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/removeBucketIamMember.js,samples/README.md) | +| Storage Remove Bucket Label. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/removeBucketLabel.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/removeBucketLabel.js,samples/README.md) | +| Remove Bucket Owner Acl | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/removeBucketOwnerAcl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/removeBucketOwnerAcl.js,samples/README.md) | +| Remove Default KMS Key. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/removeDefaultKMSKey.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/removeDefaultKMSKey.js,samples/README.md) | +| Remove File Owner Acl | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/removeFileOwnerAcl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/removeFileOwnerAcl.js,samples/README.md) | +| Remove Retention Policy | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/removeRetentionPolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/removeRetentionPolicy.js,samples/README.md) | +| Rename File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/renameFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/renameFile.js,samples/README.md) | +| Restore Soft Deleted Bucket | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/restoreSoftDeletedBucket.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/restoreSoftDeletedBucket.js,samples/README.md) | +| Restore Soft Deleted Object | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/restoreSoftDeletedObject.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/restoreSoftDeletedObject.js,samples/README.md) | +| Rotate Encryption Key | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/rotateEncryptionKey.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/rotateEncryptionKey.js,samples/README.md) | +| Set Autoclass | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setAutoclass.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setAutoclass.js,samples/README.md) | +| Set Client Endpoint | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setClientEndpoint.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setClientEndpoint.js,samples/README.md) | +| Set Event Based Hold | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setEventBasedHold.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setEventBasedHold.js,samples/README.md) | +| Set the object retention policy of a File. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setObjectRetentionPolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setObjectRetentionPolicy.js,samples/README.md) | +| Set Public Access Prevention Enforced | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setPublicAccessPreventionEnforced.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setPublicAccessPreventionEnforced.js,samples/README.md) | +| Set Public Access Prevention Inherited | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setPublicAccessPreventionInherited.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setPublicAccessPreventionInherited.js,samples/README.md) | +| Set RPO Async Turbo | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setRPOAsyncTurbo.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setRPOAsyncTurbo.js,samples/README.md) | +| Set RPO Default | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setRPODefault.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setRPODefault.js,samples/README.md) | +| Set Retention Policy | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setRetentionPolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setRetentionPolicy.js,samples/README.md) | +| Set Soft Delete Policy | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setSoftDeletePolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setSoftDeletePolicy.js,samples/README.md) | +| Set Temporary Hold | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setTemporaryHold.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setTemporaryHold.js,samples/README.md) | +| Stream File Download | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/streamFileDownload.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/streamFileDownload.js,samples/README.md) | +| Stream File Upload | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/streamFileUpload.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/streamFileUpload.js,samples/README.md) | +| Upload a directory to a bucket. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadDirectory.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadDirectory.js,samples/README.md) | +| Upload Directory With Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadDirectoryWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadDirectoryWithTransferManager.js,samples/README.md) | +| Upload Encrypted File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadEncryptedFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadEncryptedFile.js,samples/README.md) | +| Upload File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFile.js,samples/README.md) | +| Upload a File in Chunks With Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFileInChunksWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFileInChunksWithTransferManager.js,samples/README.md) | +| Upload File With Kms Key | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFileWithKmsKey.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFileWithKmsKey.js,samples/README.md) | +| Upload From Memory | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadFromMemory.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadFromMemory.js,samples/README.md) | +| Upload Many Files With Transfer Manager | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadManyFilesWithTransferManager.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadManyFilesWithTransferManager.js,samples/README.md) | +| Upload Without Authentication | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadWithoutAuthentication.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadWithoutAuthentication.js,samples/README.md) | +| Upload Without Authentication Signed Url | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/uploadWithoutAuthenticationSignedUrl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/uploadWithoutAuthenticationSignedUrl.js,samples/README.md) | +| View Bucket Iam Members | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/viewBucketIamMembers.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/viewBucketIamMembers.js,samples/README.md) | + + + +The [Google Cloud Storage Node.js Client API Reference][client-docs] documentation +also contains samples. + +## Supported Node.js Versions + +Our client libraries follow the [Node.js release schedule](https://github.com/nodejs/release#release-schedule). +Libraries are compatible with all current _active_ and _maintenance_ versions of +Node.js. +If you are using an end-of-life version of Node.js, we recommend that you update +as soon as possible to an actively supported LTS version. + +Google's client libraries support legacy versions of Node.js runtimes on a +best-efforts basis with the following warnings: + +* Legacy versions are not tested in continuous integration. +* Some security patches and features cannot be backported. +* Dependencies cannot be kept up-to-date. + +Client libraries targeting some end-of-life versions of Node.js are available, and +can be installed through npm [dist-tags](https://docs.npmjs.com/cli/dist-tag). +The dist-tags follow the naming convention `legacy-(version)`. +For example, `npm install @google-cloud/storage@legacy-8` installs client libraries +for versions compatible with Node.js 8. + +## Versioning + +This library follows [Semantic Versioning](http://semver.org/). + + + +This library is considered to be **stable**. The code surface will not change in backwards-incompatible ways +unless absolutely necessary (e.g. because of critical security issues) or with +an extensive deprecation period. Issues and requests against **stable** libraries +are addressed with the highest priority. + + + + + + +More Information: [Google Cloud Platform Launch Stages][launch_stages] + +[launch_stages]: https://cloud.google.com/terms/launch-stages + +## Contributing + +Contributions welcome! See the [Contributing Guide](https://github.com/googleapis/nodejs-storage/blob/main/CONTRIBUTING.md). + +Please note that this `README.md`, the `samples/README.md`, +and a variety of configuration files in this repository (including `.nycrc` and `tsconfig.json`) +are generated from a central template. To edit one of these files, make an edit +to its templates in +[directory](https://github.com/googleapis/synthtool). + +## License + +Apache Version 2.0 + +See [LICENSE](https://github.com/googleapis/nodejs-storage/blob/main/LICENSE) + +[client-docs]: https://cloud.google.com/nodejs/docs/reference/storage/latest +[product-docs]: https://cloud.google.com/storage +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png +[projects]: https://console.cloud.google.com/project +[billing]: https://support.google.com/cloud/answer/6293499#enable-billing +[enable_api]: https://console.cloud.google.com/flows/enableapi?apiid=storage-api.googleapis.com +[auth]: https://cloud.google.com/docs/authentication/external/set-up-adc-local diff --git a/handwritten/storage/conformance-test/conformanceCommon.ts b/handwritten/storage/conformance-test/conformanceCommon.ts new file mode 100644 index 00000000000..65da9293811 --- /dev/null +++ b/handwritten/storage/conformance-test/conformanceCommon.ts @@ -0,0 +1,235 @@ +/*! + * Copyright 2021 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as jsonToNodeApiMapping from './test-data/retryInvocationMap.json'; +import * as libraryMethods from './libraryMethods'; +import {Bucket, File, HmacKey, Notification, Storage} from '../src/'; +import * as uuid from 'uuid'; +import * as assert from 'assert'; +import {DecorateRequestOptions} from '../src/nodejs-common'; +import fetch from 'node-fetch'; + +interface RetryCase { + instructions: String[]; +} + +interface Method { + name: String; + resources: String[]; + group?: String; +} + +export interface RetryTestCase { + id: number; + description: String; + cases: RetryCase[]; + methods: Method[]; + preconditionProvided: boolean; + expectSuccess: boolean; +} + +interface ConformanceTestCreationResult { + id: string; +} + +interface ConformanceTestResult { + completed: boolean; +} + +type LibraryMethodsModuleType = typeof import('./libraryMethods'); +const methodMap: Map = new Map( + Object.entries(jsonToNodeApiMapping) +); + +const DURATION_SECONDS = 600; // 10 mins. +const TESTS_PREFIX = `storage.retry.tests.${shortUUID()}.`; +const TESTBENCH_HOST = + process.env.STORAGE_EMULATOR_HOST || 'http://localhost:9000/'; +const CONF_TEST_PROJECT_ID = 'my-project-id'; +const TIMEOUT_FOR_INDIVIDUAL_TEST = 20000; +const RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS = 0.01; + +export function executeScenario(testCase: RetryTestCase) { + for ( + let instructionNumber = 0; + instructionNumber < testCase.cases.length; + instructionNumber++ + ) { + const instructionSet: RetryCase = testCase.cases[instructionNumber]; + testCase.methods.forEach(async jsonMethod => { + const functionList = + jsonMethod?.group !== undefined + ? methodMap.get(jsonMethod?.group) + : methodMap.get(jsonMethod?.name); + functionList?.forEach(storageMethodString => { + const storageMethodObject = + libraryMethods[storageMethodString as keyof LibraryMethodsModuleType]; + let bucket: Bucket; + let file: File; + let notification: Notification; + let creationResult: {id: string}; + let storage: Storage; + let hmacKey: HmacKey; + + describe(`${storageMethodString}`, async () => { + beforeEach(async () => { + storage = new Storage({ + apiEndpoint: TESTBENCH_HOST, + projectId: CONF_TEST_PROJECT_ID, + retryOptions: { + retryDelayMultiplier: RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS, + }, + }); + creationResult = await createTestBenchRetryTest( + instructionSet.instructions, + jsonMethod?.name.toString() + ); + if (storageMethodString.includes('InstancePrecondition')) { + bucket = await createBucketForTest( + storage, + testCase.preconditionProvided, + storageMethodString + ); + file = await createFileForTest( + testCase.preconditionProvided, + storageMethodString, + bucket + ); + } else { + bucket = await createBucketForTest( + storage, + false, + storageMethodString + ); + file = await createFileForTest( + false, + storageMethodString, + bucket + ); + } + notification = bucket.notification(`${TESTS_PREFIX}`); + await notification.create(); + + [hmacKey] = await storage.createHmacKey( + `${TESTS_PREFIX}@email.com` + ); + + storage.interceptors.push({ + request: requestConfig => { + requestConfig.headers = requestConfig.headers || {}; + Object.assign(requestConfig.headers, { + 'x-retry-test-id': creationResult.id, + }); + return requestConfig as DecorateRequestOptions; + }, + }); + }); + + it(`${instructionNumber}`, async () => { + const methodParameters: libraryMethods.ConformanceTestOptions = { + bucket: bucket, + file: file, + notification: notification, + storage: storage, + hmacKey: hmacKey, + }; + if (testCase.preconditionProvided) { + methodParameters.preconditionRequired = true; + } + if (testCase.expectSuccess) { + assert.ifError(await storageMethodObject(methodParameters)); + } else { + await assert.rejects(storageMethodObject(methodParameters)); + } + const testBenchResult = await getTestBenchRetryTest( + creationResult.id + ); + assert.strictEqual(testBenchResult.completed, true); + }).timeout(TIMEOUT_FOR_INDIVIDUAL_TEST); + }); + }); + }); + } +} + +async function createBucketForTest( + storage: Storage, + preconditionShouldBeOnInstance: boolean, + storageMethodString: String +) { + const name = generateName(storageMethodString, 'bucket'); + const bucket = storage.bucket(name); + await bucket.create(); + await bucket.setRetentionPeriod(DURATION_SECONDS); + + if (preconditionShouldBeOnInstance) { + return new Bucket(storage, bucket.name, { + preconditionOpts: { + ifMetagenerationMatch: 2, + }, + }); + } + return bucket; +} + +async function createFileForTest( + preconditionShouldBeOnInstance: boolean, + storageMethodString: String, + bucket: Bucket +) { + const name = generateName(storageMethodString, 'file'); + const file = bucket.file(name); + await file.save(name); + if (preconditionShouldBeOnInstance) { + return new File(bucket, file.name, { + preconditionOpts: { + ifMetagenerationMatch: file.metadata.metageneration, + ifGenerationMatch: file.metadata.generation, + }, + }); + } + return file; +} + +function generateName(storageMethodString: String, bucketOrFile: string) { + return `${TESTS_PREFIX}${storageMethodString.toLowerCase()}${bucketOrFile}.${shortUUID()}`; +} + +async function createTestBenchRetryTest( + instructions: String[], + methodName: string +): Promise { + const requestBody = {instructions: {[methodName]: instructions}}; + const response = await fetch(`${TESTBENCH_HOST}retry_test`, { + method: 'POST', + body: JSON.stringify(requestBody), + headers: {'Content-Type': 'application/json'}, + }); + return response.json() as Promise; +} + +async function getTestBenchRetryTest( + testId: string +): Promise { + const response = await fetch(`${TESTBENCH_HOST}retry_test/${testId}`, { + method: 'GET', + }); + + return response.json() as Promise; +} + +function shortUUID() { + return uuid.v1().split('-').shift(); +} diff --git a/handwritten/storage/conformance-test/fixtures/signing-service-account.json b/handwritten/storage/conformance-test/fixtures/signing-service-account.json new file mode 100644 index 00000000000..5fdc01240ef --- /dev/null +++ b/handwritten/storage/conformance-test/fixtures/signing-service-account.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "dummy-project-id", + "private_key_id": "ffffffffffffffffffffffffffffffffffffffff", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsPzMirIottfQ2\nryjQmPWocSEeGo7f7Q4/tMQXHlXFzo93AGgU2t+clEj9L5loNhLVq+vk+qmnyDz5\nQ04y8jVWyMYzzGNNrGRW/yaYqnqlKZCy1O3bmnNjV7EDbC/jE1ZLBY0U3HaSHfn6\nS9ND8MXdgD0/ulRTWwq6vU8/w6i5tYsU7n2LLlQTl1fQ7/emO9nYcCFJezHZVa0H\nmeWsdHwWsok0skwQYQNIzP3JF9BpR5gJT2gNge6KopDesJeLoLzaX7cUnDn+CAnn\nLuLDwwSsIVKyVxhBFsFXPplgpaQRwmGzwEbf/Xpt9qo26w2UMgn30jsOaKlSeAX8\ncS6ViF+tAgMBAAECggEACKRuJCP8leEOhQziUx8Nmls8wmYqO4WJJLyk5xUMUC22\nSI4CauN1e0V8aQmxnIc0CDkFT7qc9xBmsMoF+yvobbeKrFApvlyzNyM7tEa/exh8\nDGD/IzjbZ8VfWhDcUTwn5QE9DCoon9m1sG+MBNlokB3OVOt8LieAAREdEBG43kJu\nyQTOkY9BGR2AY1FnAl2VZ/jhNDyrme3tp1sW1BJrawzR7Ujo8DzlVcS2geKA9at7\n55ua5GbHz3hfzFgjVXDfnkWzId6aHypUyqHrSn1SqGEbyXTaleKTc6Pgv0PgkJjG\nhZazWWdSuf1T5Xbs0OhAK9qraoAzT6cXXvMEvvPt6QKBgQDXcZKqJAOnGEU4b9+v\nOdoh+nssdrIOBNMu1m8mYbUVYS1aakc1iDGIIWNM3qAwbG+yNEIi2xi80a2RMw2T\n9RyCNB7yqCXXVKLBiwg9FbKMai6Vpk2bWIrzahM9on7AhCax/X2AeOp+UyYhFEy6\nUFG4aHb8THscL7b515ukSuKb5QKBgQDMq+9PuaB0eHsrmL6q4vHNi3MLgijGg/zu\nAXaPygSYAwYW8KglcuLZPvWrL6OG0+CrfmaWTLsyIZO4Uhdj7MLvX6yK7IMnagvk\nL3xjgxSklEHJAwi5wFeJ8ai/1MIuCn8p2re3CbwISKpvf7Sgs/W4196P4vKvTiAz\njcTiSYFIKQKBgCjMpkS4O0TakMlGTmsFnqyOneLmu4NyIHgfPb9cA4n/9DHKLKAT\noaWxBPgatOVWs7RgtyGYsk+XubHkpC6f3X0+15mGhFwJ+CSE6tN+l2iF9zp52vqP\nQwkjzm7+pdhZbmaIpcq9m1K+9lqPWJRz/3XXuqi+5xWIZ7NaxGvRjqaNAoGAdK2b\nutZ2y48XoI3uPFsuP+A8kJX+CtWZrlE1NtmS7tnicdd19AtfmTuUL6fz0FwfW4Su\nlQZfPT/5B339CaEiq/Xd1kDor+J7rvUHM2+5p+1A54gMRGCLRv92FQ4EON0RC1o9\nm2I4SHysdO3XmjmdXmfp4BsgAKJIJzutvtbqlakCgYB+Cb10z37NJJ+WgjDt+yT2\nyUNH17EAYgWXryfRgTyi2POHuJitd64Xzuy6oBVs3wVveYFM6PIKXlj8/DahYX5I\nR2WIzoCNLL3bEZ+nC6Jofpb4kspoAeRporj29SgesK6QBYWHWX2H645RkRGYGpDo\n51gjy9m/hSNqBbH2zmh04A==\n-----END PRIVATE KEY-----\n", + "client_email": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com", + "client_id": "000000000000000000000", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/handwritten/storage/conformance-test/globalHooks.ts b/handwritten/storage/conformance-test/globalHooks.ts new file mode 100644 index 00000000000..0775b74578e --- /dev/null +++ b/handwritten/storage/conformance-test/globalHooks.ts @@ -0,0 +1,41 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + getTestBenchDockerImage, + runTestBenchDockerImage, + stopTestBenchDockerImage, +} from './testBenchUtil'; + +const TIMEOUT_FOR_DOCKER_OPS = 60000; +const TIME_TO_WAIT_FOR_CONTAINER_READY = 10000; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function mochaGlobalSetup(this: any) { + // Increase the timeout for this before block so that the docker images have time to download and run. + this.suite._timeout = TIMEOUT_FOR_DOCKER_OPS; + await getTestBenchDockerImage(); + await runTestBenchDockerImage(); + await new Promise(resolve => + setTimeout(resolve, TIME_TO_WAIT_FOR_CONTAINER_READY) + ); +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function mochaGlobalTeardown(this: any) { + // Increase the timeout for this block so that docker has time to stop the container. + this.suite._timeout = TIMEOUT_FOR_DOCKER_OPS; + await stopTestBenchDockerImage(); +} diff --git a/handwritten/storage/conformance-test/libraryMethods.ts b/handwritten/storage/conformance-test/libraryMethods.ts new file mode 100644 index 00000000000..2dd2e586beb --- /dev/null +++ b/handwritten/storage/conformance-test/libraryMethods.ts @@ -0,0 +1,836 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {Bucket, File, Notification, Storage, HmacKey, Policy} from '../src'; +import * as path from 'path'; +import {ApiError} from '../src/nodejs-common'; +import { + createTestBuffer, + createTestFileFromBuffer, + deleteTestFile, +} from './testBenchUtil'; +import * as uuid from 'uuid'; +import {getDirName} from '../src/util.js'; + +const FILE_SIZE_BYTES = 9 * 1024 * 1024; +const CHUNK_SIZE_BYTES = 2 * 1024 * 1024; + +export interface ConformanceTestOptions { + bucket?: Bucket; + file?: File; + notification?: Notification; + storage?: Storage; + hmacKey?: HmacKey; + preconditionRequired?: boolean; +} + +///////////////////////////////////////////////// +//////////////////// BUCKET ///////////////////// +///////////////////////////////////////////////// + +export async function addLifecycleRuleInstancePrecondition( + options: ConformanceTestOptions +) { + await options.bucket!.addLifecycleRule({ + action: { + type: 'Delete', + }, + condition: { + age: 365 * 3, // Specified in days. + }, + }); +} + +export async function addLifecycleRule(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.bucket!.addLifecycleRule( + { + action: { + type: 'Delete', + }, + condition: { + age: 365 * 3, // Specified in days. + }, + }, + { + ifMetagenerationMatch: 2, + } + ); + } else { + await options.bucket!.addLifecycleRule({ + action: { + type: 'Delete', + }, + condition: { + age: 365 * 3, // Specified in days. + }, + }); + } +} + +export async function combineInstancePrecondition( + options: ConformanceTestOptions +) { + const file1 = options.bucket!.file('file1.txt'); + const file2 = options.bucket!.file('file2.txt'); + await file1.save('file1 contents'); + await file2.save('file2 contents'); + let allFiles; + const sources = [file1, file2]; + if (options.preconditionRequired) { + allFiles = options.bucket!.file('all-files.txt', { + preconditionOpts: { + ifGenerationMatch: 0, + }, + }); + } else { + allFiles = options.bucket!.file('all-files.txt'); + } + + await options.bucket!.combine(sources, allFiles); +} + +export async function combine(options: ConformanceTestOptions) { + const file1 = options.bucket!.file('file1.txt'); + const file2 = options.bucket!.file('file2.txt'); + await file1.save('file1 contents'); + await file2.save('file2 contents'); + const sources = [file1, file2]; + const allFiles = options.bucket!.file('all-files.txt'); + await allFiles.save('allfiles contents'); + if (options.preconditionRequired) { + await options.bucket!.combine(sources, allFiles, { + ifGenerationMatch: allFiles.metadata.generation!, + }); + } else { + await options.bucket!.combine(sources, allFiles); + } +} + +export async function create(options: ConformanceTestOptions) { + const [bucketExists] = await options.bucket!.exists(); + if (bucketExists) { + await options.bucket!.deleteFiles(); + await options.bucket!.delete({ + ignoreNotFound: true, + }); + } + await options.bucket!.create(); +} + +export async function createNotification(options: ConformanceTestOptions) { + await options.bucket!.createNotification('my-topic'); +} + +export async function deleteBucket(options: ConformanceTestOptions) { + await options.bucket!.deleteFiles(); + await options.bucket!.delete(); +} + +// Note: bucket.deleteFiles is missing from these tests +// Preconditions cannot be implemented with current setup. + +export async function deleteLabelsInstancePrecondition( + options: ConformanceTestOptions +) { + await options.bucket!.deleteLabels(); +} + +export async function deleteLabels(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.bucket!.deleteLabels({ + ifMetagenerationMatch: 2, + }); + } else { + await options.bucket!.deleteLabels(); + } +} + +export async function disableRequesterPaysInstancePrecondition( + options: ConformanceTestOptions +) { + await options.bucket!.disableRequesterPays(); +} + +export async function disableRequesterPays(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.bucket!.disableRequesterPays({ + ifMetagenerationMatch: 2, + }); + } else { + await options.bucket!.disableRequesterPays(); + } +} + +export async function enableLoggingInstancePrecondition( + options: ConformanceTestOptions +) { + const config = { + prefix: 'log', + }; + await options.bucket!.enableLogging(config); +} + +export async function enableLogging(options: ConformanceTestOptions) { + let config; + if (options.preconditionRequired) { + config = { + prefix: 'log', + ifMetagenerationMatch: 2, + }; + } else { + config = { + prefix: 'log', + }; + } + await options.bucket!.enableLogging(config); +} + +export async function enableRequesterPaysInstancePrecondition( + options: ConformanceTestOptions +) { + await options.bucket!.enableRequesterPays(); +} + +export async function enableRequesterPays(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.bucket!.enableRequesterPays({ + ifMetagenerationMatch: 2, + }); + } else { + await options.bucket!.enableRequesterPays(); + } +} + +export async function bucketExists(options: ConformanceTestOptions) { + await options.bucket!.exists(); +} + +export async function bucketGet(options: ConformanceTestOptions) { + await options.bucket!.get(); +} + +export async function getFilesStream(options: ConformanceTestOptions) { + return new Promise((resolve, reject) => { + options + .bucket!.getFilesStream() + .on('data', () => {}) + .on('end', () => resolve(undefined)) + .on('error', (err: ApiError) => reject(err)); + }); +} + +export async function getLabels(options: ConformanceTestOptions) { + await options.bucket!.getLabels(); +} + +export async function bucketGetMetadata(options: ConformanceTestOptions) { + await options.bucket!.getMetadata(); +} + +export async function getNotifications(options: ConformanceTestOptions) { + await options.bucket!.getNotifications(); +} + +export async function lock(options: ConformanceTestOptions) { + const metageneration = 0; + await options.bucket!.lock(metageneration); +} + +export async function bucketMakePrivateInstancePrecondition( + options: ConformanceTestOptions +) { + await options.bucket!.makePrivate(); +} + +export async function bucketMakePrivate(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.bucket!.makePrivate({ + preconditionOpts: {ifMetagenerationMatch: 2}, + }); + } else { + await options.bucket!.makePrivate(); + } +} + +export async function bucketMakePublic(options: ConformanceTestOptions) { + await options.bucket!.makePublic(); +} + +export async function removeRetentionPeriodInstancePrecondition( + options: ConformanceTestOptions +) { + await options.bucket!.removeRetentionPeriod(); +} + +export async function removeRetentionPeriod(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.bucket!.removeRetentionPeriod({ + ifMetagenerationMatch: 2, + }); + } else { + await options.bucket!.removeRetentionPeriod(); + } +} + +export async function setCorsConfigurationInstancePrecondition( + options: ConformanceTestOptions +) { + const corsConfiguration = [{maxAgeSeconds: 3600}]; // 1 hour + await options.bucket!.setCorsConfiguration(corsConfiguration); +} + +export async function setCorsConfiguration(options: ConformanceTestOptions) { + const corsConfiguration = [{maxAgeSeconds: 3600}]; // 1 hour + if (options.preconditionRequired) { + await options.bucket!.setCorsConfiguration(corsConfiguration, { + ifMetagenerationMatch: 2, + }); + } else { + await options.bucket!.setCorsConfiguration(corsConfiguration); + } +} + +export async function setLabelsInstancePrecondition( + options: ConformanceTestOptions +) { + const labels = { + labelone: 'labelonevalue', + labeltwo: 'labeltwovalue', + }; + await options.bucket!.setLabels(labels); +} + +export async function setLabels(options: ConformanceTestOptions) { + const labels = { + labelone: 'labelonevalue', + labeltwo: 'labeltwovalue', + }; + if (options.preconditionRequired) { + await options.bucket!.setLabels(labels, { + ifMetagenerationMatch: 2, + }); + } else { + await options.bucket!.setLabels(labels); + } +} + +export async function bucketSetMetadataInstancePrecondition( + options: ConformanceTestOptions +) { + const metadata = { + website: { + mainPageSuffix: 'http://example.com', + notFoundPage: 'http://example.com/404.html', + }, + }; + await options.bucket!.setMetadata(metadata); +} + +export async function bucketSetMetadata(options: ConformanceTestOptions) { + const metadata = { + website: { + mainPageSuffix: 'http://example.com', + notFoundPage: 'http://example.com/404.html', + }, + }; + if (options.preconditionRequired) { + await options.bucket!.setMetadata(metadata, { + ifMetagenerationMatch: 2, + }); + } else { + await options.bucket!.setMetadata(metadata); + } +} + +export async function setRetentionPeriodInstancePrecondition( + options: ConformanceTestOptions +) { + const DURATION_SECONDS = 15780000; // 6 months. + await options.bucket!.setRetentionPeriod(DURATION_SECONDS); +} + +export async function setRetentionPeriod(options: ConformanceTestOptions) { + const DURATION_SECONDS = 15780000; // 6 months. + if (options.preconditionRequired) { + await options.bucket!.setRetentionPeriod(DURATION_SECONDS, { + ifMetagenerationMatch: 2, + }); + } else { + await options.bucket!.setRetentionPeriod(DURATION_SECONDS); + } +} + +export async function bucketSetStorageClassInstancePrecondition( + options: ConformanceTestOptions +) { + await options.bucket!.setStorageClass('nearline'); +} + +export async function bucketSetStorageClass(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.bucket!.setStorageClass('nearline', { + ifMetagenerationMatch: 2, + }); + } else { + await options.bucket!.setStorageClass('nearline'); + } +} + +export async function bucketUploadResumableInstancePrecondition( + options: ConformanceTestOptions +) { + const filePath = path.join( + getDirName(), + `../conformance-test/test-data/tmp-${uuid.v4()}.txt` + ); + createTestFileFromBuffer(FILE_SIZE_BYTES, filePath); + if (options.bucket!.instancePreconditionOpts) { + options.bucket!.instancePreconditionOpts.ifGenerationMatch = 0; + delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch; + } + await options.bucket!.upload(filePath, { + resumable: true, + chunkSize: CHUNK_SIZE_BYTES, + metadata: {contentLength: FILE_SIZE_BYTES}, + }); + deleteTestFile(filePath); +} + +export async function bucketUploadResumable(options: ConformanceTestOptions) { + const filePath = path.join( + getDirName(), + `../conformance-test/test-data/tmp-${uuid.v4()}.txt` + ); + createTestFileFromBuffer(FILE_SIZE_BYTES, filePath); + if (options.preconditionRequired) { + await options.bucket!.upload(filePath, { + resumable: true, + chunkSize: CHUNK_SIZE_BYTES, + metadata: {contentLength: FILE_SIZE_BYTES}, + preconditionOpts: {ifGenerationMatch: 0}, + }); + } else { + await options.bucket!.upload(filePath, { + resumable: true, + chunkSize: CHUNK_SIZE_BYTES, + metadata: {contentLength: FILE_SIZE_BYTES}, + }); + } + deleteTestFile(filePath); +} + +export async function bucketUploadMultipartInstancePrecondition( + options: ConformanceTestOptions +) { + if (options.bucket!.instancePreconditionOpts) { + delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch; + options.bucket!.instancePreconditionOpts.ifGenerationMatch = 0; + } + await options.bucket!.upload( + path.join( + getDirName(), + '../../../conformance-test/test-data/retryStrategyTestData.json' + ), + {resumable: false} + ); +} + +export async function bucketUploadMultipart(options: ConformanceTestOptions) { + if (options.bucket!.instancePreconditionOpts) { + delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch; + } + + if (options.preconditionRequired) { + await options.bucket!.upload( + path.join( + getDirName(), + '../../../conformance-test/test-data/retryStrategyTestData.json' + ), + {resumable: false, preconditionOpts: {ifGenerationMatch: 0}} + ); + } else { + await options.bucket!.upload( + path.join( + getDirName(), + '../../../conformance-test/test-data/retryStrategyTestData.json' + ), + {resumable: false} + ); + } +} + +///////////////////////////////////////////////// +//////////////////// FILE ///////////////////// +///////////////////////////////////////////////// + +export async function copy(options: ConformanceTestOptions) { + const newFile = new File(options.bucket!, 'a-different-file.png'); + await newFile.save('a-different-file.png'); + + if (options.preconditionRequired) { + await options.file!.copy('a-different-file.png', { + preconditionOpts: { + ifGenerationMatch: newFile.metadata.generation!, + }, + }); + } else { + await options.file!.copy('a-different-file.png'); + } +} + +export async function createReadStream(options: ConformanceTestOptions) { + return new Promise((resolve, reject) => { + options + .file!.createReadStream() + .on('data', () => {}) + .on('end', () => resolve(undefined)) + .on('error', (err: ApiError) => reject(err)); + }); +} + +export async function createResumableUploadInstancePrecondition( + options: ConformanceTestOptions +) { + await options.file!.createResumableUpload(); +} + +export async function createResumableUpload(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.file!.createResumableUpload({ + preconditionOpts: {ifGenerationMatch: 0}, + }); + } else { + await options.file!.createResumableUpload(); + } +} + +export async function fileDeleteInstancePrecondition( + options: ConformanceTestOptions +) { + await options.file!.delete(); +} + +export async function fileDelete(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.file!.delete({ + ifGenerationMatch: options.file!.metadata.generation, + }); + } else { + await options.file!.delete(); + } +} + +export async function download(options: ConformanceTestOptions) { + await options.file!.download(); +} + +export async function exists(options: ConformanceTestOptions) { + await options.file!.exists(); +} + +export async function get(options: ConformanceTestOptions) { + await options.file!.get(); +} + +export async function getExpirationDate(options: ConformanceTestOptions) { + await options.file!.getExpirationDate(); +} + +export async function getMetadata(options: ConformanceTestOptions) { + await options.file!.getMetadata(); +} + +export async function isPublic(options: ConformanceTestOptions) { + await options.file!.isPublic(); +} + +export async function fileMakePrivateInstancePrecondition( + options: ConformanceTestOptions +) { + await options.file!.makePrivate(); +} + +export async function fileMakePrivate(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.file!.makePrivate({ + preconditionOpts: { + ifMetagenerationMatch: options.file!.metadata.metageneration, + }, + }); + } else { + await options.file!.makePrivate(); + } +} + +export async function fileMakePublic(options: ConformanceTestOptions) { + await options.file!.makePublic(); +} + +export async function move(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.file!.move('new-file', { + preconditionOpts: {ifGenerationMatch: 0}, + }); + } else { + await options.file!.move('new-file'); + } +} + +export async function rename(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.file!.rename('new-name', { + preconditionOpts: {ifGenerationMatch: 0}, + }); + } else { + await options.file!.rename('new-name'); + } +} + +export async function rotateEncryptionKey(options: ConformanceTestOptions) { + const crypto = require('crypto'); + const buffer = crypto.randomBytes(32); + const newKey = buffer.toString('base64'); + if (options.preconditionRequired) { + await options.file!.rotateEncryptionKey({ + encryptionKey: Buffer.from(newKey, 'base64'), + preconditionOpts: {ifGenerationMatch: options.file!.metadata.generation}, + }); + } else { + await options.file!.rotateEncryptionKey({ + encryptionKey: Buffer.from(newKey, 'base64'), + }); + } +} + +export async function saveResumableInstancePrecondition( + options: ConformanceTestOptions +) { + const buf = createTestBuffer(FILE_SIZE_BYTES); + await options.file!.save(buf, { + resumable: true, + chunkSize: CHUNK_SIZE_BYTES, + metadata: {contentLength: FILE_SIZE_BYTES}, + }); +} + +export async function saveResumable(options: ConformanceTestOptions) { + const buf = createTestBuffer(FILE_SIZE_BYTES); + if (options.preconditionRequired) { + await options.file!.save(buf, { + resumable: true, + chunkSize: CHUNK_SIZE_BYTES, + metadata: {contentLength: FILE_SIZE_BYTES}, + preconditionOpts: { + ifGenerationMatch: options.file!.metadata.generation, + ifMetagenerationMatch: options.file!.metadata.metageneration, + }, + }); + } else { + await options.file!.save(buf, { + resumable: true, + chunkSize: CHUNK_SIZE_BYTES, + metadata: {contentLength: FILE_SIZE_BYTES}, + }); + } +} + +export async function saveMultipartInstancePrecondition( + options: ConformanceTestOptions +) { + await options.file!.save('testdata', {resumable: false}); +} + +export async function saveMultipart(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.file!.save('testdata', { + resumable: false, + preconditionOpts: { + ifGenerationMatch: options.file!.metadata.generation, + }, + }); + } else { + await options.file!.save('testdata', { + resumable: false, + }); + } +} + +export async function setMetadataInstancePrecondition( + options: ConformanceTestOptions +) { + const metadata = { + contentType: 'application/x-font-ttf', + metadata: { + my: 'custom', + properties: 'go here', + }, + }; + await options.file!.setMetadata(metadata); +} + +export async function setMetadata(options: ConformanceTestOptions) { + const metadata = { + contentType: 'application/x-font-ttf', + metadata: { + my: 'custom', + properties: 'go here', + }, + }; + if (options.preconditionRequired) { + await options.file!.setMetadata(metadata, { + ifMetagenerationMatch: options.file!.metadata.metageneration, + }); + } else { + await options.file!.setMetadata(metadata); + } +} + +export async function setStorageClass(options: ConformanceTestOptions) { + if (options.preconditionRequired) { + await options.file!.setStorageClass('nearline', { + preconditionOpts: { + ifGenerationMatch: options.file!.metadata.generation, + }, + }); + } else { + await options.file!.setStorageClass('nearline'); + } +} + +// ///////////////////////////////////////////////// +// /////////////////// HMAC KEY //////////////////// +// ///////////////////////////////////////////////// + +export async function deleteHMAC(options: ConformanceTestOptions) { + const metadata = { + state: 'INACTIVE', + }; + await options.hmacKey!.setMetadata(metadata); + await options.hmacKey!.delete(); +} + +export async function getHMAC(options: ConformanceTestOptions) { + await options.hmacKey!.get(); +} + +export async function getMetadataHMAC(options: ConformanceTestOptions) { + await options.hmacKey!.getMetadata(); +} + +export async function setMetadataHMAC(options: ConformanceTestOptions) { + const metadata = { + state: 'INACTIVE', + }; + await options.hmacKey!.setMetadata(metadata); +} + +///////////////////////////////////////////////// +////////////////////// IAM ////////////////////// +///////////////////////////////////////////////// + +export async function iamGetPolicy(options: ConformanceTestOptions) { + await options.bucket!.iam.getPolicy({requestedPolicyVersion: 1}); +} + +export async function iamSetPolicy(options: ConformanceTestOptions) { + const testPolicy: Policy = { + bindings: [ + { + role: 'roles/storage.admin', + members: ['serviceAccount:myotherproject@appspot.gserviceaccount.com'], + }, + ], + }; + if (options.preconditionRequired) { + const currentPolicy = await options.bucket!.iam.getPolicy(); + testPolicy.etag = currentPolicy[0].etag; + } + await options.bucket!.iam.setPolicy(testPolicy); +} + +export async function iamTestPermissions(options: ConformanceTestOptions) { + const permissionToTest = 'storage.buckets.delete'; + await options.bucket!.iam.testPermissions(permissionToTest); +} + +///////////////////////////////////////////////// +///////////////// NOTIFICATION ////////////////// +///////////////////////////////////////////////// + +export async function notificationDelete(options: ConformanceTestOptions) { + await options.notification!.delete(); +} + +export async function notificationCreate(options: ConformanceTestOptions) { + await options.notification!.create(); +} + +export async function notificationExists(options: ConformanceTestOptions) { + await options.notification!.exists(); +} + +export async function notificationGet(options: ConformanceTestOptions) { + await options.notification!.get(); +} + +export async function notificationGetMetadata(options: ConformanceTestOptions) { + await options.notification!.getMetadata(); +} + +///////////////////////////////////////////////// +/////////////////// STORAGE ///////////////////// +///////////////////////////////////////////////// + +export async function createBucket(options: ConformanceTestOptions) { + const bucket = options.storage!.bucket('test-creating-bucket'); + const [exists] = await bucket.exists(); + if (exists) { + bucket.delete(); + } + await options.storage!.createBucket('test-creating-bucket'); +} + +export async function createHMACKey(options: ConformanceTestOptions) { + const serviceAccountEmail = 'my-service-account@appspot.gserviceaccount.com'; + await options.storage!.createHmacKey(serviceAccountEmail); +} + +export async function getBuckets(options: ConformanceTestOptions) { + await options.storage!.getBuckets(); +} + +export async function getBucketsStream(options: ConformanceTestOptions) { + return new Promise((resolve, reject) => { + options + .storage!.getBucketsStream() + .on('data', () => {}) + .on('end', () => resolve(undefined)) + .on('error', err => reject(err)); + }); +} + +export function getHMACKeyStream(options: ConformanceTestOptions) { + return new Promise((resolve, reject) => { + options + .storage!.getHmacKeysStream() + .on('data', () => {}) + .on('end', () => resolve(undefined)) + .on('error', err => reject(err)); + }); +} + +export async function getServiceAccount(options: ConformanceTestOptions) { + await options.storage!.getServiceAccount(); +} diff --git a/handwritten/storage/conformance-test/scenarios/scenarioFive.ts b/handwritten/storage/conformance-test/scenarios/scenarioFive.ts new file mode 100644 index 00000000000..9c3a3b57215 --- /dev/null +++ b/handwritten/storage/conformance-test/scenarios/scenarioFive.ts @@ -0,0 +1,28 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as testFile from '../test-data/retryStrategyTestData.json'; +import {executeScenario, RetryTestCase} from '../conformanceCommon'; +import assert from 'assert'; + +const SCENARIO_NUMBER_TO_TEST = 5; +const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find( + test => test.id === SCENARIO_NUMBER_TO_TEST +); + +describe(`Scenario ${SCENARIO_NUMBER_TO_TEST}`, () => { + assert(retryTestCase); + executeScenario(retryTestCase); +}); diff --git a/handwritten/storage/conformance-test/scenarios/scenarioFour.ts b/handwritten/storage/conformance-test/scenarios/scenarioFour.ts new file mode 100644 index 00000000000..0072461e40f --- /dev/null +++ b/handwritten/storage/conformance-test/scenarios/scenarioFour.ts @@ -0,0 +1,28 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as testFile from '../test-data/retryStrategyTestData.json'; +import {executeScenario, RetryTestCase} from '../conformanceCommon'; +import assert from 'assert'; + +const SCENARIO_NUMBER_TO_TEST = 4; +const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find( + test => test.id === SCENARIO_NUMBER_TO_TEST +); + +describe(`Scenario ${SCENARIO_NUMBER_TO_TEST}`, () => { + assert(retryTestCase); + executeScenario(retryTestCase); +}); diff --git a/handwritten/storage/conformance-test/scenarios/scenarioOne.ts b/handwritten/storage/conformance-test/scenarios/scenarioOne.ts new file mode 100644 index 00000000000..981da527b87 --- /dev/null +++ b/handwritten/storage/conformance-test/scenarios/scenarioOne.ts @@ -0,0 +1,28 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as testFile from '../test-data/retryStrategyTestData.json'; +import {executeScenario, RetryTestCase} from '../conformanceCommon'; +import assert from 'assert'; + +const SCENARIO_NUMBER_TO_TEST = 1; +const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find( + test => test.id === SCENARIO_NUMBER_TO_TEST +); + +describe(`Scenario ${SCENARIO_NUMBER_TO_TEST}`, () => { + assert(retryTestCase); + executeScenario(retryTestCase); +}); diff --git a/handwritten/storage/conformance-test/scenarios/scenarioSeven.ts b/handwritten/storage/conformance-test/scenarios/scenarioSeven.ts new file mode 100644 index 00000000000..d1204d3b48d --- /dev/null +++ b/handwritten/storage/conformance-test/scenarios/scenarioSeven.ts @@ -0,0 +1,28 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as testFile from '../test-data/retryStrategyTestData.json'; +import {executeScenario, RetryTestCase} from '../conformanceCommon'; +import assert from 'assert'; + +const SCENARIO_NUMBER_TO_TEST = 7; +const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find( + test => test.id === SCENARIO_NUMBER_TO_TEST +); + +describe(`Scenario ${SCENARIO_NUMBER_TO_TEST}`, () => { + assert(retryTestCase); + executeScenario(retryTestCase); +}); diff --git a/handwritten/storage/conformance-test/scenarios/scenarioSix.ts b/handwritten/storage/conformance-test/scenarios/scenarioSix.ts new file mode 100644 index 00000000000..6d2b452ff7b --- /dev/null +++ b/handwritten/storage/conformance-test/scenarios/scenarioSix.ts @@ -0,0 +1,28 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as testFile from '../test-data/retryStrategyTestData.json'; +import {executeScenario, RetryTestCase} from '../conformanceCommon'; +import assert from 'assert'; + +const SCENARIO_NUMBER_TO_TEST = 6; +const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find( + test => test.id === SCENARIO_NUMBER_TO_TEST +); + +describe(`Scenario ${SCENARIO_NUMBER_TO_TEST}`, () => { + assert(retryTestCase); + executeScenario(retryTestCase); +}); diff --git a/handwritten/storage/conformance-test/scenarios/scenarioThree.ts b/handwritten/storage/conformance-test/scenarios/scenarioThree.ts new file mode 100644 index 00000000000..7b6c9002184 --- /dev/null +++ b/handwritten/storage/conformance-test/scenarios/scenarioThree.ts @@ -0,0 +1,28 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as testFile from '../test-data/retryStrategyTestData.json'; +import {executeScenario, RetryTestCase} from '../conformanceCommon'; +import assert from 'assert'; + +const SCENARIO_NUMBER_TO_TEST = 3; +const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find( + test => test.id === SCENARIO_NUMBER_TO_TEST +); + +describe(`Scenario ${SCENARIO_NUMBER_TO_TEST}`, () => { + assert(retryTestCase); + executeScenario(retryTestCase); +}); diff --git a/handwritten/storage/conformance-test/scenarios/scenarioTwo.ts b/handwritten/storage/conformance-test/scenarios/scenarioTwo.ts new file mode 100644 index 00000000000..fe2e6fb117e --- /dev/null +++ b/handwritten/storage/conformance-test/scenarios/scenarioTwo.ts @@ -0,0 +1,28 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as testFile from '../test-data/retryStrategyTestData.json'; +import {executeScenario, RetryTestCase} from '../conformanceCommon'; +import assert from 'assert'; + +const SCENARIO_NUMBER_TO_TEST = 2; +const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find( + test => test.id === SCENARIO_NUMBER_TO_TEST +); + +describe(`Scenario ${SCENARIO_NUMBER_TO_TEST}`, () => { + assert(retryTestCase); + executeScenario(retryTestCase); +}); diff --git a/handwritten/storage/conformance-test/test-data/retryInvocationMap.json b/handwritten/storage/conformance-test/test-data/retryInvocationMap.json new file mode 100644 index 00000000000..8dea345f12c --- /dev/null +++ b/handwritten/storage/conformance-test/test-data/retryInvocationMap.json @@ -0,0 +1,147 @@ +{ + "storage.buckets.delete": [ + "deleteBucket" + ], + "storage.buckets.patch": [ + "addLifecycleRuleInstancePrecondition", + "bucketMakePrivateInstancePrecondition", + "deleteLabelsInstancePrecondition", + "disableRequesterPaysInstancePrecondition", + "enableRequesterPaysInstancePrecondition", + "enableLoggingInstancePrecondition", + "removeRetentionPeriodInstancePrecondition", + "setCorsConfigurationInstancePrecondition", + "setLabelsInstancePrecondition", + "setRetentionPeriodInstancePrecondition", + "bucketSetStorageClassInstancePrecondition", + "bucketSetMetadataInstancePrecondition", + "addLifecycleRule", + "bucketMakePrivate", + "deleteLabels", + "disableRequesterPays", + "enableRequesterPays", + "enableLogging", + "removeRetentionPeriod", + "setCorsConfiguration", + "setLabels", + "setRetentionPeriod", + "bucketSetStorageClass", + "bucketSetMetadata" + ], + "storage.buckets.insert": [ + "create", + "createBucket" + ], + "storage.buckets.get": [ + "bucketGet", + "getLabels", + "bucketGetMetadata", + "bucketExists" + ], + "storage.objects.list": [ + "getFilesStream" + ], + "storage.notifications.list": [ + "getNotifications" + ], + "storage.buckets.lockRententionPolicy": [ + "lock" + ], + "storage.objects.patch": [ + "fileMakePrivateInstancePrecondition", + "setMetadataInstancePrecondition", + "fileMakePrivate", + "setMetadata" + ], + "storage.bucket_acl.insert": [ + "bucketMakePublic" + ], + "storage.object_acl.insert": [ + "fileMakePublic" + ], + "storage.objects.rewrite": [ + "copy", + "move", + "rename", + "rotateEncryptionKey", + "setStorageClass" + ], + "storage.objects.insert": [ + "bucketUploadResumableInstancePrecondition", + "saveResumableInstancePrecondition", + "bucketUploadMultipartInstancePrecondition", + "saveMultipartInstancePrecondition", + "createResumableUploadInstancePrecondition", + "bucketUploadResumable", + "saveResumable", + "bucketUploadMultipart", + "saveMultipart", + "createResumableUpload" + ], + "storage.resumable.upload": [ + "bucketUploadResumableInstancePrecondition", + "saveResumableInstancePrecondition", + "bucketUploadResumable", + "saveResumable" + ], + "storage.objects.delete": [ + "fileDeleteInstancePrecondition", + "fileDelete" + ], + "storage.objects.get": [ + "get", + "download", + "exists", + "getExpirationDate", + "getMetadata", + "isPublic", + "createReadStream" + ], + "storage.notifications.delete": [ + "notificationDelete" + ], + "storage.notifications.insert": [ + "createNotification", + "notificationCreate" + ], + "storage.notifications.get": [ + "notificationExists", + "notificationGet", + "notificationGetMetadata" + ], + "storage.buckets.getIamPolicy": [ + "iamGetPolicy" + ], + "storage.buckets.setIamPolicy": [ + "iamSetPolicy" + ], + "storage.buckets.testIamPermission": [ + "iamTestPermissions" + ], + "storage.buckets.list": [ + "getBuckets", + "getBucketsStream" + ], + "storage.objects.compose": [ + "combineInstancePrecondition", + "combine" + ], + "storage.hmacKey.delete": [ + "deleteHMAC" + ], + "storage.hmacKey.get": [ + "getHMAC", + "getMetadataHMAC" + ], + "storage.hmacKey.update": [ + ], + "storage.hmacKey.create": [ + "createHMACKey" + ], + "storage.hmacKey.list": [ + "getHMACKeyStream" + ], + "storage.serviceaccount.get": [ + "getServiceAccount" + ] +} diff --git a/handwritten/storage/conformance-test/test-data/retryStrategyTestData.json b/handwritten/storage/conformance-test/test-data/retryStrategyTestData.json new file mode 100644 index 00000000000..b302c007272 --- /dev/null +++ b/handwritten/storage/conformance-test/test-data/retryStrategyTestData.json @@ -0,0 +1,267 @@ +{ + "retryTests": [ + { + "id": 1, + "description": "always_idempotent", + "cases": [ + { + "instructions": ["return-503", "return-503"] + }, + { + "instructions": ["return-reset-connection", "return-reset-connection"] + }, + { + "instructions": ["return-reset-connection", "return-503"] + } + ], + "methods": [ + {"name": "storage.bucket_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.delete", "resources": ["BUCKET"]}, + {"name": "storage.buckets.get", "resources": ["BUCKET"]}, + {"name": "storage.buckets.getIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.insert", "resources": []}, + {"name": "storage.buckets.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.lockRetentionPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.testIamPermissions", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.delete", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.get", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.list", "resources": ["HMAC_KEY"]}, + {"name": "storage.notifications.delete", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.get", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.list", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.object_acl.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.serviceaccount.get", "resources": []} + ], + "preconditionProvided": false, + "expectSuccess": true + }, + { + "id": 2, + "description": "conditionally_idempotent_retries_when_precondition_is_present", + "cases": [ + { + "instructions": ["return-503", "return-503"] + }, + { + "instructions": ["return-reset-connection", "return-reset-connection"] + }, + { + "instructions": ["return-reset-connection", "return-503"] + } + ], + "methods": [ + {"name": "storage.buckets.patch", "resources": ["BUCKET"]}, + {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.update", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]}, + {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.insert", "resources": ["BUCKET"]}, + {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]} + ], + "preconditionProvided": true, + "expectSuccess": true + }, + { + "id": 3, + "description": "conditionally_idempotent_no_retries_when_precondition_is_absent", + "cases": [ + { + "instructions": ["return-503"] + }, + { + "instructions": ["return-reset-connection"] + } + ], + "methods": [ + {"name": "storage.buckets.patch", "resources": ["BUCKET"]}, + {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.update", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]}, + {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.insert", "resources": ["BUCKET"]}, + {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]} + ], + "preconditionProvided": false, + "expectSuccess": false + }, + { + "id": 4, + "description": "non_idempotent", + "cases": [ + { + "instructions": ["return-503"] + }, + { + "instructions": ["return-reset-connection"] + } + ], + "methods": [ + {"name": "storage.bucket_acl.delete", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.insert", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.patch", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.update", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.delete", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.insert", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.patch", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.update", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.create", "resources": []}, + {"name": "storage.notifications.insert", "resources": ["BUCKET"]}, + {"name": "storage.object_acl.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.insert", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.update", "resources": ["BUCKET", "OBJECT"]} + ], + "preconditionProvided": false, + "expectSuccess": false + }, + { + "id": 5, + "description": "non-retryable errors", + "cases": [ + { + "instructions": ["return-400"] + }, + { + "instructions": ["return-401"] + } + ], + "methods": [ + {"name": "storage.bucket_acl.delete", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.insert", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.patch", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.update", "resources": ["BUCKET"]}, + {"name": "storage.buckets.delete", "resources": ["BUCKET"]}, + {"name": "storage.buckets.get", "resources": ["BUCKET"]}, + {"name": "storage.buckets.getIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.insert", "resources": ["BUCKET"]}, + {"name": "storage.buckets.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.lockRetentionPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.patch", "resources": ["BUCKET"]}, + {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.testIamPermissions", "resources": ["BUCKET"]}, + {"name": "storage.buckets.update", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.delete", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.insert", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.patch", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.update", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.create", "resources": []}, + {"name": "storage.hmacKey.delete", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.get", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.list", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]}, + {"name": "storage.notifications.delete", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.get", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.insert", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.list", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.object_acl.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.insert", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.update", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.insert", "resources": ["BUCKET"]}, + {"name": "storage.objects.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.serviceaccount.get", "resources": []} + ], + "preconditionProvided": false, + "expectSuccess": false + }, + { + "id": 6, + "description": "mix_retryable_non_retryable_errors", + "cases": [ + { + "instructions": ["return-503", "return-400"] + }, + { + "instructions": ["return-reset-connection", "return-401"] + } + ], + "methods": [ + {"name": "storage.bucket_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.bucket_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.delete", "resources": ["BUCKET"]}, + {"name": "storage.buckets.get", "resources": ["BUCKET"]}, + {"name": "storage.buckets.getIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.insert", "resources": []}, + {"name": "storage.buckets.list", "resources": ["BUCKET"]}, + {"name": "storage.buckets.lockRetentionPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.patch", "resources": ["BUCKET"]}, + {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]}, + {"name": "storage.buckets.testIamPermissions", "resources": ["BUCKET"]}, + {"name": "storage.buckets.update", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.get", "resources": ["BUCKET"]}, + {"name": "storage.default_object_acl.list", "resources": ["BUCKET"]}, + {"name": "storage.hmacKey.delete", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.get", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.list", "resources": ["HMAC_KEY"]}, + {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]}, + {"name": "storage.notifications.delete", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.get", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.notifications.list", "resources": ["BUCKET", "NOTIFICATION"]}, + {"name": "storage.object_acl.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.object_acl.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.get", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.list", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.insert", "resources": ["BUCKET"]}, + {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]}, + {"name": "storage.serviceaccount.get", "resources": []} + ], + "preconditionProvided": true, + "expectSuccess": false + }, + { + "id": 7, + "description": "resumable_uploads_handle_complex_retries", + "cases": [ + { + "instructions": ["return-reset-connection", "return-503"] + }, + { + "instructions": ["return-408"] + }, + { + "instructions": ["return-503-after-256K"] + }, + { + "instructions": ["return-503-after-8192K"] + } + ], + "methods": [ + {"name": "storage.objects.insert", "group": "storage.resumable.upload", "resources": ["BUCKET"]} + ], + "preconditionProvided": true, + "expectSuccess": true + } + ] +} diff --git a/handwritten/storage/conformance-test/test-data/v4SignedUrl.json b/handwritten/storage/conformance-test/test-data/v4SignedUrl.json new file mode 100644 index 00000000000..86a51e0ba3c --- /dev/null +++ b/handwritten/storage/conformance-test/test-data/v4SignedUrl.json @@ -0,0 +1,698 @@ +{ + "signingV4Tests": [ + { + "description": "Simple GET", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Simple PUT", + "bucket": "test-bucket", + "object": "test-object", + "method": "PUT", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=8adff1d4285739e31aa68e73767a46bc5511fde377497dbe08481bf5ceb34e29cc9a59921748d8ec3dd4085b7e9b7772a952afedfcdaecb3ae8352275b8b7c867f204e3db85076220a3127a8a9589302fc1181eae13b9b7fe41109ec8cdc93c1e8bac2d7a0cc32a109ca02d06957211326563ab3d3e678a0ba296e298b5fc5e14593c99d444c94724cc4be97015dbff1dca377b508fa0cb7169195de98d0e4ac96c42b918d28c8d92d33e1bd125ce0fb3cd7ad2c45dae65c22628378f6584971b8bf3945b26f2611eb651e9b6a8648970c1ecf386bb71327b082e7296c4e1ee2fc0bdd8983da80af375c817fb1ad491d0bc22c0f51dba0d66e2cffbc90803e47", + "scheme": "https", + "expectedCanonicalRequest": "PUT\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n78742860705da91404222d5d66ff89850292471199c3c2808d116ad12e6177b4" + }, + { + "description": "POST for resumable uploads", + "bucket": "test-bucket", + "object": "test-object", + "method": "POST", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-resumable&X-Goog-Signature=4a6d39b23343cedf4c30782aed4b384001828c79ffa3a080a481ea01a640dea0a0ceb58d67a12cef3b243c3f036bb3799c6ee88e8db3eaf7d0bdd4b70a228d0736e07eaa1ee076aff5c6ce09dff1f1f03a0d8ead0d2893408dd3604fdabff553aa6d7af2da67cdba6790006a70240f96717b98f1a6ccb24f00940749599be7ef72aaa5358db63ddd54b2de9e2d6d6a586eac4fe25f36d86fc6ab150418e9c6fa01b732cded226c6d62fc95b72473a4cc55a8257482583fe66d9ab6ede909eb41516a8690946c3e87b0f2052eb0e97e012a14b2f721c42e6e19b8a1cd5658ea36264f10b9b1ada66b8ed5bf7ed7d1708377ac6e5fe608ae361fb594d2e5b24c54", + "headers": { + "X-Goog-Resumable": "start" + }, + "scheme": "https", + "expectedCanonicalRequest": "POST\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-resumable\nhost:storage.googleapis.com\nx-goog-resumable:start\n\nhost;x-goog-resumable\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n877f8b40179d2753296f2fd6de815ab40503c7a3c446a7b44aa4e74422ff4daf" + }, + { + "description": "Vary expiration and timestamp", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 20, + "timestamp": "2019-03-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190301%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190301T090000Z&X-Goog-Expires=20&X-Goog-SignedHeaders=host&X-Goog-Signature=9669ed5b10664dc594c758296580662912cf4bcc5a4ba0b6bf055bcbf6f34eed7bdad664f534962174a924741a0c273a4f67bc1847cef20192a6beab44223bd9d4fbbd749c407b79997598c30f82ddc269ff47ec09fa3afe74e00616d438df0d96a7d8ad0adacfad1dc3286f864d924fe919fb0dce45d3d975c5afe8e13af2db9cc37ba77835f92f7669b61e94c6d562196c1274529e76cfff1564cc2cad7d5387dc8e12f7a5dfd925685fe92c30b43709eee29fa2f66067472cee5423d1a3a4182fe8cea75c9329d181dc6acad7c393cd04f8bf5bc0515127d8ebd65d80c08e19ad03316053ea60033fd1b1fd85a69c576415da3bf0a3718d9ea6d03e0d66f0", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190301%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190301T090000Z&X-Goog-Expires=20&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190301T090000Z\n20190301/auto/storage/goog4_request\n779f19fdb6fd381390e2d5af04947cf21750277ee3c20e0c97b7e46a1dff8907" + }, + { + "description": "Vary bucket and object", + "bucket": "test-bucket2", + "object": "test-object2", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket2/test-object2?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=36e3d58dfd3ec1d2dd2f24b5ee372a71e811ffaa2162a2b871d26728d0354270bc116face87127532969c4a3967ed05b7309af741e19c7202f3167aa8c2ac420b61417d6451442bb91d7c822cd17be8783f01e05372769c88913561d27e6660dd8259f0081a71f831be6c50283626cbf04494ac10c394b29bb3bce74ab91548f58a37118a452693cf0483d77561fc9cac8f1765d2c724994cca46a83517a10157ee0347a233a2aaeae6e6ab5e204ff8fc5f54f90a3efdb8301d9fff5475d58cd05b181affd657f48203f4fb133c3a3d355b8eefbd10d5a0a5fd70d06e9515460ad74e22334b2cba4b29cae4f6f285cdb92d8f3126d7a1479ca3bdb69c207d860", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket2/test-object2\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\na139afbf35ac30e9864f63197f79609731ab1b0ca166e2a456dba156fcd3f9ce" + }, + { + "description": "Slashes in object name should not be URL encoded", + "bucket": "test-bucket", + "object": "path/with/slashes/under_score/amper&sand/file.ext", + "headers": { + "header/name/with/slash": "should-be-encoded" + }, + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/path/with/slashes/under_score/amper%26sand/file.ext?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=header%2Fname%2Fwith%2Fslash%3Bhost&X-Goog-Signature=2a9a82e84e39f5d2c0d980514db17f8c3dece473c9a5743d54e8453f9811927b1b99ce548c534cababd8fa339183e75b410e12e32a4c72f5ff176e95651fabed0072e59e7e236eb7e26f52c0ce599db1c47ae07af1a98d20872b6fde23432c0a5fcf4fb2dda735169198c80cd5cc51be9904f7e5eef2cc489ff44ac5697c529e4b34ac08709a7d2e425619377212c64561ed8b4d2fcb70a26e4f9236f995ab4658d240ac85c7a353bae6b2d39d5fc0716afa435a1f6e100db5504612b5e610db370623ab4b8eba3c03c98f23dcb4b9ffd518f2212abb2f93649d25385d71603d470cff0b7631adb9d0849d38609dedb3097761c8f47ec0d57777bb063611c05b", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/path/with/slashes/under_score/amper%26sand/file.ext\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=header%2Fname%2Fwith%2Fslash%3Bhost\nheader/name/with/slash:should-be-encoded\nhost:storage.googleapis.com\n\nheader/name/with/slash;host\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\nf1d206dd8cbe1b892d4081ccddae0927d9f5fee5653fb2a2f43e7c20ed455cad" + }, + { + "description": "Forward Slashes should not be stripped", + "bucket": "test-bucket", + "object": "/path/with/slashes/under_score/amper&sand/file.ext", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket//path/with/slashes/under_score/amper%26sand/file.ext?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=2db8b70e3f85b39be7824f6d02be31af2e6a2eb63f6bb41254851f7ef51bdad8963a9d2b254f8379c1780c8e6898be002d4100a0abd3d45f1437687fed65d15dd237c3a6f3c399c64ffd4e4cea7ef1c2f0391d35ecbeeaf3e3148d23c6f24c839cfcd92c1496332f5bfbbf1ed1e957eb45fad57df24828c96cf243eec23fba014d277c22a572708beb355888c5a8c0047cb3015d7f62cc90285676e7e34626fd0ce9ba5e0da39fc3de0035cc3ad120c46cb73db87246ae123f7a342c235e9480bd7d7e00c13b1e1bb7be5e2bce74d59a53505172463b48aefeedb48281d90874aa4177c881d3596ed1067f02eaac13d810a7aed234c41978b1394d0ce3662f76", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket//path/with/slashes/under_score/amper%26sand/file.ext\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n63c601ecd6ccfec84f1113fc906609cbdf7651395f4300cecd96ddd2c35164f8" + }, + { + "description": "Simple headers", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost&X-Goog-Signature=68ecd3b008328ed30d91e2fe37444ed7b9b03f28ed4424555b5161980531ef87db1c3a5bc0265aad5640af30f96014c94fb2dba7479c41bfe1c020eb90c0c6d387d4dd09d4a5df8b60ea50eb6b01cdd786a1e37020f5f95eb8f9b6cd3f65a1f8a8a65c9fcb61ea662959efd9cd73b683f8d8804ef4d6d9b2852419b013368842731359d7f9e6d1139032ceca75d5e67cee5fd0192ea2125e5f2955d38d3d50cf116f3a52e6a62de77f6207f5b95aaa1d7d0f8a46de89ea72e7ea30f21286318d7eba0142232b0deb3a1dc9e1e812a981c66b5ffda3c6b01a8a9d113155792309fd53a3acfd054ca7776e8eec28c26480cd1e3c812f67f91d14217f39a606669d", + "headers": { + "BAR": "BAR-value", + "foo": "foo-value" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost\nbar:BAR-value\nfoo:foo-value\nhost:storage.googleapis.com\n\nbar;foo;host\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n59c1ac1a6ee7d773d5c4487ecc861d60b71c4871dd18fc7d8485fac09df1d296" + }, + { + "description": "Headers with colons", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-Signature=30b831c18b5cdef6dceaa476a395a28d80002ad70b4419af8fd63eaad02c2fbb4b4829b3a4e33e7796a9ce92735498dfc20e0fbc177172f7c8ab6a07736512c7c923ef2f28a2e72d727fd61ca89495c9e62d51b93a2f7061451240c909ed8d05a7bcf616c1ad90fa5cdbc27c4724dec6b29db04129b32402db4ddf7b5b554724481bfdbf41cb24c3c6b9e33bb411c864077d6a19a750a90eb5ad9370d2b171df2813c9a864b40b2ee215ae9790d7916155de863708aa5121bca42e4695def5322f3726f8e1a7ec56da7a1a4f6b959253513a10f7edf6594c02340021b8cc709b0177ec6bb127fc2fb705f508bde045ed94603471c19c1c6af165f559a3c4741b&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost", + "headers": { + "BAR": "2023-02-10T03:", + "foo": "2023-02-10T02:00:00Z" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost\nbar:2023-02-10T03:\nfoo:2023-02-10T02:00:00Z\nhost:storage.googleapis.com\n\nbar;foo;host\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\na2a6df7e6bd818894e1f60ac3c393901b512ca1cf1061ba602dace3fb38c19a6" + }, + { + "description": "Headers should be trimmed", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=collapsed%3Bhost%3Bleading%3Btabs%3Btrailing&X-Goog-Signature=75d77a3ed2d9b74ff7e1e23b2fd7cc714ad4cc32518c65f3a8197827cd87d302623bab990cf2ff3a633bfaae69b6c2d897add78c105aa68411229610421c4239579add4aff6bdbd5067a0fd61c3aa0029d7de0f8ae88fa3458fa70f875e841d6df9598597d9012b9f848c6857e08f2704ca2f332c71738490ffdda2ed928f9340549d7295745725062d28dc1696eab7cb3b88ac4fd445e951423f645d680a60dd8033d65b65f4c10286f59f4258dbb2bcf36a76ffdd40574104cbbf0b76901c24df5854f24c42e9192fcedc386d85704fec6a6bad3a5201e1fb6c491a4c43371b0913420743580daf3504e99204c6ec894b4d70cd27bc60c3fe2850e8bf3ed22", + "headers": { + "collapsed": "abc def", + "leading": " xyz", + "trailing": "abc ", + "tabs": "\tabc\t\t\t\tdef\t" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=collapsed%3Bhost%3Bleading%3Btabs%3Btrailing\ncollapsed:abc def\nhost:storage.googleapis.com\nleading:xyz\ntabs:abc def\ntrailing:abc\n\ncollapsed;host;leading;tabs;trailing\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n19153e83555808dbfeb8969043cc8ce8d5db0cce91dc11fb9df58b8130f09d42" + }, + { + "description": "Header value with multiple inline values", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bmultiple&X-Goog-Signature=5cc113735625341f59c7203f0c2c9febc95ba6af6b9c38814f8e523214712087dc0996e4960d273ae1889f248ac1e58d4d19cb3a69ad7670e9a8ca1b434e878f59339dc7006cf32dfd715337e9f593e0504371839174962a08294586e0c78160a7aa303397888c8350637c6af3b32ac310886cc4590bfda9ca561ee58fb5b8ec56bc606d2ada6e7df31f4276e9dcb96bcaea39dc2cd096f3fad774f9c4b30e317ad43736c05f76831437f44e8726c1e90d3f6c9827dc273f211f32fc85658dfc5d357eb606743a6b00a29e519eef1bebaf9db3e8f4b1f5f9afb648ad06e60bc42fa8b57025056697c874c9ea76f5a73201c9717ea43e54713ff3502ff3fc626b", + "headers": { + "multiple": " xyz , abc, def , xyz " + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bmultiple\nhost:storage.googleapis.com\nmultiple:xyz , abc, def , xyz\n\nhost;multiple\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4df8e486146c31f1c8cd4e4c730554cde4326791ba48ec11fa969a3de064cd7f" + }, + { + "description": "Customer-supplied encryption key", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-encryption-algorithm%3Bx-goog-encryption-key%3Bx-goog-encryption-key-sha256&X-Goog-Signature=278a1c5a3bad248637054a047014760353942433955871031ed08f515b54588654ad033e91f046ab202b68673030e117d1b786c325e870238b035ba75b3feed560a17aff9bab6bddebd4a31a52cb68b214e27d3b0bd886502c6b36b164306fe88b5a07c6063592afe746b2a5d205dbe90dd5386b94f0a78f75d9f53ee884e18f476e8fc2eb1dd910ce0b4ae1f5d7b09876ef9bf983f539c028429e14bad3c75dbd4ed1ae37856f6d6f8a1805eaf8b52a0d6fc993902e4c1ee8de477661f7b67c3663000474cb00e178189789b2a3ed6bd21b4ade684fca8108ac4dd106acb17f5954d045775f7aa5a98ebda5d3075e11a8ea49c64c6ad1481e463e8c9f11f704", + "headers": { + "X-Goog-Encryption-Algorithm": "AES256", + "X-Goog-Encryption-Key": "key", + "X-Goog-Encryption-Key-Sha256": "key-hash" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-encryption-algorithm%3Bx-goog-encryption-key%3Bx-goog-encryption-key-sha256\nhost:storage.googleapis.com\nx-goog-encryption-algorithm:AES256\nx-goog-encryption-key:key\nx-goog-encryption-key-sha256:key-hash\n\nhost;x-goog-encryption-algorithm;x-goog-encryption-key;x-goog-encryption-key-sha256\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n66a45104eba8bdd9748723b45cbd54c3f0f6dba337a5deb9fb6a66334223dc06" + }, + { + "description": "List Objects", + "bucket": "test-bucket", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=6dbe94f8e52b2b8a9a476b1c857efa474e09944e2b52b925800316e094a7169d8dbe0df9c0ac08dabb22ac7e827470ceccd65f5a3eadba2a4fb9beebfe37f0d9bb1e552b851fa31a25045bdf019e507f5feb44f061551ef1aeb18dcec0e38ba2e2f77d560a46eaace9c56ed9aa642281301a9d848b0eb30749e34bc7f73a3d596240533466ff9b5f289cd0d4c845c7d96b82a35a5abd0c3aff83e4440ee6873e796087f43545544dc8c01afe1d79c726696b6f555371e491980e7ec145cca0803cf562c38f3fa1d724242f5dea25aac91d74ec9ddd739ff65523627763eaef25cd1f95ad985aaf0079b7c74eb5bcb2870a9b137a7b2c8e41fbe838c95872f75b", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n51a7426c2a6c6ab80f336855fc629461ff182fb1d2cb552ac68e5ce8e25db487" + }, + { + "description": "Query Parameter Encoding", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&aA0%C3%A9%2F%3D%25-_.~=~%20._-%25%3D%2F%C3%A90Aa&X-Goog-Signature=0e4f289c28626a32af95d78f5a0d2c75e5f0bef1cfe5035a532a8185e3ad45b1d5e8ad5ae0fb1898420e40a1576479861c6579db7e3b28ef8386160995ac1374be85e780ac3dfcc62538e9b2e2c51555950fb6fd0d7ecc509d3ccd9c02af5a6c6eb930d21a7383792300eb50a093a597c1da2a290ed56b01844251f8271d0d5e61fc3f4273a0c26f80e061c06124d93346ea1c50388e3fe73494c05ac27a54caedc04d1476c276c7602554e4cc1933e41df31ea523f4009e879c92333f98b4313755470d7e400bbbde9f5bef36f40b2a1c4a85edbd62aaa84a516e3df055d5d3f575b9ea0f1c24f057240852d9e618403e1dd2cd0fa7fc16b071b07322172256", + "queryParameters": { + "aA0é/=%-_.~": "~ ._-%=/é0Aa" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&aA0%C3%A9%2F%3D%25-_.~=~%20._-%25%3D%2F%C3%A90Aa\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n448f96c23dafa8210900554e138b2b5fd55bc53ef53b8637cecc3edec45a8fcf" + }, + { + "description": "Query Parameter Ordering", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-Meta-Foo=bar&X-Goog-SignedHeaders=host&prefix=%2Ffoo&X-Goog-Signature=a07745e1b3d59b85cbe11aa766df72c22468959e7217615dccb7f030234f66b60b37e480f30725ed51f29816362ca8286c619ebb66448ff1d370be2a4a48aacf20d3d2d6200ed17341a5791baf2ee5cd9c2823adacc6264f66c8a54fa887e1bce3c55cf78fb2f6a52618cf09d6f945f63d148052a7b66a75e075ff5065828a806b84bdc49a42399be7483225c720d5e18a6160f79d815f433e7921694fe1d041099851793c2581db0e5ca503cfb566e414f900ceede5f9b22030edd32ab20b6f7f9fb2afba89098b9364e03397c03a94eac3a140c99979b8786844fb4f6c62c1985378939dd1bbaea8e41b9100dda85a27733171cc78d96ee362ea2c3432f4d8", + "queryParameters": { + "prefix": "/foo", + "X-Goog-Meta-Foo": "bar" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-Meta-Foo=bar&X-Goog-SignedHeaders=host&prefix=%2Ffoo\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4dafe74ad142f32b7c25fc4e6b38fd3b8a6339d7f112247573fb0066f637db6c" + }, + { + "description": "Header Ordering", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-date&X-Goog-Signature=55a28435997457f1498291e878fd39c5f321973057d2541886020fdfd212b1467d9eeffdc70951ea952d634cb4193e657ed5b7860c46d37f7d904774680a16e518aa9dff273e8441d6893de615eb592e3113d682ad64a87eb0e0c48df17c30f899e7f940ba230530b30f725ab9ec38789682413752de6a026ae69dd858843100645f3ec986aed618d229f8844d378e0e66e907ede6dff7aac56723f51eb830e8877a56100c86a876173424602abefe6c22b6540a2b36634860b2e89137f297cca8f080bdf3433a9d614c5ab2ec84f65412b45516b30500886a2300f23c3423ae0e91546e3471ee08d06894bddc76203a418d46f35bf0b4574f7b24a693fb046c", + "headers": { + "X-Goog-Date": "20190201T090000Z" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-date\nhost:storage.googleapis.com\nx-goog-date:20190201T090000Z\n\nhost;x-goog-date\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4052143280d90d5f4a8c878ff7418be6fee5d34e50b1da28d8081a094b88fa61" + }, + { + "description": "Signed Payload Instead of UNSIGNED-PAYLOAD", + "bucket": "test-bucket", + "object": "test-object", + "method": "PUT", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-content-sha256%3Bx-testcasemetadata-payload-value&X-Goog-Signature=3e5a9669e9aa162888dff1553d24c159bad4f16d444987f6a1b26d8ad0cb7927f15bfaf79c205324d2138fd1f62edb255430c77a03c0d6e9601399e2519014f9e1a7051d9be735cde530022c84602b1c4c25c86cb1e1584489e49d511c9a618a1a8443af31626ca5b2ad105eda1e4499f52b4043f3c1a3bd40c06c0cae36bb19a50ed8671e5d2cdbb148a196ce5a8c14d6970c08225da293e1ef400c92e7a3d5ba0a29ad0893827c96b203a04b04ebd51929bf99b323beba93097dfee700ee2c1bd97013779e5c8f156e56175d4d07e453b2eb0d616086f9f4753dde63507efe88b0dec29c872d25d9465f07778b16b532814148c578ee7e64ed8437006fa551", + "headers": { + "X-Goog-Content-SHA256": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b982", + "X-TestCaseMetadata-Payload-Value": "hello" + }, + "scheme": "https", + "expectedCanonicalRequest": "PUT\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-content-sha256%3Bx-testcasemetadata-payload-value\nhost:storage.googleapis.com\nx-goog-content-sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b982\nx-testcasemetadata-payload-value:hello\n\nhost;x-goog-content-sha256;x-testcasemetadata-payload-value\n2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b982", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\nbe21a0841a897930ff5cf72e6e74ec5274efd76c3fe4cde6678f24a0a3d6dbec" + }, + { + "description": "Virtual Hosted Style", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://test-bucket.storage.googleapis.com/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=06c633ea060b0eda81ee58dd3337b01b0d243a44f18cb03ec948861f533a129e579c7fd4c856d187f1c7b86e5715ea0abf6a1c6ba32b69274d22b1b0406df6847dc87f0d289fe8dc0682351574849b8b13e4b66922f39441af96becb73ea4c56cd5e3eeb30bc91fe84e8bd205adca8639253bdb65b2fcaf2598a230c6d8f6d8177c9e58a61b6e826767f594056b490184d676897c4bbedc15d6fbf08c3fa82a406c62e74db661e6c5d7d3ced29e0619ee719dce4b8136360345b8dce120b9f1debd511c8dac3e6d874ee05bfda8c8f1c4fedd0c07fc6d98f5f18a349bb204d8ff401402a025194e2792df8a09282141157e4ca51d26a8d0d142a01c805321911", + "scheme": "https", + "urlStyle": "VIRTUAL_HOSTED_STYLE", + "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:test-bucket.storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n89eeae48258eccdcb1f592fb908008e3f5d36a949c002c1e614c94356dc18fc6" + }, + { + "description": "HTTP Bucket Bound Hostname Support", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56", + "scheme": "http", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld", + "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:mydomain.tld\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\nd6c309924b51a5abbe4d6356f7bf29c2120c6b14649b1e97b3bc9309adca7d4b" + }, + { + "description": "HTTPS Bucket Bound Hostname Support", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56", + "scheme": "https", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld", + "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:mydomain.tld\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\nd6c309924b51a5abbe4d6356f7bf29c2120c6b14649b1e97b3bc9309adca7d4b" + }, + { + "description": "Simple GET with hostname", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74", + "scheme": "https", + "hostname": "storage.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Simple GET with non-default hostname", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://localhost:8080/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=66c8c0371ca7d933a0d50f110abdf4fc7e3e379329134f272ebe4aa8100ccd5f21cd56ca5ccffae5ed36c8d6840e7cac80c2e7d786cd85b10d0faea34cddf09d2e7eb7f5c7c53934e4bf8f5cd654bc3c1b5ee9e3f8ca2189cd225b445bb866563fc4bd0d0b4d116111655611d12ec18f2d854fd7142d9afcc977dbd8f6d0524e4170506abf2b119bbe00d17697321d225162fabddb4ddae77781b4f3277a6b6fccfeb47d70b88537e5efb416001274aaeb1535b5aae757c997edc66d03898a5d08f767313d018d10992981d00e2a18ed9a6839b8a1ac7b3be1cab2e9511ba91e14a786443b59e9f21e1ae74a2c60106180646a764531fbe1fcd9c1e40550e56e", + "scheme": "http", + "hostname": "localhost:8080", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Simple GET with endpoint on client", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com:443/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74", + "scheme": "https", + "clientEndpoint": "storage.googleapis.com:443", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Endpoint on client with scheme", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://localhost:8080/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=66c8c0371ca7d933a0d50f110abdf4fc7e3e379329134f272ebe4aa8100ccd5f21cd56ca5ccffae5ed36c8d6840e7cac80c2e7d786cd85b10d0faea34cddf09d2e7eb7f5c7c53934e4bf8f5cd654bc3c1b5ee9e3f8ca2189cd225b445bb866563fc4bd0d0b4d116111655611d12ec18f2d854fd7142d9afcc977dbd8f6d0524e4170506abf2b119bbe00d17697321d225162fabddb4ddae77781b4f3277a6b6fccfeb47d70b88537e5efb416001274aaeb1535b5aae757c997edc66d03898a5d08f767313d018d10992981d00e2a18ed9a6839b8a1ac7b3be1cab2e9511ba91e14a786443b59e9f21e1ae74a2c60106180646a764531fbe1fcd9c1e40550e56e", + "scheme": "http", + "clientEndpoint": "http://localhost:8080", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Emulator host", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://xyz.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=53b20003ff2c552b3194a6bccc25024663662392554b3334e989e2704f3a0308455eaacf45c335a78c0186a5cf8eef78bf5781a7267465d28a35c9e1291f87ff340e9ee40b3b9bdce70561bf000887ce38ccd7d2445a8749453960a8f11d37576dfd5942f92d6f4527bbeffb90526b5de9653b6ca16136e9f19bcb65d984ddaf22c4ade45d6a168bb4752a43de33ab121206f50d994612824407711bff720cb1b207b61b613c44c85d3ce16dc4fc6eba24e494e176b0780d0ab85a800b13fcbf31434ddf51992efae1efde330ebda0617d1c20078ef22a4f10a7bcbed961237442d9a8db78d7aeb777a4994b50efdd41e07c4966e912a30f92a7426f207e9545", + "emulatorHostname": "https://xyz.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Endpoint on client takes precedence over emulator", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://localhost:8080/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=66c8c0371ca7d933a0d50f110abdf4fc7e3e379329134f272ebe4aa8100ccd5f21cd56ca5ccffae5ed36c8d6840e7cac80c2e7d786cd85b10d0faea34cddf09d2e7eb7f5c7c53934e4bf8f5cd654bc3c1b5ee9e3f8ca2189cd225b445bb866563fc4bd0d0b4d116111655611d12ec18f2d854fd7142d9afcc977dbd8f6d0524e4170506abf2b119bbe00d17697321d225162fabddb4ddae77781b4f3277a6b6fccfeb47d70b88537e5efb416001274aaeb1535b5aae757c997edc66d03898a5d08f767313d018d10992981d00e2a18ed9a6839b8a1ac7b3be1cab2e9511ba91e14a786443b59e9f21e1ae74a2c60106180646a764531fbe1fcd9c1e40550e56e", + "scheme": "http", + "clientEndpoint": "http://localhost:8080", + "emulatorHostname": "https://xyz.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Hostname takes precendence over endpoint and emulator", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://xyz.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=53b20003ff2c552b3194a6bccc25024663662392554b3334e989e2704f3a0308455eaacf45c335a78c0186a5cf8eef78bf5781a7267465d28a35c9e1291f87ff340e9ee40b3b9bdce70561bf000887ce38ccd7d2445a8749453960a8f11d37576dfd5942f92d6f4527bbeffb90526b5de9653b6ca16136e9f19bcb65d984ddaf22c4ade45d6a168bb4752a43de33ab121206f50d994612824407711bff720cb1b207b61b613c44c85d3ce16dc4fc6eba24e494e176b0780d0ab85a800b13fcbf31434ddf51992efae1efde330ebda0617d1c20078ef22a4f10a7bcbed961237442d9a8db78d7aeb777a4994b50efdd41e07c4966e912a30f92a7426f207e9545", + "emulatorHostname": "http://localhost:9000", + "clientEndpoint": "http://localhost:8080", + "hostname": "https://xyz.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Universe domain", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.domain.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=8cd0d479a88fb7d791a2dcc8fc5b5f020ca817eeef5b5a5cb3260eb63cf47ecd271faa238d0fa31efca35bc2a9244bd122178c520749f922c0235726a5a6be099bf4f33a0d54187eee2e0208964c2a13104b03e235cdeb4f07b3eb566b8a33259cf7540a3fe823be601ace2a54a79acd6834cb646380c4cfc7ef0fd95d3ebbc1f97d840f6fe1dceed4269ecb4e91ff7e6633f38adab82049a965968367b9e7c362cec868d804bd42abbb6d2e837ce5d45ee9e1d92c7acc09623acaae3df6128ca15f9f80bb6543944e8c997f691c35113b9e9f44e86fd343524343b08dd8f887685588acc103e0b432f24912e7e1c63e086aeed1890e41b75beb64164fe6bfcf", + "universeDomain": "domain.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Universe domain with virtual hosted style", + "bucket": "test-bucket", + "object": "test-object", + "urlStyle": "VIRTUAL_HOSTED_STYLE", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://test-bucket.storage.domain.com/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=25820e3a60856596cba594511d7d4039239b2728a9738f15d3a7acce8d70aa5435d0c91f99a9318f932afc73355ac562e014cb654e16ed5524b403536f1cba74489701fdc0c088b8826fccf20a648d3b2b704bd6661e01786d4132174c21441d0752be07e8af93e84e24b87799ee91fabef24a0a58d0889263280c3d37423fab677bd4d98469ab01aa36efaad62ff81ca27bf7fc92f14e20faa71e34de9ffbc5eb4ecf1b0361de42270665bb78367bd0a8cc6a604a8e347f0c864754bf14514aac3106fe73572a6c068ce2c380cc2a943b35502093d162ba9ae8de9abbbc9541ef765d5679857a89d36cc01be30cf1e04c4a477bbcd59a02955dcc1a903d8baa", + "universeDomain": "domain.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + } + ], + "postPolicyV4Tests": [ + { + "description": "POST Policy Simple", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902670-h3q7wvodjor6bc7y", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z" + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902670-h3q7wvodjor6bc7y/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "9eb947e08891be62a6c26d214c5b639506a57c20e1242a135dce45326a821aa9d8eaf5a8b6961a7ba87ab4da09668eb303b615dabdd6344970a024d4697aa3c467ec527cb8318d6c971782c6f82555c8cdd791589226962f0c26d5c415bf9d257cf9ed266a43e1b74891c35f009be741cef42a1eec150ffa5783de217e3704e8f278209250423f62c57864c50c6e525e6e9b724fcafc3844898c6a219effedfaf1204cb95eb135f63307db86726b8d6c12b8a75df61f93143b5e798acdffee0b5ad481075fa89a9d49a470cea339e5e43b61a0d87ef54f355dc401a659b583e7906c20bf84b8da7941d2e932acab21ffe92416985262fe772a117483b6e70410", + "policy": "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcwLWgzcTd3dm9kam9yNmJjN3kifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"bucket\":\"rsaposttest-1579902670-h3q7wvodjor6bc7y\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Simple Virtual Hosted Style", + "policyInput": { + "scheme": "https", + "urlStyle": "VIRTUAL_HOSTED_STYLE", + "bucket": "rsaposttest-1579902670-h3q7wvodjor6bc7y", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z" + }, + "policyOutput": { + "url": "https://rsaposttest-1579902670-h3q7wvodjor6bc7y.storage.googleapis.com/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "9eb947e08891be62a6c26d214c5b639506a57c20e1242a135dce45326a821aa9d8eaf5a8b6961a7ba87ab4da09668eb303b615dabdd6344970a024d4697aa3c467ec527cb8318d6c971782c6f82555c8cdd791589226962f0c26d5c415bf9d257cf9ed266a43e1b74891c35f009be741cef42a1eec150ffa5783de217e3704e8f278209250423f62c57864c50c6e525e6e9b724fcafc3844898c6a219effedfaf1204cb95eb135f63307db86726b8d6c12b8a75df61f93143b5e798acdffee0b5ad481075fa89a9d49a470cea339e5e43b61a0d87ef54f355dc401a659b583e7906c20bf84b8da7941d2e932acab21ffe92416985262fe772a117483b6e70410", + "policy": "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcwLWgzcTd3dm9kam9yNmJjN3kifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"bucket\":\"rsaposttest-1579902670-h3q7wvodjor6bc7y\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Simple Bucket Bound Hostname", + "policyInput": { + "scheme": "https", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld", + "bucket": "rsaposttest-1579902670-h3q7wvodjor6bc7y", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z" + }, + "policyOutput": { + "url": "https://mydomain.tld/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "9eb947e08891be62a6c26d214c5b639506a57c20e1242a135dce45326a821aa9d8eaf5a8b6961a7ba87ab4da09668eb303b615dabdd6344970a024d4697aa3c467ec527cb8318d6c971782c6f82555c8cdd791589226962f0c26d5c415bf9d257cf9ed266a43e1b74891c35f009be741cef42a1eec150ffa5783de217e3704e8f278209250423f62c57864c50c6e525e6e9b724fcafc3844898c6a219effedfaf1204cb95eb135f63307db86726b8d6c12b8a75df61f93143b5e798acdffee0b5ad481075fa89a9d49a470cea339e5e43b61a0d87ef54f355dc401a659b583e7906c20bf84b8da7941d2e932acab21ffe92416985262fe772a117483b6e70410", + "policy": "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcwLWgzcTd3dm9kam9yNmJjN3kifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"bucket\":\"rsaposttest-1579902670-h3q7wvodjor6bc7y\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Simple Bucket Bound Hostname HTTP", + "policyInput": { + "scheme": "http", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld", + "bucket": "rsaposttest-1579902670-h3q7wvodjor6bc7y", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z" + }, + "policyOutput": { + "url": "http://mydomain.tld/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "9eb947e08891be62a6c26d214c5b639506a57c20e1242a135dce45326a821aa9d8eaf5a8b6961a7ba87ab4da09668eb303b615dabdd6344970a024d4697aa3c467ec527cb8318d6c971782c6f82555c8cdd791589226962f0c26d5c415bf9d257cf9ed266a43e1b74891c35f009be741cef42a1eec150ffa5783de217e3704e8f278209250423f62c57864c50c6e525e6e9b724fcafc3844898c6a219effedfaf1204cb95eb135f63307db86726b8d6c12b8a75df61f93143b5e798acdffee0b5ad481075fa89a9d49a470cea339e5e43b61a0d87ef54f355dc401a659b583e7906c20bf84b8da7941d2e932acab21ffe92416985262fe772a117483b6e70410", + "policy": "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcwLWgzcTd3dm9kam9yNmJjN3kifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"bucket\":\"rsaposttest-1579902670-h3q7wvodjor6bc7y\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy ACL matching", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902662-x2kd7kjwh2w5izcw", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "conditions": { + "startsWith": [ + "$acl", + "public" + ] + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902662-x2kd7kjwh2w5izcw/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "134b88d3d7ed57fd7ba79fe53b52191f2ccf46bde0ed1b1fe1557025dc76388fab367b55e887e7a2a23425c98c72d5d1095700568ca5e71f0bd38042b8d94ace6d75bce2c17760b53a8af7706944fd9f382b3a78c9651d9214a5d6252f099c18a759cd98f8dd3e32ac024d5e1b63d4c01d44954d4e943254f3a4cc4cab74cd251a733d4794a22e5840993b6d2970aa050f403c68c25019e91d133d47ff7188facf13560f918ae8efe49ec9ebcc4080d141154554f65cc6d9d6ef0e8bc12119c7491800d79769b5f27707ea9fe78c7af3c39df82608ca78f6f60b638510fd45a14404ed0224365c7ea45b839d91db99a7f8af50a64b754817318fae7bb94b3574", + "policy": "eyJjb25kaXRpb25zIjpbWyJzdGFydHMtd2l0aCIsIiRhY2wiLCJwdWJsaWMiXSx7ImJ1Y2tldCI6InJzYXBvc3R0ZXN0LTE1Nzk5MDI2NjIteDJrZDdrandoMnc1aXpjdyJ9LHsia2V5IjoidGVzdC1vYmplY3QifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAxMjNUMDQzNTMwWiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0LWlhbS1jcmVkZW50aWFsc0BkdW1teS1wcm9qZWN0LWlkLmlhbS5nc2VydmljZWFjY291bnQuY29tLzIwMjAwMTIzL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDEtMjNUMDQ6MzU6NDBaIn0=" + }, + "expectedDecodedPolicy": "{\"conditions\":[[\"starts-with\",\"$acl\",\"public\"],{\"bucket\":\"rsaposttest-1579902662-x2kd7kjwh2w5izcw\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Within Content-Range", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902672-lpd47iogn6hx4sle", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "conditions": { + "contentLengthRange": [ + 246, + 266 + ] + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902672-lpd47iogn6hx4sle/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "7f81e41a67c60dda3578c3e4f6457692e9cdb7b920da30204de893dac10e625fabea5f8002c8e1891bedc47e6136c8a012dbb8df4e532aea3e186b6c98050cdaf8d82d6cf01495ed87d97af6909589e456cd4d1a624462166cdd4c3bb3a6f945a69d69768bef4d8542add6c2971c3d9993af805f9e4cf2ad8abc69cf8dc3a99eb658eb51030d84037583a5991de29cad34edf7c0d88f6e904f8a00b86022394cc35e9552ed7bdcec3f04e46165952f78cd8bfcabb2def569b6076d0009f6ccc79b94fd67497481711dea1351e82f9e2626c9de374c81aa418000bff039f97367d021afb85d228230b4f3cd5ffe58ccb140bebc62a34e45fc42ba75aec4335035", + "policy": "eyJjb25kaXRpb25zIjpbWyJjb250ZW50LWxlbmd0aC1yYW5nZSIsMjQ2LDI2Nl0seyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcyLWxwZDQ3aW9nbjZoeDRzbGUifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[[\"content-length-range\",246,266],{\"bucket\":\"rsaposttest-1579902672-lpd47iogn6hx4sle\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Cache-Control File Header", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902669-nwk5s7vvfjgdjs62", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "acl": "public-read", + "cache-control": "public,max-age=86400" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902669-nwk5s7vvfjgdjs62/", + "fields": { + "key": "test-object", + "acl": "public-read", + "cache-control": "public,max-age=86400", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "99f29892dca68c13a1a9939392efa0891e68def63376ee6de4cf978e1537fcf79e1f8ef8ed068bc00a1c6a311fafbcc95eee5ad59aee502fe443b06905a6942f04e9f516e6bdd4162571b989a45378850d0721956a1808d5f4e3d531b6c20c654886b6910acd4c334127f78f8e6bfcb38ed82c65ecd2b0283e4e17275cbae40c43619ccecfe47cea81dad5ec180bbebc239c1c323af6719df4916e85db2b0a7f9a931ccb8ffe4d6f23899359339593c92f246be884324a1959327a5108a88f48da5be22444c943ff58493b3d1579f4dc734a7b14b3759b8e4a9350666e55e187a3b14f8b6388cf474ec8b7c7ed67cd5e21d0e13c1cf09884fdf5deb045b8bb6f", + "policy": "eyJjb25kaXRpb25zIjpbeyJhY2wiOiJwdWJsaWMtcmVhZCJ9LHsiY2FjaGUtY29udHJvbCI6InB1YmxpYyxtYXgtYWdlPTg2NDAwIn0seyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjY5LW53azVzN3Z2ZmpnZGpzNjIifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"acl\":\"public-read\"},{\"cache-control\":\"public,max-age=86400\"},{\"bucket\":\"rsaposttest-1579902669-nwk5s7vvfjgdjs62\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Success With Status", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902678-pt5yms55j47r6qy4", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "success_action_status": "200" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902678-pt5yms55j47r6qy4/", + "fields": { + "key": "test-object", + "success_action_status": "200", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "4b93fb02fa09f8eca92fc27eaa79028ae56c4b61fef83715833ebf7601607dbee627d6101e078881e4b24d795e9e2062ddb7b01470a742b9f6b1aac5c7b7c86d17bf298259189fd6ae0d6ae952b993a6a9f4eaf218bcc462d8dbc0b8553ca4d00349714e1143655a8eed18b02c71e9e53558055976cf3dc58f5946c9e9d6bda9305eed0575f7be80abff41d7a02fe2ab9a2abe87ab7040314734c1179e3a8edb0a024f227509391ca1ef4705140252a1a0bd6022096e9ef0ef5789639bce5953d5c4595b81b262768dbbfe2b7f68e3ebd2cf42f746897fe7c0ac8ec08c6cb85db8b1737a98d25bfa4a4022be72c4e17a1687856c1020b4fdd9438e91949437be", + "policy": "eyJjb25kaXRpb25zIjpbeyJzdWNjZXNzX2FjdGlvbl9zdGF0dXMiOiIyMDAifSx7ImJ1Y2tldCI6InJzYXBvc3R0ZXN0LTE1Nzk5MDI2NzgtcHQ1eW1zNTVqNDdyNnF5NCJ9LHsia2V5IjoidGVzdC1vYmplY3QifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAxMjNUMDQzNTMwWiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0LWlhbS1jcmVkZW50aWFsc0BkdW1teS1wcm9qZWN0LWlkLmlhbS5nc2VydmljZWFjY291bnQuY29tLzIwMjAwMTIzL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDEtMjNUMDQ6MzU6NDBaIn0=" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"success_action_status\":\"200\"},{\"bucket\":\"rsaposttest-1579902678-pt5yms55j47r6qy4\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Success With Redirect", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902671-6ldm6caw4se52vrx", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "success_action_redirect": "http://www.google.com/" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902671-6ldm6caw4se52vrx/", + "fields": { + "key": "test-object", + "success_action_redirect": "http://www.google.com/", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "6e83c57e73b1794eb6c5903a1eebdb5c29b0adc333010a5cd623fcb1a9716a31c680c94d43fd9f1f18134b864c380c78f8c1ab048038e743f9da080148365acaa01374dc7aa626cc93c73010a67b79c6776faf5edb8eb7ad56c8f9b9c998a1dab7ea1de675f2b315951c4ca2f54a3d21570896aaa66d8980ed09adf4e4240b49478bdabdaf51b720124569e94b1918856893c14c119c529fcb2e01838198b5d18042994d180fd4b9e26aef1d97fe5646c328e15a05decf6005e1c64cb7783811811f4cd5a720cbd6aa4cfc27ac81fc0b163ee9719c53af5019fd2be83b87e0da6d285f0270bc94f1e8788993794c309745c22709ee0dbad0e463f06830aabbbf", + "policy": "eyJjb25kaXRpb25zIjpbeyJzdWNjZXNzX2FjdGlvbl9yZWRpcmVjdCI6Imh0dHA6Ly93d3cuZ29vZ2xlLmNvbS8ifSx7ImJ1Y2tldCI6InJzYXBvc3R0ZXN0LTE1Nzk5MDI2NzEtNmxkbTZjYXc0c2U1MnZyeCJ9LHsia2V5IjoidGVzdC1vYmplY3QifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAxMjNUMDQzNTMwWiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0LWlhbS1jcmVkZW50aWFsc0BkdW1teS1wcm9qZWN0LWlkLmlhbS5nc2VydmljZWFjY291bnQuY29tLzIwMjAwMTIzL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDEtMjNUMDQ6MzU6NDBaIn0=" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"success_action_redirect\":\"http://www.google.com/\"},{\"bucket\":\"rsaposttest-1579902671-6ldm6caw4se52vrx\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Character Escaping", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902671-6ldm6caw4se52vrx", + "object": "$test-object-é", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "success_action_redirect": "http://www.google.com/", + "x-goog-meta-custom-1": "$test-object-é-metadata" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902671-6ldm6caw4se52vrx/", + "fields": { + "key": "$test-object-é", + "success_action_redirect": "http://www.google.com/", + "x-goog-meta-custom-1": "$test-object-é-metadata", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "a79ab836c4a36c3cd1dd3253c9482e2bb65ab461ac61fccee8757169da32c77f07a0da4c47745314fc9fba24d93679c864bb197e6eff26caddf3099b72db962451131afa969bf901d6d4ef63db3e36d48af4040b743b37ab8a08174e63cb5da39082490c03a8a28f7ede43f847a8f4447bb82b73434a1bcd365c8e6f62ae09a7b30a0706745787542a919096632840925d5677f668800220e6dbe83c8a42dc8343c85c16499b7179b96a677cfb35af6cf0face1b0409f40f41fd159df50d9fe4dd915439bd34d98ae22f4e2376e6b6c86654abe147083f2766fa75cc2cee9241f0ea5bcb8daa431712952f1038e7c596568500d80834957988be69560de5ce5d", + "policy": "eyJjb25kaXRpb25zIjpbeyJzdWNjZXNzX2FjdGlvbl9yZWRpcmVjdCI6Imh0dHA6Ly93d3cuZ29vZ2xlLmNvbS8ifSx7IngtZ29vZy1tZXRhLWN1c3RvbS0xIjoiJHRlc3Qtb2JqZWN0LVx1MDBlOS1tZXRhZGF0YSJ9LHsiYnVja2V0IjoicnNhcG9zdHRlc3QtMTU3OTkwMjY3MS02bGRtNmNhdzRzZTUydnJ4In0seyJrZXkiOiIkdGVzdC1vYmplY3QtXHUwMGU5In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"success_action_redirect\":\"http://www.google.com/\"},{\"x-goog-meta-custom-1\":\"$test-object-\u00e9-metadata\"},{\"bucket\":\"rsaposttest-1579902671-6ldm6caw4se52vrx\"},{\"key\":\"$test-object-\u00e9\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy With Additional Metadata", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902671-6ldm6caw4se52vrx", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "content-disposition": "attachment; filename=\"~._-%=/é0Aa\"", + "content-encoding": "gzip", + "content-type": "text/plain", + "success_action_redirect": "http://www.google.com/" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902671-6ldm6caw4se52vrx/", + "fields": { + "content-disposition": "attachment; filename=\"~._-%=/é0Aa\"", + "content-encoding": "gzip", + "content-type": "text/plain", + "key": "test-object", + "success_action_redirect": "http://www.google.com/", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "10e881a484d257672192a50892f7373ef243e1ff0e9043e47b3487d8280e4a27e85b0b16a60e5f9f539fc04c5b6141ca8a568fd2b66555000061cad696d6841cb31dc78862dbf0f66b7d55e72156c21a2ffa116923f86df523e4b16ef686acb46bc2665a7827c5dfafc26d7a6919ffea7f2d7803aa61f93d6389731adface622a848e663b5106858754e06e1a63d55feca12d814e1bcbcf5c42cd573950f53c0e9aa9bf2e746aa1287d0a293e07c24cf15698d42f11639cbd385ba8d9fc7db17dffdcab6d4b4be2e2219f7b98a58303294087858c120a0bc550bad31e4f101615066b9e946f0d54bcd7ae8e1306608b539213c809c13deae16a2a5d62b2e9cb7", + "policy": "eyJjb25kaXRpb25zIjpbeyJjb250ZW50LWRpc3Bvc2l0aW9uIjoiYXR0YWNobWVudDsgZmlsZW5hbWU9XCJ+Ll8tJT0vXHUwMGU5MEFhXCIifSx7ImNvbnRlbnQtZW5jb2RpbmciOiJnemlwIn0seyJjb250ZW50LXR5cGUiOiJ0ZXh0L3BsYWluIn0seyJzdWNjZXNzX2FjdGlvbl9yZWRpcmVjdCI6Imh0dHA6Ly93d3cuZ29vZ2xlLmNvbS8ifSx7ImJ1Y2tldCI6InJzYXBvc3R0ZXN0LTE1Nzk5MDI2NzEtNmxkbTZjYXc0c2U1MnZyeCJ9LHsia2V5IjoidGVzdC1vYmplY3QifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAxMjNUMDQzNTMwWiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0LWlhbS1jcmVkZW50aWFsc0BkdW1teS1wcm9qZWN0LWlkLmlhbS5nc2VydmljZWFjY291bnQuY29tLzIwMjAwMTIzL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDEtMjNUMDQ6MzU6NDBaIn0=" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"content-disposition\":\"attachment; filename=\\\"~._-%=/é0Aa\\\"\"},{\"content-encoding\":\"gzip\"},{\"content-type\":\"text/plain\"},{\"success_action_redirect\":\"http://www.google.com/\"},{\"bucket\":\"rsaposttest-1579902671-6ldm6caw4se52vrx\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + } + ] +} diff --git a/handwritten/storage/conformance-test/testBenchUtil.ts b/handwritten/storage/conformance-test/testBenchUtil.ts new file mode 100644 index 00000000000..b66f8309458 --- /dev/null +++ b/handwritten/storage/conformance-test/testBenchUtil.ts @@ -0,0 +1,54 @@ +/*! + * Copyright 2021 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import {execSync} from 'child_process'; +import {unlinkSync, writeFileSync} from 'fs'; +import {URL} from 'url'; + +const HOST = process.env.STORAGE_EMULATOR_HOST || 'http://localhost:9000'; +const PORT = new URL(HOST).port; +const CONTAINER_NAME = 'storage-testbench'; +const DEFAULT_IMAGE_NAME = + 'gcr.io/cloud-devrel-public-resources/storage-testbench'; +const DEFAULT_IMAGE_TAG = 'v0.35.0'; +const DOCKER_IMAGE = `${DEFAULT_IMAGE_NAME}:${DEFAULT_IMAGE_TAG}`; +const PULL_CMD = `docker pull ${DOCKER_IMAGE}`; +const RUN_CMD = `docker run --rm -d -p ${PORT}:${PORT} --name ${CONTAINER_NAME} ${DOCKER_IMAGE} && sleep 1`; +const STOP_CMD = `docker stop ${CONTAINER_NAME};`; + +export async function getTestBenchDockerImage(): Promise { + return execSync(PULL_CMD); +} + +export async function runTestBenchDockerImage(): Promise { + return execSync(RUN_CMD); +} + +export async function stopTestBenchDockerImage(): Promise { + return execSync(STOP_CMD); +} + +export function createTestBuffer(sizeInBytes: number): Buffer { + return Buffer.alloc(sizeInBytes, 'testdata'); +} + +export function createTestFileFromBuffer(sizeInMb: number, path: string): void { + const buf = createTestBuffer(sizeInMb); + writeFileSync(path, buf); +} + +export function deleteTestFile(path: string): void { + unlinkSync(path); +} diff --git a/handwritten/storage/conformance-test/v4SignedUrl.ts b/handwritten/storage/conformance-test/v4SignedUrl.ts new file mode 100644 index 00000000000..ecf378bd7d6 --- /dev/null +++ b/handwritten/storage/conformance-test/v4SignedUrl.ts @@ -0,0 +1,287 @@ +/*! + * Copyright 2019 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as assert from 'assert'; +import {describe, it} from 'mocha'; +import * as fs from 'fs'; +import {OutgoingHttpHeaders} from 'http'; +import * as path from 'path'; +import * as sinon from 'sinon'; +import * as querystring from 'querystring'; + +import {Storage, GenerateSignedPostPolicyV4Options} from '../src/'; +import * as url from 'url'; +import {getDirName} from '../src/util.js'; + +export enum UrlStyle { + PATH_STYLE = 'PATH_STYLE', + VIRTUAL_HOSTED_STYLE = 'VIRTUAL_HOSTED_STYLE', + BUCKET_BOUND_HOSTNAME = 'BUCKET_BOUND_HOSTNAME', +} + +interface V4SignedURLTestCase { + description: string; + hostname?: string; + emulatorHostname?: string; + clientEndpoint?: string; + universeDomain?: string; + bucket: string; + object?: string; + urlStyle?: keyof typeof UrlStyle; + bucketBoundHostname?: string; + scheme?: 'https' | 'http'; + headers?: OutgoingHttpHeaders; + queryParameters?: {[key: string]: string}; + method: string; + expiration: number; + timestamp: string; + expectedUrl: string; + expectedCanonicalRequest: string; + expectedStringToSign: string; +} + +interface V4SignedPolicyTestCase { + description: string; + policyInput: PolicyInput; + policyOutput: PolicyOutput; +} + +interface PolicyInput { + scheme: 'https' | 'http'; + bucket: string; + object: string; + expiration: number; + timestamp: string; + urlStyle?: UrlStyle; + bucketBoundHostname?: string; + conditions?: Conditions; + fields?: {[key: string]: string}; +} + +interface Conditions { + contentLengthRange: [number, number]; + startsWith: [string, string]; + acl: string; +} + +interface PolicyOutput { + url: string; + fields: {[key: string]: string}; + expectedDecodedPolicy: string; +} + +interface FileAction { + [key: string]: 'read' | 'resumable' | 'write' | 'delete'; +} + +interface BucketAction { + [key: string]: 'list'; +} + +const testFile = fs.readFileSync( + path.join( + getDirName(), + '../../../conformance-test/test-data/v4SignedUrl.json' + ), + 'utf-8' +); + +const testCases = JSON.parse(testFile); +const v4SignedUrlCases: V4SignedURLTestCase[] = testCases.signingV4Tests; +const v4SignedPolicyCases: V4SignedPolicyTestCase[] = + testCases.postPolicyV4Tests; + +const SERVICE_ACCOUNT = path.join( + getDirName(), + '../../../conformance-test/fixtures/signing-service-account.json' +); + +let storage: Storage; + +describe('v4 conformance test', () => { + let fakeTimer: sinon.SinonFakeTimers; + + beforeEach(() => { + delete process.env.STORAGE_EMULATOR_HOST; + }); + + afterEach(() => { + fakeTimer.restore(); + }); + + describe('v4 signed url', () => { + v4SignedUrlCases.forEach(testCase => { + it(testCase.description, async () => { + const NOW = new Date(testCase.timestamp); + fakeTimer = sinon.useFakeTimers(NOW); + + if (testCase.emulatorHostname) { + process.env.STORAGE_EMULATOR_HOST = testCase.emulatorHostname; + } + + storage = new Storage({ + keyFilename: SERVICE_ACCOUNT, + apiEndpoint: testCase.clientEndpoint, + universeDomain: testCase.universeDomain, + }); + + const bucket = storage.bucket(testCase.bucket); + const expires = NOW.valueOf() + testCase.expiration * 1000; + const version = 'v4' as const; + const host = testCase.hostname + ? new URL( + (testCase.scheme ? testCase.scheme + '://' : '') + + testCase.hostname + ) + : undefined; + const origin = testCase.bucketBoundHostname + ? `${testCase.scheme}://${testCase.bucketBoundHostname}` + : undefined; + const {bucketBoundHostname, virtualHostedStyle} = parseUrlStyle( + testCase.urlStyle, + origin + ); + const extensionHeaders = testCase.headers; + const queryParams = testCase.queryParameters; + const baseConfig = { + extensionHeaders, + version, + expires, + cname: bucketBoundHostname, + virtualHostedStyle, + queryParams, + host, + } as const; + let signedUrl: string; + + if (testCase.object) { + const file = bucket.file(testCase.object); + + const action = ( + { + GET: 'read', + POST: 'resumable', + PUT: 'write', + DELETE: 'delete', + } as FileAction + )[testCase.method]; + + [signedUrl] = await file.getSignedUrl({ + action, + ...baseConfig, + }); + } else { + // bucket operation + const action = ( + { + GET: 'list', + } as BucketAction + )[testCase.method]; + + [signedUrl] = await bucket.getSignedUrl({ + action, + ...baseConfig, + }); + } + + const expected = new url.URL(testCase.expectedUrl); + const actual = new url.URL(signedUrl); + + assert.strictEqual(actual.origin, expected.origin); + assert.strictEqual(actual.pathname, expected.pathname); + // Order-insensitive comparison of query params + assert.deepStrictEqual( + querystring.parse(actual.search), + querystring.parse(expected.search) + ); + }); + }); + }); + + describe('v4 signed policy', () => { + v4SignedPolicyCases.forEach(testCase => { + it(testCase.description, async () => { + const input = testCase.policyInput; + const NOW = new Date(input.timestamp); + fakeTimer = sinon.useFakeTimers(NOW); + + storage = new Storage({ + keyFilename: SERVICE_ACCOUNT, + }); + + const bucket = storage.bucket(input.bucket); + const expires = NOW.valueOf() + input.expiration * 1000; + const options: GenerateSignedPostPolicyV4Options = { + expires, + }; + + const conditions: (string | number)[][] = []; + if (input.conditions) { + if (input.conditions.startsWith) { + const variable = input.conditions.startsWith[0]; + const prefix = input.conditions.startsWith[1]; + conditions.push(['starts-with', variable, prefix]); + } + + if (input.conditions.contentLengthRange) { + const min = input.conditions.contentLengthRange[0]; + const max = input.conditions.contentLengthRange[1]; + conditions.push(['content-length-range', min, max]); + } + } + + const origin = input.bucketBoundHostname + ? `${input.scheme}://${input.bucketBoundHostname}` + : undefined; + const {bucketBoundHostname, virtualHostedStyle} = parseUrlStyle( + input.urlStyle, + origin + ); + options.virtualHostedStyle = virtualHostedStyle; + options.bucketBoundHostname = bucketBoundHostname; + options.fields = input.fields; + options.conditions = conditions; + + const file = bucket.file(input.object); + const [policy] = await file.generateSignedPostPolicyV4(options); + + assert.strictEqual(policy.url, testCase.policyOutput.url); + const outputFields = testCase.policyOutput.fields; + const decodedPolicy = JSON.parse( + Buffer.from(policy.fields.policy, 'base64').toString() + ); + assert.deepStrictEqual( + decodedPolicy, + JSON.parse(testCase.policyOutput.expectedDecodedPolicy) + ); + + assert.deepStrictEqual(policy.fields, outputFields); + }); + }); + }); +}); + +function parseUrlStyle( + style?: keyof typeof UrlStyle, + origin?: string +): {bucketBoundHostname?: string; virtualHostedStyle?: boolean} { + if (style === UrlStyle.BUCKET_BOUND_HOSTNAME) { + return {bucketBoundHostname: origin}; + } else if (style === UrlStyle.VIRTUAL_HOSTED_STYLE) { + return {virtualHostedStyle: true}; + } else { + return {virtualHostedStyle: false}; + } +} diff --git a/handwritten/storage/internal-tooling/README.md b/handwritten/storage/internal-tooling/README.md new file mode 100644 index 00000000000..9a40bb4c97a --- /dev/null +++ b/handwritten/storage/internal-tooling/README.md @@ -0,0 +1,43 @@ +# nodejs-storage benchmarking + +**This is not a supported Google product** + +This benchmarking script intended for use by Storage client library maintainers to benchmark various workloads and collect metrics in order to improve performance of the library. Currently the benchmarking runs a Write-1-Read-3 workload and measures throughput. + +## Run example: +This runs 10K iterations of Write-1-Read-3 on 5KiB to 2GiB files, and generates output to a CSV file: +```bash +$ cd nodejs-storage +$ npm install +$ cd build/internal-tooling +$ node performanceTest.js --iterations 10000 +``` + +## CLI parameters + +| Parameter | Description | Possible values | Default | +| --------- | ----------- | --------------- |:-------:| +| --iterations | number of iterations to run | any positive integer | `100` | +| --numthreads | number of threads to run | any positive integer | `1` | +| --bucket | bucket to upload/download to/from | any string bucket name | `nodejs-perf-metrics` | +| --small | number of bytes for lower bound file size | any positive integer | `5120` | +| --large | number of bytes for upper bound file size | any positive integer | `2.147e9` | +| --projectid | project ID to use | any string project ID | `undefined` | + + +## Workload definition and CSV headers + +For each invocation of the benchmark, write a new object of random size between `small` and `large` . After the successful write, download the object in full three times. For each of the 4 operations record the following fields: + +| Field | Description | +| ----- | ----------- | +| Op | the name of the operations (WRITE, READ[{0,1,2}]) | +| ObjectSize | the number of bytes of the object | +| LibBufferSize | configured to use the library default of 100 MiB | +| Crc32cEnabled | whether crc32c was computed for the operation | +| MD5Enabled | whether MD5 was computed for the operation | +| ApiName | default to JSON| +| ElapsedTimeUs | the elapsed time in microseconds the operation took | +| Status | completion state of the operation [OK, FAIL] | +| AppBufferSize | N/A | +| CpuTimeUs | N/A | \ No newline at end of file diff --git a/handwritten/storage/internal-tooling/helpers/package.cjs.json b/handwritten/storage/internal-tooling/helpers/package.cjs.json new file mode 100644 index 00000000000..6a0d2ef2aa8 --- /dev/null +++ b/handwritten/storage/internal-tooling/helpers/package.cjs.json @@ -0,0 +1,3 @@ +{ + "type": "commonjs" +} \ No newline at end of file diff --git a/handwritten/storage/internal-tooling/performApplicationPerformanceTest.ts b/handwritten/storage/internal-tooling/performApplicationPerformanceTest.ts new file mode 100644 index 00000000000..99b0f151f93 --- /dev/null +++ b/handwritten/storage/internal-tooling/performApplicationPerformanceTest.ts @@ -0,0 +1,191 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable n/no-unsupported-features/node-builtins */ +import yargs from 'yargs'; +import {promises as fsp, rmSync} from 'fs'; +import { + Bucket, + DownloadOptions, + DownloadResponse, + UploadOptions, + UploadResponse, +} from '../src/index.js'; +import {performance} from 'perf_hooks'; +import {parentPort} from 'worker_threads'; +import { + NODE_DEFAULT_HIGHWATER_MARK_BYTES, + generateRandomDirectoryStructure, + getValidationType, + performanceTestSetup, + TestResult, + performanceTestCommand, + getLowHighFileSize, + PERFORMANCE_TEST_TYPES, +} from './performanceUtils.js'; + +const TEST_NAME_STRING = 'nodejs-perf-metrics-application'; + +let bucket: Bucket; + +const checkType = getValidationType(); + +const argv = yargs(process.argv.slice(2)) + .command(performanceTestCommand) + .parseSync(); + +/** + * Main entry point. This function performs a test iteration and posts the message back + * to the parent thread. + */ +async function main() { + let result: TestResult | undefined = undefined; + + ({bucket} = await performanceTestSetup( + argv.project! as string, + argv.bucket! as string + )); + + switch (argv.test_type) { + case PERFORMANCE_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS: + result = await performWriteTest(); + break; + case PERFORMANCE_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS: + result = await performReadTest(); + break; + // case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD: + // result = await performLargeReadTest(); + // break; + default: + break; + } + parentPort?.postMessage(result); +} + +async function uploadInParallel( + bucket: Bucket, + paths: string[], + options: UploadOptions +) { + const promises: Promise[] = []; + for (const index in paths) { + const path = paths[index]; + const stat = await fsp.lstat(path); + if (stat.isDirectory()) { + continue; + } + options.destination = path; + promises.push(bucket.upload(path, options)); + } + await Promise.all(promises).catch(console.error); +} + +async function downloadInParallel(bucket: Bucket, options: DownloadOptions) { + const promises: Promise[] = []; + const [files] = await bucket.getFiles(); + files.forEach(file => { + promises.push(file.download(options)); + }); + await Promise.all(promises).catch(console.error); +} + +/** + * Performs an iteration of the Write multiple objects test. + * + * @returns {Promise} Promise that resolves to a test result of an iteration. + */ +async function performWriteTest(): Promise { + await bucket.deleteFiles(); //start clean + + const fileSizeRange = getLowHighFileSize(argv.object_size as string); + const creationInfo = generateRandomDirectoryStructure( + argv.num_objects as number, + TEST_NAME_STRING, + fileSizeRange.low, + fileSizeRange.high + ); + + const start = performance.now(); + await uploadInParallel(bucket, creationInfo.paths, {validation: checkType}); + const end = performance.now(); + + await bucket.deleteFiles(); //cleanup files + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); + + const result: TestResult = { + op: 'WRITE', + objectSize: creationInfo.totalSizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + api: 'JSON', + elapsedTimeUs: Math.round((end - start) * 1000), + cpuTimeUs: -1, + status: 'OK', + chunkSize: creationInfo.totalSizeInBytes, + workers: argv.workers as number, + library: 'nodejs', + transferSize: creationInfo.totalSizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + return result; +} + +/** + * Performs an iteration of the read multiple objects test. + * + * @returns {Promise} Promise that resolves to an array of test results for the iteration. + */ +async function performReadTest(): Promise { + const fileSizeRange = getLowHighFileSize(argv.object_size as string); + await bucket.deleteFiles(); // start clean + const creationInfo = generateRandomDirectoryStructure( + argv.num_objects as number, + TEST_NAME_STRING, + fileSizeRange.low, + fileSizeRange.high + ); + await uploadInParallel(bucket, creationInfo.paths, {validation: checkType}); + + const start = performance.now(); + await downloadInParallel(bucket, {validation: checkType}); + const end = performance.now(); + + const result: TestResult = { + op: 'READ[0]', + objectSize: creationInfo.totalSizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + api: 'JSON', + elapsedTimeUs: Math.round((end - start) * 1000), + cpuTimeUs: -1, + status: 'OK', + chunkSize: creationInfo.totalSizeInBytes, + workers: argv.workers as number, + library: 'nodejs', + transferSize: creationInfo.totalSizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); + await bucket.deleteFiles({force: true}); //cleanup + return result; +} + +main(); diff --git a/handwritten/storage/internal-tooling/performPerformanceTest.ts b/handwritten/storage/internal-tooling/performPerformanceTest.ts new file mode 100644 index 00000000000..a23f547c83a --- /dev/null +++ b/handwritten/storage/internal-tooling/performPerformanceTest.ts @@ -0,0 +1,217 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yargs from 'yargs'; +import {performance} from 'perf_hooks'; +import {parentPort} from 'worker_threads'; +import * as path from 'path'; +import { + cleanupFile, + generateRandomFile, + generateRandomFileName, + getLowHighFileSize, + getValidationType, + NODE_DEFAULT_HIGHWATER_MARK_BYTES, + performanceTestCommand, + performanceTestSetup, + PERFORMANCE_TEST_TYPES, + TestResult, +} from './performanceUtils.js'; +import {Bucket} from '../src/index.js'; +import {getDirName} from '../src/util.js'; + +const TEST_NAME_STRING = 'nodejs-perf-metrics'; +const DEFAULT_NUMBER_OF_WRITES = 1; +const DEFAULT_NUMBER_OF_READS = 3; +const DEFAULT_RANGE_READS = 3; + +let bucket: Bucket; +const checkType = getValidationType(); + +const argv = yargs(process.argv.slice(2)) + .command(performanceTestCommand) + .parseSync(); + +/** + * Main entry point. This function performs a test iteration and posts the message back + * to the parent thread. + */ +async function main() { + let results: TestResult[] = []; + + ({bucket} = await performanceTestSetup( + argv.project! as string, + argv.bucket! as string + )); + + switch (argv.test_type) { + case PERFORMANCE_TEST_TYPES.WRITE_ONE_READ_THREE: + results = await performWriteReadTest(); + break; + case PERFORMANCE_TEST_TYPES.RANGE_READ: + results = await performRangedReadTest(); + break; + default: + break; + } + + parentPort?.postMessage(results); +} + +/** + * Performs an iteration of a ranged read test. Only the last result will be reported. + * + * @returns {Promise} Promise that resolves to an array of test results for the iteration. + */ +async function performRangedReadTest(): Promise { + const results: TestResult[] = []; + const fileSizeRange = getLowHighFileSize(argv.object_size as string); + const fileName = generateRandomFileName(TEST_NAME_STRING); + generateRandomFile( + fileName, + fileSizeRange.low, + fileSizeRange.high, + getDirName() + ); + const file = bucket.file(`${fileName}`); + const destinationFileName = generateRandomFileName(TEST_NAME_STRING); + const destination = path.join(getDirName(), destinationFileName); + + const iterationResult: TestResult = { + op: 'READ[0]', + objectSize: argv.range_read_size as number, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: false, + md5Enabled: false, + api: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: 'OK', + chunkSize: argv.range_read_size as number, + workers: argv.workers as number, + library: 'nodejs', + transferSize: argv.range_read_size as number, + transferOffset: 0, + bucketName: bucket.name, + }; + + await bucket.upload(`${getDirName()}/${fileName}`); + cleanupFile(fileName); + + for (let i = 0; i < DEFAULT_RANGE_READS; i++) { + const start = performance.now(); + await file.download({ + start: 0, + end: argv.range_read_size as number, + destination, + }); + const end = performance.now(); + cleanupFile(destinationFileName); + iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); + } + + await file.delete({ignoreNotFound: true}); + results.push(iterationResult); + return results; +} + +/** + * Performs an iteration of the Write 1 / Read 3 performance measuring test. + * + * @returns {Promise} Promise that resolves to an array of test results for the iteration. + */ +async function performWriteReadTest(): Promise { + const results: TestResult[] = []; + const fileSizeRange = getLowHighFileSize(argv.object_size as string); + const fileName = generateRandomFileName(TEST_NAME_STRING); + const file = bucket.file(`${fileName}`); + const sizeInBytes = generateRandomFile( + fileName, + fileSizeRange.low, + fileSizeRange.high, + getDirName() + ); + + for (let j = 0; j < DEFAULT_NUMBER_OF_WRITES; j++) { + let start = 0; + let end = 0; + + const iterationResult: TestResult = { + op: 'WRITE', + objectSize: sizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + api: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: 'OK', + chunkSize: sizeInBytes, + workers: argv.workers as number, + library: 'nodejs', + transferSize: sizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + + start = performance.now(); + await bucket.upload(`${getDirName()}/${fileName}`, {validation: checkType}); + end = performance.now(); + + iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); + results.push(iterationResult); + cleanupFile(fileName); + } + + const iterationResult: TestResult = { + op: 'READ[0]', + objectSize: sizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + api: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: 'OK', + chunkSize: sizeInBytes, + workers: argv.workers as number, + library: 'nodejs', + transferSize: sizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + + for (let j = 0; j < DEFAULT_NUMBER_OF_READS; j++) { + let start = 0; + let end = 0; + + const destinationFileName = generateRandomFileName(TEST_NAME_STRING); + const destination = path.join(getDirName(), destinationFileName); + + start = performance.now(); + await file.download({validation: checkType, destination}); + end = performance.now(); + + cleanupFile(destinationFileName); + iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); + } + + await file.delete({ignoreNotFound: true}); + results.push(iterationResult); + return results; +} + +main(); diff --git a/handwritten/storage/internal-tooling/performTransferManagerTest.ts b/handwritten/storage/internal-tooling/performTransferManagerTest.ts new file mode 100644 index 00000000000..097e999f752 --- /dev/null +++ b/handwritten/storage/internal-tooling/performTransferManagerTest.ts @@ -0,0 +1,279 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable n/no-unsupported-features/node-builtins */ +import yargs from 'yargs'; +import {parentPort} from 'worker_threads'; +import {Bucket, File, TransferManager} from '../src/index.js'; +import { + cleanupFile, + generateRandomDirectoryStructure, + generateRandomFile, + generateRandomFileName, + getValidationType, + NODE_DEFAULT_HIGHWATER_MARK_BYTES, + performanceTestSetup, + TestResult, + performanceTestCommand, + getLowHighFileSize, + PERFORMANCE_TEST_TYPES, +} from './performanceUtils.js'; +import {performance} from 'perf_hooks'; +import {rmSync} from 'fs'; +import * as path from 'path'; +import {getDirName} from '../src/util.js'; + +const TEST_NAME_STRING = 'tm-perf-metrics'; +const DIRECTORY_PROBABILITY = 0.1; + +let bucket: Bucket; +let transferManager: TransferManager; +const checkType = getValidationType(); + +const argv = yargs(process.argv.slice(2)) + .command(performanceTestCommand) + .parseSync(); + +/** + * Main entry point. This function performs a test iteration and posts the message back + * to the parent thread. + */ +async function main() { + let results: TestResult[] = []; + + ({bucket, transferManager} = await performanceTestSetup( + argv.project! as string, + argv.bucket! as string + )); + + switch (argv.test_type) { + case PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MANY_FILES: + results = await performUploadManyFilesTest(); + break; + case PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MANY_FILES: + results = await performDownloadManyFilesTest(); + break; + case PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_CHUNKED_FILE_DOWNLOAD: + results = await performChunkUploadDownloadTest(); + break; + default: + break; + } + parentPort?.postMessage(results); +} + +/** + * Cleans up after a test is complete by removing files from the bucket + */ +async function performTestCleanup(fileOrFiles: File[] | File | string[]) { + const filesToDelete = Array.isArray(fileOrFiles) + ? fileOrFiles + : [fileOrFiles]; + const promises = filesToDelete.map(f => { + let fileToDelete = f; + if (typeof f === 'string') { + fileToDelete = bucket.file(f); + } + (fileToDelete as File).delete({ignoreNotFound: true}); + }); + return Promise.all(promises); +} + +/** + * Performs a test where multiple objects are uploaded in parallel to a bucket. + * + * @returns {Promise} A promise that resolves containing information about the test results. + */ +async function performUploadManyFilesTest(): Promise { + const fileSizeRange = getLowHighFileSize(argv.object_size as string); + const creationInfo = generateRandomDirectoryStructure( + argv.num_objects as number, + TEST_NAME_STRING, + fileSizeRange.low, + fileSizeRange.high, + DIRECTORY_PROBABILITY + ); + + const start = performance.now(); + await transferManager.uploadManyFiles(creationInfo.paths, { + concurrencyLimit: argv.workers as number, + passthroughOptions: { + validation: checkType, + }, + }); + const end = performance.now(); + + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); + + const result: TestResult = { + op: 'WRITE', + objectSize: creationInfo.totalSizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + api: 'JSON', + elapsedTimeUs: Math.round((end - start) * 1000), + cpuTimeUs: -1, + status: 'OK', + chunkSize: creationInfo.totalSizeInBytes, + workers: argv.workers as number, + library: 'nodejs', + transferSize: creationInfo.totalSizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + + await performTestCleanup(creationInfo.paths); + + return [result]; +} + +/** + * Performs a test where multiple objects are downloaded in parallel from a bucket. + * + * @returns {Promise} A promise that resolves containing information about the test results. + */ +async function performDownloadManyFilesTest(): Promise { + const fileSizeRange = getLowHighFileSize(argv.object_size as string); + const creationInfo = generateRandomDirectoryStructure( + argv.num_objects as number, + TEST_NAME_STRING, + fileSizeRange.low, + fileSizeRange.high, + DIRECTORY_PROBABILITY + ); + + await transferManager.uploadManyFiles(creationInfo.paths, { + concurrencyLimit: argv.workers as number, + passthroughOptions: { + validation: checkType, + }, + }); + const start = performance.now(); + await transferManager.downloadManyFiles(TEST_NAME_STRING, { + prefix: path.join(getDirName(), '..', '..'), + concurrencyLimit: argv.workers as number, + passthroughOptions: { + validation: checkType, + }, + }); + const end = performance.now(); + + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); + + const result: TestResult = { + op: 'READ[0]', + objectSize: creationInfo.totalSizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + api: 'JSON', + elapsedTimeUs: Math.round((end - start) * 1000), + cpuTimeUs: -1, + status: 'OK', + chunkSize: creationInfo.totalSizeInBytes, + workers: argv.workers as number, + library: 'nodejs', + transferSize: creationInfo.totalSizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + + await performTestCleanup(creationInfo.paths); + + return [result]; +} + +/** + * Performs a test where a large file is uploaded and downloaded as chunks in parallel. + * + * @returns {Promise} A promise that resolves containing information about the test results. + */ +async function performChunkUploadDownloadTest(): Promise { + const results: TestResult[] = []; + const fileSizeRange = getLowHighFileSize(argv.object_size as string); + const fileName = generateRandomFileName(TEST_NAME_STRING); + const sizeInBytes = generateRandomFile( + fileName, + fileSizeRange.low, + fileSizeRange.high, + getDirName() + ); + const file = bucket.file(`${fileName}`); + let result: TestResult = { + op: 'WRITE', + objectSize: sizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: false, + api: 'JSON', + elapsedTimeUs: -1, + cpuTimeUs: -1, + status: 'OK', + chunkSize: argv.range_read_size as number, + workers: argv.workers as number, + library: 'nodejs', + transferSize: sizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + + let start = performance.now(); + await transferManager.uploadFileInChunks(`${getDirName()}/${fileName}`, { + concurrencyLimit: argv.workers as number, + chunkSizeBytes: argv.range_read_size as number, + }); + let end = performance.now(); + result.elapsedTimeUs = Math.round((end - start) * 1000); + results.push(result); + cleanupFile(fileName); + + start = performance.now(); + await transferManager.downloadFileInChunks(file, { + concurrencyLimit: argv.workers as number, + chunkSizeBytes: argv.range_read_size as number, + destination: path.join(getDirName(), fileName), + validation: checkType === 'crc32c' ? checkType : false, + }); + end = performance.now(); + + cleanupFile(fileName); + + result = { + op: 'READ[0]', + objectSize: sizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: false, + api: 'JSON', + elapsedTimeUs: Math.round((end - start) * 1000), + cpuTimeUs: -1, + status: 'OK', + chunkSize: argv.range_read_size as number, + workers: argv.workers as number, + library: 'nodejs', + transferSize: sizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + results.push(result); + + await performTestCleanup(file); + + return results; +} + +main(); diff --git a/handwritten/storage/internal-tooling/performanceTest.ts b/handwritten/storage/internal-tooling/performanceTest.ts new file mode 100644 index 00000000000..995eaf96784 --- /dev/null +++ b/handwritten/storage/internal-tooling/performanceTest.ts @@ -0,0 +1,133 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable n/no-process-exit */ +import yargs from 'yargs'; +import {appendFile} from 'fs/promises'; +import {Worker} from 'worker_threads'; +import { + convertToCloudMonitoringFormat, + TestResult, + log, + performanceTestCommand, + PERFORMANCE_TEST_TYPES, +} from './performanceUtils.js'; +import {getDirName} from '../src/util.js'; + +const argv = yargs(process.argv.slice(2)) + .command(performanceTestCommand) + .parseSync(); + +let iterationsRemaining: number = argv.samples as number; + +/** + * Main entry point for performing a Write 1 Read 3 performance measurement test. + * This function will create the number of threads supplied in the numthreads argument or + * default to 1 if no argument is provided. The test will be run for the number of iterations + * specified by the iterations parameter or 100 if not specified. + */ +function main() { + let numThreads: number = argv.workers as number; + if (numThreads > iterationsRemaining) { + log( + `${numThreads} is greater than number of iterations (${iterationsRemaining}). Using ${iterationsRemaining} threads instead.`, + argv.debug as boolean + ); + numThreads = iterationsRemaining; + } + if (argv.test_type !== PERFORMANCE_TEST_TYPES.WRITE_ONE_READ_THREE) { + numThreads = 1; + } + for (let i = 0; i < numThreads; i++) { + createWorker(); + } +} + +/** + * Creates a new worker thread and performs a test iteration in that worker. + * When the worker passes back the results, they are appended to the results file. + */ +function createWorker() { + const dirName = getDirName().replace('/src', '/internal-tooling'); + iterationsRemaining--; + log( + `Starting new iteration. Current iterations remaining: ${iterationsRemaining}`, + argv.debug as boolean + ); + let testPath = ''; + if ( + argv.test_type === PERFORMANCE_TEST_TYPES.WRITE_ONE_READ_THREE || + argv.test_type === PERFORMANCE_TEST_TYPES.RANGE_READ + ) { + testPath = `${dirName}/performPerformanceTest.js`; + } else if ( + argv.test_type === + PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MANY_FILES || + argv.test_type === + PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_CHUNKED_FILE_DOWNLOAD || + argv.test_type === + PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MANY_FILES + ) { + testPath = `${dirName}/performTransferManagerTest.js`; + } else if ( + argv.test_type === + PERFORMANCE_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS || + argv.test_type === PERFORMANCE_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD || + argv.test_type === + PERFORMANCE_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS + ) { + testPath = `${dirName}/performApplicationPerformanceTest.js`; + } + + const w = new Worker(testPath, { + argv: process.argv.slice(2), + }); + + w.on('message', data => { + log('Successfully completed iteration.', argv.debug as boolean); + recordResult(data); + if (iterationsRemaining > 0) { + createWorker(); + } + }); + w.on('error', e => { + log(e, true, true); + // BBMC will not report errors unless the process is terminated with a non zero code. + // eslint-disable-next-line no-process-exit + process.exit(1); + }); +} + +/** + * Records the test result to the appropriate place based on specified command line arguments. + * + * @param {TestResult[]} results result of a test iteration. + */ +async function recordResult(results: TestResult[] | TestResult) { + const resultsToAppend: TestResult[] = Array.isArray(results) + ? results + : [results]; + + for await (const outputString of convertToCloudMonitoringFormat( + resultsToAppend + )) { + argv.file_name + ? await appendFile(argv.file_name as string, `${outputString}\n`) + : log(outputString, true); + } +} + +main(); diff --git a/handwritten/storage/internal-tooling/performanceUtils.ts b/handwritten/storage/internal-tooling/performanceUtils.ts new file mode 100644 index 00000000000..b0ff2d1d587 --- /dev/null +++ b/handwritten/storage/internal-tooling/performanceUtils.ts @@ -0,0 +1,410 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {execSync} from 'child_process'; +import {mkdirSync, mkdtempSync, unlinkSync} from 'fs'; +import * as path from 'path'; +import * as yargs from 'yargs'; +import {Bucket, Storage, TransferManager} from '../src/index.js'; +import {getDirName} from '../src/util.js'; + +export const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; +export const DEFAULT_DIRECTORY_PROBABILITY = 0.1; +export const DEFAULT_NUMBER_OF_OBJECTS = 1000; + +export const OUTPUT_FORMATS = { + CLOUD_MONITORING: 'cloud-monitoring', +} as const; + +export const PERFORMANCE_TEST_TYPES = { + WRITE_ONE_READ_THREE: 'w1r3', + RANGE_READ: 'range-read', + TRANSFER_MANAGER_UPLOAD_MANY_FILES: 'tm-upload', + TRANSFER_MANAGER_DOWNLOAD_MANY_FILES: 'tm-download', + TRANSFER_MANAGER_CHUNKED_FILE_DOWNLOAD: 'tm-chunked', + APPLICATION_LARGE_FILE_DOWNLOAD: 'application-large', + APPLICATION_UPLOAD_MULTIPLE_OBJECTS: 'application-upload', + APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS: 'application-download', +} as const; + +const APIS = { + JSON: 'json', +} as const; + +const DEFAULT_SAMPLES = 8000; +const DEFAULT_WORKERS = 16; +const SSB_SIZE_THRESHOLD_BYTES = 1048576; +const DEFAULT_OBJECT_RANGE_SIZE_BYTES = '1048576..1048576'; +const DEFAULT_RANGE_READ_SIZE_BYTES = 0; //0 means read the full object +const DEFAULT_MINIMUM_READ_OFFSET_BYTES = 0; +const DEFAULT_MAXIMUM_READ_OFFSET_BYTES = 0; + +export interface TestResult { + library: 'nodejs'; + op: 'WRITE' | 'READ[0]' | 'READ[1]' | 'READ[2]'; + objectSize: number; + transferSize: number; + transferOffset: number; + appBufferSize: number; + crc32cEnabled: boolean; + md5Enabled: boolean; + api: 'JSON' | 'XML' | 'GRPC' | 'DirectPath'; + cpuTimeUs: number; + status: 'OK' | 'FAIL' | 'TIMEOUT'; + chunkSize: number; + workers: number; + bucketName: string; + elapsedTimeUs: number; +} + +export interface Arguments { + project?: string; + bucket?: string; + output_type: string; + samples: number; + workers: number; + api: string; + object_size: string; + range_read_size: number; + minimum_read_offset: number; + maximum_read_offset: number; + debug: boolean; + file_name: string | undefined; + num_objects: number; + test_type: string; +} + +export const performanceTestCommand: yargs.CommandModule<{}, Arguments> = { + command: 'performance-test', + builder(yargs) { + return yargs + .option('project', {type: 'string', demand: true}) + .option('bucket', {type: 'string', demand: true}) + .option('output_type', { + type: 'string', + choices: [OUTPUT_FORMATS.CLOUD_MONITORING], + default: OUTPUT_FORMATS.CLOUD_MONITORING, + }) + .option('samples', {type: 'number', default: DEFAULT_SAMPLES}) + .option('workers', {type: 'number', default: DEFAULT_WORKERS}) + .option('api', { + type: 'string', + choices: [APIS.JSON], + default: APIS.JSON, + }) + .option('object_size', { + type: 'string', + default: DEFAULT_OBJECT_RANGE_SIZE_BYTES, + }) + .option('range_read_size', { + type: 'number', + default: DEFAULT_RANGE_READ_SIZE_BYTES, + }) + .option('minimum_read_offset', { + type: 'number', + default: DEFAULT_MINIMUM_READ_OFFSET_BYTES, + }) + .option('maximum_read_offset', { + type: 'number', + default: DEFAULT_MAXIMUM_READ_OFFSET_BYTES, + }) + .option('debug', {type: 'boolean', default: false}) + .option('file_name', {type: 'string'}) + .option('num_objects', { + type: 'number', + default: DEFAULT_NUMBER_OF_OBJECTS, + }) + .option('test_type', { + type: 'string', + choices: [ + PERFORMANCE_TEST_TYPES.WRITE_ONE_READ_THREE, + PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MANY_FILES, + PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MANY_FILES, + PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_CHUNKED_FILE_DOWNLOAD, + PERFORMANCE_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS, + PERFORMANCE_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD, + PERFORMANCE_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS, + ], + default: PERFORMANCE_TEST_TYPES.WRITE_ONE_READ_THREE, + }); + }, + async handler() {}, +}; + +export interface RandomDirectoryCreationInformation { + paths: string[]; + totalSizeInBytes: number; +} + +export interface PerformanceTestSetupResults { + storage: Storage; + bucket: Bucket; + transferManager: TransferManager; +} + +/** + * Create a uniformly distributed random integer beween the inclusive min and max provided. + * + * @param {number} minInclusive lower bound (inclusive) of the range of random integer to return. + * @param {number} maxInclusive upper bound (inclusive) of the range of random integer to return. + * @returns {number} returns a random integer between minInclusive and maxInclusive + */ +export function randomInteger(minInclusive: number, maxInclusive: number) { + // Utilizing Math.random will generate uniformly distributed random numbers. + return ( + Math.floor(Math.random() * (maxInclusive - minInclusive + 1)) + minInclusive + ); +} + +/** + * Returns a boolean value with the provided probability + * + * @param {number} trueProbablity the probability the value will be true + * + * @returns {boolean} a boolean value with the probablity provided. + */ +export function weightedRandomBoolean(trueProbablity: number): boolean { + return Math.random() <= trueProbablity ? true : false; +} + +/** + * Return a string of 6 random characters + * + * @returns {string} a random string value with length of 6 + */ +export function randomString(): string { + return Math.random().toString(36).slice(-6); +} + +/** + * Creates a random file name by appending a UUID to the baseName. + * + * @param {string} baseName the base file name. A random uuid will be appended to this value. + * + * @returns {string} random file name that was generated. + */ +export function generateRandomFileName(baseName: string): string { + return `${baseName}.${randomString()}`; +} + +/** + * Creates a file with a size between the small (default 5120 bytes) and large (2.147e9 bytes) parameters. + * The file is filled with random data. + * + * @param {string} fileName name of the file to generate. + * @param {number} fileSizeLowerBoundBytes minimum size of file to generate. + * @param {number} fileSizeUpperBoundBytes maximum size of file to generate. + * @param {string} currentDirectory the directory in which to generate the file. + * + * @returns {number} the size of the file generated. + */ +export function generateRandomFile( + fileName: string, + fileSizeLowerBoundBytes: number = getLowHighFileSize( + DEFAULT_OBJECT_RANGE_SIZE_BYTES + ).low, + fileSizeUpperBoundBytes: number = getLowHighFileSize( + DEFAULT_OBJECT_RANGE_SIZE_BYTES + ).high, + currentDirectory: string = mkdtempSync(randomString()) +): number { + const fileSizeBytes = randomInteger( + fileSizeLowerBoundBytes, + fileSizeUpperBoundBytes + ); + + execSync( + `head --bytes=${fileSizeBytes} /dev/urandom > ${currentDirectory}/${fileName}` + ); + + return fileSizeBytes; +} + +/** + * Creates a random directory structure consisting of subdirectories and random files. + * + * @param {number} maxObjects the total number of subdirectories and files to generate. + * @param {string} baseName the starting directory under which everything else is added. File names will have this value prepended. + * @param {number} fileSizeLowerBoundBytes minimum size of file to generate. + * @param {number} fileSizeUpperBoundBytes maximum size of file to generate. + * + * @returns {array} an array of all the generated paths + */ +export function generateRandomDirectoryStructure( + maxObjects: number, + baseName: string, + fileSizeLowerBoundBytes: number = getLowHighFileSize( + DEFAULT_OBJECT_RANGE_SIZE_BYTES + ).low, + fileSizeUpperBoundBytes: number = getLowHighFileSize( + DEFAULT_OBJECT_RANGE_SIZE_BYTES + ).high, + directoryProbability: number = DEFAULT_DIRECTORY_PROBABILITY +): RandomDirectoryCreationInformation { + let curPath = baseName; + const creationInfo: RandomDirectoryCreationInformation = { + paths: [], + totalSizeInBytes: 0, + }; + + mkdirSync(curPath); + for (let i = 0; i < maxObjects; i++) { + if (weightedRandomBoolean(directoryProbability)) { + curPath = path.join(curPath, randomString()); + mkdirSync(curPath, {recursive: true}); + creationInfo.paths.push(curPath); + } else { + const randomName = randomString(); + creationInfo.totalSizeInBytes += generateRandomFile( + randomName, + fileSizeLowerBoundBytes, + fileSizeUpperBoundBytes, + curPath + ); + creationInfo.paths.push(path.join(curPath, randomName)); + } + } + + return creationInfo; +} + +/** + * Deletes the file specified by the fileName parameter. + * + * @param {string} fileName name of the file to delete. + */ +export function cleanupFile( + fileName: string, + directoryName: string = getDirName() +): void { + unlinkSync(`${directoryName}/${fileName}`); +} + +/** + * Creates the necessary structures for performing a performance test. + * + * @param {string} projectId the project ID to use. + * @param {string} bucketName the name of the bucket to use. + * @returns {object} object containing the created storage, bucket, and transfer manager instance. + */ +export async function performanceTestSetup( + projectId: string, + bucketName: string +): Promise { + const storage = new Storage({projectId}); + const bucket = storage.bucket(bucketName, { + preconditionOpts: { + ifGenerationMatch: 0, + }, + }); + if (!(await bucket.exists())[0]) { + await bucket.create(); + } + const transferManager = new TransferManager(bucket); + return { + storage, + bucket, + transferManager, + }; +} + +/** + * Randomly returns the type of validation check to run on upload / download + * + * @returns {string | boolean | undefined} the type of validation to run (crc32c, md5, or none). + */ +export function getValidationType(): 'md5' | 'crc32c' | boolean | undefined { + const checkType = randomInteger(0, 2); + if (checkType === 0) { + return false; + } else if (checkType === 1) { + return 'crc32c'; + } else { + return 'md5'; + } +} + +/** + * Converts the supplied test results from javascript objects to a cloud monitoring formatted string. + * + * @param {TestResult[]} results An array of test iteration result objects that will be converted to cloud monitoring format. + * @param {string} bucket The bucket name used for the test. + * + * @returns {AsyncGenerator} A string containing the results of the conversion to cloud monitoring format. + */ +export async function* convertToCloudMonitoringFormat( + results: TestResult[] +): AsyncGenerator { + for (const curResult of results) { + const throughput = + // If the object size is greater than the defined threshold, report in MiB/s, otherwise report in KiB/s. + curResult.objectSize >= SSB_SIZE_THRESHOLD_BYTES + ? curResult.objectSize / + 1024 / + 1024 / + (curResult.elapsedTimeUs / 1000000) + : curResult.objectSize / 1024 / (curResult.elapsedTimeUs / 1000000); + yield `throughput{\ + library="${curResult.library}",\ + api="${curResult.api}",\ + op="${curResult.op}",\ + object_size="${curResult.objectSize}",\ + transfer_offset="${curResult.transferOffset}",\ + transfer_size="${curResult.chunkSize}",\ + app_buffer_size="${curResult.appBufferSize}",\ + crc32c_enabled="${curResult.crc32cEnabled}",\ + md5_enabled="${curResult.md5Enabled}",\ + cpu_time_us="${curResult.cpuTimeUs}",\ + bucket_name="${curResult.bucketName}",\ + workers="${curResult.workers}",\ + status_code="${curResult.status}"} ${throughput}`; + } +} + +/** + * Logs the provided message if debugging is enabled. + * + * @param {string | error} messageOrError the message or error object to be printed utilizing console.debug. + * @param {boolean} shouldLog flag indicating whether or not the message should be logged to stdout or stderr. + * @param {boolean} isError if set to true and shouldLog is true, write the output to stderr instead of stdout. + */ +export function log( + messageOrError: string | Error, + shouldLog: boolean, + isError = false +): void { + if (shouldLog) { + isError ? console.error(messageOrError) : console.log(messageOrError); + } +} + +/** + * Converts the provided rangeSize from string format to an object containing the low and high size values. + * + * @param {string} rangeSize a string in the format low..high. + * + * @returns {object} An object containing integers low and high. + */ +export function getLowHighFileSize(rangeSize: string): { + low: number; + high: number; +} { + const split = rangeSize.split('..'); + return { + low: parseInt(split[0]), + high: parseInt(split[1]), + }; +} diff --git a/handwritten/storage/linkinator.config.json b/handwritten/storage/linkinator.config.json new file mode 100644 index 00000000000..165d1cbf45c --- /dev/null +++ b/handwritten/storage/linkinator.config.json @@ -0,0 +1,12 @@ +{ + "recurse": true, + "skip": [ + "https://codecov.io/gh/googleapis/", + "www.googleapis.com", + "img.shields.io", + "node_modules/", + "CHANGELOG.md" + ], + "silent": true, + "concurrency": 10 +} diff --git a/handwritten/storage/owlbot.py b/handwritten/storage/owlbot.py new file mode 100644 index 00000000000..647f898996e --- /dev/null +++ b/handwritten/storage/owlbot.py @@ -0,0 +1,43 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import synthtool as s +import synthtool.gcp as gcp +import synthtool.languages.node_mono_repo as node +import logging + +logging.basicConfig(level=logging.DEBUG) + +common_templates = gcp.CommonTemplates() +templates = common_templates.node_mono_repo_library(relative_dir="handwriten/storage", source_location='build/src') +s.copy(templates, destination="handwritten/storage", excludes=['.jsdoc.js', + '.github/release-please.yml', + '.github/sync-repo-settings.yaml', + '.github/workflows/ci.yaml', + '.prettierrc.js', + '.mocharc.js', + '.kokoro/continuous/node14/system-test.cfg', + '.kokoro/presubmit/node14/system-test.cfg', + '.kokoro/release/publish.cfg', + '.kokoro/system-test.sh', + '.kokoro/samples-test.sh', + ]) + +# Create .config directory under $HOME to get around permissions issues +# with resumable upload. +s.replace( + "handwriten/storage/.circleci/config.yml", + "command: npm run system-test", + "command: mkdir $HOME/.config && npm run system-test") +node.fix_hermetic(relative_dir="handwriten/storage") diff --git a/handwritten/storage/package.json b/handwritten/storage/package.json new file mode 100644 index 00000000000..63a55ea7373 --- /dev/null +++ b/handwritten/storage/package.json @@ -0,0 +1,136 @@ +{ + "name": "@google-cloud/storage", + "description": "Cloud Storage Client Library for Node.js", + "version": "7.19.0", + "license": "Apache-2.0", + "author": "Google Inc.", + "engines": { + "node": ">=14" + }, + "repository": { + "type": "git", + "directory": "handwritten/storage", + "url": "https://github.com/googleapis/google-cloud-node.git" + }, + "main": "./build/cjs/src/index.js", + "types": "./build/cjs/src/index.d.ts", + "type": "module", + "exports": { + ".": { + "import": { + "types": "./build/esm/src/index.d.ts", + "default": "./build/esm/src/index.js" + }, + "require": { + "types": "./build/cjs/src/index.d.ts", + "default": "./build/cjs/src/index.js" + } + } + }, + "files": [ + "build/cjs/src", + "build/cjs/package.json", + "!build/cjs/src/**/*.map", + "build/esm/src", + "!build/esm/src/**/*.map" + ], + "keywords": [ + "google apis client", + "google api client", + "google apis", + "google api", + "google", + "google cloud platform", + "google cloud", + "cloud", + "google storage", + "storage" + ], + "scripts": { + "all-test": "npm test && npm run system-test && npm run samples-test", + "benchwrapper": "node bin/benchwrapper.js", + "check": "gts check", + "clean": "rm -rf build/", + "compile:cjs": "tsc -p ./tsconfig.cjs.json", + "compile:esm": "tsc -p .", + "compile": "npm run compile:cjs && npm run compile:esm", + "conformance-test": "mocha --parallel build/cjs/conformance-test/ --require build/cjs/conformance-test/globalHooks.js", + "docs-test": "linkinator docs", + "docs": "jsdoc -c .jsdoc.json", + "fix": "gts fix", + "lint": "gts check", + "postcompile": "cp ./src/package-json-helper.cjs ./build/cjs/src && cp ./src/package-json-helper.cjs ./build/esm/src", + "postcompile:cjs": "babel --plugins gapic-tools/build/src/replaceImportMetaUrl,gapic-tools/build/src/toggleESMFlagVariable build/cjs/src/util.js -o build/cjs/src/util.js && cp internal-tooling/helpers/package.cjs.json build/cjs/package.json", + "precompile": "rm -rf build/", + "preconformance-test": "npm run compile:cjs -- --sourceMap", + "predocs-test": "npm run docs", + "predocs": "npm run compile:cjs -- --sourceMap", + "prelint": "cd samples; npm link ../; npm install", + "prepare": "npm run compile", + "presystem-test:esm": "npm run compile:esm", + "presystem-test": "npm run compile -- --sourceMap", + "pretest": "npm run compile -- --sourceMap", + "samples-test": "npm link && cd samples/ && npm link ../ && npm test && cd ../", + "system-test:esm": "mocha build/esm/system-test --timeout 600000 --exit", + "system-test": "mocha build/cjs/system-test --timeout 600000 --exit", + "test": "c8 mocha build/cjs/test" + }, + "dependencies": { + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "<4.1.0", + "abort-controller": "^3.0.0", + "async-retry": "^1.3.3", + "duplexify": "^4.1.3", + "fast-xml-parser": "^5.3.4", + "gaxios": "^6.0.2", + "google-auth-library": "^9.6.3", + "html-entities": "^2.5.2", + "mime": "^3.0.0", + "p-limit": "^3.0.1", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0", + "uuid": "^8.0.0" + }, + "devDependencies": { + "@babel/cli": "^7.22.10", + "@babel/core": "^7.22.11", + "@google-cloud/pubsub": "^4.0.0", + "@grpc/grpc-js": "^1.0.3", + "@grpc/proto-loader": "^0.8.0", + "@types/async-retry": "^1.4.3", + "@types/duplexify": "^3.6.4", + "@types/mime": "^3.0.0", + "@types/mocha": "^9.1.1", + "@types/mockery": "^1.4.29", + "@types/node": "^24.0.0", + "@types/node-fetch": "^2.1.3", + "@types/proxyquire": "^1.3.28", + "@types/request": "^2.48.4", + "@types/sinon": "^17.0.0", + "@types/tmp": "0.2.6", + "@types/uuid": "^8.0.0", + "@types/yargs": "^17.0.10", + "c8": "^9.0.0", + "form-data": "^4.0.4", + "gapic-tools": "^0.4.0", + "gts": "^5.0.0", + "jsdoc": "^4.0.4", + "jsdoc-fresh": "^5.0.0", + "jsdoc-region-tag": "^4.0.0", + "linkinator": "^3.0.0", + "mocha": "^9.2.2", + "mockery": "^2.1.0", + "nock": "~13.5.0", + "node-fetch": "^2.6.7", + "pack-n-play": "^2.0.0", + "proxyquire": "^2.1.3", + "sinon": "^18.0.0", + "nise": "6.0.0", + "path-to-regexp": "6.3.0", + "tmp": "^0.2.0", + "typescript": "^5.1.6", + "yargs": "^17.3.1" + }, + "homepage": "https://github.com/googleapis/google-cloud-node/tree/main/handwritten/storage" +} diff --git a/handwritten/storage/src/acl.ts b/handwritten/storage/src/acl.ts new file mode 100644 index 00000000000..9776b0340e0 --- /dev/null +++ b/handwritten/storage/src/acl.ts @@ -0,0 +1,914 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + BodyResponseCallback, + DecorateRequestOptions, + BaseMetadata, +} from './nodejs-common/index.js'; +import {promisifyAll} from '@google-cloud/promisify'; + +export interface AclOptions { + pathPrefix: string; + request: ( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ) => void; +} + +export type GetAclResponse = [ + AccessControlObject | AccessControlObject[], + AclMetadata, +]; +export interface GetAclCallback { + ( + err: Error | null, + acl?: AccessControlObject | AccessControlObject[] | null, + apiResponse?: AclMetadata + ): void; +} +export interface GetAclOptions { + entity: string; + generation?: number; + userProject?: string; +} + +export interface UpdateAclOptions { + entity: string; + role: string; + generation?: number; + userProject?: string; +} +export type UpdateAclResponse = [AccessControlObject, AclMetadata]; +export interface UpdateAclCallback { + ( + err: Error | null, + acl?: AccessControlObject | null, + apiResponse?: AclMetadata + ): void; +} + +export interface AddAclOptions { + entity: string; + role: string; + generation?: number; + userProject?: string; +} +export type AddAclResponse = [AccessControlObject, AclMetadata]; +export interface AddAclCallback { + ( + err: Error | null, + acl?: AccessControlObject | null, + apiResponse?: AclMetadata + ): void; +} +export type RemoveAclResponse = [AclMetadata]; +export interface RemoveAclCallback { + (err: Error | null, apiResponse?: AclMetadata): void; +} +export interface RemoveAclOptions { + entity: string; + generation?: number; + userProject?: string; +} + +interface AclQuery { + generation: number; + userProject: string; +} + +export interface AccessControlObject { + entity: string; + role: string; + projectTeam: string; +} + +export interface AclMetadata extends BaseMetadata { + bucket?: string; + domain?: string; + entity?: string; + entityId?: string; + generation?: string; + object?: string; + projectTeam?: { + projectNumber?: string; + team?: 'editors' | 'owners' | 'viewers'; + }; + role?: 'OWNER' | 'READER' | 'WRITER' | 'FULL_CONTROL'; + [key: string]: unknown; +} + +/** + * Attach functionality to a {@link Storage.acl} instance. This will add an + * object for each role group (owners, readers, and writers), with each object + * containing methods to add or delete a type of entity. + * + * As an example, here are a few methods that are created. + * + * myBucket.acl.readers.deleteGroup('groupId', function(err) {}); + * + * myBucket.acl.owners.addUser('email@example.com', function(err, acl) {}); + * + * myBucket.acl.writers.addDomain('example.com', function(err, acl) {}); + * + * @private + */ +class AclRoleAccessorMethods { + private static accessMethods = ['add', 'delete']; + + private static entities = [ + // Special entity groups that do not require further specification. + 'allAuthenticatedUsers', + 'allUsers', + + // Entity groups that require specification, e.g. `user-email@example.com`. + 'domain-', + 'group-', + 'project-', + 'user-', + ]; + + private static roles = ['OWNER', 'READER', 'WRITER']; + + owners = {}; + + readers = {}; + + writers = {}; + + constructor() { + /** + * An object of convenience methods to add or delete owner ACL permissions + * for a given entity. + * + * The supported methods include: + * + * - `myFile.acl.owners.addAllAuthenticatedUsers` + * - `myFile.acl.owners.deleteAllAuthenticatedUsers` + * - `myFile.acl.owners.addAllUsers` + * - `myFile.acl.owners.deleteAllUsers` + * - `myFile.acl.owners.addDomain` + * - `myFile.acl.owners.deleteDomain` + * - `myFile.acl.owners.addGroup` + * - `myFile.acl.owners.deleteGroup` + * - `myFile.acl.owners.addProject` + * - `myFile.acl.owners.deleteProject` + * - `myFile.acl.owners.addUser` + * - `myFile.acl.owners.deleteUser` + * + * @name Acl#owners + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const myBucket = storage.bucket('my-bucket'); + * const myFile = myBucket.file('my-file'); + * + * //- + * // Add a user as an owner of a file. + * //- + * const myBucket = gcs.bucket('my-bucket'); + * const myFile = myBucket.file('my-file'); + * myFile.acl.owners.addUser('email@example.com', function(err, aclObject) + * {}); + * + * //- + * // For reference, the above command is the same as running the following. + * //- + * myFile.acl.add({ + * entity: 'user-email@example.com', + * role: gcs.acl.OWNER_ROLE + * }, function(err, aclObject) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * myFile.acl.owners.addUser('email@example.com').then(function(data) { + * const aclObject = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + this.owners = {}; + + /** + * An object of convenience methods to add or delete reader ACL permissions + * for a given entity. + * + * The supported methods include: + * + * - `myFile.acl.readers.addAllAuthenticatedUsers` + * - `myFile.acl.readers.deleteAllAuthenticatedUsers` + * - `myFile.acl.readers.addAllUsers` + * - `myFile.acl.readers.deleteAllUsers` + * - `myFile.acl.readers.addDomain` + * - `myFile.acl.readers.deleteDomain` + * - `myFile.acl.readers.addGroup` + * - `myFile.acl.readers.deleteGroup` + * - `myFile.acl.readers.addProject` + * - `myFile.acl.readers.deleteProject` + * - `myFile.acl.readers.addUser` + * - `myFile.acl.readers.deleteUser` + * + * @name Acl#readers + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const myBucket = storage.bucket('my-bucket'); + * const myFile = myBucket.file('my-file'); + * + * //- + * // Add a user as a reader of a file. + * //- + * myFile.acl.readers.addUser('email@example.com', function(err, aclObject) + * {}); + * + * //- + * // For reference, the above command is the same as running the following. + * //- + * myFile.acl.add({ + * entity: 'user-email@example.com', + * role: gcs.acl.READER_ROLE + * }, function(err, aclObject) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * myFile.acl.readers.addUser('email@example.com').then(function(data) { + * const aclObject = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + this.readers = {}; + + /** + * An object of convenience methods to add or delete writer ACL permissions + * for a given entity. + * + * The supported methods include: + * + * - `myFile.acl.writers.addAllAuthenticatedUsers` + * - `myFile.acl.writers.deleteAllAuthenticatedUsers` + * - `myFile.acl.writers.addAllUsers` + * - `myFile.acl.writers.deleteAllUsers` + * - `myFile.acl.writers.addDomain` + * - `myFile.acl.writers.deleteDomain` + * - `myFile.acl.writers.addGroup` + * - `myFile.acl.writers.deleteGroup` + * - `myFile.acl.writers.addProject` + * - `myFile.acl.writers.deleteProject` + * - `myFile.acl.writers.addUser` + * - `myFile.acl.writers.deleteUser` + * + * @name Acl#writers + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const myBucket = storage.bucket('my-bucket'); + * const myFile = myBucket.file('my-file'); + * + * //- + * // Add a user as a writer of a file. + * //- + * myFile.acl.writers.addUser('email@example.com', function(err, aclObject) + * {}); + * + * //- + * // For reference, the above command is the same as running the following. + * //- + * myFile.acl.add({ + * entity: 'user-email@example.com', + * role: gcs.acl.WRITER_ROLE + * }, function(err, aclObject) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * myFile.acl.writers.addUser('email@example.com').then(function(data) { + * const aclObject = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + this.writers = {}; + AclRoleAccessorMethods.roles.forEach(this._assignAccessMethods.bind(this)); + } + + _assignAccessMethods(role: string) { + const accessMethods = AclRoleAccessorMethods.accessMethods; + const entities = AclRoleAccessorMethods.entities; + const roleGroup = role.toLowerCase() + 's'; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this as any)[roleGroup] = entities.reduce((acc, entity) => { + const isPrefix = entity.charAt(entity.length - 1) === '-'; + + accessMethods.forEach(accessMethod => { + let method = + accessMethod + entity[0].toUpperCase() + entity.substring(1); + + if (isPrefix) { + method = method.replace('-', ''); + } + + // Wrap the parent accessor method (e.g. `add` or `delete`) to avoid the + // more complex API of specifying an `entity` and `role`. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (acc as any)[method] = ( + entityId: string, + options: {}, + callback: Function | {} + ) => { + let apiEntity; + + if (typeof options === 'function') { + callback = options; + options = {}; + } + + if (isPrefix) { + apiEntity = entity + entityId; + } else { + // If the entity is not a prefix, it is a special entity group + // that does not require further details. The accessor methods + // only accept a callback. + apiEntity = entity; + callback = entityId; + } + + options = Object.assign( + { + entity: apiEntity, + role, + }, + options + ); + + const args = [options]; + + if (typeof callback === 'function') { + args.push(callback); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (this as any)[accessMethod].apply(this, args); + }; + }); + + return acc; + }, {}); + } +} + +/** + * Cloud Storage uses access control lists (ACLs) to manage object and + * bucket access. ACLs are the mechanism you use to share objects with other + * users and allow other users to access your buckets and objects. + * + * An ACL consists of one or more entries, where each entry grants permissions + * to an entity. Permissions define the actions that can be performed against an + * object or bucket (for example, `READ` or `WRITE`); the entity defines who the + * permission applies to (for example, a specific user or group of users). + * + * Where an `entity` value is accepted, we follow the format the Cloud Storage + * API expects. + * + * Refer to + * https://cloud.google.com/storage/docs/json_api/v1/defaultObjectAccessControls + * for the most up-to-date values. + * + * - `user-userId` + * - `user-email` + * - `group-groupId` + * - `group-email` + * - `domain-domain` + * - `project-team-projectId` + * - `allUsers` + * - `allAuthenticatedUsers` + * + * Examples: + * + * - The user "liz@example.com" would be `user-liz@example.com`. + * - The group "example@googlegroups.com" would be + * `group-example@googlegroups.com`. + * - To refer to all members of the Google Apps for Business domain + * "example.com", the entity would be `domain-example.com`. + * + * For more detailed information, see + * {@link http://goo.gl/6qBBPO| About Access Control Lists}. + * + * @constructor Acl + * @mixin + * @param {object} options Configuration options. + */ +class Acl extends AclRoleAccessorMethods { + default!: Acl; + pathPrefix: string; + request_: ( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ) => void; + + constructor(options: AclOptions) { + super(); + this.pathPrefix = options.pathPrefix; + this.request_ = options.request; + } + + add(options: AddAclOptions): Promise; + add(options: AddAclOptions, callback: AddAclCallback): void; + /** + * @typedef {array} AddAclResponse + * @property {object} 0 The Acl Objects. + * @property {object} 1 The full API response. + */ + /** + * @callback AddAclCallback + * @param {?Error} err Request error, if any. + * @param {object} acl The Acl Objects. + * @param {object} apiResponse The full API response. + */ + /** + * Add access controls on a {@link Bucket} or {@link File}. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert| BucketAccessControls: insert API Documentation} + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/insert| ObjectAccessControls: insert API Documentation} + * + * @param {object} options Configuration options. + * @param {string} options.entity Whose permissions will be added. + * @param {string} options.role Permissions allowed for the defined entity. + * See {@link https://cloud.google.com/storage/docs/access-control Access + * Control}. + * @param {number} [options.generation] **File Objects Only** Select a specific + * revision of this file (as opposed to the latest version, the default). + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {AddAclCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const myBucket = storage.bucket('my-bucket'); + * const myFile = myBucket.file('my-file'); + * + * const options = { + * entity: 'user-useremail@example.com', + * role: gcs.acl.OWNER_ROLE + * }; + * + * myBucket.acl.add(options, function(err, aclObject, apiResponse) {}); + * + * //- + * // For file ACL operations, you can also specify a `generation` property. + * // Here is how you would grant ownership permissions to a user on a + * specific + * // revision of a file. + * //- + * myFile.acl.add({ + * entity: 'user-useremail@example.com', + * role: gcs.acl.OWNER_ROLE, + * generation: 1 + * }, function(err, aclObject, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * myBucket.acl.add(options).then(function(data) { + * const aclObject = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/acl.js + * region_tag:storage_add_file_owner + * Example of adding an owner to a file: + * + * @example include:samples/acl.js + * region_tag:storage_add_bucket_owner + * Example of adding an owner to a bucket: + * + * @example include:samples/acl.js + * region_tag:storage_add_bucket_default_owner + * Example of adding a default owner to a bucket: + */ + add( + options: AddAclOptions, + callback?: AddAclCallback + ): void | Promise { + const query = {} as AclQuery; + + if (options.generation) { + query.generation = options.generation; + } + + if (options.userProject) { + query.userProject = options.userProject; + } + + this.request( + { + method: 'POST', + uri: '', + qs: query, + maxRetries: 0, //explicitly set this value since this is a non-idempotent function + json: { + entity: options.entity, + role: options.role.toUpperCase(), + }, + }, + (err, resp) => { + if (err) { + callback!(err, null, resp); + return; + } + + callback!(null, this.makeAclObject_(resp), resp); + } + ); + } + + delete(options: RemoveAclOptions): Promise; + delete(options: RemoveAclOptions, callback: RemoveAclCallback): void; + /** + * @typedef {array} RemoveAclResponse + * @property {object} 0 The full API response. + */ + /** + * @callback RemoveAclCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * Delete access controls on a {@link Bucket} or {@link File}. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/delete| BucketAccessControls: delete API Documentation} + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/delete| ObjectAccessControls: delete API Documentation} + * + * @param {object} options Configuration object. + * @param {string} options.entity Whose permissions will be revoked. + * @param {int} [options.generation] **File Objects Only** Select a specific + * revision of this file (as opposed to the latest version, the default). + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {RemoveAclCallback} callback The callback function. + * @returns {Promise} + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const myBucket = storage.bucket('my-bucket'); + * const myFile = myBucket.file('my-file'); + * + * myBucket.acl.delete({ + * entity: 'user-useremail@example.com' + * }, function(err, apiResponse) {}); + * + * //- + * // For file ACL operations, you can also specify a `generation` property. + * //- + * myFile.acl.delete({ + * entity: 'user-useremail@example.com', + * generation: 1 + * }, function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * myFile.acl.delete().then(function(data) { + * const apiResponse = data[0]; + * }); + * + * ``` + * @example include:samples/acl.js + * region_tag:storage_remove_bucket_owner + * Example of removing an owner from a bucket: + * + * @example include:samples/acl.js + * region_tag:storage_remove_bucket_default_owner + * Example of removing a default owner from a bucket: + * + * @example include:samples/acl.js + * region_tag:storage_remove_file_owner + * Example of removing an owner from a bucket: + */ + delete( + options: RemoveAclOptions, + callback?: RemoveAclCallback + ): void | Promise { + const query = {} as AclQuery; + + if (options.generation) { + query.generation = options.generation; + } + + if (options.userProject) { + query.userProject = options.userProject; + } + + this.request( + { + method: 'DELETE', + uri: '/' + encodeURIComponent(options.entity), + qs: query, + }, + (err, resp) => { + callback!(err, resp); + } + ); + } + + get(options?: GetAclOptions): Promise; + get(options: GetAclOptions, callback: GetAclCallback): void; + get(callback: GetAclCallback): void; + /** + * @typedef {array} GetAclResponse + * @property {object|object[]} 0 Single or array of Acl Objects. + * @property {object} 1 The full API response. + */ + /** + * @callback GetAclCallback + * @param {?Error} err Request error, if any. + * @param {object|object[]} acl Single or array of Acl Objects. + * @param {object} apiResponse The full API response. + */ + /** + * Get access controls on a {@link Bucket} or {@link File}. If + * an entity is omitted, you will receive an array of all applicable access + * controls. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/get| BucketAccessControls: get API Documentation} + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/get| ObjectAccessControls: get API Documentation} + * + * @param {object|function} [options] Configuration options. If you want to + * receive a list of all access controls, pass the callback function as + * the only argument. + * @param {string} options.entity Whose permissions will be fetched. + * @param {number} [options.generation] **File Objects Only** Select a specific + * revision of this file (as opposed to the latest version, the default). + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {GetAclCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const myBucket = storage.bucket('my-bucket'); + * const myFile = myBucket.file('my-file'); + * + * myBucket.acl.get({ + * entity: 'user-useremail@example.com' + * }, function(err, aclObject, apiResponse) {}); + * + * //- + * // Get all access controls. + * //- + * myBucket.acl.get(function(err, aclObjects, apiResponse) { + * // aclObjects = [ + * // { + * // entity: 'user-useremail@example.com', + * // role: 'owner' + * // } + * // ] + * }); + * + * //- + * // For file ACL operations, you can also specify a `generation` property. + * //- + * myFile.acl.get({ + * entity: 'user-useremail@example.com', + * generation: 1 + * }, function(err, aclObject, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * myBucket.acl.get().then(function(data) { + * const aclObject = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/acl.js + * region_tag:storage_print_file_acl + * Example of printing a file's ACL: + * + * @example include:samples/acl.js + * region_tag:storage_print_file_acl_for_user + * Example of printing a file's ACL for a specific user: + * + * @example include:samples/acl.js + * region_tag:storage_print_bucket_acl + * Example of printing a bucket's ACL: + * + * @example include:samples/acl.js + * region_tag:storage_print_bucket_acl_for_user + * Example of printing a bucket's ACL for a specific user: + */ + get( + optionsOrCallback?: GetAclOptions | GetAclCallback, + cb?: GetAclCallback + ): void | Promise { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : null; + const callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : cb; + let path = ''; + const query = {} as AclQuery; + + if (options) { + path = '/' + encodeURIComponent(options.entity); + + if (options.generation) { + query.generation = options.generation; + } + + if (options.userProject) { + query.userProject = options.userProject; + } + } + + this.request( + { + uri: path, + qs: query, + }, + (err, resp) => { + if (err) { + callback!(err, null, resp); + return; + } + + let results; + + if (resp.items) { + results = resp.items.map(this.makeAclObject_); + } else { + results = this.makeAclObject_(resp); + } + + callback!(null, results, resp); + } + ); + } + + update(options: UpdateAclOptions): Promise; + update(options: UpdateAclOptions, callback: UpdateAclCallback): void; + /** + * @typedef {array} UpdateAclResponse + * @property {object} 0 The updated Acl Objects. + * @property {object} 1 The full API response. + */ + /** + * @callback UpdateAclCallback + * @param {?Error} err Request error, if any. + * @param {object} acl The updated Acl Objects. + * @param {object} apiResponse The full API response. + */ + /** + * Update access controls on a {@link Bucket} or {@link File}. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/update| BucketAccessControls: update API Documentation} + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/update| ObjectAccessControls: update API Documentation} + * + * @param {object} options Configuration options. + * @param {string} options.entity Whose permissions will be updated. + * @param {string} options.role Permissions allowed for the defined entity. + * See {@link Storage.acl}. + * @param {number} [options.generation] **File Objects Only** Select a specific + * revision of this file (as opposed to the latest version, the default). + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {UpdateAclCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const myBucket = storage.bucket('my-bucket'); + * const myFile = myBucket.file('my-file'); + * + * const options = { + * entity: 'user-useremail@example.com', + * role: gcs.acl.WRITER_ROLE + * }; + * + * myBucket.acl.update(options, function(err, aclObject, apiResponse) {}); + * + * //- + * // For file ACL operations, you can also specify a `generation` property. + * //- + * myFile.acl.update({ + * entity: 'user-useremail@example.com', + * role: gcs.acl.WRITER_ROLE, + * generation: 1 + * }, function(err, aclObject, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * myFile.acl.update(options).then(function(data) { + * const aclObject = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + update( + options: UpdateAclOptions, + callback?: UpdateAclCallback + ): void | Promise { + const query = {} as AclQuery; + + if (options.generation) { + query.generation = options.generation; + } + + if (options.userProject) { + query.userProject = options.userProject; + } + + this.request( + { + method: 'PUT', + uri: '/' + encodeURIComponent(options.entity), + qs: query, + json: { + role: options.role.toUpperCase(), + }, + }, + (err, resp) => { + if (err) { + callback!(err, null, resp); + return; + } + + callback!(null, this.makeAclObject_(resp), resp); + } + ); + } + + /** + * Transform API responses to a consistent object format. + * + * @private + */ + makeAclObject_( + accessControlObject: AccessControlObject + ): AccessControlObject { + const obj = { + entity: accessControlObject.entity, + role: accessControlObject.role, + } as AccessControlObject; + + if (accessControlObject.projectTeam) { + obj.projectTeam = accessControlObject.projectTeam; + } + + return obj; + } + + /** + * Patch requests up to the bucket's request object. + * + * @private + * + * @param {string} method Action. + * @param {string} path Request path. + * @param {*} query Request query object. + * @param {*} body Request body contents. + * @param {function} callback Callback function. + */ + request( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ): void { + reqOpts.uri = this.pathPrefix + reqOpts.uri; + this.request_(reqOpts, callback); + } +} + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +promisifyAll(Acl, { + exclude: ['request'], +}); + +export {Acl, AclRoleAccessorMethods}; diff --git a/handwritten/storage/src/bucket.ts b/handwritten/storage/src/bucket.ts new file mode 100644 index 00000000000..d354209057e --- /dev/null +++ b/handwritten/storage/src/bucket.ts @@ -0,0 +1,4603 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + ApiError, + BodyResponseCallback, + DecorateRequestOptions, + DeleteCallback, + ExistsCallback, + GetConfig, + MetadataCallback, + ServiceObject, + SetMetadataResponse, + util, +} from './nodejs-common/index.js'; +import {RequestResponse} from './nodejs-common/service-object.js'; +import {paginator} from '@google-cloud/paginator'; +import {promisifyAll} from '@google-cloud/promisify'; +import * as fs from 'fs'; +import * as http from 'http'; +import mime from 'mime'; +import * as path from 'path'; +import pLimit from 'p-limit'; +import {promisify} from 'util'; +import AsyncRetry from 'async-retry'; +import {convertObjKeysToSnakeCase} from './util.js'; + +import {Acl, AclMetadata} from './acl.js'; +import {Channel} from './channel.js'; +import { + File, + FileOptions, + CreateResumableUploadOptions, + CreateWriteStreamOptions, + FileMetadata, +} from './file.js'; +import {Iam} from './iam.js'; +import {Notification, NotificationMetadata} from './notification.js'; +import { + Storage, + Cors, + PreconditionOptions, + IdempotencyStrategy, + BucketOptions, +} from './storage.js'; +import { + GetSignedUrlResponse, + GetSignedUrlCallback, + SignerGetSignedUrlConfig, + URLSigner, + Query, +} from './signer.js'; +import {Readable} from 'stream'; +import {CRC32CValidatorGenerator} from './crc32c.js'; +import {URL} from 'url'; +import { + BaseMetadata, + SetMetadataOptions, +} from './nodejs-common/service-object.js'; + +interface SourceObject { + name: string; + generation?: number; +} + +interface CreateNotificationQuery { + userProject?: string; +} + +interface MetadataOptions { + predefinedAcl: string; + userProject?: string; + ifGenerationMatch?: number | string; + ifGenerationNotMatch?: number | string; + ifMetagenerationMatch?: number | string; + ifMetagenerationNotMatch?: number | string; +} + +export type GetFilesResponse = [ + File[], + (GetFilesOptions | {}) & Partial>, + unknown, +]; +export interface GetFilesCallback { + ( + err: Error | null, + files?: File[], + nextQuery?: {}, + apiResponse?: unknown + ): void; +} + +interface WatchAllOptions { + delimiter?: string; + maxResults?: number; + pageToken?: string; + prefix?: string; + projection?: string; + userProject?: string; + versions?: boolean; +} + +export interface AddLifecycleRuleOptions extends PreconditionOptions { + append?: boolean; +} + +export interface LifecycleAction { + type: 'Delete' | 'SetStorageClass' | 'AbortIncompleteMultipartUpload'; + storageClass?: string; +} +export interface LifecycleCondition { + age?: number; + createdBefore?: Date | string; + customTimeBefore?: Date | string; + daysSinceCustomTime?: number; + daysSinceNoncurrentTime?: number; + isLive?: boolean; + matchesPrefix?: string[]; + matchesSuffix?: string[]; + matchesStorageClass?: string[]; + noncurrentTimeBefore?: Date | string; + numNewerVersions?: number; +} + +export interface LifecycleRule { + action: LifecycleAction; + condition: LifecycleCondition; +} + +export interface LifecycleCondition { + age?: number; + createdBefore?: Date | string; + customTimeBefore?: Date | string; + daysSinceCustomTime?: number; + daysSinceNoncurrentTime?: number; + isLive?: boolean; + matchesPrefix?: string[]; + matchesSuffix?: string[]; + matchesStorageClass?: string[]; + noncurrentTimeBefore?: Date | string; + numNewerVersions?: number; +} + +export interface LifecycleRule { + action: LifecycleAction; + condition: LifecycleCondition; +} + +export interface EnableLoggingOptions extends PreconditionOptions { + bucket?: string | Bucket; + prefix: string; +} + +export interface GetFilesOptions { + autoPaginate?: boolean; + delimiter?: string; + endOffset?: string; + includeFoldersAsPrefixes?: boolean; + includeTrailingDelimiter?: boolean; + prefix?: string; + matchGlob?: string; + maxApiCalls?: number; + maxResults?: number; + pageToken?: string; + softDeleted?: boolean; + startOffset?: string; + userProject?: string; + versions?: boolean; + fields?: string; +} + +export interface CombineOptions extends PreconditionOptions { + kmsKeyName?: string; + userProject?: string; +} + +export interface CombineCallback { + (err: Error | null, newFile: File | null, apiResponse: unknown): void; +} + +export type CombineResponse = [File, unknown]; + +export interface CreateChannelConfig extends WatchAllOptions { + address: string; +} + +export interface CreateChannelOptions { + userProject?: string; +} + +export type CreateChannelResponse = [Channel, unknown]; + +export interface CreateChannelCallback { + (err: Error | null, channel: Channel | null, apiResponse: unknown): void; +} + +export interface CreateNotificationOptions { + customAttributes?: {[key: string]: string}; + eventTypes?: string[]; + objectNamePrefix?: string; + payloadFormat?: string; + userProject?: string; +} + +export interface CreateNotificationCallback { + ( + err: Error | null, + notification: Notification | null, + apiResponse: unknown + ): void; +} + +export type CreateNotificationResponse = [Notification, unknown]; + +export interface DeleteBucketOptions { + ignoreNotFound?: boolean; + userProject?: string; +} + +export type DeleteBucketResponse = [unknown]; + +export interface DeleteBucketCallback extends DeleteCallback { + (err: Error | null, apiResponse: unknown): void; +} + +export interface DeleteFilesOptions + extends GetFilesOptions, + PreconditionOptions { + force?: boolean; +} + +export interface DeleteFilesCallback { + (err: Error | Error[] | null, apiResponse?: object): void; +} + +export type DeleteLabelsResponse = [unknown]; + +export type DeleteLabelsCallback = SetLabelsCallback; + +export type DeleteLabelsOptions = PreconditionOptions; + +export type DisableRequesterPaysOptions = PreconditionOptions; + +export type DisableRequesterPaysResponse = [unknown]; + +export interface DisableRequesterPaysCallback { + (err?: Error | null, apiResponse?: object): void; +} + +export type EnableRequesterPaysResponse = [unknown]; + +export interface EnableRequesterPaysCallback { + (err?: Error | null, apiResponse?: unknown): void; +} + +export type EnableRequesterPaysOptions = PreconditionOptions; +export interface BucketExistsOptions extends GetConfig { + userProject?: string; +} + +export type BucketExistsResponse = [boolean]; + +export type BucketExistsCallback = ExistsCallback; + +export interface GetBucketOptions extends GetConfig { + userProject?: string; +} + +export type GetBucketResponse = [Bucket, unknown]; + +export interface GetBucketCallback { + (err: ApiError | null, bucket: Bucket | null, apiResponse: unknown): void; +} + +export interface GetLabelsOptions { + userProject?: string; +} + +export type GetLabelsResponse = [unknown]; + +export interface GetLabelsCallback { + (err: Error | null, labels: object | null): void; +} + +export interface RestoreOptions { + generation: string; + projection?: 'full' | 'noAcl'; +} +export interface BucketMetadata extends BaseMetadata { + acl?: AclMetadata[] | null; + autoclass?: { + enabled?: boolean; + toggleTime?: string; + terminalStorageClass?: string; + terminalStorageClassUpdateTime?: string; + }; + billing?: { + requesterPays?: boolean; + }; + cors?: Cors[]; + customPlacementConfig?: { + dataLocations?: string[]; + }; + defaultEventBasedHold?: boolean; + defaultObjectAcl?: AclMetadata[]; + encryption?: { + defaultKmsKeyName?: string; + } | null; + hierarchicalNamespace?: { + enabled?: boolean; + }; + iamConfiguration?: { + publicAccessPrevention?: string; + uniformBucketLevelAccess?: { + enabled?: boolean; + lockedTime?: string; + }; + }; + labels?: { + [key: string]: string | null; + }; + lifecycle?: { + rule?: LifecycleRule[]; + } | null; + location?: string; + locationType?: string; + logging?: { + logBucket?: string; + logObjectPrefix?: string; + }; + generation?: string; + metageneration?: string; + name?: string; + objectRetention?: { + mode?: string; + }; + owner?: { + entity?: string; + entityId?: string; + }; + projectNumber?: string | number; + retentionPolicy?: { + effectiveTime?: string; + isLocked?: boolean; + retentionPeriod?: string | number; + } | null; + rpo?: string; + softDeleteTime?: string; + hardDeleteTime?: string; + softDeletePolicy?: { + retentionDurationSeconds?: string | number; + readonly effectiveTime?: string; + }; + storageClass?: string; + timeCreated?: string; + updated?: string; + versioning?: { + enabled?: boolean; + }; + website?: { + mainPageSuffix?: string; + notFoundPage?: string; + }; +} + +export type GetBucketMetadataResponse = [BucketMetadata, unknown]; + +export interface GetBucketMetadataCallback { + ( + err: ApiError | null, + metadata: BucketMetadata | null, + apiResponse: unknown + ): void; +} + +export interface GetBucketMetadataOptions { + userProject?: string; +} + +export interface GetBucketSignedUrlConfig + extends Pick { + action: 'list'; + version?: 'v2' | 'v4'; + cname?: string; + virtualHostedStyle?: boolean; + expires: string | number | Date; + extensionHeaders?: http.OutgoingHttpHeaders; + queryParams?: Query; +} + +export enum BucketActionToHTTPMethod { + list = 'GET', +} + +export enum AvailableServiceObjectMethods { + setMetadata, + delete, +} + +export interface GetNotificationsOptions { + userProject?: string; +} + +export interface GetNotificationsCallback { + ( + err: Error | null, + notifications: Notification[] | null, + apiResponse: unknown + ): void; +} + +export type GetNotificationsResponse = [Notification[], unknown]; + +export interface MakeBucketPrivateOptions { + includeFiles?: boolean; + force?: boolean; + metadata?: BucketMetadata; + userProject?: string; + preconditionOpts?: PreconditionOptions; +} + +interface MakeBucketPrivateRequest extends MakeBucketPrivateOptions { + private?: boolean; +} + +export type MakeBucketPrivateResponse = [File[]]; + +export interface MakeBucketPrivateCallback { + (err?: Error | null, files?: File[]): void; +} + +export interface MakeBucketPublicOptions { + includeFiles?: boolean; + force?: boolean; +} + +export interface MakeBucketPublicCallback { + (err?: Error | null, files?: File[]): void; +} + +export type MakeBucketPublicResponse = [File[]]; + +export interface SetBucketMetadataOptions extends PreconditionOptions { + userProject?: string; +} + +export type SetBucketMetadataResponse = [BucketMetadata]; + +export interface SetBucketMetadataCallback { + (err?: Error | null, metadata?: BucketMetadata): void; +} + +export interface BucketLockCallback { + (err?: Error | null, apiResponse?: unknown): void; +} + +export type BucketLockResponse = [unknown]; + +export interface Labels { + [key: string]: string; +} + +export interface SetLabelsOptions extends PreconditionOptions { + userProject?: string; +} + +export type SetLabelsResponse = [unknown]; + +export interface SetLabelsCallback { + (err?: Error | null, metadata?: unknown): void; +} + +export interface SetBucketStorageClassOptions extends PreconditionOptions { + userProject?: string; +} + +export interface SetBucketStorageClassCallback { + (err?: Error | null): void; +} + +export type UploadResponse = [File, unknown]; + +export interface UploadCallback { + (err: Error | null, file?: File | null, apiResponse?: unknown): void; +} + +export interface UploadOptions + extends CreateResumableUploadOptions, + CreateWriteStreamOptions { + destination?: string | File; + encryptionKey?: string | Buffer; + kmsKeyName?: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + onUploadProgress?: (progressEvent: any) => void; +} + +export interface MakeAllFilesPublicPrivateOptions { + force?: boolean; + private?: boolean; + public?: boolean; + userProject?: string; +} + +interface MakeAllFilesPublicPrivateCallback { + (err?: Error | Error[] | null, files?: File[]): void; +} + +type MakeAllFilesPublicPrivateResponse = [File[]]; + +export enum BucketExceptionMessages { + PROVIDE_SOURCE_FILE = 'You must provide at least one source file.', + DESTINATION_FILE_NOT_SPECIFIED = 'A destination file must be specified.', + CHANNEL_ID_REQUIRED = 'An ID is required to create a channel.', + TOPIC_NAME_REQUIRED = 'A valid topic name is required.', + CONFIGURATION_OBJECT_PREFIX_REQUIRED = 'A configuration object with a prefix is required.', + SPECIFY_FILE_NAME = 'A file name must be specified.', + METAGENERATION_NOT_PROVIDED = 'A metageneration must be provided.', + SUPPLY_NOTIFICATION_ID = 'You must supply a notification ID.', +} + +/** + * @callback Crc32cGeneratorToStringCallback + * A method returning the CRC32C as a base64-encoded string. + * + * @returns {string} + * + * @example + * Hashing the string 'data' should return 'rth90Q==' + * + * ```js + * const buffer = Buffer.from('data'); + * crc32c.update(buffer); + * crc32c.toString(); // 'rth90Q==' + * ``` + **/ +/** + * @callback Crc32cGeneratorValidateCallback + * A method validating a base64-encoded CRC32C string. + * + * @param {string} [value] base64-encoded CRC32C string to validate + * @returns {boolean} + * + * @example + * Should return `true` if the value matches, `false` otherwise + * + * ```js + * const buffer = Buffer.from('data'); + * crc32c.update(buffer); + * crc32c.validate('DkjKuA=='); // false + * crc32c.validate('rth90Q=='); // true + * ``` + **/ +/** + * @callback Crc32cGeneratorUpdateCallback + * A method for passing `Buffer`s for CRC32C generation. + * + * @param {Buffer} [data] data to update CRC32C value with + * @returns {undefined} + * + * @example + * Hashing buffers from 'some ' and 'text\n' + * + * ```js + * const buffer1 = Buffer.from('some '); + * crc32c.update(buffer1); + * + * const buffer2 = Buffer.from('text\n'); + * crc32c.update(buffer2); + * + * crc32c.toString(); // 'DkjKuA==' + * ``` + **/ +/** + * @typedef {object} CRC32CValidator + * @property {Crc32cGeneratorToStringCallback} + * @property {Crc32cGeneratorValidateCallback} + * @property {Crc32cGeneratorUpdateCallback} + */ +/** + * A function that generates a CRC32C Validator. Defaults to {@link CRC32C} + * + * @name Bucket#crc32cGenerator + * @type {CRC32CValidator} + */ +/** + * Get and set IAM policies for your bucket. + * + * @name Bucket#iam + * @mixes Iam + * + * See {@link https://cloud.google.com/storage/docs/access-control/iam#short_title_iam_management| Cloud Storage IAM Management} + * See {@link https://cloud.google.com/iam/docs/granting-changing-revoking-access| Granting, Changing, and Revoking Access} + * See {@link https://cloud.google.com/iam/docs/understanding-roles| IAM Roles} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * //- + * // Get the IAM policy for your bucket. + * //- + * bucket.iam.getPolicy(function(err, policy) { + * console.log(policy); + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.iam.getPolicy().then(function(data) { + * const policy = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/iam.js + * region_tag:storage_view_bucket_iam_members + * Example of retrieving a bucket's IAM policy: + * + * @example include:samples/iam.js + * region_tag:storage_add_bucket_iam_member + * Example of adding to a bucket's IAM policy: + * + * @example include:samples/iam.js + * region_tag:storage_remove_bucket_iam_member + * Example of removing from a bucket's IAM policy: + */ +/** + * Cloud Storage uses access control lists (ACLs) to manage object and + * bucket access. ACLs are the mechanism you use to share objects with other + * users and allow other users to access your buckets and objects. + * + * An ACL consists of one or more entries, where each entry grants permissions + * to an entity. Permissions define the actions that can be performed against + * an object or bucket (for example, `READ` or `WRITE`); the entity defines + * who the permission applies to (for example, a specific user or group of + * users). + * + * The `acl` object on a Bucket instance provides methods to get you a list of + * the ACLs defined on your bucket, as well as set, update, and delete them. + * + * Buckets also have + * {@link https://cloud.google.com/storage/docs/access-control/lists#default| default ACLs} + * for all created files. Default ACLs specify permissions that all new + * objects added to the bucket will inherit by default. You can add, delete, + * get, and update entities and permissions for these as well with + * {@link Bucket#acl.default}. + * + * See {@link http://goo.gl/6qBBPO| About Access Control Lists} + * See {@link https://cloud.google.com/storage/docs/access-control/lists#default| Default ACLs} + * + * @name Bucket#acl + * @mixes Acl + * @property {Acl} default Cloud Storage Buckets have + * {@link https://cloud.google.com/storage/docs/access-control/lists#default| default ACLs} + * for all created files. You can add, delete, get, and update entities and + * permissions for these as well. The method signatures and examples are all + * the same, after only prefixing the method call with `default`. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * + * //- + * // Make a bucket's contents publicly readable. + * //- + * const myBucket = storage.bucket('my-bucket'); + * + * const options = { + * entity: 'allUsers', + * role: storage.acl.READER_ROLE + * }; + * + * myBucket.acl.add(options, function(err, aclObject) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * myBucket.acl.add(options).then(function(data) { + * const aclObject = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/acl.js + * region_tag:storage_print_bucket_acl + * Example of printing a bucket's ACL: + * + * @example include:samples/acl.js + * region_tag:storage_print_bucket_acl_for_user + * Example of printing a bucket's ACL for a specific user: + * + * @example include:samples/acl.js + * region_tag:storage_add_bucket_owner + * Example of adding an owner to a bucket: + * + * @example include:samples/acl.js + * region_tag:storage_remove_bucket_owner + * Example of removing an owner from a bucket: + * + * @example include:samples/acl.js + * region_tag:storage_add_bucket_default_owner + * Example of adding a default owner to a bucket: + * + * @example include:samples/acl.js + * region_tag:storage_remove_bucket_default_owner + * Example of removing a default owner from a bucket: + */ +/** + * The API-formatted resource description of the bucket. + * + * Note: This is not guaranteed to be up-to-date when accessed. To get the + * latest record, call the `getMetadata()` method. + * + * @name Bucket#metadata + * @type {object} + */ +/** + * The bucket's name. + * @name Bucket#name + * @type {string} + */ +/** + * Get {@link File} objects for the files currently in the bucket as a + * readable object stream. + * + * @method Bucket#getFilesStream + * @param {GetFilesOptions} [query] Query object for listing files. + * @returns {ReadableStream} A readable stream that emits {@link File} instances. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.getFilesStream() + * .on('error', console.error) + * .on('data', function(file) { + * // file is a File object. + * }) + * .on('end', function() { + * // All files retrieved. + * }); + * + * //- + * // If you anticipate many results, you can end a stream early to prevent + * // unnecessary processing and API requests. + * //- + * bucket.getFilesStream() + * .on('data', function(file) { + * this.end(); + * }); + * + * //- + * // If you're filtering files with a delimiter, you should use + * // {@link Bucket#getFiles} and set `autoPaginate: false` in order to + * // preserve the `apiResponse` argument. + * //- + * const prefixes = []; + * + * function callback(err, files, nextQuery, apiResponse) { + * prefixes = prefixes.concat(apiResponse.prefixes); + * + * if (nextQuery) { + * bucket.getFiles(nextQuery, callback); + * } else { + * // prefixes = The finished array of prefixes. + * } + * } + * + * bucket.getFiles({ + * autoPaginate: false, + * delimiter: '/' + * }, callback); + * ``` + */ +/** + * Create a Bucket object to interact with a Cloud Storage bucket. + * + * @class + * @hideconstructor + * + * @param {Storage} storage A {@link Storage} instance. + * @param {string} name The name of the bucket. + * @param {object} [options] Configuration object. + * @param {string} [options.userProject] User project. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * ``` + */ +class Bucket extends ServiceObject { + name: string; + + /** + * A reference to the {@link Storage} associated with this {@link Bucket} + * instance. + * @name Bucket#storage + * @type {Storage} + */ + storage: Storage; + + /** + * A user project to apply to each request from this bucket. + * @name Bucket#userProject + * @type {string} + */ + userProject?: string; + + acl: Acl; + iam: Iam; + crc32cGenerator: CRC32CValidatorGenerator; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + getFilesStream(query?: GetFilesOptions): Readable { + // placeholder body, overwritten in constructor + return new Readable(); + } + signer?: URLSigner; + + private instanceRetryValue?: boolean; + instancePreconditionOpts?: PreconditionOptions; + + /** + * Indicates whether this Bucket object is a placeholder for an item + * that the API failed to retrieve (unreachable) due to partial failure. + * Consumers must check this flag before accessing other properties. + */ + unreachable = false; + + constructor(storage: Storage, name: string, options?: BucketOptions) { + options = options || {}; + + // Allow for "gs://"-style input, and strip any trailing slashes. + name = name.replace(/^gs:\/\//, '').replace(/\/+$/, ''); + + const requestQueryObject: { + userProject?: string; + ifGenerationMatch?: number | string; + ifGenerationNotMatch?: number | string; + ifMetagenerationMatch?: number | string; + ifMetagenerationNotMatch?: number | string; + } = {}; + + if (options?.preconditionOpts?.ifGenerationMatch) { + requestQueryObject.ifGenerationMatch = + options.preconditionOpts.ifGenerationMatch; + } + if (options?.preconditionOpts?.ifGenerationNotMatch) { + requestQueryObject.ifGenerationNotMatch = + options.preconditionOpts.ifGenerationNotMatch; + } + if (options?.preconditionOpts?.ifMetagenerationMatch) { + requestQueryObject.ifMetagenerationMatch = + options.preconditionOpts.ifMetagenerationMatch; + } + if (options?.preconditionOpts?.ifMetagenerationNotMatch) { + requestQueryObject.ifMetagenerationNotMatch = + options.preconditionOpts.ifMetagenerationNotMatch; + } + + const userProject = options.userProject; + if (typeof userProject === 'string') { + requestQueryObject.userProject = userProject; + } + + const methods = { + /** + * Create a bucket. + * + * @method Bucket#create + * @param {CreateBucketRequest} [metadata] Metadata to set for the bucket. + * @param {CreateBucketCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * bucket.create(function(err, bucket, apiResponse) { + * if (!err) { + * // The bucket was created successfully. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.create().then(function(data) { + * const bucket = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + create: { + reqOpts: { + qs: requestQueryObject, + }, + }, + /** + * IamDeleteBucketOptions Configuration options. + * @property {boolean} [ignoreNotFound = false] Ignore an error if + * the bucket does not exist. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @typedef {array} DeleteBucketResponse + * @property {object} 0 The full API response. + */ + /** + * @callback DeleteBucketCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * Delete the bucket. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/delete| Buckets: delete API Documentation} + * + * @method Bucket#delete + * @param {DeleteBucketOptions} [options] Configuration options. + * @param {boolean} [options.ignoreNotFound = false] Ignore an error if + * the bucket does not exist. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {DeleteBucketCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * bucket.delete(function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.delete().then(function(data) { + * const apiResponse = data[0]; + * }); + * + * ``` + * @example include:samples/buckets.js + * region_tag:storage_delete_bucket + * Another example: + */ + delete: { + reqOpts: { + qs: requestQueryObject, + }, + }, + /** + * @typedef {object} BucketExistsOptions Configuration options for Bucket#exists(). + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @typedef {array} BucketExistsResponse + * @property {boolean} 0 Whether the {@link Bucket} exists. + */ + /** + * @callback BucketExistsCallback + * @param {?Error} err Request error, if any. + * @param {boolean} exists Whether the {@link Bucket} exists. + */ + /** + * Check if the bucket exists. + * + * @method Bucket#exists + * @param {BucketExistsOptions} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {BucketExistsCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.exists(function(err, exists) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.exists().then(function(data) { + * const exists = data[0]; + * }); + * ``` + */ + exists: { + reqOpts: { + qs: requestQueryObject, + }, + }, + /** + * @typedef {object} [GetBucketOptions] Configuration options for Bucket#get() + * @property {boolean} [autoCreate] Automatically create the object if + * it does not exist. Default: `false` + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @typedef {array} GetBucketResponse + * @property {Bucket} 0 The {@link Bucket}. + * @property {object} 1 The full API response. + */ + /** + * @callback GetBucketCallback + * @param {?Error} err Request error, if any. + * @param {Bucket} bucket The {@link Bucket}. + * @param {object} apiResponse The full API response. + */ + /** + * Get a bucket if it exists. + * + * You may optionally use this to "get or create" an object by providing + * an object with `autoCreate` set to `true`. Any extra configuration that + * is normally required for the `create` method must be contained within + * this object as well. + * + * @method Bucket#get + * @param {GetBucketOptions} [options] Configuration options. + * @param {boolean} [options.autoCreate] Automatically create the object if + * it does not exist. Default: `false` + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {GetBucketCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.get(function(err, bucket, apiResponse) { + * // `bucket.metadata` has been populated. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.get().then(function(data) { + * const bucket = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + get: { + reqOpts: { + qs: requestQueryObject, + }, + }, + /** + * @typedef {array} GetBucketMetadataResponse + * @property {object} 0 The bucket metadata. + * @property {object} 1 The full API response. + */ + /** + * @callback GetBucketMetadataCallback + * @param {?Error} err Request error, if any. + * @param {object} metadata The bucket metadata. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {object} GetBucketMetadataOptions Configuration options for Bucket#getMetadata(). + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * Get the bucket's metadata. + * + * To set metadata, see {@link Bucket#setMetadata}. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/get| Buckets: get API Documentation} + * + * @method Bucket#getMetadata + * @param {GetBucketMetadataOptions} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {GetBucketMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.getMetadata(function(err, metadata, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.getMetadata().then(function(data) { + * const metadata = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/requesterPays.js + * region_tag:storage_get_requester_pays_status + * Example of retrieving the requester pays status of a bucket: + */ + getMetadata: { + reqOpts: { + qs: requestQueryObject, + }, + }, + /** + * @typedef {object} SetBucketMetadataOptions Configuration options for Bucket#setMetadata(). + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @typedef {array} SetBucketMetadataResponse + * @property {object} apiResponse The full API response. + */ + /** + * @callback SetBucketMetadataCallback + * @param {?Error} err Request error, if any. + * @param {object} metadata The bucket metadata. + */ + /** + * Set the bucket's metadata. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/patch| Buckets: patch API Documentation} + * + * @method Bucket#setMetadata + * @param {object} metadata The metadata you wish to set. + * @param {SetBucketMetadataOptions} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {SetBucketMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * //- + * // Set website metadata field on the bucket. + * //- + * const metadata = { + * website: { + * mainPageSuffix: 'http://example.com', + * notFoundPage: 'http://example.com/404.html' + * } + * }; + * + * bucket.setMetadata(metadata, function(err, apiResponse) {}); + * + * //- + * // Enable versioning for your bucket. + * //- + * bucket.setMetadata({ + * versioning: { + * enabled: true + * } + * }, function(err, apiResponse) {}); + * + * //- + * // Enable KMS encryption for objects within this bucket. + * //- + * bucket.setMetadata({ + * encryption: { + * defaultKmsKeyName: 'projects/grape-spaceship-123/...' + * } + * }, function(err, apiResponse) {}); + * + * //- + * // Set the default event-based hold value for new objects in this + * // bucket. + * //- + * bucket.setMetadata({ + * defaultEventBasedHold: true + * }, function(err, apiResponse) {}); + * + * //- + * // Remove object lifecycle rules. + * //- + * bucket.setMetadata({ + * lifecycle: null + * }, function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.setMetadata(metadata).then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + setMetadata: { + reqOpts: { + qs: requestQueryObject, + }, + }, + }; + + super({ + parent: storage, + baseUrl: '/b', + id: name, + createMethod: storage.createBucket.bind(storage), + methods, + }); + + this.name = name; + + this.storage = storage; + + this.userProject = options.userProject; + + this.acl = new Acl({ + request: this.request.bind(this), + pathPrefix: '/acl', + }); + + this.acl.default = new Acl({ + request: this.request.bind(this), + pathPrefix: '/defaultObjectAcl', + }); + + this.crc32cGenerator = + options.crc32cGenerator || this.storage.crc32cGenerator; + + this.iam = new Iam(this); + + this.getFilesStream = paginator.streamify('getFiles'); + + this.instanceRetryValue = storage.retryOptions.autoRetry; + this.instancePreconditionOpts = options?.preconditionOpts; + } + + /** + * The bucket's Cloud Storage URI (`gs://`) + * + * @example + * ```ts + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * + * // `gs://my-bucket` + * const href = bucket.cloudStorageURI.href; + * ``` + */ + get cloudStorageURI(): URL { + const uri = new URL('gs://'); + + uri.host = this.name; + + return uri; + } + + addLifecycleRule( + rule: LifecycleRule | LifecycleRule[], + options?: AddLifecycleRuleOptions + ): Promise; + addLifecycleRule( + rule: LifecycleRule | LifecycleRule[], + options: AddLifecycleRuleOptions, + callback: SetBucketMetadataCallback + ): void; + addLifecycleRule( + rule: LifecycleRule | LifecycleRule[], + callback: SetBucketMetadataCallback + ): void; + /** + * @typedef {object} AddLifecycleRuleOptions Configuration options for Bucket#addLifecycleRule(). + * @property {boolean} [append=true] The new rules will be appended to any + * pre-existing rules. + */ + /** + * + * @typedef {object} LifecycleRule The new lifecycle rule to be added to objects + * in this bucket. + * @property {string|object} action The action to be taken upon matching of + * all the conditions 'delete', 'setStorageClass', or 'AbortIncompleteMultipartUpload'. + * **Note**: For configuring a raw-formatted rule object to be passed as `action` + * please refer to the [examples]{@link https://cloud.google.com/storage/docs/managing-lifecycles#configexamples}. + * @property {object} condition Condition a bucket must meet before the + * action occurs on the bucket. Refer to following supported [conditions]{@link https://cloud.google.com/storage/docs/lifecycle#conditions}. + * @property {string} [storageClass] When using the `setStorageClass` + * action, provide this option to dictate which storage class the object + * should update to. Please see + * [SetStorageClass option documentation]{@link https://cloud.google.com/storage/docs/lifecycle#setstorageclass} for supported transitions. + */ + /** + * Add an object lifecycle management rule to the bucket. + * + * By default, an Object Lifecycle Management rule provided to this method + * will be included to the existing policy. To replace all existing rules, + * supply the `options` argument, setting `append` to `false`. + * + * To add multiple rules, pass a list to the `rule` parameter. Calling this + * function multiple times asynchronously does not guarantee that all rules + * are added correctly. + * + * See {@link https://cloud.google.com/storage/docs/lifecycle| Object Lifecycle Management} + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/patch| Buckets: patch API Documentation} + * + * @param {LifecycleRule|LifecycleRule[]} rule The new lifecycle rule or rules to be added to objects + * in this bucket. + * @param {string|object} rule.action The action to be taken upon matching of + * all the conditions 'delete', 'setStorageClass', or 'AbortIncompleteMultipartUpload'. + * **Note**: For configuring a raw-formatted rule object to be passed as `action` + * please refer to the [examples]{@link https://cloud.google.com/storage/docs/managing-lifecycles#configexamples}. + * @param {object} rule.condition Condition a bucket must meet before the + * action occurs on the bucket. Refer to following supported [conditions]{@link https://cloud.google.com/storage/docs/lifecycle#conditions}. + * @param {string} [rule.storageClass] When using the `setStorageClass` + * action, provide this option to dictate which storage class the object + * should update to. + * @param {AddLifecycleRuleOptions} [options] Configuration object. + * @param {boolean} [options.append=true] Append the new rule to the existing + * policy. + * @param {SetBucketMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * //- + * // Automatically have an object deleted from this bucket once it is 3 years + * // of age. + * //- + * bucket.addLifecycleRule({ + * action: 'delete', + * condition: { + * age: 365 * 3 // Specified in days. + * } + * }, function(err, apiResponse) { + * if (err) { + * // Error handling omitted. + * } + * + * const lifecycleRules = bucket.metadata.lifecycle.rule; + * + * // Iterate over the Object Lifecycle Management rules on this bucket. + * lifecycleRules.forEach(lifecycleRule => {}); + * }); + * + * //- + * // By default, the rule you provide will be added to the existing policy. + * // Optionally, you can disable this behavior to replace all of the + * // pre-existing rules. + * //- + * const options = { + * append: false + * }; + * + * bucket.addLifecycleRule({ + * action: 'delete', + * condition: { + * age: 365 * 3 // Specified in days. + * } + * }, options, function(err, apiResponse) { + * if (err) { + * // Error handling omitted. + * } + * + * // All rules have been replaced with the new "delete" rule. + * + * // Iterate over the Object Lifecycle Management rules on this bucket. + * lifecycleRules.forEach(lifecycleRule => {}); + * }); + * + * //- + * // For objects created before 2018, "downgrade" the storage class. + * //- + * bucket.addLifecycleRule({ + * action: 'setStorageClass', + * storageClass: 'COLDLINE', + * condition: { + * createdBefore: new Date('2018') + * } + * }, function(err, apiResponse) {}); + * + * //- + * // Delete objects created before 2016 which have the Coldline storage + * // class. + * //- + * bucket.addLifecycleRule({ + * action: 'delete', + * condition: { + * matchesStorageClass: [ + * 'COLDLINE' + * ], + * createdBefore: new Date('2016') + * } + * }, function(err, apiResponse) {}); + * + * //- + * // Delete object that has a noncurrent timestamp that is at least 100 days. + * //- + * bucket.addLifecycleRule({ + * action: 'delete', + * condition: { + * daysSinceNoncurrentTime: 100 + * } + * }, function(err, apiResponse) {}); + * + * //- + * // Delete object that has a noncurrent timestamp before 2020-01-01. + * //- + * bucket.addLifecycleRule({ + * action: 'delete', + * condition: { + * noncurrentTimeBefore: new Date('2020-01-01') + * } + * }, function(err, apiResponse) {}); + * + * //- + * // Delete object that has a customTime that is at least 100 days. + * //- + * bucket.addLifecycleRule({ + * action: 'delete', + * condition: { + * daysSinceCustomTime: 100 + * } + * }, function(err, apiResponse) ()); + * + * //- + * // Delete object that has a customTime before 2020-01-01. + * //- + * bucket.addLifecycleRule({ + * action: 'delete', + * condition: { + * customTimeBefore: new Date('2020-01-01') + * } + * }, function(err, apiResponse) {}); + * ``` + */ + addLifecycleRule( + rule: LifecycleRule | LifecycleRule[], + optionsOrCallback?: AddLifecycleRuleOptions | SetBucketMetadataCallback, + callback?: SetBucketMetadataCallback + ): Promise | void { + let options: AddLifecycleRuleOptions = {}; + + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + options = options || {}; + + const rules = Array.isArray(rule) ? rule : [rule]; + for (const curRule of rules) { + if (curRule.condition.createdBefore instanceof Date) { + curRule.condition.createdBefore = curRule.condition.createdBefore + .toISOString() + .replace(/T.+$/, ''); + } + if (curRule.condition.customTimeBefore instanceof Date) { + curRule.condition.customTimeBefore = curRule.condition.customTimeBefore + .toISOString() + .replace(/T.+$/, ''); + } + if (curRule.condition.noncurrentTimeBefore instanceof Date) { + curRule.condition.noncurrentTimeBefore = + curRule.condition.noncurrentTimeBefore + .toISOString() + .replace(/T.+$/, ''); + } + } + + if (options.append === false) { + this.setMetadata({lifecycle: {rule: rules}}, options, callback!); + return; + } + + // The default behavior appends the previously-defined lifecycle rules with + // the new ones just passed in by the user. + this.getMetadata((err: ApiError | null, metadata: BucketMetadata) => { + if (err) { + callback!(err); + return; + } + + const currentLifecycleRules = Array.isArray(metadata.lifecycle?.rule) + ? metadata.lifecycle?.rule + : []; + + this.setMetadata( + { + lifecycle: {rule: currentLifecycleRules!.concat(rules)}, + }, + options as AddLifecycleRuleOptions, + callback! + ); + }); + } + + combine( + sources: string[] | File[], + destination: string | File, + options?: CombineOptions + ): Promise; + combine( + sources: string[] | File[], + destination: string | File, + options: CombineOptions, + callback: CombineCallback + ): void; + combine( + sources: string[] | File[], + destination: string | File, + callback: CombineCallback + ): void; + /** + * @typedef {object} CombineOptions + * @property {string} [kmsKeyName] Resource name of the Cloud KMS key, of + * the form + * `projects/my-project/locations/location/keyRings/my-kr/cryptoKeys/my-key`, + * that will be used to encrypt the object. Overwrites the object + * metadata's `kms_key_name` value, if any. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @callback CombineCallback + * @param {?Error} err Request error, if any. + * @param {File} newFile The new {@link File}. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {array} CombineResponse + * @property {File} 0 The new {@link File}. + * @property {object} 1 The full API response. + */ + /** + * Combine multiple files into one new file. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/compose| Objects: compose API Documentation} + * + * @throws {Error} if a non-array is provided as sources argument. + * @throws {Error} if no sources are provided. + * @throws {Error} if no destination is provided. + * + * @param {string[]|File[]} sources The source files that will be + * combined. + * @param {string|File} destination The file you would like the + * source files combined into. + * @param {CombineOptions} [options] Configuration options. + * @param {string} [options.kmsKeyName] Resource name of the Cloud KMS key, of + * the form + * `projects/my-project/locations/location/keyRings/my-kr/cryptoKeys/my-key`, + * that will be used to encrypt the object. Overwrites the object + * metadata's `kms_key_name` value, if any. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + + * @param {CombineCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const logBucket = storage.bucket('log-bucket'); + * + * const sources = [ + * logBucket.file('2013-logs.txt'), + * logBucket.file('2014-logs.txt') + * ]; + * + * const allLogs = logBucket.file('all-logs.txt'); + * + * logBucket.combine(sources, allLogs, function(err, newFile, apiResponse) { + * // newFile === allLogs + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * logBucket.combine(sources, allLogs).then(function(data) { + * const newFile = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + combine( + sources: string[] | File[], + destination: string | File, + optionsOrCallback?: CombineOptions | CombineCallback, + callback?: CombineCallback + ): Promise | void { + if (!Array.isArray(sources) || sources.length === 0) { + throw new Error(BucketExceptionMessages.PROVIDE_SOURCE_FILE); + } + + if (!destination) { + throw new Error(BucketExceptionMessages.DESTINATION_FILE_NOT_SPECIFIED); + } + + let options: CombineOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + this.disableAutoRetryConditionallyIdempotent_( + this.methods.setMetadata, // Not relevant but param is required + AvailableServiceObjectMethods.setMetadata, // Same as above + options + ); + + const convertToFile = (file: string | File): File => { + if (file instanceof File) { + return file; + } + return this.file(file); + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + sources = (sources as any).map(convertToFile); + const destinationFile = convertToFile(destination); + callback = callback || util.noop; + + if (!destinationFile.metadata.contentType) { + const destinationContentType = + mime.getType(destinationFile.name) || undefined; + + if (destinationContentType) { + destinationFile.metadata.contentType = destinationContentType; + } + } + + let maxRetries = this.storage.retryOptions.maxRetries; + if ( + (destinationFile?.instancePreconditionOpts?.ifGenerationMatch === + undefined && + options.ifGenerationMatch === undefined && + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryConditional) || + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryNever + ) { + maxRetries = 0; + } + + if (options.ifGenerationMatch === undefined) { + Object.assign(options, destinationFile.instancePreconditionOpts, options); + } + + // Make the request from the destination File object. + destinationFile.request( + { + method: 'POST', + uri: '/compose', + maxRetries, + json: { + destination: { + contentType: destinationFile.metadata.contentType, + contentEncoding: destinationFile.metadata.contentEncoding, + }, + sourceObjects: (sources as File[]).map(source => { + const sourceObject = { + name: source.name, + } as SourceObject; + + if (source.metadata && source.metadata.generation) { + sourceObject.generation = parseInt( + source.metadata.generation.toString() + ); + } + + return sourceObject; + }), + }, + qs: options, + }, + (err, resp) => { + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + if (err) { + callback!(err, null, resp); + return; + } + + callback!(null, destinationFile, resp); + } + ); + } + + createChannel( + id: string, + config: CreateChannelConfig, + options?: CreateChannelOptions + ): Promise; + createChannel( + id: string, + config: CreateChannelConfig, + callback: CreateChannelCallback + ): void; + createChannel( + id: string, + config: CreateChannelConfig, + options: CreateChannelOptions, + callback: CreateChannelCallback + ): void; + /** + * See a {@link https://cloud.google.com/storage/docs/json_api/v1/objects/watchAll| Objects: watchAll request body}. + * + * @typedef {object} CreateChannelConfig + * @property {string} address The address where notifications are + * delivered for this channel. + * @property {string} [delimiter] Returns results in a directory-like mode. + * @property {number} [maxResults] Maximum number of `items` plus `prefixes` + * to return in a single page of responses. + * @property {string} [pageToken] A previously-returned page token + * representing part of the larger set of results to view. + * @property {string} [prefix] Filter results to objects whose names begin + * with this prefix. + * @property {string} [projection=noAcl] Set of properties to return. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + * @property {boolean} [versions=false] If `true`, lists all versions of an object + * as distinct results. + */ + /** + * @typedef {object} CreateChannelOptions + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @typedef {array} CreateChannelResponse + * @property {Channel} 0 The new {@link Channel}. + * @property {object} 1 The full API response. + */ + /** + * @callback CreateChannelCallback + * @param {?Error} err Request error, if any. + * @param {Channel} channel The new {@link Channel}. + * @param {object} apiResponse The full API response. + */ + /** + * Create a channel that will be notified when objects in this bucket changes. + * + * @throws {Error} If an ID is not provided. + * @throws {Error} If an address is not provided. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/watchAll| Objects: watchAll API Documentation} + * + * @param {string} id The ID of the channel to create. + * @param {CreateChannelConfig} config Configuration for creating channel. + * @param {string} config.address The address where notifications are + * delivered for this channel. + * @param {string} [config.delimiter] Returns results in a directory-like mode. + * @param {number} [config.maxResults] Maximum number of `items` plus `prefixes` + * to return in a single page of responses. + * @param {string} [config.pageToken] A previously-returned page token + * representing part of the larger set of results to view. + * @param {string} [config.prefix] Filter results to objects whose names begin + * with this prefix. + * @param {string} [config.projection=noAcl] Set of properties to return. + * @param {string} [config.userProject] The ID of the project which will be + * billed for the request. + * @param {boolean} [config.versions=false] If `true`, lists all versions of an object + * as distinct results. + * @param {CreateChannelOptions} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {CreateChannelCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * const id = 'new-channel-id'; + * + * const config = { + * address: 'https://...' + * }; + * + * bucket.createChannel(id, config, function(err, channel, apiResponse) { + * if (!err) { + * // Channel created successfully. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.createChannel(id, config).then(function(data) { + * const channel = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + createChannel( + id: string, + config: CreateChannelConfig, + optionsOrCallback?: CreateChannelOptions | CreateChannelCallback, + callback?: CreateChannelCallback + ): Promise | void { + if (typeof id !== 'string') { + throw new Error(BucketExceptionMessages.CHANNEL_ID_REQUIRED); + } + + let options: CreateChannelOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + this.request( + { + method: 'POST', + uri: '/o/watch', + json: Object.assign( + { + id, + type: 'web_hook', + }, + config + ), + qs: options, + }, + (err, apiResponse) => { + if (err) { + callback!(err, null, apiResponse); + return; + } + + const resourceId = apiResponse.resourceId; + const channel = this.storage.channel(id, resourceId); + + channel.metadata = apiResponse; + + callback!(null, channel, apiResponse); + } + ); + } + + createNotification( + topic: string, + options?: CreateNotificationOptions + ): Promise; + createNotification( + topic: string, + options: CreateNotificationOptions, + callback: CreateNotificationCallback + ): void; + createNotification(topic: string, callback: CreateNotificationCallback): void; + /** + * Metadata to set for the Notification. + * + * @typedef {object} CreateNotificationOptions + * @property {object} [customAttributes] An optional list of additional + * attributes to attach to each Cloud PubSub message published for this + * notification subscription. + * @property {string[]} [eventTypes] If present, only send notifications about + * listed event types. If empty, sent notifications for all event types. + * @property {string} [objectNamePrefix] If present, only apply this + * notification configuration to object names that begin with this prefix. + * @property {string} [payloadFormat] The desired content of the Payload. + * Defaults to `JSON_API_V1`. + * + * Acceptable values are: + * - `JSON_API_V1` + * + * - `NONE` + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @callback CreateNotificationCallback + * @param {?Error} err Request error, if any. + * @param {Notification} notification The new {@link Notification}. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {array} CreateNotificationResponse + * @property {Notification} 0 The new {@link Notification}. + * @property {object} 1 The full API response. + */ + /** + * Creates a notification subscription for the bucket. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/notifications/insert| Notifications: insert} + * + * @param {Topic|string} topic The Cloud PubSub topic to which this + * subscription publishes. If the project ID is omitted, the current + * project ID will be used. + * + * Acceptable formats are: + * - `projects/grape-spaceship-123/topics/my-topic` + * + * - `my-topic` + * @param {CreateNotificationOptions} [options] Metadata to set for the + * notification. + * @param {object} [options.customAttributes] An optional list of additional + * attributes to attach to each Cloud PubSub message published for this + * notification subscription. + * @param {string[]} [options.eventTypes] If present, only send notifications about + * listed event types. If empty, sent notifications for all event types. + * @param {string} [options.objectNamePrefix] If present, only apply this + * notification configuration to object names that begin with this prefix. + * @param {string} [options.payloadFormat] The desired content of the Payload. + * Defaults to `JSON_API_V1`. + * + * Acceptable values are: + * - `JSON_API_V1` + * + * - `NONE` + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {CreateNotificationCallback} [callback] Callback function. + * @returns {Promise} + * @throws {Error} If a valid topic is not provided. + * @see Notification#create + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const callback = function(err, notification, apiResponse) { + * if (!err) { + * // The notification was created successfully. + * } + * }; + * + * myBucket.createNotification('my-topic', callback); + * + * //- + * // Configure the notification by providing Notification metadata. + * //- + * const metadata = { + * objectNamePrefix: 'prefix-' + * }; + * + * myBucket.createNotification('my-topic', metadata, callback); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * myBucket.createNotification('my-topic').then(function(data) { + * const notification = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/createNotification.js + * region_tag:storage_create_bucket_notifications + * Another example: + */ + createNotification( + topic: string, + optionsOrCallback?: CreateNotificationOptions | CreateNotificationCallback, + callback?: CreateNotificationCallback + ): Promise | void { + let options: CreateNotificationOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + const topicIsObject = topic !== null && typeof topic === 'object'; + if (topicIsObject && util.isCustomType(topic, 'pubsub/topic')) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + topic = (topic as any).name; + } + + if (typeof topic !== 'string') { + throw new Error(BucketExceptionMessages.TOPIC_NAME_REQUIRED); + } + + const body = Object.assign({topic}, options); + + if (body.topic.indexOf('projects') !== 0) { + body.topic = 'projects/{{projectId}}/topics/' + body.topic; + } + + body.topic = `//pubsub.${this.storage.universeDomain}/` + body.topic; + + if (!body.payloadFormat) { + body.payloadFormat = 'JSON_API_V1'; + } + + const query = {} as CreateNotificationQuery; + + if (body.userProject) { + query.userProject = body.userProject; + delete body.userProject; + } + + this.request( + { + method: 'POST', + uri: '/notificationConfigs', + json: convertObjKeysToSnakeCase(body), + qs: query, + maxRetries: 0, //explicitly set this value since this is a non-idempotent function + }, + (err, apiResponse) => { + if (err) { + callback!(err, null, apiResponse); + return; + } + + const notification = this.notification(apiResponse.id); + + notification.metadata = apiResponse; + + callback!(null, notification, apiResponse); + } + ); + } + + deleteFiles(query?: DeleteFilesOptions): Promise; + deleteFiles(callback: DeleteFilesCallback): void; + deleteFiles(query: DeleteFilesOptions, callback: DeleteFilesCallback): void; + /** + * @typedef {object} DeleteFilesOptions Query object. See {@link Bucket#getFiles} + * for all of the supported properties. + * @property {boolean} [force] Suppress errors until all files have been + * processed. + */ + /** + * @callback DeleteFilesCallback + * @param {?Error|?Error[]} err Request error, if any, or array of errors from + * files that were not able to be deleted. + * @param {object} [apiResponse] The full API response. + */ + /** + * Iterate over the bucket's files, calling `file.delete()` on each. + * + * This is not an atomic request. A delete attempt will be + * made for each file individually. Any one can fail, in which case only a + * portion of the files you intended to be deleted would have. + * + * Operations are performed in parallel, up to 10 at once. The first error + * breaks the loop and will execute the provided callback with it. Specify + * `{ force: true }` to suppress the errors until all files have had a chance + * to be processed. + * + * File preconditions cannot be passed to this function. It will not retry unless + * the idempotency strategy is set to retry always. + * + * The `query` object passed as the first argument will also be passed to + * {@link Bucket#getFiles}. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/delete| Objects: delete API Documentation} + * + * @param {DeleteFilesOptions} [query] Query object. See {@link Bucket#getFiles} + * @param {boolean} [query.force] Suppress errors until all files have been + * processed. + * @param {DeleteFilesCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * //- + * // Delete all of the files in the bucket. + * //- + * bucket.deleteFiles(function(err) {}); + * + * //- + * // By default, if a file cannot be deleted, this method will stop deleting + * // files from your bucket. You can override this setting with `force: + * // true`. + * //- + * bucket.deleteFiles({ + * force: true + * }, function(errors) { + * // `errors`: + * // Array of errors if any occurred, otherwise null. + * }); + * + * //- + * // The first argument to this method acts as a query to + * // {@link Bucket#getFiles}. As an example, you can delete files + * // which match a prefix. + * //- + * bucket.deleteFiles({ + * prefix: 'images/' + * }, function(err) { + * if (!err) { + * // All files in the `images` directory have been deleted. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.deleteFiles().then(function() {}); + * ``` + */ + deleteFiles( + queryOrCallback?: DeleteFilesOptions | DeleteFilesCallback, + callback?: DeleteFilesCallback + ): Promise | void { + let query: DeleteFilesOptions = {}; + if (typeof queryOrCallback === 'function') { + callback = queryOrCallback; + } else if (queryOrCallback) { + query = queryOrCallback; + } + + const MAX_PARALLEL_LIMIT = 10; + const MAX_QUEUE_SIZE = 1000; + const errors = [] as Error[]; + + const deleteFile = (file: File) => { + return file.delete(query).catch(err => { + if (!query.force) { + throw err; + } + errors.push(err); + }); + }; + + (async () => { + try { + let promises = []; + const limit = pLimit(MAX_PARALLEL_LIMIT); + const filesStream = this.getFilesStream(query); + + for await (const curFile of filesStream) { + if (promises.length >= MAX_QUEUE_SIZE) { + await Promise.all(promises); + promises = []; + } + promises.push( + limit(() => deleteFile(curFile)).catch(e => { + filesStream.destroy(); + throw e; + }) + ); + } + + await Promise.all(promises); + callback!(errors.length > 0 ? errors : null); + } catch (e) { + callback!(e as Error); + return; + } + })(); + } + + deleteLabels(labels?: string | string[]): Promise; + deleteLabels(options: DeleteLabelsOptions): Promise; + deleteLabels(callback: DeleteLabelsCallback): void; + deleteLabels( + labels: string | string[], + options: DeleteLabelsOptions + ): Promise; + deleteLabels(labels: string | string[], callback: DeleteLabelsCallback): void; + deleteLabels( + labels: string | string[], + options: DeleteLabelsOptions, + callback: DeleteLabelsCallback + ): void; + /** + * @deprecated + * @typedef {array} DeleteLabelsResponse + * @property {object} 0 The full API response. + */ + /** + * @deprecated + * @callback DeleteLabelsCallback + * @param {?Error} err Request error, if any. + * @param {object} metadata Bucket's metadata. + */ + /** + * @deprecated Use setMetadata directly + * Delete one or more labels from this bucket. + * + * @param {string|string[]} [labels] The labels to delete. If no labels are + * provided, all of the labels are removed. + * @param {DeleteLabelsCallback} [callback] Callback function. + * @param {DeleteLabelsOptions} [options] Options, including precondition options + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * //- + * // Delete all of the labels from this bucket. + * //- + * bucket.deleteLabels(function(err, apiResponse) {}); + * + * //- + * // Delete a single label. + * //- + * bucket.deleteLabels('labelone', function(err, apiResponse) {}); + * + * //- + * // Delete a specific set of labels. + * //- + * bucket.deleteLabels([ + * 'labelone', + * 'labeltwo' + * ], function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.deleteLabels().then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + deleteLabels( + labelsOrCallbackOrOptions?: + | string + | string[] + | DeleteLabelsCallback + | DeleteLabelsOptions, + optionsOrCallback?: DeleteLabelsCallback | DeleteLabelsOptions, + callback?: DeleteLabelsCallback + ): Promise | void { + let labels = new Array(); + let options: DeleteLabelsOptions = {}; + + if (typeof labelsOrCallbackOrOptions === 'function') { + callback = labelsOrCallbackOrOptions; + } else if (typeof labelsOrCallbackOrOptions === 'string') { + labels = [labelsOrCallbackOrOptions]; + } else if (Array.isArray(labelsOrCallbackOrOptions)) { + labels = labelsOrCallbackOrOptions; + } else if (labelsOrCallbackOrOptions) { + options = labelsOrCallbackOrOptions; + } + + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + const deleteLabels = (labels: string[]) => { + const nullLabelMap = labels.reduce((nullLabelMap, labelKey) => { + (nullLabelMap as {[index: string]: null})[labelKey] = null; + return nullLabelMap; + }, {}); + + if (options?.ifMetagenerationMatch !== undefined) { + this.setLabels(nullLabelMap, options, callback!); + } else { + this.setLabels(nullLabelMap, callback!); + } + }; + + if (labels.length === 0) { + this.getLabels((err, labels) => { + if (err) { + callback!(err); + return; + } + deleteLabels(Object.keys(labels!)); + }); + } else { + deleteLabels(labels); + } + } + + disableRequesterPays( + options?: DisableRequesterPaysOptions + ): Promise; + disableRequesterPays(callback: DisableRequesterPaysCallback): void; + disableRequesterPays( + options: DisableRequesterPaysOptions, + callback: DisableRequesterPaysCallback + ): void; + /** + * @typedef {array} DisableRequesterPaysResponse + * @property {object} 0 The full API response. + */ + /** + * @callback DisableRequesterPaysCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + *
+ * Early Access Testers Only + *

+ * This feature is not yet widely-available. + *

+ *
+ * + * Disable `requesterPays` functionality from this bucket. + * + * @param {DisableRequesterPaysCallback} [callback] Callback function. + * @param {DisableRequesterPaysOptions} [options] Options, including precondition options + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.disableRequesterPays(function(err, apiResponse) { + * if (!err) { + * // requesterPays functionality disabled successfully. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.disableRequesterPays().then(function(data) { + * const apiResponse = data[0]; + * }); + * + * ``` + * @example include:samples/requesterPays.js + * region_tag:storage_disable_requester_pays + * Example of disabling requester pays: + */ + disableRequesterPays( + optionsOrCallback?: + | DisableRequesterPaysOptions + | DisableRequesterPaysCallback, + callback?: DisableRequesterPaysCallback + ): Promise | void { + let options: DisableRequesterPaysOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + this.setMetadata( + { + billing: { + requesterPays: false, + }, + }, + options, + callback! + ); + } + + enableLogging( + config: EnableLoggingOptions + ): Promise; + enableLogging( + config: EnableLoggingOptions, + callback: SetBucketMetadataCallback + ): void; + /** + * Configuration object for enabling logging. + * + * @typedef {object} EnableLoggingOptions + * @property {string|Bucket} [bucket] The bucket for the log entries. By + * default, the current bucket is used. + * @property {string} prefix A unique prefix for log object names. + */ + /** + * Enable logging functionality for this bucket. This will make two API + * requests, first to grant Cloud Storage WRITE permission to the bucket, then + * to set the appropriate configuration on the Bucket's metadata. + * + * @param {EnableLoggingOptions} config Configuration options. + * @param {string|Bucket} [config.bucket] The bucket for the log entries. By + * default, the current bucket is used. + * @param {string} config.prefix A unique prefix for log object names. + * @param {SetBucketMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * const config = { + * prefix: 'log' + * }; + * + * bucket.enableLogging(config, function(err, apiResponse) { + * if (!err) { + * // Logging functionality enabled successfully. + * } + * }); + * + * ``` + * @example + * Optionally, provide a destination bucket. + * ``` + * const config = { + * prefix: 'log', + * bucket: 'destination-bucket' + * }; + * + * bucket.enableLogging(config, function(err, apiResponse) {}); + * ``` + * + * @example + * If the callback is omitted, we'll return a Promise. + * ``` + * bucket.enableLogging(config).then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + enableLogging( + config: EnableLoggingOptions, + callback?: SetBucketMetadataCallback + ): Promise | void { + if ( + !config || + typeof config === 'function' || + typeof config.prefix === 'undefined' + ) { + throw new Error( + BucketExceptionMessages.CONFIGURATION_OBJECT_PREFIX_REQUIRED + ); + } + + let logBucket = this.id; + if (config.bucket && config.bucket instanceof Bucket) { + logBucket = config.bucket.id; + } else if (config.bucket && typeof config.bucket === 'string') { + logBucket = config.bucket; + } + + const options: PreconditionOptions = {}; + if (config?.ifMetagenerationMatch) { + options.ifMetagenerationMatch = config.ifMetagenerationMatch; + } + if (config?.ifMetagenerationNotMatch) { + options.ifMetagenerationNotMatch = config.ifMetagenerationNotMatch; + } + (async () => { + try { + const [policy] = await this.iam.getPolicy(); + policy.bindings.push({ + members: ['group:cloud-storage-analytics@google.com'], + role: 'roles/storage.objectCreator', + }); + await this.iam.setPolicy(policy); + this.setMetadata( + { + logging: { + logBucket, + logObjectPrefix: config.prefix, + }, + }, + options, + callback! + ); + } catch (e) { + callback!(e as Error); + return; + } + })(); + } + + enableRequesterPays( + options?: EnableRequesterPaysOptions + ): Promise; + enableRequesterPays(callback: EnableRequesterPaysCallback): void; + enableRequesterPays( + options: EnableRequesterPaysOptions, + callback: EnableRequesterPaysCallback + ): void; + + /** + * @typedef {array} EnableRequesterPaysResponse + * @property {object} 0 The full API response. + */ + /** + * @callback EnableRequesterPaysCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + *
+ * Early Access Testers Only + *

+ * This feature is not yet widely-available. + *

+ *
+ * + * Enable `requesterPays` functionality for this bucket. This enables you, the + * bucket owner, to have the requesting user assume the charges for the access + * to your bucket and its contents. + * + * @param {EnableRequesterPaysCallback | EnableRequesterPaysOptions} [optionsOrCallback] + * Callback function or precondition options. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.enableRequesterPays(function(err, apiResponse) { + * if (!err) { + * // requesterPays functionality enabled successfully. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.enableRequesterPays().then(function(data) { + * const apiResponse = data[0]; + * }); + * + * ``` + * @example include:samples/requesterPays.js + * region_tag:storage_enable_requester_pays + * Example of enabling requester pays: + */ + enableRequesterPays( + optionsOrCallback?: + | EnableRequesterPaysCallback + | EnableRequesterPaysOptions, + cb?: EnableRequesterPaysCallback + ): Promise | void { + let options: EnableRequesterPaysOptions = {}; + if (typeof optionsOrCallback === 'function') { + cb = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + this.setMetadata( + { + billing: { + requesterPays: true, + }, + }, + options, + cb! + ); + } + + /** + * Create a {@link File} object. See {@link File} to see how to handle + * the different use cases you may have. + * + * @param {string} name The name of the file in this bucket. + * @param {FileOptions} [options] Configuration options. + * @param {string|number} [options.generation] Only use a specific revision of + * this file. + * @param {string} [options.encryptionKey] A custom encryption key. See + * {@link https://cloud.google.com/storage/docs/encryption#customer-supplied| Customer-supplied Encryption Keys}. + * @param {string} [options.kmsKeyName] The name of the Cloud KMS key that will + * be used to encrypt the object. Must be in the format: + * `projects/my-project/locations/location/keyRings/my-kr/cryptoKeys/my-key`. + * KMS key ring must use the same location as the bucket. + * @param {string} [options.userProject] The ID of the project which will be + * billed for all requests made from File object. + * @returns {File} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * const file = bucket.file('my-existing-file.png'); + * ``` + */ + file(name: string, options?: FileOptions): File { + if (!name) { + throw Error(BucketExceptionMessages.SPECIFY_FILE_NAME); + } + + return new File(this, name, options); + } + + getFiles(query?: GetFilesOptions): Promise; + getFiles(query: GetFilesOptions, callback: GetFilesCallback): void; + getFiles(callback: GetFilesCallback): void; + /** + * @typedef {array} GetFilesResponse + * @property {File[]} 0 Array of {@link File} instances. + * @param {object} nextQuery 1 A query object to receive more results. + * @param {object} apiResponse 2 The full API response. + */ + /** + * @callback GetFilesCallback + * @param {?Error} err Request error, if any. + * @param {File[]} files Array of {@link File} instances. + * @param {object} nextQuery A query object to receive more results. + * @param {object} apiResponse The full API response. + */ + /** + * Query object for listing files. + * + * @typedef {object} GetFilesOptions + * @property {boolean} [autoPaginate=true] Have pagination handled + * automatically. + * @property {string} [delimiter] Results will contain only objects whose + * names, aside from the prefix, do not contain delimiter. Objects whose + * names, aside from the prefix, contain delimiter will have their name + * truncated after the delimiter, returned in `apiResponse.prefixes`. + * Duplicate prefixes are omitted. + * @property {string} [endOffset] Filter results to objects whose names are + * lexicographically before endOffset. If startOffset is also set, the objects + * listed have names between startOffset (inclusive) and endOffset (exclusive). + * @property {boolean} [includeFoldersAsPrefixes] If true, includes folders and + * managed folders in the set of prefixes returned by the query. Only applicable if + * delimiter is set to / and autoPaginate is set to false. + * See: https://cloud.google.com/storage/docs/managed-folders + * @property {boolean} [includeTrailingDelimiter] If true, objects that end in + * exactly one instance of delimiter have their metadata included in items[] + * in addition to the relevant part of the object name appearing in prefixes[]. + * @property {string} [prefix] Filter results to objects whose names begin + * with this prefix. + * @property {string} [matchGlob] A glob pattern used to filter results, + * for example foo*bar + * @property {number} [maxApiCalls] Maximum number of API calls to make. + * @property {number} [maxResults] Maximum number of items plus prefixes to + * return per call. + * Note: By default will handle pagination automatically + * if more than 1 page worth of results are requested per call. + * When `autoPaginate` is set to `false` the smaller of `maxResults` + * or 1 page of results will be returned per call. + * @property {string} [pageToken] A previously-returned page token + * representing part of the larger set of results to view. + * @property {boolean} [softDeleted] If true, only soft-deleted object versions will be + * listed as distinct results in order of generation number. Note `soft_deleted` and + * `versions` cannot be set to true simultaneously. + * @property {string} [startOffset] Filter results to objects whose names are + * lexicographically equal to or after startOffset. If endOffset is also set, + * the objects listed have names between startOffset (inclusive) and endOffset (exclusive). + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + * @property {boolean} [versions] If true, returns File objects scoped to + * their versions. + */ + /** + * Get {@link File} objects for the files currently in the bucket. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/list| Objects: list API Documentation} + * + * @param {GetFilesOptions} [query] Query object for listing files. + * @param {boolean} [query.autoPaginate=true] Have pagination handled + * automatically. + * @param {string} [query.delimiter] Results will contain only objects whose + * names, aside from the prefix, do not contain delimiter. Objects whose + * names, aside from the prefix, contain delimiter will have their name + * truncated after the delimiter, returned in `apiResponse.prefixes`. + * Duplicate prefixes are omitted. + * @param {string} [query.endOffset] Filter results to objects whose names are + * lexicographically before endOffset. If startOffset is also set, the objects + * listed have names between startOffset (inclusive) and endOffset (exclusive). + * @param {boolean} [query.includeFoldersAsPrefixes] If true, includes folders and + * managed folders in the set of prefixes returned by the query. Only applicable if + * delimiter is set to / and autoPaginate is set to false. + * See: https://cloud.google.com/storage/docs/managed-folders + * @param {boolean} [query.includeTrailingDelimiter] If true, objects that end in + * exactly one instance of delimiter have their metadata included in items[] + * in addition to the relevant part of the object name appearing in prefixes[]. + * @param {string} [query.prefix] Filter results to objects whose names begin + * with this prefix. + * @param {number} [query.maxApiCalls] Maximum number of API calls to make. + * @param {number} [query.maxResults] Maximum number of items plus prefixes to + * return per call. + * Note: By default will handle pagination automatically + * if more than 1 page worth of results are requested per call. + * When `autoPaginate` is set to `false` the smaller of `maxResults` + * or 1 page of results will be returned per call. + * @param {string} [query.pageToken] A previously-returned page token + * representing part of the larger set of results to view. + * @param {boolean} [query.softDeleted] If true, only soft-deleted object versions will be + * listed as distinct results in order of generation number. Note `soft_deleted` and + * `versions` cannot be set to true simultaneously. + * @param {string} [query.startOffset] Filter results to objects whose names are + * lexicographically equal to or after startOffset. If endOffset is also set, + * the objects listed have names between startOffset (inclusive) and endOffset (exclusive). + * @param {string} [query.userProject] The ID of the project which will be + * billed for the request. + * @param {boolean} [query.versions] If true, returns File objects scoped to + * their versions. + * @param {GetFilesCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.getFiles(function(err, files) { + * if (!err) { + * // files is an array of File objects. + * } + * }); + * + * //- + * // If your bucket has versioning enabled, you can get all of your files + * // scoped to their generation. + * //- + * bucket.getFiles({ + * versions: true + * }, function(err, files) { + * // Each file is scoped to its generation. + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- + * const callback = function(err, files, nextQuery, apiResponse) { + * if (nextQuery) { + * // More results exist. + * bucket.getFiles(nextQuery, callback); + * } + * + * // The `metadata` property is populated for you with the metadata at the + * // time of fetching. + * files[0].metadata; + * + * // However, in cases where you are concerned the metadata could have + * // changed, use the `getMetadata` method. + * files[0].getMetadata(function(err, metadata) {}); + * }; + * + * bucket.getFiles({ + * autoPaginate: false + * }, callback); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.getFiles().then(function(data) { + * const files = data[0]; + * }); + * + * ``` + * @example + *
Simulating a File System

With `autoPaginate: false`, it's possible to iterate over files which incorporate a common structure using a delimiter.

Consider the following remote objects:

  1. "a"
  2. "a/b/c/d"
  3. "b/d/e"

Using a delimiter of `/` will return a single file, "a".

`apiResponse.prefixes` will return the "sub-directories" that were found:

  1. "a/"
  2. "b/"
+ * ``` + * bucket.getFiles({ + * autoPaginate: false, + * delimiter: '/' + * }, function(err, files, nextQuery, apiResponse) { + * // files = [ + * // {File} // File object for file "a" + * // ] + * + * // apiResponse.prefixes = [ + * // 'a/', + * // 'b/' + * // ] + * }); + * ``` + * + * @example + * Using prefixes, it's now possible to simulate a file system with follow-up requests. + * ``` + * bucket.getFiles({ + * autoPaginate: false, + * delimiter: '/', + * prefix: 'a/' + * }, function(err, files, nextQuery, apiResponse) { + * // No files found within "directory" a. + * // files = [] + * + * // However, a "sub-directory" was found. + * // This prefix can be used to continue traversing the "file system". + * // apiResponse.prefixes = [ + * // 'a/b/' + * // ] + * }); + * ``` + * + * @example include:samples/files.js + * region_tag:storage_list_files + * Another example: + * + * @example include:samples/files.js + * region_tag:storage_list_files_with_prefix + * Example of listing files, filtered by a prefix: + */ + getFiles( + queryOrCallback?: GetFilesOptions | GetFilesCallback, + callback?: GetFilesCallback + ): void | Promise { + let query = typeof queryOrCallback === 'object' ? queryOrCallback : {}; + if (!callback) { + callback = queryOrCallback as GetFilesCallback; + } + query = Object.assign({}, query); + if ( + query.fields && + query.autoPaginate && + !query.fields.includes('nextPageToken') + ) { + query.fields = `${query.fields},nextPageToken`; + } + + this.request( + { + uri: '/o', + qs: query, + }, + (err, resp) => { + if (err) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (callback as any)(err, null, null, resp); + return; + } + + const itemsArray = resp.items ? resp.items : []; + const files = itemsArray.map((file: FileMetadata) => { + const options = {} as FileOptions; + + if (query.fields) { + const fileInstance = file; + return fileInstance; + } + + if (query.versions) { + options.generation = file.generation; + } + + if (file.kmsKeyName) { + options.kmsKeyName = file.kmsKeyName; + } + + const fileInstance = this.file(file.name!, options); + fileInstance.metadata = file; + + return fileInstance; + }); + + let nextQuery: object | null = null; + if (resp.nextPageToken) { + nextQuery = Object.assign({}, query, { + pageToken: resp.nextPageToken, + }); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (callback as any)(null, files, nextQuery, resp); + } + ); + } + + getLabels(options?: GetLabelsOptions): Promise; + getLabels(callback: GetLabelsCallback): void; + getLabels(options: GetLabelsOptions, callback: GetLabelsCallback): void; + /** + * @deprecated + * @typedef {object} GetLabelsOptions Configuration options for Bucket#getLabels(). + * @param {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @deprecated + * @typedef {array} GetLabelsResponse + * @property {object} 0 Object of labels currently set on this bucket. + */ + /** + * @deprecated + * @callback GetLabelsCallback + * @param {?Error} err Request error, if any. + * @param {object} labels Object of labels currently set on this bucket. + */ + /** + * @deprecated Use getMetadata directly. + * Get the labels currently set on this bucket. + * + * @param {object} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {GetLabelsCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.getLabels(function(err, labels) { + * if (err) { + * // Error handling omitted. + * } + * + * // labels = { + * // label: 'labelValue', + * // ... + * // } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.getLabels().then(function(data) { + * const labels = data[0]; + * }); + * ``` + */ + getLabels( + optionsOrCallback?: GetLabelsOptions | GetLabelsCallback, + callback?: GetLabelsCallback + ): Promise | void { + let options: GetLabelsOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + this.getMetadata( + options, + (err: ApiError | null, metadata: BucketMetadata | undefined) => { + if (err) { + callback!(err, null); + return; + } + + callback!(null, metadata?.labels || {}); + } + ); + } + + getNotifications( + options?: GetNotificationsOptions + ): Promise; + getNotifications(callback: GetNotificationsCallback): void; + getNotifications( + options: GetNotificationsOptions, + callback: GetNotificationsCallback + ): void; + /** + * @typedef {object} GetNotificationsOptions Configuration options for Bucket#getNotification(). + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @callback GetNotificationsCallback + * @param {?Error} err Request error, if any. + * @param {Notification[]} notifications Array of {@link Notification} + * instances. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {array} GetNotificationsResponse + * @property {Notification[]} 0 Array of {@link Notification} instances. + * @property {object} 1 The full API response. + */ + /** + * Retrieves a list of notification subscriptions for a given bucket. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/notifications/list| Notifications: list} + * + * @param {GetNotificationsOptions} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {GetNotificationsCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * + * bucket.getNotifications(function(err, notifications, apiResponse) { + * if (!err) { + * // notifications is an array of Notification objects. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.getNotifications().then(function(data) { + * const notifications = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/listNotifications.js + * region_tag:storage_list_bucket_notifications + * Another example: + */ + getNotifications( + optionsOrCallback?: GetNotificationsOptions | GetNotificationsCallback, + callback?: GetNotificationsCallback + ): Promise | void { + let options: GetNotificationsOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + this.request( + { + uri: '/notificationConfigs', + qs: options, + }, + (err, resp) => { + if (err) { + callback!(err, null, resp); + return; + } + const itemsArray = resp.items ? resp.items : []; + const notifications = itemsArray.map( + (notification: NotificationMetadata) => { + const notificationInstance = this.notification(notification.id!); + notificationInstance.metadata = notification; + return notificationInstance; + } + ); + + callback!(null, notifications, resp); + } + ); + } + + getSignedUrl(cfg: GetBucketSignedUrlConfig): Promise; + getSignedUrl( + cfg: GetBucketSignedUrlConfig, + callback: GetSignedUrlCallback + ): void; + /** + * @typedef {array} GetSignedUrlResponse + * @property {object} 0 The signed URL. + */ + /** + * @callback GetSignedUrlCallback + * @param {?Error} err Request error, if any. + * @param {object} url The signed URL. + */ + /** + * @typedef {object} GetBucketSignedUrlConfig + * @property {string} action Only listing objects within a bucket (HTTP: GET) is supported for bucket-level signed URLs. + * @property {*} expires A timestamp when this link will expire. Any value + * given is passed to `new Date()`. + * Note: 'v4' supports maximum duration of 7 days (604800 seconds) from now. + * @property {string} [version='v2'] The signing version to use, either + * 'v2' or 'v4'. + * @property {boolean} [virtualHostedStyle=false] Use virtual hosted-style + * URLs ('https://mybucket.storage.googleapis.com/...') instead of path-style + * ('https://storage.googleapis.com/mybucket/...'). Virtual hosted-style URLs + * should generally be preferred instead of path-style URL. + * Currently defaults to `false` for path-style, although this may change in a + * future major-version release. + * @property {string} [cname] The cname for this bucket, i.e., + * "https://cdn.example.com". + * See [reference]{@link https://cloud.google.com/storage/docs/access-control/signed-urls#example} + * @property {object} [extensionHeaders] If these headers are used, the + * server will check to make sure that the client provides matching + * values. See {@link https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers| Canonical extension headers} + * for the requirements of this feature, most notably: + * - The header name must be prefixed with `x-goog-` + * - The header name must be all lowercase + * + * Note: Multi-valued header passed as an array in the extensionHeaders + * object is converted into a string, delimited by `,` with + * no space. Requests made using the signed URL will need to + * delimit multi-valued headers using a single `,` as well, or + * else the server will report a mismatched signature. + * @property {object} [queryParams] Additional query parameters to include + * in the signed URL. + */ + /** + * Get a signed URL to allow limited time access to a bucket. + * + * In Google Cloud Platform environments, such as Cloud Functions and App + * Engine, you usually don't provide a `keyFilename` or `credentials` during + * instantiation. In those environments, we call the + * {@link https://cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob| signBlob API} + * to create a signed URL. That API requires either the + * `https://www.googleapis.com/auth/iam` or + * `https://www.googleapis.com/auth/cloud-platform` scope, so be sure they are + * enabled. + * + * See {@link https://cloud.google.com/storage/docs/access-control/signed-urls| Signed URLs Reference} + * + * @throws {Error} if an expiration timestamp from the past is given. + * + * @param {GetBucketSignedUrlConfig} config Configuration object. + * @param {string} config.action Currently only supports "list" (HTTP: GET). + * @param {*} config.expires A timestamp when this link will expire. Any value + * given is passed to `new Date()`. + * Note: 'v4' supports maximum duration of 7 days (604800 seconds) from now. + * @param {string} [config.version='v2'] The signing version to use, either + * 'v2' or 'v4'. + * @param {boolean} [config.virtualHostedStyle=false] Use virtual hosted-style + * URLs ('https://mybucket.storage.googleapis.com/...') instead of path-style + * ('https://storage.googleapis.com/mybucket/...'). Virtual hosted-style URLs + * should generally be preferred instead of path-style URL. + * Currently defaults to `false` for path-style, although this may change in a + * future major-version release. + * @param {string} [config.cname] The cname for this bucket, i.e., + * "https://cdn.example.com". + * See [reference]{@link https://cloud.google.com/storage/docs/access-control/signed-urls#example} + * @param {object} [config.extensionHeaders] If these headers are used, the + * server will check to make sure that the client provides matching + * values. See {@link https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers| Canonical extension headers} + * for the requirements of this feature, most notably: + * - The header name must be prefixed with `x-goog-` + * - The header name must be all lowercase + * + * Note: Multi-valued header passed as an array in the extensionHeaders + * object is converted into a string, delimited by `,` with + * no space. Requests made using the signed URL will need to + * delimit multi-valued headers using a single `,` as well, or + * else the server will report a mismatched signature. + * @property {object} [config.queryParams] Additional query parameters to include + * in the signed URL. + * @param {GetSignedUrlCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * //- + * // Generate a URL that allows temporary access to list files in a bucket. + * //- + * const request = require('request'); + * + * const config = { + * action: 'list', + * expires: '03-17-2025' + * }; + * + * bucket.getSignedUrl(config, function(err, url) { + * if (err) { + * console.error(err); + * return; + * } + * + * // The bucket is now available to be listed from this URL. + * request(url, function(err, resp) { + * // resp.statusCode = 200 + * }); + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.getSignedUrl(config).then(function(data) { + * const url = data[0]; + * }); + * ``` + */ + getSignedUrl( + cfg: GetBucketSignedUrlConfig, + callback?: GetSignedUrlCallback + ): void | Promise { + const method = BucketActionToHTTPMethod[cfg.action]; + + const signConfig: SignerGetSignedUrlConfig = { + method, + expires: cfg.expires, + version: cfg.version, + cname: cfg.cname, + extensionHeaders: cfg.extensionHeaders || {}, + queryParams: cfg.queryParams || {}, + host: cfg.host, + signingEndpoint: cfg.signingEndpoint, + }; + + if (!this.signer) { + this.signer = new URLSigner( + this.storage.authClient, + this, + undefined, + this.storage + ); + } + + this.signer + .getSignedUrl(signConfig) + .then(signedUrl => callback!(null, signedUrl), callback!); + } + + lock(metageneration: number | string): Promise; + lock(metageneration: number | string, callback: BucketLockCallback): void; + /** + * @callback BucketLockCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * Lock a previously-defined retention policy. This will prevent changes to + * the policy. + * + * @throws {Error} if a metageneration is not provided. + * + * @param {number|string} metageneration The bucket's metageneration. This is + * accessible from calling {@link File#getMetadata}. + * @param {BucketLockCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const bucket = storage.bucket('albums'); + * + * const metageneration = 2; + * + * bucket.lock(metageneration, function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.lock(metageneration).then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + lock( + metageneration: number | string, + callback?: BucketLockCallback + ): Promise | void { + const metatype = typeof metageneration; + if (metatype !== 'number' && metatype !== 'string') { + throw new Error(BucketExceptionMessages.METAGENERATION_NOT_PROVIDED); + } + + this.request( + { + method: 'POST', + uri: '/lockRetentionPolicy', + qs: { + ifMetagenerationMatch: metageneration, + }, + }, + callback! + ); + } + + /** + * @typedef {object} RestoreOptions Options for Bucket#restore(). See an + * {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/restore#resource| Object resource}. + * @param {number} [generation] If present, selects a specific revision of this object. + * @param {string} [projection] Specifies the set of properties to return. If used, must be 'full' or 'noAcl'. + */ + /** + * Restores a soft-deleted bucket + * @param {RestoreOptions} options Restore options. + * @returns {Promise} + */ + async restore(options: RestoreOptions): Promise { + const [bucket] = await this.request({ + method: 'POST', + uri: '/restore', + qs: options, + }); + + return bucket as Bucket; + } + + makePrivate( + options?: MakeBucketPrivateOptions + ): Promise; + makePrivate(callback: MakeBucketPrivateCallback): void; + makePrivate( + options: MakeBucketPrivateOptions, + callback: MakeBucketPrivateCallback + ): void; + /** + * @typedef {array} MakeBucketPrivateResponse + * @property {File[]} 0 List of files made private. + */ + /** + * @callback MakeBucketPrivateCallback + * @param {?Error} err Request error, if any. + * @param {File[]} files List of files made private. + */ + /** + * @typedef {object} MakeBucketPrivateOptions + * @property {boolean} [includeFiles=false] Make each file in the bucket + * private. + * @property {Metadata} [metadata] Define custom metadata properties to define + * along with the operation. + * @property {boolean} [force] Queue errors occurred while making files + * private until all files have been processed. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * Make the bucket listing private. + * + * You may also choose to make the contents of the bucket private by + * specifying `includeFiles: true`. This will automatically run + * {@link File#makePrivate} for every file in the bucket. + * + * When specifying `includeFiles: true`, use `force: true` to delay execution + * of your callback until all files have been processed. By default, the + * callback is executed after the first error. Use `force` to queue such + * errors until all files have been processed, after which they will be + * returned as an array as the first argument to your callback. + * + * NOTE: This may cause the process to be long-running and use a high number + * of requests. Use with caution. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/patch| Buckets: patch API Documentation} + * + * @param {MakeBucketPrivateOptions} [options] Configuration options. + * @param {boolean} [options.includeFiles=false] Make each file in the bucket + * private. + * @param {Metadata} [options.metadata] Define custom metadata properties to define + * along with the operation. + * @param {boolean} [options.force] Queue errors occurred while making files + * private until all files have been processed. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {MakeBucketPrivateCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * //- + * // Make the bucket private. + * //- + * bucket.makePrivate(function(err) {}); + * + * //- + * // Make the bucket and its contents private. + * //- + * const opts = { + * includeFiles: true + * }; + * + * bucket.makePrivate(opts, function(err, files) { + * // `err`: + * // The first error to occur, otherwise null. + * // + * // `files`: + * // Array of files successfully made private in the bucket. + * }); + * + * //- + * // Make the bucket and its contents private, using force to suppress errors + * // until all files have been processed. + * //- + * const opts = { + * includeFiles: true, + * force: true + * }; + * + * bucket.makePrivate(opts, function(errors, files) { + * // `errors`: + * // Array of errors if any occurred, otherwise null. + * // + * // `files`: + * // Array of files successfully made private in the bucket. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.makePrivate(opts).then(function(data) { + * const files = data[0]; + * }); + * ``` + */ + makePrivate( + optionsOrCallback?: MakeBucketPrivateOptions | MakeBucketPrivateCallback, + callback?: MakeBucketPrivateCallback + ): Promise | void { + const options: MakeBucketPrivateRequest = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + options.private = true; + + const query: MetadataOptions = { + predefinedAcl: 'projectPrivate', + }; + + if (options.userProject) { + query.userProject = options.userProject; + } + + if (options.preconditionOpts?.ifGenerationMatch) { + query.ifGenerationMatch = options.preconditionOpts.ifGenerationMatch; + } + + if (options.preconditionOpts?.ifGenerationNotMatch) { + query.ifGenerationNotMatch = + options.preconditionOpts.ifGenerationNotMatch; + } + + if (options.preconditionOpts?.ifMetagenerationMatch) { + query.ifMetagenerationMatch = + options.preconditionOpts.ifMetagenerationMatch; + } + + if (options.preconditionOpts?.ifMetagenerationNotMatch) { + query.ifMetagenerationNotMatch = + options.preconditionOpts.ifMetagenerationNotMatch; + } + + // You aren't allowed to set both predefinedAcl & acl properties on a bucket + // so acl must explicitly be nullified. + const metadata = {...options.metadata, acl: null}; + + this.setMetadata(metadata, query, (err: Error | null | undefined) => { + if (err) { + callback!(err); + } + const internalCall = () => { + if (options.includeFiles) { + return promisify( + this.makeAllFilesPublicPrivate_ + ).call(this, options); + } + return Promise.resolve([] as File[]); + }; + internalCall() + .then(files => callback!(null, files)) + .catch(callback!); + }); + } + + makePublic( + options?: MakeBucketPublicOptions + ): Promise; + makePublic(callback: MakeBucketPublicCallback): void; + makePublic( + options: MakeBucketPublicOptions, + callback: MakeBucketPublicCallback + ): void; + /** + * @typedef {object} MakeBucketPublicOptions + * @property {boolean} [includeFiles=false] Make each file in the bucket + * private. + * @property {boolean} [force] Queue errors occurred while making files + * private until all files have been processed. + */ + /** + * @callback MakeBucketPublicCallback + * @param {?Error} err Request error, if any. + * @param {File[]} files List of files made public. + */ + /** + * @typedef {array} MakeBucketPublicResponse + * @property {File[]} 0 List of files made public. + */ + /** + * Make the bucket publicly readable. + * + * You may also choose to make the contents of the bucket publicly readable by + * specifying `includeFiles: true`. This will automatically run + * {@link File#makePublic} for every file in the bucket. + * + * When specifying `includeFiles: true`, use `force: true` to delay execution + * of your callback until all files have been processed. By default, the + * callback is executed after the first error. Use `force` to queue such + * errors until all files have been processed, after which they will be + * returned as an array as the first argument to your callback. + * + * NOTE: This may cause the process to be long-running and use a high number + * of requests. Use with caution. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/patch| Buckets: patch API Documentation} + * + * @param {MakeBucketPublicOptions} [options] Configuration options. + * @param {boolean} [options.includeFiles=false] Make each file in the bucket + * private. + * @param {boolean} [options.force] Queue errors occurred while making files + * private until all files have been processed. + * @param {MakeBucketPublicCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * //- + * // Make the bucket publicly readable. + * //- + * bucket.makePublic(function(err) {}); + * + * //- + * // Make the bucket and its contents publicly readable. + * //- + * const opts = { + * includeFiles: true + * }; + * + * bucket.makePublic(opts, function(err, files) { + * // `err`: + * // The first error to occur, otherwise null. + * // + * // `files`: + * // Array of files successfully made public in the bucket. + * }); + * + * //- + * // Make the bucket and its contents publicly readable, using force to + * // suppress errors until all files have been processed. + * //- + * const opts = { + * includeFiles: true, + * force: true + * }; + * + * bucket.makePublic(opts, function(errors, files) { + * // `errors`: + * // Array of errors if any occurred, otherwise null. + * // + * // `files`: + * // Array of files successfully made public in the bucket. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.makePublic(opts).then(function(data) { + * const files = data[0]; + * }); + * ``` + */ + makePublic( + optionsOrCallback?: MakeBucketPublicOptions | MakeBucketPublicCallback, + callback?: MakeBucketPublicCallback + ): Promise | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + const req = {public: true, ...options}; + + this.acl + .add({ + entity: 'allUsers', + role: 'READER', + }) + .then(() => { + return this.acl.default!.add({ + entity: 'allUsers', + role: 'READER', + }); + }) + .then(() => { + if (req.includeFiles) { + return promisify( + this.makeAllFilesPublicPrivate_ + ).call(this, req); + } + return []; + }) + .then(files => callback!(null, files), callback); + } + + /** + * Get a reference to a Cloud Pub/Sub Notification. + * + * @param {string} id ID of notification. + * @returns {Notification} + * @see Notification + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * const notification = bucket.notification('1'); + * ``` + */ + notification(id: string): Notification { + if (!id) { + throw new Error(BucketExceptionMessages.SUPPLY_NOTIFICATION_ID); + } + + return new Notification(this, id); + } + + removeRetentionPeriod( + options?: SetBucketMetadataOptions + ): Promise; + removeRetentionPeriod(callback: SetBucketMetadataCallback): void; + removeRetentionPeriod( + options: SetBucketMetadataOptions, + callback: SetBucketMetadataCallback + ): void; + /** + * Remove an already-existing retention policy from this bucket, if it is not + * locked. + * + * @param {SetBucketMetadataCallback} [callback] Callback function. + * @param {SetBucketMetadataOptions} [options] Options, including precondition options + * @returns {Promise} + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const bucket = storage.bucket('albums'); + * + * bucket.removeRetentionPeriod(function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.removeRetentionPeriod().then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + removeRetentionPeriod( + optionsOrCallback?: SetBucketMetadataOptions | SetBucketMetadataCallback, + callback?: SetBucketMetadataCallback + ): Promise | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + this.setMetadata( + { + retentionPolicy: null, + }, + options, + callback! + ); + } + + request(reqOpts: DecorateRequestOptions): Promise; + request( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ): void; + /** + * Makes request and applies userProject query parameter if necessary. + * + * @private + * + * @param {object} reqOpts - The request options. + * @param {function} callback - The callback function. + */ + request( + reqOpts: DecorateRequestOptions, + callback?: BodyResponseCallback + ): void | Promise { + if (this.userProject && (!reqOpts.qs || !reqOpts.qs.userProject)) { + reqOpts.qs = {...reqOpts.qs, userProject: this.userProject}; + } + return super.request(reqOpts, callback!); + } + + setLabels( + labels: Labels, + options?: SetLabelsOptions + ): Promise; + setLabels(labels: Labels, callback: SetLabelsCallback): void; + setLabels( + labels: Labels, + options: SetLabelsOptions, + callback: SetLabelsCallback + ): void; + /** + * @deprecated + * @typedef {array} SetLabelsResponse + * @property {object} 0 The bucket metadata. + */ + /** + * @deprecated + * @callback SetLabelsCallback + * @param {?Error} err Request error, if any. + * @param {object} metadata The bucket metadata. + */ + /** + * @deprecated + * @typedef {object} SetLabelsOptions Configuration options for Bucket#setLabels(). + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @deprecated Use setMetadata directly. + * Set labels on the bucket. + * + * This makes an underlying call to {@link Bucket#setMetadata}, which + * is a PATCH request. This means an individual label can be overwritten, but + * unmentioned labels will not be touched. + * + * @param {object} labels Labels to set on the bucket. + * @param {SetLabelsOptions} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {SetLabelsCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * const labels = { + * labelone: 'labelonevalue', + * labeltwo: 'labeltwovalue' + * }; + * + * bucket.setLabels(labels, function(err, metadata) { + * if (!err) { + * // Labels set successfully. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.setLabels(labels).then(function(data) { + * const metadata = data[0]; + * }); + * ``` + */ + setLabels( + labels: Labels, + optionsOrCallback?: SetLabelsOptions | SetLabelsCallback, + callback?: SetLabelsCallback + ): Promise | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + callback = callback || util.noop; + + this.setMetadata({labels}, options, callback); + } + + setMetadata( + metadata: BucketMetadata, + options?: SetMetadataOptions + ): Promise>; + setMetadata( + metadata: BucketMetadata, + callback: MetadataCallback + ): void; + setMetadata( + metadata: BucketMetadata, + options: SetMetadataOptions, + callback: MetadataCallback + ): void; + setMetadata( + metadata: BucketMetadata, + optionsOrCallback: SetMetadataOptions | MetadataCallback, + cb?: MetadataCallback + ): Promise> | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + cb = + typeof optionsOrCallback === 'function' + ? (optionsOrCallback as MetadataCallback) + : cb; + + this.disableAutoRetryConditionallyIdempotent_( + this.methods.setMetadata, + AvailableServiceObjectMethods.setMetadata, + options + ); + + super + .setMetadata(metadata, options) + .then(resp => cb!(null, ...resp)) + .catch(cb!) + .finally(() => { + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + }); + } + + setRetentionPeriod( + duration: number, + options?: SetBucketMetadataOptions + ): Promise; + setRetentionPeriod( + duration: number, + callback: SetBucketMetadataCallback + ): void; + setRetentionPeriod( + duration: number, + options: SetBucketMetadataOptions, + callback: SetBucketMetadataCallback + ): void; + /** + * Lock all objects contained in the bucket, based on their creation time. Any + * attempt to overwrite or delete objects younger than the retention period + * will result in a `PERMISSION_DENIED` error. + * + * An unlocked retention policy can be modified or removed from the bucket via + * {@link File#removeRetentionPeriod} and {@link File#setRetentionPeriod}. A + * locked retention policy cannot be removed or shortened in duration for the + * lifetime of the bucket. Attempting to remove or decrease period of a locked + * retention policy will result in a `PERMISSION_DENIED` error. You can still + * increase the policy. + * + * @param {*} duration In seconds, the minimum retention time for all objects + * contained in this bucket. + * @param {SetBucketMetadataCallback} [callback] Callback function. + * @param {SetBucketMetadataCallback} [options] Options, including precondition options. + * @returns {Promise} + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const bucket = storage.bucket('albums'); + * + * const DURATION_SECONDS = 15780000; // 6 months. + * + * //- + * // Lock the objects in this bucket for 6 months. + * //- + * bucket.setRetentionPeriod(DURATION_SECONDS, function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.setRetentionPeriod(DURATION_SECONDS).then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + setRetentionPeriod( + duration: number, + optionsOrCallback?: SetBucketMetadataOptions | SetBucketMetadataCallback, + callback?: SetBucketMetadataCallback + ): Promise | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + this.setMetadata( + { + retentionPolicy: { + retentionPeriod: duration.toString(), + }, + }, + options, + callback! + ); + } + + setCorsConfiguration( + corsConfiguration: Cors[], + options?: SetBucketMetadataOptions + ): Promise; + setCorsConfiguration( + corsConfiguration: Cors[], + callback: SetBucketMetadataCallback + ): void; + setCorsConfiguration( + corsConfiguration: Cors[], + options: SetBucketMetadataOptions, + callback: SetBucketMetadataCallback + ): void; + /** + * + * @typedef {object} Cors + * @property {number} [maxAgeSeconds] The number of seconds the browser is + * allowed to make requests before it must repeat the preflight request. + * @property {string[]} [method] HTTP method allowed for cross origin resource + * sharing with this bucket. + * @property {string[]} [origin] an origin allowed for cross origin resource + * sharing with this bucket. + * @property {string[]} [responseHeader] A header allowed for cross origin + * resource sharing with this bucket. + */ + /** + * This can be used to set the CORS configuration on the bucket. + * + * The configuration will be overwritten with the value passed into this. + * + * @param {Cors[]} corsConfiguration The new CORS configuration to set + * @param {number} [corsConfiguration.maxAgeSeconds] The number of seconds the browser is + * allowed to make requests before it must repeat the preflight request. + * @param {string[]} [corsConfiguration.method] HTTP method allowed for cross origin resource + * sharing with this bucket. + * @param {string[]} [corsConfiguration.origin] an origin allowed for cross origin resource + * sharing with this bucket. + * @param {string[]} [corsConfiguration.responseHeader] A header allowed for cross origin + * resource sharing with this bucket. + * @param {SetBucketMetadataCallback} [callback] Callback function. + * @param {SetBucketMetadataOptions} [options] Options, including precondition options. + * @returns {Promise} + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const bucket = storage.bucket('albums'); + * + * const corsConfiguration = [{maxAgeSeconds: 3600}]; // 1 hour + * bucket.setCorsConfiguration(corsConfiguration); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.setCorsConfiguration(corsConfiguration).then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + setCorsConfiguration( + corsConfiguration: Cors[], + optionsOrCallback?: SetBucketMetadataOptions | SetBucketMetadataCallback, + callback?: SetBucketMetadataCallback + ): Promise | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + this.setMetadata( + { + cors: corsConfiguration, + }, + options, + callback! + ); + } + + setStorageClass( + storageClass: string, + options?: SetBucketStorageClassOptions + ): Promise; + setStorageClass( + storageClass: string, + callback: SetBucketStorageClassCallback + ): void; + setStorageClass( + storageClass: string, + options: SetBucketStorageClassOptions, + callback: SetBucketStorageClassCallback + ): void; + /** + * @typedef {object} SetBucketStorageClassOptions + * @property {string} [userProject] - The ID of the project which will be + * billed for the request. + */ + /** + * @callback SetBucketStorageClassCallback + * @param {?Error} err Request error, if any. + */ + /** + * Set the default storage class for new files in this bucket. + * + * See {@link https://cloud.google.com/storage/docs/storage-classes| Storage Classes} + * + * @param {string} storageClass The new storage class. (`standard`, + * `nearline`, `coldline`, or `archive`). + * **Note:** The storage classes `multi_regional`, `regional`, and + * `durable_reduced_availability` are now legacy and will be deprecated in + * the future. + * @param {object} [options] Configuration options. + * @param {string} [options.userProject] - The ID of the project which will be + * billed for the request. + * @param {SetStorageClassCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.setStorageClass('nearline', function(err, apiResponse) { + * if (err) { + * // Error handling omitted. + * } + * + * // The storage class was updated successfully. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.setStorageClass('nearline').then(function() {}); + * ``` + */ + setStorageClass( + storageClass: string, + optionsOrCallback?: + | SetBucketStorageClassOptions + | SetBucketStorageClassCallback, + callback?: SetBucketStorageClassCallback + ): Promise | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + // In case we get input like `storageClass`, convert to `storage_class`. + storageClass = storageClass + .replace(/-/g, '_') + .replace(/([a-z])([A-Z])/g, (_, low, up) => { + return low + '_' + up; + }) + .toUpperCase(); + + this.setMetadata({storageClass}, options, callback!); + } + + /** + * Set a user project to be billed for all requests made from this Bucket + * object and any files referenced from this Bucket object. + * + * @param {string} userProject The user project. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * bucket.setUserProject('grape-spaceship-123'); + * ``` + */ + setUserProject(userProject: string) { + this.userProject = userProject; + + const methods = [ + 'create', + 'delete', + 'exists', + 'get', + 'getMetadata', + 'setMetadata', + ]; + methods.forEach(method => { + const methodConfig = this.methods[method]; + if (typeof methodConfig === 'object') { + if (typeof methodConfig.reqOpts === 'object') { + Object.assign(methodConfig.reqOpts.qs, {userProject}); + } else { + methodConfig.reqOpts = { + qs: {userProject}, + }; + } + } + }); + } + + upload(pathString: string, options?: UploadOptions): Promise; + upload( + pathString: string, + options: UploadOptions, + callback: UploadCallback + ): void; + upload(pathString: string, callback: UploadCallback): void; + /** + * @typedef {object} UploadOptions Configuration options for Bucket#upload(). + * @property {string|File} [destination] The place to save + * your file. If given a string, the file will be uploaded to the bucket + * using the string as a filename. When given a File object, your local + * file will be uploaded to the File object's bucket and under the File + * object's name. Lastly, when this argument is omitted, the file is uploaded + * to your bucket using the name of the local file. + * @property {string} [encryptionKey] A custom encryption key. See + * {@link https://cloud.google.com/storage/docs/encryption#customer-supplied| Customer-supplied Encryption Keys}. + * @property {boolean} [gzip] Automatically gzip the file. This will set + * `options.metadata.contentEncoding` to `gzip`. + * @property {string} [kmsKeyName] The name of the Cloud KMS key that will + * be used to encrypt the object. Must be in the format: + * `projects/my-project/locations/location/keyRings/my-kr/cryptoKeys/my-key`. + * @property {object} [metadata] See an + * {@link https://cloud.google.com/storage/docs/json_api/v1/objects/insert#request_properties_JSON| Objects: insert request body}. + * @property {string} [offset] The starting byte of the upload stream, for + * resuming an interrupted upload. Defaults to 0. + * @property {string} [predefinedAcl] Apply a predefined set of access + * controls to this object. + * + * Acceptable values are: + * - **`authenticatedRead`** - Object owner gets `OWNER` access, and + * `allAuthenticatedUsers` get `READER` access. + * + * - **`bucketOwnerFullControl`** - Object owner gets `OWNER` access, and + * project team owners get `OWNER` access. + * + * - **`bucketOwnerRead`** - Object owner gets `OWNER` access, and project + * team owners get `READER` access. + * + * - **`private`** - Object owner gets `OWNER` access. + * + * - **`projectPrivate`** - Object owner gets `OWNER` access, and project + * team members get access according to their roles. + * + * - **`publicRead`** - Object owner gets `OWNER` access, and `allUsers` + * get `READER` access. + * @property {boolean} [private] Make the uploaded file private. (Alias for + * `options.predefinedAcl = 'private'`) + * @property {boolean} [public] Make the uploaded file public. (Alias for + * `options.predefinedAcl = 'publicRead'`) + * @property {boolean} [resumable=true] Resumable uploads are automatically + * enabled and must be shut off explicitly by setting to false. + * @property {number} [timeout=60000] Set the HTTP request timeout in + * milliseconds. This option is not available for resumable uploads. + * Default: `60000` + * @property {string} [uri] The URI for an already-created resumable + * upload. See {@link File#createResumableUpload}. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + * @property {string|boolean} [validation] Possible values: `"md5"`, + * `"crc32c"`, or `false`. By default, data integrity is validated with an + * MD5 checksum for maximum reliability. CRC32c will provide better + * performance with less reliability. You may also choose to skip + * validation completely, however this is **not recommended**. + */ + /** + * @typedef {array} UploadResponse + * @property {object} 0 The uploaded {@link File}. + * @property {object} 1 The full API response. + */ + /** + * @callback UploadCallback + * @param {?Error} err Request error, if any. + * @param {object} file The uploaded {@link File}. + * @param {object} apiResponse The full API response. + */ + /** + * Upload a file to the bucket. This is a convenience method that wraps + * {@link File#createWriteStream}. + * + * Resumable uploads are enabled by default + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#uploads| Upload Options (Simple or Resumable)} + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/insert| Objects: insert API Documentation} + * + * @param {string} pathString The fully qualified path to the file you + * wish to upload to your bucket. + * @param {UploadOptions} [options] Configuration options. + * @param {string|File} [options.destination] The place to save + * your file. If given a string, the file will be uploaded to the bucket + * using the string as a filename. When given a File object, your local + * file will be uploaded to the File object's bucket and under the File + * object's name. Lastly, when this argument is omitted, the file is uploaded + * to your bucket using the name of the local file. + * @param {string} [options.encryptionKey] A custom encryption key. See + * {@link https://cloud.google.com/storage/docs/encryption#customer-supplied| Customer-supplied Encryption Keys}. + * @param {boolean} [options.gzip] Automatically gzip the file. This will set + * `options.metadata.contentEncoding` to `gzip`. + * @param {string} [options.kmsKeyName] The name of the Cloud KMS key that will + * be used to encrypt the object. Must be in the format: + * `projects/my-project/locations/location/keyRings/my-kr/cryptoKeys/my-key`. + * @param {object} [options.metadata] See an + * {@link https://cloud.google.com/storage/docs/json_api/v1/objects/insert#request_properties_JSON| Objects: insert request body}. + * @param {string} [options.offset] The starting byte of the upload stream, for + * resuming an interrupted upload. Defaults to 0. + * @param {string} [options.predefinedAcl] Apply a predefined set of access + * controls to this object. + * Acceptable values are: + * - **`authenticatedRead`** - Object owner gets `OWNER` access, and + * `allAuthenticatedUsers` get `READER` access. + * + * - **`bucketOwnerFullControl`** - Object owner gets `OWNER` access, and + * project team owners get `OWNER` access. + * + * - **`bucketOwnerRead`** - Object owner gets `OWNER` access, and project + * team owners get `READER` access. + * + * - **`private`** - Object owner gets `OWNER` access. + * + * - **`projectPrivate`** - Object owner gets `OWNER` access, and project + * team members get access according to their roles. + * + * - **`publicRead`** - Object owner gets `OWNER` access, and `allUsers` + * get `READER` access. + * @param {boolean} [options.private] Make the uploaded file private. (Alias for + * `options.predefinedAcl = 'private'`) + * @param {boolean} [options.public] Make the uploaded file public. (Alias for + * `options.predefinedAcl = 'publicRead'`) + * @param {boolean} [options.resumable=true] Resumable uploads are automatically + * enabled and must be shut off explicitly by setting to false. + * @param {number} [options.timeout=60000] Set the HTTP request timeout in + * milliseconds. This option is not available for resumable uploads. + * Default: `60000` + * @param {string} [options.uri] The URI for an already-created resumable + * upload. See {@link File#createResumableUpload}. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {string|boolean} [options.validation] Possible values: `"md5"`, + * `"crc32c"`, or `false`. By default, data integrity is validated with an + * MD5 checksum for maximum reliability. CRC32c will provide better + * performance with less reliability. You may also choose to skip + * validation completely, however this is **not recommended**. + * @param {UploadCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * + * //- + * // Upload a file from a local path. + * //- + * bucket.upload('/local/path/image.png', function(err, file, apiResponse) { + * // Your bucket now contains: + * // - "image.png" (with the contents of `/local/path/image.png') + * + * // `file` is an instance of a File object that refers to your new file. + * }); + * + * + * //- + * // It's not always that easy. You will likely want to specify the filename + * // used when your new file lands in your bucket. + * // + * // You may also want to set metadata or customize other options. + * //- + * const options = { + * destination: 'new-image.png', + * validation: 'crc32c', + * metadata: { + * metadata: { + * event: 'Fall trip to the zoo' + * } + * } + * }; + * + * bucket.upload('local-image.png', options, function(err, file) { + * // Your bucket now contains: + * // - "new-image.png" (with the contents of `local-image.png') + * + * // `file` is an instance of a File object that refers to your new file. + * }); + * + * //- + * // You can also have a file gzip'd on the fly. + * //- + * bucket.upload('index.html', { gzip: true }, function(err, file) { + * // Your bucket now contains: + * // - "index.html" (automatically compressed with gzip) + * + * // Downloading the file with `file.download` will automatically decode + * the + * // file. + * }); + * + * //- + * // You may also re-use a File object, {File}, that references + * // the file you wish to create or overwrite. + * //- + * const options = { + * destination: bucket.file('existing-file.png'), + * resumable: false + * }; + * + * bucket.upload('local-img.png', options, function(err, newFile) { + * // Your bucket now contains: + * // - "existing-file.png" (with the contents of `local-img.png') + * + * // Note: + * // The `newFile` parameter is equal to `file`. + * }); + * + * //- + * // To use + * // + * // Customer-supplied Encryption Keys, provide the `encryptionKey` + * option. + * //- + * const crypto = require('crypto'); + * const encryptionKey = crypto.randomBytes(32); + * + * bucket.upload('img.png', { + * encryptionKey: encryptionKey + * }, function(err, newFile) { + * // `img.png` was uploaded with your custom encryption key. + * + * // `newFile` is already configured to use the encryption key when making + * // operations on the remote object. + * + * // However, to use your encryption key later, you must create a `File` + * // instance with the `key` supplied: + * const file = bucket.file('img.png', { + * encryptionKey: encryptionKey + * }); + * + * // Or with `file#setEncryptionKey`: + * const file = bucket.file('img.png'); + * file.setEncryptionKey(encryptionKey); + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.upload('local-image.png').then(function(data) { + * const file = data[0]; + * }); + * + * To upload a file from a URL, use {@link File#createWriteStream}. + * + * ``` + * @example include:samples/files.js + * region_tag:storage_upload_file + * Another example: + * + * @example include:samples/encryption.js + * region_tag:storage_upload_encrypted_file + * Example of uploading an encrypted file: + */ + upload( + pathString: string, + optionsOrCallback?: UploadOptions | UploadCallback, + callback?: UploadCallback + ): Promise | void { + const upload = (numberOfRetries: number | undefined) => { + const returnValue = AsyncRetry( + async (bail: (err: Error) => void) => { + await new Promise((resolve, reject) => { + if ( + numberOfRetries === 0 && + newFile?.storage?.retryOptions?.autoRetry + ) { + newFile.storage.retryOptions.autoRetry = false; + } + const writable = newFile.createWriteStream(options); + if (options.onUploadProgress) { + writable.on('progress', options.onUploadProgress); + } + fs.createReadStream(pathString) + .on('error', bail) + .pipe(writable) + .on('error', err => { + if ( + this.storage.retryOptions.autoRetry && + this.storage.retryOptions.retryableErrorFn!(err) + ) { + return reject(err); + } else { + return bail(err); + } + }) + .on('finish', () => { + return resolve(); + }); + }); + }, + { + retries: numberOfRetries, + factor: this.storage.retryOptions.retryDelayMultiplier, + maxTimeout: this.storage.retryOptions.maxRetryDelay! * 1000, //convert to milliseconds + maxRetryTime: this.storage.retryOptions.totalTimeout! * 1000, //convert to milliseconds + } + ); + + if (!callback) { + return returnValue; + } else { + return returnValue + .then(() => { + if (callback) { + return callback!(null, newFile, newFile.metadata); + } + }) + .catch(callback); + } + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((global as any)['GCLOUD_SANDBOX_ENV']) { + return; + } + + let options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + options = Object.assign( + { + metadata: {}, + }, + options + ); + + // Do not retry if precondition option ifGenerationMatch is not set + // because this is a file operation + let maxRetries = this.storage.retryOptions.maxRetries; + if ( + (options?.preconditionOpts?.ifGenerationMatch === undefined && + this.instancePreconditionOpts?.ifGenerationMatch === undefined && + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryConditional) || + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryNever + ) { + maxRetries = 0; + } + + let newFile: File; + if (options.destination instanceof File) { + newFile = options.destination; + } else if ( + options.destination !== null && + typeof options.destination === 'string' + ) { + // Use the string as the name of the file. + newFile = this.file(options.destination, { + encryptionKey: options.encryptionKey, + kmsKeyName: options.kmsKeyName, + preconditionOpts: this.instancePreconditionOpts, + }); + } else { + // Resort to using the name of the incoming file. + const destination = path.basename(pathString); + newFile = this.file(destination, { + encryptionKey: options.encryptionKey, + kmsKeyName: options.kmsKeyName, + preconditionOpts: this.instancePreconditionOpts, + }); + } + + upload(maxRetries); + } + + makeAllFilesPublicPrivate_( + options?: MakeAllFilesPublicPrivateOptions + ): Promise; + makeAllFilesPublicPrivate_(callback: MakeAllFilesPublicPrivateCallback): void; + makeAllFilesPublicPrivate_( + options: MakeAllFilesPublicPrivateOptions, + callback: MakeAllFilesPublicPrivateCallback + ): void; + /** + * @private + * + * @typedef {object} MakeAllFilesPublicPrivateOptions + * @property {boolean} [force] Suppress errors until all files have been + * processed. + * @property {boolean} [private] Make files private. + * @property {boolean} [public] Make files public. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @private + * + * @callback SetBucketMetadataCallback + * @param {?Error} err Request error, if any. + * @param {File[]} files Files that were updated. + */ + /** + * @typedef {array} MakeAllFilesPublicPrivateResponse + * @property {File[]} 0 List of files affected. + */ + /** + * Iterate over all of a bucket's files, calling `file.makePublic()` (public) + * or `file.makePrivate()` (private) on each. + * + * Operations are performed in parallel, up to 10 at once. The first error + * breaks the loop, and will execute the provided callback with it. Specify + * `{ force: true }` to suppress the errors. + * + * @private + * + * @param {MakeAllFilesPublicPrivateOptions} [options] Configuration options. + * @param {boolean} [options.force] Suppress errors until all files have been + * processed. + * @param {boolean} [options.private] Make files private. + * @param {boolean} [options.public] Make files public. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + + * @param {MakeAllFilesPublicPrivateCallback} callback Callback function. + * + * @return {Promise} + */ + makeAllFilesPublicPrivate_( + optionsOrCallback?: + | MakeAllFilesPublicPrivateOptions + | MakeAllFilesPublicPrivateCallback, + callback?: MakeAllFilesPublicPrivateCallback + ): Promise | void { + const MAX_PARALLEL_LIMIT = 10; + const errors = [] as Error[]; + const updatedFiles = [] as File[]; + + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + const processFile = async (file: File) => { + try { + await (options.public ? file.makePublic() : file.makePrivate(options)); + updatedFiles.push(file); + } catch (e) { + if (!options.force) { + throw e; + } + errors.push(e as Error); + } + }; + + this.getFiles(options) + .then(([files]) => { + const limit = pLimit(MAX_PARALLEL_LIMIT); + const promises = files.map(file => { + return limit(() => processFile(file)); + }); + return Promise.all(promises); + }) + .then( + () => callback!(errors.length > 0 ? errors : null, updatedFiles), + err => callback!(err, updatedFiles) + ); + } + + getId(): string { + return this.id!; + } + + disableAutoRetryConditionallyIdempotent_( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + coreOpts: any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + methodType: AvailableServiceObjectMethods, + localPreconditionOptions?: PreconditionOptions + ): void { + if ( + typeof coreOpts === 'object' && + coreOpts?.reqOpts?.qs?.ifMetagenerationMatch === undefined && + localPreconditionOptions?.ifMetagenerationMatch === undefined && + (methodType === AvailableServiceObjectMethods.setMetadata || + methodType === AvailableServiceObjectMethods.delete) && + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryConditional + ) { + this.storage.retryOptions.autoRetry = false; + } else if ( + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryNever + ) { + this.storage.retryOptions.autoRetry = false; + } + } +} + +/*! Developer Documentation + * + * These methods can be auto-paginated. + */ +paginator.extend(Bucket, 'getFiles'); + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +promisifyAll(Bucket, { + exclude: ['cloudStorageURI', 'request', 'file', 'notification', 'restore'], +}); + +/** + * Reference to the {@link Bucket} class. + * @name module:@google-cloud/storage.Bucket + * @see Bucket + */ +export {Bucket}; diff --git a/handwritten/storage/src/channel.ts b/handwritten/storage/src/channel.ts new file mode 100644 index 00000000000..ee0c10984b4 --- /dev/null +++ b/handwritten/storage/src/channel.ts @@ -0,0 +1,126 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {BaseMetadata, ServiceObject, util} from './nodejs-common/index.js'; +import {promisifyAll} from '@google-cloud/promisify'; + +import {Storage} from './storage.js'; + +export interface StopCallback { + (err: Error | null, apiResponse?: unknown): void; +} + +/** + * Create a channel object to interact with a Cloud Storage channel. + * + * See {@link https://cloud.google.com/storage/docs/object-change-notification| Object Change Notification} + * + * @class + * + * @param {string} id The ID of the channel. + * @param {string} resourceId The resource ID of the channel. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const channel = storage.channel('id', 'resource-id'); + * ``` + */ +class Channel extends ServiceObject { + constructor(storage: Storage, id: string, resourceId: string) { + const config = { + parent: storage, + baseUrl: '/channels', + + // An ID shouldn't be included in the API requests. + // RE: + // https://github.com/GoogleCloudPlatform/google-cloud-node/issues/1145 + id: '', + + methods: { + // Only need `request`. + }, + }; + + super(config); + + this.metadata.id = id; + this.metadata.resourceId = resourceId; + } + + stop(): Promise; + stop(callback: StopCallback): void; + /** + * @typedef {array} StopResponse + * @property {object} 0 The full API response. + */ + /** + * @callback StopCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * Stop this channel. + * + * @param {StopCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const channel = storage.channel('id', 'resource-id'); + * channel.stop(function(err, apiResponse) { + * if (!err) { + * // Channel stopped successfully. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * channel.stop().then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + stop(callback?: StopCallback): Promise | void { + callback = callback || util.noop; + this.request( + { + method: 'POST', + uri: '/stop', + json: this.metadata, + }, + (err, apiResponse) => { + callback!(err, apiResponse); + } + ); + } +} + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +promisifyAll(Channel); + +/** + * Reference to the {@link Channel} class. + * @name module:@google-cloud/storage.Channel + * @see Channel + */ +export {Channel}; diff --git a/handwritten/storage/src/crc32c.ts b/handwritten/storage/src/crc32c.ts new file mode 100644 index 00000000000..929cc1d5a95 --- /dev/null +++ b/handwritten/storage/src/crc32c.ts @@ -0,0 +1,339 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {PathLike, createReadStream} from 'fs'; + +/** + * Ported from {@link https://github.com/google/crc32c/blob/21fc8ef30415a635e7351ffa0e5d5367943d4a94/src/crc32c_portable.cc#L16-L59 github.com/google/crc32c} + */ +const CRC32C_EXTENSIONS = [ + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, + 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, + 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, + 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, + 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, + 0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc, 0xb3109ebf, + 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, + 0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, + 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, + 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, + 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, + 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7, 0xfa44e0b4, + 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, + 0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, + 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, + 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, + 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, + 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, + 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, + 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, + 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, + 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351, +] as const; + +const CRC32C_EXTENSION_TABLE = new Int32Array(CRC32C_EXTENSIONS); + +/** An interface for CRC32C hashing and validation */ +interface CRC32CValidator { + /** + * A method returning the CRC32C as a base64-encoded string. + * + * @example + * Hashing the string 'data' should return 'rth90Q==' + * + * ```js + * const buffer = Buffer.from('data'); + * crc32c.update(buffer); + * crc32c.toString(); // 'rth90Q==' + * ``` + **/ + toString: () => string; + /** + * A method validating a base64-encoded CRC32C string. + * + * @example + * Should return `true` if the value matches, `false` otherwise + * + * ```js + * const buffer = Buffer.from('data'); + * crc32c.update(buffer); + * crc32c.validate('DkjKuA=='); // false + * crc32c.validate('rth90Q=='); // true + * ``` + */ + validate: (value: string) => boolean; + /** + * A method for passing `Buffer`s for CRC32C generation. + * + * @example + * Hashing buffers from 'some ' and 'text\n' + * + * ```js + * const buffer1 = Buffer.from('some '); + * crc32c.update(buffer1); + * + * const buffer2 = Buffer.from('text\n'); + * crc32c.update(buffer2); + * + * crc32c.toString(); // 'DkjKuA==' + * ``` + */ + update: (data: Buffer) => void; +} + +/** A function that generates a CRC32C Validator */ +interface CRC32CValidatorGenerator { + /** Should return a new, ready-to-use `CRC32CValidator` */ + (): CRC32CValidator; +} + +const CRC32C_DEFAULT_VALIDATOR_GENERATOR: CRC32CValidatorGenerator = () => + new CRC32C(); + +const CRC32C_EXCEPTION_MESSAGES = { + INVALID_INIT_BASE64_RANGE: (l: number) => + `base64-encoded data expected to equal 4 bytes, not ${l}`, + INVALID_INIT_BUFFER_LENGTH: (l: number) => + `Buffer expected to equal 4 bytes, not ${l}`, + INVALID_INIT_INTEGER: (l: number) => + `Number expected to be a safe, unsigned 32-bit integer, not ${l}`, +} as const; + +class CRC32C implements CRC32CValidator { + /** Current CRC32C value */ + #crc32c = 0; + + /** + * Constructs a new `CRC32C` object. + * + * Reconstruction is recommended via the `CRC32C.from` static method. + * + * @param initialValue An initial CRC32C value - a signed 32-bit integer. + */ + constructor(initialValue = 0) { + this.#crc32c = initialValue; + } + + /** + * Calculates a CRC32C from a provided buffer. + * + * Implementation inspired from: + * - {@link https://github.com/google/crc32c/blob/21fc8ef30415a635e7351ffa0e5d5367943d4a94/src/crc32c_portable.cc github.com/google/crc32c} + * - {@link https://github.com/googleapis/python-crc32c/blob/a595e758c08df445a99c3bf132ee8e80a3ec4308/src/google_crc32c/python.py github.com/googleapis/python-crc32c} + * - {@link https://github.com/googleapis/java-storage/pull/1376/files github.com/googleapis/java-storage} + * + * @param data The `Buffer` to generate the CRC32C from + */ + update(data: Buffer) { + let current = this.#crc32c ^ 0xffffffff; + + for (const d of data) { + const tablePoly = CRC32C.CRC32C_EXTENSION_TABLE[(d ^ current) & 0xff]; + current = tablePoly ^ (current >>> 8); + } + + this.#crc32c = current ^ 0xffffffff; + } + + /** + * Validates a provided input to the current CRC32C value. + * + * @param input A Buffer, `CRC32C`-compatible object, base64-encoded data (string), or signed 32-bit integer + */ + validate(input: Buffer | CRC32CValidator | string | number): boolean { + if (typeof input === 'number') { + return input === this.#crc32c; + } else if (typeof input === 'string') { + return input === this.toString(); + } else if (Buffer.isBuffer(input)) { + return Buffer.compare(input, this.toBuffer()) === 0; + } else { + // `CRC32C`-like object + return input.toString() === this.toString(); + } + } + + /** + * Returns a `Buffer` representation of the CRC32C value + */ + toBuffer(): Buffer { + const buffer = Buffer.alloc(4); + buffer.writeInt32BE(this.#crc32c); + + return buffer; + } + + /** + * Returns a JSON-compatible, base64-encoded representation of the CRC32C value. + * + * See {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify `JSON#stringify`} + */ + toJSON(): string { + return this.toString(); + } + + /** + * Returns a base64-encoded representation of the CRC32C value. + * + * See {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/toString `Object#toString`} + */ + toString(): string { + return this.toBuffer().toString('base64'); + } + + /** + * Returns the `number` representation of the CRC32C value as a signed 32-bit integer + * + * See {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/valueOf `Object#valueOf`} + */ + valueOf(): number { + return this.#crc32c; + } + + static readonly CRC32C_EXTENSIONS = CRC32C_EXTENSIONS; + static readonly CRC32C_EXTENSION_TABLE = CRC32C_EXTENSION_TABLE; + + /** + * Generates a `CRC32C` from a compatible buffer format. + * + * @param value 4-byte `ArrayBufferView`/`Buffer`/`TypedArray` + */ + private static fromBuffer( + value: ArrayBuffer | ArrayBufferView | Buffer + ): CRC32C { + let buffer: Buffer; + + if (Buffer.isBuffer(value)) { + buffer = value; + } else if ('buffer' in value) { + // `ArrayBufferView` + buffer = Buffer.from(value.buffer); + } else { + // `ArrayBuffer` + buffer = Buffer.from(value); + } + + if (buffer.byteLength !== 4) { + throw new RangeError( + CRC32C_EXCEPTION_MESSAGES.INVALID_INIT_BUFFER_LENGTH(buffer.byteLength) + ); + } + + return new CRC32C(buffer.readInt32BE()); + } + + static async fromFile(file: PathLike) { + const crc32c = new CRC32C(); + + await new Promise((resolve, reject) => { + createReadStream(file) + .on('data', (d: string | Buffer) => { + if (typeof d === 'string') { + crc32c.update(Buffer.from(d)); + } else { + crc32c.update(d); + } + }) + .on('end', () => resolve()) + .on('error', reject); + }); + + return crc32c; + } + + /** + * Generates a `CRC32C` from 4-byte base64-encoded data (string). + * + * @param value 4-byte base64-encoded data (string) + */ + private static fromString(value: string): CRC32C { + const buffer = Buffer.from(value, 'base64'); + + if (buffer.byteLength !== 4) { + throw new RangeError( + CRC32C_EXCEPTION_MESSAGES.INVALID_INIT_BASE64_RANGE(buffer.byteLength) + ); + } + + return this.fromBuffer(buffer); + } + + /** + * Generates a `CRC32C` from a safe, unsigned 32-bit integer. + * + * @param value an unsigned 32-bit integer + */ + private static fromNumber(value: number): CRC32C { + if (!Number.isSafeInteger(value) || value > 2 ** 32 || value < -(2 ** 32)) { + throw new RangeError( + CRC32C_EXCEPTION_MESSAGES.INVALID_INIT_INTEGER(value) + ); + } + + return new CRC32C(value); + } + + /** + * Generates a `CRC32C` from a variety of compatable types. + * Note: strings are treated as input, not as file paths to read from. + * + * @param value A number, 4-byte `ArrayBufferView`/`Buffer`/`TypedArray`, or 4-byte base64-encoded data (string) + */ + static from( + value: ArrayBuffer | ArrayBufferView | CRC32CValidator | string | number + ): CRC32C { + if (typeof value === 'number') { + return this.fromNumber(value); + } else if (typeof value === 'string') { + return this.fromString(value); + } else if ('byteLength' in value) { + // `ArrayBuffer` | `Buffer` | `ArrayBufferView` + return this.fromBuffer(value); + } else { + // `CRC32CValidator`/`CRC32C`-like + return this.fromString(value.toString()); + } + } +} + +export { + CRC32C, + CRC32C_DEFAULT_VALIDATOR_GENERATOR, + CRC32C_EXCEPTION_MESSAGES, + CRC32C_EXTENSIONS, + CRC32C_EXTENSION_TABLE, + CRC32CValidator, + CRC32CValidatorGenerator, +}; diff --git a/handwritten/storage/src/file.ts b/handwritten/storage/src/file.ts new file mode 100644 index 00000000000..7a85ae96ea3 --- /dev/null +++ b/handwritten/storage/src/file.ts @@ -0,0 +1,4640 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + BodyResponseCallback, + DecorateRequestOptions, + GetConfig, + Interceptor, + MetadataCallback, + ServiceObject, + SetMetadataResponse, + util, +} from './nodejs-common/index.js'; +import {promisifyAll} from '@google-cloud/promisify'; + +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import mime from 'mime'; +import * as resumableUpload from './resumable-upload.js'; +import {Writable, Readable, pipeline, Transform, PipelineSource} from 'stream'; +import * as zlib from 'zlib'; +import * as http from 'http'; + +import { + ExceptionMessages, + IdempotencyStrategy, + PreconditionOptions, + Storage, +} from './storage.js'; +import {AvailableServiceObjectMethods, Bucket} from './bucket.js'; +import {Acl, AclMetadata} from './acl.js'; +import { + GetSignedUrlResponse, + SigningError, + GetSignedUrlCallback, + URLSigner, + SignerGetSignedUrlConfig, + Query, +} from './signer.js'; +import { + ResponseBody, + ApiError, + Duplexify, + GCCL_GCS_CMD_KEY, +} from './nodejs-common/util.js'; +import duplexify from 'duplexify'; +import { + normalize, + objectKeyToLowercase, + unicodeJSONStringify, + formatAsUTCISO, + PassThroughShim, +} from './util.js'; +import {CRC32C, CRC32CValidatorGenerator} from './crc32c.js'; +import {HashStreamValidator} from './hash-stream-validator.js'; +import {URL} from 'url'; + +import AsyncRetry from 'async-retry'; +import { + BaseMetadata, + DeleteCallback, + DeleteOptions, + GetResponse, + InstanceResponseCallback, + RequestResponse, + SetMetadataOptions, +} from './nodejs-common/service-object.js'; +import * as r from 'teeny-request'; + +export type GetExpirationDateResponse = [Date]; +export interface GetExpirationDateCallback { + ( + err: Error | null, + expirationDate?: Date | null, + apiResponse?: unknown + ): void; +} + +export interface PolicyDocument { + string: string; + base64: string; + signature: string; +} + +export type SaveData = + | string + | Buffer + | Uint8Array + | PipelineSource; + +export type GenerateSignedPostPolicyV2Response = [PolicyDocument]; + +export interface GenerateSignedPostPolicyV2Callback { + (err: Error | null, policy?: PolicyDocument): void; +} + +export interface GenerateSignedPostPolicyV2Options { + equals?: string[] | string[][]; + expires: string | number | Date; + startsWith?: string[] | string[][]; + acl?: string; + successRedirect?: string; + successStatus?: string; + contentLengthRange?: {min?: number; max?: number}; + /** + * @example + * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + */ + signingEndpoint?: string; +} + +export interface PolicyFields { + [key: string]: string; +} + +export interface GenerateSignedPostPolicyV4Options { + expires: string | number | Date; + bucketBoundHostname?: string; + virtualHostedStyle?: boolean; + conditions?: object[]; + fields?: PolicyFields; + /** + * @example + * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + */ + signingEndpoint?: string; +} + +export interface GenerateSignedPostPolicyV4Callback { + (err: Error | null, output?: SignedPostPolicyV4Output): void; +} + +export type GenerateSignedPostPolicyV4Response = [SignedPostPolicyV4Output]; + +export interface SignedPostPolicyV4Output { + url: string; + fields: PolicyFields; +} + +export interface GetSignedUrlConfig + extends Pick { + action: 'read' | 'write' | 'delete' | 'resumable'; + version?: 'v2' | 'v4'; + virtualHostedStyle?: boolean; + cname?: string; + contentMd5?: string; + contentType?: string; + expires: string | number | Date; + accessibleAt?: string | number | Date; + extensionHeaders?: http.OutgoingHttpHeaders; + promptSaveAs?: string; + responseDisposition?: string; + responseType?: string; + queryParams?: Query; +} + +export interface GetFileMetadataOptions { + userProject?: string; +} + +export type GetFileMetadataResponse = [FileMetadata, unknown]; + +export interface GetFileMetadataCallback { + (err: Error | null, metadata?: FileMetadata, apiResponse?: unknown): void; +} + +export interface GetFileOptions extends GetConfig { + userProject?: string; + generation?: number; + restoreToken?: string; + softDeleted?: boolean; +} + +export type GetFileResponse = [File, unknown]; + +export interface GetFileCallback { + (err: Error | null, file?: File, apiResponse?: unknown): void; +} + +export interface FileExistsOptions { + userProject?: string; +} + +export type FileExistsResponse = [boolean]; + +export interface FileExistsCallback { + (err: Error | null, exists?: boolean): void; +} + +export interface DeleteFileOptions { + ignoreNotFound?: boolean; + userProject?: string; +} + +export type DeleteFileResponse = [unknown]; + +export interface DeleteFileCallback { + (err: Error | null, apiResponse?: unknown): void; +} + +export type PredefinedAcl = + | 'authenticatedRead' + | 'bucketOwnerFullControl' + | 'bucketOwnerRead' + | 'private' + | 'projectPrivate' + | 'publicRead'; + +type PublicResumableUploadOptions = + | 'chunkSize' + | 'highWaterMark' + | 'isPartialUpload' + | 'metadata' + | 'origin' + | 'offset' + | 'predefinedAcl' + | 'private' + | 'public' + | 'uri' + | 'userProject'; + +export interface CreateResumableUploadOptions + extends Pick { + /** + * A CRC32C to resume from when continuing a previous upload. It is recommended + * to capture the `crc32c` event from previous upload sessions to provide in + * subsequent requests in order to accurately track the upload. This is **required** + * when validating a final portion of the uploaded object. + * + * @see {@link CRC32C.from} for possible values. + */ + resumeCRC32C?: Parameters<(typeof CRC32C)['from']>[0]; + preconditionOpts?: PreconditionOptions; + [GCCL_GCS_CMD_KEY]?: resumableUpload.UploadConfig[typeof GCCL_GCS_CMD_KEY]; +} + +export type CreateResumableUploadResponse = [string]; + +export interface CreateResumableUploadCallback { + (err: Error | null, uri?: string): void; +} + +export interface CreateWriteStreamOptions extends CreateResumableUploadOptions { + contentType?: string; + gzip?: string | boolean; + resumable?: boolean; + timeout?: number; + validation?: string | boolean; +} + +export interface MakeFilePrivateOptions { + metadata?: FileMetadata; + strict?: boolean; + userProject?: string; + preconditionOpts?: PreconditionOptions; +} + +export type MakeFilePrivateResponse = [unknown]; + +export type MakeFilePrivateCallback = SetFileMetadataCallback; + +export interface IsPublicCallback { + (err: Error | null, resp?: boolean): void; +} + +export type IsPublicResponse = [boolean]; + +export type MakeFilePublicResponse = [unknown]; + +export interface MakeFilePublicCallback { + (err?: Error | null, apiResponse?: unknown): void; +} + +interface MoveFileAtomicQuery { + userProject?: string; + ifGenerationMatch?: number | string; + ifGenerationNotMatch?: number | string; + ifMetagenerationMatch?: number | string; + ifMetagenerationNotMatch?: number | string; +} + +export type MoveResponse = [unknown]; + +export interface MoveCallback { + ( + err: Error | null, + destinationFile?: File | null, + apiResponse?: unknown + ): void; +} + +export interface MoveOptions { + userProject?: string; + preconditionOpts?: PreconditionOptions; +} + +export type MoveFileAtomicOptions = MoveOptions; +export type MoveFileAtomicCallback = MoveCallback; +export type MoveFileAtomicResponse = MoveResponse; + +export type RenameOptions = MoveOptions; +export type RenameResponse = MoveResponse; +export type RenameCallback = MoveCallback; + +export type RotateEncryptionKeyOptions = string | Buffer | EncryptionKeyOptions; + +export interface EncryptionKeyOptions { + encryptionKey?: string | Buffer; + kmsKeyName?: string; + preconditionOpts?: PreconditionOptions; +} + +export type RotateEncryptionKeyCallback = CopyCallback; + +export type RotateEncryptionKeyResponse = CopyResponse; + +export enum ActionToHTTPMethod { + read = 'GET', + write = 'PUT', + delete = 'DELETE', + resumable = 'POST', +} + +/** + * @deprecated - no longer used + */ +export const STORAGE_POST_POLICY_BASE_URL = 'https://storage.googleapis.com'; + +/** + * @private + */ +const GS_URL_REGEXP = /^gs:\/\/([a-z0-9_.-]+)\/(.+)$/; + +/** + * @private + * This regex will match compressible content types. These are primarily text/*, +json, +text, +xml content types. + * This was based off of mime-db and may periodically need to be updated if new compressible content types become + * standards. + */ +const COMPRESSIBLE_MIME_REGEX = new RegExp( + [ + /^text\/|application\/ecmascript|application\/javascript|application\/json/, + /|application\/postscript|application\/rtf|application\/toml|application\/vnd.dart/, + /|application\/vnd.ms-fontobject|application\/wasm|application\/x-httpd-php|application\/x-ns-proxy-autoconfig/, + /|application\/x-sh(?!ockwave-flash)|application\/x-tar|application\/x-virtualbox-hdd|application\/x-virtualbox-ova|application\/x-virtualbox-ovf/, + /|^application\/x-virtualbox-vbox$|application\/x-virtualbox-vdi|application\/x-virtualbox-vhd|application\/x-virtualbox-vmdk/, + /|application\/xml|application\/xml-dtd|font\/otf|font\/ttf|image\/bmp|image\/vnd.adobe.photoshop|image\/vnd.microsoft.icon/, + /|image\/vnd.ms-dds|image\/x-icon|image\/x-ms-bmp|message\/rfc822|model\/gltf-binary|\+json|\+text|\+xml|\+yaml/, + ] + .map(r => r.source) + .join(''), + 'i' +); + +export interface FileOptions { + crc32cGenerator?: CRC32CValidatorGenerator; + encryptionKey?: string | Buffer; + generation?: number | string; + restoreToken?: string; + kmsKeyName?: string; + preconditionOpts?: PreconditionOptions; + userProject?: string; +} + +export interface CopyOptions { + cacheControl?: string; + contentEncoding?: string; + contentType?: string; + contentDisposition?: string; + destinationKmsKeyName?: string; + metadata?: { + [key: string]: string | boolean | number | null; + }; + predefinedAcl?: string; + token?: string; + userProject?: string; + preconditionOpts?: PreconditionOptions; +} + +export type CopyResponse = [File, unknown]; + +export interface CopyCallback { + (err: Error | null, file?: File | null, apiResponse?: unknown): void; +} + +export type DownloadResponse = [Buffer]; + +export type DownloadCallback = ( + err: RequestError | null, + contents: Buffer +) => void; + +export interface DownloadOptions extends CreateReadStreamOptions { + destination?: string; + encryptionKey?: string | Buffer; +} + +interface CopyQuery { + sourceGeneration?: number; + rewriteToken?: string; + userProject?: string; + destinationKmsKeyName?: string; + destinationPredefinedAcl?: string; + ifGenerationMatch?: number | string; + ifGenerationNotMatch?: number | string; + ifMetagenerationMatch?: number | string; + ifMetagenerationNotMatch?: number | string; +} + +interface FileQuery { + alt: string; + generation?: number; + userProject?: string; +} + +export interface CreateReadStreamOptions { + userProject?: string; + validation?: 'md5' | 'crc32c' | false | true; + start?: number; + end?: number; + decompress?: boolean; + [GCCL_GCS_CMD_KEY]?: string; +} + +export interface SaveOptions extends CreateWriteStreamOptions { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + onUploadProgress?: (progressEvent: any) => void; +} + +export interface SaveCallback { + (err?: Error | null): void; +} + +export interface SetFileMetadataOptions { + userProject?: string; +} + +export interface SetFileMetadataCallback { + (err?: Error | null, apiResponse?: unknown): void; +} + +export type SetFileMetadataResponse = [unknown]; + +export type SetStorageClassResponse = [unknown]; + +export interface SetStorageClassOptions { + userProject?: string; + preconditionOpts?: PreconditionOptions; +} + +export interface SetStorageClassCallback { + (err?: Error | null, apiResponse?: unknown): void; +} + +export interface RestoreOptions extends PreconditionOptions { + generation: number; + restoreToken?: string; + projection?: 'full' | 'noAcl'; +} + +export interface FileMetadata extends BaseMetadata { + acl?: AclMetadata[] | null; + bucket?: string; + cacheControl?: string; + componentCount?: number; + contentDisposition?: string; + contentEncoding?: string; + contentLanguage?: string; + contentType?: string; + crc32c?: string; + customerEncryption?: { + encryptionAlgorithm?: string; + keySha256?: string; + }; + customTime?: string; + eventBasedHold?: boolean | null; + readonly eventBasedHoldReleaseTime?: string; + generation?: string | number; + restoreToken?: string; + hardDeleteTime?: string; + kmsKeyName?: string; + md5Hash?: string; + mediaLink?: string; + metadata?: { + [key: string]: string | boolean | number | null; + }; + metageneration?: string | number; + name?: string; + owner?: { + entity?: string; + entityId?: string; + }; + retention?: { + retainUntilTime?: string; + mode?: string; + } | null; + retentionExpirationTime?: string; + size?: string | number; + softDeleteTime?: string; + storageClass?: string; + temporaryHold?: boolean | null; + timeCreated?: string; + timeDeleted?: string; + timeStorageClassUpdated?: string; + updated?: string; +} + +export class RequestError extends Error { + code?: string; + errors?: Error[]; +} + +const SEVEN_DAYS = 7 * 24 * 60 * 60; +const GS_UTIL_URL_REGEX = /(gs):\/\/([a-z0-9_.-]+)\/(.+)/g; +const HTTPS_PUBLIC_URL_REGEX = + /(https):\/\/(storage\.googleapis\.com)\/([a-z0-9_.-]+)\/(.+)/g; + +export enum FileExceptionMessages { + EXPIRATION_TIME_NA = 'An expiration time is not available.', + DESTINATION_NO_NAME = 'Destination file should have a name.', + INVALID_VALIDATION_FILE_RANGE = 'Cannot use validation with file ranges (start/end).', + MD5_NOT_AVAILABLE = 'MD5 verification was specified, but is not available for the requested object. MD5 is not available for composite objects.', + EQUALS_CONDITION_TWO_ELEMENTS = 'Equals condition must be an array of 2 elements.', + STARTS_WITH_TWO_ELEMENTS = 'StartsWith condition must be an array of 2 elements.', + CONTENT_LENGTH_RANGE_MIN_MAX = 'ContentLengthRange must have numeric min & max fields.', + DOWNLOAD_MISMATCH = 'The downloaded data did not match the data from the server. To be sure the content is the same, you should download the file again.', + UPLOAD_MISMATCH_DELETE_FAIL = `The uploaded data did not match the data from the server. + As a precaution, we attempted to delete the file, but it was not successful. + To be sure the content is the same, you should try removing the file manually, + then uploading the file again. + \n\nThe delete attempt failed with this message:\n\n `, + UPLOAD_MISMATCH = `The uploaded data did not match the data from the server. + As a precaution, the file has been deleted. + To be sure the content is the same, you should try uploading the file again.`, + MD5_RESUMED_UPLOAD = 'MD5 cannot be used with a continued resumable upload as MD5 cannot be extended from an existing value', + MISSING_RESUME_CRC32C_FINAL_UPLOAD = 'The CRC32C is missing for the final portion of a resumed upload, which is required for validation. Please provide `resumeCRC32C` if validation is required, or disable `validation`.', +} + +/** + * A File object is created from your {@link Bucket} object using + * {@link Bucket#file}. + * + * @class + */ +class File extends ServiceObject { + acl: Acl; + crc32cGenerator: CRC32CValidatorGenerator; + bucket: Bucket; + storage: Storage; + kmsKeyName?: string; + userProject?: string; + signer?: URLSigner; + name: string; + + generation?: number; + restoreToken?: string; + parent!: Bucket; + + private encryptionKey?: string | Buffer; + private encryptionKeyBase64?: string; + private encryptionKeyHash?: string; + private encryptionKeyInterceptor?: Interceptor; + private instanceRetryValue?: boolean; + instancePreconditionOpts?: PreconditionOptions; + + /** + * Cloud Storage uses access control lists (ACLs) to manage object and + * bucket access. ACLs are the mechanism you use to share objects with other + * users and allow other users to access your buckets and objects. + * + * An ACL consists of one or more entries, where each entry grants permissions + * to an entity. Permissions define the actions that can be performed against + * an object or bucket (for example, `READ` or `WRITE`); the entity defines + * who the permission applies to (for example, a specific user or group of + * users). + * + * The `acl` object on a File instance provides methods to get you a list of + * the ACLs defined on your bucket, as well as set, update, and delete them. + * + * See {@link http://goo.gl/6qBBPO| About Access Control lists} + * + * @name File#acl + * @mixes Acl + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * //- + * // Make a file publicly readable. + * //- + * const options = { + * entity: 'allUsers', + * role: storage.acl.READER_ROLE + * }; + * + * file.acl.add(options, function(err, aclObject) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.acl.add(options).then(function(data) { + * const aclObject = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + /** + * The API-formatted resource description of the file. + * + * Note: This is not guaranteed to be up-to-date when accessed. To get the + * latest record, call the `getMetadata()` method. + * + * @name File#metadata + * @type {object} + */ + /** + * The file's name. + * @name File#name + * @type {string} + */ + /** + * @callback Crc32cGeneratorToStringCallback + * A method returning the CRC32C as a base64-encoded string. + * + * @returns {string} + * + * @example + * Hashing the string 'data' should return 'rth90Q==' + * + * ```js + * const buffer = Buffer.from('data'); + * crc32c.update(buffer); + * crc32c.toString(); // 'rth90Q==' + * ``` + **/ + /** + * @callback Crc32cGeneratorValidateCallback + * A method validating a base64-encoded CRC32C string. + * + * @param {string} [value] base64-encoded CRC32C string to validate + * @returns {boolean} + * + * @example + * Should return `true` if the value matches, `false` otherwise + * + * ```js + * const buffer = Buffer.from('data'); + * crc32c.update(buffer); + * crc32c.validate('DkjKuA=='); // false + * crc32c.validate('rth90Q=='); // true + * ``` + **/ + /** + * @callback Crc32cGeneratorUpdateCallback + * A method for passing `Buffer`s for CRC32C generation. + * + * @param {Buffer} [data] data to update CRC32C value with + * @returns {undefined} + * + * @example + * Hashing buffers from 'some ' and 'text\n' + * + * ```js + * const buffer1 = Buffer.from('some '); + * crc32c.update(buffer1); + * + * const buffer2 = Buffer.from('text\n'); + * crc32c.update(buffer2); + * + * crc32c.toString(); // 'DkjKuA==' + * ``` + **/ + /** + * @typedef {object} CRC32CValidator + * @property {Crc32cGeneratorToStringCallback} + * @property {Crc32cGeneratorValidateCallback} + * @property {Crc32cGeneratorUpdateCallback} + */ + /** + * @callback Crc32cGeneratorCallback + * @returns {CRC32CValidator} + */ + /** + * @typedef {object} FileOptions Options passed to the File constructor. + * @property {string} [encryptionKey] A custom encryption key. + * @property {number} [generation] Generation to scope the file to. + * @property {string} [kmsKeyName] Cloud KMS Key used to encrypt this + * object, if the object is encrypted by such a key. Limited availability; + * usable only by enabled projects. + * @property {string} [userProject] The ID of the project which will be + * billed for all requests made from File object. + * @property {Crc32cGeneratorCallback} [callback] A function that generates a CRC32C Validator. Defaults to {@link CRC32C} + */ + /** + * Constructs a file object. + * + * @param {Bucket} bucket The Bucket instance this file is + * attached to. + * @param {string} name The name of the remote file. + * @param {FileOptions} [options] Configuration options. + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * ``` + */ + constructor(bucket: Bucket, name: string, options: FileOptions = {}) { + const requestQueryObject: { + generation?: number; + userProject?: string; + ifGenerationMatch?: number; + ifGenerationNotMatch?: number; + ifMetagenerationMatch?: number; + ifMetagenerationNotMatch?: number; + } = {}; + + let generation: number; + if (options.generation !== null) { + if (typeof options.generation === 'string') { + generation = Number(options.generation); + } else { + generation = options.generation!; + } + if (!isNaN(generation)) { + requestQueryObject.generation = generation; + } + } + + Object.assign(requestQueryObject, options.preconditionOpts); + + const userProject = options.userProject || bucket.userProject; + if (typeof userProject === 'string') { + requestQueryObject.userProject = userProject; + } + + const methods = { + /** + * @typedef {array} DeleteFileResponse + * @property {object} 0 The full API response. + */ + /** + * @callback DeleteFileCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * Delete the file. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/delete| Objects: delete API Documentation} + * + * @method File#delete + * @param {object} [options] Configuration options. + * @param {boolean} [options.ignoreNotFound = false] Ignore an error if + * the file does not exist. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {DeleteFileCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * file.delete(function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.delete().then(function(data) { + * const apiResponse = data[0]; + * }); + * + * ``` + * @example include:samples/files.js + * region_tag:storage_delete_file + * Another example: + */ + delete: { + reqOpts: { + qs: requestQueryObject, + }, + }, + /** + * @typedef {array} FileExistsResponse + * @property {boolean} 0 Whether the {@link File} exists. + */ + /** + * @callback FileExistsCallback + * @param {?Error} err Request error, if any. + * @param {boolean} exists Whether the {@link File} exists. + */ + /** + * Check if the file exists. + * + * @method File#exists + * @param {options} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {FileExistsCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * file.exists(function(err, exists) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.exists().then(function(data) { + * const exists = data[0]; + * }); + * ``` + */ + exists: { + reqOpts: { + qs: requestQueryObject, + }, + }, + /** + * @typedef {array} GetFileResponse + * @property {File} 0 The {@link File}. + * @property {object} 1 The full API response. + */ + /** + * @callback GetFileCallback + * @param {?Error} err Request error, if any. + * @param {File} file The {@link File}. + * @param {object} apiResponse The full API response. + */ + /** + * Get a file object and its metadata if it exists. + * + * @method File#get + * @param {options} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {number} [options.generation] The generation number to get + * @param {string} [options.restoreToken] If this is a soft-deleted object in an HNS-enabled bucket, returns the restore token which will + * be necessary to restore it if there's a name conflict with another object. + * @param {boolean} [options.softDeleted] If true, returns the soft-deleted object. + Object `generation` is required if `softDeleted` is set to True. + * @param {GetFileCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * file.get(function(err, file, apiResponse) { + * // file.metadata` has been populated. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.get().then(function(data) { + * const file = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + get: { + reqOpts: { + qs: requestQueryObject, + }, + }, + /** + * @typedef {array} GetFileMetadataResponse + * @property {object} 0 The {@link File} metadata. + * @property {object} 1 The full API response. + */ + /** + * @callback GetFileMetadataCallback + * @param {?Error} err Request error, if any. + * @param {object} metadata The {@link File} metadata. + * @param {object} apiResponse The full API response. + */ + /** + * Get the file's metadata. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/get| Objects: get API Documentation} + * + * @method File#getMetadata + * @param {object} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {GetFileMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * file.getMetadata(function(err, metadata, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.getMetadata().then(function(data) { + * const metadata = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/files.js + * region_tag:storage_get_metadata + * Another example: + */ + getMetadata: { + reqOpts: { + qs: requestQueryObject, + }, + }, + /** + * @typedef {object} SetFileMetadataOptions Configuration options for File#setMetadata(). + * @param {string} [userProject] The ID of the project which will be billed for the request. + */ + /** + * @callback SetFileMetadataCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {array} SetFileMetadataResponse + * @property {object} 0 The full API response. + */ + /** + * Merge the given metadata with the current remote file's metadata. This + * will set metadata if it was previously unset or update previously set + * metadata. To unset previously set metadata, set its value to null. + * + * You can set custom key/value pairs in the metadata key of the given + * object, however the other properties outside of this object must adhere + * to the {@link https://goo.gl/BOnnCK| official API documentation}. + * + * + * See the examples below for more information. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/patch| Objects: patch API Documentation} + * + * @method File#setMetadata + * @param {object} [metadata] The metadata you wish to update. + * @param {SetFileMetadataOptions} [options] Configuration options. + * @param {SetFileMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * const metadata = { + * contentType: 'application/x-font-ttf', + * metadata: { + * my: 'custom', + * properties: 'go here' + * } + * }; + * + * file.setMetadata(metadata, function(err, apiResponse) {}); + * + * // Assuming current metadata = { hello: 'world', unsetMe: 'will do' } + * file.setMetadata({ + * metadata: { + * abc: '123', // will be set. + * unsetMe: null, // will be unset (deleted). + * hello: 'goodbye' // will be updated from 'world' to 'goodbye'. + * } + * }, function(err, apiResponse) { + * // metadata should now be { abc: '123', hello: 'goodbye' } + * }); + * + * //- + * // Set a temporary hold on this file from its bucket's retention period + * // configuration. + * // + * file.setMetadata({ + * temporaryHold: true + * }, function(err, apiResponse) {}); + * + * //- + * // Alternatively, you may set a temporary hold. This will follow the + * // same behavior as an event-based hold, with the exception that the + * // bucket's retention policy will not renew for this file from the time + * // the hold is released. + * //- + * file.setMetadata({ + * eventBasedHold: true + * }, function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.setMetadata(metadata).then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + setMetadata: { + reqOpts: { + qs: requestQueryObject, + }, + }, + }; + + super({ + parent: bucket, + baseUrl: '/o', + id: encodeURIComponent(name), + methods, + }); + + this.bucket = bucket; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.storage = (bucket as any).parent as Storage; + + // @TODO Can this duplicate code from above be avoided? + if (options.generation !== null) { + let generation: number; + if (typeof options.generation === 'string') { + generation = Number(options.generation); + } else { + generation = options.generation!; + } + if (!isNaN(generation)) { + this.generation = generation; + } + } + this.kmsKeyName = options.kmsKeyName; + this.userProject = userProject; + + this.name = name; + + if (options.encryptionKey) { + this.setEncryptionKey(options.encryptionKey); + } + + this.acl = new Acl({ + request: this.request.bind(this), + pathPrefix: '/acl', + }); + + this.crc32cGenerator = + options.crc32cGenerator || this.bucket.crc32cGenerator; + + this.instanceRetryValue = this.storage?.retryOptions?.autoRetry; + this.instancePreconditionOpts = options?.preconditionOpts; + } + + /** + * The object's Cloud Storage URI (`gs://`) + * + * @example + * ```ts + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * const file = bucket.file('image.png'); + * + * // `gs://my-bucket/image.png` + * const href = file.cloudStorageURI.href; + * ``` + */ + get cloudStorageURI(): URL { + const uri = this.bucket.cloudStorageURI; + + uri.pathname = this.name; + + return uri; + } + + /** + * A helper method for determining if a request should be retried based on preconditions. + * This should only be used for methods where the idempotency is determined by + * `ifGenerationMatch` + * @private + * + * A request should not be retried under the following conditions: + * - if precondition option `ifGenerationMatch` is not set OR + * - if `idempotencyStrategy` is set to `RetryNever` + */ + private shouldRetryBasedOnPreconditionAndIdempotencyStrat( + options?: PreconditionOptions + ): boolean { + return !( + (options?.ifGenerationMatch === undefined && + this.instancePreconditionOpts?.ifGenerationMatch === undefined && + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryConditional) || + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryNever + ); + } + + copy( + destination: string | Bucket | File, + options?: CopyOptions + ): Promise; + copy(destination: string | Bucket | File, callback: CopyCallback): void; + copy( + destination: string | Bucket | File, + options: CopyOptions, + callback: CopyCallback + ): void; + /** + * @typedef {array} CopyResponse + * @property {File} 0 The copied {@link File}. + * @property {object} 1 The full API response. + */ + /** + * @callback CopyCallback + * @param {?Error} err Request error, if any. + * @param {File} copiedFile The copied {@link File}. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {object} CopyOptions Configuration options for File#copy(). See an + * {@link https://cloud.google.com/storage/docs/json_api/v1/objects#resource| Object resource}. + * @property {string} [cacheControl] The cacheControl setting for the new file. + * @property {string} [contentEncoding] The contentEncoding setting for the new file. + * @property {string} [contentType] The contentType setting for the new file. + * @property {string} [destinationKmsKeyName] Resource name of the Cloud + * KMS key, of the form + * `projects/my-project/locations/location/keyRings/my-kr/cryptoKeys/my-key`, + * that will be used to encrypt the object. Overwrites the object + * metadata's `kms_key_name` value, if any. + * @property {Metadata} [metadata] Metadata to specify on the copied file. + * @property {string} [predefinedAcl] Set the ACL for the new file. + * @property {string} [token] A previously-returned `rewriteToken` from an + * unfinished rewrite request. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * Copy this file to another file. By default, this will copy the file to the + * same bucket, but you can choose to copy it to another Bucket by providing + * a Bucket or File object or a URL starting with "gs://". + * The generation of the file will not be preserved. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite| Objects: rewrite API Documentation} + * + * @throws {Error} If the destination file is not provided. + * + * @param {string|Bucket|File} destination Destination file. + * @param {CopyOptions} [options] Configuration options. See an + * @param {CopyCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * + * //- + * // You can pass in a variety of types for the destination. + * // + * // For all of the below examples, assume we are working with the following + * // Bucket and File objects. + * //- + * const bucket = storage.bucket('my-bucket'); + * const file = bucket.file('my-image.png'); + * + * //- + * // If you pass in a string for the destination, the file is copied to its + * // current bucket, under the new name provided. + * //- + * file.copy('my-image-copy.png', function(err, copiedFile, apiResponse) { + * // `my-bucket` now contains: + * // - "my-image.png" + * // - "my-image-copy.png" + * + * // `copiedFile` is an instance of a File object that refers to your new + * // file. + * }); + * + * //- + * // If you pass in a string starting with "gs://" for the destination, the + * // file is copied to the other bucket and under the new name provided. + * //- + * const newLocation = 'gs://another-bucket/my-image-copy.png'; + * file.copy(newLocation, function(err, copiedFile, apiResponse) { + * // `my-bucket` still contains: + * // - "my-image.png" + * // + * // `another-bucket` now contains: + * // - "my-image-copy.png" + * + * // `copiedFile` is an instance of a File object that refers to your new + * // file. + * }); + * + * //- + * // If you pass in a Bucket object, the file will be copied to that bucket + * // using the same name. + * //- + * const anotherBucket = storage.bucket('another-bucket'); + * file.copy(anotherBucket, function(err, copiedFile, apiResponse) { + * // `my-bucket` still contains: + * // - "my-image.png" + * // + * // `another-bucket` now contains: + * // - "my-image.png" + * + * // `copiedFile` is an instance of a File object that refers to your new + * // file. + * }); + * + * //- + * // If you pass in a File object, you have complete control over the new + * // bucket and filename. + * //- + * const anotherFile = anotherBucket.file('my-awesome-image.png'); + * file.copy(anotherFile, function(err, copiedFile, apiResponse) { + * // `my-bucket` still contains: + * // - "my-image.png" + * // + * // `another-bucket` now contains: + * // - "my-awesome-image.png" + * + * // Note: + * // The `copiedFile` parameter is equal to `anotherFile`. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.copy(newLocation).then(function(data) { + * const newFile = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/files.js + * region_tag:storage_copy_file + * Another example: + */ + copy( + destination: string | Bucket | File, + optionsOrCallback?: CopyOptions | CopyCallback, + callback?: CopyCallback + ): Promise | void { + const noDestinationError = new Error( + FileExceptionMessages.DESTINATION_NO_NAME + ); + + if (!destination) { + throw noDestinationError; + } + + let options: CopyOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = {...optionsOrCallback}; + } + + callback = callback || util.noop; + + let destBucket: Bucket; + let destName: string; + let newFile: File; + + if (typeof destination === 'string') { + const parsedDestination = GS_URL_REGEXP.exec(destination); + if (parsedDestination !== null && parsedDestination.length === 3) { + destBucket = this.storage.bucket(parsedDestination[1]); + destName = parsedDestination[2]; + } else { + destBucket = this.bucket; + destName = destination; + } + } else if (destination instanceof Bucket) { + destBucket = destination; + destName = this.name; + } else if (destination instanceof File) { + destBucket = destination.bucket; + destName = destination.name; + newFile = destination; + } else { + throw noDestinationError; + } + + const query = {} as CopyQuery; + if (this.generation !== undefined) { + query.sourceGeneration = this.generation; + } + if (options.token !== undefined) { + query.rewriteToken = options.token; + } + if (options.userProject !== undefined) { + query.userProject = options.userProject; + delete options.userProject; + } + if (options.predefinedAcl !== undefined) { + query.destinationPredefinedAcl = options.predefinedAcl; + delete options.predefinedAcl; + } + + newFile = newFile! || destBucket.file(destName); + + const headers: {[index: string]: string | undefined} = {}; + + if (this.encryptionKey !== undefined) { + headers['x-goog-copy-source-encryption-algorithm'] = 'AES256'; + headers['x-goog-copy-source-encryption-key'] = this.encryptionKeyBase64; + headers['x-goog-copy-source-encryption-key-sha256'] = + this.encryptionKeyHash; + } + + if (newFile.encryptionKey !== undefined) { + this.setEncryptionKey(newFile.encryptionKey!); + } else if (options.destinationKmsKeyName !== undefined) { + query.destinationKmsKeyName = options.destinationKmsKeyName; + delete options.destinationKmsKeyName; + } else if (newFile.kmsKeyName !== undefined) { + query.destinationKmsKeyName = newFile.kmsKeyName; + } + + if (query.destinationKmsKeyName) { + this.kmsKeyName = query.destinationKmsKeyName; + + const keyIndex = this.interceptors.indexOf( + this.encryptionKeyInterceptor! + ); + if (keyIndex > -1) { + this.interceptors.splice(keyIndex, 1); + } + } + + if ( + !this.shouldRetryBasedOnPreconditionAndIdempotencyStrat( + options?.preconditionOpts + ) + ) { + this.storage.retryOptions.autoRetry = false; + } + + if (options.preconditionOpts?.ifGenerationMatch !== undefined) { + query.ifGenerationMatch = options.preconditionOpts?.ifGenerationMatch; + delete options.preconditionOpts; + } + + this.request( + { + method: 'POST', + uri: `/rewriteTo/b/${destBucket.name}/o/${encodeURIComponent( + newFile.name + )}`, + qs: query, + json: options, + headers, + }, + (err, resp) => { + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + if (err) { + callback!(err, null, resp); + return; + } + + if (resp.rewriteToken) { + const options = { + token: resp.rewriteToken, + } as CopyOptions; + + if (query.userProject) { + options.userProject = query.userProject; + } + + if (query.destinationKmsKeyName) { + options.destinationKmsKeyName = query.destinationKmsKeyName; + } + + this.copy(newFile, options, callback!); + return; + } + + callback!(null, newFile, resp); + } + ); + } + + /** + * @typedef {object} CreateReadStreamOptions Configuration options for File#createReadStream. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + * @property {string|boolean} [validation] Possible values: `"md5"`, + * `"crc32c"`, or `false`. By default, data integrity is validated with a + * CRC32c checksum. You may use MD5 if preferred, but that hash is not + * supported for composite objects. An error will be raised if MD5 is + * specified but is not available. You may also choose to skip validation + * completely, however this is **not recommended**. + * @property {number} [start] A byte offset to begin the file's download + * from. Default is 0. NOTE: Byte ranges are inclusive; that is, + * `options.start = 0` and `options.end = 999` represent the first 1000 + * bytes in a file or object. NOTE: when specifying a byte range, data + * integrity is not available. + * @property {number} [end] A byte offset to stop reading the file at. + * NOTE: Byte ranges are inclusive; that is, `options.start = 0` and + * `options.end = 999` represent the first 1000 bytes in a file or object. + * NOTE: when specifying a byte range, data integrity is not available. + * @property {boolean} [decompress=true] Disable auto decompression of the + * received data. By default this option is set to `true`. + * Applicable in cases where the data was uploaded with + * `gzip: true` option. See {@link File#createWriteStream}. + */ + /** + * Create a readable stream to read the contents of the remote file. It can be + * piped to a writable stream or listened to for 'data' events to read a + * file's contents. + * + * In the unlikely event there is a mismatch between what you downloaded and + * the version in your Bucket, your error handler will receive an error with + * code "CONTENT_DOWNLOAD_MISMATCH". If you receive this error, the best + * recourse is to try downloading the file again. + * + * NOTE: Readable streams will emit the `end` event when the file is fully + * downloaded. + * + * @param {CreateReadStreamOptions} [options] Configuration options. + * @returns {ReadableStream} + * + * @example + * ``` + * //- + * //

Downloading a File

+ * // + * // The example below demonstrates how we can reference a remote file, then + * // pipe its contents to a local file. This is effectively creating a local + * // backup of your remote data. + * //- + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * + * const fs = require('fs'); + * const remoteFile = bucket.file('image.png'); + * const localFilename = '/Users/stephen/Photos/image.png'; + * + * remoteFile.createReadStream() + * .on('error', function(err) {}) + * .on('response', function(response) { + * // Server connected and responded with the specified status and headers. + * }) + * .on('end', function() { + * // The file is fully downloaded. + * }) + * .pipe(fs.createWriteStream(localFilename)); + * + * //- + * // To limit the downloaded data to only a byte range, pass an options + * // object. + * //- + * const logFile = myBucket.file('access_log'); + * logFile.createReadStream({ + * start: 10000, + * end: 20000 + * }) + * .on('error', function(err) {}) + * .pipe(fs.createWriteStream('/Users/stephen/logfile.txt')); + * + * //- + * // To read a tail byte range, specify only `options.end` as a negative + * // number. + * //- + * const logFile = myBucket.file('access_log'); + * logFile.createReadStream({ + * end: -100 + * }) + * .on('error', function(err) {}) + * .pipe(fs.createWriteStream('/Users/stephen/logfile.txt')); + * ``` + */ + createReadStream(options: CreateReadStreamOptions = {}): Readable { + options = Object.assign({decompress: true}, options); + const rangeRequest = + typeof options.start === 'number' || typeof options.end === 'number'; + const tailRequest = options.end! < 0; + + let validateStream: HashStreamValidator | undefined = undefined; + let request: r.Request | undefined = undefined; + + const throughStream = new PassThroughShim(); + + let crc32c = true; + let md5 = false; + + if (typeof options.validation === 'string') { + const value = options.validation.toLowerCase().trim(); + + crc32c = value === 'crc32c'; + md5 = value === 'md5'; + } else if (options.validation === false) { + crc32c = false; + } + + const shouldRunValidation = !rangeRequest && (crc32c || md5); + + if (rangeRequest) { + if ( + typeof options.validation === 'string' || + options.validation === true + ) { + throw new Error(FileExceptionMessages.INVALID_VALIDATION_FILE_RANGE); + } + // Range requests can't receive data integrity checks. + crc32c = false; + md5 = false; + } + + const onComplete = (err: Error | null) => { + if (err) { + // There is an issue with node-fetch 2.x that if the stream errors the underlying socket connection is not closed. + // This causes a memory leak, so cleanup the sockets manually here by destroying the agent. + if (request?.agent) { + request.agent.destroy(); + } + throughStream.destroy(err); + } + }; + + // We listen to the response event from the request stream so that we + // can... + // + // 1) Intercept any data from going to the user if an error occurred. + // 2) Calculate the hashes from the http.IncomingMessage response + // stream, + // which will return the bytes from the source without decompressing + // gzip'd content. We then send it through decompressed, if + // applicable, to the user. + const onResponse = ( + err: Error | null, + _body: ResponseBody, + rawResponseStream: unknown + ) => { + if (err) { + // Get error message from the body. + this.getBufferFromReadable(rawResponseStream as Readable).then(body => { + err.message = body.toString('utf8'); + throughStream.destroy(err); + }); + + return; + } + + request = (rawResponseStream as r.Response).request; + const headers = (rawResponseStream as ResponseBody).toJSON().headers; + const isCompressed = headers['content-encoding'] === 'gzip'; + const hashes: {crc32c?: string; md5?: string} = {}; + + // The object is safe to validate if: + // 1. It was stored gzip and returned to us gzip OR + // 2. It was never stored as gzip + const safeToValidate = + (headers['x-goog-stored-content-encoding'] === 'gzip' && + isCompressed) || + headers['x-goog-stored-content-encoding'] === 'identity'; + + const transformStreams: Transform[] = []; + + if (shouldRunValidation) { + // The x-goog-hash header should be set with a crc32c and md5 hash. + // ex: headers['x-goog-hash'] = 'crc32c=xxxx,md5=xxxx' + if (typeof headers['x-goog-hash'] === 'string') { + headers['x-goog-hash'] + .split(',') + .forEach((hashKeyValPair: string) => { + const delimiterIndex = hashKeyValPair.indexOf('='); + const hashType = hashKeyValPair.substring(0, delimiterIndex); + const hashValue = hashKeyValPair.substring(delimiterIndex + 1); + hashes[hashType as 'crc32c' | 'md5'] = hashValue; + }); + } + + validateStream = new HashStreamValidator({ + crc32c, + md5, + crc32cGenerator: this.crc32cGenerator, + crc32cExpected: hashes.crc32c, + md5Expected: hashes.md5, + }); + } + + if (md5 && !hashes.md5) { + const hashError = new RequestError( + FileExceptionMessages.MD5_NOT_AVAILABLE + ); + hashError.code = 'MD5_NOT_AVAILABLE'; + throughStream.destroy(hashError); + return; + } + + if (safeToValidate && shouldRunValidation && validateStream) { + transformStreams.push(validateStream); + } + + if (isCompressed && options.decompress) { + transformStreams.push(zlib.createGunzip()); + } + + pipeline( + rawResponseStream as Readable, + ...(transformStreams as [Transform]), + throughStream, + onComplete + ); + }; + + // Authenticate the request, then pipe the remote API request to the stream + // returned to the user. + const makeRequest = () => { + const query: FileQuery = {alt: 'media'}; + + if (this.generation) { + query.generation = this.generation; + } + + if (options.userProject) { + query.userProject = options.userProject; + } + + interface Headers { + [index: string]: string; + } + + const headers = { + 'Accept-Encoding': 'gzip', + 'Cache-Control': 'no-store', + } as Headers; + + if (rangeRequest) { + const start = typeof options.start === 'number' ? options.start : '0'; + const end = typeof options.end === 'number' ? options.end : ''; + + headers.Range = `bytes=${tailRequest ? end : `${start}-${end}`}`; + } + + const reqOpts: DecorateRequestOptions = { + uri: '', + headers, + qs: query, + }; + + if (options[GCCL_GCS_CMD_KEY]) { + reqOpts[GCCL_GCS_CMD_KEY] = options[GCCL_GCS_CMD_KEY]; + } + + this.requestStream(reqOpts) + .on('error', err => { + throughStream.destroy(err); + }) + .on('response', res => { + throughStream.emit('response', res); + util.handleResp(null, res, null, onResponse); + }) + .resume(); + }; + throughStream.on('reading', makeRequest); + + return throughStream; + } + + createResumableUpload( + options?: CreateResumableUploadOptions + ): Promise; + createResumableUpload( + options: CreateResumableUploadOptions, + callback: CreateResumableUploadCallback + ): void; + createResumableUpload(callback: CreateResumableUploadCallback): void; + /** + * @callback CreateResumableUploadCallback + * @param {?Error} err Request error, if any. + * @param {string} uri The resumable upload's unique session URI. + */ + /** + * @typedef {array} CreateResumableUploadResponse + * @property {string} 0 The resumable upload's unique session URI. + */ + /** + * @typedef {object} CreateResumableUploadOptions + * @property {object} [metadata] Metadata to set on the file. + * @property {number} [offset] The starting byte of the upload stream for resuming an interrupted upload. + * @property {string} [origin] Origin header to set for the upload. + * @property {string} [predefinedAcl] Apply a predefined set of access + * controls to this object. + * + * Acceptable values are: + * - **`authenticatedRead`** - Object owner gets `OWNER` access, and + * `allAuthenticatedUsers` get `READER` access. + * + * - **`bucketOwnerFullControl`** - Object owner gets `OWNER` access, and + * project team owners get `OWNER` access. + * + * - **`bucketOwnerRead`** - Object owner gets `OWNER` access, and project + * team owners get `READER` access. + * + * - **`private`** - Object owner gets `OWNER` access. + * + * - **`projectPrivate`** - Object owner gets `OWNER` access, and project + * team members get access according to their roles. + * + * - **`publicRead`** - Object owner gets `OWNER` access, and `allUsers` + * get `READER` access. + * @property {boolean} [private] Make the uploaded file private. (Alias for + * `options.predefinedAcl = 'private'`) + * @property {boolean} [public] Make the uploaded file public. (Alias for + * `options.predefinedAcl = 'publicRead'`) + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + * @property {string} [chunkSize] Create a separate request per chunk. This + * value is in bytes and should be a multiple of 256 KiB (2^18). + * {@link https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload| We recommend using at least 8 MiB for the chunk size.} + */ + /** + * Create a unique resumable upload session URI. This is the first step when + * performing a resumable upload. + * + * See the {@link https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload| Resumable upload guide} + * for more on how the entire process works. + * + *

Note

+ * + * If you are just looking to perform a resumable upload without worrying + * about any of the details, see {@link File#createWriteStream}. Resumable + * uploads are performed by default. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload| Resumable upload guide} + * + * @param {CreateResumableUploadOptions} [options] Configuration options. + * @param {CreateResumableUploadCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * file.createResumableUpload(function(err, uri) { + * if (!err) { + * // `uri` can be used to PUT data to. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.createResumableUpload().then(function(data) { + * const uri = data[0]; + * }); + * ``` + */ + createResumableUpload( + optionsOrCallback?: + | CreateResumableUploadOptions + | CreateResumableUploadCallback, + callback?: CreateResumableUploadCallback + ): void | Promise { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + const retryOptions = this.storage.retryOptions; + if ( + (options?.preconditionOpts?.ifGenerationMatch === undefined && + this.instancePreconditionOpts?.ifGenerationMatch === undefined && + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryConditional) || + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryNever + ) { + retryOptions.autoRetry = false; + } + + resumableUpload.createURI( + { + authClient: this.storage.authClient, + apiEndpoint: this.storage.apiEndpoint, + bucket: this.bucket.name, + customRequestOptions: this.getRequestInterceptors().reduce( + (reqOpts, interceptorFn) => interceptorFn(reqOpts), + {} + ), + file: this.name, + generation: this.generation, + key: this.encryptionKey, + kmsKeyName: this.kmsKeyName, + metadata: options.metadata, + offset: options.offset, + origin: options.origin, + predefinedAcl: options.predefinedAcl, + private: options.private, + public: options.public, + userProject: options.userProject || this.userProject, + retryOptions: retryOptions, + params: options?.preconditionOpts || this.instancePreconditionOpts, + universeDomain: this.bucket.storage.universeDomain, + useAuthWithCustomEndpoint: this.storage.useAuthWithCustomEndpoint, + [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY], + }, + callback! + ); + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + } + + /** + * @typedef {object} CreateWriteStreamOptions Configuration options for File#createWriteStream(). + * @property {string} [contentType] Alias for + * `options.metadata.contentType`. If set to `auto`, the file name is used + * to determine the contentType. + * @property {string|boolean} [gzip] If true, automatically gzip the file. + * If set to `auto`, the contentType is used to determine if the file + * should be gzipped. This will set `options.metadata.contentEncoding` to + * `gzip` if necessary. + * @property {object} [metadata] See the examples below or + * {@link https://cloud.google.com/storage/docs/json_api/v1/objects/insert#request_properties_JSON| Objects: insert request body} + * for more details. + * @property {number} [offset] The starting byte of the upload stream, for + * resuming an interrupted upload. Defaults to 0. + * @property {string} [predefinedAcl] Apply a predefined set of access + * controls to this object. + * + * Acceptable values are: + * - **`authenticatedRead`** - Object owner gets `OWNER` access, and + * `allAuthenticatedUsers` get `READER` access. + * + * - **`bucketOwnerFullControl`** - Object owner gets `OWNER` access, and + * project team owners get `OWNER` access. + * + * - **`bucketOwnerRead`** - Object owner gets `OWNER` access, and project + * team owners get `READER` access. + * + * - **`private`** - Object owner gets `OWNER` access. + * + * - **`projectPrivate`** - Object owner gets `OWNER` access, and project + * team members get access according to their roles. + * + * - **`publicRead`** - Object owner gets `OWNER` access, and `allUsers` + * get `READER` access. + * @property {boolean} [private] Make the uploaded file private. (Alias for + * `options.predefinedAcl = 'private'`) + * @property {boolean} [public] Make the uploaded file public. (Alias for + * `options.predefinedAcl = 'publicRead'`) + * @property {boolean} [resumable] Force a resumable upload. NOTE: When + * working with streams, the file format and size is unknown until it's + * completely consumed. Because of this, it's best for you to be explicit + * for what makes sense given your input. + * @property {number} [timeout=60000] Set the HTTP request timeout in + * milliseconds. This option is not available for resumable uploads. + * Default: `60000` + * @property {string} [uri] The URI for an already-created resumable + * upload. See {@link File#createResumableUpload}. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + * @property {string|boolean} [validation] Possible values: `"md5"`, + * `"crc32c"`, or `false`. By default, data integrity is validated with a + * CRC32c checksum. You may use MD5 if preferred, but that hash is not + * supported for composite objects. An error will be raised if MD5 is + * specified but is not available. You may also choose to skip validation + * completely, however this is **not recommended**. In addition to specifying + * validation type, providing `metadata.crc32c` or `metadata.md5Hash` will + * cause the server to perform validation in addition to client validation. + * NOTE: Validation is automatically skipped for objects that were + * uploaded using the `gzip` option and have already compressed content. + */ + /** + * Create a writable stream to overwrite the contents of the file in your + * bucket. + * + * A File object can also be used to create files for the first time. + * + * Resumable uploads are automatically enabled and must be shut off explicitly + * by setting `options.resumable` to `false`. + * + * + *

+ * There is some overhead when using a resumable upload that can cause + * noticeable performance degradation while uploading a series of small + * files. When uploading files less than 10MB, it is recommended that the + * resumable feature is disabled. + *

+ * + * NOTE: Writable streams will emit the `finish` event when the file is fully + * uploaded. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload Upload Options (Simple or Resumable)} + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/insert Objects: insert API Documentation} + * + * @param {CreateWriteStreamOptions} [options] Configuration options. + * @returns {WritableStream} + * + * @example + * ``` + * const fs = require('fs'); + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * //- + * //

Uploading a File

+ * // + * // Now, consider a case where we want to upload a file to your bucket. You + * // have the option of using {@link Bucket#upload}, but that is just + * // a convenience method which will do the following. + * //- + * fs.createReadStream('/Users/stephen/Photos/birthday-at-the-zoo/panda.jpg') + * .pipe(file.createWriteStream()) + * .on('error', function(err) {}) + * .on('finish', function() { + * // The file upload is complete. + * }); + * + * //- + * //

Uploading a File with gzip compression

+ * //- + * fs.createReadStream('/Users/stephen/site/index.html') + * .pipe(file.createWriteStream({ gzip: true })) + * .on('error', function(err) {}) + * .on('finish', function() { + * // The file upload is complete. + * }); + * + * //- + * // Downloading the file with `createReadStream` will automatically decode + * // the file. + * //- + * + * //- + * //

Uploading a File with Metadata

+ * // + * // One last case you may run into is when you want to upload a file to your + * // bucket and set its metadata at the same time. Like above, you can use + * // {@link Bucket#upload} to do this, which is just a wrapper around + * // the following. + * //- + * fs.createReadStream('/Users/stephen/Photos/birthday-at-the-zoo/panda.jpg') + * .pipe(file.createWriteStream({ + * metadata: { + * contentType: 'image/jpeg', + * metadata: { + * custom: 'metadata' + * } + * } + * })) + * .on('error', function(err) {}) + * .on('finish', function() { + * // The file upload is complete. + * }); + * ``` + * + * //- + * //

Continuing a Resumable Upload

+ * // + * // One can capture a `uri` from a resumable upload to reuse later. + * // Additionally, for validation, one can also capture and pass `crc32c`. + * //- + * let uri: string | undefined = undefined; + * let resumeCRC32C: string | undefined = undefined; + * + * fs.createWriteStream() + * .on('uri', link => {uri = link}) + * .on('crc32', crc32c => {resumeCRC32C = crc32c}); + * + * // later... + * fs.createWriteStream({uri, resumeCRC32C}); + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + createWriteStream(options: CreateWriteStreamOptions = {}): Writable { + options.metadata ??= {}; + + if (options.contentType) { + options!.metadata!.contentType = options.contentType; + } + + if ( + !options!.metadata!.contentType || + options!.metadata!.contentType === 'auto' + ) { + const detectedContentType = mime.getType(this.name); + if (detectedContentType) { + options!.metadata!.contentType = detectedContentType; + } + } + + let gzip = options.gzip; + + if (gzip === 'auto') { + gzip = COMPRESSIBLE_MIME_REGEX.test(options!.metadata!.contentType || ''); + } + + if (gzip) { + options!.metadata!.contentEncoding = 'gzip'; + } + + let crc32c = true; + let md5 = false; + + if (typeof options.validation === 'string') { + options.validation = options.validation.toLowerCase(); + crc32c = options.validation === 'crc32c'; + md5 = options.validation === 'md5'; + } else if (options.validation === false) { + crc32c = false; + md5 = false; + } + + if (options.offset) { + if (md5) { + throw new RangeError(FileExceptionMessages.MD5_RESUMED_UPLOAD); + } + + if (crc32c && !options.isPartialUpload && !options.resumeCRC32C) { + throw new RangeError( + FileExceptionMessages.MISSING_RESUME_CRC32C_FINAL_UPLOAD + ); + } + } + + /** + * A callback for determining when the underlying pipeline is complete. + * It's possible the pipeline callback could error before the write stream + * calls `final` so by default this will destroy the write stream unless the + * write stream sets this callback via its `final` handler. + * @param error An optional error + */ + let pipelineCallback: (error?: Error | null) => void = error => { + writeStream.destroy(error || undefined); + }; + + // A stream for consumer to write to + const writeStream = new Writable({ + final(cb) { + // Set the pipeline callback to this callback so the pipeline's results + // can be populated to the consumer + pipelineCallback = cb; + + emitStream.end(); + }, + write(chunk, encoding, cb) { + emitStream.write(chunk, encoding, cb); + }, + }); + // If the write stream, which is returned to the caller, catches an error we need to make sure that + // at least one of the streams in the pipeline below gets notified so that they + // all get cleaned up / destroyed. + writeStream.once('error', e => { + emitStream.destroy(e); + }); + // If the write stream is closed, cleanup the pipeline below by calling destroy on one of the streams. + writeStream.once('close', () => { + emitStream.destroy(); + }); + + const transformStreams: Transform[] = []; + + if (gzip) { + transformStreams.push(zlib.createGzip()); + } + + const emitStream = new PassThroughShim(); + + // If `writeStream` is destroyed before the `writing` event, `emitStream` will not have any listeners. This prevents an unhandled error. + const noop = () => {}; + emitStream.on('error', noop); + + let hashCalculatingStream: HashStreamValidator | null = null; + + if (crc32c || md5) { + const crc32cInstance = options.resumeCRC32C + ? CRC32C.from(options.resumeCRC32C) + : undefined; + + hashCalculatingStream = new HashStreamValidator({ + crc32c, + crc32cInstance, + md5, + crc32cGenerator: this.crc32cGenerator, + updateHashesOnly: true, + }); + + transformStreams.push(hashCalculatingStream); + } + + const fileWriteStream = duplexify(); + let fileWriteStreamMetadataReceived = false; + + // Handing off emitted events to users + emitStream.on('reading', () => writeStream.emit('reading')); + emitStream.on('writing', () => writeStream.emit('writing')); + fileWriteStream.on('uri', evt => writeStream.emit('uri', evt)); + fileWriteStream.on('progress', evt => writeStream.emit('progress', evt)); + fileWriteStream.on('response', resp => writeStream.emit('response', resp)); + fileWriteStream.once('metadata', () => { + fileWriteStreamMetadataReceived = true; + }); + + writeStream.once('writing', () => { + if (options.resumable === false) { + this.startSimpleUpload_(fileWriteStream, options); + } else { + this.startResumableUpload_(fileWriteStream, options); + } + + // remove temporary noop listener as we now create a pipeline that handles the errors + emitStream.removeListener('error', noop); + + pipeline( + emitStream, + ...(transformStreams as [Transform]), + fileWriteStream, + async e => { + if (e) { + return pipelineCallback(e); + } + + // If this is a partial upload, we don't expect final metadata yet. + if (options.isPartialUpload) { + // Emit CRC32c for this completed chunk if hash validation is active. + if (hashCalculatingStream?.crc32c) { + writeStream.emit('crc32c', hashCalculatingStream.crc32c); + } + // Resolve the pipeline for this *partial chunk*. + return pipelineCallback(); + } + + // We want to make sure we've received the metadata from the server in order + // to properly validate the object's integrity. Depending on the type of upload, + // the stream could close before the response is returned. + if (!fileWriteStreamMetadataReceived) { + try { + await new Promise((resolve, reject) => { + fileWriteStream.once('metadata', resolve); + fileWriteStream.once('error', reject); + }); + } catch (e) { + return pipelineCallback(e as Error); + } + } + + // Emit the local CRC32C value for future validation, if validation is enabled. + if (hashCalculatingStream?.crc32c) { + writeStream.emit('crc32c', hashCalculatingStream.crc32c); + } + + try { + // Metadata may not be ready if the upload is a partial upload, + // nothing to validate yet. + const metadataNotReady = options.isPartialUpload && !this.metadata; + + if (hashCalculatingStream && !metadataNotReady) { + await this.#validateIntegrity(hashCalculatingStream, { + crc32c, + md5, + }); + } + + pipelineCallback(); + } catch (e) { + pipelineCallback(e as Error); + } + } + ); + }); + + return writeStream; + } + + /** + * Delete the object. + * + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.apiResponse - The full API response. + */ + delete(options?: DeleteOptions): Promise<[r.Response]>; + delete(options: DeleteOptions, callback: DeleteCallback): void; + delete(callback: DeleteCallback): void; + delete( + optionsOrCallback?: DeleteOptions | DeleteCallback, + cb?: DeleteCallback + ): Promise<[r.Response]> | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + cb = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb; + + this.disableAutoRetryConditionallyIdempotent_( + this.methods.delete, + AvailableServiceObjectMethods.delete, + options + ); + + super + .delete(options) + .then(resp => cb!(null, ...resp)) + .catch(cb!) + .finally(() => { + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + }); + } + + download(options?: DownloadOptions): Promise; + download(options: DownloadOptions, callback: DownloadCallback): void; + download(callback: DownloadCallback): void; + /** + * @typedef {array} DownloadResponse + * @property [0] The contents of a File. + */ + /** + * @callback DownloadCallback + * @param err Request error, if any. + * @param contents The contents of a File. + */ + /** + * Convenience method to download a file into memory or to a local + * destination. + * + * @param {object} [options] Configuration options. The arguments match those + * passed to {@link File#createReadStream}. + * @param {string} [options.destination] Local file path to write the file's + * contents to. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {DownloadCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * //- + * // Download a file into memory. The contents will be available as the + * second + * // argument in the demonstration below, `contents`. + * //- + * file.download(function(err, contents) {}); + * + * //- + * // Download a file to a local destination. + * //- + * file.download({ + * destination: '/Users/me/Desktop/file-backup.txt' + * }, function(err) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.download().then(function(data) { + * const contents = data[0]; + * }); + * + * ``` + * @example include:samples/files.js + * region_tag:storage_download_file + * Another example: + * + * @example include:samples/encryption.js + * region_tag:storage_download_encrypted_file + * Example of downloading an encrypted file: + * + * @example include:samples/requesterPays.js + * region_tag:storage_download_file_requester_pays + * Example of downloading a file where the requester pays: + */ + download( + optionsOrCallback?: DownloadOptions | DownloadCallback, + cb?: DownloadCallback + ): Promise | void { + let options: DownloadOptions; + if (typeof optionsOrCallback === 'function') { + cb = optionsOrCallback as DownloadCallback; + options = {}; + } else { + options = Object.assign({}, optionsOrCallback); + } + + let called = false; + const callback = ((...args) => { + if (!called) cb!(...args); + called = true; + }) as DownloadCallback; + + const destination = options.destination; + delete options.destination; + + if (options.encryptionKey) { + this.setEncryptionKey(options.encryptionKey); + delete options.encryptionKey; + } + + const fileStream = this.createReadStream(options); + let receivedData = false; + + if (destination) { + fileStream + .on('error', callback) + .once('data', data => { + receivedData = true; + // We know that the file exists the server - now we can truncate/write to a file + const writable = fs.createWriteStream(destination); + writable.write(data); + fileStream + .pipe(writable) + .on('error', (err: Error) => { + callback(err, Buffer.from('')); + }) + .on('finish', () => { + callback(null, data); + }); + }) + .on('end', () => { + // In the case of an empty file no data will be received before the end event fires + if (!receivedData) { + const data = Buffer.alloc(0); + + try { + fs.writeFileSync(destination, data); + callback(null, data); + } catch (e) { + callback(e as Error, data); + } + } + }); + } else { + this.getBufferFromReadable(fileStream) + .then(contents => callback?.(null, contents)) + .catch(callback as (err: RequestError) => void); + } + } + + /** + * The Storage API allows you to use a custom key for server-side encryption. + * + * See {@link https://cloud.google.com/storage/docs/encryption#customer-supplied| Customer-supplied Encryption Keys} + * + * @param {string|buffer} encryptionKey An AES-256 encryption key. + * @returns {File} + * + * @example + * ``` + * const crypto = require('crypto'); + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const encryptionKey = crypto.randomBytes(32); + * + * const fileWithCustomEncryption = myBucket.file('my-file'); + * fileWithCustomEncryption.setEncryptionKey(encryptionKey); + * + * const fileWithoutCustomEncryption = myBucket.file('my-file'); + * + * fileWithCustomEncryption.save('data', function(err) { + * // Try to download with the File object that hasn't had + * // `setEncryptionKey()` called: + * fileWithoutCustomEncryption.download(function(err) { + * // We will receive an error: + * // err.message === 'Bad Request' + * + * // Try again with the File object we called `setEncryptionKey()` on: + * fileWithCustomEncryption.download(function(err, contents) { + * // contents.toString() === 'data' + * }); + * }); + * }); + * + * ``` + * @example include:samples/encryption.js + * region_tag:storage_upload_encrypted_file + * Example of uploading an encrypted file: + * + * @example include:samples/encryption.js + * region_tag:storage_download_encrypted_file + * Example of downloading an encrypted file: + */ + setEncryptionKey(encryptionKey: string | Buffer) { + this.encryptionKey = encryptionKey; + this.encryptionKeyBase64 = Buffer.from(encryptionKey as string).toString( + 'base64' + ); + this.encryptionKeyHash = crypto + .createHash('sha256') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .update(this.encryptionKeyBase64, 'base64' as any) + .digest('base64'); + + this.encryptionKeyInterceptor = { + request: reqOpts => { + reqOpts.headers = reqOpts.headers || {}; + reqOpts.headers['x-goog-encryption-algorithm'] = 'AES256'; + reqOpts.headers['x-goog-encryption-key'] = this.encryptionKeyBase64; + reqOpts.headers['x-goog-encryption-key-sha256'] = + this.encryptionKeyHash; + return reqOpts as DecorateRequestOptions; + }, + }; + + this.interceptors.push(this.encryptionKeyInterceptor!); + + return this; + } + + /** + * Gets a reference to a Cloud Storage {@link File} file from the provided URL in string format. + * @param {string} publicUrlOrGsUrl the URL as a string. Must be of the format gs://bucket/file + * or https://storage.googleapis.com/bucket/file. + * @param {Storage} storageInstance an instance of a Storage object. + * @param {FileOptions} [options] Configuration options + * @returns {File} + */ + static from( + publicUrlOrGsUrl: string, + storageInstance: Storage, + options?: FileOptions + ): File { + const gsMatches = [...publicUrlOrGsUrl.matchAll(GS_UTIL_URL_REGEX)]; + const httpsMatches = [...publicUrlOrGsUrl.matchAll(HTTPS_PUBLIC_URL_REGEX)]; + + if (gsMatches.length > 0) { + const bucket = new Bucket(storageInstance, gsMatches[0][2]); + return new File(bucket, gsMatches[0][3], options); + } else if (httpsMatches.length > 0) { + const bucket = new Bucket(storageInstance, httpsMatches[0][3]); + return new File(bucket, httpsMatches[0][4], options); + } else { + throw new Error( + 'URL string must be of format gs://bucket/file or https://storage.googleapis.com/bucket/file' + ); + } + } + + get(options?: GetFileOptions): Promise>; + get(callback: InstanceResponseCallback): void; + get(options: GetFileOptions, callback: InstanceResponseCallback): void; + get( + optionsOrCallback?: GetFileOptions | InstanceResponseCallback, + cb?: InstanceResponseCallback + ): Promise> | void { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const options: any = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + cb = + typeof optionsOrCallback === 'function' + ? (optionsOrCallback as InstanceResponseCallback) + : cb; + + super + .get(options) + .then(resp => cb!(null, ...resp)) + .catch(cb!); + } + + getExpirationDate(): Promise; + getExpirationDate(callback: GetExpirationDateCallback): void; + /** + * @typedef {array} GetExpirationDateResponse + * @property {date} 0 A Date object representing the earliest time this file's + * retention policy will expire. + */ + /** + * @callback GetExpirationDateCallback + * @param {?Error} err Request error, if any. + * @param {date} expirationDate A Date object representing the earliest time + * this file's retention policy will expire. + */ + /** + * If this bucket has a retention policy defined, use this method to get a + * Date object representing the earliest time this file will expire. + * + * @param {GetExpirationDateCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const storage = require('@google-cloud/storage')(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * file.getExpirationDate(function(err, expirationDate) { + * // expirationDate is a Date object. + * }); + * ``` + */ + getExpirationDate( + callback?: GetExpirationDateCallback + ): void | Promise { + this.getMetadata( + (err: ApiError | null, metadata: FileMetadata, apiResponse: unknown) => { + if (err) { + callback!(err, null, apiResponse); + return; + } + + if (!metadata.retentionExpirationTime) { + const error = new Error(FileExceptionMessages.EXPIRATION_TIME_NA); + callback!(error, null, apiResponse); + return; + } + + callback!( + null, + new Date(metadata.retentionExpirationTime), + apiResponse + ); + } + ); + } + + generateSignedPostPolicyV2( + options: GenerateSignedPostPolicyV2Options + ): Promise; + generateSignedPostPolicyV2( + options: GenerateSignedPostPolicyV2Options, + callback: GenerateSignedPostPolicyV2Callback + ): void; + generateSignedPostPolicyV2( + callback: GenerateSignedPostPolicyV2Callback + ): void; + /** + * @typedef {array} GenerateSignedPostPolicyV2Response + * @property {object} 0 The document policy. + */ + /** + * @callback GenerateSignedPostPolicyV2Callback + * @param {?Error} err Request error, if any. + * @param {object} policy The document policy. + */ + /** + * Get a signed policy document to allow a user to upload data with a POST + * request. + * + * In Google Cloud Platform environments, such as Cloud Functions and App + * Engine, you usually don't provide a `keyFilename` or `credentials` during + * instantiation. In those environments, we call the + * {@link https://cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob| signBlob API} + * to create a signed policy. That API requires either the + * `https://www.googleapis.com/auth/iam` or + * `https://www.googleapis.com/auth/cloud-platform` scope, so be sure they are + * enabled. + * + * See {@link https://cloud.google.com/storage/docs/xml-api/post-object-v2| POST Object with the V2 signing process} + * + * @throws {Error} If an expiration timestamp from the past is given. + * @throws {Error} If options.equals has an array with less or more than two + * members. + * @throws {Error} If options.startsWith has an array with less or more than two + * members. + * + * @param {object} options Configuration options. + * @param {array|array[]} [options.equals] Array of request parameters and + * their expected value (e.g. [['$', '']]). Values are + * translated into equality constraints in the conditions field of the + * policy document (e.g. ['eq', '$', '']). If only one + * equality condition is to be specified, options.equals can be a one- + * dimensional array (e.g. ['$', '']). + * @param {*} options.expires - A timestamp when this policy will expire. Any + * value given is passed to `new Date()`. + * @param {array|array[]} [options.startsWith] Array of request parameters and + * their expected prefixes (e.g. [['$', '']). Values are + * translated into starts-with constraints in the conditions field of the + * policy document (e.g. ['starts-with', '$', '']). If only + * one prefix condition is to be specified, options.startsWith can be a + * one- dimensional array (e.g. ['$', '']). + * @param {string} [options.acl] ACL for the object from possibly predefined + * ACLs. + * @param {string} [options.successRedirect] The URL to which the user client + * is redirected if the upload is successful. + * @param {string} [options.successStatus] - The status of the Google Storage + * response if the upload is successful (must be string). + * @param {object} [options.contentLengthRange] + * @param {number} [options.contentLengthRange.min] Minimum value for the + * request's content length. + * @param {number} [options.contentLengthRange.max] Maximum value for the + * request's content length. + * @param {GenerateSignedPostPolicyV2Callback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * const options = { + * equals: ['$Content-Type', 'image/jpeg'], + * expires: '10-25-2022', + * contentLengthRange: { + * min: 0, + * max: 1024 + * } + * }; + * + * file.generateSignedPostPolicyV2(options, function(err, policy) { + * // policy.string: the policy document in plain text. + * // policy.base64: the policy document in base64. + * // policy.signature: the policy signature in base64. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.generateSignedPostPolicyV2(options).then(function(data) { + * const policy = data[0]; + * }); + * ``` + */ + generateSignedPostPolicyV2( + optionsOrCallback?: + | GenerateSignedPostPolicyV2Options + | GenerateSignedPostPolicyV2Callback, + cb?: GenerateSignedPostPolicyV2Callback + ): void | Promise { + const args = normalize( + optionsOrCallback, + cb + ); + let options = args.options; + const callback = args.callback; + const expires = new Date( + (options as GenerateSignedPostPolicyV2Options).expires + ); + + if (isNaN(expires.getTime())) { + throw new Error(ExceptionMessages.EXPIRATION_DATE_INVALID); + } + + if (expires.valueOf() < Date.now()) { + throw new Error(ExceptionMessages.EXPIRATION_DATE_PAST); + } + + options = Object.assign({}, options); + + const conditions = [ + ['eq', '$key', this.name], + { + bucket: this.bucket.name, + }, + ] as object[]; + + if (Array.isArray(options.equals)) { + if (!Array.isArray((options.equals as string[][])[0])) { + options.equals = [options.equals as string[]]; + } + (options.equals as string[][]).forEach(condition => { + if (!Array.isArray(condition) || condition.length !== 2) { + throw new Error(FileExceptionMessages.EQUALS_CONDITION_TWO_ELEMENTS); + } + conditions.push(['eq', condition[0], condition[1]]); + }); + } + + if (Array.isArray(options.startsWith)) { + if (!Array.isArray((options.startsWith as string[][])[0])) { + options.startsWith = [options.startsWith as string[]]; + } + (options.startsWith as string[][]).forEach(condition => { + if (!Array.isArray(condition) || condition.length !== 2) { + throw new Error(FileExceptionMessages.STARTS_WITH_TWO_ELEMENTS); + } + conditions.push(['starts-with', condition[0], condition[1]]); + }); + } + + if (options.acl) { + conditions.push({ + acl: options.acl, + }); + } + + if (options.successRedirect) { + conditions.push({ + success_action_redirect: options.successRedirect, + }); + } + + if (options.successStatus) { + conditions.push({ + success_action_status: options.successStatus, + }); + } + + if (options.contentLengthRange) { + const min = options.contentLengthRange.min; + const max = options.contentLengthRange.max; + if (typeof min !== 'number' || typeof max !== 'number') { + throw new Error(FileExceptionMessages.CONTENT_LENGTH_RANGE_MIN_MAX); + } + conditions.push(['content-length-range', min, max]); + } + + const policy = { + expiration: expires.toISOString(), + conditions, + }; + + const policyString = JSON.stringify(policy); + const policyBase64 = Buffer.from(policyString).toString('base64'); + + this.storage.authClient.sign(policyBase64, options.signingEndpoint).then( + signature => { + callback(null, { + string: policyString, + base64: policyBase64, + signature, + }); + }, + err => { + callback(new SigningError(err.message)); + } + ); + } + + generateSignedPostPolicyV4( + options: GenerateSignedPostPolicyV4Options + ): Promise; + generateSignedPostPolicyV4( + options: GenerateSignedPostPolicyV4Options, + callback: GenerateSignedPostPolicyV4Callback + ): void; + generateSignedPostPolicyV4( + callback: GenerateSignedPostPolicyV4Callback + ): void; + /** + * @typedef {object} SignedPostPolicyV4Output + * @property {string} url The request URL. + * @property {object} fields The form fields to include in the POST request. + */ + /** + * @typedef {array} GenerateSignedPostPolicyV4Response + * @property {SignedPostPolicyV4Output} 0 An object containing the request URL and form fields. + */ + /** + * @callback GenerateSignedPostPolicyV4Callback + * @param {?Error} err Request error, if any. + * @param {SignedPostPolicyV4Output} output An object containing the request URL and form fields. + */ + /** + * Get a v4 signed policy document to allow a user to upload data with a POST + * request. + * + * In Google Cloud Platform environments, such as Cloud Functions and App + * Engine, you usually don't provide a `keyFilename` or `credentials` during + * instantiation. In those environments, we call the + * {@link https://cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob| signBlob API} + * to create a signed policy. That API requires either the + * `https://www.googleapis.com/auth/iam` or + * `https://www.googleapis.com/auth/cloud-platform` scope, so be sure they are + * enabled. + * + * See {@link https://cloud.google.com/storage/docs/xml-api/post-object#policydocument| Policy Document Reference} + * + * @param {object} options Configuration options. + * @param {Date|number|string} options.expires - A timestamp when this policy will expire. Any + * value given is passed to `new Date()`. + * @param {boolean} [config.virtualHostedStyle=false] Use virtual hosted-style + * URLs ('https://mybucket.storage.googleapis.com/...') instead of path-style + * ('https://storage.googleapis.com/mybucket/...'). Virtual hosted-style URLs + * should generally be preferred instead of path-style URL. + * Currently defaults to `false` for path-style, although this may change in a + * future major-version release. + * @param {string} [config.bucketBoundHostname] The bucket-bound hostname to return in + * the result, e.g. "https://cdn.example.com". + * @param {object} [config.fields] [Form fields]{@link https://cloud.google.com/storage/docs/xml-api/post-object#policydocument} + * to include in the signed policy. Any fields with key beginning with 'x-ignore-' + * will not be included in the policy to be signed. + * @param {object[]} [config.conditions] [Conditions]{@link https://cloud.google.com/storage/docs/authentication/signatures#policy-document} + * to include in the signed policy. All fields given in `config.fields` are + * automatically included in the conditions array, adding the same entry + * in both `fields` and `conditions` will result in duplicate entries. + * + * @param {GenerateSignedPostPolicyV4Callback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * const options = { + * expires: '10-25-2022', + * conditions: [ + * ['eq', '$Content-Type', 'image/jpeg'], + * ['content-length-range', 0, 1024], + * ], + * fields: { + * acl: 'public-read', + * 'x-goog-meta-foo': 'bar', + * 'x-ignore-mykey': 'data' + * } + * }; + * + * file.generateSignedPostPolicyV4(options, function(err, response) { + * // response.url The request URL + * // response.fields The form fields (including the signature) to include + * // to be used to upload objects by HTML forms. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.generateSignedPostPolicyV4(options).then(function(data) { + * const response = data[0]; + * // response.url The request URL + * // response.fields The form fields (including the signature) to include + * // to be used to upload objects by HTML forms. + * }); + * ``` + */ + generateSignedPostPolicyV4( + optionsOrCallback?: + | GenerateSignedPostPolicyV4Options + | GenerateSignedPostPolicyV4Callback, + cb?: GenerateSignedPostPolicyV4Callback + ): void | Promise { + const args = normalize< + GenerateSignedPostPolicyV4Options, + GenerateSignedPostPolicyV4Callback + >(optionsOrCallback, cb); + let options = args.options; + const callback = args.callback; + const expires = new Date( + (options as GenerateSignedPostPolicyV4Options).expires + ); + + if (isNaN(expires.getTime())) { + throw new Error(ExceptionMessages.EXPIRATION_DATE_INVALID); + } + + if (expires.valueOf() < Date.now()) { + throw new Error(ExceptionMessages.EXPIRATION_DATE_PAST); + } + + if (expires.valueOf() - Date.now() > SEVEN_DAYS * 1000) { + throw new Error( + `Max allowed expiration is seven days (${SEVEN_DAYS} seconds).` + ); + } + + options = Object.assign({}, options); + let fields = Object.assign({}, options.fields); + + const now = new Date(); + const nowISO = formatAsUTCISO(now, true); + const todayISO = formatAsUTCISO(now); + + const sign = async () => { + const {client_email} = await this.storage.authClient.getCredentials(); + const credential = `${client_email}/${todayISO}/auto/storage/goog4_request`; + + fields = { + ...fields, + bucket: this.bucket.name, + key: this.name, + 'x-goog-date': nowISO, + 'x-goog-credential': credential, + 'x-goog-algorithm': 'GOOG4-RSA-SHA256', + }; + + const conditions = options.conditions || []; + + Object.entries(fields).forEach(([key, value]) => { + if (!key.startsWith('x-ignore-')) { + conditions.push({[key]: value}); + } + }); + + delete fields.bucket; + + const expiration = formatAsUTCISO(expires, true, '-', ':'); + + const policy = { + conditions, + expiration, + }; + + const policyString = unicodeJSONStringify(policy); + const policyBase64 = Buffer.from(policyString).toString('base64'); + + try { + const signature = await this.storage.authClient.sign( + policyBase64, + options.signingEndpoint + ); + const signatureHex = Buffer.from(signature, 'base64').toString('hex'); + const universe = this.parent.storage.universeDomain; + fields['policy'] = policyBase64; + fields['x-goog-signature'] = signatureHex; + + let url: string; + + const EMULATOR_HOST = process.env.STORAGE_EMULATOR_HOST; + + if (this.storage.customEndpoint && typeof EMULATOR_HOST === 'string') { + url = `${this.storage.apiEndpoint}/${this.bucket.name}`; + } else if (this.storage.customEndpoint) { + url = this.storage.apiEndpoint; + } else if (options.virtualHostedStyle) { + url = `https://${this.bucket.name}.storage.${universe}/`; + } else if (options.bucketBoundHostname) { + url = `${options.bucketBoundHostname}/`; + } else { + url = `https://storage.${universe}/${this.bucket.name}/`; + } + + return { + url, + fields, + }; + } catch (err) { + throw new SigningError((err as Error).message); + } + }; + + sign().then(res => callback!(null, res), callback!); + } + + getSignedUrl(cfg: GetSignedUrlConfig): Promise; + getSignedUrl(cfg: GetSignedUrlConfig, callback: GetSignedUrlCallback): void; + /** + * @typedef {array} GetSignedUrlResponse + * @property {object} 0 The signed URL. + */ + /** + * @callback GetSignedUrlCallback + * @param {?Error} err Request error, if any. + * @param {object} url The signed URL. + */ + /** + * Get a signed URL to allow limited time access to the file. + * + * In Google Cloud Platform environments, such as Cloud Functions and App + * Engine, you usually don't provide a `keyFilename` or `credentials` during + * instantiation. In those environments, we call the + * {@link https://cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob| signBlob API} + * to create a signed URL. That API requires either the + * `https://www.googleapis.com/auth/iam` or + * `https://www.googleapis.com/auth/cloud-platform` scope, so be sure they are + * enabled. + * + * See {@link https://cloud.google.com/storage/docs/access-control/signed-urls| Signed URLs Reference} + * + * @throws {Error} if an expiration timestamp from the past is given. + * + * @param {object} config Configuration object. + * @param {string} config.action "read" (HTTP: GET), "write" (HTTP: PUT), or + * "delete" (HTTP: DELETE), "resumable" (HTTP: POST). + * When using "resumable", the header `X-Goog-Resumable: start` has + * to be sent when making a request with the signed URL. + * @param {*} config.expires A timestamp when this link will expire. Any value + * given is passed to `new Date()`. + * Note: 'v4' supports maximum duration of 7 days (604800 seconds) from now. + * See [reference]{@link https://cloud.google.com/storage/docs/access-control/signed-urls#example} + * @param {string} [config.version='v2'] The signing version to use, either + * 'v2' or 'v4'. + * @param {boolean} [config.virtualHostedStyle=false] Use virtual hosted-style + * URLs (e.g. 'https://mybucket.storage.googleapis.com/...') instead of path-style + * (e.g. 'https://storage.googleapis.com/mybucket/...'). Virtual hosted-style URLs + * should generally be preferred instead of path-style URL. + * Currently defaults to `false` for path-style, although this may change in a + * future major-version release. + * @param {string} [config.cname] The cname for this bucket, i.e., + * "https://cdn.example.com". + * @param {string} [config.contentMd5] The MD5 digest value in base64. Just like + * if you provide this, the client must provide this HTTP header with this same + * value in its request, so to if this parameter is not provided here, + * the client must not provide any value for this HTTP header in its request. + * @param {string} [config.contentType] Just like if you provide this, the client + * must provide this HTTP header with this same value in its request, so to if + * this parameter is not provided here, the client must not provide any value + * for this HTTP header in its request. + * @param {object} [config.extensionHeaders] If these headers are used, the + * server will check to make sure that the client provides matching + * values. See {@link https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers| Canonical extension headers} + * for the requirements of this feature, most notably: + * - The header name must be prefixed with `x-goog-` + * - The header name must be all lowercase + * + * Note: Multi-valued header passed as an array in the extensionHeaders + * object is converted into a string, delimited by `,` with + * no space. Requests made using the signed URL will need to + * delimit multi-valued headers using a single `,` as well, or + * else the server will report a mismatched signature. + * @param {object} [config.queryParams] Additional query parameters to include + * in the signed URL. + * @param {string} [config.promptSaveAs] The filename to prompt the user to + * save the file as when the signed url is accessed. This is ignored if + * `config.responseDisposition` is set. + * @param {string} [config.responseDisposition] The + * {@link http://goo.gl/yMWxQV| response-content-disposition parameter} of the + * signed url. + * @param {*} [config.accessibleAt=Date.now()] A timestamp when this link became usable. Any value + * given is passed to `new Date()`. + * Note: Use for 'v4' only. + * @param {string} [config.responseType] The response-content-type parameter + * of the signed url. + * @param {GetSignedUrlCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * //- + * // Generate a URL that allows temporary access to download your file. + * //- + * const request = require('request'); + * + * const config = { + * action: 'read', + * expires: '03-17-2025', + * }; + * + * file.getSignedUrl(config, function(err, url) { + * if (err) { + * console.error(err); + * return; + * } + * + * // The file is now available to read from this URL. + * request(url, function(err, resp) { + * // resp.statusCode = 200 + * }); + * }); + * + * //- + * // Generate a URL that allows temporary access to download your file. + * // Access will begin at accessibleAt and end at expires. + * //- + * const request = require('request'); + * + * const config = { + * action: 'read', + * expires: '03-17-2025', + * accessibleAt: '03-13-2025' + * }; + * + * file.getSignedUrl(config, function(err, url) { + * if (err) { + * console.error(err); + * return; + * } + * + * // The file will be available to read from this URL from 03-13-2025 to 03-17-2025. + * request(url, function(err, resp) { + * // resp.statusCode = 200 + * }); + * }); + * + * //- + * // Generate a URL to allow write permissions. This means anyone with this + * URL + * // can send a POST request with new data that will overwrite the file. + * //- + * file.getSignedUrl({ + * action: 'write', + * expires: '03-17-2025' + * }, function(err, url) { + * if (err) { + * console.error(err); + * return; + * } + * + * // The file is now available to be written to. + * const writeStream = request.put(url); + * writeStream.end('New data'); + * + * writeStream.on('complete', function(resp) { + * // Confirm the new content was saved. + * file.download(function(err, fileContents) { + * console.log('Contents:', fileContents.toString()); + * // Contents: New data + * }); + * }); + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.getSignedUrl(config).then(function(data) { + * const url = data[0]; + * }); + * + * ``` + * @example include:samples/files.js + * region_tag:storage_generate_signed_url + * Another example: + */ + getSignedUrl( + cfg: GetSignedUrlConfig, + callback?: GetSignedUrlCallback + ): void | Promise { + const method = ActionToHTTPMethod[cfg.action]; + const extensionHeaders = objectKeyToLowercase(cfg.extensionHeaders || {}); + if (cfg.action === 'resumable') { + extensionHeaders['x-goog-resumable'] = 'start'; + } + + const queryParams = Object.assign({}, cfg.queryParams); + if (typeof cfg.responseType === 'string') { + queryParams['response-content-type'] = cfg.responseType!; + } + if (typeof cfg.promptSaveAs === 'string') { + queryParams['response-content-disposition'] = + 'attachment; filename="' + cfg.promptSaveAs + '"'; + } + if (typeof cfg.responseDisposition === 'string') { + queryParams['response-content-disposition'] = cfg.responseDisposition!; + } + if (this.generation) { + queryParams['generation'] = this.generation.toString(); + } + + const signConfig: SignerGetSignedUrlConfig = { + method, + expires: cfg.expires, + accessibleAt: cfg.accessibleAt, + extensionHeaders, + queryParams, + contentMd5: cfg.contentMd5, + contentType: cfg.contentType, + host: cfg.host, + }; + + if (cfg.cname) { + signConfig.cname = cfg.cname; + } + + if (cfg.version) { + signConfig.version = cfg.version; + } + + if (cfg.virtualHostedStyle) { + signConfig.virtualHostedStyle = cfg.virtualHostedStyle; + } + + if (!this.signer) { + this.signer = new URLSigner( + this.storage.authClient, + this.bucket, + this, + this.storage + ); + } + + this.signer + .getSignedUrl(signConfig) + .then(signedUrl => callback!(null, signedUrl), callback!); + } + + isPublic(): Promise; + isPublic(callback: IsPublicCallback): void; + /** + * @callback IsPublicCallback + * @param {?Error} err Request error, if any. + * @param {boolean} resp Whether file is public or not. + */ + /** + * @typedef {array} IsPublicResponse + * @property {boolean} 0 Whether file is public or not. + */ + /** + * Check whether this file is public or not by sending + * a HEAD request without credentials. + * No errors from the server indicates that the current + * file is public. + * A 403-Forbidden error {@link https://cloud.google.com/storage/docs/json_api/v1/status-codes#403_Forbidden} + * indicates that file is private. + * Any other non 403 error is propagated to user. + * + * @param {IsPublicCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * //- + * // Check whether the file is publicly accessible. + * //- + * file.isPublic(function(err, resp) { + * if (err) { + * console.error(err); + * return; + * } + * console.log(`the file ${file.id} is public: ${resp}`) ; + * }) + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.isPublic().then(function(data) { + * const resp = data[0]; + * }); + * ``` + */ + + isPublic(callback?: IsPublicCallback): Promise | void { + // Build any custom headers based on the defined interceptors on the parent + // storage object and this object + const storageInterceptors = this.storage?.interceptors || []; + const fileInterceptors = this.interceptors || []; + const allInterceptors = storageInterceptors.concat(fileInterceptors); + const headers = allInterceptors.reduce((acc, curInterceptor) => { + const currentHeaders = curInterceptor.request({ + uri: `${this.storage.apiEndpoint}/${ + this.bucket.name + }/${encodeURIComponent(this.name)}`, + }); + + Object.assign(acc, currentHeaders.headers); + return acc; + }, {}); + + util.makeRequest( + { + method: 'GET', + uri: `${this.storage.apiEndpoint}/${ + this.bucket.name + }/${encodeURIComponent(this.name)}`, + headers, + }, + { + retryOptions: this.storage.retryOptions, + }, + (err: Error | ApiError | null) => { + if (err) { + const apiError = err as ApiError; + if (apiError.code === 403) { + callback!(null, false); + } else { + callback!(err); + } + } else { + callback!(null, true); + } + } + ); + } + + makePrivate( + options?: MakeFilePrivateOptions + ): Promise; + makePrivate(callback: MakeFilePrivateCallback): void; + makePrivate( + options: MakeFilePrivateOptions, + callback: MakeFilePrivateCallback + ): void; + /** + * @typedef {object} MakeFilePrivateOptions Configuration options for File#makePrivate(). + * @property {Metadata} [metadata] Define custom metadata properties to define + * along with the operation. + * @property {boolean} [strict] If true, set the file to be private to + * only the owner user. Otherwise, it will be private to the project. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @callback MakeFilePrivateCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {array} MakeFilePrivateResponse + * @property {object} 0 The full API response. + */ + /** + * Make a file private to the project and remove all other permissions. + * Set `options.strict` to true to make the file private to only the owner. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/patch| Objects: patch API Documentation} + * + * @param {MakeFilePrivateOptions} [options] Configuration options. + * @param {MakeFilePrivateCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * //- + * // Set the file private so only project maintainers can see and modify it. + * //- + * file.makePrivate(function(err) {}); + * + * //- + * // Set the file private so only the owner can see and modify it. + * //- + * file.makePrivate({ strict: true }, function(err) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.makePrivate().then(function(data) { + * const apiResponse = data[0]; + * }); + * ``` + */ + makePrivate( + optionsOrCallback?: MakeFilePrivateOptions | MakeFilePrivateCallback, + callback?: MakeFilePrivateCallback + ): Promise | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + const query = { + predefinedAcl: options.strict ? 'private' : 'projectPrivate', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + if (options.preconditionOpts?.ifMetagenerationMatch !== undefined) { + query.ifMetagenerationMatch = + options.preconditionOpts?.ifMetagenerationMatch; + delete options.preconditionOpts; + } + + if (options.userProject) { + query.userProject = options.userProject; + } + + // You aren't allowed to set both predefinedAcl & acl properties on a file, + // so acl must explicitly be nullified, destroying all previous acls on the + // file. + const metadata = {...options.metadata, acl: null}; + + this.setMetadata(metadata, query, callback!); + } + + makePublic(): Promise; + makePublic(callback: MakeFilePublicCallback): void; + /** + * @typedef {array} MakeFilePublicResponse + * @property {object} 0 The full API response. + */ + /** + * @callback MakeFilePublicCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * Set a file to be publicly readable and maintain all previous permissions. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/insert| ObjectAccessControls: insert API Documentation} + * + * @param {MakeFilePublicCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * + * file.makePublic(function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.makePublic().then(function(data) { + * const apiResponse = data[0]; + * }); + * + * ``` + * @example include:samples/files.js + * region_tag:storage_make_public + * Another example: + */ + makePublic( + callback?: MakeFilePublicCallback + ): Promise | void { + callback = callback || util.noop; + this.acl.add( + { + entity: 'allUsers', + role: 'READER', + }, + (err, acl, resp) => { + callback!(err, resp); + } + ); + } + + /** + * The public URL of this File + * Use {@link File#makePublic} to enable anonymous access via the returned URL. + * + * @returns {string} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * const file = bucket.file('my-file'); + * + * // publicUrl will be "https://storage.googleapis.com/albums/my-file" + * const publicUrl = file.publicUrl(); + * ``` + */ + publicUrl(): string { + return `${this.storage.apiEndpoint}/${ + this.bucket.name + }/${encodeURIComponent(this.name)}`; + } + + moveFileAtomic( + destination: string | File, + options?: MoveFileAtomicOptions + ): Promise; + moveFileAtomic( + destination: string | File, + callback: MoveFileAtomicCallback + ): void; + moveFileAtomic( + destination: string | File, + options: MoveFileAtomicOptions, + callback: MoveFileAtomicCallback + ): void; + /** + * @typedef {array} MoveFileAtomicResponse + * @property {File} 0 The moved {@link File}. + * @property {object} 1 The full API response. + */ + /** + * @callback MoveFileAtomicCallback + * @param {?Error} err Request error, if any. + * @param {File} movedFile The moved {@link File}. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {object} MoveFileAtomicOptions Configuration options for File#moveFileAtomic(). See an + * {@link https://cloud.google.com/storage/docs/json_api/v1/objects#resource| Object resource}. + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + * @property {object} [preconditionOpts] Precondition options. + * @property {number} [preconditionOpts.ifGenerationMatch] Makes the operation conditional on whether the object's current generation matches the given value. + */ + /** + * Move this file within the same bucket. + * The source object must exist and be a live object. + * The source and destination object IDs must be different. + * Overwriting the destination object is allowed by default, but can be prevented + * using preconditions. + * If the destination path includes non-existent parent folders, they will be created. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/move| Objects: move API Documentation} + * + * @throws {Error} If the destination file is not provided. + * + * @param {string|File} destination Destination file name or File object within the same bucket.. + * @param {MoveFileAtomicOptions} [options] Configuration options. See an + * @param {MoveFileAtomicCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * + * //- + * // Assume 'my-bucket' is a bucket. + * //- + * const bucket = storage.bucket('my-bucket'); + * const file = bucket.file('my-image.png'); + * + * //- + * // If you pass in a string for the destination, the file is copied to its + * // current bucket, under the new name provided. + * //- + * file.moveFileAtomic('moved-image.png', function(err, movedFile, apiResponse) { + * // `my-bucket` now contains: + * // - "moved-image.png" + * + * // `movedFile` is an instance of a File object that refers to your new + * // file. + * }); + * + * //- + * // Move the file to a subdirectory, creating parent folders if necessary. + * //- + * file.moveFileAtomic('new-folder/subfolder/moved-image.png', function(err, movedFile, apiResponse) { + * // `my-bucket` now contains: + * // - "new-folder/subfolder/moved-image.png" + * }); + * + * //- + * // Prevent overwriting an existing destination object using preconditions. + * //- + * file.moveFileAtomic('existing-destination.png', { + * preconditionOpts: { + * ifGenerationMatch: 0 // Fails if the destination object exists. + * } + * }, function(err, movedFile, apiResponse) { + * if (err) { + * // Handle the error (e.g., the destination object already exists). + * } else { + * // Move successful. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.moveFileAtomic('moved-image.png).then(function(data) { + * const newFile = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/files.js + * region_tag:storage_move_file + * Another example: + */ + moveFileAtomic( + destination: string | File, + optionsOrCallback?: MoveFileAtomicOptions | MoveFileAtomicCallback, + callback?: MoveFileAtomicCallback + ): Promise | void { + const noDestinationError = new Error( + FileExceptionMessages.DESTINATION_NO_NAME + ); + + if (!destination) { + throw noDestinationError; + } + + let options: MoveFileAtomicOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = {...optionsOrCallback}; + } + + callback = callback || util.noop; + + let destName: string; + let newFile: File; + + if (typeof destination === 'string') { + const parsedDestination = GS_URL_REGEXP.exec(destination); + if (parsedDestination !== null && parsedDestination.length === 3) { + destName = parsedDestination[2]; + } else { + destName = destination; + } + } else if (destination instanceof File) { + destName = destination.name; + newFile = destination; + } else { + throw noDestinationError; + } + + newFile = newFile! || this.bucket.file(destName); + + if ( + !this.shouldRetryBasedOnPreconditionAndIdempotencyStrat( + options?.preconditionOpts + ) + ) { + this.storage.retryOptions.autoRetry = false; + } + const query = {} as MoveFileAtomicQuery; + if (options.userProject !== undefined) { + query.userProject = options.userProject; + delete options.userProject; + } + if (options.preconditionOpts?.ifGenerationMatch !== undefined) { + query.ifGenerationMatch = options.preconditionOpts?.ifGenerationMatch; + delete options.preconditionOpts; + } + + this.request( + { + method: 'POST', + uri: `/moveTo/o/${encodeURIComponent(newFile.name)}`, + qs: query, + json: options, + }, + (err, resp) => { + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + if (err) { + callback!(err, null, resp); + return; + } + + callback!(null, newFile, resp); + } + ); + } + + move( + destination: string | Bucket | File, + options?: MoveOptions + ): Promise; + move(destination: string | Bucket | File, callback: MoveCallback): void; + move( + destination: string | Bucket | File, + options: MoveOptions, + callback: MoveCallback + ): void; + /** + * @typedef {array} MoveResponse + * @property {File} 0 The destination File. + * @property {object} 1 The full API response. + */ + /** + * @callback MoveCallback + * @param {?Error} err Request error, if any. + * @param {?File} destinationFile The destination File. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {object} MoveOptions Configuration options for File#move(). See an + * {@link https://cloud.google.com/storage/docs/json_api/v1/objects#resource| Object resource}. + * @param {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * Move this file to another location. By default, this will rename the file + * and keep it in the same bucket, but you can choose to move it to another + * Bucket by providing a Bucket or File object or a URL beginning with + * "gs://". + * + * **Warning**: + * There is currently no atomic `move` method in the Cloud Storage API, + * so this method is a composition of {@link File#copy} (to the new + * location) and {@link File#delete} (from the old location). While + * unlikely, it is possible that an error returned to your callback could be + * triggered from either one of these API calls failing, which could leave a + * duplicate file lingering. The error message will indicate what operation + * has failed. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/copy| Objects: copy API Documentation} + * + * @throws {Error} If the destination file is not provided. + * + * @param {string|Bucket|File} destination Destination file. + * @param {MoveCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * //- + * // You can pass in a variety of types for the destination. + * // + * // For all of the below examples, assume we are working with the following + * // Bucket and File objects. + * //- + * const bucket = storage.bucket('my-bucket'); + * const file = bucket.file('my-image.png'); + * + * //- + * // If you pass in a string for the destination, the file is moved to its + * // current bucket, under the new name provided. + * //- + * file.move('my-image-new.png', function(err, destinationFile, apiResponse) { + * // `my-bucket` no longer contains: + * // - "my-image.png" + * // but contains instead: + * // - "my-image-new.png" + * + * // `destinationFile` is an instance of a File object that refers to your + * // new file. + * }); + * + * //- + * // If you pass in a string starting with "gs://" for the destination, the + * // file is copied to the other bucket and under the new name provided. + * //- + * const newLocation = 'gs://another-bucket/my-image-new.png'; + * file.move(newLocation, function(err, destinationFile, apiResponse) { + * // `my-bucket` no longer contains: + * // - "my-image.png" + * // + * // `another-bucket` now contains: + * // - "my-image-new.png" + * + * // `destinationFile` is an instance of a File object that refers to your + * // new file. + * }); + * + * //- + * // If you pass in a Bucket object, the file will be moved to that bucket + * // using the same name. + * //- + * const anotherBucket = gcs.bucket('another-bucket'); + * + * file.move(anotherBucket, function(err, destinationFile, apiResponse) { + * // `my-bucket` no longer contains: + * // - "my-image.png" + * // + * // `another-bucket` now contains: + * // - "my-image.png" + * + * // `destinationFile` is an instance of a File object that refers to your + * // new file. + * }); + * + * //- + * // If you pass in a File object, you have complete control over the new + * // bucket and filename. + * //- + * const anotherFile = anotherBucket.file('my-awesome-image.png'); + * + * file.move(anotherFile, function(err, destinationFile, apiResponse) { + * // `my-bucket` no longer contains: + * // - "my-image.png" + * // + * // `another-bucket` now contains: + * // - "my-awesome-image.png" + * + * // Note: + * // The `destinationFile` parameter is equal to `anotherFile`. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.move('my-image-new.png').then(function(data) { + * const destinationFile = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/files.js + * region_tag:storage_move_file + * Another example: + */ + move( + destination: string | Bucket | File, + optionsOrCallback?: MoveOptions | MoveCallback, + callback?: MoveCallback + ): Promise | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + callback = callback || util.noop; + + this.copy(destination, options, (err, destinationFile, copyApiResponse) => { + if (err) { + err.message = 'file#copy failed with an error - ' + err.message; + callback!(err, null, copyApiResponse); + return; + } + + if ( + this.name !== destinationFile!.name || + this.bucket.name !== destinationFile!.bucket.name + ) { + this.delete(options, (err, apiResponse) => { + if (err) { + err.message = 'file#delete failed with an error - ' + err.message; + callback!(err, destinationFile, apiResponse); + return; + } + callback!(null, destinationFile, copyApiResponse); + }); + } else { + callback!(null, destinationFile, copyApiResponse); + } + }); + } + + rename( + destinationFile: string | File, + options?: RenameOptions + ): Promise; + rename(destinationFile: string | File, callback: RenameCallback): void; + rename( + destinationFile: string | File, + options: RenameOptions, + callback: RenameCallback + ): void; + /** + * @typedef {array} RenameResponse + * @property {File} 0 The destination File. + * @property {object} 1 The full API response. + */ + /** + * @callback RenameCallback + * @param {?Error} err Request error, if any. + * @param {?File} destinationFile The destination File. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {object} RenameOptions Configuration options for File#move(). See an + * {@link https://cloud.google.com/storage/docs/json_api/v1/objects#resource| Object resource}. + * @param {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * Rename this file. + * + * **Warning**: + * There is currently no atomic `rename` method in the Cloud Storage API, + * so this method is an alias of {@link File#move}, which in turn is a + * composition of {@link File#copy} (to the new location) and + * {@link File#delete} (from the old location). While + * unlikely, it is possible that an error returned to your callback could be + * triggered from either one of these API calls failing, which could leave a + * duplicate file lingering. The error message will indicate what operation + * has failed. + * + * @param {string|File} destinationFile Destination file. + * @param {RenameCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * + * //- + * // You can pass in a string or a File object. + * // + * // For all of the below examples, assume we are working with the following + * // Bucket and File objects. + * //- + * + * const bucket = storage.bucket('my-bucket'); + * const file = bucket.file('my-image.png'); + * + * //- + * // You can pass in a string for the destinationFile. + * //- + * file.rename('renamed-image.png', function(err, renamedFile, apiResponse) { + * // `my-bucket` no longer contains: + * // - "my-image.png" + * // but contains instead: + * // - "renamed-image.png" + * + * // `renamedFile` is an instance of a File object that refers to your + * // renamed file. + * }); + * + * //- + * // You can pass in a File object. + * //- + * const anotherFile = anotherBucket.file('my-awesome-image.png'); + * + * file.rename(anotherFile, function(err, renamedFile, apiResponse) { + * // `my-bucket` no longer contains: + * // - "my-image.png" + * + * // Note: + * // The `renamedFile` parameter is equal to `anotherFile`. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.rename('my-renamed-image.png').then(function(data) { + * const renamedFile = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + rename( + destinationFile: string | File, + optionsOrCallback?: RenameOptions | RenameCallback, + callback?: RenameCallback + ): Promise | void { + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + + callback = callback || util.noop; + + this.move(destinationFile, options, callback); + } + + /** + * @typedef {object} RestoreOptions Options for File#restore(). See an + * {@link https://cloud.google.com/storage/docs/json_api/v1/objects#resource| Object resource}. + * @param {string} [userProject] The ID of the project which will be + * billed for the request. + * @param {number} [generation] If present, selects a specific revision of this object. + * @param {string} [restoreToken] Returns an option that must be specified when getting a soft-deleted object from an HNS-enabled + * bucket that has a naming and generation conflict with another object in the same bucket. + * @param {string} [projection] Specifies the set of properties to return. If used, must be 'full' or 'noAcl'. + * @param {string | number} [ifGenerationMatch] Request proceeds if the generation of the target resource + * matches the value used in the precondition. + * If the values don't match, the request fails with a 412 Precondition Failed response. + * @param {string | number} [ifGenerationNotMatch] Request proceeds if the generation of the target resource does + * not match the value used in the precondition. If the values match, the request fails with a 304 Not Modified response. + * @param {string | number} [ifMetagenerationMatch] Request proceeds if the meta-generation of the target resource + * matches the value used in the precondition. + * If the values don't match, the request fails with a 412 Precondition Failed response. + * @param {string | number} [ifMetagenerationNotMatch] Request proceeds if the meta-generation of the target resource does + * not match the value used in the precondition. If the values match, the request fails with a 304 Not Modified response. + */ + /** + * Restores a soft-deleted file + * @param {RestoreOptions} options Restore options. + * @returns {Promise} + */ + async restore(options: RestoreOptions): Promise { + const [file] = await this.request({ + method: 'POST', + uri: '/restore', + qs: options, + }); + + return file as File; + } + + request(reqOpts: DecorateRequestOptions): Promise; + request( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ): void; + /** + * Makes request and applies userProject query parameter if necessary. + * + * @private + * + * @param {object} reqOpts - The request options. + * @param {function} callback - The callback function. + */ + request( + reqOpts: DecorateRequestOptions, + callback?: BodyResponseCallback + ): void | Promise { + return this.parent.request.call(this, reqOpts, callback!); + } + + rotateEncryptionKey( + options?: RotateEncryptionKeyOptions + ): Promise; + rotateEncryptionKey(callback: RotateEncryptionKeyCallback): void; + rotateEncryptionKey( + options: RotateEncryptionKeyOptions, + callback: RotateEncryptionKeyCallback + ): void; + /** + * @callback RotateEncryptionKeyCallback + * @extends CopyCallback + */ + /** + * @typedef RotateEncryptionKeyResponse + * @extends CopyResponse + */ + /** + * @param {string|buffer|object} RotateEncryptionKeyOptions Configuration options + * for File#rotateEncryptionKey(). + * If a string or Buffer is provided, it is interpreted as an AES-256, + * customer-supplied encryption key. If you'd like to use a Cloud KMS key + * name, you must specify an options object with the property name: + * `kmsKeyName`. + * @param {string|buffer} [options.encryptionKey] An AES-256 encryption key. + * @param {string} [options.kmsKeyName] A Cloud KMS key name. + */ + /** + * This method allows you to update the encryption key associated with this + * file. + * + * See {@link https://cloud.google.com/storage/docs/encryption#customer-supplied| Customer-supplied Encryption Keys} + * + * @param {RotateEncryptionKeyOptions} [options] - Configuration options. + * @param {RotateEncryptionKeyCallback} [callback] + * @returns {Promise} + * + * @example include:samples/encryption.js + * region_tag:storage_rotate_encryption_key + * Example of rotating the encryption key for this file: + */ + rotateEncryptionKey( + optionsOrCallback?: + | RotateEncryptionKeyOptions + | RotateEncryptionKeyCallback, + callback?: RotateEncryptionKeyCallback + ): Promise | void { + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + let options: EncryptionKeyOptions = {}; + if ( + typeof optionsOrCallback === 'string' || + optionsOrCallback instanceof Buffer + ) { + options = { + encryptionKey: optionsOrCallback, + }; + } else if (typeof optionsOrCallback === 'object') { + options = optionsOrCallback as EncryptionKeyOptions; + } + + const newFile = this.bucket.file(this.id!, options); + const copyOptions = + options.preconditionOpts?.ifGenerationMatch !== undefined + ? {preconditionOpts: options.preconditionOpts} + : {}; + this.copy(newFile, copyOptions, callback!); + } + + save(data: SaveData, options?: SaveOptions): Promise; + save(data: SaveData, callback: SaveCallback): void; + save(data: SaveData, options: SaveOptions, callback: SaveCallback): void; + /** + * @typedef {object} SaveOptions + * @extends CreateWriteStreamOptions + */ + /** + * @callback SaveCallback + * @param {?Error} err Request error, if any. + */ + /** + * Write strings or buffers to a file. + * + * *This is a convenience method which wraps {@link File#createWriteStream}.* + * To upload arbitrary data to a file, please use {@link File#createWriteStream} directly. + * + * Resumable uploads are automatically enabled and must be shut off explicitly + * by setting `options.resumable` to `false`. + * + * Multipart uploads with retryable error codes will be retried 3 times with exponential backoff. + * + *

+ * There is some overhead when using a resumable upload that can cause + * noticeable performance degradation while uploading a series of small + * files. When uploading files less than 10MB, it is recommended that the + * resumable feature is disabled. + *

+ * + * @param {SaveData} data The data to write to a file. + * @param {SaveOptions} [options] See {@link File#createWriteStream}'s `options` + * parameter. + * @param {SaveCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const file = myBucket.file('my-file'); + * const contents = 'This is the contents of the file.'; + * + * file.save(contents, function(err) { + * if (!err) { + * // File written successfully. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.save(contents).then(function() {}); + * ``` + */ + save( + data: SaveData, + optionsOrCallback?: SaveOptions | SaveCallback, + callback?: SaveCallback + ): Promise | void { + // tslint:enable:no-any + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + + let maxRetries = this.storage.retryOptions.maxRetries; + if ( + !this.shouldRetryBasedOnPreconditionAndIdempotencyStrat( + options?.preconditionOpts + ) + ) { + maxRetries = 0; + } + const returnValue = AsyncRetry( + async (bail: (err: Error) => void) => { + return new Promise((resolve, reject) => { + if (maxRetries === 0) { + this.storage.retryOptions.autoRetry = false; + } + const writable = this.createWriteStream(options); + + if (options.onUploadProgress) { + writable.on('progress', options.onUploadProgress); + } + + const handleError = (err: Error) => { + if ( + this.storage.retryOptions.autoRetry && + this.storage.retryOptions.retryableErrorFn!(err) + ) { + return reject(err); + } + + return bail(err); + }; + + if ( + typeof data === 'string' || + Buffer.isBuffer(data) || + data instanceof Uint8Array + ) { + writable + .on('error', handleError) + .on('finish', () => resolve()) + .end(data); + } else { + pipeline(data, writable, err => { + if (err) { + if (typeof data !== 'function') { + // Only PipelineSourceFunction can be retried. Async-iterables + // and Readable streams can only be consumed once. + return bail(err); + } + + handleError(err); + } else { + resolve(); + } + }); + } + }); + }, + { + retries: maxRetries, + factor: this.storage.retryOptions.retryDelayMultiplier, + maxTimeout: this.storage.retryOptions.maxRetryDelay! * 1000, //convert to milliseconds + maxRetryTime: this.storage.retryOptions.totalTimeout! * 1000, //convert to milliseconds + } + ); + if (!callback) { + return returnValue; + } else { + return returnValue + .then(() => { + if (callback) { + return callback(); + } + }) + .catch(callback); + } + } + + setMetadata( + metadata: FileMetadata, + options?: SetMetadataOptions + ): Promise>; + setMetadata( + metadata: FileMetadata, + callback: MetadataCallback + ): void; + setMetadata( + metadata: FileMetadata, + options: SetMetadataOptions, + callback: MetadataCallback + ): void; + setMetadata( + metadata: FileMetadata, + optionsOrCallback: SetMetadataOptions | MetadataCallback, + cb?: MetadataCallback + ): Promise> | void { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const options: any = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + cb = + typeof optionsOrCallback === 'function' + ? (optionsOrCallback as MetadataCallback) + : cb; + + this.disableAutoRetryConditionallyIdempotent_( + this.methods.setMetadata, + AvailableServiceObjectMethods.setMetadata, + options + ); + + super + .setMetadata(metadata, options) + .then(resp => cb!(null, ...resp)) + .catch(cb!) + .finally(() => { + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + }); + } + + setStorageClass( + storageClass: string, + options?: SetStorageClassOptions + ): Promise; + setStorageClass( + storageClass: string, + options: SetStorageClassOptions, + callback: SetStorageClassCallback + ): void; + setStorageClass( + storageClass: string, + callback?: SetStorageClassCallback + ): void; + /** + * @typedef {array} SetStorageClassResponse + * @property {object} 0 The full API response. + */ + /** + * @typedef {object} SetStorageClassOptions Configuration options for File#setStorageClass(). + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @callback SetStorageClassCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * Set the storage class for this file. + * + * See {@link https://cloud.google.com/storage/docs/per-object-storage-class| Per-Object Storage Class} + * See {@link https://cloud.google.com/storage/docs/storage-classes| Storage Classes} + * + * @param {string} storageClass The new storage class. (`standard`, + * `nearline`, `coldline`, or `archive`) + * **Note:** The storage classes `multi_regional` and `regional` + * are now legacy and will be deprecated in the future. + * @param {SetStorageClassOptions} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {SetStorageClassCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * file.setStorageClass('nearline', function(err, apiResponse) { + * if (err) { + * // Error handling omitted. + * } + * + * // The storage class was updated successfully. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * file.setStorageClass('nearline').then(function() {}); + * ``` + */ + setStorageClass( + storageClass: string, + optionsOrCallback?: SetStorageClassOptions | SetStorageClassCallback, + callback?: SetStorageClassCallback + ): Promise | void { + callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : callback; + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + + const req = { + ...options, + // In case we get input like `storageClass`, convert to `storage_class`. + storageClass: storageClass + .replace(/-/g, '_') + .replace(/([a-z])([A-Z])/g, (_, low, up) => { + return low + '_' + up; + }) + .toUpperCase(), + }; + + this.copy(this, req, (err, file, apiResponse) => { + if (err) { + callback!(err, apiResponse!); + return; + } + + this.metadata = file!.metadata; + + callback!(null, apiResponse!); + }); + } + + /** + * Set a user project to be billed for all requests made from this File + * object. + * + * @param {string} userProject The user project. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('albums'); + * const file = bucket.file('my-file'); + * + * file.setUserProject('grape-spaceship-123'); + * ``` + */ + setUserProject(userProject: string): void { + this.bucket.setUserProject.call(this, userProject); + } + + /** + * This creates a resumable-upload upload stream. + * + * @param {Duplexify} stream - Duplexify stream of data to pipe to the file. + * @param {object=} options - Configuration object. + * + * @private + */ + startResumableUpload_( + dup: Duplexify, + options: CreateResumableUploadOptions = {} + ): void { + options.metadata ??= {}; + + const retryOptions = this.storage.retryOptions; + if ( + !this.shouldRetryBasedOnPreconditionAndIdempotencyStrat( + options.preconditionOpts + ) + ) { + retryOptions.autoRetry = false; + } + const cfg = { + authClient: this.storage.authClient, + apiEndpoint: this.storage.apiEndpoint, + bucket: this.bucket.name, + customRequestOptions: this.getRequestInterceptors().reduce( + (reqOpts, interceptorFn) => interceptorFn(reqOpts), + {} + ), + file: this.name, + generation: this.generation, + isPartialUpload: options.isPartialUpload, + key: this.encryptionKey, + kmsKeyName: this.kmsKeyName, + metadata: options.metadata, + offset: options.offset, + predefinedAcl: options.predefinedAcl, + private: options.private, + public: options.public, + uri: options.uri, + userProject: options.userProject || this.userProject, + retryOptions: {...retryOptions}, + params: options?.preconditionOpts || this.instancePreconditionOpts, + chunkSize: options?.chunkSize, + highWaterMark: options?.highWaterMark, + universeDomain: this.bucket.storage.universeDomain, + [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY], + }; + + let uploadStream: resumableUpload.Upload; + + try { + uploadStream = resumableUpload.upload(cfg); + } catch (error) { + dup.destroy(error as Error); + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + return; + } + + uploadStream + .on('response', resp => { + dup.emit('response', resp); + }) + .on('uri', uri => { + dup.emit('uri', uri); + }) + .on('metadata', metadata => { + this.metadata = metadata; + dup.emit('metadata'); + }) + .on('finish', () => { + dup.emit('complete'); + }) + .on('progress', evt => dup.emit('progress', evt)); + + dup.setWritable(uploadStream); + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + } + + /** + * Takes a readable stream and pipes it to a remote file. Unlike + * `startResumableUpload_`, which uses the resumable upload technique, this + * method uses a simple upload (all or nothing). + * + * @param {Duplexify} dup - Duplexify stream of data to pipe to the file. + * @param {object=} options - Configuration object. + * + * @private + */ + startSimpleUpload_( + dup: Duplexify, + options: CreateWriteStreamOptions = {} + ): void { + options.metadata ??= {}; + + const apiEndpoint = this.storage.apiEndpoint; + const bucketName = this.bucket.name; + const uri = `${apiEndpoint}/upload/storage/v1/b/${bucketName}/o`; + + const reqOpts: DecorateRequestOptions = { + qs: { + name: this.name, + }, + uri: uri, + [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY], + }; + + if (this.generation !== undefined) { + reqOpts.qs.ifGenerationMatch = this.generation; + } + + if (this.kmsKeyName !== undefined) { + reqOpts.qs.kmsKeyName = this.kmsKeyName; + } + + if (typeof options.timeout === 'number') { + reqOpts.timeout = options.timeout; + } + + if (options.userProject || this.userProject) { + reqOpts.qs.userProject = options.userProject || this.userProject; + } + + if (options.predefinedAcl) { + reqOpts.qs.predefinedAcl = options.predefinedAcl; + } else if (options.private) { + reqOpts.qs.predefinedAcl = 'private'; + } else if (options.public) { + reqOpts.qs.predefinedAcl = 'publicRead'; + } + + Object.assign( + reqOpts.qs, + this.instancePreconditionOpts, + options.preconditionOpts + ); + + util.makeWritableStream(dup, { + makeAuthenticatedRequest: (reqOpts: object) => { + this.request(reqOpts as DecorateRequestOptions, (err, body, resp) => { + if (err) { + dup.destroy(err); + return; + } + + this.metadata = body; + dup.emit('metadata', body); + dup.emit('response', resp); + dup.emit('complete'); + }); + }, + metadata: options.metadata, + request: reqOpts, + }); + } + + disableAutoRetryConditionallyIdempotent_( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + coreOpts: any, + methodType: AvailableServiceObjectMethods, + localPreconditionOptions?: PreconditionOptions + ): void { + if ( + (typeof coreOpts === 'object' && + coreOpts?.reqOpts?.qs?.ifGenerationMatch === undefined && + localPreconditionOptions?.ifGenerationMatch === undefined && + methodType === AvailableServiceObjectMethods.delete && + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryConditional) || + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryNever + ) { + this.storage.retryOptions.autoRetry = false; + } + + if ( + (typeof coreOpts === 'object' && + coreOpts?.reqOpts?.qs?.ifMetagenerationMatch === undefined && + localPreconditionOptions?.ifMetagenerationMatch === undefined && + methodType === AvailableServiceObjectMethods.setMetadata && + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryConditional) || + this.storage.retryOptions.idempotencyStrategy === + IdempotencyStrategy.RetryNever + ) { + this.storage.retryOptions.autoRetry = false; + } + } + + private async getBufferFromReadable(readable: Readable): Promise { + const buf = []; + for await (const chunk of readable) { + buf.push(chunk); + } + + return Buffer.concat(buf); + } + + /** + * + * @param hashCalculatingStream + * @param verify + * @returns {boolean} Returns `true` if valid, throws with error otherwise + */ + async #validateIntegrity( + hashCalculatingStream: HashStreamValidator, + verify: {crc32c?: boolean; md5?: boolean} = {} + ) { + const metadata = this.metadata; + + // If we're doing validation, assume the worst + let dataMismatch = !!(verify.crc32c || verify.md5); + + if (verify.crc32c && metadata.crc32c) { + dataMismatch = !hashCalculatingStream.test('crc32c', metadata.crc32c); + } + + if (verify.md5 && metadata.md5Hash) { + dataMismatch = !hashCalculatingStream.test('md5', metadata.md5Hash); + } + + if (dataMismatch) { + const errors: Error[] = []; + let code = ''; + let message = ''; + + try { + await this.delete(); + + if (verify.md5 && !metadata.md5Hash) { + code = 'MD5_NOT_AVAILABLE'; + message = FileExceptionMessages.MD5_NOT_AVAILABLE; + } else { + code = 'FILE_NO_UPLOAD'; + message = FileExceptionMessages.UPLOAD_MISMATCH; + } + } catch (e) { + const error = e as Error; + + code = 'FILE_NO_UPLOAD_DELETE'; + message = `${FileExceptionMessages.UPLOAD_MISMATCH_DELETE_FAIL}${error.message}`; + + errors.push(error); + } + + const error = new RequestError(message); + error.code = code; + error.errors = errors; + + throw error; + } + + return true; + } +} + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +promisifyAll(File, { + exclude: [ + 'cloudStorageURI', + 'publicUrl', + 'request', + 'save', + 'setEncryptionKey', + 'shouldRetryBasedOnPreconditionAndIdempotencyStrat', + 'getBufferFromReadable', + 'restore', + ], +}); + +/** + * Reference to the {@link File} class. + * @name module:@google-cloud/storage.File + * @see File + */ +export {File}; diff --git a/handwritten/storage/src/hash-stream-validator.ts b/handwritten/storage/src/hash-stream-validator.ts new file mode 100644 index 00000000000..9e9f67d330c --- /dev/null +++ b/handwritten/storage/src/hash-stream-validator.ts @@ -0,0 +1,161 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {createHash, Hash} from 'crypto'; +import {Transform} from 'stream'; + +import { + CRC32CValidatorGenerator, + CRC32C_DEFAULT_VALIDATOR_GENERATOR, + CRC32CValidator, +} from './crc32c.js'; +import {FileExceptionMessages, RequestError} from './file.js'; + +interface HashStreamValidatorOptions { + /** Enables CRC32C calculation. To validate a provided value use `crc32cExpected`. */ + crc32c: boolean; + /** Enables MD5 calculation. To validate a provided value use `md5Expected`. */ + md5: boolean; + /** A CRC32C instance for validation. To validate a provided value use `crc32cExpected`. */ + crc32cInstance: CRC32CValidator; + /** Set a custom CRC32C generator. Used if `crc32cInstance` has not been provided. */ + crc32cGenerator: CRC32CValidatorGenerator; + /** Sets the expected CRC32C value to verify once all data has been consumed. Also sets the `crc32c` option to `true` */ + crc32cExpected?: string; + /** Sets the expected MD5 value to verify once all data has been consumed. Also sets the `md5` option to `true` */ + md5Expected?: string; + /** Indicates whether or not to run a validation check or only update the hash values */ + updateHashesOnly?: boolean; +} +class HashStreamValidator extends Transform { + readonly crc32cEnabled: boolean; + readonly md5Enabled: boolean; + readonly crc32cExpected: string | undefined; + readonly md5Expected: string | undefined; + readonly updateHashesOnly: boolean = false; + + #crc32cHash?: CRC32CValidator = undefined; + #md5Hash?: Hash = undefined; + #md5Digest = ''; + + constructor(options: Partial = {}) { + super(); + + this.crc32cEnabled = !!options.crc32c; + this.md5Enabled = !!options.md5; + this.updateHashesOnly = !!options.updateHashesOnly; + this.crc32cExpected = options.crc32cExpected; + this.md5Expected = options.md5Expected; + + if (this.crc32cEnabled) { + if (options.crc32cInstance) { + this.#crc32cHash = options.crc32cInstance; + } else { + const crc32cGenerator = + options.crc32cGenerator || CRC32C_DEFAULT_VALIDATOR_GENERATOR; + + this.#crc32cHash = crc32cGenerator(); + } + } + + if (this.md5Enabled) { + this.#md5Hash = createHash('md5'); + } + } + + /** + * Return the current CRC32C value, if available. + */ + get crc32c() { + return this.#crc32cHash?.toString(); + } + + /** + * Return the calculated MD5 value, if available. + */ + get md5Digest(): string | undefined { + if (this.#md5Hash && !this.#md5Digest) { + this.#md5Digest = this.#md5Hash.digest('base64'); + } + return this.#md5Digest; + } + + _flush(callback: (error?: Error | null | undefined) => void) { + // Triggers the getter logic to finalize and cache the MD5 digest + this.md5Digest; + + if (this.updateHashesOnly) { + callback(); + return; + } + + // If we're doing validation, assume the worst-- a data integrity + // mismatch. If not, these tests won't be performed, and we can assume + // the best. + // We must check if the server decompressed the data on serve because hash + // validation is not possible in this case. + let failed = this.crc32cEnabled || this.md5Enabled; + + if (this.crc32cEnabled && this.crc32cExpected) { + failed = !this.test('crc32c', this.crc32cExpected); + } + + if (this.md5Enabled && this.md5Expected) { + failed = !this.test('md5', this.md5Expected); + } + + if (failed) { + const mismatchError = new RequestError( + FileExceptionMessages.DOWNLOAD_MISMATCH + ); + mismatchError.code = 'CONTENT_DOWNLOAD_MISMATCH'; + + callback(mismatchError); + } else { + callback(); + } + } + + _transform( + chunk: Buffer, + encoding: BufferEncoding, + callback: (e?: Error) => void + ) { + this.push(chunk, encoding); + + try { + if (this.#crc32cHash) this.#crc32cHash.update(chunk); + if (this.#md5Hash) this.#md5Hash.update(chunk); + callback(); + } catch (e) { + callback(e as Error); + } + } + + test(hash: 'crc32c' | 'md5', sum: Buffer | string): boolean { + const check = Buffer.isBuffer(sum) ? sum.toString('base64') : sum; + + if (hash === 'crc32c' && this.#crc32cHash) { + return this.#crc32cHash.validate(check); + } + + if (hash === 'md5' && this.#md5Hash) { + return this.#md5Digest === check; + } + + return false; + } +} + +export {HashStreamValidator, HashStreamValidatorOptions}; diff --git a/handwritten/storage/src/hmacKey.ts b/handwritten/storage/src/hmacKey.ts new file mode 100644 index 00000000000..9b79daa5e8c --- /dev/null +++ b/handwritten/storage/src/hmacKey.ts @@ -0,0 +1,419 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + ServiceObject, + Methods, + MetadataCallback, + SetMetadataResponse, +} from './nodejs-common/index.js'; +import { + BaseMetadata, + SetMetadataOptions, +} from './nodejs-common/service-object.js'; +import {IdempotencyStrategy, Storage} from './storage.js'; +import {promisifyAll} from '@google-cloud/promisify'; + +export interface HmacKeyOptions { + projectId?: string; +} + +export interface HmacKeyMetadata extends BaseMetadata { + accessId?: string; + etag?: string; + projectId?: string; + serviceAccountEmail?: string; + state?: string; + timeCreated?: string; + updated?: string; +} + +export interface SetHmacKeyMetadataOptions { + /** + * This parameter is currently ignored. + */ + userProject?: string; +} + +export interface SetHmacKeyMetadata { + state?: 'ACTIVE' | 'INACTIVE'; + etag?: string; +} + +export interface HmacKeyMetadataCallback { + (err: Error | null, metadata?: HmacKeyMetadata, apiResponse?: unknown): void; +} + +export type HmacKeyMetadataResponse = [HmacKeyMetadata, unknown]; + +/** + * The API-formatted resource description of the HMAC key. + * + * Note: This is not guaranteed to be up-to-date when accessed. To get the + * latest record, call the `getMetadata()` method. + * + * @name HmacKey#metadata + * @type {object} + */ +/** + * An HmacKey object contains metadata of an HMAC key created from a + * service account through the {@link Storage} client using + * {@link Storage#createHmacKey}. + * + * See {@link https://cloud.google.com/storage/docs/authentication/hmackeys| HMAC keys documentation} + * + * @class + */ +export class HmacKey extends ServiceObject { + /** + * A reference to the {@link Storage} associated with this {@link HmacKey} + * instance. + * @name HmacKey#storage + * @type {Storage} + */ + storage: Storage; + private instanceRetryValue?: boolean; + + /** + * @typedef {object} HmacKeyOptions + * @property {string} [projectId] The project ID of the project that owns + * the service account of the requested HMAC key. If not provided, + * the project ID used to instantiate the Storage client will be used. + */ + /** + * Constructs an HmacKey object. + * + * Note: this only create a local reference to an HMAC key, to create + * an HMAC key, use {@link Storage#createHmacKey}. + * + * @param {Storage} storage The Storage instance this HMAC key is + * attached to. + * @param {string} accessId The unique accessId for this HMAC key. + * @param {HmacKeyOptions} options Constructor configurations. + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const hmacKey = storage.hmacKey('access-id'); + * ``` + */ + constructor(storage: Storage, accessId: string, options?: HmacKeyOptions) { + const methods = { + /** + * @typedef {object} DeleteHmacKeyOptions + * @property {string} [userProject] This parameter is currently ignored. + */ + /** + * @typedef {array} DeleteHmacKeyResponse + * @property {object} 0 The full API response. + */ + /** + * @callback DeleteHmacKeyCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ + /** + * Deletes an HMAC key. + * Key state must be set to `INACTIVE` prior to deletion. + * Caution: HMAC keys cannot be recovered once you delete them. + * + * The authenticated user must have `storage.hmacKeys.delete` permission for the project in which the key exists. + * + * @method HmacKey#delete + * @param {DeleteHmacKeyOptions} [options] Configuration options. + * @param {DeleteHmacKeyCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * + * //- + * // Delete HMAC key after making the key inactive. + * //- + * const hmacKey = storage.hmacKey('ACCESS_ID'); + * hmacKey.setMetadata({state: 'INACTIVE'}, (err, hmacKeyMetadata) => { + * if (err) { + * // The request was an error. + * console.error(err); + * return; + * } + * hmacKey.delete((err) => { + * if (err) { + * console.error(err); + * return; + * } + * // The HMAC key is deleted. + * }); + * }); + * + * //- + * // If the callback is omitted, a promise is returned. + * //- + * const hmacKey = storage.hmacKey('ACCESS_ID'); + * hmacKey + * .setMetadata({state: 'INACTIVE'}) + * .then(() => { + * return hmacKey.delete(); + * }); + * ``` + */ + delete: true, + /** + * @callback GetHmacKeyCallback + * @param {?Error} err Request error, if any. + * @param {HmacKey} hmacKey this {@link HmacKey} instance. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {array} GetHmacKeyResponse + * @property {HmacKey} 0 This {@link HmacKey} instance. + * @property {object} 1 The full API response. + */ + /** + * @typedef {object} GetHmacKeyOptions + * @property {string} [userProject] This parameter is currently ignored. + */ + /** + * Retrieves and populate an HMAC key's metadata, and return + * this {@link HmacKey} instance. + * + * HmacKey.get() does not give the HMAC key secret, as + * it is only returned on creation. + * + * The authenticated user must have `storage.hmacKeys.get` permission + * for the project in which the key exists. + * + * @method HmacKey#get + * @param {GetHmacKeyOptions} [options] Configuration options. + * @param {GetHmacKeyCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * + * //- + * // Get the HmacKey's Metadata. + * //- + * storage.hmacKey('ACCESS_ID') + * .get((err, hmacKey) => { + * if (err) { + * // The request was an error. + * console.error(err); + * return; + * } + * // do something with the returned HmacKey object. + * }); + * + * //- + * // If the callback is omitted, a promise is returned. + * //- + * storage.hmacKey('ACCESS_ID') + * .get() + * .then((data) => { + * const hmacKey = data[0]; + * }); + * ``` + */ + get: true, + /** + * @typedef {object} GetHmacKeyMetadataOptions + * @property {string} [userProject] This parameter is currently ignored. + */ + /** + * Retrieves and populate an HMAC key's metadata, and return + * the HMAC key's metadata as an object. + * + * HmacKey.getMetadata() does not give the HMAC key secret, as + * it is only returned on creation. + * + * The authenticated user must have `storage.hmacKeys.get` permission + * for the project in which the key exists. + * + * @method HmacKey#getMetadata + * @param {GetHmacKeyMetadataOptions} [options] Configuration options. + * @param {HmacKeyMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * + * //- + * // Get the HmacKey's metadata and populate to the metadata property. + * //- + * storage.hmacKey('ACCESS_ID') + * .getMetadata((err, hmacKeyMetadata) => { + * if (err) { + * // The request was an error. + * console.error(err); + * return; + * } + * console.log(hmacKeyMetadata); + * }); + * + * //- + * // If the callback is omitted, a promise is returned. + * //- + * storage.hmacKey('ACCESS_ID') + * .getMetadata() + * .then((data) => { + * const hmacKeyMetadata = data[0]; + * console.log(hmacKeyMetadata); + * }); + * ``` + */ + getMetadata: true, + /** + * @typedef {object} SetHmacKeyMetadata Subset of {@link HmacKeyMetadata} to update. + * @property {string} state New state of the HmacKey. Either 'ACTIVE' or 'INACTIVE'. + * @property {string} [etag] Include an etag from a previous get HMAC key request + * to perform safe read-modify-write. + */ + /** + * @typedef {object} SetHmacKeyMetadataOptions + * @property {string} [userProject] This parameter is currently ignored. + */ + /** + * @callback HmacKeyMetadataCallback + * @param {?Error} err Request error, if any. + * @param {HmacKeyMetadata} metadata The updated {@link HmacKeyMetadata} object. + * @param {object} apiResponse The full API response. + */ + /** + * @typedef {array} HmacKeyMetadataResponse + * @property {HmacKeyMetadata} 0 The updated {@link HmacKeyMetadata} object. + * @property {object} 1 The full API response. + */ + /** + * Updates the state of an HMAC key. See {@link SetHmacKeyMetadata} for + * valid states. + * + * @method HmacKey#setMetadata + * @param {SetHmacKeyMetadata} metadata The new metadata. + * @param {SetHmacKeyMetadataOptions} [options] Configuration options. + * @param {HmacKeyMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * + * const metadata = { + * state: 'INACTIVE', + * }; + * + * storage.hmacKey('ACCESS_ID') + * .setMetadata(metadata, (err, hmacKeyMetadata) => { + * if (err) { + * // The request was an error. + * console.error(err); + * return; + * } + * console.log(hmacKeyMetadata); + * }); + * + * //- + * // If the callback is omitted, a promise is returned. + * //- + * storage.hmacKey('ACCESS_ID') + * .setMetadata(metadata) + * .then((data) => { + * const hmacKeyMetadata = data[0]; + * console.log(hmacKeyMetadata); + * }); + * ``` + */ + setMetadata: { + reqOpts: { + method: 'PUT', + }, + }, + } as Methods; + + const projectId = (options && options.projectId) || storage.projectId; + + super({ + parent: storage, + id: accessId, + baseUrl: `/projects/${projectId}/hmacKeys`, + methods, + }); + + this.storage = storage; + this.instanceRetryValue = storage.retryOptions.autoRetry; + } + + /** + * Set the metadata for this object. + * + * @param {object} metadata - The metadata to set on this object. + * @param {object=} options - Configuration options. + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.apiResponse - The full API response. + */ + setMetadata( + metadata: HmacKeyMetadata, + options?: SetMetadataOptions + ): Promise>; + setMetadata( + metadata: HmacKeyMetadata, + callback: MetadataCallback + ): void; + setMetadata( + metadata: HmacKeyMetadata, + options: SetMetadataOptions, + callback: MetadataCallback + ): void; + setMetadata( + metadata: HmacKeyMetadata, + optionsOrCallback: SetMetadataOptions | MetadataCallback, + cb?: MetadataCallback + ): Promise> | void { + // ETag preconditions are not currently supported. Retries should be disabled if the idempotency strategy is not set to RetryAlways + if ( + this.storage.retryOptions.idempotencyStrategy !== + IdempotencyStrategy.RetryAlways + ) { + this.storage.retryOptions.autoRetry = false; + } + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + cb = + typeof optionsOrCallback === 'function' + ? (optionsOrCallback as MetadataCallback) + : cb; + + super + .setMetadata(metadata, options) + .then(resp => cb!(null, ...resp)) + .catch(cb!) + .finally(() => { + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + }); + } +} + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +promisifyAll(HmacKey); diff --git a/handwritten/storage/src/iam.ts b/handwritten/storage/src/iam.ts new file mode 100644 index 00000000000..8f6ee5d76d3 --- /dev/null +++ b/handwritten/storage/src/iam.ts @@ -0,0 +1,497 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + BodyResponseCallback, + DecorateRequestOptions, +} from './nodejs-common/index.js'; +import {promisifyAll} from '@google-cloud/promisify'; + +import {Bucket} from './bucket.js'; +import {normalize} from './util.js'; + +export interface GetPolicyOptions { + userProject?: string; + requestedPolicyVersion?: number; +} + +export type GetPolicyResponse = [Policy, unknown]; + +/** + * @callback GetPolicyCallback + * @param {?Error} err Request error, if any. + * @param {object} acl The policy. + * @param {object} apiResponse The full API response. + */ +export interface GetPolicyCallback { + (err?: Error | null, acl?: Policy, apiResponse?: unknown): void; +} + +/** + * @typedef {object} SetPolicyOptions + * @param {string} [userProject] The ID of the project which will be + * billed for the request. + */ +export interface SetPolicyOptions { + userProject?: string; +} + +/** + * @typedef {array} SetPolicyResponse + * @property {object} 0 The policy. + * @property {object} 1 The full API response. + */ +export type SetPolicyResponse = [Policy, unknown]; + +/** + * @callback SetPolicyCallback + * @param {?Error} err Request error, if any. + * @param {object} acl The policy. + * @param {object} apiResponse The full API response. + */ +export interface SetPolicyCallback { + (err?: Error | null, acl?: Policy, apiResponse?: object): void; +} + +export interface Policy { + bindings: PolicyBinding[]; + version?: number; + etag?: string; +} + +export interface PolicyBinding { + role: string; + members: string[]; + condition?: Expr; +} + +export interface Expr { + title?: string; + description?: string; + expression: string; +} + +/** + * @typedef {array} TestIamPermissionsResponse + * @property {object} 0 A subset of permissions that the caller is allowed. + * @property {object} 1 The full API response. + */ +export type TestIamPermissionsResponse = [{[key: string]: boolean}, unknown]; + +/** + * @callback TestIamPermissionsCallback + * @param {?Error} err Request error, if any. + * @param {object} acl A subset of permissions that the caller is allowed. + * @param {object} apiResponse The full API response. + */ +export interface TestIamPermissionsCallback { + ( + err?: Error | null, + acl?: {[key: string]: boolean} | null, + apiResponse?: unknown + ): void; +} + +/** + * @typedef {object} TestIamPermissionsOptions Configuration options for Iam#testPermissions(). + * @param {string} [userProject] The ID of the project which will be + * billed for the request. + */ +export interface TestIamPermissionsOptions { + userProject?: string; +} + +interface GetPolicyRequest { + userProject?: string; + optionsRequestedPolicyVersion?: number; +} + +export enum IAMExceptionMessages { + POLICY_OBJECT_REQUIRED = 'A policy object is required.', + PERMISSIONS_REQUIRED = 'Permissions are required.', +} + +/** + * Get and set IAM policies for your Cloud Storage bucket. + * + * See {@link https://cloud.google.com/storage/docs/access-control/iam#short_title_iam_management| Cloud Storage IAM Management} + * See {@link https://cloud.google.com/iam/docs/granting-changing-revoking-access| Granting, Changing, and Revoking Access} + * See {@link https://cloud.google.com/iam/docs/understanding-roles| IAM Roles} + * + * @constructor Iam + * + * @param {Bucket} bucket The parent instance. + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * // bucket.iam + * ``` + */ +class Iam { + private request_: ( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ) => void; + private resourceId_: string; + + constructor(bucket: Bucket) { + this.request_ = bucket.request.bind(bucket); + this.resourceId_ = 'buckets/' + bucket.getId(); + } + + getPolicy(options?: GetPolicyOptions): Promise; + getPolicy(options: GetPolicyOptions, callback: GetPolicyCallback): void; + getPolicy(callback: GetPolicyCallback): void; + /** + * @typedef {object} GetPolicyOptions Requested options for IAM#getPolicy(). + * @property {number} [requestedPolicyVersion] The version of IAM policies to + * request. If a policy with a condition is requested without setting + * this, the server will return an error. This must be set to a value + * of 3 to retrieve IAM policies containing conditions. This is to + * prevent client code that isn't aware of IAM conditions from + * interpreting and modifying policies incorrectly. The service might + * return a policy with version lower than the one that was requested, + * based on the feature syntax in the policy fetched. + * See {@link https://cloud.google.com/iam/docs/policies#versions| IAM Policy versions} + * @property {string} [userProject] The ID of the project which will be + * billed for the request. + */ + /** + * @typedef {array} GetPolicyResponse + * @property {Policy} 0 The policy. + * @property {object} 1 The full API response. + */ + /** + * @typedef {object} Policy + * @property {PolicyBinding[]} policy.bindings Bindings associate members with roles. + * @property {string} [policy.etag] Etags are used to perform a read-modify-write. + * @property {number} [policy.version] The syntax schema version of the Policy. + * To set an IAM policy with conditional binding, this field must be set to + * 3 or greater. + * See {@link https://cloud.google.com/iam/docs/policies#versions| IAM Policy versions} + */ + /** + * @typedef {object} PolicyBinding + * @property {string} role Role that is assigned to members. + * @property {string[]} members Specifies the identities requesting access for the bucket. + * @property {Expr} [condition] The condition that is associated with this binding. + */ + /** + * @typedef {object} Expr + * @property {string} [title] An optional title for the expression, i.e. a + * short string describing its purpose. This can be used e.g. in UIs + * which allow to enter the expression. + * @property {string} [description] An optional description of the + * expression. This is a longer text which describes the expression, + * e.g. when hovered over it in a UI. + * @property {string} expression Textual representation of an expression in + * Common Expression Language syntax. The application context of the + * containing message determines which well-known feature set of CEL + * is supported.The condition that is associated with this binding. + * + * @see [Condition] https://cloud.google.com/storage/docs/access-control/iam#conditions + */ + /** + * Get the IAM policy. + * + * @param {GetPolicyOptions} [options] Request options. + * @param {GetPolicyCallback} [callback] Callback function. + * @returns {Promise} + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/getIamPolicy| Buckets: setIamPolicy API Documentation} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * + * bucket.iam.getPolicy( + * {requestedPolicyVersion: 3}, + * function(err, policy, apiResponse) { + * + * }, + * ); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.iam.getPolicy({requestedPolicyVersion: 3}) + * .then(function(data) { + * const policy = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/iam.js + * region_tag:storage_view_bucket_iam_members + * Example of retrieving a bucket's IAM policy: + */ + getPolicy( + optionsOrCallback?: GetPolicyOptions | GetPolicyCallback, + callback?: GetPolicyCallback + ): Promise | void { + const {options, callback: cb} = normalize< + GetPolicyOptions, + GetPolicyCallback + >(optionsOrCallback, callback); + + const qs: GetPolicyRequest = {}; + if (options.userProject) { + qs.userProject = options.userProject; + } + + if ( + options.requestedPolicyVersion !== null && + options.requestedPolicyVersion !== undefined + ) { + qs.optionsRequestedPolicyVersion = options.requestedPolicyVersion; + } + + this.request_( + { + uri: '/iam', + qs, + }, + cb! + ); + } + + setPolicy( + policy: Policy, + options?: SetPolicyOptions + ): Promise; + setPolicy(policy: Policy, callback: SetPolicyCallback): void; + setPolicy( + policy: Policy, + options: SetPolicyOptions, + callback: SetPolicyCallback + ): void; + /** + * Set the IAM policy. + * + * @throws {Error} If no policy is provided. + * + * @param {Policy} policy The policy. + * @param {SetPolicyOptions} [options] Configuration options. + * @param {SetPolicyCallback} callback Callback function. + * @returns {Promise} + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/setIamPolicy| Buckets: setIamPolicy API Documentation} + * See {@link https://cloud.google.com/iam/docs/understanding-roles| IAM Roles} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * + * const myPolicy = { + * bindings: [ + * { + * role: 'roles/storage.admin', + * members: + * ['serviceAccount:myotherproject@appspot.gserviceaccount.com'] + * } + * ] + * }; + * + * bucket.iam.setPolicy(myPolicy, function(err, policy, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.iam.setPolicy(myPolicy).then(function(data) { + * const policy = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/iam.js + * region_tag:storage_add_bucket_iam_member + * Example of adding to a bucket's IAM policy: + * + * @example include:samples/iam.js + * region_tag:storage_remove_bucket_iam_member + * Example of removing from a bucket's IAM policy: + */ + setPolicy( + policy: Policy, + optionsOrCallback?: SetPolicyOptions | SetPolicyCallback, + callback?: SetPolicyCallback + ): Promise | void { + if (policy === null || typeof policy !== 'object') { + throw new Error(IAMExceptionMessages.POLICY_OBJECT_REQUIRED); + } + + const {options, callback: cb} = normalize< + SetPolicyOptions, + SetPolicyCallback + >(optionsOrCallback, callback); + + let maxRetries; + if (policy.etag === undefined) { + maxRetries = 0; + } + + this.request_( + { + method: 'PUT', + uri: '/iam', + maxRetries, + json: Object.assign( + { + resourceId: this.resourceId_, + }, + policy + ), + qs: options, + }, + cb + ); + } + + testPermissions( + permissions: string | string[], + options?: TestIamPermissionsOptions + ): Promise; + testPermissions( + permissions: string | string[], + callback: TestIamPermissionsCallback + ): void; + testPermissions( + permissions: string | string[], + options: TestIamPermissionsOptions, + callback: TestIamPermissionsCallback + ): void; + /** + * Test a set of permissions for a resource. + * + * @throws {Error} If permissions are not provided. + * + * @param {string|string[]} permissions The permission(s) to test for. + * @param {TestIamPermissionsOptions} [options] Configuration object. + * @param {TestIamPermissionsCallback} [callback] Callback function. + * @returns {Promise} + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/testIamPermissions| Buckets: testIamPermissions API Documentation} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * + * //- + * // Test a single permission. + * //- + * const test = 'storage.buckets.delete'; + * + * bucket.iam.testPermissions(test, function(err, permissions, apiResponse) { + * console.log(permissions); + * // { + * // "storage.buckets.delete": true + * // } + * }); + * + * //- + * // Test several permissions at once. + * //- + * const tests = [ + * 'storage.buckets.delete', + * 'storage.buckets.get' + * ]; + * + * bucket.iam.testPermissions(tests, function(err, permissions) { + * console.log(permissions); + * // { + * // "storage.buckets.delete": false, + * // "storage.buckets.get": true + * // } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * bucket.iam.testPermissions(test).then(function(data) { + * const permissions = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + testPermissions( + permissions: string | string[], + optionsOrCallback?: TestIamPermissionsOptions | TestIamPermissionsCallback, + callback?: TestIamPermissionsCallback + ): Promise | void { + if (!Array.isArray(permissions) && typeof permissions !== 'string') { + throw new Error(IAMExceptionMessages.PERMISSIONS_REQUIRED); + } + + const {options, callback: cb} = normalize< + TestIamPermissionsOptions, + TestIamPermissionsCallback + >(optionsOrCallback, callback); + + const permissionsArray = Array.isArray(permissions) + ? permissions + : [permissions]; + + const req = Object.assign( + { + permissions: permissionsArray, + }, + options + ); + + this.request_( + { + uri: '/iam/testPermissions', + qs: req, + useQuerystring: true, + }, + (err, resp) => { + if (err) { + cb!(err, null, resp); + return; + } + + const availablePermissions = Array.isArray(resp.permissions) + ? resp.permissions + : []; + + const permissionsHash = permissionsArray.reduce( + (acc: {[index: string]: boolean}, permission) => { + acc[permission] = availablePermissions.indexOf(permission) > -1; + return acc; + }, + {} + ); + + cb!(null, permissionsHash, resp); + } + ); + } +} + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +promisifyAll(Iam); + +export {Iam}; diff --git a/handwritten/storage/src/index.ts b/handwritten/storage/src/index.ts new file mode 100644 index 00000000000..32d2728bdeb --- /dev/null +++ b/handwritten/storage/src/index.ts @@ -0,0 +1,272 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * The `@google-cloud/storage` package has a single named export which is the + * {@link Storage} (ES6) class, which should be instantiated with `new`. + * + * See {@link Storage} and {@link ClientConfig} for client methods and + * configuration options. + * + * @module {Storage} @google-cloud/storage + * @alias nodejs-storage + * + * @example + * Install the client library with npm: + * ``` + * npm install --save @google-cloud/storage + * ``` + * + * @example + * Import the client library + * ``` + * const {Storage} = require('@google-cloud/storage'); + * ``` + * + * @example + * Create a client that uses Application + * Default Credentials (ADC): + * ``` + * const storage = new Storage(); + * ``` + * + * @example + * Create a client with explicit + * credentials: + * ``` + * const storage = new Storage({ projectId: + * 'your-project-id', keyFilename: '/path/to/keyfile.json' + * }); + * ``` + * + * @example include:samples/quickstart.js + * region_tag:storage_quickstart + * Full quickstart example: + */ +export {ApiError} from './nodejs-common/index.js'; +export { + BucketCallback, + BucketOptions, + CreateBucketQuery, + CreateBucketRequest, + CreateBucketResponse, + CreateHmacKeyCallback, + CreateHmacKeyOptions, + CreateHmacKeyResponse, + GetBucketsCallback, + GetBucketsRequest, + GetBucketsResponse, + GetHmacKeysCallback, + GetHmacKeysOptions, + GetHmacKeysResponse, + GetServiceAccountCallback, + GetServiceAccountOptions, + GetServiceAccountResponse, + HmacKeyResourceResponse, + IdempotencyStrategy, + PreconditionOptions, + RETRYABLE_ERR_FN_DEFAULT, + ServiceAccount, + Storage, + StorageOptions, +} from './storage.js'; +export { + AclMetadata, + AccessControlObject, + AclOptions, + AddAclCallback, + AddAclOptions, + AddAclResponse, + GetAclCallback, + GetAclOptions, + GetAclResponse, + RemoveAclCallback, + RemoveAclOptions, + RemoveAclResponse, + UpdateAclCallback, + UpdateAclOptions, + UpdateAclResponse, +} from './acl.js'; +export { + Bucket, + BucketExistsCallback, + BucketExistsOptions, + BucketExistsResponse, + BucketLockCallback, + BucketLockResponse, + BucketMetadata, + CombineCallback, + CombineOptions, + CombineResponse, + CreateChannelCallback, + CreateChannelConfig, + CreateChannelOptions, + CreateChannelResponse, + CreateNotificationCallback, + CreateNotificationOptions, + CreateNotificationResponse, + DeleteBucketCallback, + DeleteBucketOptions, + DeleteBucketResponse, + DeleteFilesCallback, + DeleteFilesOptions, + DeleteLabelsCallback, + DeleteLabelsResponse, + DisableRequesterPaysCallback, + DisableRequesterPaysResponse, + EnableRequesterPaysCallback, + EnableRequesterPaysResponse, + GetBucketCallback, + GetBucketMetadataCallback, + GetBucketMetadataOptions, + GetBucketMetadataResponse, + GetBucketOptions, + GetBucketResponse, + GetBucketSignedUrlConfig, + GetFilesCallback, + GetFilesOptions, + GetFilesResponse, + GetLabelsCallback, + GetLabelsOptions, + GetLabelsResponse, + GetNotificationsCallback, + GetNotificationsOptions, + GetNotificationsResponse, + Labels, + LifecycleAction, + LifecycleCondition, + LifecycleRule, + MakeBucketPrivateCallback, + MakeBucketPrivateOptions, + MakeBucketPrivateResponse, + MakeBucketPublicCallback, + MakeBucketPublicOptions, + MakeBucketPublicResponse, + SetBucketMetadataCallback, + SetBucketMetadataOptions, + SetBucketMetadataResponse, + SetBucketStorageClassCallback, + SetBucketStorageClassOptions, + SetLabelsCallback, + SetLabelsOptions, + SetLabelsResponse, + UploadCallback, + UploadOptions, + UploadResponse, +} from './bucket.js'; +export * from './crc32c.js'; +export {Channel, StopCallback} from './channel.js'; +export { + CopyCallback, + CopyOptions, + CopyResponse, + CreateReadStreamOptions, + CreateResumableUploadCallback, + CreateResumableUploadOptions, + CreateResumableUploadResponse, + CreateWriteStreamOptions, + DeleteFileCallback, + DeleteFileOptions, + DeleteFileResponse, + DownloadCallback, + DownloadOptions, + DownloadResponse, + EncryptionKeyOptions, + File, + FileExistsCallback, + FileExistsOptions, + FileExistsResponse, + FileMetadata, + FileOptions, + GetExpirationDateCallback, + GetExpirationDateResponse, + GetFileCallback, + GetFileMetadataCallback, + GetFileMetadataOptions, + GetFileMetadataResponse, + GetFileOptions, + GetFileResponse, + GenerateSignedPostPolicyV2Callback, + GenerateSignedPostPolicyV2Options, + GenerateSignedPostPolicyV2Response, + GenerateSignedPostPolicyV4Callback, + GenerateSignedPostPolicyV4Options, + GenerateSignedPostPolicyV4Response, + GetSignedUrlConfig, + MakeFilePrivateCallback, + MakeFilePrivateOptions, + MakeFilePrivateResponse, + MakeFilePublicCallback, + MakeFilePublicResponse, + MoveCallback, + MoveOptions, + MoveResponse, + MoveFileAtomicOptions, + MoveFileAtomicCallback, + MoveFileAtomicResponse, + PolicyDocument, + PolicyFields, + PredefinedAcl, + RotateEncryptionKeyCallback, + RotateEncryptionKeyOptions, + RotateEncryptionKeyResponse, + SaveCallback, + SaveData, + SaveOptions, + SetFileMetadataCallback, + SetFileMetadataOptions, + SetFileMetadataResponse, + SetStorageClassCallback, + SetStorageClassOptions, + SetStorageClassResponse, + SignedPostPolicyV4Output, +} from './file.js'; +export * from './hash-stream-validator.js'; +export { + HmacKey, + HmacKeyMetadata, + HmacKeyMetadataCallback, + HmacKeyMetadataResponse, + SetHmacKeyMetadata, + SetHmacKeyMetadataOptions, +} from './hmacKey.js'; +export { + GetPolicyCallback, + GetPolicyOptions, + GetPolicyResponse, + Iam, + Policy, + SetPolicyCallback, + SetPolicyOptions, + SetPolicyResponse, + TestIamPermissionsCallback, + TestIamPermissionsOptions, + TestIamPermissionsResponse, +} from './iam.js'; +export { + DeleteNotificationCallback, + DeleteNotificationOptions, + GetNotificationCallback, + GetNotificationMetadataCallback, + GetNotificationMetadataOptions, + GetNotificationMetadataResponse, + GetNotificationOptions, + GetNotificationResponse, + Notification, + NotificationMetadata, +} from './notification.js'; +export {GetSignedUrlCallback, GetSignedUrlResponse} from './signer.js'; +export * from './transfer-manager.js'; diff --git a/handwritten/storage/src/nodejs-common/index.ts b/handwritten/storage/src/nodejs-common/index.ts new file mode 100644 index 00000000000..89ed3ea815e --- /dev/null +++ b/handwritten/storage/src/nodejs-common/index.ts @@ -0,0 +1,50 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export {GoogleAuthOptions} from 'google-auth-library'; + +export { + Service, + ServiceConfig, + ServiceOptions, + StreamRequestOptions, +} from './service.js'; + +export { + BaseMetadata, + DeleteCallback, + ExistsCallback, + GetConfig, + InstanceResponseCallback, + Interceptor, + MetadataCallback, + MetadataResponse, + Methods, + ResponseCallback, + ServiceObject, + ServiceObjectConfig, + ServiceObjectParent, + SetMetadataResponse, +} from './service-object.js'; + +export { + Abortable, + AbortableDuplex, + ApiError, + BodyResponseCallback, + DecorateRequestOptions, + ResponseBody, + util, +} from './util.js'; diff --git a/handwritten/storage/src/nodejs-common/service-object.ts b/handwritten/storage/src/nodejs-common/service-object.ts new file mode 100644 index 00000000000..a4e369a3f5e --- /dev/null +++ b/handwritten/storage/src/nodejs-common/service-object.ts @@ -0,0 +1,621 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import {promisifyAll} from '@google-cloud/promisify'; +import {EventEmitter} from 'events'; +import * as r from 'teeny-request'; + +import {StreamRequestOptions} from './service.js'; +import { + ApiError, + BodyResponseCallback, + DecorateRequestOptions, + ResponseBody, + util, +} from './util.js'; + +export type RequestResponse = [unknown, r.Response]; + +export interface ServiceObjectParent { + interceptors: Interceptor[]; + getRequestInterceptors(): Function[]; + requestStream(reqOpts: DecorateRequestOptions): r.Request; + request( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ): void; +} + +export interface Interceptor { + request(opts: r.Options): DecorateRequestOptions; +} + +export type GetMetadataOptions = object; + +export type MetadataResponse = [K, r.Response]; +export type MetadataCallback = ( + err: Error | null, + metadata?: K, + apiResponse?: r.Response +) => void; + +export type ExistsOptions = object; +export interface ExistsCallback { + (err: Error | null, exists?: boolean): void; +} + +export interface ServiceObjectConfig { + /** + * The base URL to make API requests to. + */ + baseUrl?: string; + + /** + * The method which creates this object. + */ + createMethod?: Function; + + /** + * The identifier of the object. For example, the name of a Storage bucket or + * Pub/Sub topic. + */ + id?: string; + + /** + * A map of each method name that should be inherited. + */ + methods?: Methods; + + /** + * The parent service instance. For example, an instance of Storage if the + * object is Bucket. + */ + parent: ServiceObjectParent; + + /** + * Override of projectId, used to allow access to resources in another project. + * For example, a BigQuery dataset in another project to which the user has been + * granted permission. + */ + projectId?: string; +} + +export interface Methods { + [methodName: string]: {reqOpts?: r.CoreOptions} | boolean; +} + +export interface InstanceResponseCallback { + (err: ApiError | null, instance?: T | null, apiResponse?: r.Response): void; +} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface CreateOptions {} +// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unused-vars +export type CreateResponse = any[]; +export interface CreateCallback { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (err: ApiError | null, instance?: T | null, ...args: any[]): void; +} + +export type DeleteOptions = { + ignoreNotFound?: boolean; + ifGenerationMatch?: number | string; + ifGenerationNotMatch?: number | string; + ifMetagenerationMatch?: number | string; + ifMetagenerationNotMatch?: number | string; +} & object; +export interface DeleteCallback { + (err: Error | null, apiResponse?: r.Response): void; +} + +export interface GetConfig { + /** + * Create the object if it doesn't already exist. + */ + autoCreate?: boolean; +} +export type GetOrCreateOptions = GetConfig & CreateOptions; +export type GetResponse = [T, r.Response]; + +export interface ResponseCallback { + (err?: Error | null, apiResponse?: r.Response): void; +} + +export type SetMetadataResponse = [K]; +export type SetMetadataOptions = object; + +export interface BaseMetadata { + id?: string; + kind?: string; + etag?: string; + selfLink?: string; + [key: string]: unknown; +} + +/** + * ServiceObject is a base class, meant to be inherited from by a "service + * object," like a BigQuery dataset or Storage bucket. + * + * Most of the time, these objects share common functionality; they can be + * created or deleted, and you can get or set their metadata. + * + * By inheriting from this class, a service object will be extended with these + * shared behaviors. Note that any method can be overridden when the service + * object requires specific behavior. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +class ServiceObject extends EventEmitter { + metadata: K; + baseUrl?: string; + parent: ServiceObjectParent; + id?: string; + private createMethod?: Function; + protected methods: Methods; + interceptors: Interceptor[]; + projectId?: string; + + /* + * @constructor + * @alias module:common/service-object + * + * @private + * + * @param {object} config - Configuration object. + * @param {string} config.baseUrl - The base URL to make API requests to. + * @param {string} config.createMethod - The method which creates this object. + * @param {string=} config.id - The identifier of the object. For example, the + * name of a Storage bucket or Pub/Sub topic. + * @param {object=} config.methods - A map of each method name that should be inherited. + * @param {object} config.methods[].reqOpts - Default request options for this + * particular method. A common use case is when `setMetadata` requires a + * `PUT` method to override the default `PATCH`. + * @param {object} config.parent - The parent service instance. For example, an + * instance of Storage if the object is Bucket. + */ + constructor(config: ServiceObjectConfig) { + super(); + this.metadata = {} as K; + this.baseUrl = config.baseUrl; + this.parent = config.parent; // Parent class. + this.id = config.id; // Name or ID (e.g. dataset ID, bucket name, etc). + this.createMethod = config.createMethod; + this.methods = config.methods || {}; + this.interceptors = []; + this.projectId = config.projectId; + + if (config.methods) { + // This filters the ServiceObject instance (e.g. a "File") to only have + // the configured methods. We make a couple of exceptions for core- + // functionality ("request()" and "getRequestInterceptors()") + Object.getOwnPropertyNames(ServiceObject.prototype) + .filter(methodName => { + return ( + // All ServiceObjects need `request` and `getRequestInterceptors`. + // clang-format off + !/^request/.test(methodName) && + !/^getRequestInterceptors/.test(methodName) && + // clang-format on + // The ServiceObject didn't redefine the method. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this as any)[methodName] === + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (ServiceObject.prototype as any)[methodName] && + // This method isn't wanted. + !config.methods![methodName] + ); + }) + .forEach(methodName => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this as any)[methodName] = undefined; + }); + } + } + + /** + * Create the object. + * + * @param {object=} options - Configuration object. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.instance - The instance. + * @param {object} callback.apiResponse - The full API response. + */ + create(options?: CreateOptions): Promise>; + create(options: CreateOptions, callback: CreateCallback): void; + create(callback: CreateCallback): void; + create( + optionsOrCallback?: CreateOptions | CreateCallback, + callback?: CreateCallback + ): void | Promise> { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + const args = [this.id] as Array<{}>; + + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback as CreateCallback; + } + + if (typeof optionsOrCallback === 'object') { + args.push(optionsOrCallback); + } + + // Wrap the callback to return *this* instance of the object, not the + // newly-created one. + // tslint: disable-next-line no-any + function onCreate(...args: [Error, ServiceObject]) { + const [err, instance] = args; + if (!err) { + self.metadata = instance.metadata; + if (self.id && instance.metadata) { + self.id = instance.metadata.id; + } + args[1] = self; // replace the created `instance` with this one. + } + callback!(...(args as {} as [Error, T])); + } + args.push(onCreate); + // eslint-disable-next-line prefer-spread + this.createMethod!.apply(null, args); + } + + /** + * Delete the object. + * + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.apiResponse - The full API response. + */ + delete(options?: DeleteOptions): Promise<[r.Response]>; + delete(options: DeleteOptions, callback: DeleteCallback): void; + delete(callback: DeleteCallback): void; + delete( + optionsOrCallback?: DeleteOptions | DeleteCallback, + cb?: DeleteCallback + ): Promise<[r.Response]> | void { + const [options, callback] = util.maybeOptionsOrCallback< + DeleteOptions, + DeleteCallback + >(optionsOrCallback, cb); + + const ignoreNotFound = options.ignoreNotFound!; + delete options.ignoreNotFound; + + const methodConfig = + (typeof this.methods.delete === 'object' && this.methods.delete) || {}; + + const reqOpts = { + method: 'DELETE', + uri: '', + ...methodConfig.reqOpts, + qs: { + ...methodConfig.reqOpts?.qs, + ...options, + }, + }; + + // The `request` method may have been overridden to hold any special + // behavior. Ensure we call the original `request` method. + ServiceObject.prototype.request.call( + this, + reqOpts, + (err: ApiError | null, body?: ResponseBody, res?: r.Response) => { + if (err) { + if (err.code === 404 && ignoreNotFound) { + err = null; + } + } + callback(err, res); + } + ); + } + + /** + * Check if the object exists. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {boolean} callback.exists - Whether the object exists or not. + */ + exists(options?: ExistsOptions): Promise<[boolean]>; + exists(options: ExistsOptions, callback: ExistsCallback): void; + exists(callback: ExistsCallback): void; + exists( + optionsOrCallback?: ExistsOptions | ExistsCallback, + cb?: ExistsCallback + ): void | Promise<[boolean]> { + const [options, callback] = util.maybeOptionsOrCallback< + ExistsOptions, + ExistsCallback + >(optionsOrCallback, cb); + + this.get(options, err => { + if (err) { + if (err.code === 404) { + callback!(null, false); + } else { + callback!(err); + } + return; + } + callback!(null, true); + }); + } + + /** + * Get the object if it exists. Optionally have the object created if an + * options object is provided with `autoCreate: true`. + * + * @param {object=} options - The configuration object that will be used to + * create the object if necessary. + * @param {boolean} options.autoCreate - Create the object if it doesn't already exist. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.instance - The instance. + * @param {object} callback.apiResponse - The full API response. + */ + get(options?: GetOrCreateOptions): Promise>; + get(callback: InstanceResponseCallback): void; + get(options: GetOrCreateOptions, callback: InstanceResponseCallback): void; + get( + optionsOrCallback?: GetOrCreateOptions | InstanceResponseCallback, + cb?: InstanceResponseCallback + ): Promise> | void { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + + const [opts, callback] = util.maybeOptionsOrCallback< + GetOrCreateOptions, + InstanceResponseCallback + >(optionsOrCallback, cb); + const options = Object.assign({}, opts); + + const autoCreate = options.autoCreate && typeof this.create === 'function'; + delete options.autoCreate; + + function onCreate( + err: ApiError | null, + instance: T, + apiResponse: r.Response + ) { + if (err) { + if (err.code === 409) { + self.get(options, callback!); + return; + } + callback!(err, null, apiResponse); + return; + } + callback!(null, instance, apiResponse); + } + + this.getMetadata(options, (err: ApiError | null, metadata) => { + if (err) { + if (err.code === 404 && autoCreate) { + const args: Array = []; + if (Object.keys(options).length > 0) { + args.push(options); + } + args.push(onCreate); + self.create(...args); + return; + } + callback!(err, null, metadata as unknown as r.Response); + return; + } + callback!(null, self as {} as T, metadata as unknown as r.Response); + }); + } + + /** + * Get the metadata of this object. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.metadata - The metadata for this object. + * @param {object} callback.apiResponse - The full API response. + */ + getMetadata(options?: GetMetadataOptions): Promise>; + getMetadata(options: GetMetadataOptions, callback: MetadataCallback): void; + getMetadata(callback: MetadataCallback): void; + getMetadata( + optionsOrCallback: GetMetadataOptions | MetadataCallback, + cb?: MetadataCallback + ): Promise> | void { + const [options, callback] = util.maybeOptionsOrCallback< + GetMetadataOptions, + MetadataCallback + >(optionsOrCallback, cb); + + const methodConfig = + (typeof this.methods.getMetadata === 'object' && + this.methods.getMetadata) || + {}; + const reqOpts = { + uri: '', + ...methodConfig.reqOpts, + qs: { + ...methodConfig.reqOpts?.qs, + ...options, + }, + }; + + // The `request` method may have been overridden to hold any special + // behavior. Ensure we call the original `request` method. + ServiceObject.prototype.request.call( + this, + reqOpts, + (err: Error | null, body?: ResponseBody, res?: r.Response) => { + this.metadata = body; + callback!(err, this.metadata, res); + } + ); + } + + /** + * Return the user's custom request interceptors. + */ + getRequestInterceptors(): Function[] { + // Interceptors should be returned in the order they were assigned. + const localInterceptors = this.interceptors + .filter(interceptor => typeof interceptor.request === 'function') + .map(interceptor => interceptor.request); + return this.parent.getRequestInterceptors().concat(localInterceptors); + } + + /** + * Set the metadata for this object. + * + * @param {object} metadata - The metadata to set on this object. + * @param {object=} options - Configuration options. + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.apiResponse - The full API response. + */ + setMetadata( + metadata: K, + options?: SetMetadataOptions + ): Promise>; + setMetadata(metadata: K, callback: MetadataCallback): void; + setMetadata( + metadata: K, + options: SetMetadataOptions, + callback: MetadataCallback + ): void; + setMetadata( + metadata: K, + optionsOrCallback: SetMetadataOptions | MetadataCallback, + cb?: MetadataCallback + ): Promise> | void { + const [options, callback] = util.maybeOptionsOrCallback< + SetMetadataOptions, + MetadataCallback + >(optionsOrCallback, cb); + const methodConfig = + (typeof this.methods.setMetadata === 'object' && + this.methods.setMetadata) || + {}; + + const reqOpts = { + method: 'PATCH', + uri: '', + ...methodConfig.reqOpts, + json: { + ...methodConfig.reqOpts?.json, + ...metadata, + }, + qs: { + ...methodConfig.reqOpts?.qs, + ...options, + }, + }; + + // The `request` method may have been overridden to hold any special + // behavior. Ensure we call the original `request` method. + ServiceObject.prototype.request.call( + this, + reqOpts, + (err: Error | null, body?: ResponseBody, res?: r.Response) => { + this.metadata = body; + callback!(err, this.metadata, res); + } + ); + } + + /** + * Make an authenticated API request. + * + * @private + * + * @param {object} reqOpts - Request options that are passed to `request`. + * @param {string} reqOpts.uri - A URI relative to the baseUrl. + * @param {function} callback - The callback function passed to `request`. + */ + private request_(reqOpts: StreamRequestOptions): r.Request; + private request_( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ): void; + private request_( + reqOpts: DecorateRequestOptions | StreamRequestOptions, + callback?: BodyResponseCallback + ): void | r.Request { + reqOpts = {...reqOpts}; + + if (this.projectId) { + reqOpts.projectId = this.projectId; + } + + const isAbsoluteUrl = reqOpts.uri.indexOf('http') === 0; + const uriComponents = [this.baseUrl, this.id || '', reqOpts.uri]; + + if (isAbsoluteUrl) { + uriComponents.splice(0, uriComponents.indexOf(reqOpts.uri)); + } + + reqOpts.uri = uriComponents + .filter(x => x!.trim()) // Limit to non-empty strings. + .map(uriComponent => { + const trimSlashesRegex = /^\/*|\/*$/g; + return uriComponent!.replace(trimSlashesRegex, ''); + }) + .join('/'); + + const childInterceptors = Array.isArray(reqOpts.interceptors_) + ? reqOpts.interceptors_ + : []; + const localInterceptors = [].slice.call(this.interceptors); + + reqOpts.interceptors_ = childInterceptors.concat(localInterceptors); + + if (reqOpts.shouldReturnStream) { + return this.parent.requestStream(reqOpts); + } + this.parent.request(reqOpts, callback!); + } + + /** + * Make an authenticated API request. + * + * @param {object} reqOpts - Request options that are passed to `request`. + * @param {string} reqOpts.uri - A URI relative to the baseUrl. + * @param {function} callback - The callback function passed to `request`. + */ + request(reqOpts: DecorateRequestOptions): Promise; + request( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ): void; + request( + reqOpts: DecorateRequestOptions, + callback?: BodyResponseCallback + ): void | Promise { + this.request_(reqOpts, callback!); + } + + /** + * Make an authenticated API request. + * + * @param {object} reqOpts - Request options that are passed to `request`. + * @param {string} reqOpts.uri - A URI relative to the baseUrl. + */ + requestStream(reqOpts: DecorateRequestOptions): r.Request { + const opts = {...reqOpts, shouldReturnStream: true}; + return this.request_(opts as StreamRequestOptions); + } +} + +promisifyAll(ServiceObject, {exclude: ['getRequestInterceptors']}); + +export {ServiceObject}; diff --git a/handwritten/storage/src/nodejs-common/service.ts b/handwritten/storage/src/nodejs-common/service.ts new file mode 100644 index 00000000000..9ade0478e04 --- /dev/null +++ b/handwritten/storage/src/nodejs-common/service.ts @@ -0,0 +1,316 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + AuthClient, + DEFAULT_UNIVERSE, + GoogleAuth, + GoogleAuthOptions, +} from 'google-auth-library'; +import * as r from 'teeny-request'; +import * as uuid from 'uuid'; + +import {Interceptor} from './service-object.js'; +import { + BodyResponseCallback, + DecorateRequestOptions, + GCCL_GCS_CMD_KEY, + MakeAuthenticatedRequest, + PackageJson, + util, +} from './util.js'; +import { + getRuntimeTrackingString, + getUserAgentString, + getModuleFormat, +} from '../util.js'; + +export const DEFAULT_PROJECT_ID_TOKEN = '{{projectId}}'; + +export interface StreamRequestOptions extends DecorateRequestOptions { + shouldReturnStream: true; +} + +export interface ServiceConfig { + /** + * The base URL to make API requests to. + */ + baseUrl: string; + + /** + * The API Endpoint to use when connecting to the service. + * Example: storage.googleapis.com + */ + apiEndpoint: string; + + /** + * The scopes required for the request. + */ + scopes: string[]; + + projectIdRequired?: boolean; + packageJson: PackageJson; + + /** + * Reuse an existing `AuthClient` or `GoogleAuth` client instead of creating a new one. + */ + authClient?: AuthClient | GoogleAuth; + + /** + * Set to true if the endpoint is a custom URL + */ + customEndpoint?: boolean; + + /** + * Controls whether or not to use authentication when using a custom endpoint. + */ + useAuthWithCustomEndpoint?: boolean; +} + +export interface ServiceOptions extends Omit { + authClient?: AuthClient | GoogleAuth; + interceptors_?: Interceptor[]; + email?: string; + token?: string; + timeout?: number; // http.request.options.timeout + userAgent?: string; + useAuthWithCustomEndpoint?: boolean; +} + +export class Service { + baseUrl: string; + private globalInterceptors: Interceptor[]; + interceptors: Interceptor[]; + private packageJson: PackageJson; + projectId: string; + private projectIdRequired: boolean; + providedUserAgent?: string; + makeAuthenticatedRequest: MakeAuthenticatedRequest; + authClient: GoogleAuth; + apiEndpoint: string; + timeout?: number; + universeDomain: string; + customEndpoint: boolean; + useAuthWithCustomEndpoint?: boolean; + + /** + * Service is a base class, meant to be inherited from by a "service," like + * BigQuery or Storage. + * + * This handles making authenticated requests by exposing a `makeReq_` + * function. + * + * @constructor + * @alias module:common/service + * + * @param {object} config - Configuration object. + * @param {string} config.baseUrl - The base URL to make API requests to. + * @param {string[]} config.scopes - The scopes required for the request. + * @param {object=} options - [Configuration object](#/docs). + */ + constructor(config: ServiceConfig, options: ServiceOptions = {}) { + this.baseUrl = config.baseUrl; + this.apiEndpoint = config.apiEndpoint; + this.timeout = options.timeout; + this.globalInterceptors = Array.isArray(options.interceptors_) + ? options.interceptors_ + : []; + this.interceptors = []; + this.packageJson = config.packageJson; + this.projectId = options.projectId || DEFAULT_PROJECT_ID_TOKEN; + this.projectIdRequired = config.projectIdRequired !== false; + this.providedUserAgent = options.userAgent; + this.universeDomain = options.universeDomain || DEFAULT_UNIVERSE; + this.customEndpoint = config.customEndpoint || false; + this.useAuthWithCustomEndpoint = config.useAuthWithCustomEndpoint; + + this.makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory({ + ...config, + projectIdRequired: this.projectIdRequired, + projectId: this.projectId, + authClient: options.authClient || config.authClient, + credentials: options.credentials, + keyFile: options.keyFilename, + email: options.email, + clientOptions: { + universeDomain: options.universeDomain, + ...options.clientOptions, + }, + }); + this.authClient = this.makeAuthenticatedRequest.authClient; + + const isCloudFunctionEnv = !!process.env.FUNCTION_NAME; + + if (isCloudFunctionEnv) { + this.interceptors.push({ + request(reqOpts: DecorateRequestOptions) { + reqOpts.forever = false; + return reqOpts; + }, + }); + } + } + + /** + * Return the user's custom request interceptors. + */ + getRequestInterceptors(): Function[] { + // Interceptors should be returned in the order they were assigned. + return ([] as Interceptor[]).slice + .call(this.globalInterceptors) + .concat(this.interceptors) + .filter(interceptor => typeof interceptor.request === 'function') + .map(interceptor => interceptor.request); + } + + /** + * Get and update the Service's project ID. + * + * @param {function} callback - The callback function. + */ + getProjectId(): Promise; + getProjectId(callback: (err: Error | null, projectId?: string) => void): void; + getProjectId( + callback?: (err: Error | null, projectId?: string) => void + ): Promise | void { + if (!callback) { + return this.getProjectIdAsync(); + } + this.getProjectIdAsync().then(p => callback(null, p), callback); + } + + protected async getProjectIdAsync(): Promise { + const projectId = await this.authClient.getProjectId(); + if (this.projectId === DEFAULT_PROJECT_ID_TOKEN && projectId) { + this.projectId = projectId; + } + return this.projectId; + } + + /** + * Make an authenticated API request. + * + * @private + * + * @param {object} reqOpts - Request options that are passed to `request`. + * @param {string} reqOpts.uri - A URI relative to the baseUrl. + * @param {function} callback - The callback function passed to `request`. + */ + private request_(reqOpts: StreamRequestOptions): r.Request; + private request_( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ): void; + private request_( + reqOpts: DecorateRequestOptions | StreamRequestOptions, + callback?: BodyResponseCallback + ): void | r.Request { + reqOpts = {...reqOpts, timeout: this.timeout}; + const isAbsoluteUrl = reqOpts.uri.indexOf('http') === 0; + const uriComponents = [this.baseUrl]; + + if (this.projectIdRequired) { + if (reqOpts.projectId) { + uriComponents.push('projects'); + uriComponents.push(reqOpts.projectId); + } else { + uriComponents.push('projects'); + uriComponents.push(this.projectId); + } + } + + uriComponents.push(reqOpts.uri); + + if (isAbsoluteUrl) { + uriComponents.splice(0, uriComponents.indexOf(reqOpts.uri)); + } + + reqOpts.uri = uriComponents + .map(uriComponent => { + const trimSlashesRegex = /^\/*|\/*$/g; + return uriComponent.replace(trimSlashesRegex, ''); + }) + .join('/') + // Some URIs have colon separators. + // Bad: https://.../projects/:list + // Good: https://.../projects:list + .replace(/\/:/g, ':'); + + const requestInterceptors = this.getRequestInterceptors(); + const interceptorArray = Array.isArray(reqOpts.interceptors_) + ? reqOpts.interceptors_ + : []; + interceptorArray.forEach(interceptor => { + if (typeof interceptor.request === 'function') { + requestInterceptors.push(interceptor.request); + } + }); + + requestInterceptors.forEach(requestInterceptor => { + reqOpts = requestInterceptor(reqOpts); + }); + + delete reqOpts.interceptors_; + + const pkg = this.packageJson; + let userAgent = getUserAgentString(); + if (this.providedUserAgent) { + userAgent = `${this.providedUserAgent} ${userAgent}`; + } + reqOpts.headers = { + ...reqOpts.headers, + 'User-Agent': userAgent, + 'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${ + pkg.version + }-${getModuleFormat()} gccl-invocation-id/${uuid.v4()}`, + }; + + if (reqOpts[GCCL_GCS_CMD_KEY]) { + reqOpts.headers['x-goog-api-client'] += + ` gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`; + } + + if (reqOpts.shouldReturnStream) { + return this.makeAuthenticatedRequest(reqOpts) as {} as r.Request; + } else { + this.makeAuthenticatedRequest(reqOpts, callback); + } + } + + /** + * Make an authenticated API request. + * + * @param {object} reqOpts - Request options that are passed to `request`. + * @param {string} reqOpts.uri - A URI relative to the baseUrl. + * @param {function} callback - The callback function passed to `request`. + */ + request( + reqOpts: DecorateRequestOptions, + callback: BodyResponseCallback + ): void { + Service.prototype.request_.call(this, reqOpts, callback); + } + + /** + * Make an authenticated API request. + * + * @param {object} reqOpts - Request options that are passed to `request`. + * @param {string} reqOpts.uri - A URI relative to the baseUrl. + */ + requestStream(reqOpts: DecorateRequestOptions): r.Request { + const opts = {...reqOpts, shouldReturnStream: true}; + return (Service.prototype.request_ as Function).call(this, opts); + } +} diff --git a/handwritten/storage/src/nodejs-common/util.ts b/handwritten/storage/src/nodejs-common/util.ts new file mode 100644 index 00000000000..aec0b2f5064 --- /dev/null +++ b/handwritten/storage/src/nodejs-common/util.ts @@ -0,0 +1,1056 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module common/util + */ + +import { + replaceProjectIdToken, + MissingProjectIdError, +} from '@google-cloud/projectify'; +import * as htmlEntities from 'html-entities'; +import {AuthClient, GoogleAuth, GoogleAuthOptions} from 'google-auth-library'; +import {CredentialBody} from 'google-auth-library'; +import * as r from 'teeny-request'; +import retryRequest from 'retry-request'; +import {Duplex, DuplexOptions, Readable, Transform, Writable} from 'stream'; +import {teenyRequest} from 'teeny-request'; +import {Interceptor} from './service-object.js'; +import * as uuid from 'uuid'; +import {DEFAULT_PROJECT_ID_TOKEN} from './service.js'; +import { + getModuleFormat, + getRuntimeTrackingString, + getUserAgentString, +} from '../util.js'; +import duplexify from 'duplexify'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import {getPackageJSON} from '../package-json-helper.cjs'; + +const packageJson = getPackageJSON(); + +/** + * A unique symbol for providing a `gccl-gcs-cmd` value + * for the `X-Goog-API-Client` header. + * + * E.g. the `V` in `X-Goog-API-Client: gccl-gcs-cmd/V` + **/ +export const GCCL_GCS_CMD_KEY = Symbol.for('GCCL_GCS_CMD'); + +const requestDefaults: r.CoreOptions = { + timeout: 60000, + gzip: true, + forever: true, + pool: { + maxSockets: Infinity, + }, +}; + +/** + * Default behavior: Automatically retry retriable server errors. + * + * @const {boolean} + * @private + */ +const AUTO_RETRY_DEFAULT = true; + +/** + * Default behavior: Only attempt to retry retriable errors 3 times. + * + * @const {number} + * @private + */ +const MAX_RETRY_DEFAULT = 3; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type ResponseBody = any; + +// Directly copy over Duplexify interfaces +export interface DuplexifyOptions extends DuplexOptions { + autoDestroy?: boolean; + end?: boolean; +} + +export interface Duplexify extends Duplex { + readonly destroyed: boolean; + setWritable(writable: Writable | false | null): void; + setReadable(readable: Readable | false | null): void; +} + +export interface DuplexifyConstructor { + obj( + writable?: Writable | false | null, + readable?: Readable | false | null, + options?: DuplexifyOptions + ): Duplexify; + new ( + writable?: Writable | false | null, + readable?: Readable | false | null, + options?: DuplexifyOptions + ): Duplexify; + ( + writable?: Writable | false | null, + readable?: Readable | false | null, + options?: DuplexifyOptions + ): Duplexify; +} + +export interface ParsedHttpRespMessage { + resp: r.Response; + err?: ApiError; +} + +export interface MakeAuthenticatedRequest { + (reqOpts: DecorateRequestOptions): Duplexify; + ( + reqOpts: DecorateRequestOptions, + options?: MakeAuthenticatedRequestOptions + ): void | Abortable; + ( + reqOpts: DecorateRequestOptions, + callback?: BodyResponseCallback + ): void | Abortable; + ( + reqOpts: DecorateRequestOptions, + optionsOrCallback?: MakeAuthenticatedRequestOptions | BodyResponseCallback + ): void | Abortable | Duplexify; + getCredentials: ( + callback: (err?: Error | null, credentials?: CredentialBody) => void + ) => void; + authClient: GoogleAuth; +} + +export interface Abortable { + abort(): void; +} +export type AbortableDuplex = Duplexify & Abortable; + +export interface PackageJson { + name: string; + version: string; +} + +export interface MakeAuthenticatedRequestFactoryConfig + extends Omit { + /** + * Automatically retry requests if the response is related to rate limits or + * certain intermittent server errors. We will exponentially backoff + * subsequent requests by default. (default: true) + */ + autoRetry?: boolean; + + /** + * If true, just return the provided request options. Default: false. + */ + customEndpoint?: boolean; + + /** + * If true, will authenticate when using a custom endpoint. Default: false. + */ + useAuthWithCustomEndpoint?: boolean; + + /** + * Account email address, required for PEM/P12 usage. + */ + email?: string; + + /** + * Maximum number of automatic retries attempted before returning the error. + * (default: 3) + */ + maxRetries?: number; + + stream?: Duplexify; + + /** + * A pre-instantiated `AuthClient` or `GoogleAuth` client that should be used. + * A new client will be created if this is not set. + */ + authClient?: AuthClient | GoogleAuth; + + /** + * Determines if a projectId is required for authenticated requests. Defaults to `true`. + */ + projectIdRequired?: boolean; +} + +export interface MakeAuthenticatedRequestOptions { + onAuthenticated: OnAuthenticatedCallback; +} + +export interface OnAuthenticatedCallback { + (err: Error | null, reqOpts?: DecorateRequestOptions): void; +} + +export interface GoogleErrorBody { + code: number; + errors?: GoogleInnerError[]; + response: r.Response; + message?: string; +} + +export interface GoogleInnerError { + reason?: string; + message?: string; +} + +export interface MakeWritableStreamOptions { + /** + * A connection instance used to get a token with and send the request + * through. + */ + connection?: {}; + + /** + * Metadata to send at the head of the request. + */ + metadata?: {contentType?: string}; + + /** + * Request object, in the format of a standard Node.js http.request() object. + */ + request?: r.Options; + + makeAuthenticatedRequest( + reqOpts: r.OptionsWithUri & { + [GCCL_GCS_CMD_KEY]?: string; + }, + fnobj: { + onAuthenticated( + err: Error | null, + authenticatedReqOpts?: r.Options + ): void; + } + ): void; +} + +export interface DecorateRequestOptions extends r.CoreOptions { + autoPaginate?: boolean; + autoPaginateVal?: boolean; + objectMode?: boolean; + maxRetries?: number; + uri: string; + interceptors_?: Interceptor[]; + shouldReturnStream?: boolean; + projectId?: string; + [GCCL_GCS_CMD_KEY]?: string; +} + +export interface ParsedHttpResponseBody { + body: ResponseBody; + err?: Error; +} + +/** + * Custom error type for API errors. + * + * @param {object} errorBody - Error object. + */ +export class ApiError extends Error { + code?: number; + errors?: GoogleInnerError[]; + response?: r.Response; + constructor(errorMessage: string); + constructor(errorBody: GoogleErrorBody); + constructor(errorBodyOrMessage?: GoogleErrorBody | string) { + super(); + if (typeof errorBodyOrMessage !== 'object') { + this.message = errorBodyOrMessage || ''; + return; + } + const errorBody = errorBodyOrMessage; + + this.code = errorBody.code; + this.errors = errorBody.errors; + this.response = errorBody.response; + + try { + this.errors = JSON.parse(this.response.body).error.errors; + } catch (e) { + this.errors = errorBody.errors; + } + + this.message = ApiError.createMultiErrorMessage(errorBody, this.errors); + Error.captureStackTrace(this); + } + /** + * Pieces together an error message by combining all unique error messages + * returned from a single GoogleError + * + * @private + * + * @param {GoogleErrorBody} err The original error. + * @param {GoogleInnerError[]} [errors] Inner errors, if any. + * @returns {string} + */ + static createMultiErrorMessage( + err: GoogleErrorBody, + errors?: GoogleInnerError[] + ): string { + const messages: Set = new Set(); + + if (err.message) { + messages.add(err.message); + } + + if (errors && errors.length) { + errors.forEach(({message}) => messages.add(message!)); + } else if (err.response && err.response.body) { + messages.add(htmlEntities.decode(err.response.body.toString())); + } else if (!err.message) { + messages.add('A failure occurred during this request.'); + } + + let messageArr: string[] = Array.from(messages); + + if (messageArr.length > 1) { + messageArr = messageArr.map((message, i) => ` ${i + 1}. ${message}`); + messageArr.unshift( + 'Multiple errors occurred during the request. Please see the `errors` array for complete details.\n' + ); + messageArr.push('\n'); + } + + return messageArr.join('\n'); + } +} + +/** + * Custom error type for partial errors returned from the API. + * + * @param {object} b - Error object. + */ +export class PartialFailureError extends Error { + errors?: GoogleInnerError[]; + response?: r.Response; + constructor(b: GoogleErrorBody) { + super(); + const errorObject = b; + + this.errors = errorObject.errors; + this.name = 'PartialFailureError'; + this.response = errorObject.response; + + this.message = ApiError.createMultiErrorMessage(errorObject, this.errors); + } +} + +export interface BodyResponseCallback { + (err: Error | ApiError | null, body?: ResponseBody, res?: r.Response): void; +} + +export interface RetryOptions { + retryDelayMultiplier?: number; + totalTimeout?: number; + maxRetryDelay?: number; + autoRetry?: boolean; + maxRetries?: number; + retryableErrorFn?: (err: ApiError) => boolean; +} + +export interface MakeRequestConfig { + /** + * Automatically retry requests if the response is related to rate limits or + * certain intermittent server errors. We will exponentially backoff + * subsequent requests by default. (default: true) + */ + autoRetry?: boolean; + + /** + * Maximum number of automatic retries attempted before returning the error. + * (default: 3) + */ + maxRetries?: number; + + retries?: number; + + retryOptions?: RetryOptions; + + stream?: Duplexify; + + shouldRetryFn?: (response?: r.Response) => boolean; +} + +export class Util { + ApiError = ApiError; + PartialFailureError = PartialFailureError; + + /** + * No op. + * + * @example + * function doSomething(callback) { + * callback = callback || noop; + * } + */ + noop() {} + + /** + * Uniformly process an API response. + * + * @param {*} err - Error value. + * @param {*} resp - Response value. + * @param {*} body - Body value. + * @param {function} callback - The callback function. + */ + handleResp( + err: Error | null, + resp?: r.Response | null, + body?: ResponseBody, + callback?: BodyResponseCallback + ) { + callback = callback || util.noop; + + const parsedResp = { + err: err || null, + ...(resp && util.parseHttpRespMessage(resp)), + ...(body && util.parseHttpRespBody(body)), + }; + + // Assign the parsed body to resp.body, even if { json: false } was passed + // as a request option. + // We assume that nobody uses the previously unparsed value of resp.body. + if (!parsedResp.err && resp && typeof parsedResp.body === 'object') { + parsedResp.resp.body = parsedResp.body; + } + + if (parsedResp.err && resp) { + parsedResp.err.response = resp; + } + + callback(parsedResp.err, parsedResp.body, parsedResp.resp); + } + + /** + * Sniff an incoming HTTP response message for errors. + * + * @param {object} httpRespMessage - An incoming HTTP response message from `request`. + * @return {object} parsedHttpRespMessage - The parsed response. + * @param {?error} parsedHttpRespMessage.err - An error detected. + * @param {object} parsedHttpRespMessage.resp - The original response object. + */ + parseHttpRespMessage(httpRespMessage: r.Response) { + const parsedHttpRespMessage = { + resp: httpRespMessage, + } as ParsedHttpRespMessage; + + if (httpRespMessage.statusCode < 200 || httpRespMessage.statusCode > 299) { + // Unknown error. Format according to ApiError standard. + parsedHttpRespMessage.err = new ApiError({ + errors: new Array(), + code: httpRespMessage.statusCode, + message: httpRespMessage.statusMessage, + response: httpRespMessage, + }); + } + + return parsedHttpRespMessage; + } + + /** + * Parse the response body from an HTTP request. + * + * @param {object} body - The response body. + * @return {object} parsedHttpRespMessage - The parsed response. + * @param {?error} parsedHttpRespMessage.err - An error detected. + * @param {object} parsedHttpRespMessage.body - The original body value provided + * will try to be JSON.parse'd. If it's successful, the parsed value will + * be returned here, otherwise the original value and an error will be returned. + */ + parseHttpRespBody(body: ResponseBody) { + const parsedHttpRespBody: ParsedHttpResponseBody = { + body, + }; + + if (typeof body === 'string') { + try { + parsedHttpRespBody.body = JSON.parse(body); + } catch (err) { + parsedHttpRespBody.body = body; + } + } + + if (parsedHttpRespBody.body && parsedHttpRespBody.body.error) { + // Error from JSON API. + parsedHttpRespBody.err = new ApiError(parsedHttpRespBody.body.error); + } + + return parsedHttpRespBody; + } + + /** + * Take a Duplexify stream, fetch an authenticated connection header, and + * create an outgoing writable stream. + * + * @param {Duplexify} dup - Duplexify stream. + * @param {object} options - Configuration object. + * @param {module:common/connection} options.connection - A connection instance used to get a token with and send the request through. + * @param {object} options.metadata - Metadata to send at the head of the request. + * @param {object} options.request - Request object, in the format of a standard Node.js http.request() object. + * @param {string=} options.request.method - Default: "POST". + * @param {string=} options.request.qs.uploadType - Default: "multipart". + * @param {string=} options.streamContentType - Default: "application/octet-stream". + * @param {function} onComplete - Callback, executed after the writable Request stream has completed. + */ + makeWritableStream( + dup: Duplexify, + options: MakeWritableStreamOptions, + onComplete?: Function + ) { + onComplete = onComplete || util.noop; + + const writeStream = new ProgressStream(); + writeStream.on('progress', evt => dup.emit('progress', evt)); + dup.setWritable(writeStream); + + const defaultReqOpts = { + method: 'POST', + qs: { + uploadType: 'multipart', + }, + timeout: 0, + maxRetries: 0, + }; + + const metadata = options.metadata || {}; + + const reqOpts = { + ...defaultReqOpts, + ...options.request, + qs: { + ...defaultReqOpts.qs, + ...options.request?.qs, + }, + multipart: [ + { + 'Content-Type': 'application/json', + body: JSON.stringify(metadata), + }, + { + 'Content-Type': metadata.contentType || 'application/octet-stream', + body: writeStream, + }, + ], + } as {} as r.OptionsWithUri & { + [GCCL_GCS_CMD_KEY]?: string; + }; + + options.makeAuthenticatedRequest(reqOpts, { + onAuthenticated(err, authenticatedReqOpts) { + if (err) { + dup.destroy(err); + return; + } + + requestDefaults.headers = util._getDefaultHeaders( + reqOpts[GCCL_GCS_CMD_KEY] + ); + const request = teenyRequest.defaults(requestDefaults); + request(authenticatedReqOpts!, (err, resp, body) => { + util.handleResp(err, resp, body, (err, data) => { + if (err) { + dup.destroy(err); + return; + } + dup.emit('response', resp); + onComplete!(data); + }); + }); + }, + }); + } + + /** + * Returns true if the API request should be retried, given the error that was + * given the first time the request was attempted. This is used for rate limit + * related errors as well as intermittent server errors. + * + * @param {error} err - The API error to check if it is appropriate to retry. + * @return {boolean} True if the API request should be retried, false otherwise. + */ + shouldRetryRequest(err?: ApiError) { + if (err) { + if ([408, 429, 500, 502, 503, 504].indexOf(err.code!) !== -1) { + return true; + } + + if (err.errors) { + for (const e of err.errors) { + const reason = e.reason; + if (reason === 'rateLimitExceeded') { + return true; + } + if (reason === 'userRateLimitExceeded') { + return true; + } + if (reason && reason.includes('EAI_AGAIN')) { + return true; + } + } + } + } + + return false; + } + + /** + * Get a function for making authenticated requests. + * + * @param {object} config - Configuration object. + * @param {boolean=} config.autoRetry - Automatically retry requests if the + * response is related to rate limits or certain intermittent server + * errors. We will exponentially backoff subsequent requests by default. + * (default: true) + * @param {object=} config.credentials - Credentials object. + * @param {boolean=} config.customEndpoint - If true, just return the provided request options. Default: false. + * @param {boolean=} config.useAuthWithCustomEndpoint - If true, will authenticate when using a custom endpoint. Default: false. + * @param {string=} config.email - Account email address, required for PEM/P12 usage. + * @param {number=} config.maxRetries - Maximum number of automatic retries attempted before returning the error. (default: 3) + * @param {string=} config.keyFile - Path to a .json, .pem, or .p12 keyfile. + * @param {array} config.scopes - Array of scopes required for the API. + */ + makeAuthenticatedRequestFactory( + config: MakeAuthenticatedRequestFactoryConfig + ) { + const googleAutoAuthConfig = {...config}; + if (googleAutoAuthConfig.projectId === DEFAULT_PROJECT_ID_TOKEN) { + delete googleAutoAuthConfig.projectId; + } + + let authClient: GoogleAuth; + + if (googleAutoAuthConfig.authClient instanceof GoogleAuth) { + // Use an existing `GoogleAuth` + authClient = googleAutoAuthConfig.authClient; + } else { + // Pass an `AuthClient` & `clientOptions` to `GoogleAuth`, if available + authClient = new GoogleAuth({ + ...googleAutoAuthConfig, + authClient: googleAutoAuthConfig.authClient, + clientOptions: googleAutoAuthConfig.clientOptions, + }); + } + + /** + * The returned function that will make an authenticated request. + * + * @param {type} reqOpts - Request options in the format `request` expects. + * @param {object|function} options - Configuration object or callback function. + * @param {function=} options.onAuthenticated - If provided, a request will + * not be made. Instead, this function is passed the error & + * authenticated request options. + */ + function makeAuthenticatedRequest( + reqOpts: DecorateRequestOptions + ): Duplexify; + function makeAuthenticatedRequest( + reqOpts: DecorateRequestOptions, + options?: MakeAuthenticatedRequestOptions + ): void | Abortable; + function makeAuthenticatedRequest( + reqOpts: DecorateRequestOptions, + callback?: BodyResponseCallback + ): void | Abortable; + function makeAuthenticatedRequest( + reqOpts: DecorateRequestOptions, + optionsOrCallback?: MakeAuthenticatedRequestOptions | BodyResponseCallback + ): void | Abortable | Duplexify { + let stream: Duplexify; + let projectId: string; + const reqConfig = {...config}; + let activeRequest_: void | Abortable | null; + + if (!optionsOrCallback) { + stream = duplexify(); + reqConfig.stream = stream; + } + + const options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : undefined; + const callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : undefined; + + async function setProjectId() { + projectId = await authClient.getProjectId(); + } + + const onAuthenticated = async ( + err: Error | null, + authenticatedReqOpts?: DecorateRequestOptions + ) => { + const authLibraryError = err; + const autoAuthFailed = + err && + typeof err.message === 'string' && + err.message.indexOf('Could not load the default credentials') > -1; + + if (autoAuthFailed) { + // Even though authentication failed, the API might not actually + // care. + authenticatedReqOpts = reqOpts; + } + + if (!err || autoAuthFailed) { + try { + // Try with existing `projectId` value + authenticatedReqOpts = util.decorateRequest( + authenticatedReqOpts!, + projectId + ); + + err = null; + } catch (e) { + if (e instanceof MissingProjectIdError) { + // A `projectId` was required, but we don't have one. + try { + // Attempt to get the `projectId` + await setProjectId(); + + authenticatedReqOpts = util.decorateRequest( + authenticatedReqOpts!, + projectId + ); + + err = null; + } catch (e) { + // Re-use the "Could not load the default credentials error" if + // auto auth failed. + err = err || (e as Error); + } + } else { + // Some other error unrelated to missing `projectId` + err = err || (e as Error); + } + } + } + + if (err) { + if (stream) { + stream.destroy(err); + } else { + const fn = + options && options.onAuthenticated + ? options.onAuthenticated + : callback; + (fn as Function)(err); + } + return; + } + + if (options && options.onAuthenticated) { + options.onAuthenticated(null, authenticatedReqOpts); + } else { + activeRequest_ = util.makeRequest( + authenticatedReqOpts!, + reqConfig, + (apiResponseError, ...params) => { + if ( + apiResponseError && + (apiResponseError as ApiError).code === 401 && + authLibraryError + ) { + // Re-use the "Could not load the default credentials error" if + // the API request failed due to missing credentials. + apiResponseError = authLibraryError; + } + callback!(apiResponseError, ...params); + } + ); + } + }; + + const prepareRequest = async () => { + try { + const getProjectId = async () => { + if ( + config.projectId && + config.projectId !== DEFAULT_PROJECT_ID_TOKEN + ) { + // The user provided a project ID. We don't need to check with the + // auth client, it could be incorrect. + return config.projectId; + } + + if (config.projectIdRequired === false) { + // A projectId is not required. Return the default. + return DEFAULT_PROJECT_ID_TOKEN; + } + + return setProjectId(); + }; + + const authorizeRequest = async () => { + if ( + reqConfig.customEndpoint && + !reqConfig.useAuthWithCustomEndpoint + ) { + // Using a custom API override. Do not use `google-auth-library` for + // authentication. (ex: connecting to a local Datastore server) + return reqOpts; + } else { + return authClient.authorizeRequest(reqOpts); + } + }; + + const [_projectId, authorizedReqOpts] = await Promise.all([ + getProjectId(), + authorizeRequest(), + ]); + + if (_projectId) { + projectId = _projectId; + } + + return onAuthenticated( + null, + authorizedReqOpts as DecorateRequestOptions + ); + } catch (e) { + return onAuthenticated(e as Error); + } + }; + + prepareRequest(); + + if (stream!) { + return stream!; + } + + return { + abort() { + setImmediate(() => { + if (activeRequest_) { + activeRequest_.abort(); + activeRequest_ = null; + } + }); + }, + }; + } + const mar = makeAuthenticatedRequest as MakeAuthenticatedRequest; + mar.getCredentials = authClient.getCredentials.bind(authClient); + mar.authClient = authClient; + return mar; + } + + /** + * Make a request through the `retryRequest` module with built-in error + * handling and exponential back off. + * + * @param {object} reqOpts - Request options in the format `request` expects. + * @param {object=} config - Configuration object. + * @param {boolean=} config.autoRetry - Automatically retry requests if the + * response is related to rate limits or certain intermittent server + * errors. We will exponentially backoff subsequent requests by default. + * (default: true) + * @param {number=} config.maxRetries - Maximum number of automatic retries + * attempted before returning the error. (default: 3) + * @param {object=} config.request - HTTP module for request calls. + * @param {function} callback - The callback function. + */ + makeRequest( + reqOpts: DecorateRequestOptions, + config: MakeRequestConfig, + callback: BodyResponseCallback + ): void | Abortable { + let autoRetryValue = AUTO_RETRY_DEFAULT; + if (config.autoRetry !== undefined) { + autoRetryValue = config.autoRetry; + } else if (config.retryOptions?.autoRetry !== undefined) { + autoRetryValue = config.retryOptions.autoRetry; + } + + let maxRetryValue = MAX_RETRY_DEFAULT; + if (config.maxRetries !== undefined) { + maxRetryValue = config.maxRetries; + } else if (config.retryOptions?.maxRetries !== undefined) { + maxRetryValue = config.retryOptions.maxRetries; + } + + requestDefaults.headers = this._getDefaultHeaders( + reqOpts[GCCL_GCS_CMD_KEY] + ); + const options = { + request: teenyRequest.defaults(requestDefaults), + retries: autoRetryValue !== false ? maxRetryValue : 0, + noResponseRetries: autoRetryValue !== false ? maxRetryValue : 0, + shouldRetryFn(httpRespMessage: r.Response) { + const err = util.parseHttpRespMessage(httpRespMessage).err; + if (config.retryOptions?.retryableErrorFn) { + return err && config.retryOptions?.retryableErrorFn(err); + } + return err && util.shouldRetryRequest(err); + }, + maxRetryDelay: config.retryOptions?.maxRetryDelay, + retryDelayMultiplier: config.retryOptions?.retryDelayMultiplier, + totalTimeout: config.retryOptions?.totalTimeout, + } as {} as retryRequest.Options; + + if (typeof reqOpts.maxRetries === 'number') { + options.retries = reqOpts.maxRetries; + options.noResponseRetries = reqOpts.maxRetries; + } + + if (!config.stream) { + return retryRequest( + reqOpts, + options, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (err: Error | null, response: {}, body: any) => { + util.handleResp(err, response as {} as r.Response, body, callback!); + } + ); + } + const dup = config.stream as AbortableDuplex; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let requestStream: any; + const isGetRequest = (reqOpts.method || 'GET').toUpperCase() === 'GET'; + + if (isGetRequest) { + requestStream = retryRequest(reqOpts, options); + dup.setReadable(requestStream); + } else { + // Streaming writable HTTP requests cannot be retried. + requestStream = (options.request as unknown as Function)!(reqOpts); + dup.setWritable(requestStream); + } + + // Replay the Request events back to the stream. + requestStream + .on('error', dup.destroy.bind(dup)) + .on('response', dup.emit.bind(dup, 'response')) + .on('complete', dup.emit.bind(dup, 'complete')); + + dup.abort = requestStream.abort; + return dup; + } + + /** + * Decorate the options about to be made in a request. + * + * @param {object} reqOpts - The options to be passed to `request`. + * @param {string} projectId - The project ID. + * @return {object} reqOpts - The decorated reqOpts. + */ + decorateRequest(reqOpts: DecorateRequestOptions, projectId: string) { + delete reqOpts.autoPaginate; + delete reqOpts.autoPaginateVal; + delete reqOpts.objectMode; + + if (reqOpts.qs !== null && typeof reqOpts.qs === 'object') { + delete reqOpts.qs.autoPaginate; + delete reqOpts.qs.autoPaginateVal; + reqOpts.qs = replaceProjectIdToken(reqOpts.qs, projectId); + } + + if (Array.isArray(reqOpts.multipart)) { + reqOpts.multipart = (reqOpts.multipart as []).map(part => { + return replaceProjectIdToken(part, projectId); + }); + } + + if (reqOpts.json !== null && typeof reqOpts.json === 'object') { + delete reqOpts.json.autoPaginate; + delete reqOpts.json.autoPaginateVal; + reqOpts.json = replaceProjectIdToken(reqOpts.json, projectId); + } + + reqOpts.uri = replaceProjectIdToken(reqOpts.uri, projectId); + + return reqOpts; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + isCustomType(unknown: any, module: string) { + function getConstructorName(obj: Function) { + return obj.constructor && obj.constructor.name.toLowerCase(); + } + + const moduleNameParts = module.split('/'); + + const parentModuleName = + moduleNameParts[0] && moduleNameParts[0].toLowerCase(); + const subModuleName = + moduleNameParts[1] && moduleNameParts[1].toLowerCase(); + + if (subModuleName && getConstructorName(unknown) !== subModuleName) { + return false; + } + + let walkingModule = unknown; + // eslint-disable-next-line no-constant-condition + while (true) { + if (getConstructorName(walkingModule) === parentModuleName) { + return true; + } + walkingModule = walkingModule.parent; + if (!walkingModule) { + return false; + } + } + } + + /** + * Given two parameters, figure out if this is either: + * - Just a callback function + * - An options object, and then a callback function + * @param optionsOrCallback An options object or callback. + * @param cb A potentially undefined callback. + */ + maybeOptionsOrCallback void>( + optionsOrCallback?: T | C, + cb?: C + ): [T, C] { + return typeof optionsOrCallback === 'function' + ? [{} as T, optionsOrCallback as C] + : [optionsOrCallback as T, cb as C]; + } + + _getDefaultHeaders(gcclGcsCmd?: string) { + const headers = { + 'User-Agent': getUserAgentString(), + 'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${ + packageJson.version + }-${getModuleFormat()} gccl-invocation-id/${uuid.v4()}`, + }; + + if (gcclGcsCmd) { + headers['x-goog-api-client'] += ` gccl-gcs-cmd/${gcclGcsCmd}`; + } + + return headers; + } +} + +/** + * Basic Passthrough Stream that records the number of bytes read + * every time the cursor is moved. + */ +class ProgressStream extends Transform { + bytesRead = 0; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + _transform(chunk: any, encoding: string, callback: Function) { + this.bytesRead += chunk.length; + this.emit('progress', {bytesWritten: this.bytesRead, contentLength: '*'}); + this.push(chunk); + callback(); + } +} + +const util = new Util(); +export {util}; diff --git a/handwritten/storage/src/notification.ts b/handwritten/storage/src/notification.ts new file mode 100644 index 00000000000..6d63a899f2e --- /dev/null +++ b/handwritten/storage/src/notification.ts @@ -0,0 +1,362 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {BaseMetadata, ServiceObject} from './nodejs-common/index.js'; +import {ResponseBody} from './nodejs-common/util.js'; +import {promisifyAll} from '@google-cloud/promisify'; + +import {Bucket} from './bucket.js'; + +export interface DeleteNotificationOptions { + userProject?: string; +} + +export interface GetNotificationMetadataOptions { + userProject?: string; +} + +/** + * @typedef {array} GetNotificationMetadataResponse + * @property {object} 0 The notification metadata. + * @property {object} 1 The full API response. + */ +export type GetNotificationMetadataResponse = [ResponseBody, unknown]; + +/** + * @callback GetNotificationMetadataCallback + * @param {?Error} err Request error, if any. + * @param {object} files The notification metadata. + * @param {object} apiResponse The full API response. + */ +export interface GetNotificationMetadataCallback { + (err: Error | null, metadata?: ResponseBody, apiResponse?: unknown): void; +} + +/** + * @typedef {array} GetNotificationResponse + * @property {Notification} 0 The {@link Notification} + * @property {object} 1 The full API response. + */ +export type GetNotificationResponse = [Notification, unknown]; + +export interface GetNotificationOptions { + /** + * Automatically create the object if it does not exist. Default: `false`. + */ + autoCreate?: boolean; + + /** + * The ID of the project which will be billed for the request. + */ + userProject?: string; +} + +/** + * @callback GetNotificationCallback + * @param {?Error} err Request error, if any. + * @param {Notification} notification The {@link Notification}. + * @param {object} apiResponse The full API response. + */ +export interface GetNotificationCallback { + ( + err: Error | null, + notification?: Notification | null, + apiResponse?: unknown + ): void; +} + +/** + * @callback DeleteNotificationCallback + * @param {?Error} err Request error, if any. + * @param {object} apiResponse The full API response. + */ +export interface DeleteNotificationCallback { + (err: Error | null, apiResponse?: unknown): void; +} + +export interface NotificationMetadata extends BaseMetadata { + custom_attributes?: { + [key: string]: string; + }; + event_types?: string[]; + object_name_prefix?: string; + payload_format?: 'JSON_API_V1' | 'NONE'; + topic?: string; +} + +/** + * The API-formatted resource description of the notification. + * + * Note: This is not guaranteed to be up-to-date when accessed. To get the + * latest record, call the `getMetadata()` method. + * + * @name Notification#metadata + * @type {object} + */ +/** + * A Notification object is created from your {@link Bucket} object using + * {@link Bucket#notification}. Use it to interact with Cloud Pub/Sub + * notifications. + * + * See {@link https://cloud.google.com/storage/docs/pubsub-notifications| Cloud Pub/Sub Notifications for Google Cloud Storage} + * + * @class + * @hideconstructor + * + * @param {Bucket} bucket The bucket instance this notification is attached to. + * @param {string} id The ID of the notification. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * + * const notification = myBucket.notification('1'); + * ``` + */ +class Notification extends ServiceObject { + constructor(bucket: Bucket, id: string) { + const requestQueryObject: { + ifGenerationMatch?: number; + ifGenerationNotMatch?: number; + ifMetagenerationMatch?: number; + ifMetagenerationNotMatch?: number; + } = {}; + + const methods = { + /** + * Creates a notification subscription for the bucket. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/notifications/insert| Notifications: insert} + * @method Notification#create + * + * @param {Topic|string} topic The Cloud PubSub topic to which this + * subscription publishes. If the project ID is omitted, the current + * project ID will be used. + * + * Acceptable formats are: + * - `projects/grape-spaceship-123/topics/my-topic` + * + * - `my-topic` + * @param {CreateNotificationRequest} [options] Metadata to set for + * the notification. + * @param {CreateNotificationCallback} [callback] Callback function. + * @returns {Promise} + * @throws {Error} If a valid topic is not provided. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * const notification = myBucket.notification('1'); + * + * notification.create(function(err, notification, apiResponse) { + * if (!err) { + * // The notification was created successfully. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * notification.create().then(function(data) { + * const notification = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + create: true, + + /** + * @typedef {array} DeleteNotificationResponse + * @property {object} 0 The full API response. + */ + /** + * Permanently deletes a notification subscription. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/notifications/delete| Notifications: delete API Documentation} + * + * @param {object} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {DeleteNotificationCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * const notification = myBucket.notification('1'); + * + * notification.delete(function(err, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * notification.delete().then(function(data) { + * const apiResponse = data[0]; + * }); + * + * ``` + * @example include:samples/deleteNotification.js + * region_tag:storage_delete_bucket_notification + * Another example: + */ + delete: { + reqOpts: { + qs: requestQueryObject, + }, + }, + + /** + * Get a notification and its metadata if it exists. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/notifications/get| Notifications: get API Documentation} + * + * @param {object} [options] Configuration options. + * See {@link Bucket#createNotification} for create options. + * @param {boolean} [options.autoCreate] Automatically create the object if + * it does not exist. Default: `false`. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {GetNotificationCallback} [callback] Callback function. + * @return {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * const notification = myBucket.notification('1'); + * + * notification.get(function(err, notification, apiResponse) { + * // `notification.metadata` has been populated. + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * notification.get().then(function(data) { + * const notification = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ + get: { + reqOpts: { + qs: requestQueryObject, + }, + }, + + /** + * Get the notification's metadata. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/notifications/get| Notifications: get API Documentation} + * + * @param {object} [options] Configuration options. + * @param {string} [options.userProject] The ID of the project which will be + * billed for the request. + * @param {GetNotificationMetadataCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * const notification = myBucket.notification('1'); + * + * notification.getMetadata(function(err, metadata, apiResponse) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * notification.getMetadata().then(function(data) { + * const metadata = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/getMetadataNotifications.js + * region_tag:storage_print_pubsub_bucket_notification + * Another example: + */ + getMetadata: { + reqOpts: { + qs: requestQueryObject, + }, + }, + + /** + * @typedef {array} NotificationExistsResponse + * @property {boolean} 0 Whether the notification exists or not. + */ + /** + * @callback NotificationExistsCallback + * @param {?Error} err Request error, if any. + * @param {boolean} exists Whether the notification exists or not. + */ + /** + * Check if the notification exists. + * + * @method Notification#exists + * @param {NotificationExistsCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const myBucket = storage.bucket('my-bucket'); + * const notification = myBucket.notification('1'); + * + * notification.exists(function(err, exists) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * notification.exists().then(function(data) { + * const exists = data[0]; + * }); + * ``` + */ + exists: true, + }; + + super({ + parent: bucket, + baseUrl: '/notificationConfigs', + id: id.toString(), + createMethod: bucket.createNotification.bind(bucket), + methods, + }); + } +} + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +promisifyAll(Notification); + +/** + * Reference to the {@link Notification} class. + * @name module:@google-cloud/storage.Notification + * @see Notification + */ +export {Notification}; diff --git a/handwritten/storage/src/package-json-helper.cjs b/handwritten/storage/src/package-json-helper.cjs new file mode 100644 index 00000000000..794923b5f96 --- /dev/null +++ b/handwritten/storage/src/package-json-helper.cjs @@ -0,0 +1,21 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* eslint-disable node/no-missing-require */ + +function getPackageJSON() { + return require('../../../package.json'); +} + +exports.getPackageJSON = getPackageJSON; diff --git a/handwritten/storage/src/resumable-upload.ts b/handwritten/storage/src/resumable-upload.ts new file mode 100644 index 00000000000..07b5a28f191 --- /dev/null +++ b/handwritten/storage/src/resumable-upload.ts @@ -0,0 +1,1488 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import AbortController from 'abort-controller'; +import {createHash} from 'crypto'; +import { + GaxiosOptions, + GaxiosPromise, + GaxiosResponse, + GaxiosError, +} from 'gaxios'; +import * as gaxios from 'gaxios'; +import { + DEFAULT_UNIVERSE, + GoogleAuth, + GoogleAuthOptions, +} from 'google-auth-library'; +import {Readable, Writable, WritableOptions} from 'stream'; +import AsyncRetry from 'async-retry'; +import {RetryOptions, PreconditionOptions} from './storage.js'; +import * as uuid from 'uuid'; +import { + getRuntimeTrackingString, + getModuleFormat, + getUserAgentString, +} from './util.js'; +import {GCCL_GCS_CMD_KEY} from './nodejs-common/util.js'; +import {FileExceptionMessages, FileMetadata, RequestError} from './file.js'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import {getPackageJSON} from './package-json-helper.cjs'; +import {HashStreamValidator} from './hash-stream-validator.js'; + +const NOT_FOUND_STATUS_CODE = 404; +const RESUMABLE_INCOMPLETE_STATUS_CODE = 308; +const packageJson = getPackageJSON(); + +export const PROTOCOL_REGEX = /^(\w*):\/\//; + +export interface ErrorWithCode extends Error { + code: number; + status?: number | string; +} + +export type CreateUriCallback = (err: Error | null, uri?: string) => void; +export interface Encryption { + key: {}; + hash: {}; +} + +export type PredefinedAcl = + | 'authenticatedRead' + | 'bucketOwnerFullControl' + | 'bucketOwnerRead' + | 'private' + | 'projectPrivate' + | 'publicRead'; + +export interface QueryParameters extends PreconditionOptions { + contentEncoding?: string; + kmsKeyName?: string; + predefinedAcl?: PredefinedAcl; + projection?: 'full' | 'noAcl'; + userProject?: string; +} + +export interface UploadConfig extends Pick { + /** + * The API endpoint used for the request. + * Defaults to `storage.googleapis.com`. + * + * **Warning**: + * If this value does not match the current GCP universe an emulator context + * will be assumed and authentication will be bypassed. + */ + apiEndpoint?: string; + + /** + * The name of the destination bucket. + */ + bucket: string; + + /** + * The name of the destination file. + */ + file: string; + + /** + * The GoogleAuthOptions passed to google-auth-library + */ + authConfig?: GoogleAuthOptions; + + /** + * If you want to re-use an auth client from google-auto-auth, pass an + * instance here. + * Defaults to GoogleAuth and gets automatically overridden if an + * emulator context is detected. + */ + authClient?: { + request: ( + opts: GaxiosOptions + ) => Promise> | GaxiosPromise; + }; + + /** + * Create a separate request per chunk. + * + * This value is in bytes and should be a multiple of 256 KiB (2^18). + * We recommend using at least 8 MiB for the chunk size. + * + * @link https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload + */ + chunkSize?: number; + + /** + * For each API request we send, you may specify custom request options that + * we'll add onto the request. The request options follow the gaxios API: + * https://github.com/googleapis/gaxios#request-options. + */ + customRequestOptions?: GaxiosOptions; + + /** + * This will cause the upload to fail if the current generation of the remote + * object does not match the one provided here. + */ + generation?: number; + + /** + * Set to `true` if the upload is only a subset of the overall object to upload. + * This can be used when planning to continue the upload an object in another + * session. + * + * **Must be used with {@link UploadConfig.chunkSize} != `0`**. + * + * If this is a continuation of a previous upload, {@link UploadConfig.offset} + * should be set. + * + * @see {@link checkUploadStatus} for checking the status of an existing upload. + */ + isPartialUpload?: boolean; + + clientCrc32c?: string; + clientMd5Hash?: string; + /** + * Enables CRC32C calculation on the client side. + * The calculated hash will be sent in the final PUT request if `clientCrc32c` is not provided. + */ + crc32c?: boolean; + /** + * Enables MD5 calculation on the client side. + * The calculated hash will be sent in the final PUT request if `clientMd5Hash` is not provided. + */ + md5?: boolean; + + /** + * A customer-supplied encryption key. See + * https://cloud.google.com/storage/docs/encryption#customer-supplied. + */ + key?: string | Buffer; + + /** + * Resource name of the Cloud KMS key, of the form + * `projects/my-project/locations/global/keyRings/my-kr/cryptoKeys/my-key`, + * that will be used to encrypt the object. Overrides the object metadata's + * `kms_key_name` value, if any. + */ + kmsKeyName?: string; + + /** + * Any metadata you wish to set on the object. + */ + metadata?: ConfigMetadata; + + /** + * The starting byte in relation to the final uploaded object. + * **Must be used with {@link UploadConfig.uri}**. + * + * If resuming an interrupted stream, do not supply this argument unless you + * know the exact number of bytes the service has AND the provided stream's + * first byte is a continuation from that provided offset. If resuming an + * interrupted stream and this option has not been provided, we will treat + * the provided upload stream as the object to upload - where the first byte + * of the upload stream is the first byte of the object to upload; skipping + * any bytes that are already present on the server. + * + * @see {@link checkUploadStatus} for checking the status of an existing upload. + * @see {@link https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload#resume-upload.} + */ + offset?: number; + + /** + * Set an Origin header when creating the resumable upload URI. + */ + origin?: string; + + /** + * Specify query parameters that go along with the initial upload request. See + * https://cloud.google.com/storage/docs/json_api/v1/objects/insert#parameters + */ + params?: QueryParameters; + + /** + * Apply a predefined set of access controls to the created file. + */ + predefinedAcl?: PredefinedAcl; + + /** + * Make the uploaded file private. (Alias for config.predefinedAcl = + * 'private') + */ + private?: boolean; + + /** + * Make the uploaded file public. (Alias for config.predefinedAcl = + * 'publicRead') + */ + public?: boolean; + + /** + * The service domain for a given Cloud universe. + */ + universeDomain?: string; + + /** + * If you already have a resumable URI from a previously-created resumable + * upload, just pass it in here and we'll use that. + * + * If resuming an interrupted stream and the {@link UploadConfig.offset} + * option has not been provided, we will treat the provided upload stream as + * the object to upload - where the first byte of the upload stream is the + * first byte of the object to upload; skipping any bytes that are already + * present on the server. + * + * @see {@link checkUploadStatus} for checking the status of an existing upload. + */ + uri?: string; + + /** + * If the bucket being accessed has requesterPays functionality enabled, this + * can be set to control which project is billed for the access of this file. + */ + userProject?: string; + + /** + * Configuration options for retrying retryable errors. + */ + retryOptions: RetryOptions; + + /** + * Controls whether or not to use authentication when using a custom endpoint. + */ + useAuthWithCustomEndpoint?: boolean; + + [GCCL_GCS_CMD_KEY]?: string; +} + +export interface ConfigMetadata { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + [key: string]: any; + + /** + * Set the length of the object being uploaded. If uploading a partial + * object, this is the overall size of the finalized object. + */ + contentLength?: number; + + /** + * Set the content type of the incoming data. + */ + contentType?: string; +} + +export interface GoogleInnerError { + reason?: string; +} + +export interface ApiError extends Error { + code?: number; + errors?: GoogleInnerError[]; +} + +export interface CheckUploadStatusConfig { + /** + * Set to `false` to disable retries within this method. + * + * @defaultValue `true` + */ + retry?: boolean; +} + +export class Upload extends Writable { + bucket: string; + file: string; + apiEndpoint: string; + baseURI: string; + authConfig?: {scopes?: string[]}; + /* + * Defaults to GoogleAuth and gets automatically overridden if an + * emulator context is detected. + */ + authClient: { + request: ( + opts: GaxiosOptions + ) => Promise> | GaxiosPromise; + }; + cacheKey: string; + chunkSize?: number; + customRequestOptions: GaxiosOptions; + generation?: number; + key?: string | Buffer; + kmsKeyName?: string; + metadata: ConfigMetadata; + offset?: number; + origin?: string; + params: QueryParameters; + predefinedAcl?: PredefinedAcl; + private?: boolean; + public?: boolean; + uri?: string; + userProject?: string; + encryption?: Encryption; + uriProvidedManually: boolean; + numBytesWritten = 0; + numRetries = 0; + contentLength: number | '*'; + retryOptions: RetryOptions; + timeOfFirstRequest: number; + isPartialUpload: boolean; + + private currentInvocationId = { + checkUploadStatus: uuid.v4(), + chunk: uuid.v4(), + uri: uuid.v4(), + }; + /** + * A cache of buffers written to this instance, ready for consuming + */ + private writeBuffers: Buffer[] = []; + private numChunksReadInRequest = 0; + + #hashValidator?: HashStreamValidator; + #clientCrc32c?: string; + #clientMd5Hash?: string; + + /** + * An array of buffers used for caching the most recent upload chunk. + * We should not assume that the server received all bytes sent in the request. + * - https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload + */ + private localWriteCache: Buffer[] = []; + private localWriteCacheByteLength = 0; + private upstreamEnded = false; + #gcclGcsCmd?: string; + + constructor(cfg: UploadConfig) { + super(cfg); + cfg = cfg || {}; + + if (!cfg.bucket || !cfg.file) { + throw new Error('A bucket and file name are required'); + } + + if (cfg.offset && !cfg.uri) { + throw new RangeError( + 'Cannot provide an `offset` without providing a `uri`' + ); + } + + if (cfg.isPartialUpload && !cfg.chunkSize) { + throw new RangeError( + 'Cannot set `isPartialUpload` without providing a `chunkSize`' + ); + } + + cfg.authConfig = cfg.authConfig || {}; + cfg.authConfig.scopes = [ + 'https://www.googleapis.com/auth/devstorage.full_control', + ]; + this.authClient = cfg.authClient || new GoogleAuth(cfg.authConfig); + + const universe = cfg.universeDomain || DEFAULT_UNIVERSE; + + this.apiEndpoint = `https://storage.${universe}`; + if (cfg.apiEndpoint && cfg.apiEndpoint !== this.apiEndpoint) { + this.apiEndpoint = this.sanitizeEndpoint(cfg.apiEndpoint); + + const hostname = new URL(this.apiEndpoint).hostname; + + // check if it is a domain of a known universe + const isDomain = hostname === universe; + const isDefaultUniverseDomain = hostname === DEFAULT_UNIVERSE; + + // check if it is a subdomain of a known universe + // by checking a last (universe's length + 1) of a hostname + const isSubDomainOfUniverse = + hostname.slice(-(universe.length + 1)) === `.${universe}`; + const isSubDomainOfDefaultUniverse = + hostname.slice(-(DEFAULT_UNIVERSE.length + 1)) === + `.${DEFAULT_UNIVERSE}`; + + if ( + !isDomain && + !isDefaultUniverseDomain && + !isSubDomainOfUniverse && + !isSubDomainOfDefaultUniverse + ) { + // Check if we should use auth with custom endpoint + if (cfg.useAuthWithCustomEndpoint !== true) { + // Only bypass auth if explicitly not requested + this.authClient = gaxios; + } + // Otherwise keep the authenticated client + } + } + + this.baseURI = `${this.apiEndpoint}/upload/storage/v1/b`; + this.bucket = cfg.bucket; + + const cacheKeyElements = [cfg.bucket, cfg.file]; + if (typeof cfg.generation === 'number') { + cacheKeyElements.push(`${cfg.generation}`); + } + + this.cacheKey = cacheKeyElements.join('/'); + + this.customRequestOptions = cfg.customRequestOptions || {}; + this.file = cfg.file; + this.generation = cfg.generation; + this.kmsKeyName = cfg.kmsKeyName; + this.metadata = cfg.metadata || {}; + this.offset = cfg.offset; + this.origin = cfg.origin; + this.params = cfg.params || {}; + this.userProject = cfg.userProject; + this.chunkSize = cfg.chunkSize; + this.retryOptions = cfg.retryOptions; + this.isPartialUpload = cfg.isPartialUpload ?? false; + + this.#clientCrc32c = cfg.clientCrc32c; + this.#clientMd5Hash = cfg.clientMd5Hash; + + const calculateCrc32c = !cfg.clientCrc32c && cfg.crc32c; + const calculateMd5 = !cfg.clientMd5Hash && cfg.md5; + + if (calculateCrc32c || calculateMd5) { + this.#hashValidator = new HashStreamValidator({ + crc32c: calculateCrc32c, + md5: calculateMd5, + updateHashesOnly: true, + }); + } + + if (cfg.key) { + if (typeof cfg.key === 'string') { + const base64Key = Buffer.from(cfg.key).toString('base64'); + this.encryption = { + key: base64Key, + hash: createHash('sha256').update(cfg.key).digest('base64'), + }; + } else { + const base64Key = cfg.key.toString('base64'); + this.encryption = { + key: base64Key, + hash: createHash('sha256').update(cfg.key).digest('base64'), + }; + } + } + + this.predefinedAcl = cfg.predefinedAcl; + if (cfg.private) this.predefinedAcl = 'private'; + if (cfg.public) this.predefinedAcl = 'publicRead'; + + const autoRetry = cfg.retryOptions.autoRetry; + this.uriProvidedManually = !!cfg.uri; + this.uri = cfg.uri; + + if (this.offset) { + // we're resuming an incomplete upload + this.numBytesWritten = this.offset; + } + + this.numRetries = 0; // counter for number of retries currently executed + if (!autoRetry) { + cfg.retryOptions.maxRetries = 0; + } + + this.timeOfFirstRequest = Date.now(); + + const contentLength = cfg.metadata + ? Number(cfg.metadata.contentLength) + : NaN; + this.contentLength = isNaN(contentLength) ? '*' : contentLength; + + this.#gcclGcsCmd = cfg[GCCL_GCS_CMD_KEY]; + + this.once('writing', () => { + if (this.uri) { + this.continueUploading(); + } else { + this.createURI(err => { + if (err) { + return this.destroy(err); + } + this.startUploading(); + return; + }); + } + }); + } + + /** + * Prevent 'finish' event until the upload has succeeded. + * + * @param fireFinishEvent The finish callback + */ + _final(fireFinishEvent = () => {}) { + this.upstreamEnded = true; + + this.once('uploadFinished', fireFinishEvent); + + process.nextTick(() => { + this.emit('upstreamFinished'); + // it's possible `_write` may not be called - namely for empty object uploads + this.emit('writing'); + }); + } + + /** + * Handles incoming data from upstream + * + * @param chunk The chunk to append to the buffer + * @param encoding The encoding of the chunk + * @param readCallback A callback for when the buffer has been read downstream + */ + _write( + chunk: Buffer | string, + encoding: BufferEncoding, + readCallback = () => {} + ) { + // Backwards-compatible event + this.emit('writing'); + + const bufferChunk = + typeof chunk === 'string' ? Buffer.from(chunk, encoding) : chunk; + + if (this.#hashValidator) { + try { + this.#hashValidator.write(bufferChunk); + } catch (e) { + this.destroy(e as Error); + return; + } + } + + this.writeBuffers.push(bufferChunk); + + this.once('readFromChunkBuffer', readCallback); + + process.nextTick(() => this.emit('wroteToChunkBuffer')); + } + + #resetLocalBuffersCache() { + this.localWriteCache = []; + this.localWriteCacheByteLength = 0; + } + + #addLocalBufferCache(buf: Buffer) { + this.localWriteCache.push(buf); + this.localWriteCacheByteLength += buf.byteLength; + } + + /** + * Compares the client's calculated or provided hash against the server's + * returned hash for a specific checksum type. Destroys the stream on mismatch. + * @param clientHash The client's calculated or provided hash (Base64). + * @param serverHash The hash returned by the server (Base64). + * @param hashType The type of hash ('CRC32C' or 'MD5'). + */ + #validateChecksum( + clientHash: string | undefined, + serverHash: string | undefined, + hashType: 'CRC32C' | 'MD5' + ): boolean { + // Only validate if both client and server hashes are present. + if (clientHash && serverHash) { + if (clientHash !== serverHash) { + const detailMessage = `${hashType} checksum mismatch. Client calculated: ${clientHash}, Server returned: ${serverHash}`; + const detailError = new Error(detailMessage); + const error = new RequestError(FileExceptionMessages.UPLOAD_MISMATCH); + error.code = 'FILE_NO_UPLOAD'; + error.errors = [detailError]; + + this.destroy(error); + return true; + } + } + return false; + } + + /** + * Builds and applies the X-Goog-Hash header to the request options + * using either calculated hashes from #hashValidator or pre-calculated + * client-side hashes. This should only be called on the final request. + * + * @param headers The headers object to modify. + */ + #applyChecksumHeaders(headers: GaxiosOptions['headers']) { + const checksums: string[] = []; + + if (this.#hashValidator?.crc32cEnabled) { + checksums.push(`crc32c=${this.#hashValidator.crc32c!}`); + } else if (this.#clientCrc32c) { + checksums.push(`crc32c=${this.#clientCrc32c}`); + } + + if (this.#hashValidator?.md5Enabled) { + checksums.push(`md5=${this.#hashValidator.md5Digest!}`); + } else if (this.#clientMd5Hash) { + checksums.push(`md5=${this.#clientMd5Hash}`); + } + + if (checksums.length > 0) { + headers!['X-Goog-Hash'] = checksums.join(','); + } + } + + /** + * Prepends the local buffer to write buffer and resets it. + * + * @param keepLastBytes number of bytes to keep from the end of the local buffer. + */ + private prependLocalBufferToUpstream(keepLastBytes?: number) { + // Typically, the upstream write buffers should be smaller than the local + // cache, so we can save time by setting the local cache as the new + // upstream write buffer array and appending the old array to it + let initialBuffers: Buffer[] = []; + + if (keepLastBytes) { + // we only want the last X bytes + let bytesKept = 0; + + while (keepLastBytes > bytesKept) { + // load backwards because we want the last X bytes + // note: `localWriteCacheByteLength` is reset below + let buf = this.localWriteCache.pop(); + if (!buf) break; + + bytesKept += buf.byteLength; + + if (bytesKept > keepLastBytes) { + // we have gone over the amount desired, let's keep the last X bytes + // of this buffer + const diff = bytesKept - keepLastBytes; + buf = buf.subarray(diff); + bytesKept -= diff; + } + + initialBuffers.unshift(buf); + } + } else { + // we're keeping all of the local cache, simply use it as the initial buffer + initialBuffers = this.localWriteCache; + } + + // Append the old upstream to the new + const append = this.writeBuffers; + this.writeBuffers = initialBuffers; + + for (const buf of append) { + this.writeBuffers.push(buf); + } + + // reset last buffers sent + this.#resetLocalBuffersCache(); + } + + /** + * Retrieves data from upstream's buffer. + * + * @param limit The maximum amount to return from the buffer. + */ + private *pullFromChunkBuffer(limit: number) { + while (limit) { + const buf = this.writeBuffers.shift(); + if (!buf) break; + + let bufToYield = buf; + + if (buf.byteLength > limit) { + bufToYield = buf.subarray(0, limit); + this.writeBuffers.unshift(buf.subarray(limit)); + limit = 0; + } else { + limit -= buf.byteLength; + } + + yield bufToYield; + + // Notify upstream we've read from the buffer and we're able to consume + // more. It can also potentially send more data down as we're currently + // iterating. + this.emit('readFromChunkBuffer'); + } + } + + /** + * A handler for determining if data is ready to be read from upstream. + * + * @returns If there will be more chunks to read in the future + */ + private async waitForNextChunk(): Promise { + const willBeMoreChunks = await new Promise(resolve => { + // There's data available - it should be digested + if (this.writeBuffers.length) { + return resolve(true); + } + + // The upstream writable ended, we shouldn't expect any more data. + if (this.upstreamEnded) { + return resolve(false); + } + + // Nothing immediate seems to be determined. We need to prepare some + // listeners to determine next steps... + + const wroteToChunkBufferCallback = () => { + removeListeners(); + return resolve(true); + }; + + const upstreamFinishedCallback = () => { + removeListeners(); + + // this should be the last chunk, if there's anything there + if (this.writeBuffers.length) return resolve(true); + + return resolve(false); + }; + + // Remove listeners when we're ready to callback. + const removeListeners = () => { + this.removeListener('wroteToChunkBuffer', wroteToChunkBufferCallback); + this.removeListener('upstreamFinished', upstreamFinishedCallback); + }; + + // If there's data recently written it should be digested + this.once('wroteToChunkBuffer', wroteToChunkBufferCallback); + + // If the upstream finishes let's see if there's anything to grab + this.once('upstreamFinished', upstreamFinishedCallback); + }); + + return willBeMoreChunks; + } + + /** + * Reads data from upstream up to the provided `limit`. + * Ends when the limit has reached or no data is expected to be pushed from upstream. + * + * @param limit The most amount of data this iterator should return. `Infinity` by default. + */ + private async *upstreamIterator(limit = Infinity) { + // read from upstream chunk buffer + while (limit && (await this.waitForNextChunk())) { + // read until end or limit has been reached + for (const chunk of this.pullFromChunkBuffer(limit)) { + limit -= chunk.byteLength; + yield chunk; + } + } + } + + createURI(): Promise; + createURI(callback: CreateUriCallback): void; + createURI(callback?: CreateUriCallback): void | Promise { + if (!callback) { + return this.createURIAsync(); + } + this.createURIAsync().then(r => callback(null, r), callback); + } + + protected async createURIAsync(): Promise { + const metadata = {...this.metadata}; + const headers: gaxios.Headers = {}; + + // Delete content length and content type from metadata if they exist. + // These are headers and should not be sent as part of the metadata. + if (metadata.contentLength) { + headers['X-Upload-Content-Length'] = metadata.contentLength.toString(); + delete metadata.contentLength; + } + + if (metadata.contentType) { + headers!['X-Upload-Content-Type'] = metadata.contentType; + delete metadata.contentType; + } + + let googAPIClient = `${getRuntimeTrackingString()} gccl/${ + packageJson.version + }-${getModuleFormat()} gccl-invocation-id/${this.currentInvocationId.uri}`; + + if (this.#gcclGcsCmd) { + googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`; + } + + // Check if headers already exist before creating new ones + const reqOpts: GaxiosOptions = { + method: 'POST', + url: [this.baseURI, this.bucket, 'o'].join('/'), + params: Object.assign( + { + name: this.file, + uploadType: 'resumable', + }, + this.params + ), + data: metadata, + headers: { + 'User-Agent': getUserAgentString(), + 'x-goog-api-client': googAPIClient, + ...headers, + }, + }; + + if (metadata.contentLength) { + reqOpts.headers!['X-Upload-Content-Length'] = + metadata.contentLength.toString(); + } + + if (metadata.contentType) { + reqOpts.headers!['X-Upload-Content-Type'] = metadata.contentType; + } + + if (typeof this.generation !== 'undefined') { + reqOpts.params.ifGenerationMatch = this.generation; + } + + if (this.kmsKeyName) { + reqOpts.params.kmsKeyName = this.kmsKeyName; + } + + if (this.predefinedAcl) { + reqOpts.params.predefinedAcl = this.predefinedAcl; + } + + if (this.origin) { + reqOpts.headers!.Origin = this.origin; + } + const uri = await AsyncRetry( + async (bail: (err: Error) => void) => { + try { + const res = await this.makeRequest(reqOpts); + // We have successfully got a URI we can now create a new invocation id + this.currentInvocationId.uri = uuid.v4(); + return res.headers.location; + } catch (err) { + const e = err as GaxiosError; + const apiError = { + code: e.response?.status, + name: e.response?.statusText, + message: e.response?.statusText, + errors: [ + { + reason: e.code as string, + }, + ], + }; + if ( + this.retryOptions.maxRetries! > 0 && + this.retryOptions.retryableErrorFn!(apiError as ApiError) + ) { + throw e; + } else { + return bail(e); + } + } + }, + { + retries: this.retryOptions.maxRetries, + factor: this.retryOptions.retryDelayMultiplier, + maxTimeout: this.retryOptions.maxRetryDelay! * 1000, //convert to milliseconds + maxRetryTime: this.retryOptions.totalTimeout! * 1000, //convert to milliseconds + } + ); + + this.uri = uri; + this.offset = 0; + + // emit the newly generated URI for future reuse, if necessary. + this.emit('uri', uri); + + return uri; + } + + private async continueUploading() { + this.offset ?? (await this.getAndSetOffset()); + + return this.startUploading(); + } + + async startUploading() { + const multiChunkMode = !!this.chunkSize; + let responseReceived = false; + this.numChunksReadInRequest = 0; + + if (!this.offset) { + this.offset = 0; + } + + // Check if the offset (server) is too far behind the current stream + if (this.offset < this.numBytesWritten) { + const delta = this.numBytesWritten - this.offset; + const message = `The offset is lower than the number of bytes written. The server has ${this.offset} bytes and while ${this.numBytesWritten} bytes has been uploaded - thus ${delta} bytes are missing. Stopping as this could result in data loss. Initiate a new upload to continue.`; + + this.emit('error', new RangeError(message)); + return; + } + + // Check if we should 'fast-forward' to the relevant data to upload + if (this.numBytesWritten < this.offset) { + // 'fast-forward' to the byte where we need to upload. + // only push data from the byte after the one we left off on + const fastForwardBytes = this.offset - this.numBytesWritten; + + for await (const _chunk of this.upstreamIterator(fastForwardBytes)) { + _chunk; // discard the data up until the point we want + } + + this.numBytesWritten = this.offset; + } + + let expectedUploadSize: number | undefined = undefined; + + // Set `expectedUploadSize` to `contentLength - this.numBytesWritten`, if available + if (typeof this.contentLength === 'number') { + expectedUploadSize = this.contentLength - this.numBytesWritten; + } + + // `expectedUploadSize` should be no more than the `chunkSize`. + // It's possible this is the last chunk request for a multiple + // chunk upload, thus smaller than the chunk size. + if (this.chunkSize) { + expectedUploadSize = expectedUploadSize + ? Math.min(this.chunkSize, expectedUploadSize) + : this.chunkSize; + } + + // A queue for the upstream data + const upstreamQueue = this.upstreamIterator(expectedUploadSize); + + // The primary read stream for this request. This stream retrieves no more + // than the exact requested amount from upstream. + const requestStream = new Readable({ + read: async () => { + // Don't attempt to retrieve data upstream if we already have a response + if (responseReceived) requestStream.push(null); + + const result = await upstreamQueue.next(); + + if (result.value) { + this.numChunksReadInRequest++; + + if (multiChunkMode) { + // save ever buffer used in the request in multi-chunk mode + this.#addLocalBufferCache(result.value); + } else { + this.#resetLocalBuffersCache(); + this.#addLocalBufferCache(result.value); + } + + this.numBytesWritten += result.value.byteLength; + + this.emit('progress', { + bytesWritten: this.numBytesWritten, + contentLength: this.contentLength, + }); + + requestStream.push(result.value); + } + + if (result.done) { + requestStream.push(null); + } + }, + }); + + let googAPIClient = `${getRuntimeTrackingString()} gccl/${ + packageJson.version + }-${getModuleFormat()} gccl-invocation-id/${ + this.currentInvocationId.chunk + }`; + + if (this.#gcclGcsCmd) { + googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`; + } + + const headers: GaxiosOptions['headers'] = { + 'User-Agent': getUserAgentString(), + 'x-goog-api-client': googAPIClient, + }; + + // If using multiple chunk upload, set appropriate header + if (multiChunkMode) { + // We need to know how much data is available upstream to set the `Content-Range` header. + // https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload + for await (const chunk of this.upstreamIterator(expectedUploadSize)) { + // This will conveniently track and keep the size of the buffers. + // We will reach either the expected upload size or the remainder of the stream. + this.#addLocalBufferCache(chunk); + } + + // This is the sum from the `#addLocalBufferCache` calls + const bytesToUpload = this.localWriteCacheByteLength; + + // Important: we want to know if the upstream has ended and the queue is empty before + // unshifting data back into the queue. This way we will know if this is the last request or not. + const isLastChunkOfUpload = !(await this.waitForNextChunk()); + + if (isLastChunkOfUpload && this.#hashValidator) { + this.#hashValidator.end(); + } + + // Important: put the data back in the queue for the actual upload + this.prependLocalBufferToUpstream(); + + let totalObjectSize = this.contentLength; + + if ( + typeof this.contentLength !== 'number' && + isLastChunkOfUpload && + !this.isPartialUpload + ) { + // Let's let the server know this is the last chunk of the object since we didn't set it before. + totalObjectSize = bytesToUpload + this.numBytesWritten; + } + + // `- 1` as the ending byte is inclusive in the request. + const endingByte = bytesToUpload + this.numBytesWritten - 1; + + // `Content-Length` for multiple chunk uploads is the size of the chunk, + // not the overall object + headers['Content-Length'] = bytesToUpload; + headers['Content-Range'] = + `bytes ${this.offset}-${endingByte}/${totalObjectSize}`; + + // Apply X-Goog-Hash header ONLY on the final chunk (WriteObject call) + if (isLastChunkOfUpload) { + this.#applyChecksumHeaders(headers); + } + } else { + headers['Content-Range'] = `bytes ${this.offset}-*/${this.contentLength}`; + + if (this.#hashValidator) { + this.#hashValidator.end(); + } + this.#applyChecksumHeaders(headers); + } + + const reqOpts: GaxiosOptions = { + method: 'PUT', + url: this.uri, + headers, + body: requestStream, + }; + + try { + const resp = await this.makeRequestStream(reqOpts); + if (resp) { + responseReceived = true; + await this.responseHandler(resp); + } + } catch (e) { + const err = e as ApiError; + + if (this.retryOptions.retryableErrorFn!(err)) { + this.attemptDelayedRetry({ + status: NaN, + data: err, + }); + return; + } + + this.destroy(err); + } + } + + // Process the API response to look for errors that came in + // the response body. + private async responseHandler(resp: GaxiosResponse) { + if (resp.data.error) { + this.destroy(resp.data.error); + return; + } + + // At this point we can safely create a new id for the chunk + this.currentInvocationId.chunk = uuid.v4(); + + const moreDataToUpload = await this.waitForNextChunk(); + + const shouldContinueWithNextMultiChunkRequest = + this.chunkSize && + resp.status === RESUMABLE_INCOMPLETE_STATUS_CODE && + resp.headers.range && + moreDataToUpload; + + /** + * This is true when we're expecting to upload more data in a future request, + * yet the upstream for the upload session has been exhausted. + */ + const shouldContinueUploadInAnotherRequest = + this.isPartialUpload && + resp.status === RESUMABLE_INCOMPLETE_STATUS_CODE && + !moreDataToUpload; + + if (shouldContinueWithNextMultiChunkRequest) { + // Use the upper value in this header to determine where to start the next chunk. + // We should not assume that the server received all bytes sent in the request. + // https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload + const range: string = resp.headers.range; + this.offset = Number(range.split('-')[1]) + 1; + + // We should not assume that the server received all bytes sent in the request. + // - https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload + const missingBytes = this.numBytesWritten - this.offset; + if (missingBytes) { + // As multi-chunk uploads send one chunk per request and pulls one + // chunk into the pipeline, prepending the missing bytes back should + // be fine for the next request. + this.prependLocalBufferToUpstream(missingBytes); + this.numBytesWritten -= missingBytes; + } else { + // No bytes missing - no need to keep the local cache + this.#resetLocalBuffersCache(); + } + + // continue uploading next chunk + this.continueUploading(); + } else if ( + !this.isSuccessfulResponse(resp.status) && + !shouldContinueUploadInAnotherRequest + ) { + const err: ApiError = new Error('Upload failed'); + err.code = resp.status; + err.name = 'Upload failed'; + if (resp?.data) { + err.errors = [resp?.data]; + } + + this.destroy(err); + } else if (this.isSuccessfulResponse(resp.status)) { + const serverCrc32c = resp.data.crc32c; + const serverMd5 = resp.data.md5Hash; + + if (this.#hashValidator) { + this.#hashValidator.end(); + } + + const clientCrc32cToValidate = + this.#hashValidator?.crc32c || this.#clientCrc32c; + const clientMd5HashToValidate = + this.#hashValidator?.md5Digest || this.#clientMd5Hash; + if ( + this.#validateChecksum( + clientCrc32cToValidate, + serverCrc32c, + 'CRC32C' + ) || + this.#validateChecksum(clientMd5HashToValidate, serverMd5, 'MD5') + ) { + return; + } + + // no need to keep the cache + this.#resetLocalBuffersCache(); + + if (resp && resp.data) { + resp.data.size = Number(resp.data.size); + } + this.emit('metadata', resp.data); + + // Allow the object (Upload) to continue naturally so the user's + // "finish" event fires. + this.emit('uploadFinished'); + } else { + // Handles the case where shouldContinueUploadInAnotherRequest is true + // and the response is not successful (e.g., 308 for a partial upload). + // This is the expected behavior for partial uploads that have finished their chunk. + this.emit('uploadFinished'); + } + } + + /** + * Check the status of an existing resumable upload. + * + * @param cfg A configuration to use. `uri` is required. + * @returns the current upload status + */ + async checkUploadStatus( + config: CheckUploadStatusConfig = {} + ): Promise> { + let googAPIClient = `${getRuntimeTrackingString()} gccl/${ + packageJson.version + }-${getModuleFormat()} gccl-invocation-id/${ + this.currentInvocationId.checkUploadStatus + }`; + + if (this.#gcclGcsCmd) { + googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`; + } + + const opts: GaxiosOptions = { + method: 'PUT', + url: this.uri, + headers: { + 'Content-Length': 0, + 'Content-Range': 'bytes */*', + 'User-Agent': getUserAgentString(), + 'x-goog-api-client': googAPIClient, + }, + }; + + try { + const resp = await this.makeRequest(opts); + + // Successfully got the offset we can now create a new offset invocation id + this.currentInvocationId.checkUploadStatus = uuid.v4(); + + return resp; + } catch (e) { + if ( + config.retry === false || + !(e instanceof Error) || + !this.retryOptions.retryableErrorFn!(e) + ) { + throw e; + } + + const retryDelay = this.getRetryDelay(); + + if (retryDelay <= 0) { + throw e; + } + + await new Promise(res => setTimeout(res, retryDelay)); + + return this.checkUploadStatus(config); + } + } + + private async getAndSetOffset() { + try { + // we want to handle retries in this method. + const resp = await this.checkUploadStatus({retry: false}); + + if (resp.status === RESUMABLE_INCOMPLETE_STATUS_CODE) { + if (typeof resp.headers.range === 'string') { + this.offset = Number(resp.headers.range.split('-')[1]) + 1; + return; + } + } + this.offset = 0; + } catch (e) { + const err = e as ApiError; + + if (this.retryOptions.retryableErrorFn!(err)) { + this.attemptDelayedRetry({ + status: NaN, + data: err, + }); + return; + } + + this.destroy(err); + } + } + + private async makeRequest(reqOpts: GaxiosOptions): GaxiosPromise { + if (this.encryption) { + reqOpts.headers = reqOpts.headers || {}; + reqOpts.headers['x-goog-encryption-algorithm'] = 'AES256'; + reqOpts.headers['x-goog-encryption-key'] = this.encryption.key.toString(); + reqOpts.headers['x-goog-encryption-key-sha256'] = + this.encryption.hash.toString(); + } + + if (this.userProject) { + reqOpts.params = reqOpts.params || {}; + reqOpts.params.userProject = this.userProject; + } + // Let gaxios know we will handle a 308 error code ourselves. + reqOpts.validateStatus = (status: number) => { + return ( + this.isSuccessfulResponse(status) || + status === RESUMABLE_INCOMPLETE_STATUS_CODE + ); + }; + + const combinedReqOpts = { + ...this.customRequestOptions, + ...reqOpts, + headers: { + ...this.customRequestOptions.headers, + ...reqOpts.headers, + }, + }; + + const res = await this.authClient.request<{error?: object}>( + combinedReqOpts + ); + if (res.data && res.data.error) { + throw res.data.error; + } + return res; + } + + private async makeRequestStream(reqOpts: GaxiosOptions) { + const controller = new AbortController(); + const errorCallback = () => controller.abort(); + this.once('error', errorCallback); + + if (this.userProject) { + reqOpts.params = reqOpts.params || {}; + reqOpts.params.userProject = this.userProject; + } + reqOpts.signal = controller.signal; + reqOpts.validateStatus = () => true; + + const combinedReqOpts = { + ...this.customRequestOptions, + ...reqOpts, + headers: { + ...this.customRequestOptions.headers, + ...reqOpts.headers, + }, + }; + const res = await this.authClient.request(combinedReqOpts); + const successfulRequest = this.onResponse(res); + this.removeListener('error', errorCallback); + + return successfulRequest ? res : null; + } + + /** + * @return {bool} is the request good? + */ + private onResponse(resp: GaxiosResponse) { + if ( + resp.status !== 200 && + this.retryOptions.retryableErrorFn!({ + code: resp.status, + message: resp.statusText, + name: resp.statusText, + }) + ) { + this.attemptDelayedRetry(resp); + return false; + } + + this.emit('response', resp); + return true; + } + + /** + * @param resp GaxiosResponse object from previous attempt + */ + private attemptDelayedRetry(resp: Pick) { + if (this.numRetries < this.retryOptions.maxRetries!) { + if ( + resp.status === NOT_FOUND_STATUS_CODE && + this.numChunksReadInRequest === 0 + ) { + this.startUploading(); + } else { + const retryDelay = this.getRetryDelay(); + + if (retryDelay <= 0) { + this.destroy( + new Error( + `Retry total time limit exceeded - ${JSON.stringify(resp.data)}` + ) + ); + return; + } + + // Unshift the local cache back in case it's needed for the next request. + this.numBytesWritten -= this.localWriteCacheByteLength; + this.prependLocalBufferToUpstream(); + + // We don't know how much data has been received by the server. + // `continueUploading` will recheck the offset via `getAndSetOffset`. + // If `offset` < `numberBytesReceived` then we will raise a RangeError + // as we've streamed too much data that has been missed - this should + // not be the case for multi-chunk uploads as `lastChunkSent` is the + // body of the entire request. + this.offset = undefined; + + setTimeout(this.continueUploading.bind(this), retryDelay); + } + this.numRetries++; + } else { + this.destroy( + new Error(`Retry limit exceeded - ${JSON.stringify(resp.data)}`) + ); + } + } + + /** + * The amount of time to wait before retrying the request, in milliseconds. + * If negative, do not retry. + * + * @returns the amount of time to wait, in milliseconds. + */ + private getRetryDelay(): number { + const randomMs = Math.round(Math.random() * 1000); + const waitTime = + Math.pow(this.retryOptions.retryDelayMultiplier!, this.numRetries) * + 1000 + + randomMs; + const maxAllowableDelayMs = + this.retryOptions.totalTimeout! * 1000 - + (Date.now() - this.timeOfFirstRequest); + const maxRetryDelayMs = this.retryOptions.maxRetryDelay! * 1000; + + return Math.min(waitTime, maxRetryDelayMs, maxAllowableDelayMs); + } + + /* + * Prepare user-defined API endpoint for compatibility with our API. + */ + private sanitizeEndpoint(url: string) { + if (!PROTOCOL_REGEX.test(url)) { + url = `https://${url}`; + } + return url.replace(/\/+$/, ''); // Remove trailing slashes + } + + /** + * Check if a given status code is 2xx + * + * @param status The status code to check + * @returns if the status is 2xx + */ + public isSuccessfulResponse(status: number): boolean { + return status >= 200 && status < 300; + } +} + +export function upload(cfg: UploadConfig) { + return new Upload(cfg); +} + +export function createURI(cfg: UploadConfig): Promise; +export function createURI(cfg: UploadConfig, callback: CreateUriCallback): void; +export function createURI( + cfg: UploadConfig, + callback?: CreateUriCallback +): void | Promise { + const up = new Upload(cfg); + if (!callback) { + return up.createURI(); + } + up.createURI().then(r => callback(null, r), callback); +} + +/** + * Check the status of an existing resumable upload. + * + * @param cfg A configuration to use. `uri` is required. + * @returns the current upload status + */ +export function checkUploadStatus( + cfg: UploadConfig & Required> +) { + const up = new Upload(cfg); + + return up.checkUploadStatus(); +} diff --git a/handwritten/storage/src/signer.ts b/handwritten/storage/src/signer.ts new file mode 100644 index 00000000000..879bc4d2a57 --- /dev/null +++ b/handwritten/storage/src/signer.ts @@ -0,0 +1,491 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as crypto from 'crypto'; +import * as http from 'http'; +import * as url from 'url'; +import {ExceptionMessages, Storage} from './storage.js'; +import {encodeURI, qsStringify, objectEntries, formatAsUTCISO} from './util.js'; +import {GoogleAuth} from 'google-auth-library'; + +type GoogleAuthLike = Pick; + +/** + * @deprecated Use {@link GoogleAuth} instead + */ +export interface AuthClient { + sign(blobToSign: string): Promise; + getCredentials(): Promise<{ + client_email?: string; + }>; +} + +export interface BucketI { + name: string; +} + +export interface FileI { + name: string; +} + +export interface Query { + [key: string]: string; +} + +export interface GetSignedUrlConfigInternal { + expiration: number; + accessibleAt?: Date; + method: string; + extensionHeaders?: http.OutgoingHttpHeaders; + queryParams?: Query; + cname?: string; + contentMd5?: string; + contentType?: string; + bucket: string; + file?: string; + /** + * The host for the generated signed URL + * + * @example + * 'https://localhost:8080/' + */ + host?: string | URL; + /** + * An endpoint for generating the signed URL + * + * @example + * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + */ + signingEndpoint?: string | URL; +} + +interface SignedUrlQuery { + generation?: number; + 'response-content-type'?: string; + 'response-content-disposition'?: string; +} + +interface V2SignedUrlQuery extends SignedUrlQuery { + GoogleAccessId: string; + Expires: number; + Signature: string; +} + +export interface SignerGetSignedUrlConfig { + method: 'GET' | 'PUT' | 'DELETE' | 'POST'; + expires: string | number | Date; + accessibleAt?: string | number | Date; + virtualHostedStyle?: boolean; + version?: 'v2' | 'v4'; + cname?: string; + extensionHeaders?: http.OutgoingHttpHeaders; + queryParams?: Query; + contentMd5?: string; + contentType?: string; + /** + * The host for the generated signed URL + * + * @example + * 'https://localhost:8080/' + */ + host?: string | URL; + /** + * An endpoint for generating the signed URL + * + * @example + * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + */ + signingEndpoint?: string | URL; +} + +export type SignerGetSignedUrlResponse = string; + +export type GetSignedUrlResponse = [SignerGetSignedUrlResponse]; + +export interface GetSignedUrlCallback { + (err: Error | null, url?: string): void; +} +type ValueOf = T[keyof T]; +type HeaderValue = ValueOf; + +export enum SignerExceptionMessages { + ACCESSIBLE_DATE_INVALID = 'The accessible at date provided was invalid.', + EXPIRATION_BEFORE_ACCESSIBLE_DATE = 'An expiration date cannot be before accessible date.', + X_GOOG_CONTENT_SHA256 = 'The header X-Goog-Content-SHA256 must be a hexadecimal string.', +} + +/* + * Default signing version for getSignedUrl is 'v2'. + */ +const DEFAULT_SIGNING_VERSION = 'v2'; + +const SEVEN_DAYS = 7 * 24 * 60 * 60; + +/** + * @const {string} + * @deprecated - unused + */ +export const PATH_STYLED_HOST = 'https://storage.googleapis.com'; + +export class URLSigner { + constructor( + private auth: AuthClient | GoogleAuthLike, + private bucket: BucketI, + private file?: FileI, + /** + * A {@link Storage} object. + * + * @privateRemarks + * + * Technically this is a required field, however it would be a breaking change to + * move it before optional properties. In the next major we should refactor the + * constructor of this class to only accept a config object. + */ + private storage: Storage = new Storage() + ) {} + + getSignedUrl( + cfg: SignerGetSignedUrlConfig + ): Promise { + const expiresInSeconds = this.parseExpires(cfg.expires); + const method = cfg.method; + const accessibleAtInSeconds = this.parseAccessibleAt(cfg.accessibleAt); + + if (expiresInSeconds < accessibleAtInSeconds) { + throw new Error( + SignerExceptionMessages.EXPIRATION_BEFORE_ACCESSIBLE_DATE + ); + } + + let customHost: string | undefined; + // Default style is `path`. + const isVirtualHostedStyle = cfg.virtualHostedStyle || false; + + if (cfg.cname) { + customHost = cfg.cname; + } else if (isVirtualHostedStyle) { + customHost = `https://${this.bucket.name}.storage.${this.storage.universeDomain}`; + } + + const secondsToMilliseconds = 1000; + const config: GetSignedUrlConfigInternal = Object.assign({}, cfg, { + method, + expiration: expiresInSeconds, + accessibleAt: new Date(secondsToMilliseconds * accessibleAtInSeconds), + bucket: this.bucket.name, + file: this.file ? encodeURI(this.file.name, false) : undefined, + }); + + if (customHost) { + config.cname = customHost; + } + + const version = cfg.version || DEFAULT_SIGNING_VERSION; + + let promise: Promise; + if (version === 'v2') { + promise = this.getSignedUrlV2(config); + } else if (version === 'v4') { + promise = this.getSignedUrlV4(config); + } else { + throw new Error( + `Invalid signed URL version: ${version}. Supported versions are 'v2' and 'v4'.` + ); + } + + return promise.then(query => { + query = Object.assign(query, cfg.queryParams); + + const signedUrl = new url.URL( + cfg.host?.toString() || config.cname || this.storage.apiEndpoint + ); + + signedUrl.pathname = this.getResourcePath( + !!config.cname, + this.bucket.name, + config.file + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + signedUrl.search = qsStringify(query as any); + return signedUrl.href; + }); + } + + private getSignedUrlV2( + config: GetSignedUrlConfigInternal + ): Promise { + const canonicalHeadersString = this.getCanonicalHeaders( + config.extensionHeaders || {} + ); + const resourcePath = this.getResourcePath( + false, + config.bucket, + config.file + ); + + const blobToSign = [ + config.method, + config.contentMd5 || '', + config.contentType || '', + config.expiration, + canonicalHeadersString + resourcePath, + ].join('\n'); + + const sign = async () => { + const auth = this.auth; + try { + const signature = await auth.sign( + blobToSign, + config.signingEndpoint?.toString() + ); + const credentials = await auth.getCredentials(); + + return { + GoogleAccessId: credentials.client_email!, + Expires: config.expiration, + Signature: signature, + } as V2SignedUrlQuery; + } catch (err) { + const error = err as Error; + const signingErr = new SigningError(error.message); + signingErr.stack = error.stack; + throw signingErr; + } + }; + return sign(); + } + + private getSignedUrlV4( + config: GetSignedUrlConfigInternal + ): Promise { + config.accessibleAt = config.accessibleAt + ? config.accessibleAt + : new Date(); + const millisecondsToSeconds = 1.0 / 1000.0; + const expiresPeriodInSeconds = + config.expiration - config.accessibleAt.valueOf() * millisecondsToSeconds; + + // v4 limit expiration to be 7 days maximum + if (expiresPeriodInSeconds > SEVEN_DAYS) { + throw new Error( + `Max allowed expiration is seven days (${SEVEN_DAYS} seconds).` + ); + } + + const extensionHeaders = Object.assign({}, config.extensionHeaders); + const fqdn = new url.URL( + config.host?.toString() || config.cname || this.storage.apiEndpoint + ); + extensionHeaders.host = fqdn.hostname; + if (config.contentMd5) { + extensionHeaders['content-md5'] = config.contentMd5; + } + if (config.contentType) { + extensionHeaders['content-type'] = config.contentType; + } + + let contentSha256: string; + const sha256Header = extensionHeaders['x-goog-content-sha256']; + if (sha256Header) { + if ( + typeof sha256Header !== 'string' || + !/[A-Fa-f0-9]{40}/.test(sha256Header) + ) { + throw new Error(SignerExceptionMessages.X_GOOG_CONTENT_SHA256); + } + contentSha256 = sha256Header; + } + + const signedHeaders = Object.keys(extensionHeaders) + .map(header => header.toLowerCase()) + .sort() + .join(';'); + + const extensionHeadersString = this.getCanonicalHeaders(extensionHeaders); + + const datestamp = formatAsUTCISO(config.accessibleAt); + const credentialScope = `${datestamp}/auto/storage/goog4_request`; + + const sign = async () => { + const credentials = await this.auth.getCredentials(); + const credential = `${credentials.client_email}/${credentialScope}`; + const dateISO = formatAsUTCISO( + config.accessibleAt ? config.accessibleAt : new Date(), + true + ); + const queryParams: Query = { + 'X-Goog-Algorithm': 'GOOG4-RSA-SHA256', + 'X-Goog-Credential': credential, + 'X-Goog-Date': dateISO, + 'X-Goog-Expires': expiresPeriodInSeconds.toString(10), + 'X-Goog-SignedHeaders': signedHeaders, + ...(config.queryParams || {}), + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const canonicalQueryParams = this.getCanonicalQueryParams(queryParams); + + const canonicalRequest = this.getCanonicalRequest( + config.method, + this.getResourcePath(!!config.cname, config.bucket, config.file), + canonicalQueryParams, + extensionHeadersString, + signedHeaders, + contentSha256 + ); + + const hash = crypto + .createHash('sha256') + .update(canonicalRequest) + .digest('hex'); + + const blobToSign = [ + 'GOOG4-RSA-SHA256', + dateISO, + credentialScope, + hash, + ].join('\n'); + + try { + const signature = await this.auth.sign( + blobToSign, + config.signingEndpoint?.toString() + ); + const signatureHex = Buffer.from(signature, 'base64').toString('hex'); + const signedQuery: Query = Object.assign({}, queryParams, { + 'X-Goog-Signature': signatureHex, + }); + return signedQuery; + } catch (err) { + const error = err as Error; + const signingErr = new SigningError(error.message); + signingErr.stack = error.stack; + throw signingErr; + } + }; + + return sign(); + } + + /** + * Create canonical headers for signing v4 url. + * + * The canonical headers for v4-signing a request demands header names are + * first lowercased, followed by sorting the header names. + * Then, construct the canonical headers part of the request: + * + ":" + Trim() + "\n" + * .. + * + ":" + Trim() + "\n" + * + * @param headers + * @private + */ + getCanonicalHeaders(headers: http.OutgoingHttpHeaders) { + // Sort headers by their lowercased names + const sortedHeaders = objectEntries(headers) + // Convert header names to lowercase + .map<[string, HeaderValue]>(([headerName, value]) => [ + headerName.toLowerCase(), + value, + ]) + .sort((a, b) => a[0].localeCompare(b[0])); + + return sortedHeaders + .filter(([, value]) => value !== undefined) + .map(([headerName, value]) => { + // - Convert Array (multi-valued header) into string, delimited by + // ',' (no space). + // - Trim leading and trailing spaces. + // - Convert sequential (2+) spaces into a single space + const canonicalValue = `${value}`.trim().replace(/\s{2,}/g, ' '); + + return `${headerName}:${canonicalValue}\n`; + }) + .join(''); + } + + getCanonicalRequest( + method: string, + path: string, + query: string, + headers: string, + signedHeaders: string, + contentSha256?: string + ) { + return [ + method, + path, + query, + headers, + signedHeaders, + contentSha256 || 'UNSIGNED-PAYLOAD', + ].join('\n'); + } + + getCanonicalQueryParams(query: Query) { + return objectEntries(query) + .map(([key, value]) => [encodeURI(key, true), encodeURI(value, true)]) + .sort((a, b) => (a[0] < b[0] ? -1 : 1)) + .map(([key, value]) => `${key}=${value}`) + .join('&'); + } + + getResourcePath(cname: boolean, bucket: string, file?: string): string { + if (cname) { + return '/' + (file || ''); + } else if (file) { + return `/${bucket}/${file}`; + } else { + return `/${bucket}`; + } + } + + parseExpires( + expires: string | number | Date, + current: Date = new Date() + ): number { + const expiresInMSeconds = new Date(expires).valueOf(); + + if (isNaN(expiresInMSeconds)) { + throw new Error(ExceptionMessages.EXPIRATION_DATE_INVALID); + } + + if (expiresInMSeconds < current.valueOf()) { + throw new Error(ExceptionMessages.EXPIRATION_DATE_PAST); + } + + return Math.floor(expiresInMSeconds / 1000); // The API expects seconds. + } + + parseAccessibleAt(accessibleAt?: string | number | Date): number { + const accessibleAtInMSeconds = new Date( + accessibleAt || new Date() + ).valueOf(); + + if (isNaN(accessibleAtInMSeconds)) { + throw new Error(SignerExceptionMessages.ACCESSIBLE_DATE_INVALID); + } + + return Math.floor(accessibleAtInMSeconds / 1000); // The API expects seconds. + } +} + +/** + * Custom error type for errors related to getting signed errors and policies. + * + * @private + */ +export class SigningError extends Error { + name = 'SigningError'; +} diff --git a/handwritten/storage/src/storage.ts b/handwritten/storage/src/storage.ts new file mode 100644 index 00000000000..ab036e15b0e --- /dev/null +++ b/handwritten/storage/src/storage.ts @@ -0,0 +1,1634 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {ApiError, Service, ServiceOptions} from './nodejs-common/index.js'; +import {paginator} from '@google-cloud/paginator'; +import {promisifyAll} from '@google-cloud/promisify'; +import {Readable} from 'stream'; + +import {Bucket, BucketMetadata} from './bucket.js'; +import {Channel} from './channel.js'; +import {File} from './file.js'; +import {normalize} from './util.js'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import {getPackageJSON} from './package-json-helper.cjs'; +import {HmacKey, HmacKeyMetadata, HmacKeyOptions} from './hmacKey.js'; +import { + CRC32CValidatorGenerator, + CRC32C_DEFAULT_VALIDATOR_GENERATOR, +} from './crc32c.js'; +import {DEFAULT_UNIVERSE} from 'google-auth-library'; + +export interface GetServiceAccountOptions { + userProject?: string; + projectIdentifier?: string; +} +export interface ServiceAccount { + emailAddress?: string; +} +export type GetServiceAccountResponse = [ServiceAccount, unknown]; +export interface GetServiceAccountCallback { + ( + err: Error | null, + serviceAccount?: ServiceAccount, + apiResponse?: unknown + ): void; +} + +export interface CreateBucketQuery { + enableObjectRetention: boolean; + predefinedAcl?: + | 'authenticatedRead' + | 'private' + | 'projectPrivate' + | 'publicRead' + | 'publicReadWrite'; + predefinedDefaultObjectAcl?: + | 'authenticatedRead' + | 'bucketOwnerFullControl' + | 'bucketOwnerRead' + | 'private' + | 'projectPrivate' + | 'publicRead'; + project: string; + projection?: 'full' | 'noAcl'; + userProject: string; +} + +export enum IdempotencyStrategy { + RetryAlways, + RetryConditional, + RetryNever, +} + +export interface RetryOptions { + retryDelayMultiplier?: number; + totalTimeout?: number; + maxRetryDelay?: number; + autoRetry?: boolean; + maxRetries?: number; + retryableErrorFn?: (err: ApiError) => boolean; + idempotencyStrategy?: IdempotencyStrategy; +} + +export interface PreconditionOptions { + ifGenerationMatch?: number | string; + ifGenerationNotMatch?: number | string; + ifMetagenerationMatch?: number | string; + ifMetagenerationNotMatch?: number | string; +} + +export interface StorageOptions extends ServiceOptions { + /** + * The API endpoint of the service used to make requests. + * Defaults to `storage.googleapis.com`. + */ + apiEndpoint?: string; + crc32cGenerator?: CRC32CValidatorGenerator; + retryOptions?: RetryOptions; +} + +export interface BucketOptions { + crc32cGenerator?: CRC32CValidatorGenerator; + kmsKeyName?: string; + preconditionOpts?: PreconditionOptions; + userProject?: string; + generation?: number; + softDeleted?: boolean; +} + +export interface Cors { + maxAgeSeconds?: number; + method?: string[]; + origin?: string[]; + responseHeader?: string[]; +} + +interface Versioning { + enabled: boolean; +} + +/** + * Custom placement configuration. + * Initially used for dual-region buckets. + **/ +export interface CustomPlacementConfig { + dataLocations?: string[]; +} + +export interface AutoclassConfig { + enabled?: boolean; + terminalStorageClass?: 'NEARLINE' | 'ARCHIVE'; +} + +export interface CreateBucketRequest extends BucketMetadata { + archive?: boolean; + coldline?: boolean; + dataLocations?: string[]; + dra?: boolean; + enableObjectRetention?: boolean; + multiRegional?: boolean; + nearline?: boolean; + predefinedAcl?: + | 'authenticatedRead' + | 'private' + | 'projectPrivate' + | 'publicRead' + | 'publicReadWrite'; + predefinedDefaultObjectAcl?: + | 'authenticatedRead' + | 'bucketOwnerFullControl' + | 'bucketOwnerRead' + | 'private' + | 'projectPrivate' + | 'publicRead'; + projection?: 'full' | 'noAcl'; + regional?: boolean; + requesterPays?: boolean; + rpo?: string; + standard?: boolean; + storageClass?: string; + userProject?: string; + versioning?: Versioning; +} + +export type CreateBucketResponse = [Bucket, unknown]; + +export interface BucketCallback { + (err: Error | null, bucket?: Bucket | null, apiResponse?: unknown): void; +} + +export type GetBucketsResponse = [Bucket[], {}, unknown]; +export interface GetBucketsCallback { + ( + err: Error | null, + buckets: Bucket[], + nextQuery?: {}, + apiResponse?: unknown + ): void; +} +export interface GetBucketsRequest { + prefix?: string; + project?: string; + autoPaginate?: boolean; + maxApiCalls?: number; + maxResults?: number; + pageToken?: string; + userProject?: string; + softDeleted?: boolean; + generation?: number; + returnPartialSuccess?: boolean; +} + +export interface HmacKeyResourceResponse { + metadata: HmacKeyMetadata; + secret: string; +} + +export type CreateHmacKeyResponse = [HmacKey, string, HmacKeyResourceResponse]; + +export interface CreateHmacKeyOptions { + projectId?: string; + userProject?: string; +} + +export interface CreateHmacKeyCallback { + ( + err: Error | null, + hmacKey?: HmacKey | null, + secret?: string | null, + apiResponse?: HmacKeyResourceResponse + ): void; +} + +export interface GetHmacKeysOptions { + projectId?: string; + serviceAccountEmail?: string; + showDeletedKeys?: boolean; + autoPaginate?: boolean; + maxApiCalls?: number; + maxResults?: number; + pageToken?: string; + userProject?: string; +} + +export interface GetHmacKeysCallback { + ( + err: Error | null, + hmacKeys: HmacKey[] | null, + nextQuery?: {}, + apiResponse?: unknown + ): void; +} + +export enum ExceptionMessages { + EXPIRATION_DATE_INVALID = 'The expiration date provided was invalid.', + EXPIRATION_DATE_PAST = 'An expiration date cannot be in the past.', +} + +export enum StorageExceptionMessages { + BUCKET_NAME_REQUIRED = 'A bucket name is needed to use Cloud Storage.', + BUCKET_NAME_REQUIRED_CREATE = 'A name is required to create a bucket.', + HMAC_SERVICE_ACCOUNT = 'The first argument must be a service account email to create an HMAC key.', + HMAC_ACCESS_ID = 'An access ID is needed to create an HmacKey object.', +} + +export type GetHmacKeysResponse = [HmacKey[]]; + +export const PROTOCOL_REGEX = /^(\w*):\/\//; + +/** + * Default behavior: Automatically retry retriable server errors. + * + * @const {boolean} + */ +export const AUTO_RETRY_DEFAULT = true; + +/** + * Default behavior: Only attempt to retry retriable errors 3 times. + * + * @const {number} + */ +export const MAX_RETRY_DEFAULT = 3; + +/** + * Default behavior: Wait twice as long as previous retry before retrying. + * + * @const {number} + */ +export const RETRY_DELAY_MULTIPLIER_DEFAULT = 2; + +/** + * Default behavior: If the operation doesn't succeed after 600 seconds, + * stop retrying. + * + * @const {number} + */ +export const TOTAL_TIMEOUT_DEFAULT = 600; + +/** + * Default behavior: Wait no more than 64 seconds between retries. + * + * @const {number} + */ +export const MAX_RETRY_DELAY_DEFAULT = 64; + +/** + * Default behavior: Retry conditionally idempotent operations if correct preconditions are set. + * + * @const {enum} + * @private + */ +const IDEMPOTENCY_STRATEGY_DEFAULT = IdempotencyStrategy.RetryConditional; + +/** + * Returns true if the API request should be retried, given the error that was + * given the first time the request was attempted. + * @const + * @param {error} err - The API error to check if it is appropriate to retry. + * @return {boolean} True if the API request should be retried, false otherwise. + */ +export const RETRYABLE_ERR_FN_DEFAULT = function (err?: ApiError) { + const isConnectionProblem = (reason: string) => { + return ( + reason.includes('eai_again') || // DNS lookup error + reason === 'econnreset' || + reason === 'unexpected connection closure' || + reason === 'epipe' || + reason === 'socket connection timeout' + ); + }; + + if (err) { + if ([408, 429, 500, 502, 503, 504].indexOf(err.code!) !== -1) { + return true; + } + + if (typeof err.code === 'string') { + if (['408', '429', '500', '502', '503', '504'].indexOf(err.code) !== -1) { + return true; + } + const reason = (err.code as string).toLowerCase(); + if (isConnectionProblem(reason)) { + return true; + } + } + + if (err.errors) { + for (const e of err.errors) { + const reason = e?.reason?.toString().toLowerCase(); + if (reason && isConnectionProblem(reason)) { + return true; + } + } + } + } + return false; +}; + +/*! Developer Documentation + * + * Invoke this method to create a new Storage object bound with pre-determined + * configuration options. For each object that can be created (e.g., a bucket), + * there is an equivalent static and instance method. While they are classes, + * they can be instantiated without use of the `new` keyword. + */ +/** + * Cloud Storage uses access control lists (ACLs) to manage object and + * bucket access. ACLs are the mechanism you use to share objects with other + * users and allow other users to access your buckets and objects. + * + * This object provides constants to refer to the three permission levels that + * can be granted to an entity: + * + * - `gcs.acl.OWNER_ROLE` - ("OWNER") + * - `gcs.acl.READER_ROLE` - ("READER") + * - `gcs.acl.WRITER_ROLE` - ("WRITER") + * + * See {@link https://cloud.google.com/storage/docs/access-control/lists| About Access Control Lists} + * + * @name Storage#acl + * @type {object} + * @property {string} OWNER_ROLE + * @property {string} READER_ROLE + * @property {string} WRITER_ROLE + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const albums = storage.bucket('albums'); + * + * //- + * // Make all of the files currently in a bucket publicly readable. + * //- + * const options = { + * entity: 'allUsers', + * role: storage.acl.READER_ROLE + * }; + * + * albums.acl.add(options, function(err, aclObject) {}); + * + * //- + * // Make any new objects added to a bucket publicly readable. + * //- + * albums.acl.default.add(options, function(err, aclObject) {}); + * + * //- + * // Grant a user ownership permissions to a bucket. + * //- + * albums.acl.add({ + * entity: 'user-useremail@example.com', + * role: storage.acl.OWNER_ROLE + * }, function(err, aclObject) {}); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * albums.acl.add(options).then(function(data) { + * const aclObject = data[0]; + * const apiResponse = data[1]; + * }); + * ``` + */ +/** + * Get {@link Bucket} objects for all of the buckets in your project as + * a readable object stream. + * + * @method Storage#getBucketsStream + * @param {GetBucketsRequest} [query] Query object for listing buckets. + * @returns {ReadableStream} A readable stream that emits {@link Bucket} + * instances. + * + * @example + * ``` + * storage.getBucketsStream() + * .on('error', console.error) + * .on('data', function(bucket) { + * // bucket is a Bucket object. + * }) + * .on('end', function() { + * // All buckets retrieved. + * }); + * + * //- + * // If you anticipate many results, you can end a stream early to prevent + * // unnecessary processing and API requests. + * //- + * storage.getBucketsStream() + * .on('data', function(bucket) { + * this.end(); + * }); + * ``` + */ +/** + * Get {@link HmacKey} objects for all of the HMAC keys in the project in a + * readable object stream. + * + * @method Storage#getHmacKeysStream + * @param {GetHmacKeysOptions} [options] Configuration options. + * @returns {ReadableStream} A readable stream that emits {@link HmacKey} + * instances. + * + * @example + * ``` + * storage.getHmacKeysStream() + * .on('error', console.error) + * .on('data', function(hmacKey) { + * // hmacKey is an HmacKey object. + * }) + * .on('end', function() { + * // All HmacKey retrieved. + * }); + * + * //- + * // If you anticipate many results, you can end a stream early to prevent + * // unnecessary processing and API requests. + * //- + * storage.getHmacKeysStream() + * .on('data', function(bucket) { + * this.end(); + * }); + * ``` + */ +/** + *

ACLs

+ * Cloud Storage uses access control lists (ACLs) to manage object and + * bucket access. ACLs are the mechanism you use to share files with other users + * and allow other users to access your buckets and files. + * + * To learn more about ACLs, read this overview on + * {@link https://cloud.google.com/storage/docs/access-control| Access Control}. + * + * See {@link https://cloud.google.com/storage/docs/overview| Cloud Storage overview} + * See {@link https://cloud.google.com/storage/docs/access-control| Access Control} + * + * @class + */ +export class Storage extends Service { + /** + * {@link Bucket} class. + * + * @name Storage.Bucket + * @see Bucket + * @type {Constructor} + */ + static Bucket: typeof Bucket = Bucket; + + /** + * {@link Channel} class. + * + * @name Storage.Channel + * @see Channel + * @type {Constructor} + */ + static Channel: typeof Channel = Channel; + + /** + * {@link File} class. + * + * @name Storage.File + * @see File + * @type {Constructor} + */ + static File: typeof File = File; + + /** + * {@link HmacKey} class. + * + * @name Storage.HmacKey + * @see HmacKey + * @type {Constructor} + */ + static HmacKey: typeof HmacKey = HmacKey; + + static acl = { + OWNER_ROLE: 'OWNER', + READER_ROLE: 'READER', + WRITER_ROLE: 'WRITER', + }; + + /** + * Reference to {@link Storage.acl}. + * + * @name Storage#acl + * @see Storage.acl + */ + acl: typeof Storage.acl; + + crc32cGenerator: CRC32CValidatorGenerator; + + getBucketsStream(): Readable { + // placeholder body, overwritten in constructor + return new Readable(); + } + + getHmacKeysStream(): Readable { + // placeholder body, overwritten in constructor + return new Readable(); + } + + retryOptions: RetryOptions; + + /** + * @callback Crc32cGeneratorToStringCallback + * A method returning the CRC32C as a base64-encoded string. + * + * @returns {string} + * + * @example + * Hashing the string 'data' should return 'rth90Q==' + * + * ```js + * const buffer = Buffer.from('data'); + * crc32c.update(buffer); + * crc32c.toString(); // 'rth90Q==' + * ``` + **/ + /** + * @callback Crc32cGeneratorValidateCallback + * A method validating a base64-encoded CRC32C string. + * + * @param {string} [value] base64-encoded CRC32C string to validate + * @returns {boolean} + * + * @example + * Should return `true` if the value matches, `false` otherwise + * + * ```js + * const buffer = Buffer.from('data'); + * crc32c.update(buffer); + * crc32c.validate('DkjKuA=='); // false + * crc32c.validate('rth90Q=='); // true + * ``` + **/ + /** + * @callback Crc32cGeneratorUpdateCallback + * A method for passing `Buffer`s for CRC32C generation. + * + * @param {Buffer} [data] data to update CRC32C value with + * @returns {undefined} + * + * @example + * Hashing buffers from 'some ' and 'text\n' + * + * ```js + * const buffer1 = Buffer.from('some '); + * crc32c.update(buffer1); + * + * const buffer2 = Buffer.from('text\n'); + * crc32c.update(buffer2); + * + * crc32c.toString(); // 'DkjKuA==' + * ``` + **/ + /** + * @typedef {object} CRC32CValidator + * @property {Crc32cGeneratorToStringCallback} + * @property {Crc32cGeneratorValidateCallback} + * @property {Crc32cGeneratorUpdateCallback} + */ + /** + * @callback Crc32cGeneratorCallback + * @returns {CRC32CValidator} + */ + /** + * @typedef {object} StorageOptions + * @property {string} [projectId] The project ID from the Google Developer's + * Console, e.g. 'grape-spaceship-123'. We will also check the environment + * variable `GCLOUD_PROJECT` for your project ID. If your app is running + * in an environment which supports {@link + * https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application + * Application Default Credentials}, your project ID will be detected + * automatically. + * @property {string} [keyFilename] Full path to the a .json, .pem, or .p12 key + * downloaded from the Google Developers Console. If you provide a path to + * a JSON file, the `projectId` option above is not necessary. NOTE: .pem and + * .p12 require you to specify the `email` option as well. + * @property {string} [email] Account email address. Required when using a .pem + * or .p12 keyFilename. + * @property {object} [credentials] Credentials object. + * @property {string} [credentials.client_email] + * @property {string} [credentials.private_key] + * @property {object} [retryOptions] Options for customizing retries. Retriable server errors + * will be retried with exponential delay between them dictated by the formula + * max(maxRetryDelay, retryDelayMultiplier*retryNumber) until maxRetries or totalTimeout + * has been reached. Retries will only happen if autoRetry is set to true. + * @property {boolean} [retryOptions.autoRetry=true] Automatically retry requests if the + * response is related to rate limits or certain intermittent server + * errors. We will exponentially backoff subsequent requests by default. + * @property {number} [retryOptions.retryDelayMultiplier = 2] the multiplier by which to + * increase the delay time between the completion of failed requests, and the + * initiation of the subsequent retrying request. + * @property {number} [retryOptions.totalTimeout = 600] The total time, starting from + * when the initial request is sent, after which an error will + * be returned, regardless of the retrying attempts made meanwhile. + * @property {number} [retryOptions.maxRetryDelay = 64] The maximum delay time between requests. + * When this value is reached, ``retryDelayMultiplier`` will no longer be used to + * increase delay time. + * @property {number} [retryOptions.maxRetries=3] Maximum number of automatic retries + * attempted before returning the error. + * @property {function} [retryOptions.retryableErrorFn] Function that returns true if a given + * error should be retried and false otherwise. + * @property {enum} [retryOptions.idempotencyStrategy=IdempotencyStrategy.RetryConditional] Enumeration + * controls how conditionally idempotent operations are retried. Possible values are: RetryAlways - + * will respect other retry settings and attempt to retry conditionally idempotent operations. RetryConditional - + * will retry conditionally idempotent operations if the correct preconditions are set. RetryNever - never + * retry a conditionally idempotent operation. + * @property {string} [userAgent] The value to be prepended to the User-Agent + * header in API requests. + * @property {object} [authClient] `AuthClient` or `GoogleAuth` client to reuse instead of creating a new one. + * @property {number} [timeout] The amount of time in milliseconds to wait per http request before timing out. + * @property {object[]} [interceptors_] Array of custom request interceptors to be returned in the order they were assigned. + * @property {string} [apiEndpoint = storage.google.com] The API endpoint of the service used to make requests. + * @property {boolean} [useAuthWithCustomEndpoint = false] Controls whether or not to use authentication when using a custom endpoint. + * @property {Crc32cGeneratorCallback} [callback] A function that generates a CRC32C Validator. Defaults to {@link CRC32C} + */ + /** + * Constructs the Storage client. + * + * @example + * Create a client that uses Application Default Credentials + * (ADC) + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * ``` + * + * @example + * Create a client with explicit credentials + * ``` + * const storage = new Storage({ + * projectId: 'your-project-id', + * keyFilename: '/path/to/keyfile.json' + * }); + * ``` + * + * @example + * Create a client with credentials passed + * by value as a JavaScript object + * ``` + * const storage = new Storage({ + * projectId: 'your-project-id', + * credentials: { + * type: 'service_account', + * project_id: 'xxxxxxx', + * private_key_id: 'xxxx', + * private_key:'-----BEGIN PRIVATE KEY-----xxxxxxx\n-----END PRIVATE KEY-----\n', + * client_email: 'xxxx', + * client_id: 'xxx', + * auth_uri: 'https://accounts.google.com/o/oauth2/auth', + * token_uri: 'https://oauth2.googleapis.com/token', + * auth_provider_x509_cert_url: 'https://www.googleapis.com/oauth2/v1/certs', + * client_x509_cert_url: 'xxx', + * } + * }); + * ``` + * + * @example + * Create a client with credentials passed + * by loading a JSON file directly from disk + * ``` + * const storage = new Storage({ + * projectId: 'your-project-id', + * credentials: require('/path/to-keyfile.json') + * }); + * ``` + * + * @example + * Create a client with an `AuthClient` (e.g. `DownscopedClient`) + * ``` + * const {DownscopedClient} = require('google-auth-library'); + * const authClient = new DownscopedClient({...}); + * + * const storage = new Storage({authClient}); + * ``` + * + * Additional samples: + * - https://github.com/googleapis/google-auth-library-nodejs#sample-usage-1 + * - https://github.com/googleapis/google-auth-library-nodejs/blob/main/samples/downscopedclient.js + * + * @param {StorageOptions} [options] Configuration options. + */ + constructor(options: StorageOptions = {}) { + const universe = options.universeDomain || DEFAULT_UNIVERSE; + + let apiEndpoint = `https://storage.${universe}`; + let customEndpoint = false; + + // Note: EMULATOR_HOST is an experimental configuration variable. Use apiEndpoint instead. + const EMULATOR_HOST = process.env.STORAGE_EMULATOR_HOST; + if (typeof EMULATOR_HOST === 'string') { + apiEndpoint = Storage.sanitizeEndpoint(EMULATOR_HOST); + customEndpoint = true; + } + + if (options.apiEndpoint && options.apiEndpoint !== apiEndpoint) { + apiEndpoint = Storage.sanitizeEndpoint(options.apiEndpoint); + customEndpoint = true; + } + + options = Object.assign({}, options, {apiEndpoint}); + + // Note: EMULATOR_HOST is an experimental configuration variable. Use apiEndpoint instead. + const baseUrl = EMULATOR_HOST || `${options.apiEndpoint}/storage/v1`; + + const config = { + apiEndpoint: options.apiEndpoint!, + retryOptions: { + autoRetry: + options.retryOptions?.autoRetry !== undefined + ? options.retryOptions?.autoRetry + : AUTO_RETRY_DEFAULT, + maxRetries: options.retryOptions?.maxRetries + ? options.retryOptions?.maxRetries + : MAX_RETRY_DEFAULT, + retryDelayMultiplier: options.retryOptions?.retryDelayMultiplier + ? options.retryOptions?.retryDelayMultiplier + : RETRY_DELAY_MULTIPLIER_DEFAULT, + totalTimeout: options.retryOptions?.totalTimeout + ? options.retryOptions?.totalTimeout + : TOTAL_TIMEOUT_DEFAULT, + maxRetryDelay: options.retryOptions?.maxRetryDelay + ? options.retryOptions?.maxRetryDelay + : MAX_RETRY_DELAY_DEFAULT, + retryableErrorFn: options.retryOptions?.retryableErrorFn + ? options.retryOptions?.retryableErrorFn + : RETRYABLE_ERR_FN_DEFAULT, + idempotencyStrategy: + options.retryOptions?.idempotencyStrategy !== undefined + ? options.retryOptions?.idempotencyStrategy + : IDEMPOTENCY_STRATEGY_DEFAULT, + }, + baseUrl, + customEndpoint, + useAuthWithCustomEndpoint: options?.useAuthWithCustomEndpoint, + projectIdRequired: false, + scopes: [ + 'https://www.googleapis.com/auth/iam', + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/devstorage.full_control', + ], + packageJson: getPackageJSON(), + }; + + super(config, options); + + /** + * Reference to {@link Storage.acl}. + * + * @name Storage#acl + * @see Storage.acl + */ + this.acl = Storage.acl; + this.crc32cGenerator = + options.crc32cGenerator || CRC32C_DEFAULT_VALIDATOR_GENERATOR; + + this.retryOptions = config.retryOptions; + + this.getBucketsStream = paginator.streamify('getBuckets'); + this.getHmacKeysStream = paginator.streamify('getHmacKeys'); + } + + private static sanitizeEndpoint(url: string) { + if (!PROTOCOL_REGEX.test(url)) { + url = `https://${url}`; + } + return url.replace(/\/+$/, ''); // Remove trailing slashes + } + + /** + * Get a reference to a Cloud Storage bucket. + * + * @param {string} name Name of the bucket. + * @param {object} [options] Configuration object. + * @param {string} [options.kmsKeyName] A Cloud KMS key that will be used to + * encrypt objects inserted into this bucket, if no encryption method is + * specified. + * @param {string} [options.userProject] User project to be billed for all + * requests made from this Bucket object. + * @returns {Bucket} + * @see Bucket + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const albums = storage.bucket('albums'); + * const photos = storage.bucket('photos'); + * ``` + */ + bucket(name: string, options?: BucketOptions) { + if (!name) { + throw new Error(StorageExceptionMessages.BUCKET_NAME_REQUIRED); + } + return new Bucket(this, name, options); + } + + /** + * Reference a channel to receive notifications about changes to your bucket. + * + * @param {string} id The ID of the channel. + * @param {string} resourceId The resource ID of the channel. + * @returns {Channel} + * @see Channel + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const channel = storage.channel('id', 'resource-id'); + * ``` + */ + channel(id: string, resourceId: string) { + return new Channel(this, id, resourceId); + } + + createBucket( + name: string, + metadata?: CreateBucketRequest + ): Promise; + createBucket(name: string, callback: BucketCallback): void; + createBucket( + name: string, + metadata: CreateBucketRequest, + callback: BucketCallback + ): void; + createBucket( + name: string, + metadata: CreateBucketRequest, + callback: BucketCallback + ): void; + /** + * @typedef {array} CreateBucketResponse + * @property {Bucket} 0 The new {@link Bucket}. + * @property {object} 1 The full API response. + */ + /** + * @callback CreateBucketCallback + * @param {?Error} err Request error, if any. + * @param {Bucket} bucket The new {@link Bucket}. + * @param {object} apiResponse The full API response. + */ + /** + * Metadata to set for the bucket. + * + * @typedef {object} CreateBucketRequest + * @property {boolean} [archive=false] Specify the storage class as Archive. + * @property {object} [autoclass.enabled=false] Specify whether Autoclass is + * enabled for the bucket. + * @property {object} [autoclass.terminalStorageClass='NEARLINE'] The storage class that objects in an Autoclass bucket eventually transition to if + * they are not read for a certain length of time. Valid values are NEARLINE and ARCHIVE. + * @property {boolean} [coldline=false] Specify the storage class as Coldline. + * @property {Cors[]} [cors=[]] Specify the CORS configuration to use. + * @property {CustomPlacementConfig} [customPlacementConfig={}] Specify the bucket's regions for dual-region buckets. + * For more information, see {@link https://cloud.google.com/storage/docs/locations| Bucket Locations}. + * @property {boolean} [dra=false] Specify the storage class as Durable Reduced + * Availability. + * @property {boolean} [enableObjectRetention=false] Specify whether or not object retention should be enabled on this bucket. + * @property {object} [hierarchicalNamespace.enabled=false] Specify whether or not to enable hierarchical namespace on this bucket. + * @property {string} [location] Specify the bucket's location. If specifying + * a dual-region, the `customPlacementConfig` property should be set in conjunction. + * For more information, see {@link https://cloud.google.com/storage/docs/locations| Bucket Locations}. + * @property {boolean} [multiRegional=false] Specify the storage class as + * Multi-Regional. + * @property {boolean} [nearline=false] Specify the storage class as Nearline. + * @property {boolean} [regional=false] Specify the storage class as Regional. + * @property {boolean} [requesterPays=false] Force the use of the User Project metadata field to assign operational + * costs when an operation is made on a Bucket and its objects. + * @property {string} [rpo] For dual-region buckets, controls whether turbo + * replication is enabled (`ASYNC_TURBO`) or disabled (`DEFAULT`). + * @property {boolean} [standard=true] Specify the storage class as Standard. + * @property {string} [storageClass] The new storage class. (`standard`, + * `nearline`, `coldline`, or `archive`). + * **Note:** The storage classes `multi_regional`, `regional`, and + * `durable_reduced_availability` are now legacy and will be deprecated in + * the future. + * @property {Versioning} [versioning=undefined] Specify the versioning status. + * @property {string} [userProject] The ID of the project which will be billed + * for the request. + */ + /** + * Create a bucket. + * + * Cloud Storage uses a flat namespace, so you can't create a bucket with + * a name that is already in use. For more information, see + * {@link https://cloud.google.com/storage/docs/bucketnaming.html#requirements| Bucket Naming Guidelines}. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/insert| Buckets: insert API Documentation} + * See {@link https://cloud.google.com/storage/docs/storage-classes| Storage Classes} + * + * @param {string} name Name of the bucket to create. + * @param {CreateBucketRequest} [metadata] Metadata to set for the bucket. + * @param {CreateBucketCallback} [callback] Callback function. + * @returns {Promise} + * @throws {Error} If a name is not provided. + * @see Bucket#create + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const callback = function(err, bucket, apiResponse) { + * // `bucket` is a Bucket object. + * }; + * + * storage.createBucket('new-bucket', callback); + * + * //- + * // Create a bucket in a specific location and region. See the + * // Official JSON API docs for complete details on the `location` + * option. + * // + * //- + * const metadata = { + * location: 'US-CENTRAL1', + * regional: true + * }; + * + * storage.createBucket('new-bucket', metadata, callback); + * + * //- + * // Create a bucket with a retention policy of 6 months. + * //- + * const metadata = { + * retentionPolicy: { + * retentionPeriod: 15780000 // 6 months in seconds. + * } + * }; + * + * storage.createBucket('new-bucket', metadata, callback); + * + * //- + * // Enable versioning on a new bucket. + * //- + * const metadata = { + * versioning: { + * enabled: true + * } + * }; + * + * storage.createBucket('new-bucket', metadata, callback); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * storage.createBucket('new-bucket').then(function(data) { + * const bucket = data[0]; + * const apiResponse = data[1]; + * }); + * + * ``` + * @example include:samples/buckets.js + * region_tag:storage_create_bucket + * Another example: + */ + createBucket( + name: string, + metadataOrCallback?: BucketCallback | CreateBucketRequest, + callback?: BucketCallback + ): Promise | void { + if (!name) { + throw new Error(StorageExceptionMessages.BUCKET_NAME_REQUIRED_CREATE); + } + + let metadata: CreateBucketRequest; + if (!callback) { + callback = metadataOrCallback as BucketCallback; + metadata = {}; + } else { + metadata = metadataOrCallback as CreateBucketRequest; + } + + const body: CreateBucketRequest & {[index: string]: string | {} | null} = { + ...metadata, + name, + }; + + const storageClasses = { + archive: 'ARCHIVE', + coldline: 'COLDLINE', + dra: 'DURABLE_REDUCED_AVAILABILITY', + multiRegional: 'MULTI_REGIONAL', + nearline: 'NEARLINE', + regional: 'REGIONAL', + standard: 'STANDARD', + } as const; + const storageClassKeys = Object.keys( + storageClasses + ) as (keyof typeof storageClasses)[]; + + for (const storageClass of storageClassKeys) { + if (body[storageClass]) { + if (metadata.storageClass && metadata.storageClass !== storageClass) { + throw new Error( + `Both \`${storageClass}\` and \`storageClass\` were provided.` + ); + } + body.storageClass = storageClasses[storageClass]; + delete body[storageClass]; + } + } + + if (body.requesterPays) { + body.billing = { + requesterPays: body.requesterPays, + }; + delete body.requesterPays; + } + + const query = { + project: this.projectId, + } as CreateBucketQuery; + + if (body.userProject) { + query.userProject = body.userProject as string; + delete body.userProject; + } + + if (body.enableObjectRetention) { + query.enableObjectRetention = body.enableObjectRetention; + delete body.enableObjectRetention; + } + + if (body.predefinedAcl) { + query.predefinedAcl = body.predefinedAcl; + delete body.predefinedAcl; + } + + if (body.predefinedDefaultObjectAcl) { + query.predefinedDefaultObjectAcl = body.predefinedDefaultObjectAcl; + delete body.predefinedDefaultObjectAcl; + } + + if (body.projection) { + query.projection = body.projection; + delete body.projection; + } + + this.request( + { + method: 'POST', + uri: '/b', + qs: query, + json: body, + }, + (err, resp) => { + if (err) { + callback!(err, null, resp); + return; + } + + const bucket = this.bucket(name); + bucket.metadata = resp; + + callback!(null, bucket, resp); + } + ); + } + + createHmacKey( + serviceAccountEmail: string, + options?: CreateHmacKeyOptions + ): Promise; + createHmacKey( + serviceAccountEmail: string, + callback: CreateHmacKeyCallback + ): void; + createHmacKey( + serviceAccountEmail: string, + options: CreateHmacKeyOptions, + callback: CreateHmacKeyCallback + ): void; + /** + * @typedef {object} CreateHmacKeyOptions + * @property {string} [projectId] The project ID of the project that owns + * the service account of the requested HMAC key. If not provided, + * the project ID used to instantiate the Storage client will be used. + * @property {string} [userProject] This parameter is currently ignored. + */ + /** + * @typedef {object} HmacKeyMetadata + * @property {string} accessId The access id identifies which HMAC key was + * used to sign a request when authenticating with HMAC. + * @property {string} etag Used to perform a read-modify-write of the key. + * @property {string} id The resource name of the HMAC key. + * @property {string} projectId The project ID. + * @property {string} serviceAccountEmail The service account's email this + * HMAC key is created for. + * @property {string} state The state of this HMAC key. One of "ACTIVE", + * "INACTIVE" or "DELETED". + * @property {string} timeCreated The creation time of the HMAC key in + * RFC 3339 format. + * @property {string} [updated] The time this HMAC key was last updated in + * RFC 3339 format. + */ + /** + * @typedef {array} CreateHmacKeyResponse + * @property {HmacKey} 0 The HmacKey instance created from API response. + * @property {string} 1 The HMAC key's secret used to access the XML API. + * @property {object} 3 The raw API response. + */ + /** + * @callback CreateHmacKeyCallback Callback function. + * @param {?Error} err Request error, if any. + * @param {HmacKey} hmacKey The HmacKey instance created from API response. + * @param {string} secret The HMAC key's secret used to access the XML API. + * @param {object} apiResponse The raw API response. + */ + /** + * Create an HMAC key associated with an service account to authenticate + * requests to the Cloud Storage XML API. + * + * See {@link https://cloud.google.com/storage/docs/authentication/hmackeys| HMAC keys documentation} + * + * @param {string} serviceAccountEmail The service account's email address + * with which the HMAC key is created for. + * @param {CreateHmacKeyCallback} [callback] Callback function. + * @return {Promise} + * + * @example + * ``` + * const {Storage} = require('google-cloud/storage'); + * const storage = new Storage(); + * + * // Replace with your service account's email address + * const serviceAccountEmail = + * 'my-service-account@appspot.gserviceaccount.com'; + * + * storage.createHmacKey(serviceAccountEmail, function(err, hmacKey, secret) { + * if (!err) { + * // Securely store the secret for use with the XML API. + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * storage.createHmacKey(serviceAccountEmail) + * .then((response) => { + * const hmacKey = response[0]; + * const secret = response[1]; + * // Securely store the secret for use with the XML API. + * }); + * ``` + */ + createHmacKey( + serviceAccountEmail: string, + optionsOrCb?: CreateHmacKeyOptions | CreateHmacKeyCallback, + cb?: CreateHmacKeyCallback + ): Promise | void { + if (typeof serviceAccountEmail !== 'string') { + throw new Error(StorageExceptionMessages.HMAC_SERVICE_ACCOUNT); + } + + const {options, callback} = normalize< + CreateHmacKeyOptions, + CreateHmacKeyCallback + >(optionsOrCb, cb); + const query = Object.assign({}, options, {serviceAccountEmail}); + const projectId = query.projectId || this.projectId; + delete query.projectId; + + this.request( + { + method: 'POST', + uri: `/projects/${projectId}/hmacKeys`, + qs: query, + maxRetries: 0, //explicitly set this value since this is a non-idempotent function + }, + (err, resp: HmacKeyResourceResponse) => { + if (err) { + callback!(err, null, null, resp); + return; + } + + const metadata = resp.metadata; + const hmacKey = this.hmacKey(metadata.accessId!, { + projectId: metadata.projectId, + }); + hmacKey.metadata = resp.metadata; + + callback!(null, hmacKey, resp.secret, resp); + } + ); + } + + getBuckets(options?: GetBucketsRequest): Promise; + getBuckets(options: GetBucketsRequest, callback: GetBucketsCallback): void; + getBuckets(callback: GetBucketsCallback): void; + /** + * Query object for listing buckets. + * + * @typedef {object} GetBucketsRequest + * @property {boolean} [autoPaginate=true] Have pagination handled + * automatically. + * @property {number} [maxApiCalls] Maximum number of API calls to make. + * @property {number} [maxResults] Maximum number of items plus prefixes to + * return per call. + * Note: By default will handle pagination automatically + * if more than 1 page worth of results are requested per call. + * When `autoPaginate` is set to `false` the smaller of `maxResults` + * or 1 page of results will be returned per call. + * @property {string} [pageToken] A previously-returned page token + * representing part of the larger set of results to view. + * @property {string} [userProject] The ID of the project which will be billed + * for the request. + * @param {boolean} [softDeleted] If true, returns the soft-deleted object. + * Object `generation` is required if `softDeleted` is set to True. + */ + /** + * @typedef {array} GetBucketsResponse + * @property {Bucket[]} 0 Array of {@link Bucket} instances. + * @property {object} 1 nextQuery A query object to receive more results. + * @property {object} 2 The full API response. + */ + /** + * @callback GetBucketsCallback + * @param {?Error} err Request error, if any. + * @param {Bucket[]} buckets Array of {@link Bucket} instances. + * @param {object} nextQuery A query object to receive more results. + * @param {object} apiResponse The full API response. + */ + /** + * Get Bucket objects for all of the buckets in your project. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/buckets/list| Buckets: list API Documentation} + * + * @param {GetBucketsRequest} [query] Query object for listing buckets. + * @param {GetBucketsCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * storage.getBuckets(function(err, buckets) { + * if (!err) { + * // buckets is an array of Bucket objects. + * } + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- + * const callback = function(err, buckets, nextQuery, apiResponse) { + * if (nextQuery) { + * // More results exist. + * storage.getBuckets(nextQuery, callback); + * } + * + * // The `metadata` property is populated for you with the metadata at the + * // time of fetching. + * buckets[0].metadata; + * + * // However, in cases where you are concerned the metadata could have + * // changed, use the `getMetadata` method. + * buckets[0].getMetadata(function(err, metadata, apiResponse) {}); + * }; + * + * storage.getBuckets({ + * autoPaginate: false + * }, callback); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * storage.getBuckets().then(function(data) { + * const buckets = data[0]; + * }); + * + * ``` + * @example include:samples/buckets.js + * region_tag:storage_list_buckets + * Another example: + */ + getBuckets( + optionsOrCallback?: GetBucketsRequest | GetBucketsCallback, + cb?: GetBucketsCallback + ): void | Promise { + const {options, callback} = normalize( + optionsOrCallback, + cb + ); + options.project = options.project || this.projectId; + + this.request( + { + uri: '/b', + qs: options, + }, + (err, resp) => { + if (err) { + callback(err, null, null, resp); + return; + } + + const itemsArray = resp.items ? resp.items : []; + const unreachableArray = resp.unreachable ? resp.unreachable : []; + + const buckets = itemsArray.map((bucket: BucketMetadata) => { + const bucketInstance = this.bucket(bucket.id!); + bucketInstance.metadata = bucket; + + return bucketInstance; + }); + + if (unreachableArray.length > 0) { + unreachableArray.forEach((fullPath: string) => { + const name = fullPath.split('/').pop(); + if (name) { + const placeholder = this.bucket(name); + placeholder.unreachable = true; + placeholder.metadata = {}; + buckets.push(placeholder); + } + }); + } + + const nextQuery = resp.nextPageToken + ? Object.assign({}, options, {pageToken: resp.nextPageToken}) + : null; + + callback(null, buckets, nextQuery, resp); + } + ); + } + + /** + * Query object for listing HMAC keys. + * + * @typedef {object} GetHmacKeysOptions + * @property {string} [projectId] The project ID of the project that owns + * the service account of the requested HMAC key. If not provided, + * the project ID used to instantiate the Storage client will be used. + * @property {string} [serviceAccountEmail] If present, only HMAC keys for the + * given service account are returned. + * @property {boolean} [showDeletedKeys=false] If true, include keys in the DELETE + * state. Default is false. + * @property {boolean} [autoPaginate=true] Have pagination handled + * automatically. + * @property {number} [maxApiCalls] Maximum number of API calls to make. + * @property {number} [maxResults] Maximum number of items plus prefixes to + * return per call. + * Note: By default will handle pagination automatically + * if more than 1 page worth of results are requested per call. + * When `autoPaginate` is set to `false` the smaller of `maxResults` + * or 1 page of results will be returned per call. + * @property {string} [pageToken] A previously-returned page token + * representing part of the larger set of results to view. + * @property {string} [userProject] This parameter is currently ignored. + */ + /** + * @typedef {array} GetHmacKeysResponse + * @property {HmacKey[]} 0 Array of {@link HmacKey} instances. + * @param {object} nextQuery 1 A query object to receive more results. + * @param {object} apiResponse 2 The full API response. + */ + /** + * @callback GetHmacKeysCallback + * @param {?Error} err Request error, if any. + * @param {HmacKey[]} hmacKeys Array of {@link HmacKey} instances. + * @param {object} nextQuery A query object to receive more results. + * @param {object} apiResponse The full API response. + */ + /** + * Retrieves a list of HMAC keys matching the criteria. + * + * The authenticated user must have storage.hmacKeys.list permission for the project in which the key exists. + * + * @param {GetHmacKeysOption} options Configuration options. + * @param {GetHmacKeysCallback} callback Callback function. + * @return {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * storage.getHmacKeys(function(err, hmacKeys) { + * if (!err) { + * // hmacKeys is an array of HmacKey objects. + * } + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- + * const callback = function(err, hmacKeys, nextQuery, apiResponse) { + * if (nextQuery) { + * // More results exist. + * storage.getHmacKeys(nextQuery, callback); + * } + * + * // The `metadata` property is populated for you with the metadata at the + * // time of fetching. + * hmacKeys[0].metadata; + * }; + * + * storage.getHmacKeys({ + * autoPaginate: false + * }, callback); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * storage.getHmacKeys().then(function(data) { + * const hmacKeys = data[0]; + * }); + * ``` + */ + getHmacKeys(options?: GetHmacKeysOptions): Promise; + getHmacKeys(callback: GetHmacKeysCallback): void; + getHmacKeys(options: GetHmacKeysOptions, callback: GetHmacKeysCallback): void; + getHmacKeys( + optionsOrCb?: GetHmacKeysOptions | GetHmacKeysCallback, + cb?: GetHmacKeysCallback + ): Promise | void { + const {options, callback} = normalize(optionsOrCb, cb); + const query = Object.assign({}, options); + const projectId = query.projectId || this.projectId; + delete query.projectId; + + this.request( + { + uri: `/projects/${projectId}/hmacKeys`, + qs: query, + }, + (err, resp) => { + if (err) { + callback(err, null, null, resp); + return; + } + + const itemsArray = resp.items ? resp.items : []; + const hmacKeys = itemsArray.map((hmacKey: HmacKeyMetadata) => { + const hmacKeyInstance = this.hmacKey(hmacKey.accessId!, { + projectId: hmacKey.projectId, + }); + hmacKeyInstance.metadata = hmacKey; + return hmacKeyInstance; + }); + + const nextQuery = resp.nextPageToken + ? Object.assign({}, options, {pageToken: resp.nextPageToken}) + : null; + + callback(null, hmacKeys, nextQuery, resp); + } + ); + } + + getServiceAccount( + options?: GetServiceAccountOptions + ): Promise; + getServiceAccount( + options?: GetServiceAccountOptions + ): Promise; + getServiceAccount( + options: GetServiceAccountOptions, + callback: GetServiceAccountCallback + ): void; + getServiceAccount(callback: GetServiceAccountCallback): void; + /** + * @typedef {array} GetServiceAccountResponse + * @property {object} 0 The service account resource. + * @property {object} 1 The full + * {@link https://cloud.google.com/storage/docs/json_api/v1/projects/serviceAccount#resource| API response}. + */ + /** + * @callback GetServiceAccountCallback + * @param {?Error} err Request error, if any. + * @param {object} serviceAccount The serviceAccount resource. + * @param {string} serviceAccount.emailAddress The service account email + * address. + * @param {object} apiResponse The full + * {@link https://cloud.google.com/storage/docs/json_api/v1/projects/serviceAccount#resource| API response}. + */ + /** + * Get the email address of this project's Google Cloud Storage service + * account. + * + * See {@link https://cloud.google.com/storage/docs/json_api/v1/projects/serviceAccount/get| Projects.serviceAccount: get API Documentation} + * See {@link https://cloud.google.com/storage/docs/json_api/v1/projects/serviceAccount#resource| Projects.serviceAccount Resource} + * + * @param {object} [options] Configuration object. + * @param {string} [options.userProject] User project to be billed for this + * request. + * @param {GetServiceAccountCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * + * storage.getServiceAccount(function(err, serviceAccount, apiResponse) { + * if (!err) { + * const serviceAccountEmail = serviceAccount.emailAddress; + * } + * }); + * + * //- + * // If the callback is omitted, we'll return a Promise. + * //- + * storage.getServiceAccount().then(function(data) { + * const serviceAccountEmail = data[0].emailAddress; + * const apiResponse = data[1]; + * }); + * ``` + */ + getServiceAccount( + optionsOrCallback?: GetServiceAccountOptions | GetServiceAccountCallback, + cb?: GetServiceAccountCallback + ): void | Promise { + const {options, callback} = normalize( + optionsOrCallback, + cb + ); + this.request( + { + uri: `/projects/${this.projectId}/serviceAccount`, + qs: options, + }, + (err, resp) => { + if (err) { + callback(err, null, resp); + return; + } + + const camelCaseResponse = {} as {[index: string]: string}; + + for (const prop in resp) { + // eslint-disable-next-line no-prototype-builtins + if (resp.hasOwnProperty(prop)) { + const camelCaseProp = prop.replace(/_(\w)/g, (_, match) => + match.toUpperCase() + ); + camelCaseResponse[camelCaseProp] = resp[prop]; + } + } + + callback(null, camelCaseResponse, resp); + } + ); + } + + /** + * Get a reference to an HmacKey object. + * Note: this does not fetch the HMAC key's metadata. Use HmacKey#get() to + * retrieve and populate the metadata. + * + * To get a reference to an HMAC key that's not created for a service + * account in the same project used to instantiate the Storage client, + * supply the project's ID as `projectId` in the `options` argument. + * + * @param {string} accessId The HMAC key's access ID. + * @param {HmacKeyOptions} options HmacKey constructor options. + * @returns {HmacKey} + * @see HmacKey + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const hmacKey = storage.hmacKey('ACCESS_ID'); + * ``` + */ + hmacKey(accessId: string, options?: HmacKeyOptions) { + if (!accessId) { + throw new Error(StorageExceptionMessages.HMAC_ACCESS_ID); + } + + return new HmacKey(this, accessId, options); + } +} + +/*! Developer Documentation + * + * These methods can be auto-paginated. + */ +paginator.extend(Storage, ['getBuckets', 'getHmacKeys']); + +/*! Developer Documentation + * + * All async methods (except for streams) will return a Promise in the event + * that a callback is omitted. + */ +promisifyAll(Storage, { + exclude: ['bucket', 'channel', 'hmacKey'], +}); diff --git a/handwritten/storage/src/transfer-manager.ts b/handwritten/storage/src/transfer-manager.ts new file mode 100644 index 00000000000..dd4e41eeb31 --- /dev/null +++ b/handwritten/storage/src/transfer-manager.ts @@ -0,0 +1,870 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {Bucket, UploadOptions, UploadResponse} from './bucket.js'; +import { + DownloadOptions, + DownloadResponse, + File, + FileExceptionMessages, + RequestError, +} from './file.js'; +import pLimit from 'p-limit'; +import * as path from 'path'; +import {createReadStream, existsSync, promises as fsp} from 'fs'; +import {CRC32C} from './crc32c.js'; +import {GoogleAuth} from 'google-auth-library'; +import {XMLParser, XMLBuilder} from 'fast-xml-parser'; +import AsyncRetry from 'async-retry'; +import {ApiError} from './nodejs-common/index.js'; +import {GaxiosResponse, Headers} from 'gaxios'; +import {createHash} from 'crypto'; +import {GCCL_GCS_CMD_KEY} from './nodejs-common/util.js'; +import {getRuntimeTrackingString, getUserAgentString} from './util.js'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import {getPackageJSON} from './package-json-helper.cjs'; + +const packageJson = getPackageJSON(); + +/** + * Default number of concurrently executing promises to use when calling uploadManyFiles. + * + */ +const DEFAULT_PARALLEL_UPLOAD_LIMIT = 5; +/** + * Default number of concurrently executing promises to use when calling downloadManyFiles. + * + */ +const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 5; +/** + * Default number of concurrently executing promises to use when calling downloadFileInChunks. + * + */ +const DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT = 5; +/** + * The minimum size threshold in bytes at which to apply a chunked download strategy when calling downloadFileInChunks. + * + */ +const DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD = 32 * 1024 * 1024; +/** + * The chunk size in bytes to use when calling downloadFileInChunks. + * + */ +const DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024; +/** + * The chunk size in bytes to use when calling uploadFileInChunks. + * + */ +const UPLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024; +/** + * Default number of concurrently executing promises to use when calling uploadFileInChunks. + * + */ +const DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT = 5; + +const EMPTY_REGEX = '(?:)'; + +/** + * The `gccl-gcs-cmd` value for the `X-Goog-API-Client` header. + * Example: `gccl-gcs-cmd/tm.upload_many` + * + * @see {@link GCCL_GCS_CMD}. + * @see {@link GCCL_GCS_CMD_KEY}. + */ +const GCCL_GCS_CMD_FEATURE = { + UPLOAD_MANY: 'tm.upload_many', + DOWNLOAD_MANY: 'tm.download_many', + UPLOAD_SHARDED: 'tm.upload_sharded', + DOWNLOAD_SHARDED: 'tm.download_sharded', +}; + +export interface UploadManyFilesOptions { + concurrencyLimit?: number; + customDestinationBuilder?( + path: string, + options: UploadManyFilesOptions + ): string; + skipIfExists?: boolean; + prefix?: string; + passthroughOptions?: Omit; +} + +export interface DownloadManyFilesOptions { + concurrencyLimit?: number; + prefix?: string; + stripPrefix?: string; + passthroughOptions?: DownloadOptions; + skipIfExists?: boolean; +} + +export interface DownloadFileInChunksOptions { + concurrencyLimit?: number; + chunkSizeBytes?: number; + destination?: string; + validation?: 'crc32c' | false; + noReturnData?: boolean; +} + +export interface UploadFileInChunksOptions { + concurrencyLimit?: number; + chunkSizeBytes?: number; + uploadName?: string; + maxQueueSize?: number; + uploadId?: string; + autoAbortFailure?: boolean; + partsMap?: Map; + validation?: 'md5' | false; + headers?: {[key: string]: string}; +} + +export interface MultiPartUploadHelper { + bucket: Bucket; + fileName: string; + uploadId?: string; + partsMap?: Map; + initiateUpload(headers?: {[key: string]: string}): Promise; + uploadPart( + partNumber: number, + chunk: Buffer, + validation?: 'md5' | false + ): Promise; + completeUpload(): Promise; + abortUpload(): Promise; +} + +export type MultiPartHelperGenerator = ( + bucket: Bucket, + fileName: string, + uploadId?: string, + partsMap?: Map +) => MultiPartUploadHelper; + +const defaultMultiPartGenerator: MultiPartHelperGenerator = ( + bucket, + fileName, + uploadId, + partsMap +) => { + return new XMLMultiPartUploadHelper(bucket, fileName, uploadId, partsMap); +}; + +export class MultiPartUploadError extends Error { + private uploadId: string; + private partsMap: Map; + + constructor( + message: string, + uploadId: string, + partsMap: Map + ) { + super(message); + this.uploadId = uploadId; + this.partsMap = partsMap; + } +} +/** + * Class representing an implementation of MPU in the XML API. This class is not meant for public usage. + * + * @private + * + */ +class XMLMultiPartUploadHelper implements MultiPartUploadHelper { + public partsMap; + public uploadId; + public bucket; + public fileName; + + private authClient; + private xmlParser; + private xmlBuilder; + private baseUrl; + private retryOptions; + + constructor( + bucket: Bucket, + fileName: string, + uploadId?: string, + partsMap?: Map + ) { + this.authClient = bucket.storage.authClient || new GoogleAuth(); + this.uploadId = uploadId || ''; + this.bucket = bucket; + this.fileName = fileName; + this.baseUrl = `https://${bucket.name}.${ + new URL(this.bucket.storage.apiEndpoint).hostname + }/${fileName}`; + this.xmlBuilder = new XMLBuilder({arrayNodeName: 'Part'}); + this.xmlParser = new XMLParser(); + this.partsMap = partsMap || new Map(); + this.retryOptions = { + retries: this.bucket.storage.retryOptions.maxRetries, + factor: this.bucket.storage.retryOptions.retryDelayMultiplier, + maxTimeout: this.bucket.storage.retryOptions.maxRetryDelay! * 1000, + maxRetryTime: this.bucket.storage.retryOptions.totalTimeout! * 1000, + }; + } + + #setGoogApiClientHeaders(headers: Headers = {}): Headers { + let headerFound = false; + let userAgentFound = false; + + for (const [key, value] of Object.entries(headers)) { + if (key.toLocaleLowerCase().trim() === 'x-goog-api-client') { + headerFound = true; + + // Prepend command feature to value, if not already there + if (!value.includes(GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED)) { + headers[key] = + `${value} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`; + } + } else if (key.toLocaleLowerCase().trim() === 'user-agent') { + userAgentFound = true; + } + } + + // If the header isn't present, add it + if (!headerFound) { + headers['x-goog-api-client'] = `${getRuntimeTrackingString()} gccl/${ + packageJson.version + } gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`; + } + + // If the User-Agent isn't present, add it + if (!userAgentFound) { + headers['User-Agent'] = getUserAgentString(); + } + + return headers; + } + + /** + * Initiates a multipart upload (MPU) to the XML API and stores the resultant upload id. + * + * @returns {Promise} + */ + async initiateUpload(headers: Headers = {}): Promise { + const url = `${this.baseUrl}?uploads`; + return AsyncRetry(async bail => { + try { + const res = await this.authClient.request({ + headers: this.#setGoogApiClientHeaders(headers), + method: 'POST', + url, + }); + + if (res.data && res.data.error) { + throw res.data.error; + } + const parsedXML = this.xmlParser.parse(res.data); + this.uploadId = parsedXML.InitiateMultipartUploadResult.UploadId; + } catch (e) { + this.#handleErrorResponse(e as Error, bail); + } + }, this.retryOptions); + } + + /** + * Uploads the provided chunk of data to the XML API using the previously created upload id. + * + * @param {number} partNumber the sequence number of this chunk. + * @param {Buffer} chunk the chunk of data to be uploaded. + * @param {string | false} validation whether or not to include the md5 hash in the headers to cause the server + * to validate the chunk was not corrupted. + * @returns {Promise} + */ + async uploadPart( + partNumber: number, + chunk: Buffer, + validation?: 'md5' | false + ): Promise { + const url = `${this.baseUrl}?partNumber=${partNumber}&uploadId=${this.uploadId}`; + let headers: Headers = this.#setGoogApiClientHeaders(); + + if (validation === 'md5') { + const hash = createHash('md5').update(chunk).digest('base64'); + headers = { + 'Content-MD5': hash, + }; + } + + return AsyncRetry(async bail => { + try { + const res = await this.authClient.request({ + url, + method: 'PUT', + body: chunk, + headers, + }); + if (res.data && res.data.error) { + throw res.data.error; + } + this.partsMap.set(partNumber, res.headers['etag']); + } catch (e) { + this.#handleErrorResponse(e as Error, bail); + } + }, this.retryOptions); + } + + /** + * Sends the final request of the MPU to tell GCS the upload is now complete. + * + * @returns {Promise} + */ + async completeUpload(): Promise { + const url = `${this.baseUrl}?uploadId=${this.uploadId}`; + const sortedMap = new Map( + [...this.partsMap.entries()].sort((a, b) => a[0] - b[0]) + ); + const parts: {}[] = []; + for (const entry of sortedMap.entries()) { + parts.push({PartNumber: entry[0], ETag: entry[1]}); + } + const body = `${this.xmlBuilder.build( + parts + )}`; + return AsyncRetry(async bail => { + try { + const res = await this.authClient.request({ + headers: this.#setGoogApiClientHeaders(), + url, + method: 'POST', + body, + }); + if (res.data && res.data.error) { + throw res.data.error; + } + return res; + } catch (e) { + this.#handleErrorResponse(e as Error, bail); + return; + } + }, this.retryOptions); + } + + /** + * Aborts an multipart upload that is in progress. Once aborted, any parts in the process of being uploaded fail, + * and future requests using the upload ID fail. + * + * @returns {Promise} + */ + async abortUpload(): Promise { + const url = `${this.baseUrl}?uploadId=${this.uploadId}`; + return AsyncRetry(async bail => { + try { + const res = await this.authClient.request({ + url, + method: 'DELETE', + }); + if (res.data && res.data.error) { + throw res.data.error; + } + } catch (e) { + this.#handleErrorResponse(e as Error, bail); + return; + } + }, this.retryOptions); + } + + /** + * Handles error responses and calls the bail function if the error should not be retried. + * + * @param {Error} err the thrown error + * @param {Function} bail if the error can not be retried, the function to be called. + */ + #handleErrorResponse(err: Error, bail: Function) { + if ( + this.bucket.storage.retryOptions.autoRetry && + this.bucket.storage.retryOptions.retryableErrorFn!(err as ApiError) + ) { + throw err; + } else { + bail(err as Error); + } + } +} + +/** + * Create a TransferManager object to perform parallel transfer operations on a Cloud Storage bucket. + * + * @class + * @hideconstructor + * + * @param {Bucket} bucket A {@link Bucket} instance + * + */ +export class TransferManager { + bucket: Bucket; + constructor(bucket: Bucket) { + this.bucket = bucket; + } + + /** + * @typedef {object} UploadManyFilesOptions + * @property {number} [concurrencyLimit] The number of concurrently executing promises + * to use when uploading the files. + * @property {Function} [customDestinationBuilder] A function that will take the current path of a local file + * and return a string representing a custom path to be used to upload the file to GCS. + * @property {boolean} [skipIfExists] Do not upload the file if it already exists in + * the bucket. This will set the precondition ifGenerationMatch = 0. + * @property {string} [prefix] A prefix to append to all of the uploaded files. + * @property {object} [passthroughOptions] {@link UploadOptions} Options to be passed through + * to each individual upload operation. + * + */ + /** + * Upload multiple files in parallel to the bucket. This is a convenience method + * that utilizes {@link Bucket#upload} to perform the upload. + * + * @param {array | string} [filePathsOrDirectory] An array of fully qualified paths to the files or a directory name. + * If a directory name is provided, the directory will be recursively walked and all files will be added to the upload list. + * to be uploaded to the bucket + * @param {UploadManyFilesOptions} [options] Configuration options. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * const transferManager = new TransferManager(bucket); + * + * //- + * // Upload multiple files in parallel. + * //- + * const response = await transferManager.uploadManyFiles(['/local/path/file1.txt, 'local/path/file2.txt']); + * // Your bucket now contains: + * // - "local/path/file1.txt" (with the contents of '/local/path/file1.txt') + * // - "local/path/file2.txt" (with the contents of '/local/path/file2.txt') + * const response = await transferManager.uploadManyFiles('/local/directory'); + * // Your bucket will now contain all files contained in '/local/directory' maintaining the subdirectory structure. + * ``` + * + */ + async uploadManyFiles( + filePathsOrDirectory: string[] | string, + options: UploadManyFilesOptions = {} + ): Promise { + if (options.skipIfExists && options.passthroughOptions?.preconditionOpts) { + options.passthroughOptions.preconditionOpts.ifGenerationMatch = 0; + } else if ( + options.skipIfExists && + options.passthroughOptions === undefined + ) { + options.passthroughOptions = { + preconditionOpts: { + ifGenerationMatch: 0, + }, + }; + } + + const limit = pLimit( + options.concurrencyLimit || DEFAULT_PARALLEL_UPLOAD_LIMIT + ); + const promises: Promise[] = []; + let allPaths: string[] = []; + if (!Array.isArray(filePathsOrDirectory)) { + for await (const curPath of this.getPathsFromDirectory( + filePathsOrDirectory + )) { + allPaths.push(curPath); + } + } else { + allPaths = filePathsOrDirectory; + } + + for (const filePath of allPaths) { + const stat = await fsp.lstat(filePath); + if (stat.isDirectory()) { + continue; + } + + const passThroughOptionsCopy: UploadOptions = { + ...options.passthroughOptions, + [GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.UPLOAD_MANY, + }; + + passThroughOptionsCopy.destination = options.customDestinationBuilder + ? options.customDestinationBuilder(filePath, options) + : filePath.split(path.sep).join(path.posix.sep); + if (options.prefix) { + passThroughOptionsCopy.destination = path.posix.join( + ...options.prefix.split(path.sep), + passThroughOptionsCopy.destination + ); + } + + promises.push( + limit(() => + this.bucket.upload(filePath, passThroughOptionsCopy as UploadOptions) + ) + ); + } + + return Promise.all(promises); + } + + /** + * @typedef {object} DownloadManyFilesOptions + * @property {number} [concurrencyLimit] The number of concurrently executing promises + * to use when downloading the files. + * @property {string} [prefix] A prefix to append to all of the downloaded files. + * @property {string} [stripPrefix] A prefix to remove from all of the downloaded files. + * @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through + * to each individual download operation. + * @property {boolean} [skipIfExists] Do not download the file if it already exists in + * the destination. + * + */ + /** + * Download multiple files in parallel to the local filesystem. This is a convenience method + * that utilizes {@link File#download} to perform the download. + * + * @param {array | string} [filesOrFolder] An array of file name strings or file objects to be downloaded. If + * a string is provided this will be treated as a GCS prefix and all files with that prefix will be downloaded. + * @param {DownloadManyFilesOptions} [options] Configuration options. Setting options.prefix or options.stripPrefix + * or options.passthroughOptions.destination will cause the downloaded files to be written to the file system + * instead of being returned as a buffer. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * const transferManager = new TransferManager(bucket); + * + * //- + * // Download multiple files in parallel. + * //- + * const response = await transferManager.downloadManyFiles(['file1.txt', 'file2.txt']); + * // The following files have been downloaded: + * // - "file1.txt" (with the contents from my-bucket.file1.txt) + * // - "file2.txt" (with the contents from my-bucket.file2.txt) + * const response = await transferManager.downloadManyFiles([bucket.File('file1.txt'), bucket.File('file2.txt')]); + * // The following files have been downloaded: + * // - "file1.txt" (with the contents from my-bucket.file1.txt) + * // - "file2.txt" (with the contents from my-bucket.file2.txt) + * const response = await transferManager.downloadManyFiles('test-folder'); + * // All files with GCS prefix of 'test-folder' have been downloaded. + * ``` + * + */ + async downloadManyFiles( + filesOrFolder: File[] | string[] | string, + options: DownloadManyFilesOptions = {} + ): Promise { + const limit = pLimit( + options.concurrencyLimit || DEFAULT_PARALLEL_DOWNLOAD_LIMIT + ); + const promises: Promise[] = []; + let files: File[] = []; + + if (!Array.isArray(filesOrFolder)) { + const directoryFiles = await this.bucket.getFiles({ + prefix: filesOrFolder, + }); + files = directoryFiles[0]; + } else { + files = filesOrFolder.map(curFile => { + if (typeof curFile === 'string') { + return this.bucket.file(curFile); + } + return curFile; + }); + } + + const stripRegexString = options.stripPrefix + ? `^${options.stripPrefix}` + : EMPTY_REGEX; + const regex = new RegExp(stripRegexString, 'g'); + + for (const file of files) { + const passThroughOptionsCopy = { + ...options.passthroughOptions, + [GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_MANY, + }; + + if (options.prefix || passThroughOptionsCopy.destination) { + passThroughOptionsCopy.destination = path.join( + options.prefix || '', + passThroughOptionsCopy.destination || '', + file.name + ); + } + if (options.stripPrefix) { + passThroughOptionsCopy.destination = file.name.replace(regex, ''); + } + if ( + options.skipIfExists && + existsSync(passThroughOptionsCopy.destination || '') + ) { + continue; + } + + promises.push( + limit(async () => { + const destination = passThroughOptionsCopy.destination; + if (destination && destination.endsWith(path.sep)) { + await fsp.mkdir(destination, {recursive: true}); + return Promise.resolve([ + Buffer.alloc(0), + ]) as Promise; + } + + return file.download(passThroughOptionsCopy); + }) + ); + } + + return Promise.all(promises); + } + + /** + * @typedef {object} DownloadFileInChunksOptions + * @property {number} [concurrencyLimit] The number of concurrently executing promises + * to use when downloading the file. + * @property {number} [chunkSizeBytes] The size in bytes of each chunk to be downloaded. + * @property {string | boolean} [validation] Whether or not to perform a CRC32C validation check when download is complete. + * @property {boolean} [noReturnData] Whether or not to return the downloaded data. A `true` value here would be useful for files with a size that will not fit into memory. + * + */ + /** + * Download a large file in chunks utilizing parallel download operations. This is a convenience method + * that utilizes {@link File#download} to perform the download. + * + * @param {File | string} fileOrName {@link File} to download. + * @param {DownloadFileInChunksOptions} [options] Configuration options. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * const transferManager = new TransferManager(bucket); + * + * //- + * // Download a large file in chunks utilizing parallel operations. + * //- + * const response = await transferManager.downloadFileInChunks(bucket.file('large-file.txt'); + * // Your local directory now contains: + * // - "large-file.txt" (with the contents from my-bucket.large-file.txt) + * ``` + * + */ + async downloadFileInChunks( + fileOrName: File | string, + options: DownloadFileInChunksOptions = {} + ): Promise { + let chunkSize = + options.chunkSizeBytes || DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE; + let limit = pLimit( + options.concurrencyLimit || DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT + ); + const noReturnData = Boolean(options.noReturnData); + const promises: Promise[] = []; + const file: File = + typeof fileOrName === 'string' + ? this.bucket.file(fileOrName) + : fileOrName; + + const fileInfo = await file.get(); + const size = parseInt(fileInfo[0].metadata.size!.toString()); + // If the file size does not meet the threshold download it as a single chunk. + if (size < DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD) { + limit = pLimit(1); + chunkSize = size; + } + + let start = 0; + const filePath = options.destination || path.basename(file.name); + const fileToWrite = await fsp.open(filePath, 'w'); + while (start < size) { + const chunkStart = start; + let chunkEnd = start + chunkSize - 1; + chunkEnd = chunkEnd > size ? size : chunkEnd; + promises.push( + limit(async () => { + const resp = await file.download({ + start: chunkStart, + end: chunkEnd, + [GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_SHARDED, + }); + const result = await fileToWrite.write( + resp[0], + 0, + resp[0].length, + chunkStart + ); + if (noReturnData) return; + return result.buffer; + }) + ); + + start += chunkSize; + } + + let chunks: Array; + try { + chunks = await Promise.all(promises); + } finally { + await fileToWrite.close(); + } + + if (options.validation === 'crc32c' && fileInfo[0].metadata.crc32c) { + const downloadedCrc32C = await CRC32C.fromFile(filePath); + if (!downloadedCrc32C.validate(fileInfo[0].metadata.crc32c)) { + const mismatchError = new RequestError( + FileExceptionMessages.DOWNLOAD_MISMATCH + ); + mismatchError.code = 'CONTENT_DOWNLOAD_MISMATCH'; + throw mismatchError; + } + } + if (noReturnData) return; + return [Buffer.concat(chunks as Buffer[], size)]; + } + + /** + * @typedef {object} UploadFileInChunksOptions + * @property {number} [concurrencyLimit] The number of concurrently executing promises + * to use when uploading the file. + * @property {number} [chunkSizeBytes] The size in bytes of each chunk to be uploaded. + * @property {string} [uploadName] Name of the file when saving to GCS. If omitted the name is taken from the file path. + * @property {number} [maxQueueSize] The number of chunks to be uploaded to hold in memory concurrently. If not specified + * defaults to the specified concurrency limit. + * @property {string} [uploadId] If specified attempts to resume a previous upload. + * @property {Map} [partsMap] If specified alongside uploadId, attempts to resume a previous upload from the last chunk + * specified in partsMap + * @property {object} [headers] headers to be sent when initiating the multipart upload. + * See {@link https://cloud.google.com/storage/docs/xml-api/post-object-multipart#request_headers| Request Headers: Initiate a Multipart Upload} + * @property {boolean} [autoAbortFailure] boolean to indicate if an in progress upload session will be automatically aborted upon failure. If not set, + * failures will be automatically aborted. + * + */ + /** + * Upload a large file in chunks utilizing parallel upload operations. If the upload fails, an uploadId and + * map containing all the successfully uploaded parts will be returned to the caller. These arguments can be used to + * resume the upload. + * + * @param {string} [filePath] The path of the file to be uploaded + * @param {UploadFileInChunksOptions} [options] Configuration options. + * @param {MultiPartHelperGenerator} [generator] A function that will return a type that implements the MPU interface. Most users will not need to use this. + * @returns {Promise} If successful a promise resolving to void, otherwise a error containing the message, uploadId, and parts map. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * const transferManager = new TransferManager(bucket); + * + * //- + * // Upload a large file in chunks utilizing parallel operations. + * //- + * const response = await transferManager.uploadFileInChunks('large-file.txt'); + * // Your bucket now contains: + * // - "large-file.txt" + * ``` + * + * + */ + async uploadFileInChunks( + filePath: string, + options: UploadFileInChunksOptions = {}, + generator: MultiPartHelperGenerator = defaultMultiPartGenerator + ): Promise { + const chunkSize = + options.chunkSizeBytes || UPLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE; + const limit = pLimit( + options.concurrencyLimit || DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT + ); + const maxQueueSize = + options.maxQueueSize || + options.concurrencyLimit || + DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT; + const fileName = options.uploadName || path.basename(filePath); + const mpuHelper = generator( + this.bucket, + fileName, + options.uploadId, + options.partsMap + ); + let partNumber = 1; + let promises: Promise[] = []; + try { + if (options.uploadId === undefined) { + await mpuHelper.initiateUpload(options.headers); + } + const startOrResumptionByte = mpuHelper.partsMap!.size * chunkSize; + const readStream = createReadStream(filePath, { + highWaterMark: chunkSize, + start: startOrResumptionByte, + }); + // p-limit only limits the number of running promises. We do not want to hold an entire + // large file in memory at once so promises acts a queue that will hold only maxQueueSize in memory. + for await (const curChunk of readStream) { + if (promises.length >= maxQueueSize) { + await Promise.all(promises); + promises = []; + } + promises.push( + limit(() => + mpuHelper.uploadPart(partNumber++, curChunk, options.validation) + ) + ); + } + await Promise.all(promises); + return await mpuHelper.completeUpload(); + } catch (e) { + if ( + (options.autoAbortFailure === undefined || options.autoAbortFailure) && + mpuHelper.uploadId + ) { + try { + await mpuHelper.abortUpload(); + return; + } catch (e) { + throw new MultiPartUploadError( + (e as Error).message, + mpuHelper.uploadId!, + mpuHelper.partsMap! + ); + } + } + throw new MultiPartUploadError( + (e as Error).message, + mpuHelper.uploadId!, + mpuHelper.partsMap! + ); + } + } + + private async *getPathsFromDirectory( + directory: string + ): AsyncGenerator { + const filesAndSubdirectories = await fsp.readdir(directory, { + withFileTypes: true, + }); + for (const curFileOrDirectory of filesAndSubdirectories) { + const fullPath = path.join(directory, curFileOrDirectory.name); + curFileOrDirectory.isDirectory() + ? yield* this.getPathsFromDirectory(fullPath) + : yield fullPath; + } + } +} diff --git a/handwritten/storage/src/util.ts b/handwritten/storage/src/util.ts new file mode 100644 index 00000000000..259f7c0f3e8 --- /dev/null +++ b/handwritten/storage/src/util.ts @@ -0,0 +1,274 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as path from 'path'; +import * as querystring from 'querystring'; +import {PassThrough} from 'stream'; +import * as url from 'url'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import {getPackageJSON} from './package-json-helper.cjs'; + +// Done to avoid a problem with mangling of identifiers when using esModuleInterop +const fileURLToPath = url.fileURLToPath; +const isEsm = true; + +export function normalize( + optionsOrCallback?: T | U, + cb?: U +) { + const options = ( + typeof optionsOrCallback === 'object' ? optionsOrCallback : {} + ) as T; + const callback = ( + typeof optionsOrCallback === 'function' ? optionsOrCallback : cb + )! as U; + return {options, callback}; +} + +/** + * Flatten an object into an Array of arrays, [[key, value], ..]. + * Implements Object.entries() for Node.js <8 + * @internal + */ +export function objectEntries(obj: {[key: string]: T}): Array<[string, T]> { + return Object.keys(obj).map(key => [key, obj[key]] as [string, T]); +} + +/** + * Encode `str` with encodeURIComponent, plus these + * reserved characters: `! * ' ( )`. + * + * See {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent| MDN: fixedEncodeURIComponent} + * + * @param {string} str The URI component to encode. + * @return {string} The encoded string. + */ +export function fixedEncodeURIComponent(str: string): string { + return encodeURIComponent(str).replace( + /[!'()*]/g, + c => '%' + c.charCodeAt(0).toString(16).toUpperCase() + ); +} + +/** + * URI encode `uri` for generating signed URLs, using fixedEncodeURIComponent. + * + * Encode every byte except `A-Z a-Z 0-9 ~ - . _`. + * + * @param {string} uri The URI to encode. + * @param [boolean=false] encodeSlash If `true`, the "/" character is not encoded. + * @return {string} The encoded string. + */ +export function encodeURI(uri: string, encodeSlash: boolean): string { + // Split the string by `/`, and conditionally rejoin them with either + // %2F if encodeSlash is `true`, or '/' if `false`. + return uri + .split('/') + .map(fixedEncodeURIComponent) + .join(encodeSlash ? '%2F' : '/'); +} + +/** + * Serialize an object to a URL query string using util.encodeURI(uri, true). + * @param {string} url The object to serialize. + * @return {string} Serialized string. + */ +export function qsStringify(qs: querystring.ParsedUrlQueryInput): string { + return querystring.stringify(qs, '&', '=', { + encodeURIComponent: (component: string) => encodeURI(component, true), + }); +} + +export function objectKeyToLowercase(object: {[key: string]: T}) { + const newObj: {[key: string]: T} = {}; + for (let key of Object.keys(object)) { + const value = object[key]; + key = key.toLowerCase(); + newObj[key] = value; + } + return newObj; +} + +/** + * JSON encode str, with unicode \u+ representation. + * @param {object} obj The object to encode. + * @return {string} Serialized string. + */ +export function unicodeJSONStringify(obj: object) { + return JSON.stringify(obj).replace( + /[\u0080-\uFFFF]/g, + (char: string) => + '\\u' + ('0000' + char.charCodeAt(0).toString(16)).slice(-4) + ); +} + +/** + * Converts the given objects keys to snake_case + * @param {object} obj object to convert keys to snake case. + * @returns {object} object with keys converted to snake case. + */ +export function convertObjKeysToSnakeCase(obj: object): object { + if (obj instanceof Date || obj instanceof RegExp) { + return obj; + } + if (Array.isArray(obj)) { + return obj.map(convertObjKeysToSnakeCase); + } + if (obj instanceof Object) { + return Object.keys(obj).reduce((acc, cur) => { + const s = + cur[0].toLocaleLowerCase() + + cur.slice(1).replace(/([A-Z]+)/g, (match, p1) => { + return `_${p1.toLowerCase()}`; + }); + + acc[s] = convertObjKeysToSnakeCase(obj[cur as keyof Object]); + return acc; + }, Object()); + } + + return obj; +} + +/** + * Formats the provided date object as a UTC ISO string. + * @param {Date} dateTimeToFormat date object to be formatted. + * @param {boolean} includeTime flag to include hours, minutes, seconds in output. + * @param {string} dateDelimiter delimiter between date components. + * @param {string} timeDelimiter delimiter between time components. + * @returns {string} UTC ISO format of provided date object. + */ +export function formatAsUTCISO( + dateTimeToFormat: Date, + includeTime = false, + dateDelimiter = '', + timeDelimiter = '' +): string { + const year = dateTimeToFormat.getUTCFullYear(); + const month = dateTimeToFormat.getUTCMonth() + 1; + const day = dateTimeToFormat.getUTCDate(); + const hour = dateTimeToFormat.getUTCHours(); + const minute = dateTimeToFormat.getUTCMinutes(); + const second = dateTimeToFormat.getUTCSeconds(); + + let resultString = `${year.toString().padStart(4, '0')}${dateDelimiter}${month + .toString() + .padStart(2, '0')}${dateDelimiter}${day.toString().padStart(2, '0')}`; + if (includeTime) { + resultString = `${resultString}T${hour + .toString() + .padStart(2, '0')}${timeDelimiter}${minute + .toString() + .padStart(2, '0')}${timeDelimiter}${second.toString().padStart(2, '0')}Z`; + } + + return resultString; +} + +/** + * Examines the runtime environment and returns the appropriate tracking string. + * @returns {string} metrics tracking string based on the current runtime environment. + */ +export function getRuntimeTrackingString(): string { + if ( + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + globalThis.Deno && + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + globalThis.Deno.version && + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + globalThis.Deno.version.deno + ) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + return `gl-deno/${globalThis.Deno.version.deno}`; + } else { + return `gl-node/${process.versions.node}`; + } +} + +/** + * Looks at package.json and creates the user-agent string to be applied to request headers. + * @returns {string} user agent string. + */ +export function getUserAgentString(): string { + const pkg = getPackageJSON(); + const hyphenatedPackageName = pkg.name + .replace('@google-cloud', 'gcloud-node') // For legacy purposes. + .replace('/', '-'); // For UA spec-compliance purposes. + + return hyphenatedPackageName + '/' + pkg.version; +} + +export function getDirName() { + let dirToUse = ''; + try { + dirToUse = __dirname; + } catch (e) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + dirToUse = path.dirname(fileURLToPath(import.meta.url)); + } + + return dirToUse; +} + +export function getModuleFormat() { + return isEsm ? 'ESM' : 'CJS'; +} + +export class PassThroughShim extends PassThrough { + private shouldEmitReading = true; + private shouldEmitWriting = true; + + _read(size: number): void { + if (this.shouldEmitReading) { + this.emit('reading'); + this.shouldEmitReading = false; + } + super._read(size); + } + + _write( + chunk: never, + encoding: BufferEncoding, + callback: (error?: Error | null | undefined) => void + ): void { + if (this.shouldEmitWriting) { + this.emit('writing'); + this.shouldEmitWriting = false; + } + // Per the nodejs documentation, callback must be invoked on the next tick + process.nextTick(() => { + super._write(chunk, encoding, callback); + }); + } + + _final(callback: (error?: Error | null | undefined) => void): void { + // If the stream is empty (i.e. empty file) final will be invoked before _read / _write + // and we should still emit the proper events. + if (this.shouldEmitReading) { + this.emit('reading'); + this.shouldEmitReading = false; + } + if (this.shouldEmitWriting) { + this.emit('writing'); + this.shouldEmitWriting = false; + } + callback(null); + } +} diff --git a/handwritten/storage/system-test/common.ts b/handwritten/storage/system-test/common.ts new file mode 100644 index 00000000000..17bc59b6142 --- /dev/null +++ b/handwritten/storage/system-test/common.ts @@ -0,0 +1,116 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {before, describe, it} from 'mocha'; +import assert from 'assert'; +import * as http from 'http'; + +import * as common from '../src/nodejs-common/index.js'; + +describe('Common', () => { + const MOCK_HOST_PORT = 8118; + const MOCK_HOST = `http://localhost:${MOCK_HOST_PORT}`; + + describe('Service', () => { + let service: common.Service; + + before(() => { + service = new common.Service({ + baseUrl: MOCK_HOST, + apiEndpoint: MOCK_HOST, + scopes: [], + packageJson: {name: 'tests', version: '1.0.0'}, + }); + }); + + it('should send a request and receive a response', done => { + const mockResponse = 'response'; + const mockServer = new http.Server((req, res) => { + res.end(mockResponse); + }); + + mockServer.listen(MOCK_HOST_PORT); + + service.request( + { + uri: '/mock-endpoint', + }, + (err, resp) => { + assert.ifError(err); + assert.strictEqual(resp, mockResponse); + mockServer.close(done); + } + ); + }); + + it('should retry a request', function (done) { + // We've increased the timeout to accommodate the retry backoff strategy. + // The test's retry attempts and the delay between them can exceed the default timeout, + // causing a false negative (test failure due to timeout instead of a logic error). + this.timeout(90 * 1000); + + let numRequestAttempts = 0; + + const mockServer = new http.Server((req, res) => { + numRequestAttempts++; + res.statusCode = 408; + res.end(); + }); + + mockServer.listen(MOCK_HOST_PORT); + + service.request( + { + uri: '/mock-endpoint-retry', + }, + err => { + assert.strictEqual((err! as common.ApiError).code, 408); + assert.strictEqual(numRequestAttempts, 4); + mockServer.close(done); + } + ); + }); + + it('should retry non-responsive hosts', function (done) { + this.timeout(60 * 1000); + + function getMinimumRetryDelay(retryNumber: number) { + return Math.pow(2, retryNumber) * 1000; + } + + let minExpectedResponseTime = 0; + let numExpectedRetries = 2; + + while (numExpectedRetries--) { + minExpectedResponseTime += getMinimumRetryDelay(numExpectedRetries + 1); + } + + const timeRequest = Date.now(); + + service.request( + { + uri: '/mock-endpoint-no-response', + }, + err => { + assert(err?.message.includes('ECONNREFUSED')); + const timeResponse = Date.now(); + assert(timeResponse - timeRequest > minExpectedResponseTime); + } + ); + done(); + }); + }); +}); diff --git a/handwritten/storage/system-test/data/CloudPlatform_128px_Retina.png b/handwritten/storage/system-test/data/CloudPlatform_128px_Retina.png new file mode 100644 index 00000000000..86c04e4b44f Binary files /dev/null and b/handwritten/storage/system-test/data/CloudPlatform_128px_Retina.png differ diff --git a/handwritten/storage/system-test/data/empty-file.txt b/handwritten/storage/system-test/data/empty-file.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/handwritten/storage/system-test/data/long-html-file.html b/handwritten/storage/system-test/data/long-html-file.html new file mode 100644 index 00000000000..a8ea1cc650f --- /dev/null +++ b/handwritten/storage/system-test/data/long-html-file.html @@ -0,0 +1,1720 @@ + + + + + +What is the maximum compression ratio of gzip? - Super User + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + + + +
+
+ +
+ +
+ + + +
+

+your communities

+ +
+ + + + +
+
+ + + + +
+
+ + +
+ + + + + +
+ + +
+ + + +
+
+ + Take the 2-minute tour + × + +
+ Super User is a question and answer site for computer enthusiasts and power users. It's 100% free, no registration required. +
+
+ +
+ + +
+ + + +
+ + +
+
+ + + + + + + + + +
+ + +
+ + up vote + 25 + down vote + + favorite +
6
+ + +
+ +
+
+
+ +

What is the largest size a gzip (say 10kb for the sake of an example) can be decompressed to?

+
+ + + + + + + +
+
share|improve this question
+
+ + +
+
+
+
+ + + + + +
+
+ + +
+
+ +
+ + +
+
+

+ 5 Answers + 5 +

+
+ +
+
+
+ + + + + + +
+ + + + + + + + + + + + + +
+ + +
+ + up vote + 46 + down vote + + + + accepted + +
+ +
+
+

It very much depends ont he data being compressed. A quick test with a 1Gb file full for zeros give a compressed size of ~120Kb, so your 10Kb file could potentially expand into ~85Mbytes.

+ +

If the data has low redundancy to start with, for instance the archive contains images files in a format that is compressed natively (gif, jpg, png, ...), then gzip may add not further compression at all. For binary files like program executables you might see up to 2:1 compression, for plain text, HTML or other markup 3:1 or 4:1 or more is not unlikely. You might see 10:1 in some cases but the ~8700:1 seen with a file filled with a single symbol is something you are not going to see outside similarly artificial circumstances.

+ +

You can check how much data would result from unpacking a gzip file, without actually writing its uncompressed content to disk, with gunzip -c file.gz | wc --bytes - this will uncompress the file but not store the results, instead passing them to wc which will count the number of bytes as they pass then discard them. If compressed content is a tar file containing many many small files you might find that noticably more disk space is required to unpack the full archive, but in most circumstances the count returned from piping gunzip output through wc is going to be as accurate as you need.

+
+ + + + + + + + + +
+
share|improve this answer
+ + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + +
+ 1 + +   +
+
+
+ Nice. Limiting case, discussion of common cases and a a "how-to" on answering the question at hand. A model for a good answer. + –  + dmckee + May 9 '10 at 15:09 +
+
+ + + + + + + +
+    + +   +
+
+
+ I've seen HTML expand to 10x (of course x3 and x4 was the most common!).... perhaps a lot of redundant data for those ones that were exploding +8x. I think the page in question that was doing that was a php info page. + –  + Zombies + May 10 '10 at 12:10 + +
+
+ + + + + + + +
+    + +   +
+
+
+ Repetitive markup, as seen in the output of phpinfo(), compresses very well. The technical information in that output contains more direct repetition than the average chunk of natural language would too, and the alphabet distribution is probably less smooth which could help the Huffman stage get better results. + –  + David Spillett + May 10 '10 at 12:55 +
+
+ + + + + + + +
+    + +   +
+
+
+ This answer doesn't account for intentionally malicious compressed data. One can craft a malicious zip file around 10KB that can expand to a bit over 4GB. + –  + David Schwartz + Jan 2 '13 at 2:36 + +
+
+ + + + + + + +
+    + +   +
+
+
+ Zip bombs of that scale rely on nested archives though, so as a human unpacking the file you would noticed something odd before long. They can be used as an effective DoS attack against automated scanners (on mail services and so forth) though. + –  + David Spillett + Jan 2 '13 at 11:47 +
+
+
+ + +
+
+ +
+
+ + +
+ + + + + + + + + + + + + +
+ + +
+ + up vote + 7 + down vote + + + + +
+ +
+
+

Usually you don't get more than 95% compression (so that 10kB gzipped data would decompress to ~200kB), but there are specially crafted files that expand exponentially. Look for 42.zip, it decompresses to few petabytes of (meaningless) data.

+
+ + + + + + + + +
+
share|improve this answer
+ + + +
+
+
+ + + + + + + + + + + + + + +
+ + + + + + + +
+ 2 + +   +
+
+
+ Wikipedia says 42.zip is "containing five layers of nested zip files in sets of 16", so that is not a valid example for decompression (only for recursive decompression). + –  + Tgr + Jul 10 '13 at 13:59 +
+
+ + + + + + + +
+ 1 + +   +
+
+
+ Indeed, 42.zip is specifically a danger to tools that automatically scan zip files recursively, for example virus scanners. + –  + thomasrutter + Feb 5 '14 at 0:42 +
+
+
+ + +
+
+ + + +
+ + + + + + + + + + + + + +
+ + +
+ + up vote + 3 + down vote + + + + +
+ +
+
+

The compression ratio of any compression algorithm will be a function of the data being compressed (besides the length of that data).

+ +

Here is an analysis at MaximumCompression,
+Look at one of the samples like,

+ +

Summary of the multiple file compression benchmark tests

+ +
File type : Multiple file types (46 in total)
+# of files to compress in this test : 510
+Total File Size (bytes) : 316.355.757
+Average File Size (bytes) : 620,305
+Largest File (bytes) : 18,403,071
+Smallest File (bytes) : 3,554
+
+
+ + + + + + + + +
+
share|improve this answer
+ + + +
+
+
+ + + + + +
+
+ + +
+
+ + + +
+ + + + + + + + + + + + + +
+ + +
+ + up vote + 2 + down vote + + + + +
+ +
+
+

A huge file containing only one symbol will compress very well.

+
+ + + + + + + + +
+
share|improve this answer
+ + + +
+
+
+ + + + + +
+
+ + +
+
+ + + +
+ + + + + + + + + + + + + +
+ + +
+ + up vote + 1 + down vote + + + + +
+ +
+
+

10 MB of zeros in file, compress with gzip -9 to 10217. So maximal ratio looks to be around 1000x.

+
+ + + + + + + + +
+
share|improve this answer
+ + + +
+
+
+ + + + + +
+
+ + +
+
+ +
+ + + +

Your Answer

+ + + + + + + + +
+ +
+
+
+ +
+
+ +
 
+ + + + + + +
+
+
+ + +
+ + + +
+
+ + + + +
+ +
+ + discard + +

+By posting your answer, you agree to the privacy policy and terms of service.

+
+
+ + + +

+Not the answer you're looking for? Browse other questions tagged or ask your own question.

+
+
+ + + +
+ + + +
+
+ + + + + + + + diff --git a/handwritten/storage/system-test/data/three-mb-file.tif b/handwritten/storage/system-test/data/three-mb-file.tif new file mode 100644 index 00000000000..5aaa8a1745b Binary files /dev/null and b/handwritten/storage/system-test/data/three-mb-file.tif differ diff --git a/handwritten/storage/system-test/fixtures/index-cjs.js b/handwritten/storage/system-test/fixtures/index-cjs.js new file mode 100644 index 00000000000..b987f57c0d6 --- /dev/null +++ b/handwritten/storage/system-test/fixtures/index-cjs.js @@ -0,0 +1,22 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* eslint-disable node/no-missing-require, no-unused-vars, no-undef */ +const {Storage} = require('@google-cloud/storage'); + +function main() { + const storage = new Storage(); +} + +main(); diff --git a/handwritten/storage/system-test/fixtures/index-esm.js b/handwritten/storage/system-test/fixtures/index-esm.js new file mode 100644 index 00000000000..92cae36bcc5 --- /dev/null +++ b/handwritten/storage/system-test/fixtures/index-esm.js @@ -0,0 +1,22 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* eslint-disable node/no-missing-import, no-unused-vars */ +import {Storage} from '@google-cloud/storage'; + +function main() { + const storage = new Storage(); +} + +main(); diff --git a/handwritten/storage/system-test/install.ts b/handwritten/storage/system-test/install.ts new file mode 100644 index 00000000000..cb50b632ca6 --- /dev/null +++ b/handwritten/storage/system-test/install.ts @@ -0,0 +1,37 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {packNTest} from 'pack-n-play'; +import {readFileSync} from 'fs'; +import {describe, it} from 'mocha'; + +describe('pack-n-play tests', () => { + it('ESM module', async () => { + await packNTest({ + sample: { + description: 'Should be able to import the storage library in ESM', + esm: readFileSync('./system-test/fixtures/index-esm.js').toString(), + }, + }); + }); + + it('CJS module', async () => { + await packNTest({ + sample: { + description: 'Should be able to import the storage library in CJS', + cjs: readFileSync('./system-test/fixtures/index-cjs.js').toString(), + }, + }); + }); +}); diff --git a/handwritten/storage/system-test/kitchen.ts b/handwritten/storage/system-test/kitchen.ts new file mode 100644 index 00000000000..e63615f7ba8 --- /dev/null +++ b/handwritten/storage/system-test/kitchen.ts @@ -0,0 +1,371 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as assert from 'assert'; +import {describe, it, beforeEach} from 'mocha'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as tmp from 'tmp'; +import * as crypto from 'crypto'; +import * as os from 'os'; +import {pipeline, Readable} from 'stream'; +import { + checkUploadStatus, + createURI, + ErrorWithCode, + upload, +} from '../src/resumable-upload.js'; +import { + RETRY_DELAY_MULTIPLIER_DEFAULT, + TOTAL_TIMEOUT_DEFAULT, + MAX_RETRY_DELAY_DEFAULT, + AUTO_RETRY_DEFAULT, + MAX_RETRY_DEFAULT, + RETRYABLE_ERR_FN_DEFAULT, + Storage, +} from '../src/storage.js'; +import {CRC32C} from '../src/crc32c.js'; + +const bucketName = process.env.BUCKET_NAME || 'gcs-resumable-upload-test'; + +/** + * The known multiple chunk upload size, in bytes + */ +const KNOWN_MULTI_CHUNK_MULTIPLE_BYTES = 256 * 1024; +const FILE_SIZE = 1024 * 1024 * 20; + +describe('resumable-upload', () => { + const retryOptions = { + retryDelayMultiplier: RETRY_DELAY_MULTIPLIER_DEFAULT, + totalTimeout: TOTAL_TIMEOUT_DEFAULT, + maxRetryDelay: MAX_RETRY_DELAY_DEFAULT, + autoRetry: AUTO_RETRY_DEFAULT, + maxRetries: MAX_RETRY_DEFAULT, + retryableErrorFn: RETRYABLE_ERR_FN_DEFAULT, + }; + + const bucket = new Storage({retryOptions}).bucket(bucketName); + let filePath: string; + + before(async () => { + tmp.setGracefulCleanup(); + filePath = path.join(os.tmpdir(), '20MB.zip'); + + await fs.promises.writeFile(filePath, crypto.randomBytes(FILE_SIZE)); + }); + + beforeEach(() => { + upload({ + bucket: bucketName, + file: filePath, + retryOptions: retryOptions, + }); + }); + + afterEach(async () => { + await bucket.file(filePath).delete({ignoreNotFound: true}); + }); + + after(async () => { + await fs.promises.rm(filePath, {force: true}); + }); + + function createReadable(amount: number) { + async function* readableGenerator() { + while (amount > 0) { + yield crypto.randomBytes(KNOWN_MULTI_CHUNK_MULTIPLE_BYTES); + amount -= KNOWN_MULTI_CHUNK_MULTIPLE_BYTES; + } + } + + return Readable.from(readableGenerator()); + } + + async function delay(title: string, retries: number, done: Function) { + if (retries === 0) return done(); // no retry on the first failure. + // see: https://cloud.google.com/storage/docs/exponential-backoff: + const ms = Math.pow(2, retries) * 1000 + Math.random() * 2000; + console.info(`retrying "${title}" in ${ms}ms`); + setTimeout(done(), ms); + } + + it('should work', done => { + let uploadSucceeded = false; + fs.createReadStream(filePath) + .on('error', done) + .pipe( + upload({ + bucket: bucketName, + file: filePath, + retryOptions: retryOptions, + metadata: {contentType: 'image/jpg'}, + }) + ) + .on('error', done) + .on('response', resp => { + uploadSucceeded = resp.status === 200; + }) + .on('finish', () => { + assert.strictEqual(uploadSucceeded, true); + done(); + }); + }); + + let retries = 0; + it('should resume an interrupted upload', function (done) { + this.retries(3); + delay(this.test!.title, retries, () => { + retries++; + // If we've retried, delay. + fs.stat(filePath, (err, fd) => { + assert.ifError(err); + + const size = fd.size; + let uri: string | undefined = undefined; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + type DoUploadCallback = (...args: any[]) => void; + const doUpload = ( + opts: {interrupt?: boolean}, + callback: DoUploadCallback + ) => { + let sizeStreamed = 0; + let destroyed = false; + + const ws = upload({ + uri, + bucket: bucketName, + file: filePath, + metadata: {contentType: 'image/jpg'}, + retryOptions: retryOptions, + }); + + ws.on('uri', (link: string) => { + uri = link; + }); + + fs.createReadStream(filePath) + .on('error', callback) + .on('data', function (this: Readable, chunk) { + sizeStreamed += chunk.length; + + if (!destroyed && opts.interrupt && sizeStreamed >= size / 2) { + // stop sending data half way through + destroyed = true; + this.destroy(); + process.nextTick(() => ws.destroy(new Error('Interrupted'))); + } + }) + .pipe(ws) + .on('error', callback) + .on('metadata', callback.bind(null, null)); + }; + + doUpload({interrupt: true}, (err: Error) => { + assert.strictEqual(err.message, 'Interrupted'); + + doUpload( + {interrupt: false}, + (err: Error, metadata: {size: number}) => { + assert.ifError(err); + assert.ok(uri); + assert.strictEqual(metadata.size, size); + assert.strictEqual(typeof metadata.size, 'number'); + done(); + } + ); + }); + }); + }).catch(done); + }); + + it('should create an upload URI', async () => { + const uri = await createURI({ + bucket: bucketName, + file: filePath, + metadata: {contentType: 'image/jpg'}, + retryOptions: retryOptions, + }); + + const resp = await checkUploadStatus({ + bucket: bucketName, + file: filePath, + metadata: {contentType: 'image/jpg'}, + retryOptions: retryOptions, + uri, + }); + + assert.ok(!resp.data); + assert.equal(resp.headers['content-length'], '0'); + }); + + it('should return a non-resumable failed upload', done => { + const metadata = { + metadata: {largeString: 'a'.repeat(2.1e6)}, + }; + + fs.createReadStream(filePath) + .on('error', done) + .pipe( + upload({ + bucket: bucketName, + file: filePath, + metadata, + retryOptions: retryOptions, + }) + ) + .on('error', (err: ErrorWithCode) => { + assert.strictEqual(err.status, 400); + done(); + }); + }); + + it('should upload an object in multiple chunks', async () => { + const file = bucket.file(filePath); + const chunkSize = Math.floor(FILE_SIZE / 4); + + // ensure we're testing a valid size + assert.ok(chunkSize > KNOWN_MULTI_CHUNK_MULTIPLE_BYTES); + + await new Promise((resolve, reject) => + pipeline( + fs.createReadStream(filePath), + file.createWriteStream({ + chunkSize, + }), + e => (e ? reject(e) : resolve()) + ) + ); + + const [results] = await file.getMetadata(); + + assert.equal(results.size, FILE_SIZE); + }); + + it('should complete multiple, partial upload session', async () => { + const count = 4; + const file = bucket.file(filePath); + const chunkSize = Math.floor(FILE_SIZE / count); + + // ensure we're testing a valid size and multiple + assert.equal(chunkSize % KNOWN_MULTI_CHUNK_MULTIPLE_BYTES, 0); + + let uri: string | undefined = undefined; + let uriGenerated = 0; + + let resumeCRC32C = ''; + let crc32cGenerated = 0; + + for (let i = 0; i < count; i++) { + const isPartialUpload = !(i + 1 === count); + const offset = i * chunkSize; + + const readable = createReadable(chunkSize); + const writable = file.createWriteStream({ + uri, + chunkSize, + isPartialUpload, + offset, + resumeCRC32C, + }); + + writable.on('uri', link => { + uri = link; + uriGenerated++; + }); + + writable.on('crc32c', crc32c => { + resumeCRC32C = crc32c; + crc32cGenerated++; + }); + + await new Promise((resolve, reject) => + pipeline(readable, writable, e => (e ? reject(e) : resolve())) + ); + } + + const [results] = await file.getMetadata(); + + assert.ok(uri); + assert.equal(uriGenerated, 1, 'uri should be generated once'); + assert.equal( + crc32cGenerated, + count, + 'crc32c should be generated on each upload' + ); + assert.equal(results.size, FILE_SIZE); + }); + + const KNOWN_CRC32C_OF_ZEROS = 'rthIWA=='; + describe('Validation of Client Checksums Against Server Response', () => { + let crc32c: string; + + before(async () => { + crc32c = (await CRC32C.fromFile(filePath)).toString(); + }); + it('should upload successfully when crc32c calculation is enabled', done => { + let uploadSucceeded = false; + + fs.createReadStream(filePath) + .on('error', done) + .pipe( + upload({ + bucket: bucketName, + file: filePath, + crc32c: true, + clientCrc32c: crc32c, + retryOptions: retryOptions, + }) + ) + .on('error', err => { + console.log(err); + done( + new Error( + `Upload failed unexpectedly on success path: ${err.message}` + ) + ); + }) + .on('response', resp => { + uploadSucceeded = resp.status === 200; + }) + .on('finish', () => { + assert.strictEqual(uploadSucceeded, true); + done(); + }); + }); + + it('should destroy the stream on a checksum mismatch (client-provided hash mismatch)', done => { + const EXPECTED_ERROR_MESSAGE_PART = `Provided CRC32C "${KNOWN_CRC32C_OF_ZEROS}" doesn't match calculated CRC32C`; + + fs.createReadStream(filePath) + .on('error', done) + .pipe( + upload({ + bucket: bucketName, + file: filePath, + clientCrc32c: KNOWN_CRC32C_OF_ZEROS, + crc32c: true, + retryOptions: retryOptions, + }) + ) + .on('error', (err: Error) => { + assert.ok( + err.message.includes(EXPECTED_ERROR_MESSAGE_PART), + `Expected error message part "${EXPECTED_ERROR_MESSAGE_PART}" not found in: ${err.message}` + ); + done(); + }); + }); + }); +}); diff --git a/handwritten/storage/system-test/storage.ts b/handwritten/storage/system-test/storage.ts new file mode 100644 index 00000000000..15257fb59db --- /dev/null +++ b/handwritten/storage/system-test/storage.ts @@ -0,0 +1,4295 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import assert from 'assert'; +import {after, afterEach, before, beforeEach, describe, it} from 'mocha'; +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import fetch from 'node-fetch'; +import FormData from 'form-data'; +import pLimit from 'p-limit'; +import {promisify} from 'util'; +import * as path from 'path'; +import * as tmp from 'tmp'; +import * as uuid from 'uuid'; +import {ApiError} from '../src/nodejs-common/index.js'; +import { + AccessControlObject, + Bucket, + CRC32C, + DeleteBucketCallback, + File, + IdempotencyStrategy, + LifecycleRule, + Notification, + Storage, + UploadOptions, +} from '../src/index.js'; +import nock from 'nock'; +import {Transform} from 'stream'; +import {gzipSync} from 'zlib'; + +interface ErrorCallbackFunction { + (err: Error | null): void; +} +import {PubSub, Subscription, Topic} from '@google-cloud/pubsub'; +import {getDirName} from '../src/util.js'; + +class HTTPError extends Error { + code: number; + constructor(message: string, code: number) { + super(message); + this.code = code; + } +} + +// When set to true, skips all tests that is not compatible for +// running inside VPCSC. +const RUNNING_IN_VPCSC = !!process.env['GOOGLE_CLOUD_TESTS_IN_VPCSC']; + +const UNIFORM_ACCESS_TIMEOUT = 60 * 1000; // 60s see: https://cloud.google.com/storage/docs/consistency#eventually_consistent_operations +const UNIFORM_ACCESS_WAIT_TIME = 5 * 1000; // 5s +const BUCKET_METADATA_UPDATE_WAIT_TIME = 1250; // 1.25s buckets have a max rate of one metadata update per second. + +// block all attempts to chat with the metadata server (kokoro runs on GCE) +nock('http://metadata.google.internal') + .get(() => true) + .replyWithError({code: 'ENOTFOUND'}) + .persist(); + +// eslint-disable-next-line prefer-arrow-callback +describe('storage', function () { + this.retries(3); + + const USER_ACCOUNT = 'user-spsawchuk@gmail.com'; + const TESTS_PREFIX = `storage-tests-${shortUUID()}-`; + const RETENTION_DURATION_SECONDS = 10; + + const storage = new Storage({ + retryOptions: { + idempotencyStrategy: IdempotencyStrategy.RetryAlways, + }, + }); + const bucket = storage.bucket(generateName()); + + const pubsub = new PubSub({ + projectId: process.env.PROJECT_ID, + }); + let topic: Topic; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const FILES: {[index: string]: any} = { + logo: { + path: path.join( + getDirName(), + '../../../system-test/data/CloudPlatform_128px_Retina.png' + ), + }, + big: { + path: path.join( + getDirName(), + '../../../system-test/data/three-mb-file.tif' + ), + hash: undefined, + }, + html: { + path: path.join( + getDirName(), + '../../../system-test/data/long-html-file.html' + ), + }, + empty: { + path: path.join(getDirName(), '../../../system-test/data/empty-file.txt'), + }, + }; + + before(async () => { + await bucket.create(); + const data = await pubsub.createTopic(generateName()); + topic = data[0]; + await topic.iam.setPolicy({ + bindings: [ + { + role: 'roles/pubsub.editor', + members: ['allUsers'], + }, + ], + }); + }); + + after(() => { + return Promise.all([deleteAllBucketsAsync(), deleteAllTopicsAsync()]); + }); + + describe('without authentication', () => { + let privateBucket: Bucket; + let privateFile: File; + let storageWithoutAuth: Storage; + + let GOOGLE_APPLICATION_CREDENTIALS: string | undefined; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + let GOOGLE_CLOUD_PROJECT: string | undefined; + + before(async () => { + // CI authentication is done with ADC. Cache it here, restore it `after` + GOOGLE_APPLICATION_CREDENTIALS = + process.env.GOOGLE_APPLICATION_CREDENTIALS; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT; + + privateBucket = bucket; // `bucket` was created in the global `before` + privateFile = privateBucket.file('file-name'); + await privateFile.save('data'); + }); + + beforeEach(() => { + delete process.env.GOOGLE_APPLICATION_CREDENTIALS; + delete process.env.GOOGLE_CLOUD_PROJECT; + + storageWithoutAuth = new Storage({ + retryOptions: { + idempotencyStrategy: IdempotencyStrategy.RetryAlways, + retryDelayMultiplier: 3, + }, + }); + }); + + after(() => { + process.env.GOOGLE_APPLICATION_CREDENTIALS = + GOOGLE_APPLICATION_CREDENTIALS; + process.env.GOOGLE_CLOUD_PROJECT = GOOGLE_APPLICATION_CREDENTIALS; + }); + + describe('public data', () => { + before(function () { + if (RUNNING_IN_VPCSC) this.skip(); + }); + + let bucket: Bucket; + + beforeEach(() => { + bucket = storageWithoutAuth.bucket('gcp-public-data-landsat'); + }); + + it('should list and download a file', async () => { + const [files] = await bucket.getFiles({autoPaginate: false}); + const file = files[0]; + const [isPublic] = await file.isPublic(); + assert.strictEqual(isPublic, true); + assert.doesNotReject(file.download()); + }); + }); + + describe('private data', () => { + let bucket: Bucket; + let file: File; + + beforeEach(() => { + bucket = storageWithoutAuth.bucket(privateBucket.id!); + file = bucket.file(privateFile.id!); + }); + + it('should not download a file', async () => { + const [isPublic] = await file.isPublic(); + assert.strictEqual(isPublic, false); + await assert.rejects( + file.download(), + (err: Error) => + err.message.indexOf('does not have storage.objects.get') > -1 + ); + }); + + it('should not upload a file', async () => { + try { + await file.save('new data'); + } catch (e) { + const allowedErrorMessages = [ + /Could not load the default credentials/, + /does not have storage\.objects\.create access/, + ]; + assert( + allowedErrorMessages.some(msg => msg.test((e as Error).message)) + ); + } + }); + }); + }); + + describe('acls', () => { + describe('buckets', () => { + // Some bucket update operations have a rate limit. + // Introduce a delay between tests to avoid getting an error. + beforeEach(async () => { + await new Promise(resolve => + setTimeout(resolve, BUCKET_METADATA_UPDATE_WAIT_TIME) + ); + }); + + it('should get access controls', async () => { + const accessControls = await bucket.acl.get(); + assert(Array.isArray(accessControls)); + }); + + it('should add entity to default access controls', async () => { + const [accessControl] = await bucket.acl.default.add({ + entity: USER_ACCOUNT, + role: storage.acl.OWNER_ROLE, + }); + assert.strictEqual(accessControl!.role, storage.acl.OWNER_ROLE); + + const [updatedAccessControl] = await bucket.acl.default.update({ + entity: USER_ACCOUNT, + role: storage.acl.READER_ROLE, + }); + assert.strictEqual(updatedAccessControl.role, storage.acl.READER_ROLE); + await bucket.acl.default.delete({entity: USER_ACCOUNT}); + }); + + it('should get default access controls', async () => { + const accessControls = await bucket.acl.default.get(); + assert(Array.isArray(accessControls)); + }); + + it('should grant an account access', async () => { + const [accessControl] = await bucket.acl.add({ + entity: USER_ACCOUNT, + role: storage.acl.OWNER_ROLE, + }); + assert.strictEqual(accessControl!.role, storage.acl.OWNER_ROLE); + const opts = {entity: USER_ACCOUNT}; + const [accessControlGet] = await bucket.acl.get(opts); + assert.strictEqual( + (accessControlGet as AccessControlObject).role, + storage.acl.OWNER_ROLE + ); + await bucket.acl.delete(opts); + }); + + it('should update an account', async () => { + const [accessControl] = await bucket.acl.add({ + entity: USER_ACCOUNT, + role: storage.acl.OWNER_ROLE, + }); + assert.strictEqual(accessControl!.role, storage.acl.OWNER_ROLE); + const [updatedAcl] = await bucket.acl.update({ + entity: USER_ACCOUNT, + role: storage.acl.WRITER_ROLE, + }); + assert.strictEqual(updatedAcl!.role, storage.acl.WRITER_ROLE); + await bucket.acl.delete({entity: USER_ACCOUNT}); + }); + + /** + * TODO: Re-enable once the test environment allows public IAM roles. + * Currently disabled to avoid 403 errors when adding 'allUsers' or + * 'allAuthenticatedUsers' permissions. + */ + it.skip('should make a bucket public', async () => { + await bucket.makePublic(); + const [aclObject] = await bucket.acl.get({entity: 'allUsers'}); + assert.deepStrictEqual(aclObject, { + entity: 'allUsers', + role: 'READER', + }); + await new Promise(resolve => + setTimeout(resolve, BUCKET_METADATA_UPDATE_WAIT_TIME) + ); + await bucket.acl.delete({entity: 'allUsers'}); + }); + + /** + * TODO: Re-enable once the test environment allows public IAM roles. + * Currently disabled to avoid 403 errors when adding 'allUsers' or + * 'allAuthenticatedUsers' permissions. + */ + it.skip('should make files public', async () => { + await Promise.all( + ['a', 'b', 'c'].map(text => createFileWithContentPromise(text)) + ); + + await bucket.makePublic({includeFiles: true}); + const [files] = await bucket.getFiles(); + const resps = await Promise.all( + files.map(file => isFilePublicAsync(file)) + ); + resps.forEach(resp => assert.strictEqual(resp, true)); + await Promise.all([ + bucket.acl.default.delete({entity: 'allUsers'}), + bucket.deleteFiles(), + ]); + }); + + /** + * TODO: Re-enable once the test environment allows public IAM roles. + * Currently disabled to avoid 403 errors when adding 'allUsers' or + * 'allAuthenticatedUsers' permissions. + */ + it.skip('should make a bucket private', async () => { + try { + await bucket.makePublic(); + await new Promise(resolve => + setTimeout(resolve, BUCKET_METADATA_UPDATE_WAIT_TIME) + ); + await bucket.makePrivate(); + assert.rejects(bucket.acl.get({entity: 'allUsers'}), err => { + assert.strictEqual((err as ApiError).code, 404); + assert.strictEqual((err as ApiError).errors![0].reason, 'notFound'); + }); + } catch (err) { + assert.ifError(err); + } + }); + + it('should make files private', async () => { + await Promise.all( + ['a', 'b', 'c'].map(text => createFileWithContentPromise(text)) + ); + + await bucket.makePrivate({includeFiles: true}); + const [files] = await bucket.getFiles(); + const resps = await Promise.all( + files.map(file => isFilePublicAsync(file)) + ); + resps.forEach(resp => { + assert.strictEqual(resp, false); + }); + await bucket.deleteFiles(); + }); + }); + + describe('files', () => { + let file: File; + + beforeEach(async () => { + const options = { + destination: generateName() + '.png', + }; + [file] = await bucket.upload(FILES.logo.path, options); + }); + + afterEach(async () => { + await file.delete(); + }); + + it('should get access controls', async () => { + const [accessControls] = await file.acl.get(); + assert(Array.isArray(accessControls)); + }); + + it('should not expose default api', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + assert.strictEqual(typeof (file as any).default, 'undefined'); + }); + + it('should grant an account access', async () => { + const [accessControl] = await file.acl.add({ + entity: USER_ACCOUNT, + role: storage.acl.OWNER_ROLE, + }); + assert.strictEqual(accessControl!.role, storage.acl.OWNER_ROLE); + const [accessControlGet] = await file.acl.get({entity: USER_ACCOUNT}); + assert.strictEqual( + (accessControlGet as AccessControlObject).role, + storage.acl.OWNER_ROLE + ); + await file.acl.delete({entity: USER_ACCOUNT}); + }); + + it('should update an account', async () => { + const [accessControl] = await file.acl.add({ + entity: USER_ACCOUNT, + role: storage.acl.OWNER_ROLE, + }); + assert.strictEqual(accessControl!.role, storage.acl.OWNER_ROLE); + const [accessControlUpdate] = await file.acl.update({ + entity: USER_ACCOUNT, + role: storage.acl.READER_ROLE, + }); + assert.strictEqual(accessControlUpdate!.role, storage.acl.READER_ROLE); + await file.acl.delete({entity: USER_ACCOUNT}); + }); + + /** + * TODO: Re-enable once the test environment allows public IAM roles. + * Currently disabled to avoid 403 errors when adding 'allUsers' or + * 'allAuthenticatedUsers' permissions. + */ + it.skip('should make a file public', async () => { + await file.makePublic(); + const [aclObject] = await file.acl.get({entity: 'allUsers'}); + assert.deepStrictEqual(aclObject, { + entity: 'allUsers', + role: 'READER', + }); + await file.acl.delete({entity: 'allUsers'}); + }); + + it('should make a file private', async () => { + const validateMakeFilePrivateRejects = (err: ApiError) => { + assert.strictEqual(err.code, 404); + assert.strictEqual(err!.errors![0].reason, 'notFound'); + return true; + }; + assert.doesNotReject(file.makePublic()); + assert.doesNotReject(file.makePrivate()); + assert.rejects( + file.acl.get({entity: 'allUsers'}), + validateMakeFilePrivateRejects + ); + }); + + it('should set custom encryption during the upload', async () => { + const key = '12345678901234567890123456789012'; + const [file] = await bucket.upload(FILES.big.path, { + encryptionKey: key, + resumable: false, + }); + const [metadata] = await file.getMetadata(); + const encryptionAlgorithm = + metadata.customerEncryption?.encryptionAlgorithm; + assert.strictEqual(encryptionAlgorithm, 'AES256'); + }); + + it('should set custom encryption in a resumable upload', async () => { + const key = crypto.randomBytes(32); + const [file] = await bucket.upload(FILES.big.path, { + encryptionKey: key, + resumable: true, + }); + const [metadata] = await file.getMetadata(); + const encryptionAlgorithm = + metadata.customerEncryption?.encryptionAlgorithm; + assert.strictEqual(encryptionAlgorithm, 'AES256'); + }); + + /** + * TODO: Re-enable once the test environment allows public IAM roles. + * Currently disabled to avoid 403 errors when adding 'allUsers' or + * 'allAuthenticatedUsers' permissions. + */ + it.skip('should make a file public during the upload', async () => { + const [file] = await bucket.upload(FILES.big.path, { + resumable: false, + public: true, + }); + + const [aclObject] = await file.acl.get({entity: 'allUsers'}); + assert.deepStrictEqual(aclObject, { + entity: 'allUsers', + role: 'READER', + }); + }); + + /** + * TODO: Re-enable once the test environment allows public IAM roles. + * Currently disabled to avoid 403 errors when adding 'allUsers' or + * 'allAuthenticatedUsers' permissions. + */ + it.skip('should make a file public from a resumable upload', async () => { + const [file] = await bucket.upload(FILES.big.path, { + resumable: true, + public: true, + }); + const [aclObject] = await file.acl.get({entity: 'allUsers'}); + assert.deepStrictEqual(aclObject, { + entity: 'allUsers', + role: 'READER', + }); + }); + + it('should make a file private from a resumable upload', async () => { + const validateMakeFilePrivateRejects = (err: ApiError) => { + assert.strictEqual((err as ApiError)!.code, 404); + assert.strictEqual((err as ApiError).errors![0].reason, 'notFound'); + return true; + }; + assert.doesNotReject( + bucket.upload(FILES.big.path, { + resumable: true, + private: true, + }) + ); + assert.rejects( + file.acl.get({entity: 'allUsers'}), + validateMakeFilePrivateRejects + ); + }); + }); + }); + + describe('iam', () => { + let PROJECT_ID: string; + + before(async () => { + PROJECT_ID = await storage.authClient.getProjectId(); + }); + + describe('buckets', () => { + let bucket: Bucket; + + before(() => { + bucket = storage.bucket(generateName()); + return bucket.create(); + }); + + it('should get a policy', async () => { + const [policy] = await bucket.iam.getPolicy(); + assert.deepStrictEqual(policy!.bindings, [ + { + members: [ + 'projectEditor:' + PROJECT_ID, + 'projectOwner:' + PROJECT_ID, + ], + role: 'roles/storage.legacyBucketOwner', + }, + { + members: ['projectViewer:' + PROJECT_ID], + role: 'roles/storage.legacyBucketReader', + }, + ]); + }); + + /** + * TODO: Re-enable once the test environment allows public IAM roles. + * Currently disabled to avoid 403 errors when adding 'allUsers' or + * 'allAuthenticatedUsers' permissions. + */ + it.skip('should set a policy', async () => { + const [policy] = await bucket.iam.getPolicy(); + policy!.bindings.push({ + role: 'roles/storage.legacyBucketReader', + members: ['allUsers'], + }); + const [newPolicy] = await bucket.iam.setPolicy(policy); + const legacyBucketReaderBinding = newPolicy!.bindings.filter( + binding => { + return binding.role === 'roles/storage.legacyBucketReader'; + } + )[0]; + assert(legacyBucketReaderBinding.members.includes('allUsers')); + }); + + it('should get-modify-set a conditional policy', async () => { + // Uniform-bucket-level-access is required to use IAM Conditions. + await bucket.setMetadata({ + iamConfiguration: { + uniformBucketLevelAccess: { + enabled: true, + }, + }, + }); + + const [policy] = await bucket.iam.getPolicy(); + + const serviceAccount = (await storage.authClient.getCredentials()) + .client_email; + const conditionalBinding = { + role: 'roles/storage.objectViewer', + members: [`serviceAccount:${serviceAccount}`], + condition: { + title: 'always-true', + description: 'this condition is always effective', + expression: 'true', + }, + }; + + policy.version = 3; + policy.bindings.push(conditionalBinding); + + await bucket.iam.setPolicy(policy); + + const [newPolicy] = await bucket.iam.getPolicy({ + requestedPolicyVersion: 3, + }); + assert.deepStrictEqual(newPolicy.bindings, policy.bindings); + }); + + it('should test the iam permissions', async () => { + const testPermissions = [ + 'storage.buckets.get', + 'storage.buckets.getIamPolicy', + ]; + const [permissions] = await bucket.iam.testPermissions(testPermissions); + assert.deepStrictEqual(permissions, { + 'storage.buckets.get': true, + 'storage.buckets.getIamPolicy': true, + }); + }); + }); + }); + + describe('public access prevention', () => { + let bucket: Bucket; + + const PUBLIC_ACCESS_PREVENTION_INHERITED = 'inherited'; + const PUBLIC_ACCESS_PREVENTION_ENFORCED = 'enforced'; + + const createBucket = () => { + bucket = storage.bucket(generateName()); + return bucket.create(); + }; + + const setPublicAccessPrevention = ( + bucket: Bucket, + configuration: string + ) => { + return bucket.setMetadata({ + iamConfiguration: { + publicAccessPrevention: configuration, + }, + }); + }; + + const validateUnexpectedPublicAccessPreventionValueError = ( + err: ApiError + ) => { + assert.strictEqual(err.code, 400); + return true; + }; + + const validateConfiguringPublicAccessWhenPAPEnforcedError = ( + err: ApiError + ) => { + assert.strictEqual(err.code, 412); + return true; + }; + + beforeEach(createBucket); + + it('inserts a bucket with enforced public access prevention', async () => { + await setPublicAccessPrevention( + bucket, + PUBLIC_ACCESS_PREVENTION_ENFORCED + ); + const [bucketMetadata] = await bucket.getMetadata(); + const publicAccessPreventionStatus = + bucketMetadata!.iamConfiguration!.publicAccessPrevention; + return assert.strictEqual( + publicAccessPreventionStatus, + PUBLIC_ACCESS_PREVENTION_ENFORCED + ); + }); + + describe('enforced public access prevention behavior', () => { + let bucket: Bucket; + let file: File; + + before(async () => { + bucket = storage.bucket(generateName()); + await bucket.create(); + + const name = 'enforcedPAPBucketFile'; + const contents = + 'Enforced public access prevention bucket file contents'; + file = bucket.file(name); + await file.save(contents); + + await setPublicAccessPrevention( + bucket, + PUBLIC_ACCESS_PREVENTION_ENFORCED + ); + }); + + it('bucket cannot be made public', async () => { + return assert.rejects( + () => bucket.makePublic(), + validateConfiguringPublicAccessWhenPAPEnforcedError + ); + }); + + it('object cannot be made public via ACL', async () => { + return assert.rejects( + () => file.makePublic(), + validateConfiguringPublicAccessWhenPAPEnforcedError + ); + }); + }); + + it('inserts a bucket with inherited public access prevention', async () => { + await setPublicAccessPrevention( + bucket, + PUBLIC_ACCESS_PREVENTION_INHERITED + ); + const [bucketMetadata] = await bucket.getMetadata(); + const publicAccessPreventionStatus = + bucketMetadata!.iamConfiguration!.publicAccessPrevention; + return assert.strictEqual( + publicAccessPreventionStatus, + PUBLIC_ACCESS_PREVENTION_INHERITED + ); + }); + + it('makes public a bucket with inherited public access prevention', async () => { + await setPublicAccessPrevention( + bucket, + PUBLIC_ACCESS_PREVENTION_INHERITED + ); + return assert.ok(() => bucket.makePublic()); + }); + + it('should fail to insert a bucket with unexpected public access prevention value', async () => { + await assert.rejects( + () => setPublicAccessPrevention(bucket, 'unexpected value'), + validateUnexpectedPublicAccessPreventionValueError + ); + }); + + it('UBLA modification on PAP bucket does not affect pap setting', async () => { + const [bucketMetadata] = await bucket.getMetadata(); + const publicAccessPreventionStatus = + bucketMetadata!.iamConfiguration!.publicAccessPrevention; + await bucket.setMetadata({ + iamConfiguration: { + uniformBucketLevelAccess: { + enabled: true, + }, + }, + }); + const [updatedBucketMetadata] = await bucket.getMetadata(); + return assert.strictEqual( + updatedBucketMetadata!.iamConfiguration!.publicAccessPrevention, + publicAccessPreventionStatus + ); + }); + + it('PAP modification on UBLA bucket should not affect UBLA setting', async () => { + await bucket.setMetadata({ + iamConfiguration: { + uniformBucketLevelAccess: { + enabled: true, + }, + }, + }); + const [bucketMetadata] = await bucket.getMetadata(); + const ublaSetting = + bucketMetadata!.iamConfiguration!.uniformBucketLevelAccess!.enabled; + await setPublicAccessPrevention( + bucket, + PUBLIC_ACCESS_PREVENTION_INHERITED + ); + const [updatedBucketMetadata] = await bucket.getMetadata(); + return assert.strictEqual( + updatedBucketMetadata!.iamConfiguration!.uniformBucketLevelAccess! + .enabled, + ublaSetting + ); + }); + }); + + describe('turbo replication', () => { + let bucket: Bucket; + + const RPO_ASYNC_TURBO = 'ASYNC_TURBO'; + const RPO_DEFAULT = 'DEFAULT'; + + const createBucket = () => { + bucket = storage.bucket(generateName()); + return bucket.create({location: 'NAM4'}); + }; + + const setTurboReplication = ( + bucket: Bucket, + turboReplicationConfiguration: string + ) => { + return bucket.setMetadata({ + rpo: turboReplicationConfiguration, + }); + }; + + beforeEach(createBucket); + + afterEach(async () => { + await bucket.delete(); + }); + + it("sets bucket's RPO to ASYNC_TURBO", async () => { + await setTurboReplication(bucket, RPO_ASYNC_TURBO); + const [bucketMetadata] = await bucket.getMetadata(); + return assert.strictEqual(bucketMetadata.rpo, RPO_ASYNC_TURBO); + }); + + it("sets a bucket's RPO to DEFAULT", async () => { + await setTurboReplication(bucket, RPO_DEFAULT); + const [bucketMetadata] = await bucket.getMetadata(); + return assert.strictEqual(bucketMetadata.rpo, RPO_DEFAULT); + }); + }); + + describe('soft-delete', () => { + describe('buckets', () => { + let bucket: Bucket; + const SOFT_DELETE_RETENTION_SECONDS = 7 * 24 * 60 * 60; //7 days in seconds; + let generation: string; + + before(async () => { + bucket = storage.bucket(generateName()); + await bucket.create(); + await bucket.setMetadata({ + softDeletePolicy: { + retentionDurationSeconds: SOFT_DELETE_RETENTION_SECONDS, + }, + }); + + const [metadata] = await bucket.getMetadata(); + generation = metadata!.generation!; + await bucket.delete(); + }); + + after(async () => { + await bucket.delete(); + }); + + it('should be listing soft-deleted buckets', async () => { + const [buckets] = await storage.getBuckets({softDeleted: true}); + assert(buckets.length > 0); + + buckets.forEach(bucket => { + assert(bucket.name); + assert(bucket.metadata.generation); + assert(bucket.metadata.softDeleteTime); + assert(bucket.metadata.hardDeleteTime); + }); + }); + + it('should GET a soft-deleted bucket', async () => { + const [softDeletedBucket] = await storage + .bucket(bucket.name) + .getMetadata({ + softDeleted: true, + generation: generation, + }); + assert(softDeletedBucket); + assert.strictEqual(softDeletedBucket.generation, generation); + assert(softDeletedBucket.softDeleteTime); + assert(softDeletedBucket.hardDeleteTime); + }); + + it('should restore a soft-deleted bucket', async () => { + const restoredBucket = await storage.bucket(bucket.name).restore({ + generation: generation, + }); + assert(restoredBucket); + const [metadata] = await bucket.getMetadata(); + assert(metadata); + }); + }); + + describe('files', () => { + let bucket: Bucket; + let hnsBucket: Bucket; + const SOFT_DELETE_RETENTION_SECONDS = 7 * 24 * 60 * 60; //7 days in seconds; + + beforeEach(async () => { + bucket = storage.bucket(generateName()); + await bucket.create(); + await bucket.setMetadata({ + softDeletePolicy: { + retentionDurationSeconds: SOFT_DELETE_RETENTION_SECONDS, + }, + }); + + hnsBucket = storage.bucket(generateName()); + await storage.createBucket(hnsBucket.name, { + hierarchicalNamespace: {enabled: true}, + iamConfiguration: { + uniformBucketLevelAccess: { + enabled: true, + }, + }, + softDeletePolicy: { + retentionDurationSeconds: SOFT_DELETE_RETENTION_SECONDS, + }, + }); + }); + + afterEach(async () => { + await bucket.deleteFiles({force: true, versions: true}); + await bucket.delete(); + await hnsBucket.deleteFiles({force: true, versions: true}); + await hnsBucket.delete(); + }); + + it('should set softDeletePolicy correctly', async () => { + const metadata = await bucket.getMetadata(); + assert(metadata[0].softDeletePolicy); + assert(metadata[0].softDeletePolicy.effectiveTime); + assert.deepStrictEqual( + metadata[0].softDeletePolicy.retentionDurationSeconds, + SOFT_DELETE_RETENTION_SECONDS.toString() + ); + }); + + it('should LIST soft-deleted files', async () => { + const f1 = bucket.file('file1'); + const f2 = bucket.file('file2'); + await f1.save('file1'); + await f2.save('file2'); + await f1.delete(); + await f2.delete(); + const [notSoftDeletedFiles] = await bucket.getFiles(); + assert.strictEqual(notSoftDeletedFiles.length, 0); + const [softDeletedFiles] = await bucket.getFiles({softDeleted: true}); + assert.strictEqual(softDeletedFiles.length, 2); + }); + + it('should GET a soft-deleted file', async () => { + const f1 = bucket.file('file3'); + await f1.save('file3'); + const [metadata] = await f1.getMetadata(); + await f1.delete(); + const [softDeletedFile] = await f1.get({ + softDeleted: true, + generation: parseInt(metadata.generation?.toString() || '0'), + }); + assert(softDeletedFile); + assert.strictEqual( + softDeletedFile.metadata.generation, + metadata.generation + ); + }); + + it('should restore a soft-deleted file', async () => { + const f1 = bucket.file('file4'); + await f1.save('file4'); + const [metadata] = await f1.getMetadata(); + await f1.delete(); + let [files] = await bucket.getFiles(); + assert.strictEqual(files.length, 0); + const restoredFile = await f1.restore({ + generation: parseInt(metadata.generation?.toString() || '0'), + }); + assert(restoredFile); + [files] = await bucket.getFiles(); + assert.strictEqual(files.length, 1); + }); + + it('should LIST soft-deleted files with restore token', async () => { + const f1 = hnsBucket.file('file5a'); + const f2 = hnsBucket.file('file5b'); + await f1.save('file5a'); + await f2.save('file5b'); + await f1.delete(); + await f2.delete(); + const [notSoftDeletedFiles] = await hnsBucket.getFiles(); + assert.strictEqual(notSoftDeletedFiles.length, 0); + const [softDeletedFiles] = await hnsBucket.getFiles({ + softDeleted: true, + }); + assert.strictEqual(softDeletedFiles.length, 2); + assert.notStrictEqual( + softDeletedFiles![0].metadata.restoreToken, + undefined + ); + }); + + it('should GET a soft-deleted file with restore token', async () => { + const f1 = hnsBucket.file('file6'); + await f1.save('file6'); + const [metadata] = await f1.getMetadata(); + await f1.delete(); + const [softDeletedFile] = await f1.get({ + softDeleted: true, + generation: parseInt(metadata.generation?.toString() || '0'), + }); + assert(softDeletedFile); + assert.strictEqual( + softDeletedFile.metadata.generation, + metadata.generation + ); + assert.notStrictEqual(softDeletedFile.metadata.restoreToken, undefined); + }); + + it('should restore a soft-deleted file using restoreToken', async () => { + const f1 = hnsBucket.file('file7'); + await f1.save('file7'); + const [metadata] = await f1.getMetadata(); + await f1.delete(); + let [files] = await hnsBucket.getFiles(); + assert.strictEqual(files.length, 0); + const [softDeletedFile] = await f1.get({ + softDeleted: true, + generation: parseInt(metadata.generation?.toString() || '0'), + }); + assert(softDeletedFile); + const restoredFile = await f1.restore({ + generation: parseInt( + softDeletedFile.metadata.generation?.toString() || '0' + ), + restoreToken: softDeletedFile.metadata.restoreToken, + }); + assert(restoredFile); + [files] = await hnsBucket.getFiles(); + assert.strictEqual(files.length, 1); + }); + }); + }); + + describe('dual-region', () => { + let bucket: Bucket; + + const LOCATION = 'US'; + const REGION1 = 'US-EAST1'; + const REGION2 = 'US-WEST1'; + + beforeEach(() => { + bucket = storage.bucket(generateName()); + }); + + it('creates a dual-region bucket', async () => { + await bucket.create({ + location: LOCATION, + customPlacementConfig: { + dataLocations: [REGION1, REGION2], + }, + }); + + const [exists] = await bucket.exists(); + assert.strictEqual(exists, true); + + const [bucketMetadata] = await bucket.getMetadata(); + + assert.strictEqual(bucketMetadata.location, LOCATION); + + assert(bucketMetadata.customPlacementConfig); + assert(Array.isArray(bucketMetadata.customPlacementConfig.dataLocations)); + + const dataLocations = bucketMetadata.customPlacementConfig.dataLocations; + + assert(dataLocations.includes(REGION1)); + assert(dataLocations.includes(REGION2)); + + assert.strictEqual(bucketMetadata.locationType, 'dual-region'); + }); + }); + + describe('uniform bucket-level access', () => { + let bucket: Bucket; + + const customAcl = { + entity: USER_ACCOUNT, + role: storage.acl.OWNER_ROLE, + }; + + const createBucket = () => { + bucket = storage.bucket(generateName()); + return bucket.create(); + }; + + const setUniformBucketLevelAccess = (bucket: Bucket, enabled: boolean) => { + return bucket.setMetadata({ + iamConfiguration: { + uniformBucketLevelAccess: { + enabled, + }, + }, + }); + }; + + describe('files', () => { + before(createBucket); + + it('can be written to the bucket by project owner w/o configuration', async () => { + await setUniformBucketLevelAccess(bucket, true); + const file = bucket.file(`file-${uuid.v4()}`); + return assert.doesNotReject(() => file.save('data')); + }); + }); + + describe('disables file ACL', () => { + let file: File; + + const validateUniformBucketLevelAccessEnabledError = (err: ApiError) => { + assert.strictEqual(err.code, 400); + return true; + }; + + before(async () => { + await createBucket(); + await setUniformBucketLevelAccess(bucket, true); + + file = bucket.file(`file-${uuid.v4()}`); + await file.save('data', {resumable: false}); + }); + + it('should fail to get file ACL', async () => { + // Setting uniform bucket level access is eventually consistent and may take up to a minute to be reflected + for (;;) { + try { + await file.acl.get(); + await new Promise(res => setTimeout(res, UNIFORM_ACCESS_WAIT_TIME)); + } catch (err) { + assert( + validateUniformBucketLevelAccessEnabledError(err as ApiError) + ); + break; + } + } + }).timeout(UNIFORM_ACCESS_TIMEOUT); + + it('should fail to update file ACL', async () => { + // Setting uniform bucket level access is eventually consistent and may take up to a minute to be reflected + for (;;) { + try { + await file.acl.update(customAcl); + await new Promise(res => setTimeout(res, UNIFORM_ACCESS_WAIT_TIME)); + } catch (err) { + assert( + validateUniformBucketLevelAccessEnabledError(err as ApiError) + ); + break; + } + } + }).timeout(UNIFORM_ACCESS_TIMEOUT); + }); + + describe('preserves bucket/file ACL over uniform bucket-level access on/off', () => { + beforeEach(createBucket); + + it('should preserve default bucket ACL', async () => { + await bucket.acl.default.update(customAcl); + const [aclBefore] = await bucket.acl.default.get(); + + await setUniformBucketLevelAccess(bucket, true); + await setUniformBucketLevelAccess(bucket, false); + + // Setting uniform bucket level access is eventually consistent and may take up to a minute to be reflected + for (;;) { + try { + const [aclAfter] = await bucket.acl.default.get(); + assert.deepStrictEqual(aclAfter, aclBefore); + break; + } catch { + await new Promise(res => setTimeout(res, UNIFORM_ACCESS_WAIT_TIME)); + } + } + }).timeout(UNIFORM_ACCESS_TIMEOUT); + + it('should preserve file ACL', async () => { + const file = bucket.file(`file-${uuid.v4()}`); + await file.save('data', {resumable: false}); + + await file.acl.update(customAcl); + const [aclBefore] = await file.acl.get(); + + await setUniformBucketLevelAccess(bucket, true); + await setUniformBucketLevelAccess(bucket, false); + + // Setting uniform bucket level access is eventually consistent and may take up to a minute to be reflected + for (;;) { + try { + const [aclAfter] = await file.acl.get(); + assert.deepStrictEqual(aclAfter, aclBefore); + break; + } catch { + await new Promise(res => setTimeout(res, UNIFORM_ACCESS_WAIT_TIME)); + } + } + }).timeout(UNIFORM_ACCESS_TIMEOUT); + }); + }); + + describe('unicode validation', () => { + before(function () { + if (RUNNING_IN_VPCSC) this.skip(); + }); + + let bucket: Bucket; + + before(async () => { + [bucket] = await storage.createBucket(generateName()); + }); + + // Normalization form C: a single character for e-acute; + // URL should end with Cafe%CC%81 + it('should not perform normalization form C', async () => { + const name = 'Caf\u00e9'; + const expectedContents = 'Normalization Form C'; + + const file = bucket.file(name); + await file.save(expectedContents); + + return file + .get() + .then(data => { + const receivedFile = data[0]; + assert.strictEqual(receivedFile.name, name); + return receivedFile.download(); + }) + .then(contents => { + assert.strictEqual(contents.toString(), expectedContents); + }); + }); + + // Normalization form D: an ASCII character followed by U+0301 combining + // character; URL should end with Caf%C3%A9 + it('should not perform normalization form D', async () => { + const name = 'Cafe\u0301'; + const expectedContents = 'Normalization Form D'; + + const file = bucket.file(name); + await file.save(expectedContents); + + return file + .get() + .then(data => { + const receivedFile = data[0]; + assert.strictEqual(receivedFile.name, name); + return receivedFile.download(); + }) + .then(contents => { + assert.strictEqual(contents.toString(), expectedContents); + }); + }); + }); + + describe('getting buckets', () => { + const bucketsToCreate = [generateName(), generateName()]; + + before(async () => { + await Promise.all(bucketsToCreate.map(b => storage.createBucket(b))); + }); + + after(async () => { + await Promise.all( + bucketsToCreate.map(bucket => storage.bucket(bucket).delete()) + ); + }); + + it('should get buckets', async () => { + const [buckets] = await storage.getBuckets(); + const createdBuckets = buckets.filter(bucket => { + return bucketsToCreate.indexOf(bucket.name) > -1; + }); + assert.strictEqual(createdBuckets.length, bucketsToCreate.length); + }); + + it('should get buckets as a stream', done => { + let bucketEmitted = false; + + storage + .getBucketsStream() + .on('error', done) + .on('data', bucket => { + bucketEmitted = bucket instanceof Bucket; + }) + .on('end', () => { + assert.strictEqual(bucketEmitted, true); + done(); + }); + }); + }); + + describe('bucket metadata', () => { + it('should allow setting metadata on a bucket', async () => { + const metadata = { + website: { + mainPageSuffix: 'http://fakeuri', + notFoundPage: 'http://fakeuri/404.html', + }, + }; + const [meta] = await bucket.setMetadata(metadata); + assert.deepStrictEqual(meta.website, metadata.website); + }); + + it('should allow changing the storage class', async () => { + const bucket = storage.bucket(generateName()); + await bucket.create(); + let [metadata] = await bucket.getMetadata(); + assert.strictEqual(metadata.storageClass, 'STANDARD'); + await bucket.setStorageClass('coldline'); + [metadata] = await bucket.getMetadata(); + assert.strictEqual(metadata.storageClass, 'COLDLINE'); + }); + + it('should allow enabling & disabling autoclass', async () => { + const [bucket] = await storage.createBucket(generateName(), { + autoclass: { + enabled: true, + terminalStorageClass: 'ARCHIVE', + }, + }); + let [metadata] = await bucket.getMetadata(); + const timestampEnabled = metadata!.autoclass!.toggleTime; + assert.strictEqual(metadata!.autoclass!.enabled, true); + assert.strictEqual(metadata!.autoclass?.terminalStorageClass, 'ARCHIVE'); + [metadata] = await bucket.setMetadata({ + autoclass: { + enabled: false, + }, + }); + const timestampDisabled = metadata!.autoclass!.toggleTime; + assert.strictEqual(metadata!.autoclass!.enabled, false); + assert.strictEqual(timestampDisabled! > timestampEnabled!, true); + }); + + describe('locationType', () => { + const types = ['multi-region', 'region', 'dual-region']; + + beforeEach(() => { + bucket.metadata = {}; + }); + + it('should be available from getting a bucket', async () => { + const [metadata] = await bucket.getMetadata(); + assert(types.includes(metadata.locationType!)); + }); + + it('should be available from creating a bucket', async () => { + const [bucket] = await storage.createBucket(generateName()); + assert(types.includes(bucket.metadata.locationType!)); + return bucket.delete(); + }); + + it('should be available from listing buckets', async () => { + const [buckets] = await storage.getBuckets(); + + assert(buckets.length > 0); + + buckets.forEach(bucket => { + assert(types.includes(bucket.metadata.locationType!)); + }); + }); + + it('should be available from setting retention policy', async () => { + await bucket.setRetentionPeriod(RETENTION_DURATION_SECONDS); + assert(types.includes(bucket.metadata.locationType!)); + await bucket.removeRetentionPeriod(); + }); + + it('should be available from updating a bucket', async () => { + await bucket.setMetadata({labels: {a: 'b'}}); + assert(types.includes(bucket.metadata.locationType!)); + }); + }); + + describe('labels', () => { + const LABELS = { + label: 'labelvalue', // no caps or spaces allowed (?) + labeltwo: 'labelvaluetwo', + }; + + beforeEach(async () => { + const [metadata] = await bucket.getMetadata(); + const labels: {[index: string]: string | null} = {}; + if (metadata.labels) { + for (const curLabel of Object.keys(metadata.labels)) { + labels[curLabel] = null; + } + await bucket.setMetadata({labels}); + } + }); + + it('should set labels', async () => { + await bucket.setMetadata({labels: LABELS}); + const [metadata] = await bucket.getMetadata(); + assert.deepStrictEqual(metadata.labels, LABELS); + }); + + it('should update labels', async () => { + const newLabels = { + siblinglabel: 'labelvalue', + }; + await bucket.setMetadata({labels: LABELS}); + await bucket.setMetadata({labels: newLabels}); + const [metadata] = await bucket.getMetadata(); + assert.deepStrictEqual( + metadata.labels, + Object.assign({}, LABELS, newLabels) + ); + }); + + it('should delete a single label', async () => { + if (Object.keys(LABELS).length <= 1) { + throw new Error('Maintainer Error: `LABELS` needs 2 labels.'); + } + + const labelKeyToDelete = Object.keys(LABELS)[0]; + await bucket.setMetadata({labels: LABELS}); + const labelsToDelete = { + [labelKeyToDelete]: null, + }; + await bucket.setMetadata({labels: labelsToDelete}); + const [metadata] = await bucket.getMetadata(); + const expectedLabels = Object.assign({}, LABELS); + delete (expectedLabels as {[index: string]: {}})[labelKeyToDelete]; + + assert.deepStrictEqual(metadata.labels, expectedLabels); + }); + + it('should delete all labels', async () => { + let [metadata] = await bucket.getMetadata(); + if (metadata.labels) { + const labels: {[index: string]: string | null} = {}; + for (const curLabel of Object.keys(metadata.labels)) { + labels[curLabel] = null; + } + await bucket.setMetadata({labels}); + } + [metadata] = await bucket.getMetadata(); + assert.deepStrictEqual(metadata.labels, undefined); + }); + }); + }); + + describe('bucket object lifecycle management', () => { + it('should add a rule', async () => { + await bucket.addLifecycleRule({ + action: { + type: 'Delete', + }, + condition: { + age: 30, + isLive: true, + }, + }); + + const rules = [].slice.call(bucket.metadata.lifecycle?.rule); + assert.deepStrictEqual(rules.pop(), { + action: { + type: 'Delete', + }, + condition: { + age: 30, + isLive: true, + }, + }); + }); + + it('should append a new rule', async () => { + const numExistingRules = + (bucket.metadata.lifecycle && bucket.metadata.lifecycle.rule!.length) || + 0; + + await bucket.addLifecycleRule({ + action: { + type: 'Delete', + }, + condition: { + age: 30, + isLive: true, + }, + }); + await bucket.addLifecycleRule({ + action: { + type: 'SetStorageClass', + storageClass: 'coldline', + }, + condition: { + age: 60, + isLive: true, + }, + }); + assert.strictEqual( + bucket.metadata.lifecycle!.rule!.length, + numExistingRules + 2 + ); + }); + + it('should add a prefix rule', async () => { + await bucket.addLifecycleRule({ + action: { + type: 'Delete', + }, + condition: { + matchesPrefix: [TESTS_PREFIX], + }, + }); + + assert( + bucket.metadata.lifecycle!.rule!.some( + (rule: LifecycleRule) => + typeof rule.action === 'object' && + rule.action.type === 'Delete' && + typeof rule.condition.matchesPrefix === 'object' && + (rule.condition.matchesPrefix as string[]).length === 1 && + Array.isArray(rule.condition.matchesPrefix) + ) + ); + }); + + it('should add a suffix rule', async () => { + await bucket.addLifecycleRule({ + action: { + type: 'Delete', + }, + condition: { + matchesSuffix: [TESTS_PREFIX, 'test_suffix'], + }, + }); + + assert( + bucket.metadata.lifecycle!.rule!.some( + (rule: LifecycleRule) => + typeof rule.action === 'object' && + rule.action.type === 'Delete' && + Array.isArray(rule.condition.matchesPrefix) + ) + ); + }); + + it('should convert a rule with createdBefore to a date in string', async () => { + await bucket.addLifecycleRule({ + action: { + type: 'Delete', + }, + condition: { + createdBefore: new Date('2018'), + }, + }); + const rules = [].slice.call(bucket.metadata.lifecycle?.rule); + assert.deepStrictEqual(rules.pop(), { + action: { + type: 'Delete', + }, + condition: { + createdBefore: '2018-01-01', + }, + }); + }); + + it('should add a noncurrent time rule', async () => { + const NONCURRENT_TIME_BEFORE = '2020-01-01'; + + await bucket.addLifecycleRule({ + action: { + type: 'Delete', + }, + condition: { + noncurrentTimeBefore: new Date(NONCURRENT_TIME_BEFORE), + daysSinceNoncurrentTime: 100, + }, + }); + + assert( + bucket.metadata.lifecycle!.rule!.some( + (rule: LifecycleRule) => + typeof rule.action === 'object' && + rule.action.type === 'Delete' && + rule.condition.noncurrentTimeBefore === NONCURRENT_TIME_BEFORE && + rule.condition.daysSinceNoncurrentTime === 100 + ) + ); + }); + + it('should add a custom time rule', async () => { + const CUSTOM_TIME_BEFORE = '2020-01-01'; + + await bucket.addLifecycleRule({ + action: { + type: 'Delete', + }, + condition: { + customTimeBefore: new Date(CUSTOM_TIME_BEFORE), + daysSinceCustomTime: 100, + }, + }); + + assert( + bucket.metadata.lifecycle!.rule!.some( + (rule: LifecycleRule) => + typeof rule.action === 'object' && + rule.action.type === 'Delete' && + rule.condition.customTimeBefore === CUSTOM_TIME_BEFORE && + rule.condition.daysSinceCustomTime === 100 + ) + ); + }); + + it('should remove all existing rules', async () => { + await bucket.setMetadata({ + lifecycle: null, + }); + + assert.strictEqual(bucket.metadata.lifecycle, undefined); + }); + }); + + describe('cors configuration', () => { + const corsEntry = [ + { + maxAgeSeconds: 1600, + }, + { + maxAgeSeconds: 3600, + method: ['GET', 'POST'], + origin: ['*'], + responseHeader: ['Content-Type', 'Access-Control-Allow-Origin'], + }, + ]; + + describe('bucket', () => { + it('should create a bucket with a CORS configuration when passed in', async () => { + const bucket = storage.bucket(generateName()); + await storage.createBucket(bucket.name, { + cors: corsEntry, + }); + + await bucket.getMetadata(); + assert.deepStrictEqual(bucket.metadata.cors, corsEntry); + }); + + it('should set a CORS configuration', async () => { + const bucket = storage.bucket(generateName()); + await bucket.create(); + await bucket.setCorsConfiguration(corsEntry); + await bucket.getMetadata(); + assert.deepStrictEqual(bucket.metadata.cors, corsEntry); + }); + + it('should remove a CORS configuration', async () => { + const bucket = storage.bucket(generateName()); + await bucket.create(); + await bucket.setCorsConfiguration(corsEntry); + await bucket.getMetadata(); + assert.deepStrictEqual(bucket.metadata.cors, corsEntry); + + // And now test the removing + await bucket.setCorsConfiguration([]); + assert.ok(!bucket.metadata.cors); + }); + }); + }); + + describe('bucket versioning', () => { + describe('bucket', () => { + it('should create a bucket with versioning enabled', async () => { + const bucket = storage.bucket(generateName()); + await storage.createBucket(bucket.name, { + versioning: { + enabled: true, + }, + }); + await bucket.getMetadata(); + assert.strictEqual(bucket.metadata!.versioning!.enabled, true); + }); + + it('should by default create a bucket without versioning set', async () => { + const bucket = storage.bucket(generateName()); + await storage.createBucket(bucket.name); + await bucket.getMetadata(); + assert.strictEqual(bucket.metadata.versioning, undefined); + }); + }); + }); + + describe('bucket hierarchical namespace', async () => { + let bucket: Bucket; + + beforeEach(() => { + bucket = storage.bucket(generateName()); + }); + + afterEach(async () => { + try { + await bucket.delete(); + } catch { + //Ignore errors + } + }); + + it('should create a bucket without hierarchical namespace enabled (implicit)', async () => { + await storage.createBucket(bucket.name); + const [metadata] = await bucket.getMetadata(); + assert( + [undefined, false].includes(metadata?.hierarchicalNamespace?.enabled) + ); + }); + + it('should create a bucket without hierarchical namespace enabled (explicit)', async () => { + await storage.createBucket(bucket.name, { + hierarchicalNamespace: {enabled: false}, + }); + const [metadata] = await bucket.getMetadata(); + assert( + [undefined, false].includes(metadata?.hierarchicalNamespace?.enabled) + ); + }); + + it('should create a bucket with hierarchical namespace enabled', async () => { + await storage.createBucket(bucket.name, { + hierarchicalNamespace: {enabled: true}, + iamConfiguration: { + uniformBucketLevelAccess: { + enabled: true, + }, + }, + }); + const [metadata] = await bucket.getMetadata(); + assert(metadata.hierarchicalNamespace); + assert.strictEqual(metadata.hierarchicalNamespace.enabled, true); + }); + }); + + describe('bucket retention policies', () => { + describe('bucket', () => { + it('should create a bucket with a retention policy', async () => { + const bucket = storage.bucket(generateName()); + await storage.createBucket(bucket.name, { + retentionPolicy: { + retentionPeriod: RETENTION_DURATION_SECONDS, + }, + }); + await bucket.getMetadata(); + assert.strictEqual( + bucket.metadata!.retentionPolicy!.retentionPeriod, + `${RETENTION_DURATION_SECONDS}` + ); + }); + + it('should set a retention policy', async () => { + const bucket = storage.bucket(generateName()); + await bucket.create(); + await bucket.setRetentionPeriod(RETENTION_DURATION_SECONDS); + await bucket.getMetadata(); + assert.strictEqual( + bucket.metadata!.retentionPolicy!.retentionPeriod, + `${RETENTION_DURATION_SECONDS}` + ); + }); + + it('should lock the retention period', async () => { + const bucket = storage.bucket(generateName()); + await bucket.create(); + await bucket.setRetentionPeriod(RETENTION_DURATION_SECONDS); + await bucket.getMetadata(); + + await bucket.lock(bucket.metadata!.metageneration!.toString()); + await assert.rejects( + bucket.setRetentionPeriod(RETENTION_DURATION_SECONDS / 2), + (err: ApiError) => { + return err.code === 403; + } + ); + }); + + it('should remove a retention period', async () => { + const bucket = storage.bucket(generateName()); + await bucket.create(); + await bucket.setRetentionPeriod(RETENTION_DURATION_SECONDS); + await bucket.getMetadata(); + assert.strictEqual( + bucket.metadata!.retentionPolicy!.retentionPeriod, + `${RETENTION_DURATION_SECONDS}` + ); + + await bucket.removeRetentionPeriod(); + await bucket.getMetadata(); + assert.strictEqual(bucket.metadata.retentionPolicy, undefined); + }); + }); + + describe('file', () => { + const BUCKET = storage.bucket(generateName()); + const FILE = BUCKET.file(generateName()); + + const BUCKET_RETENTION_PERIOD = 1; + before(async () => { + await BUCKET.create({ + retentionPolicy: { + retentionPeriod: BUCKET_RETENTION_PERIOD, + }, + }); + await FILE.save('data'); + }); + + afterEach(() => { + return FILE.setMetadata({temporaryHold: null, eventBasedHold: null}); + }); + + after(done => { + setTimeout(() => FILE.delete(done), BUCKET_RETENTION_PERIOD * 1000); + }); + + it('should set and release an event-based hold', async () => { + await FILE.setMetadata({eventBasedHold: true}); + assert.strictEqual(FILE.metadata.eventBasedHold, true); + await FILE.setMetadata({eventBasedHold: false}); + assert.strictEqual(FILE.metadata.eventBasedHold, false); + }); + + it('should set and release a temporary hold', async () => { + await FILE.setMetadata({temporaryHold: true}); + assert.strictEqual(FILE.metadata.temporaryHold, true); + await FILE.setMetadata({temporaryHold: false}); + assert.strictEqual(FILE.metadata.temporaryHold, false); + }); + + it('should get an expiration date', async () => { + const [expirationDate] = await FILE.getExpirationDate(); + assert(expirationDate instanceof Date); + }); + }); + + describe('operations on held objects', () => { + const BUCKET = storage.bucket(generateName()); + const FILES: File[] = []; + + const RETENTION_PERIOD_SECONDS = 5; // Each test has this much time! + + async function createFile(): Promise { + const file = BUCKET.file(generateName()); + await file.save('data'); + FILES.push(file); + return file; + } + + before(async () => { + await BUCKET.create({ + retentionPolicy: { + retentionPeriod: RETENTION_PERIOD_SECONDS, + }, + }); + }); + + after(async () => { + await new Promise(resolve => + setTimeout(resolve, RETENTION_PERIOD_SECONDS * 1000) + ); + await Promise.all( + FILES.map(async file => { + return file.delete(); + }) + ); + }); + + it('should block an overwrite request', async () => { + const file = await createFile(); + assert.rejects(file.save('new data'), (err: ApiError) => { + assert.strictEqual(err.code, 403); + }); + }); + + it('should block a delete request', async () => { + const file = await createFile(); + assert.rejects(file.delete(), (err: ApiError) => { + assert.strictEqual(err.code, 403); + }); + }); + }); + }); + + describe('bucket logging', () => { + const PREFIX = 'sys-test'; + + it('should enable logging on current bucket by default', async () => { + const [metadata] = await bucket.enableLogging({prefix: PREFIX}); + assert.deepStrictEqual(metadata.logging, { + logBucket: bucket.id, + logObjectPrefix: PREFIX, + }); + }); + + it('should enable logging on another bucket', async () => { + const bucketForLogging = storage.bucket(generateName()); + await bucketForLogging.create(); + + const [metadata] = await bucket.enableLogging({ + bucket: bucketForLogging, + prefix: PREFIX, + }); + assert.deepStrictEqual(metadata.logging, { + logBucket: bucketForLogging.id, + logObjectPrefix: PREFIX, + }); + }); + }); + + describe('object retention lock', () => { + const fileName = generateName(); + let objectRetentionBucket: Bucket; + + before(async () => { + objectRetentionBucket = storage.bucket(generateName()); + }); + + after(async () => { + await objectRetentionBucket.deleteFiles({force: true}); + await objectRetentionBucket.delete(); + }); + + it('should create a bucket with object retention enabled', async () => { + const result = await objectRetentionBucket.create({ + enableObjectRetention: true, + }); + + assert.deepStrictEqual(result[0].metadata.objectRetention, { + mode: 'Enabled', + }); + }); + + it('should create a file with object retention enabled', async () => { + const time = new Date(); + time.setMinutes(time.getMinutes() + 1); + const retention = {mode: 'Unlocked', retainUntilTime: time.toISOString()}; + const file = new File(objectRetentionBucket, fileName); + await objectRetentionBucket.upload(FILES.big.path, { + metadata: { + retention, + }, + destination: fileName, + }); + const [metadata] = await file.getMetadata(); + assert.deepStrictEqual(metadata.retention, retention); + }); + + it('should disable object retention on the file', async () => { + const file = new File(objectRetentionBucket, fileName); + const [metadata] = await file.setMetadata( + {retention: null}, + {overrideUnlockedRetention: true} + ); + assert.strictEqual(metadata.retention, undefined); + }); + }); + + describe('requester pays', () => { + const HAS_2ND_PROJECT = + process.env.GCN_STORAGE_2ND_PROJECT_ID !== undefined; + let bucket: Bucket; + + before(async () => { + bucket = storage.bucket(generateName()); + + await bucket.create({ + requesterPays: true, + }); + }); + + after(async () => { + await bucket.delete(); + }); + + it('should have enabled requesterPays functionality', async () => { + const [metadata] = await bucket.getMetadata(); + assert.strictEqual(metadata.billing!.requesterPays, true); + }); + + // These tests will verify that the requesterPays functionality works from + // the perspective of another project. + (HAS_2ND_PROJECT ? describe : describe.skip)('existing bucket', () => { + const storageNonAllowList = new Storage({ + projectId: process.env.GCN_STORAGE_2ND_PROJECT_ID, + keyFilename: process.env.GCN_STORAGE_2ND_PROJECT_KEY, + retryOptions: { + idempotencyStrategy: IdempotencyStrategy.RetryAlways, + retryDelayMultiplier: 3, + }, + }); + // the source bucket, which will have requesterPays enabled. + let bucket: Bucket; + // the bucket object from the requesting user. + let bucketNonAllowList: Bucket; + + async function isRequesterPaysEnabled(): Promise { + const [metadata] = await bucket.getMetadata(); + const billing = metadata.billing || {}; + return !!billing && billing.requesterPays === true; + } + + before(async () => { + bucket = storage.bucket(generateName()); + bucketNonAllowList = storageNonAllowList.bucket(bucket.name); + await bucket.create(); + }); + + it('should enable requesterPays', async () => { + let isEnabled = await isRequesterPaysEnabled(); + assert.strictEqual(isEnabled, false); + await bucket.enableRequesterPays(); + isEnabled = await isRequesterPaysEnabled(); + assert.strictEqual(isEnabled, true); + }); + + it('should disable requesterPays', async () => { + await bucket.enableRequesterPays(); + let isEnabled = await isRequesterPaysEnabled(); + assert.strictEqual(isEnabled, true); + await bucket.disableRequesterPays(); + isEnabled = await isRequesterPaysEnabled(); + assert.strictEqual(isEnabled, false); + }); + + describe('methods that accept userProject', () => { + let file: File; + let notification: Notification; + let topicName: string; + + const USER_PROJECT_OPTIONS = { + userProject: process.env.GCN_STORAGE_2ND_PROJECT_ID, + }; + + // This acts as a test for the following methods: + // + // - file.save() + // -> file.createWriteStream() + before(async () => { + file = bucketNonAllowList.file(generateName()); + + await bucket.enableRequesterPays(); + const data = await bucket.iam.getPolicy(); + const policy = data[0]; + // Allow an absolute or relative path (from project root) + // for the key file. + let key2 = process.env.GCN_STORAGE_2ND_PROJECT_KEY; + if (key2 && key2.charAt(0) === '.') { + key2 = `${getDirName()}/../../../${key2}`; + } + // Get the service account for the "second" account (the + // one that will read the requester pays file). + const clientEmail = JSON.parse( + fs.readFileSync(key2!, 'utf-8') + ).client_email; + policy.bindings.push({ + role: 'roles/storage.admin', + members: [`serviceAccount:${clientEmail}`], + }); + await bucket.iam.setPolicy(policy); + await file.save('abc', USER_PROJECT_OPTIONS); + const data_2 = await topic.getMetadata(); + topicName = data_2[0].name!; + }); + + // This acts as a test for the following methods: + // + // - bucket.delete({ userProject: ... }) + // -> bucket.deleteFiles({ userProject: ... }) + // -> bucket.getFiles({ userProject: ... }) + // -> file.delete({ userProject: ... }) + after(done => { + deleteBucket(bucketNonAllowList, USER_PROJECT_OPTIONS, done); + }); + + beforeEach(() => { + bucketNonAllowList = storageNonAllowList.bucket(bucket.name); + file = bucketNonAllowList.file(file.name); + }); + + /** + * A type for requester pays functions. The `options` parameter + * + * Using `typeof USER_PROJECT_OPTIONS` ensures the function should + * support requester pays options, otherwise will fail early at build- + * time rather than runtime. + */ + type requesterPaysFunction< + T = {} | typeof USER_PROJECT_OPTIONS, + R = {} | void, + > = (options?: T) => Promise; + + /** + * Accepts a function and runs 2 tests - a test where the requester pays + * option is absent and another where it is present: + * - The missing requester pays test will assert the expected error + * - The added request pays test will return the function's result + * + * @param testFunction The function/method to test. + * @returns The result of the successful request pays operation. + */ + async function requesterPaysDoubleTest( + testFunction: F + ): Promise> { + const failureMessage = + 'Bucket is a requester pays bucket but no user project provided.'; + + await assert.rejects(testFunction(), err => { + assert( + (err as Error).message.includes(failureMessage), + `Expected '${ + (err as Error).message + }' to include '${failureMessage}'` + ); + return true; + }); + + // Validate the desired functionality + const results = await testFunction(USER_PROJECT_OPTIONS); + return results as ReturnType; + } + + it('bucket#combine', async () => { + const files = [ + {file: bucketNonAllowList.file('file-one.txt'), contents: '123'}, + {file: bucketNonAllowList.file('file-two.txt'), contents: '456'}, + ]; + + await Promise.all(files.map(file => createFileAsync(file))); + + const sourceFiles = files.map(x => x.file); + const destinationFile = bucketNonAllowList.file('file-one-n-two.txt'); + await bucketNonAllowList.combine( + sourceFiles, + destinationFile, + USER_PROJECT_OPTIONS + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function createFileAsync(fileObject: any) { + return fileObject.file.save( + fileObject.contents, + USER_PROJECT_OPTIONS + ); + } + }); + + it('bucket#createNotification', async () => { + const [notif] = await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.createNotification(topicName, options); + }); + + notification = notif; + }); + + it('bucket#exists', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.exists(options); + }); + }); + + it('bucket#get', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.get(options); + }); + }); + + it('bucket#getMetadata', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.getMetadata(options); + }); + }); + + it('bucket#getNotifications', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.getNotifications(options); + }); + }); + + it('bucket#makePrivate', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.makePrivate(options); + }); + }); + + it('bucket#setMetadata', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.setMetadata({newMetadata: true}, options); + }); + }); + + it('bucket#setStorageClass', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.setStorageClass( + 'multi-regional', + options + ); + }); + }); + + it('bucket#upload', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.upload(FILES.big.path, options); + }); + }); + + it('file#copy', async () => { + await requesterPaysDoubleTest(async options => { + return file.copy('new-file.txt', options); + }); + }); + + it('file#createReadStream', async () => { + await requesterPaysDoubleTest(async options => { + await new Promise((resolve, reject) => { + return file + .createReadStream(options) + .on('error', reject) + .on('end', resolve) + .on('data', () => {}); + }); + }); + }); + + it('file#createResumableUpload', async () => { + await requesterPaysDoubleTest(async options => { + const [uri] = await file.createResumableUpload(options); + + await new Promise((resolve, reject) => { + return file + .createWriteStream({uri}) + .on('error', reject) + .on('finish', resolve) + .end('Test data'); + }); + }); + }); + + it('file#download', async () => { + await requesterPaysDoubleTest(async options => { + return file.download(options); + }); + }); + + it('file#exists', async () => { + await requesterPaysDoubleTest(async options => { + return file.exists(options); + }); + }); + + it('file#get', async () => { + await requesterPaysDoubleTest(async options => { + return file.get(options); + }); + }); + + it('file#getMetadata', async () => { + await requesterPaysDoubleTest(async options => { + return file.getMetadata(options); + }); + }); + + it('file#makePrivate', async () => { + await requesterPaysDoubleTest(async options => { + return file.makePrivate(options); + }); + }); + + it('file#move', async () => { + await requesterPaysDoubleTest(async options => { + const newFile = bucketNonAllowList.file(generateName()); + + await file.move(newFile, options); + + // Re-create the file. The tests need it. + await file.save('newcontent', options); + }); + }); + + it('file#rename', async () => { + await requesterPaysDoubleTest(async options => { + const newFile = bucketNonAllowList.file(generateName()); + + await file.rename(newFile, options); + + // Re-create the file. The tests need it. + await file.save('newcontent', options); + }); + }); + + it('file#setMetadata', async () => { + await requesterPaysDoubleTest(async options => { + return file.setMetadata({newMetadata: true}, options); + }); + }); + + it('file#setStorageClass', async () => { + await requesterPaysDoubleTest(async options => { + return file.setStorageClass('multi-regional', options); + }); + }); + + it('acl#add', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.acl.add({ + entity: USER_ACCOUNT, + role: storage.acl.OWNER_ROLE, + ...options, + }); + }); + }); + + it('acl#update', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.acl.update({ + entity: USER_ACCOUNT, + role: storage.acl.WRITER_ROLE, + ...options, + }); + }); + }); + + it('acl#get', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.acl.get({ + entity: USER_ACCOUNT, + ...options, + }); + }); + }); + + it('acl#delete', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.acl.delete({ + entity: USER_ACCOUNT, + ...options, + }); + }); + }); + + it('iam#getPolicy', async () => { + await requesterPaysDoubleTest(async options => { + return bucketNonAllowList.iam.getPolicy(options); + }); + }); + + /** + * TODO: Re-enable once the test environment allows public IAM roles. + * Currently disabled to avoid 403 errors when adding 'allUsers' or + * 'allAuthenticatedUsers' permissions. + */ + it.skip('iam#setPolicy', async () => { + await requesterPaysDoubleTest(async options => { + const [policy] = await bucket.iam.getPolicy(); + + policy.bindings.push({ + role: 'roles/storage.objectViewer', + members: ['allUsers'], + }); + + return bucketNonAllowList.iam.setPolicy(policy, options); + }); + }); + + it('iam#testPermissions', async () => { + await requesterPaysDoubleTest(async options => { + const tests = ['storage.buckets.delete']; + + return bucketNonAllowList.iam.testPermissions(tests, options); + }); + }); + + it('notification#get', async () => { + await requesterPaysDoubleTest(async options => { + return notification.get(options); + }); + }); + + it('notification#getMetadata', async () => { + await requesterPaysDoubleTest(async options => { + return notification.getMetadata(options); + }); + }); + + it('notification#delete', async () => { + await requesterPaysDoubleTest(async options => { + return notification.delete(options); + }); + }); + }); + }); + }); + + describe('write, read, and remove files', () => { + const FILE_DOWNLOAD_START_BYTE = 0; + const FILE_DOWNLOAD_END_BYTE = 20; + + before(async () => { + function setHash(filesKey: string) { + const file = FILES[filesKey]; + const hash = crypto.createHash('md5'); + + return new Promise(resolve => + fs + .createReadStream(file.path) + .on('data', hash.update.bind(hash)) + .on('end', () => { + file.hash = hash.digest('base64'); + resolve(); + }) + ); + } + await Promise.all(Object.keys(FILES).map(key => setHash(key))); + }); + + it('should read/write from/to a file in a directory', done => { + const file = bucket.file('directory/file'); + const contents = 'test'; + + const writeStream = file.createWriteStream({resumable: false}); + writeStream.write(contents); + writeStream.end(); + + writeStream.on('error', done); + writeStream.on('finish', () => { + let data = Buffer.from('', 'utf8'); + + file + .createReadStream() + .on('error', done) + .on('data', (chunk: Buffer) => { + data = Buffer.concat([data, chunk]); + }) + .on('end', () => { + assert.strictEqual(data.toString(), contents); + done(); + }); + }); + }); + + it('should not push data when a file cannot be read', done => { + const file = bucket.file('non-existing-file'); + let dataEmitted = false; + + file + .createReadStream() + .on('data', () => { + dataEmitted = true; + }) + .on('error', err => { + assert.strictEqual(dataEmitted, false); + assert.strictEqual((err as ApiError).code, 404); + done(); + }); + }); + + it('should throw original error message on non JSON response on large metadata', async () => { + const largeCustomMeta = (size: number) => { + let str = ''; + for (let i = 0; i < size; i++) { + str += 'a'; + } + return str; + }; + + const file = bucket.file('large-metadata-error-test'); + await assert.rejects( + file.save('test', { + resumable: false, + metadata: { + metadata: { + custom: largeCustomMeta(2.1e6), + }, + }, + }), + /Metadata part is too large/ + ); + }); + + it('should read a byte range from a file', done => { + bucket.upload(FILES.big.path, (err: Error | null, file?: File | null) => { + assert.ifError(err); + + const fileSize = parseInt(file!.metadata.size!.toString()); + const byteRange = { + start: Math.floor((fileSize * 1) / 3), + end: Math.floor((fileSize * 2) / 3), + }; + const expectedContentSize = byteRange.start + 1; + + let sizeStreamed = 0; + file! + .createReadStream(byteRange) + .on('data', chunk => { + sizeStreamed += chunk.length; + }) + .on('error', done) + .on('end', () => { + assert.strictEqual(sizeStreamed, expectedContentSize); + file!.delete(done); + }); + }); + }); + + it('should support readable[Symbol.asyncIterator]()', async () => { + const fileContents = fs.readFileSync(FILES.big.path); + + const [file] = await bucket.upload(FILES.big.path); + const stream = file.createReadStream(); + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const remoteContents = Buffer.concat(chunks).toString(); + assert.strictEqual(String(fileContents), String(remoteContents)); + }); + + it('should download a file to memory', async () => { + const fileContents = fs.readFileSync(FILES.big.path); + const [file] = await bucket.upload(FILES.big.path); + const [remoteContents] = await file.download(); + assert.strictEqual(String(fileContents), String(remoteContents)); + }); + + it('should download an entire file if range `start:0` is provided', async () => { + const fileContents = fs.readFileSync(FILES.big.path); + const [file] = await bucket.upload(FILES.big.path); + const [result] = await file.download({start: 0}); + + assert.strictEqual(result.toString(), fileContents.toString()); + }); + + it('should download an empty file', async () => { + const fileContents = fs.readFileSync(FILES.empty.path); + const [file] = await bucket.upload(FILES.empty.path); + const [remoteContents] = await file.download(); + assert.strictEqual(String(fileContents), String(remoteContents)); + }); + + it('should download the specified bytes of a file', async () => { + const fileContents = fs.readFileSync(FILES.big.path); + const [file] = await bucket.upload(FILES.big.path); + const [remoteContents] = await file!.download({ + start: FILE_DOWNLOAD_START_BYTE, + end: FILE_DOWNLOAD_END_BYTE, + }); + assert.strictEqual( + String(fileContents).slice(0, 20), + String(remoteContents) + ); + }); + + it('should handle non-network errors', async () => { + const file = bucket.file('hi.jpg'); + assert.rejects(file.download(), (err: ApiError) => { + assert.strictEqual((err as ApiError).code, 404); + }); + }); + + it('should gzip a file on the fly and download it', async () => { + const options = { + gzip: true, + }; + const expectedContents = fs.readFileSync(FILES.html.path, 'utf-8'); + const [file] = await bucket.upload(FILES.html.path, options); + const [contents] = await file.download(); + assert.strictEqual(contents.toString(), expectedContents); + await file.delete(); + }); + + it('should upload a gzipped file and download it', async () => { + const options = { + metadata: { + contentEncoding: 'gzip', + contentType: 'text/html', + }, + }; + + const expectedContents = fs.readFileSync(FILES.html.path, 'utf-8'); + + // Prepare temporary gzip file for upload + tmp.setGracefulCleanup(); + const {name: tmpGzFilePath} = tmp.fileSync({postfix: '.gz'}); + fs.writeFileSync(tmpGzFilePath, gzipSync(expectedContents)); + + const file: File = await new Promise((resolve, reject) => { + bucket.upload(tmpGzFilePath, options, (err, file) => { + if (err || !file) return reject(err); + resolve(file); + }); + }); + + const contents: Buffer = await new Promise((resolve, reject) => { + return file.download((error, content) => { + if (error) return reject(error); + resolve(content); + }); + }); + + assert.strictEqual(contents.toString(), expectedContents); + await file.delete(); + }); + + it('should skip validation if file is served decompressed', async () => { + const filename = 'logo-gzipped.png'; + await bucket.upload(FILES.logo.path, {destination: filename, gzip: true}); + + tmp.setGracefulCleanup(); + const {name: tmpFilePath} = tmp.fileSync(); + + const file = bucket.file(filename); + + await new Promise((resolve, reject) => { + file + .createReadStream() + .on('error', reject) + .on('response', raw => { + assert.strictEqual( + raw.toJSON().headers['content-encoding'], + undefined + ); + }) + .pipe(fs.createWriteStream(tmpFilePath)) + .on('error', reject) + .on('finish', () => resolve()); + }); + + await file.delete(); + }); + + describe('simple write', () => { + it('should save arbitrary data', async () => { + const file = bucket.file('TestFile'); + const data = 'hello'; + await file!.save(data); + const [contents] = await file!.download(); + assert.strictEqual(contents.toString(), data); + }); + }); + + describe('stream write', () => { + it('should stream write, then remove file (3mb)', done => { + const file = bucket.file('LargeFile'); + fs.createReadStream(FILES.big.path) + .pipe(file!.createWriteStream({resumable: false})) + .on('error', done) + .on('finish', () => { + assert.strictEqual(file.metadata.md5Hash, FILES.big.hash); + file.delete(done); + }); + }); + + it('should write metadata', async () => { + const options = { + metadata: {contentType: 'image/png'}, + resumable: false, + }; + const [file] = await bucket.upload(FILES.logo.path, options); + const [metadata] = await file.getMetadata(); + assert.strictEqual(metadata.contentType, options.metadata.contentType); + await file.delete(); + }); + + it('should resume an upload after an interruption', done => { + fs.stat(FILES.big.path, (err, metadata) => { + assert.ifError(err); + + const file = bucket.file(generateName()); + const fileSize = metadata.size; + upload({interrupt: true}, err => { + assert.strictEqual(err!.message, 'Interrupted.'); + upload({interrupt: false}, err => { + assert.ifError(err); + assert.strictEqual(Number(file.metadata.size), fileSize); + file.delete(done); + }); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function upload(opts: any, callback: ErrorCallbackFunction) { + const ws = file.createWriteStream(); + let sizeStreamed = 0; + + const streamTransform = new Transform({ + transform(chunk, enc, next) { + sizeStreamed += chunk.length; + + if (opts.interrupt && sizeStreamed >= fileSize / 2) { + // stop sending data half way through. + this.push(chunk); + this.destroy(); + process.nextTick(() => { + ws.destroy(new Error('Interrupted.')); + }); + } else { + this.push(chunk); + next(); + } + }, + }); + fs.createReadStream(FILES.big.path) + .pipe(streamTransform) + .pipe(ws) + .on('error', callback) + .on('finish', callback); + } + }); + }); + + it('should write/read/remove from a buffer', done => { + tmp.setGracefulCleanup(); + tmp.file((err, tmpFilePath) => { + assert.ifError(err); + + const file = bucket.file('MyBuffer'); + const fileContent = 'Hello World'; + + const writable = file.createWriteStream(); + + writable.write(fileContent); + writable.end(); + + writable.on('finish', () => { + file + .createReadStream() + .on('error', done) + .pipe(fs.createWriteStream(tmpFilePath)) + .on('error', done) + .on('finish', () => { + file.delete((err: ApiError | null) => { + assert.ifError(err); + + fs.readFile(tmpFilePath, (err, data) => { + assert.strictEqual(data.toString(), fileContent); + done(); + }); + }); + }); + }); + }); + }); + }); + + describe('customer-supplied encryption keys', () => { + const encryptionKey = crypto.randomBytes(32); + + const file = bucket.file('encrypted-file', { + encryptionKey, + }); + const unencryptedFile = bucket.file(file.name); + + before(async () => { + await file.save('secret data', {resumable: false}); + }); + + it('should not get the hashes from the unencrypted file', async () => { + const [metadata] = await unencryptedFile.getMetadata(); + assert.strictEqual(metadata.crc32c, undefined); + }); + + it('should get the hashes from the encrypted file', async () => { + const [metadata] = await file.getMetadata(); + assert.notStrictEqual(metadata.crc32c, undefined); + }); + + it('should not download from the unencrypted file', async () => { + assert.rejects(unencryptedFile.download(), (err: ApiError) => { + assert( + err!.message.indexOf( + [ + 'The target object is encrypted by a', + 'customer-supplied encryption key.', + ].join(' ') + ) > -1 + ); + }); + }); + + it('should download from the encrypted file', async () => { + const [contents] = await file.download(); + assert.strictEqual(contents.toString(), 'secret data'); + }); + + it('should rotate encryption keys', async () => { + const newEncryptionKey = crypto.randomBytes(32); + await file.rotateEncryptionKey(newEncryptionKey); + const [contents] = await file.download(); + assert.strictEqual(contents.toString(), 'secret data'); + }); + }); + + describe('kms keys', () => { + const FILE_CONTENTS = 'secret data'; + + const BUCKET_LOCATION = 'us'; + let PROJECT_ID: string; + let SERVICE_ACCOUNT_EMAIL: string; + + const keyRingId = generateName(); + const cryptoKeyId = generateName(); + + const request = promisify(storage.request).bind(storage); + + let bucket: Bucket; + let kmsKeyName: string; + let keyRingsBaseUrl: string; + + function setProjectId(projectId: string) { + PROJECT_ID = projectId; + keyRingsBaseUrl = `https://cloudkms.googleapis.com/v1/projects/${PROJECT_ID}/locations/${BUCKET_LOCATION}/keyRings`; + kmsKeyName = generateKmsKeyName(cryptoKeyId); + } + + function generateKmsKeyName(cryptoKeyId: string) { + return `projects/${PROJECT_ID}/locations/${BUCKET_LOCATION}/keyRings/${keyRingId}/cryptoKeys/${cryptoKeyId}`; + } + + async function createCryptoKeyAsync(cryptoKeyId: string) { + // createCryptoKeyId + await request({ + method: 'POST', + uri: `${keyRingsBaseUrl}/${keyRingId}/cryptoKeys`, + qs: {cryptoKeyId}, + json: {purpose: 'ENCRYPT_DECRYPT'}, + }); + + // getServiceAccountEmail + if (!SERVICE_ACCOUNT_EMAIL) { + const [serviceAccount] = await storage.getServiceAccount(); + SERVICE_ACCOUNT_EMAIL = serviceAccount!.emailAddress!; + } + + await request({ + method: 'POST', + uri: `${keyRingsBaseUrl}/${keyRingId}/cryptoKeys/${cryptoKeyId}:setIamPolicy`, + json: { + policy: { + bindings: [ + { + role: 'roles/cloudkms.cryptoKeyEncrypterDecrypter', + members: `serviceAccount:${SERVICE_ACCOUNT_EMAIL}`, + }, + ], + }, + }, + }); + } + + before(async () => { + bucket = storage.bucket(generateName()); + + setProjectId(await storage.authClient.getProjectId()); + await bucket.create({location: BUCKET_LOCATION}); + + // create keyRing + await request({ + method: 'POST', + uri: keyRingsBaseUrl, + qs: {keyRingId}, + }); + + await createCryptoKeyAsync(cryptoKeyId); + }); + + describe('files', () => { + let file: File; + + before(async () => { + file = bucket.file('kms-encrypted-file', {kmsKeyName}); + await file.save(FILE_CONTENTS, {resumable: false}); + }); + + it('should have set kmsKeyName on created file', async () => { + const [metadata] = await file.getMetadata(); + + // Strip the project ID, as it could be the placeholder locally, but + // the real value upstream. + const projectIdRegExp = /^.+\/locations/; + const actualKmsKeyName = metadata!.kmsKeyName!.replace( + projectIdRegExp, + '' + ); + let expectedKmsKeyName = kmsKeyName.replace(projectIdRegExp, ''); + + // Upstream attaches a version. + expectedKmsKeyName = `${expectedKmsKeyName}/cryptoKeyVersions/1`; + + assert.strictEqual(actualKmsKeyName, expectedKmsKeyName); + }); + + it('should set kmsKeyName on resumable uploaded file', async () => { + const file = bucket.file('resumable-file', {kmsKeyName}); + await file.save(FILE_CONTENTS, {resumable: true}); + const [metadata] = await file.getMetadata(); + + // Strip the project ID, as it could be the placeholder locally, + // but the real value upstream. + const projectIdRegExp = /^.+\/locations/; + const actualKmsKeyName = metadata!.kmsKeyName!.replace( + projectIdRegExp, + '' + ); + let expectedKmsKeyName = kmsKeyName.replace(projectIdRegExp, ''); + + // Upstream attaches a version. + expectedKmsKeyName = `${expectedKmsKeyName}/cryptoKeyVersions/1`; + + assert.strictEqual(actualKmsKeyName, expectedKmsKeyName); + }); + + it('should rotate encryption keys', async () => { + const cryptoKeyId = generateName(); + const newKmsKeyName = generateKmsKeyName(cryptoKeyId); + + await createCryptoKeyAsync(cryptoKeyId); + await file.rotateEncryptionKey({kmsKeyName: newKmsKeyName}); + const [contents] = await file.download(); + assert.strictEqual(contents.toString(), FILE_CONTENTS); + }); + + it('should convert CSEK to KMS key', async () => { + const encryptionKey = crypto.randomBytes(32); + const file = bucket.file('encrypted-file', {encryptionKey}); + await file.save(FILE_CONTENTS, {resumable: false}); + await file.rotateEncryptionKey({kmsKeyName}); + const [contents] = await file.download(); + assert.strictEqual(contents.toString(), 'secret data'); + }); + }); + + describe('buckets', () => { + let bucket: Bucket; + + before(async () => { + bucket = storage.bucket(generateName(), {kmsKeyName}); + await bucket.create(); + }); + + beforeEach(async () => { + await new Promise(res => + setTimeout(res, BUCKET_METADATA_UPDATE_WAIT_TIME) + ); + await bucket.setMetadata({ + encryption: { + defaultKmsKeyName: kmsKeyName, + }, + }); + }); + + afterEach(async () => { + await new Promise(res => + setTimeout(res, BUCKET_METADATA_UPDATE_WAIT_TIME) + ); + await bucket.setMetadata({ + encryption: null, + }); + }); + + it('should have set defaultKmsKeyName on created bucket', async () => { + const [metadata] = await bucket.getMetadata(); + // Strip the project ID, as it could be the placeholder locally, but + // the real value upstream. + const projectIdRegExp = /^.+\/locations/; + const actualKmsKeyName = + metadata!.encryption!.defaultKmsKeyName!.replace( + projectIdRegExp, + '' + ); + const expectedKmsKeyName = kmsKeyName.replace(projectIdRegExp, ''); + assert.strictEqual(actualKmsKeyName, expectedKmsKeyName); + }); + + it('should update the defaultKmsKeyName', async () => { + const cryptoKeyId = generateName(); + const newKmsKeyName = generateKmsKeyName(cryptoKeyId); + + await createCryptoKeyAsync(cryptoKeyId); + await new Promise(res => + setTimeout(res, BUCKET_METADATA_UPDATE_WAIT_TIME) + ); + await bucket.setMetadata({ + encryption: { + defaultKmsKeyName: newKmsKeyName, + }, + }); + }); + + it('should insert an object that inherits the kms key name', async () => { + const file = bucket.file('kms-encrypted-file'); + const [metadata] = await bucket.getMetadata(); + await file.save(FILE_CONTENTS, {resumable: false}); + const [fileMetadata] = await file.getMetadata(); + + assert.strictEqual( + fileMetadata.kmsKeyName, + `${metadata!.encryption!.defaultKmsKeyName}/cryptoKeyVersions/1` + ); + }); + }); + }); + + it('should copy an existing file', async () => { + const opts = {destination: 'CloudLogo'}; + const [file] = await bucket.upload(FILES.logo.path, opts); + const [copiedFile] = await file.copy('CloudLogoCopy'); + await Promise.all([file.delete, copiedFile.delete()]); + }); + + it('should copy an existing file and overwrite custom metadata', async () => { + const opts = { + destination: 'CloudLogo', + metadata: { + metadata: { + originalProperty: 'true', + }, + }, + }; + const [file] = await bucket.upload(FILES.logo.path, opts); + const copyOpts = {metadata: {newProperty: 'true'}}; + const [copiedFile] = await file.copy('CloudLogoCopy', copyOpts); + const [metadata] = await copiedFile.getMetadata(); + assert.strictEqual( + typeof metadata!.metadata!.originalProperty, + 'undefined' + ); + assert.strictEqual(metadata!.metadata!.newProperty, 'true'); + await Promise.all([file.delete, copiedFile.delete()]); + }); + + it('should copy an existing file and overwrite metadata', async () => { + const opts = { + destination: 'CloudLogo', + }; + const CACHE_CONTROL = 'private'; + const CONTENT_ENCODING = 'gzip'; + const CONTENT_TYPE = 'text/plain'; + const [file] = await bucket.upload(FILES.logo.path, opts); + const copyOpts = { + cacheControl: CACHE_CONTROL, + contentEncoding: CONTENT_ENCODING, + contentType: CONTENT_TYPE, + }; + const [copiedFile] = await file.copy('CloudLogoCopy', copyOpts); + const [metadata] = await copiedFile.getMetadata(); + assert.strictEqual(metadata.contentEncoding, CONTENT_ENCODING); + assert.strictEqual(metadata.cacheControl, CACHE_CONTROL); + assert.strictEqual(metadata.contentType, CONTENT_TYPE); + await Promise.all([file.delete, copiedFile.delete()]); + }); + + /** + * TODO: Re-enable once the test environment allows public IAM roles. + * Currently disabled to avoid 403 errors when adding 'allUsers' or + * 'allAuthenticatedUsers' permissions. + */ + it.skip('should respect predefined Acl at file#copy', async () => { + const opts = {destination: 'CloudLogo'}; + const [file] = await bucket.upload(FILES.logo.path, opts); + const copyOpts = {predefinedAcl: 'publicRead'}; + const [copiedFile] = await file.copy('CloudLogoCopy', copyOpts); + const publicAcl = await isFilePublicAsync(copiedFile); + assert.strictEqual(publicAcl, true); + await Promise.all([file.delete, copiedFile.delete()]); + }); + + it('should copy a large file', async () => { + const otherBucket = storage.bucket(generateName()); + const file = bucket.file('Big'); + const copiedFile = otherBucket.file(file.name); + await bucket.upload(FILES.logo.path, {destination: file}); + await otherBucket.create({ + location: 'ASIA-EAST1', + dra: true, + }); + await file.copy(copiedFile); + await copiedFile.delete(); + await otherBucket.delete(); + await file.delete(); + }); + + it('should copy to another bucket given a gs:// URL', async () => { + const opts = {destination: 'CloudLogo'}; + const [file] = await bucket.upload(FILES.logo.path, opts); + const otherBucket = storage.bucket(generateName()); + await otherBucket.create(); + const destPath = 'gs://' + otherBucket.name + '/CloudLogoCopy'; + await file!.copy(destPath); + const [files] = await otherBucket.getFiles(); + assert.strictEqual(files!.length, 1); + const newFile = files![0]; + assert.strictEqual(newFile.name, 'CloudLogoCopy'); + }); + + it('should allow changing the storage class', async () => { + const file = bucket.file(generateName()); + await bucket.upload(FILES.logo.path, {destination: file}); + await file.setStorageClass('standard'); + const [metadata] = await file.getMetadata(); + assert.strictEqual(metadata.storageClass, 'STANDARD'); + }); + }); + + describe('resumable upload', () => { + describe('multi-chunk upload', () => { + describe('upload configurations', () => { + const filePath: string = FILES.big.path; + const fileSize = fs.statSync(filePath).size; + let crc32c: string; + + before(async () => { + // get a CRC32C value from the file + crc32c = (await CRC32C.fromFile(filePath)).toString(); + }); + + async function uploadAndVerify( + file: File, + options: Omit + ) { + await bucket.upload(filePath, { + destination: file, + ...options, + }); + + const [metadata] = await file.getMetadata(); + + // assert we uploaded the expected data + assert.equal(metadata.crc32c, crc32c); + } + + it('should support uploads where `contentLength < chunkSize`', async () => { + const file = bucket.file(generateName()); + + const metadata = {contentLength: fileSize}; + // off by +1 to ensure `contentLength < chunkSize` + const chunkSize = fileSize + 1; + + await uploadAndVerify(file, {chunkSize, metadata}); + }); + + it('should support uploads where `contentLength % chunkSize != 0`', async () => { + const file = bucket.file(generateName()); + + const metadata = {contentLength: fileSize}; + // off by -1 to ensure `contentLength % chunkSize != 0` + const chunkSize = fileSize - 1; + + await uploadAndVerify(file, {chunkSize, metadata}); + }); + + it('should support uploads where `fileSize % chunkSize != 0` && `!contentLength`', async () => { + const file = bucket.file(generateName()); + // off by +1 to ensure `fileSize % chunkSize != 0` + const chunkSize = fileSize + 1; + + await uploadAndVerify(file, {chunkSize}); + }); + + it('should support uploads where `fileSize < chunkSize && `!contentLength`', async () => { + const file = bucket.file(generateName()); + // off by `* 2 +1` to ensure `fileSize < chunkSize` + const chunkSize = fileSize * 2 + 1; + + await uploadAndVerify(file, {chunkSize}); + }); + + it('should support uploads where `fileSize > chunkSize` && `!contentLength`', async () => { + const file = bucket.file(generateName()); + // off by -1 to ensure `fileSize > chunkSize` + const chunkSize = fileSize - 1; + + await uploadAndVerify(file, {chunkSize}); + }); + }); + }); + }); + + describe('bucket upload with progress', () => { + it('show bytes sent with resumable upload', async () => { + const fileSize = fs.statSync(FILES.big.path).size; + let called = false; + function onUploadProgress(evt: {bytesWritten: number}) { + called = true; + assert.strictEqual(typeof evt.bytesWritten, 'number'); + assert.ok(evt.bytesWritten >= 0 && evt.bytesWritten <= fileSize); + } + + await bucket.upload(FILES.big.path, { + resumable: true, + onUploadProgress, + }); + + assert.strictEqual(called, true); + }); + + it('show bytes sent with simple upload', async () => { + const fileSize = fs.statSync(FILES.big.path).size; + let called = false; + function onUploadProgress(evt: {bytesWritten: number}) { + called = true; + assert.strictEqual(typeof evt.bytesWritten, 'number'); + assert.ok(evt.bytesWritten >= 0 && evt.bytesWritten <= fileSize); + } + await bucket.upload(FILES.big.path, { + resumable: false, + onUploadProgress, + }); + + assert.strictEqual(called, true); + }); + }); + + describe('channels', () => { + it('should stop a channel', async () => { + // We can't actually create a channel. But we can test to see that we're + // reaching the right endpoint with the API request. + const channel = storage.channel('id', 'resource-id'); + assert.rejects(channel.stop(), (err: ApiError) => { + assert.strictEqual((err as ApiError).code, 404); + assert.strictEqual(err!.message.indexOf("Channel 'id' not found"), 0); + }); + }); + }); + + describe('file#moveFileAtomic', async () => { + it('Should move a file to a new name within the bucket.', async () => { + // Create a source file in the bucket and save some content. + const f1 = bucket.file('move-src-obj'); + await f1.save('move-src-obj'); + assert(f1); + const [f1_metadata] = await f1.getMetadata(); + + // Move the source file to a new destination name within the same bucket. + await f1.moveFileAtomic('move-dst-obj'); + const f2 = bucket.file('move-dst-obj'); + assert(f2); + const [f2_metadata] = await f2.getMetadata(); + + // Assert that the generation of the destination file is different from the source file, + // indicating a new file was created. + assert.notStrictEqual(f1_metadata.generation, f2_metadata.generation); + + const [f1_exists] = await f1.exists(); + const [f2_exists] = await f2.exists(); + // Assert that the source file no longer exists after the move. + assert.strictEqual(f1_exists, false); + // Assert that the destination file exists after the move. + assert.strictEqual(f2_exists, true); + }); + }); + + describe('combine files', () => { + it('should combine multiple files into one', async () => { + const files = [ + {file: bucket.file('file-one.txt'), contents: '123'}, + {file: bucket.file('file-two.txt'), contents: '456'}, + ]; + + await Promise.all(files.map(file => createFileAsync(file))); + + const sourceFiles = files.map(x => x.file); + let destinationFile = bucket.file('file-one-and-two.txt'); + + [destinationFile] = await bucket.combine(sourceFiles, destinationFile); + const [contents] = await destinationFile.download(); + assert.strictEqual( + contents.toString(), + files.map(x => x.contents).join('') + ); + + await Promise.all( + sourceFiles.concat([destinationFile]).map(file => deleteFileAsync(file)) + ); + }); + }); + + // TODO: Undo this skip once https://github.com/googleapis/nodejs-storage/issues/2461 is complete. + describe.skip('HMAC keys', () => { + // This is generally a valid service account for a project. + const ALTERNATE_SERVICE_ACCOUNT = `${process.env.PROJECT_ID}@appspot.gserviceaccount.com`; + const SERVICE_ACCOUNT = + process.env.HMAC_KEY_TEST_SERVICE_ACCOUNT || ALTERNATE_SERVICE_ACCOUNT; + const HMAC_PROJECT = process.env.HMAC_KEY_TEST_SERVICE_ACCOUNT + ? process.env.HMAC_PROJECT + : process.env.PROJECT_ID; + // Second service account to test listing HMAC keys from different accounts. + const SECOND_SERVICE_ACCOUNT = + process.env.HMAC_KEY_TEST_SECOND_SERVICE_ACCOUNT; + + let accessId: string; + + const delay = async (test: Mocha.Context, accessId: string) => { + const retries = test.currentRetry(); + // see: https://cloud.google.com/storage/docs/exponential-backoff: + const ms = Math.pow(2, retries) * 1000 + Math.random() * 1000; + return new Promise(done => { + console.info( + `retrying "${test.title}" with accessId ${accessId} in ${ms}ms` + ); + setTimeout(done, ms); + }); + }; + + before(async () => { + await deleteStaleHmacKeys(SERVICE_ACCOUNT, HMAC_PROJECT!); + if (SECOND_SERVICE_ACCOUNT) { + await deleteStaleHmacKeys(SECOND_SERVICE_ACCOUNT, HMAC_PROJECT!); + } + }); + + it('should create an HMAC key for a service account', async () => { + const [hmacKey, secret] = await storage.createHmacKey(SERVICE_ACCOUNT, { + projectId: HMAC_PROJECT, + }); + // We should always get a 40 character secret, which is valid base64. + assert.strictEqual(secret.length, 40); + accessId = hmacKey.id!; + const metadata = hmacKey.metadata!; + assert.strictEqual(metadata.accessId, accessId); + assert.strictEqual(metadata.state, 'ACTIVE'); + assert.strictEqual(metadata.projectId, HMAC_PROJECT); + assert.strictEqual(metadata.serviceAccountEmail, SERVICE_ACCOUNT); + assert(typeof metadata.etag === 'string'); + assert(typeof metadata.timeCreated === 'string'); + assert(typeof metadata.updated === 'string'); + }); + + it('should get metadata for an HMAC key', async function () { + delay(this, accessId); + const hmacKey = storage.hmacKey(accessId, {projectId: HMAC_PROJECT}); + const [metadata] = await hmacKey.getMetadata(); + assert.strictEqual(metadata.accessId, accessId); + }); + + it('should show up from getHmacKeys() without serviceAccountEmail param', async () => { + const [hmacKeys] = await storage.getHmacKeys({projectId: HMAC_PROJECT}); + assert(hmacKeys.length > 0); + assert( + hmacKeys.some(hmacKey => hmacKey.id === accessId), + 'created HMAC key not found from getHmacKeys result' + ); + }); + + it('should make the key INACTIVE', async () => { + const hmacKey = storage.hmacKey(accessId, {projectId: HMAC_PROJECT}); + let [metadata] = await hmacKey.setMetadata({state: 'INACTIVE'}); + assert.strictEqual(metadata.state, 'INACTIVE'); + + [metadata] = await hmacKey.getMetadata(); + assert.strictEqual(metadata.state, 'INACTIVE'); + }); + + it('should delete the key', async () => { + const hmacKey = storage.hmacKey(accessId, {projectId: HMAC_PROJECT}); + await hmacKey.delete(); + const [metadata] = await hmacKey.getMetadata(); + assert.strictEqual(metadata.state, 'DELETED'); + assert.strictEqual(hmacKey.metadata!.state, 'DELETED'); + }); + + it('deleted key should not show up from getHmacKeys() by default', async () => { + const [hmacKeys] = await storage.getHmacKeys({ + serviceAccountEmail: SERVICE_ACCOUNT, + projectId: HMAC_PROJECT, + }); + assert(Array.isArray(hmacKeys)); + assert( + !hmacKeys.some(hmacKey => hmacKey.id === accessId), + 'deleted HMAC key is found from getHmacKeys result' + ); + }); + + describe('second service account', () => { + let accessId: string; + + before(function () { + if (!SECOND_SERVICE_ACCOUNT) { + this.skip(); + } + }); + + after(async () => { + const hmacKey = storage.hmacKey(accessId, {projectId: HMAC_PROJECT}); + await hmacKey.setMetadata({state: 'INACTIVE'}); + await hmacKey.delete(); + }); + + it('should create key for a second service account', async () => { + const [hmacKey] = await storage.createHmacKey(SECOND_SERVICE_ACCOUNT!, { + projectId: HMAC_PROJECT, + }); + accessId = hmacKey.id!; + }); + + it('get HMAC keys for both service accounts', async () => { + // Create a key for the first service account + await storage.createHmacKey(SERVICE_ACCOUNT!, { + projectId: HMAC_PROJECT, + }); + + const [hmacKeys] = await storage.getHmacKeys({projectId: HMAC_PROJECT}); + assert( + hmacKeys.some( + hmacKey => hmacKey.metadata!.serviceAccountEmail === SERVICE_ACCOUNT + ), + `Expected at least 1 key for service account: ${SERVICE_ACCOUNT}` + ); + assert( + hmacKeys.some( + hmacKey => + hmacKey.metadata!.serviceAccountEmail === SECOND_SERVICE_ACCOUNT + ), + `Expected at least 1 key for service account: ${SECOND_SERVICE_ACCOUNT}` + ); + }); + + it('filter by service account email', async () => { + const [hmacKeys] = await storage.getHmacKeys({ + serviceAccountEmail: SECOND_SERVICE_ACCOUNT, + projectId: HMAC_PROJECT, + }); + assert( + hmacKeys.every( + hmacKey => + hmacKey.metadata!.serviceAccountEmail === SECOND_SERVICE_ACCOUNT + ), + 'HMAC key belonging to other service accounts unexpected' + ); + }); + }); + }); + + describe('list files', () => { + const DIRECTORY_NAME = 'directory-name'; + + const NEW_FILES = [ + bucket.file('CloudLogo1'), + bucket.file('CloudLogo2'), + bucket.file('CloudLogo3'), + bucket.file(`${DIRECTORY_NAME}/CloudLogo4`), + bucket.file(`${DIRECTORY_NAME}/CloudLogo5`), + bucket.file(`${DIRECTORY_NAME}/inner/CloudLogo6`), + ]; + + before(async () => { + await bucket.deleteFiles(); + const originalFile = NEW_FILES[0]; + const cloneFiles = NEW_FILES.slice(1); + await bucket.upload(FILES.logo.path, { + destination: originalFile, + }); + await Promise.all(cloneFiles.map(f => originalFile.copy(f))); + }); + + after(async () => { + await Promise.all(NEW_FILES.map(file => deleteFileAsync(file))); + }); + + it('should get files', async () => { + const [files] = await bucket.getFiles(); + assert.strictEqual(files!.length, NEW_FILES.length); + }); + + it('returns file name only when fields is set as name', async () => { + const expected = [ + {name: 'CloudLogo1'}, + {name: 'CloudLogo2'}, + {name: 'CloudLogo3'}, + {name: `${DIRECTORY_NAME}/CloudLogo4`}, + {name: `${DIRECTORY_NAME}/CloudLogo5`}, + {name: `${DIRECTORY_NAME}/inner/CloudLogo6`}, + ]; + const [files] = await bucket.getFiles({fields: 'items(name)'}); + + assert.deepStrictEqual(files, expected); + }); + + it('returns folders as prefixes when includeFoldersAsPrefixes is set', async () => { + const expected = [`${DIRECTORY_NAME}/`]; + const [, , result] = await bucket.getFiles({ + delimiter: '/', + includeFoldersAsPrefixes: true, + autoPaginate: false, + }); + + assert.deepStrictEqual( + (result as {prefixes: string[]}).prefixes, + expected + ); + }); + + it('should get files as a stream', done => { + let numFilesEmitted = 0; + + bucket + .getFilesStream() + .on('error', done) + .on('data', () => { + numFilesEmitted++; + }) + .on('end', () => { + assert.strictEqual(numFilesEmitted, NEW_FILES.length); + done(); + }); + }); + + it('should only get files matching the supplied matchGlob argument', async () => { + let expectedFileNames = ['CloudLogo1', 'CloudLogo2', 'CloudLogo3']; + let [files] = await bucket.getFiles({matchGlob: 'CloudLogo*'}); + assert.strictEqual(files.length, expectedFileNames.length); + for (const curFile of files) { + assert.strictEqual(expectedFileNames.includes(curFile.name), true); + } + + expectedFileNames = [ + `${DIRECTORY_NAME}/CloudLogo4`, + `${DIRECTORY_NAME}/CloudLogo5`, + `${DIRECTORY_NAME}/inner/CloudLogo6`, + ]; + [files] = await bucket.getFiles({ + matchGlob: `${DIRECTORY_NAME}/**/CloudLogo*`, + }); + assert.strictEqual(files.length, expectedFileNames.length); + for (const curFile of files) { + assert.strictEqual(expectedFileNames.includes(curFile.name), true); + } + }); + + it('should paginate the list', async () => { + const query = { + maxResults: NEW_FILES.length - 1, + }; + + const [files, nextQuery] = await bucket.getFiles(query); + + assert.strictEqual(files!.length, NEW_FILES.length - 1); + assert(nextQuery); + const [nextFiles] = await bucket.getFiles(nextQuery); + assert.strictEqual(nextFiles!.length, 1); + }); + }); + + describe('offset', () => { + const NEW_FILES = [ + bucket.file('startOffset_file1'), + bucket.file('startOffset_file2'), + bucket.file('file3_endOffset'), + ]; + + before(async () => { + await bucket.deleteFiles(); + const originalFile = NEW_FILES[0]; + const cloneFiles = NEW_FILES.slice(1); + await bucket.upload(FILES.logo.path, { + destination: originalFile, + }); + await Promise.all(cloneFiles.map(f => originalFile.copy(f))); + }); + + after(async () => { + await Promise.all(NEW_FILES.map(file => deleteFileAsync(file))); + }); + + it('should get files with offset', async () => { + // Listing files with startOffset. + const [filesWithStartOffset] = await bucket.getFiles({ + startOffset: 'startOffset', + }); + assert.strictEqual(filesWithStartOffset!.length, 2); + + // Listing files with endOffset. + const [filesWithEndOffset] = await bucket.getFiles({ + endOffset: 'set', + }); + assert.strictEqual(filesWithEndOffset!.length, 1); + + // Listing files with startOffset and endOffset. + const [filesWithStartAndEndOffset] = await bucket.getFiles({ + startOffset: 'startOffset', + endOffset: 'endOffset', + }); + assert.strictEqual(filesWithStartAndEndOffset!.length, 0); + }); + }); + + describe('file generations', () => { + const bucketWithVersioning = storage.bucket(generateName()); + + before(async () => { + await bucketWithVersioning.create({ + versioning: { + enabled: true, + }, + }); + }); + + after(async () => { + await bucketWithVersioning.deleteFiles({ + versions: true, + }); + await bucketWithVersioning.delete(); + }); + + it('should overwrite file, then get older version', async () => { + const versionedFile = bucketWithVersioning.file(generateName()); + await versionedFile.save('a'); + const [metadata] = await versionedFile.getMetadata(); + const initialGeneration = metadata.generation; + await versionedFile.save('b'); + const firstGenFile = bucketWithVersioning.file(versionedFile.name, { + generation: initialGeneration, + }); + const [contents] = await firstGenFile.download(); + assert.strictEqual(contents.toString(), 'a'); + }); + + it('should get all files scoped to their version', async () => { + const filesToCreate = [ + {file: bucketWithVersioning.file('file-one.txt'), contents: '123'}, + {file: bucketWithVersioning.file('file-one.txt'), contents: '456'}, + ]; + + await Promise.all(filesToCreate.map(file => createFileAsync(file))); + + const [files] = await bucketWithVersioning.getFiles({versions: true}); + assert.strictEqual(files![0].name, files![1].name); + assert.notStrictEqual( + files![0].metadata.generation, + files![1].metadata.generation + ); + }); + + it('should throw an error Precondition Failed on overwrite with version 0, then save file with and without resumable', async () => { + const fileName = `test-${Date.now()}.txt`; + + await bucketWithVersioning + .file(fileName) + .save('hello1', {resumable: false}); + await assert.rejects( + bucketWithVersioning.file(fileName, {generation: 0}).save('hello2'), + (err: ApiError) => { + assert.strictEqual(err.code, 412); + assert.strictEqual(err.errors![0].reason, 'conditionNotMet'); + return true; + } + ); + await bucketWithVersioning + .file(fileName) + .save('hello3', {resumable: false}); + await bucketWithVersioning.file(fileName).save('hello4'); + }); + }); + + describe('v2 signed urls', () => { + const localFile = fs.readFileSync(FILES.logo.path); + let file: File; + + before(done => { + file = bucket.file('LogoToSign.jpg'); + fs.createReadStream(FILES.logo.path) + .pipe(file.createWriteStream()) + .on('error', done) + .on('finish', done.bind(null, null)); + }); + + it('should create a signed read url', async () => { + const [signedReadUrl] = await file.getSignedUrl({ + version: 'v2', + action: 'read', + expires: Date.now() + 5000, + }); + + const res = await fetch(signedReadUrl); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + + it('should work with multi-valued extension headers', async () => { + const HEADERS = { + 'x-goog-custom-header': ['value1', 'value2'], + }; + const [signedReadUrl] = await file.getSignedUrl({ + action: 'read', + expires: Date.now() + 5000, + extensionHeaders: HEADERS, + }); + + const res = await fetch(signedReadUrl, { + headers: {'x-goog-custom-header': 'value1,value2'}, + }); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + + it('should create a signed delete url', async () => { + await file.delete(); + const [signedDeleteUrl] = await file.getSignedUrl({ + version: 'v2', + action: 'delete', + expires: Date.now() + 5000, + }); + + await fetch(signedDeleteUrl, {method: 'DELETE'}); + assert.rejects( + () => file.getMetadata(), + (err: ApiError) => err.code === 404 + ); + }); + }); + + describe('v2 signed url with special characters in file name', () => { + const localFile = fs.readFileSync(FILES.logo.path); + let file: File; + + before(done => { + file = bucket.file("special/azAZ!*'()*%/file.jpg"); + fs.createReadStream(FILES.logo.path) + .pipe(file.createWriteStream()) + .on('error', done) + .on('finish', done.bind(null, null)); + }); + + after(() => file.delete()); + + it('should create a signed read url and fetch a file', async () => { + const [signedUrl] = await file.getSignedUrl({ + version: 'v2', + action: 'read', + expires: Date.now() + 5000, + }); + + const res = await fetch(signedUrl); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + }); + + describe('v4 signed urls', () => { + const localFile = fs.readFileSync(FILES.logo.path); + let file: File; + + before(done => { + file = bucket.file('LogoToSign.jpg'); + fs.createReadStream(FILES.logo.path) + .pipe(file.createWriteStream()) + .on('error', done) + .on('finish', done.bind(null, null)); + }); + + it('should create a signed read url', async () => { + const [signedReadUrl] = await file.getSignedUrl({ + version: 'v4', + action: 'read', + expires: Date.now() + 5000, + }); + + const res = await fetch(signedReadUrl); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + + it('should not throw with expiration of exactly 7 days', async () => { + const ACCESSIBLE_AT = new Date().setMilliseconds(999).valueOf(); + const SEVEN_DAYS_IN_SECONDS = 7 * 24 * 60 * 60; + const SEVEN_DAYS_IN_MS = SEVEN_DAYS_IN_SECONDS * 1000; + await assert.doesNotReject( + async () => { + await file.getSignedUrl({ + version: 'v4', + action: 'read', + accessibleAt: ACCESSIBLE_AT, + expires: ACCESSIBLE_AT + SEVEN_DAYS_IN_MS, + virtualHostedStyle: true, + }); + }, + err => { + assert(err instanceof Error); + assert.strictEqual( + err.message, + `Max allowed expiration is seven days (${SEVEN_DAYS_IN_SECONDS.toString()} seconds).` + ); + return true; + } + ); + }); + + it('should create a signed read url with accessibleAt in the past', async () => { + const [signedReadUrl] = await file.getSignedUrl({ + version: 'v4', + action: 'read', + accessibleAt: Date.now() - 5000, + expires: Date.now() + 5000, + }); + + const res = await fetch(signedReadUrl); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + + it('should create a signed read url with accessibleAt in the future', async () => { + const accessibleAtDate = new Date(); + const accessibleAtMinutes = accessibleAtDate.getMinutes(); + const expiresDate = new Date(); + const expiresMinutes = expiresDate.getMinutes(); + const [signedReadUrl] = await file.getSignedUrl({ + version: 'v4', + action: 'read', + accessibleAt: accessibleAtDate.setMinutes(accessibleAtMinutes + 60), + expires: expiresDate.setMinutes(expiresMinutes + 90), + }); + const res = await fetch(signedReadUrl); + assert.strictEqual(res.status, 403); + }); + + it('should work with special characters in extension headers', async () => { + const HEADERS = { + 'x-goog-custom-header': ['value1', "azAZ!*'()*%"], + }; + const [signedReadUrl] = await file.getSignedUrl({ + version: 'v4', + action: 'read', + expires: Date.now() + 5000, + extensionHeaders: HEADERS, + }); + + const res = await fetch(signedReadUrl, { + headers: {'x-goog-custom-header': "value1,azAZ!*'()*%"}, + }); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + + it('should create a virtual-hosted style URL', async () => { + const [signedUrl] = await file.getSignedUrl({ + virtualHostedStyle: true, + version: 'v4', + action: 'read', + expires: Date.now() + 5000, + }); + + const res = await fetch(signedUrl); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + + it('should create a signed delete url', async () => { + const [signedDeleteUrl] = await file.getSignedUrl({ + version: 'v4', + action: 'delete', + expires: Date.now() + 5000, + }); + await fetch(signedDeleteUrl!, {method: 'DELETE'}); + const [exists] = await file.exists(); + assert.strictEqual(exists, false); + }); + + it('should create a signed list bucket url', async () => { + const [signedUrl] = await bucket.getSignedUrl({ + version: 'v4', + action: 'list', + expires: Date.now() + 5000, + }); + const res = await fetch(signedUrl!, {method: 'GET'}); + const body = await res.text(); + assert.strictEqual(res.status, 200); + assert(body.includes('ListBucketResult')); + }); + }); + + describe('v4 signed url with special characters in file name', () => { + const localFile = fs.readFileSync(FILES.logo.path); + let file: File; + + before(done => { + file = bucket.file("special/azAZ!*'()*%/file.jpg"); + fs.createReadStream(FILES.logo.path) + .pipe(file.createWriteStream()) + .on('error', done) + .on('finish', done.bind(null, null)); + }); + + after(async () => file.delete()); + + it('should create a signed read url and fetch a file', async () => { + const [signedUrl] = await file.getSignedUrl({ + version: 'v4', + action: 'read', + expires: Date.now() + 5000, + }); + + const res = await fetch(signedUrl); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + }); + + describe('sign policy', () => { + let file: File; + + before(() => { + file = bucket.file('LogoToSign.jpg'); + }); + + beforeEach(function () { + if (!storage.projectId) { + this.skip(); + } + }); + + it('should create a V2 policy', async () => { + const expires = Date.now() + 60 * 1000; // one minute + const expectedExpiration = new Date(expires).toISOString(); + + const options = { + equals: ['$Content-Type', 'image/jpeg'], + expires, + contentLengthRange: { + min: 0, + max: 1024, + }, + }; + + const [policy] = await file.generateSignedPostPolicyV2(options); + + const policyJson = JSON.parse(policy!.string); + assert.strictEqual(policyJson.expiration, expectedExpiration); + }); + + it('should create a V4 policy', async () => { + const expires = Date.now() + 60 * 1000; // one minute + const options = { + expires, + contentLengthRange: { + min: 0, + max: 50000, + }, + fields: {'x-goog-meta-test': 'data'}, + }; + + const [policy] = await file.generateSignedPostPolicyV4(options); + const form = new FormData(); + for (const [key, value] of Object.entries(policy.fields)) { + form.append(key, value); + } + + const CONTENT = 'my-content'; + + form.append('file', CONTENT); + const res = await fetch(policy.url, {method: 'POST', body: form}); + assert.strictEqual(res.status, 204); + + const [buf] = await file.download(); + assert.strictEqual(buf.toString(), CONTENT); + }); + }); + + describe('notifications', () => { + let notification: Notification; + let subscription: Subscription; + + before(async () => { + const createNotificationData = await bucket.createNotification( + topic.name, + { + eventTypes: ['OBJECT_FINALIZE'], + } + ); + notification = createNotificationData[0]; + subscription = topic.subscription(generateName()); + await subscription.create(); + }); + + after(async () => { + await subscription.delete(); + const notifications = await bucket.getNotifications(); + const notificationsToDelete = notifications[0].map(notification => { + return notification.delete(); + }); + await Promise.all(notificationsToDelete); + }); + + it('should get an existing notification', async () => { + await notification.get(); + assert(Object.keys(notification.metadata).length > 0); + }); + + it('should get a notifications metadata', async () => { + const [metadata] = await notification.getMetadata(); + assert(metadata !== null && typeof metadata === 'object'); + }); + + it('should tell us if a notification exists', async () => { + const [exists] = await notification.exists(); + assert(exists); + }); + + it('should tell us if a notification does not exist', async () => { + const notification = bucket.notification('123'); + const [exists] = await notification.exists(); + assert.strictEqual(exists, false); + }); + + it('should get a list of notifications', async () => { + const [notifications] = await bucket.getNotifications(); + assert.strictEqual(notifications!.length, 1); + }); + + it('should emit events to a subscription', done => { + subscription + .on('error', err => { + done(err); + }) + .on('message', message => { + const attrs = message.attributes; + assert.strictEqual(attrs.eventType, 'OBJECT_FINALIZE'); + done(); + }); + + bucket.upload(FILES.logo.path, (err: Error | null) => { + if (err) { + done(err); + } + }); + }); + + it('should delete a notification', () => { + let notificationCount = 0; + let notification: Notification; + + return bucket + .createNotification(topic.name, { + eventTypes: ['OBJECT_DELETE'], + }) + .then(data => { + notification = data[0]; + return bucket.getNotifications(); + }) + .then(data => { + notificationCount = data[0].length; + return notification.delete(); + }) + .then(() => { + return bucket.getNotifications(); + }) + .then(data => { + assert.strictEqual(data[0].length, notificationCount - 1); + }); + }); + }); + + describe('CRC32C', () => { + const KNOWN_INPUT_TO_CRC32C = { + /** empty string (i.e. nothing to 'update') */ + '': 'AAAAAA==', + /** known case #1 - validated from actual GCS object upload + metadata retrieval */ + data: 'rth90Q==', + /** known case #2 - validated from actual GCS object upload + metadata retrieval */ + 'some text\n': 'DkjKuA==', + /** arbitrary large string */ + ['a'.repeat(2 ** 16)]: 'TpXtPw==', + } as const; + + it('should generate the appropriate hashes', async () => { + const file = bucket.file('crc32c-test-file'); + + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const buffer = Buffer.from(input); + const crc32c = new CRC32C(); + + await file.save(buffer); + crc32c.update(buffer); + + const [metadata] = await file.getMetadata(); + + assert.equal(metadata.crc32c, expected); + assert(crc32c.validate(metadata.crc32c!)); + } + }); + }); + + describe('universeDomainTests', () => { + let universeDomainStorage: Storage; + const bucketName = generateName(); + const localFile = fs.readFileSync(FILES.logo.path); + let file: File; + + before(async () => { + const TEST_UNIVERSE_DOMAIN = isNullOrUndefined('TEST_UNIVERSE_DOMAIN'); + const TEST_PROJECT_ID = isNullOrUndefined('TEST_UNIVERSE_PROJECT_ID'); + const TEST_UNIVERSE_LOCATION = isNullOrUndefined( + 'TEST_UNIVERSE_LOCATION' + ); + const CREDENTIAL_PATH = isNullOrUndefined( + 'TEST_UNIVERSE_DOMAIN_CREDENTIAL' + ); + // Create a client with universe domain credentials + universeDomainStorage = new Storage({ + projectId: TEST_PROJECT_ID, + keyFilename: CREDENTIAL_PATH, + universeDomain: TEST_UNIVERSE_DOMAIN, + }); + + const [bucket] = await universeDomainStorage.createBucket(bucketName, { + location: TEST_UNIVERSE_LOCATION, + }); + + file = bucket.file('LogoToSign.jpg'); + fs.createReadStream(FILES.logo.path).pipe(file.createWriteStream()); + }); + + after(async () => { + await deleteFileAsync(file); + await deleteBucketAsync(bucket); + }); + + it('should get bucket', async () => { + const [buckets] = await universeDomainStorage.getBuckets(); + const getBucket = buckets.filter(item => item.name === bucketName); + assert.strictEqual(getBucket[0].name, bucketName); + }); + + it('should get files', async () => { + const fileName = await universeDomainStorage + .bucket(bucketName) + .file(file.name).name; + assert.strictEqual(fileName, file.name); + }); + + it('should create a signed read url', async () => { + const [signedReadUrl] = await file.getSignedUrl({ + version: 'v2', + action: 'read', + expires: Date.now() + 5000, + virtualHostedStyle: true, + }); + + const res = await fetch(signedReadUrl); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + }); + + async function deleteBucketAsync(bucket: Bucket, options?: {}) { + // After files are deleted, eventual consistency may require a bit of a + // delay to ensure that the bucket recognizes that the files don't exist + // anymore. + const CONSISTENCY_DELAY_MS = 250; + + options = Object.assign({}, options, { + versions: true, + }); + + await bucket.deleteFiles(options); + await new Promise(resolve => setTimeout(resolve, CONSISTENCY_DELAY_MS)); + await bucket.delete(); + } + + function deleteBucket( + bucket: Bucket, + options: {}, + callback: DeleteBucketCallback + ): void; + function deleteBucket(bucket: Bucket, callback: DeleteBucketCallback): void; + function deleteBucket( + bucket: Bucket, + optsOrCb: {} | DeleteBucketCallback, + callback?: DeleteBucketCallback + ) { + let options = typeof optsOrCb === 'object' ? optsOrCb : {}; + callback = + typeof optsOrCb === 'function' + ? (optsOrCb as DeleteBucketCallback) + : callback; + + // After files are deleted, eventual consistency may require a bit of a + // delay to ensure that the bucket recognizes that the files don't exist + // anymore. + const CONSISTENCY_DELAY_MS = 250; + + options = Object.assign({}, options, { + versions: true, + }); + + bucket.deleteFiles(options, err => { + if (err) { + callback!(err as Error); + return; + } + + setTimeout(() => { + bucket.delete(options, callback!); + }, CONSISTENCY_DELAY_MS); + }); + } + + function deleteFileAsync(file: File) { + return file.delete(); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function deleteTopicAsync(topic: any) { + return topic.delete(); + } + + function shortUUID() { + return uuid.v1().split('-').shift(); + } + + function generateName() { + return TESTS_PREFIX + shortUUID(); + } + + async function deleteAllBucketsAsync() { + const [buckets] = await storage.getBuckets({prefix: TESTS_PREFIX}); + const limit = pLimit(10); + await new Promise(resolve => + setTimeout(resolve, RETENTION_DURATION_SECONDS * 1000) + ); + return Promise.all( + buckets.map(bucket => limit(() => deleteBucketAsync(bucket))) + ); + } + + async function deleteAllTopicsAsync() { + const [topics] = await pubsub.getTopics(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const filteredTopics = (topics as any[]).filter(topic => { + return topic.name.indexOf(TESTS_PREFIX) > -1; + }); + const limit = pLimit(10); + return Promise.all( + filteredTopics.map(topic => limit(() => deleteTopicAsync(topic))) + ); + } + + async function isFilePublicAsync(file: File) { + try { + const [aclObject] = await file.acl.get({entity: 'allUsers'}); + if ( + (aclObject as AccessControlObject).entity === 'allUsers' && + (aclObject as AccessControlObject).role === 'READER' + ) { + return true; + } else { + return false; + } + } catch (error) { + const err = error as HTTPError; + if (err.code === 404) { + return false; + } else { + throw error; + } + } + } + + async function deleteStaleHmacKeys( + serviceAccountEmail: string, + projectId: string + ) { + const old = new Date(); + old.setHours(old.getHours() - 1); + const [hmacKeys] = await storage.getHmacKeys({ + serviceAccountEmail, + projectId, + }); + + const limit = pLimit(10); + await Promise.all( + hmacKeys + .filter(hmacKey => { + const hmacKeyCreated = new Date(hmacKey.metadata!.timeCreated!); + return hmacKey.metadata!.state !== 'DELETED' && hmacKeyCreated < old; + }) + .map(hmacKey => + limit(async () => { + await hmacKey.setMetadata({state: 'INACTIVE'}); + await hmacKey.delete(); + }) + ) + ); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function createFileAsync(fileObject: any) { + return fileObject.file.save(fileObject.contents); + } + + function createFileWithContentPromise(content: string) { + return bucket.file(`${generateName()}.txt`).save(content); + } + + function isNullOrUndefined(envVarName: string) { + const value = process.env[envVarName]; + if (value === undefined || value === null) { + throw new Error(`Please set the ${envVarName} environment variable.`); + } + return value; + } +}); diff --git a/handwritten/storage/test/acl.ts b/handwritten/storage/test/acl.ts new file mode 100644 index 00000000000..5c1d73e25ae --- /dev/null +++ b/handwritten/storage/test/acl.ts @@ -0,0 +1,609 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {DecorateRequestOptions, util} from '../src/nodejs-common/index.js'; +import assert from 'assert'; +import {describe, it, before, beforeEach} from 'mocha'; +import proxyquire from 'proxyquire'; +import {Storage} from '../src/storage.js'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +let Acl: any; +let AclRoleAccessorMethods: Function; +describe('storage/acl', () => { + let promisified = false; + const fakePromisify = { + // tslint:disable-next-line:variable-name + promisifyAll(Class: Function) { + if (Class.name === 'Acl') { + promisified = true; + } + }, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let acl: any; + + const ERROR = new Error('Error.'); + const MAKE_REQ = util.noop; + const PATH_PREFIX = '/acl'; + const ROLE = Storage.acl.OWNER_ROLE; + const ENTITY = 'user-user@example.com'; + + before(() => { + const aclModule = proxyquire('../src/acl.js', { + '@google-cloud/promisify': fakePromisify, + }); + Acl = aclModule.Acl; + AclRoleAccessorMethods = aclModule.AclRoleAccessorMethods; + }); + + beforeEach(() => { + acl = new Acl({request: MAKE_REQ, pathPrefix: PATH_PREFIX}); + }); + + describe('initialization', () => { + it('should promisify all the things', () => { + assert(promisified); + }); + + it('should assign makeReq and pathPrefix', () => { + assert.strictEqual(acl.pathPrefix, PATH_PREFIX); + assert.strictEqual(acl.request_, MAKE_REQ); + }); + }); + + describe('add', () => { + it('should make the correct api request', done => { + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.method, 'POST'); + assert.strictEqual(reqOpts.uri, ''); + assert.deepStrictEqual(reqOpts.json, {entity: ENTITY, role: ROLE}); + done(); + }; + + acl.add({entity: ENTITY, role: ROLE}, assert.ifError); + }); + + it('should set the generation', done => { + const options = { + entity: ENTITY, + role: ROLE, + generation: 8, + }; + + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.generation, options.generation); + done(); + }; + + acl.add(options, assert.ifError); + }); + + it('should set the userProject', done => { + const options = { + entity: ENTITY, + role: ROLE, + userProject: 'grape-spaceship-123', + }; + + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.userProject, options.userProject); + done(); + }; + + acl.add(options, assert.ifError); + }); + + it('should execute the callback with an ACL object', done => { + const apiResponse = {entity: ENTITY, role: ROLE}; + const expectedAclObject = {entity: ENTITY, role: ROLE}; + + acl.makeAclObject_ = (obj: {}) => { + assert.deepStrictEqual(obj, apiResponse); + return expectedAclObject; + }; + + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, apiResponse); + }; + + acl.add({entity: ENTITY, role: ROLE}, (err: Error, aclObject: {}) => { + assert.ifError(err); + assert.deepStrictEqual(aclObject, expectedAclObject); + done(); + }); + }); + + it('should execute the callback with an error', done => { + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(ERROR); + }; + + acl.add({entity: ENTITY, role: ROLE}, (err: Error) => { + assert.deepStrictEqual(err, ERROR); + done(); + }); + }); + + it('should execute the callback with apiResponse', done => { + const resp = {success: true}; + + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, resp); + }; + + acl.add( + {entity: ENTITY, role: ROLE}, + (err: Error, acls: {}, apiResponse: unknown) => { + assert.deepStrictEqual(resp, apiResponse); + done(); + } + ); + }); + }); + + describe('delete', () => { + it('should make the correct api request', done => { + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.method, 'DELETE'); + assert.strictEqual(reqOpts.uri, '/' + encodeURIComponent(ENTITY)); + + done(); + }; + + acl.delete({entity: ENTITY}, assert.ifError); + }); + + it('should set the generation', done => { + const options = { + entity: ENTITY, + generation: 8, + }; + + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.generation, options.generation); + done(); + }; + + acl.delete(options, assert.ifError); + }); + + it('should set the userProject', done => { + const options = { + entity: ENTITY, + role: ROLE, + userProject: 'grape-spaceship-123', + }; + + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.userProject, options.userProject); + done(); + }; + + acl.delete(options, assert.ifError); + }); + + it('should execute the callback with an error', done => { + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(ERROR); + }; + + acl.delete({entity: ENTITY}, (err: Error) => { + assert.deepStrictEqual(err, ERROR); + done(); + }); + }); + + it('should execute the callback with apiResponse', done => { + const resp = {success: true}; + + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, resp); + }; + + acl.delete({entity: ENTITY}, (err: Error, apiResponse: unknown) => { + assert.deepStrictEqual(resp, apiResponse); + done(); + }); + }); + }); + + describe('get', () => { + describe('all ACL objects', () => { + it('should make the correct API request', done => { + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, ''); + + done(); + }; + + acl.get(assert.ifError); + }); + + it('should accept a configuration object', done => { + const generation = 1; + + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.generation, generation); + + done(); + }; + + acl.get({generation}, assert.ifError); + }); + + it('should pass an array of acl objects to the callback', done => { + const apiResponse = { + items: [ + {entity: ENTITY, role: ROLE}, + {entity: ENTITY, role: ROLE}, + {entity: ENTITY, role: ROLE}, + ], + }; + + const expectedAclObjects = [ + {entity: ENTITY, role: ROLE}, + {entity: ENTITY, role: ROLE}, + {entity: ENTITY, role: ROLE}, + ]; + + acl.makeAclObject_ = (obj: {}, index: number) => { + return expectedAclObjects[index]; + }; + + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, apiResponse); + }; + + acl.get((err: Error, aclObjects: Array<{}>) => { + assert.ifError(err); + assert.deepStrictEqual(aclObjects, expectedAclObjects); + done(); + }); + }); + }); + + describe('ACL object for an entity', () => { + it('should get a specific ACL object', done => { + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, '/' + encodeURIComponent(ENTITY)); + + done(); + }; + + acl.get({entity: ENTITY}, assert.ifError); + }); + + it('should accept a configuration object', done => { + const generation = 1; + + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.generation, generation); + + done(); + }; + + acl.get({entity: ENTITY, generation}, assert.ifError); + }); + + it('should set the userProject', done => { + const options = { + entity: ENTITY, + userProject: 'grape-spaceship-123', + }; + + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.userProject, options.userProject); + done(); + }; + + acl.get(options, assert.ifError); + }); + + it('should pass an acl object to the callback', done => { + const apiResponse = {entity: ENTITY, role: ROLE}; + const expectedAclObject = {entity: ENTITY, role: ROLE}; + + acl.makeAclObject_ = () => { + return expectedAclObject; + }; + + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, apiResponse); + }; + + acl.get({entity: ENTITY}, (err: Error, aclObject: {}) => { + assert.ifError(err); + assert.deepStrictEqual(aclObject, expectedAclObject); + done(); + }); + }); + }); + + it('should execute the callback with an error', done => { + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(ERROR); + }; + + acl.get((err: Error) => { + assert.deepStrictEqual(err, ERROR); + done(); + }); + }); + + it('should execute the callback with apiResponse', done => { + const resp = {success: true}; + + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, resp); + }; + + acl.get((err: Error, acls: Array<{}>, apiResponse: unknown) => { + assert.deepStrictEqual(resp, apiResponse); + done(); + }); + }); + }); + + describe('update', () => { + it('should make the correct API request', done => { + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.method, 'PUT'); + assert.strictEqual(reqOpts.uri, '/' + encodeURIComponent(ENTITY)); + assert.deepStrictEqual(reqOpts.json, {role: ROLE}); + + done(); + }; + + acl.update({entity: ENTITY, role: ROLE}, assert.ifError); + }); + + it('should set the generation', done => { + const options = { + entity: ENTITY, + role: ROLE, + generation: 8, + }; + + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.generation, options.generation); + done(); + }; + + acl.update(options, assert.ifError); + }); + + it('should set the userProject', done => { + const options = { + entity: ENTITY, + role: ROLE, + userProject: 'grape-spaceship-123', + }; + + acl.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.userProject, options.userProject); + done(); + }; + + acl.update(options, assert.ifError); + }); + + it('should pass an acl object to the callback', done => { + const apiResponse = {entity: ENTITY, role: ROLE}; + const expectedAclObject = {entity: ENTITY, role: ROLE}; + + acl.makeAclObject_ = () => { + return expectedAclObject; + }; + + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, apiResponse); + }; + + acl.update({entity: ENTITY, role: ROLE}, (err: Error, aclObject: {}) => { + assert.ifError(err); + assert.deepStrictEqual(aclObject, expectedAclObject); + done(); + }); + }); + + it('should execute the callback with an error', done => { + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(ERROR); + }; + + acl.update({entity: ENTITY, role: ROLE}, (err: Error) => { + assert.deepStrictEqual(err, ERROR); + done(); + }); + }); + + it('should execute the callback with apiResponse', done => { + const resp = {success: true}; + + acl.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, resp); + }; + + const config = {entity: ENTITY, role: ROLE}; + acl.update( + config, + (err: Error, acls: Array<{}>, apiResponse: unknown) => { + assert.deepStrictEqual(resp, apiResponse); + done(); + } + ); + }); + }); + + describe('makeAclObject_', () => { + it('should return an ACL object from an API response', () => { + const projectTeam = { + projectNumber: '283748374', + team: 'awesome', + }; + + const apiResponse = { + entity: ENTITY, + role: ROLE, + projectTeam, + extra: 'ignored', + things: true, + }; + + assert.deepStrictEqual(acl.makeAclObject_(apiResponse), { + entity: ENTITY, + role: ROLE, + projectTeam, + }); + }); + }); + + describe('request', () => { + it('should make the correct request', done => { + const uri = '/uri'; + + const reqOpts = { + uri, + }; + + acl.request_ = (reqOpts_: DecorateRequestOptions, callback: Function) => { + assert.strictEqual(reqOpts_, reqOpts); + assert.strictEqual(reqOpts_.uri, PATH_PREFIX + uri); + callback(); // done() + }; + + acl.request(reqOpts, done); + }); + }); +}); + +describe('storage/AclRoleAccessorMethods', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let aclEntity: any; + + beforeEach(() => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + aclEntity = new (AclRoleAccessorMethods as any)(); + }); + + describe('initialization', () => { + it('should assign access methods for every role object', () => { + const expectedApi = [ + 'addAllAuthenticatedUsers', + 'deleteAllAuthenticatedUsers', + + 'addAllUsers', + 'deleteAllUsers', + + 'addDomain', + 'deleteDomain', + + 'addGroup', + 'deleteGroup', + + 'addProject', + 'deleteProject', + + 'addUser', + 'deleteUser', + ]; + + const actualOwnersApi = Object.keys(aclEntity.owners); + assert.deepStrictEqual(actualOwnersApi, expectedApi); + + const actualReadersApi = Object.keys(aclEntity.readers); + assert.deepStrictEqual(actualReadersApi, expectedApi); + + const actualWritersApi = Object.keys(aclEntity.writers); + assert.deepStrictEqual(actualWritersApi, expectedApi); + }); + }); + + describe('_assignAccessMethods', () => { + it('should call parent method', async () => { + const userName = 'email@example.com'; + const role = 'fakerole'; + + aclEntity.add = async (options: {}) => { + assert.deepStrictEqual(options, { + entity: 'user-' + userName, + role, + }); + }; + + aclEntity.delete = async (options: {}) => { + assert.deepStrictEqual(options, { + entity: 'allUsers', + role, + }); + }; + + aclEntity._assignAccessMethods(role); + + await Promise.all([ + aclEntity.fakeroles.addUser(userName), + aclEntity.fakeroles.deleteAllUsers(), + ]); + }); + + it('should return the parent methods return value', () => { + const fakeReturn = {}; + + aclEntity.add = () => { + return fakeReturn; + }; + + aclEntity._assignAccessMethods('fakerole'); + + const value = aclEntity.fakeroles.addUser('email@example.com'); + assert.strictEqual(value, fakeReturn); + }); + + it('should not pass in the callback if undefined', done => { + aclEntity.add = (...args: Array<{}>) => { + assert.strictEqual(args.length, 1); + done(); + }; + + aclEntity._assignAccessMethods('fakerole'); + aclEntity.fakeroles.addUser('email@example.com', undefined); + }); + + it('should optionally accept options', done => { + const fakeRole = 'fakerole'; + const fakeUser = 'email@example.com'; + const fakeOptions = { + userProject: 'grape-spaceship-123', + }; + + const expectedOptions = Object.assign( + { + entity: 'user-' + fakeUser, + role: fakeRole, + }, + fakeOptions + ); + + aclEntity.add = (options: {}) => { + assert.deepStrictEqual(options, expectedOptions); + done(); + }; + + aclEntity._assignAccessMethods(fakeRole); + aclEntity.fakeroles.addUser(fakeUser, fakeOptions, assert.ifError); + }); + }); +}); diff --git a/handwritten/storage/test/bucket.ts b/handwritten/storage/test/bucket.ts new file mode 100644 index 00000000000..5b49fa518d8 --- /dev/null +++ b/handwritten/storage/test/bucket.ts @@ -0,0 +1,3303 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + BaseMetadata, + DecorateRequestOptions, + ServiceObject, + ServiceObjectConfig, + util, +} from '../src/nodejs-common/index.js'; +import assert from 'assert'; +import * as fs from 'fs'; +import {describe, it, before, beforeEach, after, afterEach} from 'mocha'; +import mime from 'mime'; +import pLimit from 'p-limit'; +import * as path from 'path'; +import proxyquire from 'proxyquire'; + +import * as stream from 'stream'; +import {Bucket, Channel, Notification, CRC32C} from '../src/index.js'; +import { + CreateWriteStreamOptions, + File, + SetFileMetadataOptions, + FileOptions, + FileMetadata, +} from '../src/file.js'; +import {PromisifyAllOptions} from '@google-cloud/promisify'; +import { + GetBucketMetadataCallback, + GetFilesOptions, + MakeAllFilesPublicPrivateOptions, + SetBucketMetadataResponse, + GetBucketSignedUrlConfig, + AvailableServiceObjectMethods, + BucketExceptionMessages, + BucketMetadata, + LifecycleRule, +} from '../src/bucket.js'; +import {AddAclOptions} from '../src/acl.js'; +import {Policy} from '../src/iam.js'; +import sinon from 'sinon'; +import {Transform} from 'stream'; +import {IdempotencyStrategy} from '../src/storage.js'; +import {convertObjKeysToSnakeCase, getDirName} from '../src/util.js'; +import {DEFAULT_UNIVERSE} from 'google-auth-library'; + +class FakeFile { + calledWith_: IArguments; + bucket: Bucket; + name: string; + options: FileOptions; + metadata: FileMetadata; + createWriteStream: Function; + delete: Function; + isSameFile = () => false; + constructor(bucket: Bucket, name: string, options?: FileOptions) { + // eslint-disable-next-line prefer-rest-params + this.calledWith_ = arguments; + this.bucket = bucket; + this.name = name; + this.options = options || {}; + this.metadata = {}; + + this.createWriteStream = (options: CreateWriteStreamOptions) => { + this.metadata = options.metadata!; + const ws = new stream.Writable(); + ws.write = () => { + ws.emit('complete'); + ws.end(); + return true; + }; + return ws; + }; + + this.delete = () => { + return Promise.resolve(); + }; + } +} + +class FakeNotification { + bucket: Bucket; + id: string; + constructor(bucket: Bucket, id: string) { + this.bucket = bucket; + this.id = id; + } +} + +let fsStatOverride: Function | null; +const fakeFs = { + ...fs, + stat: (filePath: string, callback: Function) => { + return (fsStatOverride || fs.stat)(filePath, callback); + }, +}; + +let pLimitOverride: Function | null; +const fakePLimit = (limit: number) => (pLimitOverride || pLimit)(limit); + +let promisified = false; +const fakePromisify = { + // tslint:disable-next-line:variable-name + promisifyAll(Class: Function, options: PromisifyAllOptions) { + if (Class.name !== 'Bucket') { + return; + } + + promisified = true; + assert.deepStrictEqual(options.exclude, [ + 'cloudStorageURI', + 'request', + 'file', + 'notification', + 'restore', + ]); + }, +}; + +const fakeUtil = Object.assign({}, util); +fakeUtil.noop = util.noop; + +let extended = false; +const fakePaginator = { + paginator: { + // tslint:disable-next-line:variable-name + extend(Class: Function, methods: string[]) { + if (Class.name !== 'Bucket') { + return; + } + methods = Array.isArray(methods) ? methods : [methods]; + assert.strictEqual(Class.name, 'Bucket'); + assert.deepStrictEqual(methods, ['getFiles']); + extended = true; + }, + streamify(methodName: string) { + return methodName; + }, + }, +}; + +class FakeAcl { + calledWith_: Array<{}>; + constructor(...args: Array<{}>) { + this.calledWith_ = args; + } +} + +class FakeIam { + calledWith_: Array<{}>; + constructor(...args: Array<{}>) { + this.calledWith_ = args; + } +} + +class FakeServiceObject extends ServiceObject { + calledWith_: IArguments; + constructor(config: ServiceObjectConfig) { + super(config); + // eslint-disable-next-line prefer-rest-params + this.calledWith_ = arguments; + } +} + +const fakeSigner = { + URLSigner: () => {}, +}; + +class HTTPError extends Error { + code: number; + constructor(message: string, code: number) { + super(message); + this.code = code; + } +} + +describe('Bucket', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let Bucket: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let bucket: any; + + const STORAGE = { + createBucket: util.noop, + retryOptions: { + autoRetry: true, + maxRetries: 3, + retryDelayMultiplier: 2, + totalTimeout: 600, + maxRetryDelay: 60, + retryableErrorFn: (err: HTTPError) => { + return err.code === 500; + }, + idempotencyStrategy: IdempotencyStrategy.RetryConditional, + }, + crc32cGenerator: () => new CRC32C(), + universeDomain: DEFAULT_UNIVERSE, + }; + const BUCKET_NAME = 'test-bucket'; + + before(() => { + Bucket = proxyquire('../src/bucket.js', { + fs: fakeFs, + 'p-limit': fakePLimit, + '@google-cloud/promisify': fakePromisify, + '@google-cloud/paginator': fakePaginator, + './nodejs-common': { + ServiceObject: FakeServiceObject, + util: fakeUtil, + }, + './acl.js': {Acl: FakeAcl}, + './file.js': {File: FakeFile}, + './iam.js': {Iam: FakeIam}, + './notification.js': {Notification: FakeNotification}, + './signer.js': fakeSigner, + }).Bucket; + }); + + beforeEach(() => { + fsStatOverride = null; + pLimitOverride = null; + bucket = new Bucket(STORAGE, BUCKET_NAME); + }); + + describe('instantiation', () => { + it('should extend the correct methods', () => { + assert(extended); // See `fakePaginator.extend` + }); + + it('should streamify the correct methods', () => { + assert.strictEqual(bucket.getFilesStream, 'getFiles'); + }); + + it('should promisify all the things', () => { + assert(promisified); + }); + + it('should remove a leading gs://', () => { + const bucket = new Bucket(STORAGE, 'gs://bucket-name'); + assert.strictEqual(bucket.name, 'bucket-name'); + }); + + it('should remove a trailing /', () => { + const bucket = new Bucket(STORAGE, 'bucket-name/'); + assert.strictEqual(bucket.name, 'bucket-name'); + }); + + it('should localize the name', () => { + assert.strictEqual(bucket.name, BUCKET_NAME); + }); + + it('should localize the storage instance', () => { + assert.strictEqual(bucket.storage, STORAGE); + }); + + describe('ACL objects', () => { + let _request: Function; + + before(() => { + _request = Bucket.prototype.request; + }); + + beforeEach(() => { + Bucket.prototype.request = { + bind(ctx: {}) { + return ctx; + }, + }; + + bucket = new Bucket(STORAGE, BUCKET_NAME); + }); + + after(() => { + Bucket.prototype.request = _request; + }); + + it('should create an ACL object', () => { + assert.deepStrictEqual(bucket.acl.calledWith_[0], { + request: bucket, + pathPrefix: '/acl', + }); + }); + + it('should create a default ACL object', () => { + assert.deepStrictEqual(bucket.acl.default.calledWith_[0], { + request: bucket, + pathPrefix: '/defaultObjectAcl', + }); + }); + }); + + it('should inherit from ServiceObject', done => { + const storageInstance = Object.assign({}, STORAGE, { + createBucket: { + bind(context: {}) { + assert.strictEqual(context, storageInstance); + done(); + }, + }, + }); + + const bucket = new Bucket(storageInstance, BUCKET_NAME); + // Using assert.strictEqual instead of assert to prevent + // coercing of types. + assert.strictEqual(bucket instanceof ServiceObject, true); + + const calledWith = bucket.calledWith_[0]; + + assert.strictEqual(calledWith.parent, storageInstance); + assert.strictEqual(calledWith.baseUrl, '/b'); + assert.strictEqual(calledWith.id, BUCKET_NAME); + assert.deepStrictEqual(calledWith.methods, { + create: {reqOpts: {qs: {}}}, + delete: {reqOpts: {qs: {}}}, + exists: {reqOpts: {qs: {}}}, + get: {reqOpts: {qs: {}}}, + getMetadata: {reqOpts: {qs: {}}}, + setMetadata: {reqOpts: {qs: {}}}, + }); + }); + + it('should set the correct query string with a userProject', () => { + const options = {userProject: 'user-project'}; + const bucket = new Bucket(STORAGE, BUCKET_NAME, options); + const calledWith = bucket.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + create: {reqOpts: {qs: options}}, + delete: {reqOpts: {qs: options}}, + exists: {reqOpts: {qs: options}}, + get: {reqOpts: {qs: options}}, + getMetadata: {reqOpts: {qs: options}}, + setMetadata: {reqOpts: {qs: options}}, + }); + }); + + it('should set the correct query string with ifGenerationMatch', () => { + const options = {preconditionOpts: {ifGenerationMatch: 100}}; + const bucket = new Bucket(STORAGE, BUCKET_NAME, options); + + const calledWith = bucket.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + create: {reqOpts: {qs: options.preconditionOpts}}, + delete: {reqOpts: {qs: options.preconditionOpts}}, + exists: {reqOpts: {qs: options.preconditionOpts}}, + get: {reqOpts: {qs: options.preconditionOpts}}, + getMetadata: {reqOpts: {qs: options.preconditionOpts}}, + setMetadata: {reqOpts: {qs: options.preconditionOpts}}, + }); + assert.deepStrictEqual( + bucket.instancePreconditionOpts, + options.preconditionOpts + ); + }); + + it('should set the correct query string with ifGenerationNotMatch', () => { + const options = {preconditionOpts: {ifGenerationNotMatch: 100}}; + const bucket = new Bucket(STORAGE, BUCKET_NAME, options); + + const calledWith = bucket.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + create: {reqOpts: {qs: options.preconditionOpts}}, + delete: {reqOpts: {qs: options.preconditionOpts}}, + exists: {reqOpts: {qs: options.preconditionOpts}}, + get: {reqOpts: {qs: options.preconditionOpts}}, + getMetadata: {reqOpts: {qs: options.preconditionOpts}}, + setMetadata: {reqOpts: {qs: options.preconditionOpts}}, + }); + assert.deepStrictEqual( + bucket.instancePreconditionOpts, + options.preconditionOpts + ); + }); + + it('should set the correct query string with ifMetagenerationMatch', () => { + const options = {preconditionOpts: {ifMetagenerationMatch: 100}}; + const bucket = new Bucket(STORAGE, BUCKET_NAME, options); + + const calledWith = bucket.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + create: {reqOpts: {qs: options.preconditionOpts}}, + delete: {reqOpts: {qs: options.preconditionOpts}}, + exists: {reqOpts: {qs: options.preconditionOpts}}, + get: {reqOpts: {qs: options.preconditionOpts}}, + getMetadata: {reqOpts: {qs: options.preconditionOpts}}, + setMetadata: {reqOpts: {qs: options.preconditionOpts}}, + }); + assert.deepStrictEqual( + bucket.instancePreconditionOpts, + options.preconditionOpts + ); + }); + + it('should set the correct query string with ifMetagenerationNotMatch', () => { + const options = {preconditionOpts: {ifMetagenerationNotMatch: 100}}; + const bucket = new Bucket(STORAGE, BUCKET_NAME, options); + + const calledWith = bucket.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + create: {reqOpts: {qs: options.preconditionOpts}}, + delete: {reqOpts: {qs: options.preconditionOpts}}, + exists: {reqOpts: {qs: options.preconditionOpts}}, + get: {reqOpts: {qs: options.preconditionOpts}}, + getMetadata: {reqOpts: {qs: options.preconditionOpts}}, + setMetadata: {reqOpts: {qs: options.preconditionOpts}}, + }); + assert.deepStrictEqual( + bucket.instancePreconditionOpts, + options.preconditionOpts + ); + }); + + it('should localize an Iam instance', () => { + assert(bucket.iam instanceof FakeIam); + assert.deepStrictEqual(bucket.iam.calledWith_[0], bucket); + }); + + it('should localize userProject if provided', () => { + const fakeUserProject = 'grape-spaceship-123'; + const bucket = new Bucket(STORAGE, BUCKET_NAME, { + userProject: fakeUserProject, + }); + + assert.strictEqual(bucket.userProject, fakeUserProject); + }); + + it('should accept a `crc32cGenerator`', () => { + const crc32cGenerator = () => {}; + + const bucket = new Bucket(STORAGE, 'bucket-name', {crc32cGenerator}); + assert.strictEqual(bucket.crc32cGenerator, crc32cGenerator); + }); + + it("should use storage's `crc32cGenerator` by default", () => { + assert.strictEqual(bucket.crc32cGenerator, STORAGE.crc32cGenerator); + }); + }); + + describe('cloudStorageURI', () => { + it('should return the appropriate `gs://` URI', () => { + const bucket = new Bucket(STORAGE, BUCKET_NAME); + + assert(bucket.cloudStorageURI instanceof URL); + assert.equal(bucket.cloudStorageURI.host, BUCKET_NAME); + }); + }); + + describe('addLifecycleRule', () => { + beforeEach(() => { + bucket.getMetadata = (callback: GetBucketMetadataCallback) => { + callback(null, {}, {}); + }; + }); + + it('should accept raw input', done => { + const rule = { + action: { + type: 'type', + }, + condition: {}, + }; + + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.deepStrictEqual(metadata.lifecycle!.rule, [rule]); + done(); + }; + + bucket.addLifecycleRule(rule, assert.ifError); + }); + + it('should properly set condition', done => { + const rule = { + action: { + type: 'Delete', + }, + condition: { + age: 30, + }, + }; + + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.deepStrictEqual(metadata.lifecycle?.rule, [ + { + action: { + type: 'Delete', + }, + condition: rule.condition, + }, + ]); + done(); + }; + + bucket.addLifecycleRule(rule, assert.ifError); + }); + + it('should convert Date object to date string for condition', done => { + const date = new Date(); + + const rule = { + action: { + type: 'Delete', + }, + condition: { + createdBefore: date, + }, + }; + + bucket.setMetadata = (metadata: BucketMetadata) => { + const expectedDateString = date.toISOString().replace(/T.+$/, ''); + + const rule = metadata!.lifecycle!.rule![0]; + assert.strictEqual(rule.condition.createdBefore, expectedDateString); + + done(); + }; + + bucket.addLifecycleRule(rule, assert.ifError); + }); + + it('should optionally overwrite existing rules', done => { + const rule = { + action: { + type: 'type', + }, + condition: {}, + }; + + const options = { + append: false, + }; + + bucket.getMetadata = () => { + done(new Error('Metadata should not be refreshed.')); + }; + + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.strictEqual(metadata!.lifecycle!.rule!.length, 1); + assert.deepStrictEqual(metadata.lifecycle?.rule, [rule]); + done(); + }; + + bucket.addLifecycleRule(rule, options, assert.ifError); + }); + + it('should combine rule with existing rules by default', done => { + const existingRule: LifecycleRule = { + action: { + type: 'Delete', + }, + condition: {}, + }; + + const newRule: LifecycleRule = { + action: { + type: 'Delete', + }, + condition: {}, + }; + + bucket.getMetadata = (callback: GetBucketMetadataCallback) => { + callback(null, {lifecycle: {rule: [existingRule]}}, {}); + }; + + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.strictEqual(metadata!.lifecycle!.rule!.length, 2); + assert.deepStrictEqual(metadata.lifecycle?.rule, [ + existingRule, + newRule, + ]); + done(); + }; + + bucket.addLifecycleRule(newRule, assert.ifError); + }); + + it('should accept multiple rules', done => { + const existingRule: LifecycleRule = { + action: { + type: 'Delete', + }, + condition: {}, + }; + + const newRules: LifecycleRule[] = [ + { + action: { + type: 'Delete', + }, + condition: {}, + }, + { + action: { + type: 'Delete', + }, + condition: {}, + }, + ]; + + bucket.getMetadata = (callback: GetBucketMetadataCallback) => { + callback(null, {lifecycle: {rule: [existingRule]}}, {}); + }; + + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.strictEqual(metadata!.lifecycle!.rule!.length, 3); + assert.deepStrictEqual(metadata.lifecycle?.rule, [ + existingRule, + newRules[0], + newRules[1], + ]); + done(); + }; + + bucket.addLifecycleRule(newRules, assert.ifError); + }); + + it('should pass error from getMetadata to callback', done => { + const error = new Error('from getMetadata'); + const rule = { + action: 'delete', + condition: {}, + }; + + bucket.getMetadata = (callback: Function) => { + callback(error); + }; + + bucket.setMetadata = () => { + done(new Error('Metadata should not be set.')); + }; + + bucket.addLifecycleRule(rule, (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + }); + + describe('combine', () => { + it('should throw if invalid sources are provided', () => { + assert.throws(() => { + bucket.combine(), BucketExceptionMessages.PROVIDE_SOURCE_FILE; + }); + + assert.throws(() => { + bucket.combine([]), BucketExceptionMessages.PROVIDE_SOURCE_FILE; + }); + }); + + it('should throw if a destination is not provided', () => { + assert.throws(() => { + bucket.combine(['1', '2']), + BucketExceptionMessages.DESTINATION_FILE_NOT_SPECIFIED; + }); + }); + + it('should accept string or file input for sources', done => { + const file1 = bucket.file('1.txt'); + const file2 = '2.txt'; + const destinationFileName = 'destination.txt'; + + const originalFileMethod = bucket.file; + bucket.file = (name: string) => { + const file = originalFileMethod(name); + + if (name === '2.txt') { + return file; + } + + assert.strictEqual(name, destinationFileName); + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.method, 'POST'); + assert.strictEqual(reqOpts.uri, '/compose'); + assert.strictEqual(reqOpts.json.sourceObjects[0].name, file1.name); + assert.strictEqual(reqOpts.json.sourceObjects[1].name, file2); + + done(); + }; + + return file; + }; + + bucket.combine([file1, file2], destinationFileName); + }); + + it('should use content type from the destination metadata', done => { + const destination = bucket.file('destination.txt'); + + destination.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.json.destination.contentType, + mime.getType(destination.name) + ); + + done(); + }; + + bucket.combine(['1', '2'], destination); + }); + + it('should use content type from the destination metadata', done => { + const destination = bucket.file('destination.txt'); + destination.metadata = {contentType: 'content-type'}; + + destination.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.json.destination.contentType, + destination.metadata.contentType + ); + + done(); + }; + + bucket.combine(['1', '2'], destination); + }); + + it('should detect dest content type if not in metadata', done => { + const destination = bucket.file('destination.txt'); + + destination.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.json.destination.contentType, + mime.getType(destination.name) + ); + + done(); + }; + + bucket.combine(['1', '2'], destination); + }); + + it('should make correct API request', done => { + const sources = [bucket.file('1.foo'), bucket.file('2.foo')]; + const destination = bucket.file('destination.foo'); + + destination.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, '/compose'); + assert.deepStrictEqual(reqOpts.json, { + destination: { + contentType: mime.getType(destination.name) || undefined, + contentEncoding: undefined, + }, + sourceObjects: [{name: sources[0].name}, {name: sources[1].name}], + }); + + done(); + }; + + bucket.combine(sources, destination); + }); + + it('should encode the destination file name', done => { + const sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + const destination = bucket.file('needs encoding.jpg'); + + destination.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri.indexOf(destination), -1); + done(); + }; + + bucket.combine(sources, destination); + }); + + it('should send a source generation value if available', done => { + const sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + sources[0].metadata = {generation: 1}; + sources[1].metadata = {generation: 2}; + + const destination = bucket.file('destination.txt'); + + destination.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.json.sourceObjects, [ + {name: sources[0].name, generation: sources[0].metadata.generation}, + {name: sources[1].name, generation: sources[1].metadata.generation}, + ]); + + done(); + }; + + bucket.combine(sources, destination); + }); + + it('should accept userProject option', done => { + const options = { + userProject: 'user-project-id', + }; + + const sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + const destination = bucket.file('destination.txt'); + + destination.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs, options); + done(); + }; + + bucket.combine(sources, destination, options, assert.ifError); + }); + + it('should accept precondition options', done => { + const options = { + ifGenerationMatch: 100, + ifGenerationNotMatch: 101, + ifMetagenerationMatch: 102, + ifMetagenerationNotMatch: 103, + }; + + const sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + const destination = bucket.file('destination.txt'); + + destination.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.qs.ifGenerationMatch, + options.ifGenerationMatch + ); + assert.strictEqual( + reqOpts.qs.ifGenerationNotMatch, + options.ifGenerationNotMatch + ); + assert.strictEqual( + reqOpts.qs.ifMetagenerationMatch, + options.ifMetagenerationMatch + ); + assert.strictEqual( + reqOpts.qs.ifMetagenerationNotMatch, + options.ifMetagenerationNotMatch + ); + done(); + }; + + bucket.combine(sources, destination, options, assert.ifError); + }); + + it('should execute the callback', done => { + const sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + const destination = bucket.file('destination.txt'); + + destination.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(); + }; + + bucket.combine(sources, destination, done); + }); + + it('should execute the callback with an error', done => { + const sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + const destination = bucket.file('destination.txt'); + + const error = new Error('Error.'); + + destination.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error); + }; + + bucket.combine(sources, destination, (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should execute the callback with apiResponse', done => { + const sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + const destination = bucket.file('destination.txt'); + const resp = {success: true}; + + destination.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, resp); + }; + + bucket.combine( + sources, + destination, + (err: Error, obj: {}, apiResponse: {}) => { + assert.strictEqual(resp, apiResponse); + done(); + } + ); + }); + + it('should set maxRetries to 0 when ifGenerationMatch is undefined', done => { + const sources = [bucket.file('1.txt'), bucket.file('2.txt')]; + const destination = bucket.file('destination.txt'); + + destination.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.maxRetries, 0); + callback(); + }; + + bucket.combine(sources, destination, done); + }); + }); + + describe('createChannel', () => { + const ID = 'id'; + const CONFIG = { + address: 'https://...', + }; + + it('should throw if an ID is not provided', () => { + assert.throws(() => { + bucket.createChannel(), BucketExceptionMessages.CHANNEL_ID_REQUIRED; + }); + }); + + it('should make the correct request', done => { + const config = Object.assign({}, CONFIG, { + a: 'b', + c: 'd', + }); + const originalConfig = Object.assign({}, config); + + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.method, 'POST'); + assert.strictEqual(reqOpts.uri, '/o/watch'); + + const expectedJson = Object.assign({}, config, { + id: ID, + type: 'web_hook', + }); + assert.deepStrictEqual(reqOpts.json, expectedJson); + assert.deepStrictEqual(config, originalConfig); + + done(); + }; + + bucket.createChannel(ID, config, assert.ifError); + }); + + it('should accept userProject option', done => { + const options = { + userProject: 'user-project-id', + }; + + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs, options); + done(); + }; + + bucket.createChannel(ID, CONFIG, options, assert.ifError); + }); + + describe('error', () => { + const error = new Error('Error.'); + const apiResponse = {}; + + beforeEach(() => { + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error, apiResponse); + }; + }); + + it('should execute callback with error & API response', done => { + bucket.createChannel( + ID, + CONFIG, + (err: Error, channel: Channel, apiResponse_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(channel, null); + assert.strictEqual(apiResponse_, apiResponse); + + done(); + } + ); + }); + }); + + describe('success', () => { + const apiResponse = { + resourceId: 'resource-id', + }; + + beforeEach(() => { + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, apiResponse); + }; + }); + + it('should exec a callback with Channel & API response', done => { + const channel = {}; + + bucket.storage.channel = (id: string, resourceId: string) => { + assert.strictEqual(id, ID); + assert.strictEqual(resourceId, apiResponse.resourceId); + return channel; + }; + + bucket.createChannel( + ID, + CONFIG, + (err: Error, channel_: Channel, apiResponse_: {}) => { + assert.ifError(err); + assert.strictEqual(channel_, channel); + assert.strictEqual(channel_.metadata, apiResponse); + assert.strictEqual(apiResponse_, apiResponse); + done(); + } + ); + }); + }); + }); + + describe('createNotification', () => { + const PUBSUB_SERVICE_PATH = '//pubsub.googleapis.com/'; + const TOPIC = 'my-topic'; + const FULL_TOPIC_NAME = + PUBSUB_SERVICE_PATH + 'projects/{{projectId}}/topics/' + TOPIC; + + class FakeTopic { + name: string; + constructor(name: string) { + this.name = 'projects/grape-spaceship-123/topics/' + name; + } + } + + beforeEach(() => { + fakeUtil.isCustomType = util.isCustomType; + }); + + it('should throw an error if a valid topic is not provided', () => { + assert.throws(() => { + bucket.createNotification(), + BucketExceptionMessages.TOPIC_NAME_REQUIRED; + }); + }); + + it('should make the correct request', done => { + const topic = 'projects/my-project/topics/my-topic'; + const options = {payloadFormat: 'NONE'}; + const expectedTopic = PUBSUB_SERVICE_PATH + topic; + const expectedJson = Object.assign( + {topic: expectedTopic}, + convertObjKeysToSnakeCase(options) + ); + + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.method, 'POST'); + assert.strictEqual(reqOpts.uri, '/notificationConfigs'); + assert.deepStrictEqual(reqOpts.json, expectedJson); + assert.notStrictEqual(reqOpts.json, options); + done(); + }; + + bucket.createNotification(topic, options, assert.ifError); + }); + + it('should accept incomplete topic names', done => { + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.json.topic, FULL_TOPIC_NAME); + done(); + }; + + bucket.createNotification(TOPIC, {}, assert.ifError); + }); + + it('should accept a topic object', done => { + const fakeTopic = new FakeTopic('my-topic'); + const expectedTopicName = PUBSUB_SERVICE_PATH + fakeTopic.name; + + fakeUtil.isCustomType = (topic, type) => { + assert.strictEqual(topic, fakeTopic); + assert.strictEqual(type, 'pubsub/topic'); + return true; + }; + + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.json.topic, expectedTopicName); + done(); + }; + + bucket.createNotification(fakeTopic, {}, assert.ifError); + }); + + it('should set a default payload format', done => { + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.json.payload_format, 'JSON_API_V1'); + done(); + }; + + bucket.createNotification(TOPIC, {}, assert.ifError); + }); + + it('should optionally accept options', done => { + const expectedJson = { + topic: FULL_TOPIC_NAME, + payload_format: 'JSON_API_V1', + }; + + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.json, expectedJson); + done(); + }; + + bucket.createNotification(TOPIC, assert.ifError); + }); + + it('should accept a userProject', done => { + const options = { + userProject: 'grape-spaceship-123', + }; + + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.userProject, options.userProject); + done(); + }; + + bucket.createNotification(TOPIC, options, assert.ifError); + }); + + it('should return errors to the callback', done => { + const error = new Error('err'); + const response = {}; + + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error, response); + }; + + bucket.createNotification( + TOPIC, + (err: Error, notification: Notification, resp: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(notification, null); + assert.strictEqual(resp, response); + done(); + } + ); + }); + + it('should return a notification object', done => { + const fakeId = '123'; + const response = {id: fakeId}; + const fakeNotification = {}; + + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, response); + }; + + bucket.notification = (id: string) => { + assert.strictEqual(id, fakeId); + return fakeNotification; + }; + + bucket.createNotification( + TOPIC, + (err: Error, notification: Notification, resp: {}) => { + assert.ifError(err); + assert.strictEqual(notification, fakeNotification); + assert.strictEqual(notification.metadata, response); + assert.strictEqual(resp, response); + done(); + } + ); + }); + }); + + describe('deleteFiles', () => { + let readCount: number; + + beforeEach(() => { + readCount = 0; + }); + + it('should accept only a callback', done => { + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.delete = () => { + return Promise.resolve(); + }; + return file; + }); + + const readable = new stream.Readable({ + objectMode: true, + read() { + if (readCount < 1) { + this.push(files[readCount]); + readCount++; + } else { + this.push(null); + } + }, + }); + + bucket.getFilesStream = (query: {}) => { + assert.deepStrictEqual(query, {}); + return readable; + }; + + bucket.deleteFiles(done); + }); + + it('should get files from the bucket', done => { + const query = {a: 'b', c: 'd'}; + + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.delete = () => { + return Promise.resolve(); + }; + return file; + }); + + const readable = new stream.Readable({ + objectMode: true, + read() { + if (readCount < 1) { + this.push(files[readCount]); + readCount++; + } else { + this.push(null); + } + }, + }); + + bucket.getFilesStream = (query_: {}) => { + assert.deepStrictEqual(query_, query); + return readable; + }; + + bucket.deleteFiles(query, done); + }); + + it('should process 10 files at a time', done => { + pLimitOverride = (limit: number) => { + assert.strictEqual(limit, 10); + setImmediate(done); + return () => {}; + }; + + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.delete = () => { + return Promise.resolve(); + }; + return file; + }); + + const readable = new stream.Readable({ + objectMode: true, + read() { + if (readCount < 1) { + this.push(files[readCount]); + readCount++; + } else { + this.push(null); + } + }, + }); + + bucket.getFilesStream = () => readable; + bucket.deleteFiles({}, assert.ifError); + }); + + it('should delete the files', done => { + const query = {}; + let timesCalled = 0; + + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.delete = (query_: {}) => { + timesCalled++; + assert.strictEqual(query_, query); + return Promise.resolve(); + }; + return file; + }); + + const readable = new stream.Readable({ + objectMode: true, + read() { + if (readCount < files.length) { + this.push(files[readCount]); + readCount++; + } else { + this.push(null); + } + }, + }); + + bucket.getFilesStream = (query_: {}) => { + assert.strictEqual(query_, query); + return readable; + }; + + bucket.deleteFiles(query, (err: Error) => { + assert.ifError(err); + assert.strictEqual(timesCalled, files.length); + done(); + }); + }); + + it('should execute callback with error from getting files', done => { + const error = new Error('Error.'); + const readable = new stream.Readable({ + objectMode: true, + read() { + this.destroy(error); + }, + }); + + bucket.getFilesStream = () => { + return readable; + }; + + bucket.deleteFiles({}, (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should execute callback with error from deleting file', done => { + const error = new Error('Error.'); + + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.delete = () => Promise.reject(error); + return file; + }); + + const readable = new stream.Readable({ + objectMode: true, + read() { + if (readCount < files.length) { + this.push(files[readCount]); + readCount++; + } else { + this.push(null); + } + }, + }); + + bucket.getFilesStream = () => { + return readable; + }; + + bucket.deleteFiles({}, (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should execute callback with queued errors', done => { + const error = new Error('Error.'); + + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.delete = () => Promise.reject(error); + return file; + }); + + const readable = new stream.Readable({ + objectMode: true, + read() { + if (readCount < files.length) { + this.push(files[readCount]); + readCount++; + } else { + this.push(null); + } + }, + }); + + bucket.getFilesStream = () => { + return readable; + }; + + bucket.deleteFiles({force: true}, (errs: Array<{}>) => { + assert.strictEqual(errs[0], error); + assert.strictEqual(errs[1], error); + done(); + }); + }); + }); + + describe('deleteLabels', () => { + describe('all labels', () => { + it('should get all of the label names', done => { + bucket.getLabels = () => { + done(); + }; + + bucket.deleteLabels(assert.ifError); + }); + + it('should return an error from getLabels()', done => { + const error = new Error('Error.'); + + bucket.getLabels = (callback: Function) => { + callback(error); + }; + + bucket.deleteLabels((err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should call setLabels with all label names', done => { + const labels = { + labelone: 'labelonevalue', + labeltwo: 'labeltwovalue', + }; + + bucket.getLabels = (callback: Function) => { + callback(null, labels); + }; + + bucket.setLabels = (labels: {}, callback: Function) => { + assert.deepStrictEqual(labels, { + labelone: null, + labeltwo: null, + }); + callback(); // done() + }; + + bucket.deleteLabels(done); + }); + }); + + describe('single label', () => { + const LABEL = 'labelname'; + + it('should call setLabels with a single label', done => { + bucket.setLabels = (labels: {}, callback: Function) => { + assert.deepStrictEqual(labels, { + [LABEL]: null, + }); + callback(); // done() + }; + + bucket.deleteLabels(LABEL, done); + }); + }); + + describe('multiple labels', () => { + const LABELS = ['labelonename', 'labeltwoname']; + + it('should call setLabels with multiple labels', done => { + bucket.setLabels = (labels: {}, callback: Function) => { + assert.deepStrictEqual(labels, { + labelonename: null, + labeltwoname: null, + }); + callback(); // done() + }; + + bucket.deleteLabels(LABELS, done); + }); + }); + }); + + describe('disableRequesterPays', () => { + it('should call setMetadata correctly', done => { + bucket.setMetadata = ( + metadata: {}, + _optionsOrCallback: {}, + callback: Function + ) => { + assert.deepStrictEqual(metadata, { + billing: { + requesterPays: false, + }, + }); + Promise.resolve([]).then(resp => callback(null, ...resp)); + }; + + bucket.disableRequesterPays(done); + }); + + it('should not require a callback', done => { + bucket.setMetadata = ( + metadata: {}, + optionsOrCallback: {}, + callback: Function + ) => { + assert.strictEqual(callback, undefined); + done(); + }; + + bucket.disableRequesterPays(); + }); + + it('should set autoRetry to false when ifMetagenerationMatch is undefined', done => { + bucket.setMetadata = () => { + Promise.resolve().then(() => { + assert.strictEqual(bucket.storage.retryOptions.autoRetry, false); + done(); + }); + }; + bucket.disableRequesterPays(); + }); + }); + + describe('enableLogging', () => { + const PREFIX = 'prefix'; + + beforeEach(() => { + bucket.iam = { + getPolicy: () => Promise.resolve([{bindings: []}]), + setPolicy: () => Promise.resolve(), + }; + bucket.setMetadata = () => Promise.resolve([]); + }); + + it('should throw if a config object is not provided', () => { + assert.throws(() => { + bucket.enableLogging(), + BucketExceptionMessages.CONFIGURATION_OBJECT_PREFIX_REQUIRED; + }); + }); + + it('should throw if config is a function', () => { + assert.throws(() => { + bucket.enableLogging(assert.ifError), + BucketExceptionMessages.CONFIGURATION_OBJECT_PREFIX_REQUIRED; + }); + }); + + it('should throw if a prefix is not provided', () => { + assert.throws(() => { + bucket.enableLogging( + { + bucket: 'bucket-name', + }, + assert.ifError + ), + BucketExceptionMessages.CONFIGURATION_OBJECT_PREFIX_REQUIRED; + }); + }); + + it('should add IAM permissions', done => { + const policy = { + bindings: [{}], + }; + bucket.iam = { + getPolicy: () => Promise.resolve([policy]), + setPolicy: (policy_: Policy) => { + assert.deepStrictEqual(policy, policy_); + assert.deepStrictEqual(policy_.bindings, [ + policy.bindings[0], + { + members: ['group:cloud-storage-analytics@google.com'], + role: 'roles/storage.objectCreator', + }, + ]); + setImmediate(done); + return Promise.resolve(); + }, + }; + + bucket.enableLogging({prefix: PREFIX}, assert.ifError); + }); + + it('should return an error from getting the IAM policy', done => { + const error = new Error('Error.'); + + bucket.iam.getPolicy = () => { + throw error; + }; + + bucket.enableLogging({prefix: PREFIX}, (err: Error | null) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should return an error from setting the IAM policy', done => { + const error = new Error('Error.'); + + bucket.iam.setPolicy = () => { + throw error; + }; + + bucket.enableLogging({prefix: PREFIX}, (err: Error | null) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should update the logging metadata configuration', done => { + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.deepStrictEqual(metadata.logging, { + logBucket: bucket.id, + logObjectPrefix: PREFIX, + }); + setImmediate(done); + return Promise.resolve([]); + }; + + bucket.enableLogging({prefix: PREFIX}, assert.ifError); + }); + + it('should allow a custom bucket to be provided', done => { + const bucketName = 'bucket-name'; + + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.deepStrictEqual(metadata!.logging!.logBucket, bucketName); + setImmediate(done); + return Promise.resolve([]); + }; + + bucket.enableLogging( + { + prefix: PREFIX, + bucket: bucketName, + }, + assert.ifError + ); + }); + + it('should accept a Bucket object', done => { + const bucketForLogging = new Bucket(STORAGE, 'bucket-name'); + + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.deepStrictEqual( + metadata!.logging!.logBucket, + bucketForLogging.id + ); + setImmediate(done); + return Promise.resolve([]); + }; + + bucket.enableLogging( + { + prefix: PREFIX, + bucket: bucketForLogging, + }, + assert.ifError + ); + }); + + it('should execute the callback with the setMetadata response', done => { + const setMetadataResponse = {}; + + bucket.setMetadata = ( + metadata: {}, + optionsOrCallback: {}, + callback: Function + ) => { + Promise.resolve([setMetadataResponse]).then(resp => + callback(null, ...resp) + ); + }; + + bucket.enableLogging( + {prefix: PREFIX}, + (err: Error | null, response: SetBucketMetadataResponse) => { + assert.ifError(err); + assert.strictEqual(response, setMetadataResponse); + done(); + } + ); + }); + + it('should return an error from the setMetadata call failing', done => { + const error = new Error('Error.'); + + bucket.setMetadata = () => { + throw error; + }; + + bucket.enableLogging({prefix: PREFIX}, (err: Error | null) => { + assert.strictEqual(err, error); + done(); + }); + }); + }); + + describe('enableRequesterPays', () => { + it('should call setMetadata correctly', done => { + bucket.setMetadata = ( + metadata: {}, + optionsOrCallback: {}, + callback: Function + ) => { + assert.deepStrictEqual(metadata, { + billing: { + requesterPays: true, + }, + }); + Promise.resolve([]).then(resp => callback(null, ...resp)); + }; + + bucket.enableRequesterPays(done); + }); + + it('should not require a callback', done => { + bucket.setMetadata = ( + metadata: {}, + optionsOrCallback: {}, + callback: Function + ) => { + assert.equal(callback, undefined); + done(); + }; + + bucket.enableRequesterPays(); + }); + }); + + describe('file', () => { + const FILE_NAME = 'remote-file-name.jpg'; + let file: FakeFile; + const options = {a: 'b', c: 'd'}; + + beforeEach(() => { + file = bucket.file(FILE_NAME, options); + }); + + it('should throw if no name is provided', () => { + assert.throws(() => { + bucket.file(), BucketExceptionMessages.SPECIFY_FILE_NAME; + }); + }); + + it('should return a File object', () => { + assert(file instanceof FakeFile); + }); + + it('should pass bucket to File object', () => { + assert.deepStrictEqual(file.calledWith_[0], bucket); + }); + + it('should pass filename to File object', () => { + assert.strictEqual(file.calledWith_[1], FILE_NAME); + }); + + it('should pass configuration object to File', () => { + assert.deepStrictEqual(file.calledWith_[2], options); + }); + }); + + describe('getFiles', () => { + it('should get files without a query', done => { + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, '/o'); + assert.deepStrictEqual(reqOpts.qs, {}); + done(); + }; + + bucket.getFiles(util.noop); + }); + + it('should get files with a query', done => { + const token = 'next-page-token'; + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.qs, { + maxResults: 5, + pageToken: token, + includeFoldersAsPrefixes: true, + delimiter: '/', + autoPaginate: false, + }); + done(); + }; + bucket.getFiles( + { + maxResults: 5, + pageToken: token, + includeFoldersAsPrefixes: true, + delimiter: '/', + autoPaginate: false, + }, + util.noop + ); + }); + + it('should return nextQuery if more results exist', () => { + const token = 'next-page-token'; + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, {nextPageToken: token, items: []}); + }; + bucket.getFiles( + {maxResults: 5}, + (err: Error, results: {}, nextQuery: GetFilesOptions) => { + assert.strictEqual(nextQuery.pageToken, token); + assert.strictEqual(nextQuery.maxResults, 5); + } + ); + }); + + it('should return null nextQuery if there are no more results', () => { + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, {items: []}); + }; + bucket.getFiles( + {maxResults: 5}, + (err: Error, results: {}, nextQuery: {}) => { + assert.strictEqual(nextQuery, null); + } + ); + }); + + it('should return File objects', done => { + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, { + items: [{name: 'fake-file-name', generation: 1}], + }); + }; + bucket.getFiles((err: Error, files: FakeFile[]) => { + assert.ifError(err); + assert(files[0] instanceof FakeFile); + assert.strictEqual( + typeof files[0].calledWith_[2].generation, + 'undefined' + ); + done(); + }); + }); + + it('should return versioned Files if queried for versions', done => { + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, { + items: [{name: 'fake-file-name', generation: 1}], + }); + }; + + bucket.getFiles({versions: true}, (err: Error, files: FakeFile[]) => { + assert.ifError(err); + assert(files[0] instanceof FakeFile); + assert.strictEqual(files[0].calledWith_[2].generation, 1); + done(); + }); + }); + + it('should return Files with specified values if queried for fields', done => { + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, { + items: [{name: 'fake-file-name'}], + }); + }; + + bucket.getFiles( + {fields: 'items(name)'}, + (err: Error, files: FakeFile[]) => { + assert.ifError(err); + assert.strictEqual(files[0].name, 'fake-file-name'); + done(); + } + ); + }); + + it('should add nextPageToken to fields for autoPaginate', done => { + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.qs.fields, 'items(name),nextPageToken'); + callback(null, { + items: [{name: 'fake-file-name'}], + nextPageToken: 'fake-page-token', + }); + }; + + bucket.getFiles( + {fields: 'items(name)', autoPaginate: true}, + (err: Error, files: FakeFile[], nextQuery: {pageToken: string}) => { + assert.ifError(err); + assert.strictEqual(files[0].name, 'fake-file-name'); + assert.strictEqual(nextQuery.pageToken, 'fake-page-token'); + done(); + } + ); + }); + + it('should return soft-deleted Files if queried for softDeleted', done => { + const softDeletedTime = new Date('1/1/2024').toISOString(); + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, { + items: [{name: 'fake-file-name', generation: 1, softDeletedTime}], + }); + }; + + bucket.getFiles({softDeleted: true}, (err: Error, files: FakeFile[]) => { + assert.ifError(err); + assert(files[0] instanceof FakeFile); + assert.strictEqual(files[0].metadata.softDeletedTime, softDeletedTime); + done(); + }); + }); + + it('should set kmsKeyName on file', done => { + const kmsKeyName = 'kms-key-name'; + + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, { + items: [{name: 'fake-file-name', kmsKeyName}], + }); + }; + + bucket.getFiles({versions: true}, (err: Error, files: FakeFile[]) => { + assert.ifError(err); + assert.strictEqual(files[0].calledWith_[2].kmsKeyName, kmsKeyName); + done(); + }); + }); + + it('should return apiResponse in callback', done => { + const resp = {items: [{name: 'fake-file-name'}]}; + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, resp); + }; + bucket.getFiles( + (err: Error, files: Array<{}>, nextQuery: {}, apiResponse: {}) => { + assert.deepStrictEqual(resp, apiResponse); + done(); + } + ); + }); + + it('should execute callback with error & API response', done => { + const error = new Error('Error.'); + const apiResponse = {}; + + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error, apiResponse); + }; + + bucket.getFiles( + (err: Error, files: File[], nextQuery: {}, apiResponse_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(files, null); + assert.strictEqual(nextQuery, null); + assert.strictEqual(apiResponse_, apiResponse); + + done(); + } + ); + }); + + it('should populate returned File object with metadata', done => { + const fileMetadata = { + name: 'filename', + contentType: 'x-zebra', + metadata: { + my: 'custom metadata', + }, + }; + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, {items: [fileMetadata]}); + }; + bucket.getFiles((err: Error, files: FakeFile[]) => { + assert.ifError(err); + assert.deepStrictEqual(files[0].metadata, fileMetadata); + done(); + }); + }); + }); + + describe('getLabels', () => { + it('should refresh metadata', done => { + bucket.getMetadata = () => { + done(); + }; + + bucket.getLabels(assert.ifError); + }); + + it('should accept an options object', done => { + const options = {}; + + bucket.getMetadata = (options_: {}) => { + assert.strictEqual(options_, options); + done(); + }; + + bucket.getLabels(options, assert.ifError); + }); + + it('should return error from getMetadata', done => { + const error = new Error('Error.'); + + bucket.getMetadata = (options: {}, callback: Function) => { + callback(error); + }; + + bucket.getLabels((err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should return labels metadata property', done => { + const metadata = { + labels: { + label: 'labelvalue', + }, + }; + + bucket.getMetadata = (options: {}, callback: Function) => { + callback(null, metadata); + }; + + bucket.getLabels((err: Error, labels: {}) => { + assert.ifError(err); + assert.strictEqual(labels, metadata.labels); + done(); + }); + }); + + it('should return empty object if no labels exist', done => { + const metadata = {}; + + bucket.getMetadata = (options: {}, callback: Function) => { + callback(null, metadata); + }; + + bucket.getLabels((err: Error, labels: {}) => { + assert.ifError(err); + assert.deepStrictEqual(labels, {}); + done(); + }); + }); + }); + + describe('getNotifications', () => { + it('should make the correct request', done => { + const options = {}; + + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, '/notificationConfigs'); + assert.strictEqual(reqOpts.qs, options); + done(); + }; + + bucket.getNotifications(options, assert.ifError); + }); + + it('should optionally accept options', done => { + bucket.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.qs, {}); + done(); + }; + + bucket.getNotifications(assert.ifError); + }); + + it('should return any errors to the callback', done => { + const error = new Error('err'); + const response = {}; + + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error, response); + }; + + bucket.getNotifications( + (err: Error, notifications: Notification[], resp: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(notifications, null); + assert.strictEqual(resp, response); + done(); + } + ); + }); + + it('should return a list of notification objects', done => { + const fakeItems = [{id: '1'}, {id: '2'}, {id: '3'}]; + const response = {items: fakeItems}; + + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, response); + }; + + let callCount = 0; + const fakeNotifications = [{}, {}, {}]; + + bucket.notification = (id: string) => { + const expectedId = fakeItems[callCount].id; + assert.strictEqual(id, expectedId); + return fakeNotifications[callCount++]; + }; + + bucket.getNotifications( + (err: Error, notifications: Notification[], resp: {}) => { + assert.ifError(err); + notifications.forEach((notification, i) => { + assert.strictEqual(notification, fakeNotifications[i]); + assert.strictEqual(notification.metadata, fakeItems[i]); + }); + assert.strictEqual(resp, response); + done(); + } + ); + }); + }); + + describe('getSignedUrl', () => { + const EXPECTED_SIGNED_URL = 'signed-url'; + const CNAME = 'https://www.example.com'; + + let sandbox: sinon.SinonSandbox; + let signer: {getSignedUrl: Function}; + let signerGetSignedUrlStub: sinon.SinonStub; + let urlSignerStub: sinon.SinonStub; + let SIGNED_URL_CONFIG: GetBucketSignedUrlConfig; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + + signerGetSignedUrlStub = sandbox.stub().resolves(EXPECTED_SIGNED_URL); + + signer = { + getSignedUrl: signerGetSignedUrlStub, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + urlSignerStub = (sandbox.stub as any)(fakeSigner, 'URLSigner').returns( + signer + ); + + SIGNED_URL_CONFIG = { + version: 'v4', + expires: new Date(), + action: 'list', + cname: CNAME, + }; + }); + + afterEach(() => sandbox.restore()); + + it('should construct a URLSigner and call getSignedUrl', done => { + // assert signer is lazily-initialized. + assert.strictEqual(bucket.signer, undefined); + bucket.getSignedUrl( + SIGNED_URL_CONFIG, + (err: Error | null, signedUrl: string) => { + assert.ifError(err); + assert.strictEqual(bucket.signer, signer); + assert.strictEqual(signedUrl, EXPECTED_SIGNED_URL); + + const ctorArgs = urlSignerStub.getCall(0).args; + assert.strictEqual(ctorArgs[0], bucket.storage.authClient); + assert.strictEqual(ctorArgs[1], bucket); + + const getSignedUrlArgs = signerGetSignedUrlStub.getCall(0).args; + assert.deepStrictEqual(getSignedUrlArgs[0], { + method: 'GET', + version: 'v4', + expires: SIGNED_URL_CONFIG.expires, + extensionHeaders: {}, + host: undefined, + queryParams: {}, + cname: CNAME, + signingEndpoint: undefined, + }); + done(); + } + ); + }); + }); + + describe('lock', () => { + it('should throw if a metageneration is not provided', () => { + assert.throws(() => { + bucket.lock(assert.ifError), + BucketExceptionMessages.METAGENERATION_NOT_PROVIDED; + }); + }); + + it('should make the correct request', done => { + const metageneration = 8; + + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.deepStrictEqual(reqOpts, { + method: 'POST', + uri: '/lockRetentionPolicy', + qs: { + ifMetagenerationMatch: metageneration, + }, + }); + + callback(); // done() + }; + + bucket.lock(metageneration, done); + }); + }); + + describe('makePrivate', () => { + it('should set predefinedAcl & privatize files', done => { + let didSetPredefinedAcl = false; + let didMakeFilesPrivate = false; + const opts = { + includeFiles: true, + force: true, + }; + + bucket.setMetadata = (metadata: {}, options: {}, callback: Function) => { + assert.deepStrictEqual(metadata, {acl: null}); + assert.deepStrictEqual(options, {predefinedAcl: 'projectPrivate'}); + + didSetPredefinedAcl = true; + bucket.makeAllFilesPublicPrivate_(opts, callback); + }; + + bucket.makeAllFilesPublicPrivate_ = ( + opts: MakeAllFilesPublicPrivateOptions, + callback: Function + ) => { + assert.strictEqual(opts.private, true); + assert.strictEqual(opts.force, true); + didMakeFilesPrivate = true; + callback(); + }; + + bucket.makePrivate(opts, (err: Error) => { + assert.ifError(err); + assert(didSetPredefinedAcl); + assert(didMakeFilesPrivate); + done(); + }); + }); + + it('should accept metadata', done => { + const options = { + metadata: {a: 'b', c: 'd'}, + }; + bucket.setMetadata = (metadata: {}) => { + assert.deepStrictEqual(metadata, { + acl: null, + ...options.metadata, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + assert.strictEqual(typeof (options.metadata as any).acl, 'undefined'); + done(); + }; + bucket.makePrivate(options, assert.ifError); + }); + + it('should accept userProject', done => { + const options = { + userProject: 'user-project-id', + }; + bucket.setMetadata = (metadata: {}, options_: SetFileMetadataOptions) => { + assert.strictEqual(options_.userProject, options.userProject); + done(); + }; + bucket.makePrivate(options, done); + }); + + it('should not make files private by default', done => { + bucket.parent.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(); + }; + + bucket.makeAllFilesPublicPrivate_ = () => { + throw new Error('Please, no. I do not want to be called.'); + }; + + bucket.makePrivate(done); + }); + + it('should execute callback with error', done => { + const error = new Error('Error.'); + + bucket.parent.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error); + }; + + bucket.makePrivate((err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + }); + + describe('makePublic', () => { + beforeEach(() => { + bucket.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(); + }; + }); + + it('should set ACL, default ACL, and publicize files', done => { + let didSetAcl = false; + let didSetDefaultAcl = false; + let didMakeFilesPublic = false; + + bucket.acl.add = (opts: AddAclOptions) => { + assert.strictEqual(opts.entity, 'allUsers'); + assert.strictEqual(opts.role, 'READER'); + didSetAcl = true; + return Promise.resolve(); + }; + + bucket.acl.default.add = (opts: AddAclOptions) => { + assert.strictEqual(opts.entity, 'allUsers'); + assert.strictEqual(opts.role, 'READER'); + didSetDefaultAcl = true; + return Promise.resolve(); + }; + + bucket.makeAllFilesPublicPrivate_ = ( + opts: MakeAllFilesPublicPrivateOptions, + callback: Function + ) => { + assert.strictEqual(opts.public, true); + assert.strictEqual(opts.force, true); + didMakeFilesPublic = true; + callback(); + }; + + bucket.makePublic( + { + includeFiles: true, + force: true, + }, + (err: Error) => { + assert.ifError(err); + assert(didSetAcl); + assert(didSetDefaultAcl); + assert(didMakeFilesPublic); + done(); + } + ); + }); + + it('should not make files public by default', done => { + bucket.acl.add = () => Promise.resolve(); + bucket.acl.default.add = () => Promise.resolve(); + bucket.makeAllFilesPublicPrivate_ = () => { + throw new Error('Please, no. I do not want to be called.'); + }; + bucket.makePublic(done); + }); + + it('should execute callback with error', done => { + const error = new Error('Error.'); + bucket.acl.add = () => Promise.reject(error); + bucket.makePublic((err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + }); + + describe('notification', () => { + it('should throw an error if an id is not provided', () => { + assert.throws(() => { + bucket.notification(), BucketExceptionMessages.SUPPLY_NOTIFICATION_ID; + }); + }); + + it('should return a Notification object', () => { + const fakeId = '123'; + const notification = bucket.notification(fakeId); + + assert(notification instanceof FakeNotification); + assert.strictEqual(notification.bucket, bucket); + assert.strictEqual(notification.id, fakeId); + }); + }); + + describe('removeRetentionPeriod', () => { + it('should call setMetadata correctly', done => { + bucket.setMetadata = ( + metadata: {}, + _optionsOrCallback: {}, + callback: Function + ) => { + assert.deepStrictEqual(metadata, { + retentionPolicy: null, + }); + + Promise.resolve([]).then(resp => callback(null, ...resp)); + }; + + bucket.removeRetentionPeriod(done); + }); + }); + + describe('restore', () => { + it('should pass options to underlying request call', async () => { + bucket.request = function ( + reqOpts: DecorateRequestOptions, + callback_: Function + ) { + assert.strictEqual(this, bucket); + assert.deepStrictEqual(reqOpts, { + method: 'POST', + uri: '/restore', + qs: {generation: 123456789}, + }); + assert.strictEqual(callback_, undefined); + return []; + }; + + await bucket.restore({generation: 123456789}); + }); + }); + + describe('request', () => { + const USER_PROJECT = 'grape-spaceship-123'; + + beforeEach(() => { + bucket.userProject = USER_PROJECT; + }); + + it('should set the userProject if qs is undefined', done => { + FakeServiceObject.prototype.request = (( + reqOpts: DecorateRequestOptions + ) => { + assert.strictEqual(reqOpts.qs.userProject, USER_PROJECT); + done(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }) as any; + + bucket.request({}, assert.ifError); + }); + + it('should set the userProject if field is undefined', done => { + const options = { + qs: { + foo: 'bar', + }, + }; + + FakeServiceObject.prototype.request = (( + reqOpts: DecorateRequestOptions + ) => { + assert.strictEqual(reqOpts.qs.userProject, USER_PROJECT); + assert.strictEqual(reqOpts.qs, options.qs); + done(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }) as any; + + bucket.request(options, assert.ifError); + }); + + it('should not overwrite the userProject', done => { + const fakeUserProject = 'not-grape-spaceship-123'; + const options = { + qs: { + userProject: fakeUserProject, + }, + }; + + FakeServiceObject.prototype.request = (( + reqOpts: DecorateRequestOptions + ) => { + assert.strictEqual(reqOpts.qs.userProject, fakeUserProject); + done(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }) as any; + + bucket.request(options, assert.ifError); + }); + + it('should call ServiceObject#request correctly', done => { + const options = {}; + + Object.assign(FakeServiceObject.prototype, { + request(reqOpts: DecorateRequestOptions, callback: Function) { + assert.strictEqual(this, bucket); + assert.strictEqual(reqOpts, options); + callback(); // done fn + }, + }); + + bucket.request(options, done); + }); + }); + + describe('setLabels', () => { + it('should correctly call setMetadata', done => { + const labels = {}; + bucket.setMetadata = ( + metadata: BucketMetadata, + _callbackOrOptions: {}, + callback: Function + ) => { + assert.strictEqual(metadata.labels, labels); + Promise.resolve([]).then(resp => callback(null, ...resp)); + }; + bucket.setLabels(labels, done); + }); + + it('should accept an options object', done => { + const labels = {}; + const options = {}; + bucket.setMetadata = (metadata: {}, options_: {}) => { + assert.strictEqual(options_, options); + done(); + }; + bucket.setLabels(labels, options, done); + }); + }); + + describe('setRetentionPeriod', () => { + it('should call setMetadata correctly', done => { + const duration = 90000; + + bucket.setMetadata = ( + metadata: {}, + _callbackOrOptions: {}, + callback: Function + ) => { + assert.deepStrictEqual(metadata, { + retentionPolicy: { + retentionPeriod: `${duration}`, + }, + }); + + Promise.resolve([]).then(resp => callback(null, ...resp)); + }; + + bucket.setRetentionPeriod(duration, done); + }); + }); + + describe('setCorsConfiguration', () => { + it('should call setMetadata correctly', done => { + const corsConfiguration = [{maxAgeSeconds: 3600}]; + + bucket.setMetadata = ( + metadata: {}, + _callbackOrOptions: {}, + callback: Function + ) => { + assert.deepStrictEqual(metadata, { + cors: corsConfiguration, + }); + + return Promise.resolve([]).then(resp => callback(null, ...resp)); + }; + + bucket.setCorsConfiguration(corsConfiguration, done); + }); + }); + + describe('setStorageClass', () => { + const STORAGE_CLASS = 'NEW_STORAGE_CLASS'; + const OPTIONS = {}; + const CALLBACK = util.noop; + + it('should convert camelCase to snake_case', done => { + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.strictEqual(metadata.storageClass, 'CAMEL_CASE'); + done(); + }; + + bucket.setStorageClass('camelCase', OPTIONS, CALLBACK); + }); + + it('should convert hyphenate to snake_case', done => { + bucket.setMetadata = (metadata: BucketMetadata) => { + assert.strictEqual(metadata.storageClass, 'HYPHENATED_CLASS'); + done(); + }; + + bucket.setStorageClass('hyphenated-class', OPTIONS, CALLBACK); + }); + + it('should call setMetadata correctly', () => { + bucket.setMetadata = ( + metadata: BucketMetadata, + options: {}, + callback: Function + ) => { + assert.deepStrictEqual(metadata, {storageClass: STORAGE_CLASS}); + assert.strictEqual(options, OPTIONS); + Promise.resolve([]).then(resp => callback(null, ...resp)); + }; + + bucket.setStorageClass(STORAGE_CLASS, OPTIONS, CALLBACK); + }); + }); + + describe('setUserProject', () => { + const USER_PROJECT = 'grape-spaceship-123'; + + it('should set the userProject property', () => { + bucket.setUserProject(USER_PROJECT); + assert.strictEqual(bucket.userProject, USER_PROJECT); + }); + + it('should set the userProject on the global request options', () => { + const methods = [ + 'create', + 'delete', + 'exists', + 'get', + 'getMetadata', + 'setMetadata', + ]; + methods.forEach(method => { + assert.strictEqual( + bucket.methods[method].reqOpts.qs.userProject, + undefined + ); + }); + bucket.setUserProject(USER_PROJECT); + methods.forEach(method => { + assert.strictEqual( + bucket.methods[method].reqOpts.qs.userProject, + USER_PROJECT + ); + }); + }); + }); + + describe('upload', () => { + const basename = 'testfile.json'; + const filepath = path.join( + getDirName(), + '../../../test/testdata/' + basename + ); + const nonExistentFilePath = path.join( + getDirName(), + '../../../test/testdata/', + 'non-existent-file' + ); + const metadata = { + metadata: { + a: 'b', + c: 'd', + }, + }; + + beforeEach(() => { + bucket.file = (name: string, metadata: FileMetadata) => { + return new FakeFile(bucket, name, metadata); + }; + }); + + it('should return early in snippet sandbox', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (global as any)['GCLOUD_SANDBOX_ENV'] = true; + const returnValue = bucket.upload(filepath, assert.ifError); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + delete (global as any)['GCLOUD_SANDBOX_ENV']; + assert.strictEqual(returnValue, undefined); + }); + + it('should accept a path & cb', done => { + bucket.upload(filepath, (err: Error, file: File) => { + assert.ifError(err); + assert.strictEqual(file.bucket.name, bucket.name); + assert.strictEqual(file.name, basename); + done(); + }); + }); + + it('should accept a path, metadata, & cb', done => { + const options = { + metadata, + encryptionKey: 'key', + kmsKeyName: 'kms-key-name', + }; + bucket.upload(filepath, options, (err: Error, file: FakeFile) => { + assert.ifError(err); + assert.strictEqual(file.bucket.name, bucket.name); + assert.deepStrictEqual(file.metadata, metadata); + assert.strictEqual(file.options.encryptionKey, options.encryptionKey); + assert.strictEqual(file.options.kmsKeyName, options.kmsKeyName); + done(); + }); + }); + + it('should accept a path, a string dest, & cb', done => { + const newFileName = 'new-file-name.png'; + const options = { + destination: newFileName, + encryptionKey: 'key', + kmsKeyName: 'kms-key-name', + }; + bucket.upload(filepath, options, (err: Error, file: FakeFile) => { + assert.ifError(err); + assert.strictEqual(file.bucket.name, bucket.name); + assert.strictEqual(file.name, newFileName); + assert.strictEqual(file.options.encryptionKey, options.encryptionKey); + assert.strictEqual(file.options.kmsKeyName, options.kmsKeyName); + done(); + }); + }); + + it('should accept a path, a string dest, metadata, & cb', done => { + const newFileName = 'new-file-name.png'; + const options = { + destination: newFileName, + metadata, + encryptionKey: 'key', + kmsKeyName: 'kms-key-name', + }; + bucket.upload(filepath, options, (err: Error, file: FakeFile) => { + assert.ifError(err); + assert.strictEqual(file.bucket.name, bucket.name); + assert.strictEqual(file.name, newFileName); + assert.deepStrictEqual(file.metadata, metadata); + assert.strictEqual(file.options.encryptionKey, options.encryptionKey); + assert.strictEqual(file.options.kmsKeyName, options.kmsKeyName); + done(); + }); + }); + + it('should accept a path, a File dest, & cb', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + fakeFile.isSameFile = () => { + return true; + }; + const options = {destination: fakeFile}; + bucket.upload(filepath, options, (err: Error, file: FakeFile) => { + assert.ifError(err); + assert(file.isSameFile()); + done(); + }); + }); + + it('should accept a path, a File dest, metadata, & cb', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + fakeFile.isSameFile = () => { + return true; + }; + const options = {destination: fakeFile, metadata}; + bucket.upload(filepath, options, (err: Error, file: FakeFile) => { + assert.ifError(err); + assert(file.isSameFile()); + assert.deepStrictEqual(file.metadata, metadata); + done(); + }); + }); + + describe('resumable uploads', () => { + class DelayedStream500Error extends Transform { + retryCount: number; + constructor(retryCount: number) { + super(); + this.retryCount = retryCount; + } + _transform(chunk: string | Buffer, _encoding: string, done: Function) { + this.push(chunk); + setTimeout(() => { + if (this.retryCount === 1) { + done(new HTTPError('first error', 500)); + } else { + done(); + } + }, 5); + } + } + + beforeEach(() => { + fsStatOverride = (path: string, callback: Function) => { + callback(null, {size: 1}); // Small size to guarantee simple upload + }; + }); + + it('should respect setting a resumable upload to false', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = {destination: fakeFile, resumable: false}; + fakeFile.createWriteStream = (options_: CreateWriteStreamOptions) => { + const ws = new stream.Writable(); + ws.write = () => true; + setImmediate(() => { + assert.strictEqual(options_.resumable, options.resumable); + done(); + }); + return ws; + }; + bucket.upload(filepath, options, assert.ifError); + }); + + it('should not retry a nonretryable error code', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = {destination: fakeFile, resumable: true}; + let retryCount = 0; + fakeFile.createWriteStream = (options_: CreateWriteStreamOptions) => { + class DelayedStream403Error extends Transform { + _transform( + chunk: string | Buffer, + _encoding: string, + done: Function + ) { + this.push(chunk); + setTimeout(() => { + retryCount++; + if (retryCount === 1) { + done(new HTTPError('first error', 403)); + } else { + done(); + } + }, 5); + } + } + setImmediate(() => { + assert.strictEqual(options_.resumable, true); + retryCount++; + done(); + }); + return new DelayedStream403Error(); + }; + + bucket.upload(filepath, options, (err: Error) => { + assert.strictEqual(err.message, 'first error'); + assert.ok(retryCount === 2); + done(); + }); + }); + + it('resumable upload should retry', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = {destination: fakeFile, resumable: true}; + let retryCount = 0; + fakeFile.createWriteStream = (options_: CreateWriteStreamOptions) => { + setImmediate(() => { + assert.strictEqual(options_.resumable, true); + retryCount++; + done(); + }); + return new DelayedStream500Error(retryCount); + }; + bucket.upload(filepath, options, (err: Error) => { + assert.strictEqual(err.message, 'first error'); + assert.ok(retryCount === 1); + done(); + }); + }); + }); + + describe('multipart uploads', () => { + class DelayedStream500Error extends Transform { + retryCount: number; + constructor(retryCount: number) { + super(); + this.retryCount = retryCount; + } + _transform(chunk: string | Buffer, _encoding: string, done: Function) { + this.push(chunk); + setTimeout(() => { + if (this.retryCount === 1) { + done(new HTTPError('first error', 500)); + } else { + done(); + } + }, 5); + } + } + + beforeEach(() => { + fsStatOverride = (path: string, callback: Function) => { + callback(null, {size: 1}); // Small size to guarantee simple upload + }; + }); + + it('should save with no errors', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = {destination: fakeFile, resumable: false}; + fakeFile.createWriteStream = (options_: CreateWriteStreamOptions) => { + class DelayedStreamNoError extends Transform { + _transform( + chunk: string | Buffer, + _encoding: string, + done: Function + ) { + this.push(chunk); + setTimeout(() => { + done(); + }, 5); + } + } + assert.strictEqual(options_.resumable, false); + return new DelayedStreamNoError(); + }; + bucket.upload(filepath, options, (err: Error) => { + assert.ifError(err); + done(); + }); + }); + + it('should retry on first failure', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = {destination: fakeFile, resumable: false}; + let retryCount = 0; + fakeFile.createWriteStream = (options_: CreateWriteStreamOptions) => { + setImmediate(() => { + assert.strictEqual(options_.resumable, false); + retryCount++; + done(); + }); + return new DelayedStream500Error(retryCount); + }; + bucket.upload(filepath, options, (err: Error, file: FakeFile) => { + assert.ifError(err); + assert(file.isSameFile()); + assert.deepStrictEqual(file.metadata, metadata); + assert.ok(retryCount === 2); + done(); + }); + }); + + it('should not retry if nonretryable error code', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = {destination: fakeFile, resumable: false}; + let retryCount = 0; + fakeFile.createWriteStream = (options_: CreateWriteStreamOptions) => { + class DelayedStream403Error extends Transform { + _transform( + chunk: string | Buffer, + _encoding: string, + done: Function + ) { + this.push(chunk); + setTimeout(() => { + retryCount++; + if (retryCount === 1) { + done(new HTTPError('first error', 403)); + } else { + done(); + } + }, 5); + } + } + setImmediate(() => { + assert.strictEqual(options_.resumable, false); + retryCount++; + done(); + }); + return new DelayedStream403Error(); + }; + + bucket.upload(filepath, options, (err: Error) => { + assert.strictEqual(err.message, 'first error'); + assert.ok(retryCount === 2); + done(); + }); + }); + + it('non-multipart upload should not retry', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = {destination: fakeFile, resumable: true}; + let retryCount = 0; + fakeFile.createWriteStream = (options_: CreateWriteStreamOptions) => { + setImmediate(() => { + assert.strictEqual(options_.resumable, true); + retryCount++; + done(); + }); + return new DelayedStream500Error(retryCount); + }; + bucket.upload(filepath, options, (err: Error) => { + assert.strictEqual(err.message, 'first error'); + assert.ok(retryCount === 1); + done(); + }); + }); + }); + + it('should allow overriding content type', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const metadata = {contentType: 'made-up-content-type'}; + const options = {destination: fakeFile, metadata}; + fakeFile.createWriteStream = (options: CreateWriteStreamOptions) => { + const ws = new stream.Writable(); + ws.write = () => true; + setImmediate(() => { + assert.strictEqual( + options!.metadata!.contentType, + metadata.contentType + ); + done(); + }); + return ws; + }; + bucket.upload(filepath, options, assert.ifError); + }); + + it('should pass provided options to createWriteStream', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = { + destination: fakeFile, + a: 'b', + c: 'd', + }; + fakeFile.createWriteStream = (options_: {a: {}; c: {}}) => { + const ws = new stream.Writable(); + ws.write = () => true; + setImmediate(() => { + assert.strictEqual(options_.a, options.a); + assert.strictEqual(options_.c, options.c); + done(); + }); + return ws; + }; + bucket.upload(filepath, options, assert.ifError); + }); + + it('should execute callback on error', done => { + const error = new Error('Error.'); + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = {destination: fakeFile}; + fakeFile.createWriteStream = () => { + const ws = new stream.PassThrough(); + setImmediate(() => { + ws.destroy(error); + }); + return ws; + }; + bucket.upload(filepath, options, (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should return file and metadata', done => { + const fakeFile = new FakeFile(bucket, 'file-name'); + const options = {destination: fakeFile}; + const metadata = {}; + + fakeFile.createWriteStream = () => { + const ws = new stream.PassThrough(); + setImmediate(() => { + fakeFile.metadata = metadata; + ws.end(); + }); + return ws; + }; + + bucket.upload( + filepath, + options, + (err: Error, file: File, apiResponse: {}) => { + assert.ifError(err); + assert.strictEqual(file, fakeFile); + assert.strictEqual(apiResponse, metadata); + done(); + } + ); + }); + + it('should capture and throw on non-existent files', done => { + bucket.upload(nonExistentFilePath, (err: Error) => { + assert(err); + assert(err.message.includes('ENOENT')); + done(); + }); + }); + }); + + describe('makeAllFilesPublicPrivate_', () => { + it('should get all files from the bucket', done => { + const options = {}; + bucket.getFiles = (options_: {}) => { + assert.strictEqual(options_, options); + return Promise.resolve([[]]); + }; + bucket.makeAllFilesPublicPrivate_(options, done); + }); + + it('should process 10 files at a time', done => { + pLimitOverride = (limit: number) => { + assert.strictEqual(limit, 10); + setImmediate(done); + return () => {}; + }; + + bucket.getFiles = () => Promise.resolve([[]]); + bucket.makeAllFilesPublicPrivate_({}, assert.ifError); + }); + + it('should make files public', done => { + let timesCalled = 0; + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.makePublic = () => { + timesCalled++; + return Promise.resolve(); + }; + return file; + }); + bucket.getFiles = () => Promise.resolve([files]); + bucket.makeAllFilesPublicPrivate_({public: true}, (err: Error) => { + assert.ifError(err); + assert.strictEqual(timesCalled, files.length); + done(); + }); + }); + + it('should make files private', done => { + const options = { + private: true, + }; + let timesCalled = 0; + + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.makePrivate = () => { + timesCalled++; + return Promise.resolve(); + }; + return file; + }); + + bucket.getFiles = () => Promise.resolve([files]); + bucket.makeAllFilesPublicPrivate_(options, (err: Error) => { + assert.ifError(err); + assert.strictEqual(timesCalled, files.length); + done(); + }); + }); + + it('should execute callback with error from getting files', done => { + const error = new Error('Error.'); + bucket.getFiles = () => Promise.reject(error); + bucket.makeAllFilesPublicPrivate_({}, (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should execute callback with error from changing file', done => { + const error = new Error('Error.'); + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.makePublic = () => Promise.reject(error); + return file; + }); + bucket.getFiles = () => Promise.resolve([files]); + bucket.makeAllFilesPublicPrivate_({public: true}, (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should execute callback with queued errors', done => { + const error = new Error('Error.'); + const files = [bucket.file('1'), bucket.file('2')].map(file => { + file.makePublic = () => Promise.reject(error); + return file; + }); + bucket.getFiles = () => Promise.resolve([files]); + bucket.makeAllFilesPublicPrivate_( + { + public: true, + force: true, + }, + (errs: Error[]) => { + assert.deepStrictEqual(errs, [error, error]); + done(); + } + ); + }); + + it('should execute callback with files changed', done => { + const error = new Error('Error.'); + const successFiles = [bucket.file('1'), bucket.file('2')].map(file => { + file.makePublic = () => Promise.resolve(); + return file; + }); + const errorFiles = [bucket.file('3'), bucket.file('4')].map(file => { + file.makePublic = () => Promise.reject(error); + return file; + }); + + bucket.getFiles = () => { + const files = successFiles.concat(errorFiles); + return Promise.resolve([files]); + }; + + bucket.makeAllFilesPublicPrivate_( + { + public: true, + force: true, + }, + (errs: Error[], files: File[]) => { + assert.deepStrictEqual(errs, [error, error]); + assert.deepStrictEqual(files, successFiles); + done(); + } + ); + }); + }); + describe('disableAutoRetryConditionallyIdempotent_', () => { + beforeEach(() => { + bucket.storage.retryOptions.autoRetry = true; + STORAGE.retryOptions.idempotencyStrategy = + IdempotencyStrategy.RetryConditional; + }); + + it('should set autoRetry to false when ifMetagenerationMatch is undefined (setMetadata)', done => { + bucket.disableAutoRetryConditionallyIdempotent_( + bucket.methods.setMetadata, + AvailableServiceObjectMethods.setMetadata + ); + assert.strictEqual(bucket.storage.retryOptions.autoRetry, false); + done(); + }); + + it('should set autoRetry to false when ifMetagenerationMatch is undefined (delete)', done => { + bucket.disableAutoRetryConditionallyIdempotent_( + bucket.methods.delete, + AvailableServiceObjectMethods.delete + ); + assert.strictEqual(bucket.storage.retryOptions.autoRetry, false); + done(); + }); + + it('should set autoRetry to false when IdempotencyStrategy is set to RetryNever', done => { + STORAGE.retryOptions.idempotencyStrategy = IdempotencyStrategy.RetryNever; + bucket = new Bucket(STORAGE, BUCKET_NAME, { + preconditionOpts: { + ifMetagenerationMatch: 100, + }, + }); + bucket.disableAutoRetryConditionallyIdempotent_( + bucket.methods.delete, + AvailableServiceObjectMethods.delete + ); + assert.strictEqual(bucket.storage.retryOptions.autoRetry, false); + done(); + }); + + it('autoRetry should remain true when ifMetagenerationMatch is not undefined', done => { + bucket = new Bucket(STORAGE, BUCKET_NAME, { + preconditionOpts: { + ifMetagenerationMatch: 100, + }, + }); + bucket.disableAutoRetryConditionallyIdempotent_( + bucket.methods.delete, + AvailableServiceObjectMethods.delete + ); + assert.strictEqual(bucket.storage.retryOptions.autoRetry, true); + done(); + }); + }); +}); diff --git a/handwritten/storage/test/channel.ts b/handwritten/storage/test/channel.ts new file mode 100644 index 00000000000..e70272f2045 --- /dev/null +++ b/handwritten/storage/test/channel.ts @@ -0,0 +1,139 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/*! + * @module storage/channel + */ + +import { + BaseMetadata, + DecorateRequestOptions, + ServiceObject, + ServiceObjectConfig, +} from '../src/nodejs-common/index.js'; +import assert from 'assert'; +import {describe, it, before, beforeEach} from 'mocha'; +import proxyquire from 'proxyquire'; + +let promisified = false; +const fakePromisify = { + promisifyAll(Class: Function) { + if (Class.name === 'Channel') { + promisified = true; + } + }, +}; + +class FakeServiceObject extends ServiceObject { + calledWith_: IArguments; + constructor(config: ServiceObjectConfig) { + super(config); + // eslint-disable-next-line prefer-rest-params + this.calledWith_ = arguments; + } +} + +describe('Channel', () => { + const STORAGE = {}; + const ID = 'channel-id'; + const RESOURCE_ID = 'resource-id'; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let Channel: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let channel: any; + + before(() => { + Channel = proxyquire('../src/channel.js', { + '@google-cloud/promisify': fakePromisify, + './nodejs-common': { + ServiceObject: FakeServiceObject, + }, + }).Channel; + }); + + beforeEach(() => { + channel = new Channel(STORAGE, ID, RESOURCE_ID); + }); + + describe('initialization', () => { + it('should inherit from ServiceObject', () => { + // Using assert.strictEqual instead of assert to prevent + // coercing of types. + assert.strictEqual(channel instanceof ServiceObject, true); + + const calledWith = channel.calledWith_[0]; + + assert.strictEqual(calledWith.parent, STORAGE); + assert.strictEqual(calledWith.baseUrl, '/channels'); + assert.strictEqual(calledWith.id, ''); + assert.deepStrictEqual(calledWith.methods, {}); + }); + + it('should promisify all the things', () => { + assert(promisified); + }); + + it('should set the default metadata', () => { + assert.deepStrictEqual(channel.metadata, { + id: ID, + resourceId: RESOURCE_ID, + }); + }); + }); + + describe('stop', () => { + it('should make the correct request', done => { + channel.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.method, 'POST'); + assert.strictEqual(reqOpts.uri, '/stop'); + assert.strictEqual(reqOpts.json, channel.metadata); + + done(); + }; + + channel.stop(assert.ifError); + }); + + it('should execute callback with error & API response', done => { + const error = {}; + const apiResponse = {}; + + channel.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error, apiResponse); + }; + + channel.stop((err: Error, apiResponse_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(apiResponse_, apiResponse); + done(); + }); + }); + + it('should not require a callback', done => { + channel.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.doesNotThrow(() => callback()); + done(); + }; + + channel.stop(); + }); + }); +}); diff --git a/handwritten/storage/test/crc32c.ts b/handwritten/storage/test/crc32c.ts new file mode 100644 index 00000000000..4a14af96bbc --- /dev/null +++ b/handwritten/storage/test/crc32c.ts @@ -0,0 +1,529 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + CRC32C, + CRC32CValidator, + CRC32C_EXTENSIONS, + CRC32C_EXTENSION_TABLE, + CRC32C_EXCEPTION_MESSAGES, +} from '../src/index.js'; +import assert from 'assert'; +import {join} from 'path'; +import {tmpdir} from 'os'; +import * as fs from 'fs'; + +const KNOWN_INPUT_TO_CRC32C = { + /** empty string (i.e. nothing to 'update') */ + '': 'AAAAAA==', + /** known case #1 - validated from actual GCS object upload + metadata retrieval */ + data: 'rth90Q==', + /** known case #2 - validated from actual GCS object upload + metadata retrieval */ + 'some text\n': 'DkjKuA==', + /** arbitrary large string */ + ['a'.repeat(2 ** 16)]: 'TpXtPw==', +} as const; + +describe('CRC32C', () => { + describe('instance', () => { + describe('#constructor', () => { + it('should initial value to `0`', () => { + const crc32c = new CRC32C(); + + assert.equal(crc32c.valueOf(), 0); + }); + + it('should accept an `initialValue`', () => { + const initialValue = 123; + + const crc32c = new CRC32C(initialValue); + + assert.equal(crc32c.valueOf(), initialValue); + }); + }); + + describe('#update', () => { + it('should produce the correct calculation given the input (single buffer)', () => { + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + + const buffer = Buffer.from(input); + + crc32c.update(buffer); + + const result = crc32c.toString(); + + assert.equal( + result, + expected, + `Expected '${input}' to produce \`${expected}\` - not \`${result}\`` + ); + } + }); + + it('should produce the correct calculation given the input (multiple buffers)', () => { + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + + for (const char of input) { + const buffer = Buffer.from(char); + + crc32c.update(buffer); + } + + const result = crc32c.toString(); + + assert.equal( + result, + expected, + `Expected '${input}' to produce \`${expected}\` - not \`${result}\`` + ); + } + }); + + it('should not mutate a provided buffer', () => { + const crc32c = new CRC32C(); + + const value = 'abc'; + const buffer = Buffer.from(value); + + crc32c.update(buffer); + + assert.equal(buffer.toString(), value); + }); + }); + + describe('#validate', () => { + it('should validate a provided `number`', () => { + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + const expectedNumber = Buffer.from(expected, 'base64').readInt32BE(); + + const wrongNumber = expectedNumber + 1; + + crc32c.update(Buffer.from(input)); + + assert.equal(crc32c.validate(wrongNumber), false); + assert.equal(crc32c.validate(expectedNumber), true); + } + }); + + it('should validate a provided `string`', () => { + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + const expectedString = expected; + + // Want to test against a string generated in a valid way + const crc32cForIncorrectString = new CRC32C(); + const wrongStringInput = Buffer.from(input + ' '); + crc32cForIncorrectString.update(wrongStringInput); + const wrongString = crc32cForIncorrectString.toString(); + + crc32c.update(Buffer.from(input)); + + assert.equal(crc32c.validate(wrongString), false); + assert.equal(crc32c.validate(expectedString), true); + } + }); + + it('should validate a provided `Buffer`', () => { + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + const expectedBuffer = Buffer.from(expected, 'base64'); + + // Want to test against a `Buffer` generated in a valid way + const crc32cForIncorrectString = new CRC32C(); + const wrongBufferInput = Buffer.from(input + ' '); + crc32cForIncorrectString.update(wrongBufferInput); + const wrongBuffer = crc32cForIncorrectString.toBuffer(); + + crc32c.update(Buffer.from(input)); + + assert.equal(crc32c.validate(wrongBuffer), false); + assert.equal(crc32c.validate(expectedBuffer), true); + } + }); + + it('should validate a provided `CRC32C`', () => { + for (const [input] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + const crc32cExpected = new CRC32C(); + const wrongCRC32C = new CRC32C(); + + const wrongBufferInput = Buffer.from(input + ' '); + + crc32c.update(Buffer.from(input)); + crc32cExpected.update(Buffer.from(input)); + wrongCRC32C.update(wrongBufferInput); + + assert.equal(crc32c.validate(wrongCRC32C), false); + assert.equal(crc32c.validate(crc32cExpected), true); + } + }); + + it('should validate a provided generic `CRC32CValidator`', () => { + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + const crc32cExpectedValidator: CRC32CValidator = { + toString: () => expected, + update: () => {}, + validate: () => false, + }; + const wrongCRC32CValidator: CRC32CValidator = { + toString: () => { + const crc32c = new CRC32C(); + // Want to test against a `Buffer` generated in a valid way + const wrongBufferInput = Buffer.from(input + ' '); + + crc32c.update(wrongBufferInput); + + return crc32c.toString(); + }, + update: () => {}, + validate: () => false, + }; + + crc32c.update(Buffer.from(input)); + + assert.equal(crc32c.validate(wrongCRC32CValidator), false); + assert.equal(crc32c.validate(crc32cExpectedValidator), true); + } + }); + }); + + describe('#toBuffer', () => { + it('should return a valid 4-byte buffer', () => { + // At least one of our inputs should produce a negative 32-bit number - to prove we're not using unsigned integers + // This ensures the internally we're accurately handling unsigned integers + let atLeastOneWasSigned = false; + + for (const [input] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + crc32c.update(Buffer.from(input)); + + const value = crc32c.valueOf(); + + if (value < 0) { + // this is a negative number, thus is definitely signed + atLeastOneWasSigned = true; + } + + const buffer = Buffer.alloc(4); + buffer.writeInt32BE(value); + + assert.equal(crc32c.toBuffer().byteLength, 4); + assert.equal(Buffer.compare(crc32c.toBuffer(), buffer), 0); + } + + assert(atLeastOneWasSigned); + }); + }); + + describe('#toJSON', () => { + it('should return the expected JSON', () => { + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + crc32c.update(Buffer.from(input)); + + const stringified = JSON.stringify({crc32c}); + + assert.equal(crc32c.toJSON(), expected); + assert.deepStrictEqual(JSON.parse(stringified), {crc32c: expected}); + } + }); + }); + + describe('#toString', () => { + it('should return the expected string', () => { + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + crc32c.update(Buffer.from(input)); + + const stringified = `${crc32c}`; + + assert.equal(crc32c.toString(), expected); + assert.equal(stringified, expected); + } + }); + }); + + describe('#valueOf', () => { + it('should return the expected string', () => { + for (const [input, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = new CRC32C(); + crc32c.update(Buffer.from(input)); + + const expectedNumber = Buffer.from(expected, 'base64').readInt32BE(); + + assert.equal(crc32c.valueOf(), expectedNumber); + assert.equal(+crc32c, expectedNumber); + + // All `CRC32C` values should be safe integers + assert(Number.isSafeInteger(+crc32c)); + } + }); + }); + }); + + describe('static', () => { + describe('.CRC32C_EXTENSIONS', () => { + it('should be the same as the exported object', () => { + assert.equal(CRC32C.CRC32C_EXTENSIONS, CRC32C_EXTENSIONS); + }); + }); + + describe('.CRC32C_EXTENSION_TABLE', () => { + it('should be the same as the exported object', () => { + assert.equal(CRC32C.CRC32C_EXTENSION_TABLE, CRC32C_EXTENSION_TABLE); + }); + }); + + describe('.from', () => { + describe('`ArrayBuffer`', () => { + it('should generate from `ArrayBuffer`', () => { + for (const [, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const value = Buffer.from(expected, 'base64').readInt32BE(); + + const arrayBufferView = new Int32Array(1); + const dataView = new DataView(arrayBufferView.buffer); + dataView.setInt32(0, value, false); + + // Pass an `ArrayBuffer` + const crc32c = CRC32C.from(arrayBufferView.buffer); + + assert.equal(crc32c.valueOf(), dataView.getInt32(0, false)); + + // should not update source object + crc32c.update(Buffer.from(' ')); + + assert.notEqual(crc32c.valueOf(), dataView.getInt32(0, false)); + } + }); + + it('should raise a `RangeError` on invalid buffers', () => { + for (let i = 0; i < 8; i++) { + // `Int32Array` with length of 1 are valid + if (i === 1) continue; + + const arrayBufferView = new Int32Array(i); + + const errorMessage = + CRC32C_EXCEPTION_MESSAGES.INVALID_INIT_BUFFER_LENGTH(i * 4); + const expectedError = RangeError(errorMessage); + + assert.throws( + () => CRC32C.from(arrayBufferView.buffer), + expectedError + ); + } + }); + }); + + describe('`ArrayBufferView`', () => { + it('should generate from `ArrayBufferView`', () => { + for (const [, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const value = Buffer.from(expected, 'base64').readInt32BE(); + + const arrayBufferView = new Int32Array(1); + const dataView = new DataView(arrayBufferView.buffer); + dataView.setInt32(0, value, false); + + // Pass an `ArrayBufferView` + const crc32c = CRC32C.from(arrayBufferView); + + assert.equal(crc32c.valueOf(), dataView.getInt32(0, false)); + + // should not update source object + crc32c.update(Buffer.from(' ')); + + assert.notEqual(crc32c.valueOf(), dataView.getInt32(0, false)); + } + }); + + it('should raise a `RangeError` on invalid buffers', () => { + for (let i = 0; i < 8; i++) { + // `Int32Array` with length of 1 are valid + if (i === 1) continue; + + const arrayBufferView = new Int32Array(i); + + const errorMessage = + CRC32C_EXCEPTION_MESSAGES.INVALID_INIT_BUFFER_LENGTH(i * 4); + const expectedError = RangeError(errorMessage); + + assert.throws(() => CRC32C.from(arrayBufferView), expectedError); + } + }); + }); + + describe('`Buffer`', () => { + it('should generate from `Buffer`', () => { + for (const [, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const buffer = Buffer.from(expected, 'base64'); + + const crc32c = CRC32C.from(buffer); + + assert.equal(Buffer.compare(crc32c.toBuffer(), buffer), 0); + + // should not update source object + crc32c.update(Buffer.from(' ')); + + assert.notEqual(Buffer.compare(crc32c.toBuffer(), buffer), 0); + } + }); + + it('should raise a `RangeError` on invalid buffers', () => { + for (let i = 0; i < 8; i++) { + // Buffers with length of 4 are valid + if (i === 4) continue; + + const buffer = Buffer.alloc(i); + + const errorMessage = + CRC32C_EXCEPTION_MESSAGES.INVALID_INIT_BUFFER_LENGTH(i); + const expectedError = RangeError(errorMessage); + + assert.throws(() => CRC32C.from(buffer), expectedError); + } + }); + }); + + describe('`CRC32C`', () => { + it('should generate from `CRC32C`', () => { + for (const [, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const baseCRC32C = CRC32C.from(expected); + const crc32c = CRC32C.from(baseCRC32C); + + assert.equal(crc32c.valueOf(), baseCRC32C.valueOf()); + + // should not update source object + crc32c.update(Buffer.from(' ')); + + assert.notEqual(crc32c.valueOf(), baseCRC32C.valueOf()); + } + }); + }); + + describe('`CRC32CValidator`', () => { + it('should generate from `CRC32CValidator`', () => { + for (const [, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const baseCRC32C: CRC32CValidator = { + toString: () => expected, + update: () => {}, + validate: () => false, + }; + const crc32c = CRC32C.from(baseCRC32C); + + assert.equal(crc32c.toString(), baseCRC32C.toString()); + + // should not update source object + crc32c.update(Buffer.from(' ')); + + assert.notEqual(crc32c.toString(), baseCRC32C.toString()); + } + }); + }); + + describe('`string`', () => { + it('should generate from base64-encoded data', () => { + for (const [, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const crc32c = CRC32C.from(expected); + + assert.equal(crc32c.toString(), expected); + } + }); + + it('should raise a `RangeError` on invalid strings', () => { + for (let i = 0; i < 8; i++) { + // Buffers with length of 4 are valid + if (i === 4) continue; + + const string = Buffer.alloc(i).toString('base64'); + + const errorMessage = + CRC32C_EXCEPTION_MESSAGES.INVALID_INIT_BASE64_RANGE(i); + const expectedError = RangeError(errorMessage); + + assert.throws(() => CRC32C.from(string), expectedError); + } + }); + }); + + describe('`number`', () => { + it('should generate from `number`', () => { + // At least one of our inputs should produce a negative 32-bit number - to prove we're not using unsigned integers + // This ensures the internally we're accurately handling unsigned integers + let atLeastOneWasSigned = false; + + for (const [, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + const number = Buffer.from(expected, 'base64').readInt32BE(); + + const crc32c = CRC32C.from(number); + + if (number < 0) { + // this is a negative number, thus is definitely signed + atLeastOneWasSigned = true; + } + + assert.equal(crc32c.valueOf(), number); + } + + assert(atLeastOneWasSigned); + }); + + it('should raise a `RangeError` on invalid integers', () => { + const INVALID_SET = [ + NaN, // not a safe number + 0.5, // not an integer + 2 ** 32 + 1, // too high - out of valid range + -(2 ** 32) - 1, // too low - out of valid range + ]; + + for (const number of INVALID_SET) { + const errorMessage = + CRC32C_EXCEPTION_MESSAGES.INVALID_INIT_INTEGER(number); + const expectedError = RangeError(errorMessage); + + assert.throws(() => CRC32C.from(number), expectedError); + } + }); + }); + }); + + describe('.fromFile', () => { + let tempFilePath: string; + + beforeEach(async () => { + tempFilePath = join(tmpdir(), 'test.crc32c.fromFile'); + }); + + after(async () => { + try { + await fs.promises.unlink(tempFilePath); + } catch (e) { + // errors are fine + } + }); + + it('should generate a valid `crc32c` via a file path', async () => { + for (const [key, expected] of Object.entries(KNOWN_INPUT_TO_CRC32C)) { + await fs.promises.writeFile(tempFilePath, key); + + const crc32c = await CRC32C.fromFile(tempFilePath); + assert.equal(crc32c.toString(), expected); + } + }); + }); + }); +}); diff --git a/handwritten/storage/test/file.ts b/handwritten/storage/test/file.ts new file mode 100644 index 00000000000..14b2070aa10 --- /dev/null +++ b/handwritten/storage/test/file.ts @@ -0,0 +1,5661 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + ApiError, + BodyResponseCallback, + DecorateRequestOptions, + MetadataCallback, + ServiceObject, + ServiceObjectConfig, + util, +} from '../src/nodejs-common/index.js'; +import {describe, it, before, beforeEach, afterEach} from 'mocha'; +import {PromisifyAllOptions} from '@google-cloud/promisify'; +import { + Readable, + PassThrough, + Stream, + Duplex, + Transform, + pipeline, +} from 'stream'; +import assert from 'assert'; +import * as crypto from 'crypto'; +import duplexify from 'duplexify'; +import * as fs from 'fs'; +import * as path from 'path'; +import proxyquire from 'proxyquire'; +import * as resumableUpload from '../src/resumable-upload.js'; +import * as sinon from 'sinon'; +import * as tmp from 'tmp'; +import * as zlib from 'zlib'; + +import { + Bucket, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + File, + FileOptions, + PolicyDocument, + SetFileMetadataOptions, + GetSignedUrlConfig, + GenerateSignedPostPolicyV2Options, + CRC32C, +} from '../src/index.js'; +import { + SignedPostPolicyV4Output, + GenerateSignedPostPolicyV4Options, + STORAGE_POST_POLICY_BASE_URL, + MoveOptions, + FileExceptionMessages, + FileMetadata, +} from '../src/file.js'; +import {ExceptionMessages, IdempotencyStrategy} from '../src/storage.js'; +import {formatAsUTCISO} from '../src/util.js'; +import { + BaseMetadata, + SetMetadataOptions, +} from '../src/nodejs-common/service-object.js'; +import {GCCL_GCS_CMD_KEY} from '../src/nodejs-common/util.js'; + +class HTTPError extends Error { + code: number; + constructor(message: string, code: number) { + super(message); + this.code = code; + } +} + +let promisified = false; +let makeWritableStreamOverride: Function | null; +let handleRespOverride: Function | null; +const fakeUtil = Object.assign({}, util, { + handleResp(...args: Array<{}>) { + (handleRespOverride || util.handleResp)(...args); + }, + makeWritableStream(...args: Array<{}>) { + (makeWritableStreamOverride || util.makeWritableStream)(...args); + }, + makeRequest( + reqOpts: DecorateRequestOptions, + config: object, + callback: BodyResponseCallback + ) { + callback(null); + }, +}); + +const fakePromisify = { + // tslint:disable-next-line:variable-name + promisifyAll(Class: Function, options: PromisifyAllOptions) { + if (Class.name !== 'File') { + return; + } + + promisified = true; + assert.deepStrictEqual(options.exclude, [ + 'cloudStorageURI', + 'publicUrl', + 'request', + 'save', + 'setEncryptionKey', + 'shouldRetryBasedOnPreconditionAndIdempotencyStrat', + 'getBufferFromReadable', + 'restore', + ]); + }, +}; + +const fsCached = fs; +const fakeFs = {...fsCached}; + +const zlibCached = zlib; +let createGunzipOverride: Function | null; +const fakeZlib = { + ...zlib, + createGunzip(...args: Array<{}>) { + return (createGunzipOverride || zlibCached.createGunzip)(...args); + }, +}; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const osCached = require('os'); +const fakeOs = {...osCached}; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +let resumableUploadOverride: any; +function fakeResumableUpload() { + return () => { + return resumableUploadOverride || resumableUpload; + }; +} +Object.assign(fakeResumableUpload, { + createURI( + ...args: [resumableUpload.UploadConfig, resumableUpload.CreateUriCallback] + ) { + let createURI = resumableUpload.createURI; + + if (resumableUploadOverride && resumableUploadOverride.createURI) { + createURI = resumableUploadOverride.createURI; + } + + return createURI(...args); + }, +}); +Object.assign(fakeResumableUpload, { + upload(...args: [resumableUpload.UploadConfig]) { + let upload = resumableUpload.upload; + if (resumableUploadOverride && resumableUploadOverride.upload) { + upload = resumableUploadOverride.upload; + } + return upload(...args); + }, +}); + +class FakeServiceObject extends ServiceObject { + calledWith_: IArguments; + constructor(config: ServiceObjectConfig) { + super(config); + // eslint-disable-next-line prefer-rest-params + this.calledWith_ = arguments; + } +} + +const fakeSigner = { + URLSigner: () => {}, +}; + +describe('File', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let File: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let file: any; + + const FILE_NAME = 'file-name.png'; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let directoryFile: any; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let specialCharsFile: any; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let STORAGE: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let BUCKET: any; + + const DATA = 'test data'; + // crc32c hash of 'test data' + const CRC32C_HASH = 'M3m0yg=='; + // md5 hash of 'test data' + const MD5_HASH = '63M6AMDJ0zbmVpGjerVCkw=='; + // crc32c hash of `zlib.gzipSync(Buffer.from(DATA), {level: 9})` + const GZIPPED_DATA = Buffer.from( + 'H4sIAAAAAAACEytJLS5RSEksSQQAsq4I0wkAAAA=', + 'base64' + ); + //crc32c hash of `GZIPPED_DATA` + const CRC32C_HASH_GZIP = '64jygg=='; + + before(() => { + File = proxyquire('../src/file.js', { + './nodejs-common': { + ServiceObject: FakeServiceObject, + util: fakeUtil, + }, + '@google-cloud/promisify': fakePromisify, + fs: fakeFs, + '../src/resumable-upload': fakeResumableUpload, + os: fakeOs, + './signer': fakeSigner, + zlib: fakeZlib, + }).File; + }); + + beforeEach(() => { + Object.assign(fakeFs, fsCached); + Object.assign(fakeOs, osCached); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + FakeServiceObject.prototype.request = util.noop as any; + + STORAGE = { + createBucket: util.noop, + request: util.noop, + apiEndpoint: 'https://storage.googleapis.com', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + makeAuthenticatedRequest(req: {}, callback: any) { + if (callback) { + (callback.onAuthenticated || callback)(null, req); + } + }, + bucket(name: string) { + return new Bucket(this, name); + }, + retryOptions: { + autoRetry: true, + maxRetries: 3, + retryDelayMultiplier: 2, + totalTimeout: 600, + maxRetryDelay: 60, + retryableErrorFn: (err: HTTPError) => { + return err?.code === 500; + }, + idempotencyStrategy: IdempotencyStrategy.RetryConditional, + }, + customEndpoint: false, + }; + + BUCKET = new Bucket(STORAGE, 'bucket-name'); + BUCKET.getRequestInterceptors = () => []; + + file = new File(BUCKET, FILE_NAME); + + directoryFile = new File(BUCKET, 'directory/file.jpg'); + directoryFile.request = util.noop; + + specialCharsFile = new File(BUCKET, "special/azAZ!*'()*%/file.jpg"); + specialCharsFile.request = util.noop; + + createGunzipOverride = null; + handleRespOverride = null; + makeWritableStreamOverride = null; + resumableUploadOverride = null; + }); + + describe('initialization', () => { + it('should promisify all the things', () => { + assert(promisified); + }); + + it('should assign file name', () => { + assert.strictEqual(file.name, FILE_NAME); + }); + + it('should assign the bucket instance', () => { + assert.strictEqual(file.bucket, BUCKET); + }); + + it('should assign the storage instance', () => { + assert.strictEqual(file.storage, BUCKET.storage); + }); + + it('should set instanceRetryValue to the storage instance retryOptions.autoRetry value', () => { + assert.strictEqual( + file.instanceRetryValue, + STORAGE.retryOptions.autoRetry + ); + }); + + it('should not strip leading slashes', () => { + const file = new File(BUCKET, '/name'); + assert.strictEqual(file.name, '/name'); + }); + + it('should assign KMS key name', () => { + const kmsKeyName = 'kms-key-name'; + const file = new File(BUCKET, '/name', {kmsKeyName}); + assert.strictEqual(file.kmsKeyName, kmsKeyName); + }); + + it('should accept specifying a generation', () => { + const file = new File(BUCKET, 'name', {generation: 2}); + assert.strictEqual(file.generation, 2); + }); + + it('should inherit from ServiceObject', () => { + // Using assert.strictEqual instead of assert to prevent + // coercing of types. + assert.strictEqual(file instanceof ServiceObject, true); + + const calledWith = file.calledWith_[0]; + + assert.strictEqual(calledWith.parent, BUCKET); + assert.strictEqual(calledWith.baseUrl, '/o'); + assert.strictEqual(calledWith.id, encodeURIComponent(FILE_NAME)); + assert.deepStrictEqual(calledWith.methods, { + delete: {reqOpts: {qs: {}}}, + exists: {reqOpts: {qs: {}}}, + get: {reqOpts: {qs: {}}}, + getMetadata: {reqOpts: {qs: {}}}, + setMetadata: {reqOpts: {qs: {}}}, + }); + }); + + it('should set the correct query string with a generation', () => { + const options = {generation: 2}; + const file = new File(BUCKET, 'name', options); + + const calledWith = file.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + delete: {reqOpts: {qs: options}}, + exists: {reqOpts: {qs: options}}, + get: {reqOpts: {qs: options}}, + getMetadata: {reqOpts: {qs: options}}, + setMetadata: {reqOpts: {qs: options}}, + }); + }); + + it('should set the correct query string with a userProject', () => { + const options = {userProject: 'user-project'}; + const file = new File(BUCKET, 'name', options); + + const calledWith = file.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + delete: {reqOpts: {qs: options}}, + exists: {reqOpts: {qs: options}}, + get: {reqOpts: {qs: options}}, + getMetadata: {reqOpts: {qs: options}}, + setMetadata: {reqOpts: {qs: options}}, + }); + }); + + it('should set the correct query string with ifGenerationMatch', () => { + const options = {preconditionOpts: {ifGenerationMatch: 100}}; + const file = new File(BUCKET, 'name', options); + + const calledWith = file.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + delete: {reqOpts: {qs: options.preconditionOpts}}, + exists: {reqOpts: {qs: options.preconditionOpts}}, + get: {reqOpts: {qs: options.preconditionOpts}}, + getMetadata: {reqOpts: {qs: options.preconditionOpts}}, + setMetadata: {reqOpts: {qs: options.preconditionOpts}}, + }); + assert.deepStrictEqual( + file.instancePreconditionOpts, + options.preconditionOpts + ); + }); + + it('should set the correct query string with ifGenerationNotMatch', () => { + const options = {preconditionOpts: {ifGenerationNotMatch: 100}}; + const file = new File(BUCKET, 'name', options); + + const calledWith = file.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + delete: {reqOpts: {qs: options.preconditionOpts}}, + exists: {reqOpts: {qs: options.preconditionOpts}}, + get: {reqOpts: {qs: options.preconditionOpts}}, + getMetadata: {reqOpts: {qs: options.preconditionOpts}}, + setMetadata: {reqOpts: {qs: options.preconditionOpts}}, + }); + assert.deepStrictEqual( + file.instancePreconditionOpts, + options.preconditionOpts + ); + }); + + it('should set the correct query string with ifMetagenerationMatch', () => { + const options = {preconditionOpts: {ifMetagenerationMatch: 100}}; + const file = new File(BUCKET, 'name', options); + + const calledWith = file.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + delete: {reqOpts: {qs: options.preconditionOpts}}, + exists: {reqOpts: {qs: options.preconditionOpts}}, + get: {reqOpts: {qs: options.preconditionOpts}}, + getMetadata: {reqOpts: {qs: options.preconditionOpts}}, + setMetadata: {reqOpts: {qs: options.preconditionOpts}}, + }); + assert.deepStrictEqual( + file.instancePreconditionOpts, + options.preconditionOpts + ); + }); + + it('should set the correct query string with ifMetagenerationNotMatch', () => { + const options = {preconditionOpts: {ifMetagenerationNotMatch: 100}}; + const file = new File(BUCKET, 'name', options); + + const calledWith = file.calledWith_[0]; + + assert.deepStrictEqual(calledWith.methods, { + delete: {reqOpts: {qs: options.preconditionOpts}}, + exists: {reqOpts: {qs: options.preconditionOpts}}, + get: {reqOpts: {qs: options.preconditionOpts}}, + getMetadata: {reqOpts: {qs: options.preconditionOpts}}, + setMetadata: {reqOpts: {qs: options.preconditionOpts}}, + }); + assert.deepStrictEqual( + file.instancePreconditionOpts, + options.preconditionOpts + ); + }); + + it('should not strip leading slash name in ServiceObject', () => { + const file = new File(BUCKET, '/name'); + const calledWith = file.calledWith_[0]; + + assert.strictEqual(calledWith.id, encodeURIComponent('/name')); + }); + + it('should set a custom encryption key', done => { + const key = 'key'; + const setEncryptionKey = File.prototype.setEncryptionKey; + File.prototype.setEncryptionKey = (key_: {}) => { + File.prototype.setEncryptionKey = setEncryptionKey; + assert.strictEqual(key_, key); + done(); + }; + new File(BUCKET, FILE_NAME, {encryptionKey: key}); + }); + + it('should accept a `crc32cGenerator`', () => { + const crc32cGenerator = () => {}; + + const file = new File(BUCKET, 'name', {crc32cGenerator}); + assert.strictEqual(file.crc32cGenerator, crc32cGenerator); + }); + + it("should use the bucket's `crc32cGenerator` by default", () => { + assert.strictEqual(file.crc32cGenerator, BUCKET.crc32cGenerator); + }); + + describe('userProject', () => { + const USER_PROJECT = 'grapce-spaceship-123'; + + it('should localize the Bucket#userProject', () => { + const bucket = new Bucket(STORAGE, 'bucket-name', { + userProject: USER_PROJECT, + }); + + const file = new File(bucket, '/name'); + assert.strictEqual(file.userProject, USER_PROJECT); + }); + + it('should accept a userProject option', () => { + const file = new File(BUCKET, '/name', { + userProject: USER_PROJECT, + }); + + assert.strictEqual(file.userProject, USER_PROJECT); + }); + }); + }); + + describe('cloudStorageURI', () => { + it('should return the appropriate `gs://` URI', () => { + const file = new File(BUCKET, FILE_NAME); + + assert(file.cloudStorageURI instanceof URL); + assert.equal(file.cloudStorageURI.host, BUCKET.name); + assert.equal(file.cloudStorageURI.pathname, `/${FILE_NAME}`); + }); + }); + + describe('copy', () => { + it('should throw if no destination is provided', () => { + assert.throws(() => { + file.copy(); + }, /Destination file should have a name\./); + }); + + it('should URI encode file names', done => { + const newFile = new File(BUCKET, 'nested/file.jpg'); + + const expectedPath = `/rewriteTo/b/${ + file.bucket.name + }/o/${encodeURIComponent(newFile.name)}`; + + directoryFile.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, expectedPath); + done(); + }; + + directoryFile.copy(newFile); + }); + + it('should execute callback with error & API response', done => { + const error = new Error('Error.'); + const apiResponse = {}; + + const newFile = new File(BUCKET, 'new-file'); + + file.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(error, apiResponse); + }; + + file.copy(newFile, (err: Error, file: {}, apiResponse_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(file, null); + assert.strictEqual(apiResponse_, apiResponse); + + done(); + }); + }); + + it('should send query.sourceGeneration if File has one', done => { + const versionedFile = new File(BUCKET, 'name', {generation: 1}); + const newFile = new File(BUCKET, 'new-file'); + + versionedFile.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.sourceGeneration, 1); + done(); + }; + + versionedFile.copy(newFile, assert.ifError); + }); + + it('should accept an options object', done => { + const newFile = new File(BUCKET, 'name'); + const METADATA = { + metadataKey: 'metadataValue', + }; + const options = { + option: true, + metadata: METADATA, + }; + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.json, options); + assert.strictEqual(reqOpts.json.metadata, METADATA); + done(); + }; + + file.copy(newFile, options, assert.ifError); + }); + + it('should pass through userProject', done => { + const options = { + userProject: 'user-project', + }; + const originalOptions = Object.assign({}, options); + const newFile = new File(BUCKET, 'new-file'); + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.userProject, options.userProject); + assert.strictEqual(reqOpts.json.userProject, undefined); + assert.deepStrictEqual(options, originalOptions); + done(); + }; + + file.copy(newFile, options, assert.ifError); + }); + + it('should set correct headers when file is encrypted', done => { + file.encryptionKey = {}; + file.encryptionKeyBase64 = 'base64'; + file.encryptionKeyHash = 'hash'; + + const newFile = new File(BUCKET, 'new-file'); + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.headers, { + 'x-goog-copy-source-encryption-algorithm': 'AES256', + 'x-goog-copy-source-encryption-key': file.encryptionKeyBase64, + 'x-goog-copy-source-encryption-key-sha256': file.encryptionKeyHash, + }); + done(); + }; + + file.copy(newFile, assert.ifError); + }); + + it('should set encryption key on the new File instance', done => { + const newFile = new File(BUCKET, 'new-file'); + newFile.encryptionKey = 'encryptionKey'; + + file.setEncryptionKey = (encryptionKey: {}) => { + assert.strictEqual(encryptionKey, newFile.encryptionKey); + done(); + }; + + file.copy(newFile, assert.ifError); + }); + + it('should set destination KMS key name', done => { + const newFile = new File(BUCKET, 'new-file'); + newFile.kmsKeyName = 'kms-key-name'; + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.qs.destinationKmsKeyName, + newFile.kmsKeyName + ); + assert.strictEqual(file.kmsKeyName, newFile.kmsKeyName); + done(); + }; + + file.copy(newFile, assert.ifError); + }); + + it('should set destination KMS key name from option', done => { + const newFile = new File(BUCKET, 'new-file'); + const destinationKmsKeyName = 'destination-kms-key-name'; + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.qs.destinationKmsKeyName, + destinationKmsKeyName + ); + assert.strictEqual(file.kmsKeyName, destinationKmsKeyName); + done(); + }; + + file.copy(newFile, {destinationKmsKeyName}, assert.ifError); + }); + + it('should accept predefined Acl', done => { + const options = { + predefinedAcl: 'authenticatedRead', + }; + const newFile = new File(BUCKET, 'new-file'); + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.qs.destinationPredefinedAcl, + options.predefinedAcl + ); + assert.strictEqual(reqOpts.json.destinationPredefinedAcl, undefined); + done(); + }; + + file.copy(newFile, options, assert.ifError); + }); + + it('should favor the option over the File KMS name', done => { + const newFile = new File(BUCKET, 'new-file'); + newFile.kmsKeyName = 'incorrect-kms-key-name'; + const destinationKmsKeyName = 'correct-kms-key-name'; + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.qs.destinationKmsKeyName, + destinationKmsKeyName + ); + assert.strictEqual(file.kmsKeyName, destinationKmsKeyName); + done(); + }; + + file.copy(newFile, {destinationKmsKeyName}, assert.ifError); + }); + + it('should remove custom encryption interceptor if rotating to KMS', done => { + const newFile = new File(BUCKET, 'new-file'); + const destinationKmsKeyName = 'correct-kms-key-name'; + + file.encryptionKeyInterceptor = {}; + file.interceptors = [{}, file.encryptionKeyInterceptor, {}]; + + file.request = () => { + assert.strictEqual(file.interceptors.length, 2); + assert(file.interceptors.indexOf(file.encryptionKeyInterceptor) === -1); + done(); + }; + + file.copy(newFile, {destinationKmsKeyName}, assert.ifError); + }); + + describe('destination types', () => { + function assertPathEquals( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file: any, + expectedPath: string, + callback: Function + ) { + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, expectedPath); + callback(); + }; + } + + it('should allow a string', done => { + const newFileName = 'new-file-name.png'; + const newFile = new File(BUCKET, newFileName); + const expectedPath = `/rewriteTo/b/${file.bucket.name}/o/${newFile.name}`; + assertPathEquals(file, expectedPath, done); + file.copy(newFileName); + }); + + it('should allow a string with leading slash.', done => { + const newFileName = '/new-file-name.png'; + const newFile = new File(BUCKET, newFileName); + // File uri encodes file name when calling this.request during copy + const expectedPath = `/rewriteTo/b/${ + file.bucket.name + }/o/${encodeURIComponent(newFile.name)}`; + assertPathEquals(file, expectedPath, done); + file.copy(newFileName); + }); + + it('should allow a "gs://..." string', done => { + const newFileName = 'gs://other-bucket/new-file-name.png'; + const expectedPath = '/rewriteTo/b/other-bucket/o/new-file-name.png'; + assertPathEquals(file, expectedPath, done); + file.copy(newFileName); + }); + + it('should allow a Bucket', done => { + const expectedPath = `/rewriteTo/b/${BUCKET.name}/o/${file.name}`; + assertPathEquals(file, expectedPath, done); + file.copy(BUCKET); + }); + + it('should allow a File', done => { + const newFile = new File(BUCKET, 'new-file'); + const expectedPath = `/rewriteTo/b/${BUCKET.name}/o/${newFile.name}`; + assertPathEquals(file, expectedPath, done); + file.copy(newFile); + }); + + it('should throw if a destination cannot be parsed', () => { + assert.throws(() => { + file.copy(() => {}); + }, /Destination file should have a name\./); + }); + }); + + describe('not finished copying', () => { + const apiResponse = { + rewriteToken: '...', + }; + + beforeEach(() => { + file.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, apiResponse); + }; + }); + + it('should continue attempting to copy', done => { + const newFile = new File(BUCKET, 'new-file'); + + file.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + file.copy = (newFile_: {}, options: {}, callback: Function) => { + assert.strictEqual(newFile_, newFile); + assert.deepStrictEqual(options, {token: apiResponse.rewriteToken}); + callback(); // done() + }; + + callback(null, apiResponse); + }; + + file.copy(newFile, done); + }); + + it('should pass the userProject in subsequent requests', done => { + const newFile = new File(BUCKET, 'new-file'); + const fakeOptions = { + userProject: 'grapce-spaceship-123', + }; + + file.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.copy = (newFile_: {}, options: any) => { + assert.notStrictEqual(options, fakeOptions); + assert.strictEqual(options.userProject, fakeOptions.userProject); + done(); + }; + + callback(null, apiResponse); + }; + + file.copy(newFile, fakeOptions, assert.ifError); + }); + + it('should pass the KMS key name in subsequent requests', done => { + const newFile = new File(BUCKET, 'new-file'); + const fakeOptions = { + destinationKmsKeyName: 'kms-key-name', + }; + + file.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.copy = (newFile_: {}, options: any) => { + assert.strictEqual( + options.destinationKmsKeyName, + fakeOptions.destinationKmsKeyName + ); + done(); + }; + + callback(null, apiResponse); + }; + + file.copy(newFile, fakeOptions, assert.ifError); + }); + + it('should make the subsequent correct API request', done => { + const newFile = new File(BUCKET, 'new-file'); + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.rewriteToken, apiResponse.rewriteToken); + done(); + }; + + file.copy(newFile, {token: apiResponse.rewriteToken}, assert.ifError); + }); + }); + + describe('returned File object', () => { + beforeEach(() => { + const resp = {success: true}; + file.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, resp); + }; + }); + + it('should re-use file object if one is provided', done => { + const newFile = new File(BUCKET, 'new-file'); + file.copy(newFile, (err: Error, copiedFile: {}) => { + assert.ifError(err); + assert.deepStrictEqual(copiedFile, newFile); + done(); + }); + }); + + it('should create new file on the same bucket', done => { + const newFilename = 'new-filename'; + file.copy(newFilename, (err: Error, copiedFile: File) => { + assert.ifError(err); + assert.strictEqual(copiedFile.bucket.name, BUCKET.name); + assert.strictEqual(copiedFile.name, newFilename); + done(); + }); + }); + + it('should create new file on the destination bucket', done => { + file.copy(BUCKET, (err: Error, copiedFile: File) => { + assert.ifError(err); + assert.strictEqual(copiedFile.bucket.name, BUCKET.name); + assert.strictEqual(copiedFile.name, file.name); + done(); + }); + }); + + it('should pass apiResponse into callback', done => { + file.copy(BUCKET, (err: Error, copiedFile: File, apiResponse: {}) => { + assert.ifError(err); + assert.deepStrictEqual({success: true}, apiResponse); + done(); + }); + }); + }); + }); + + describe('createReadStream', () => { + function getFakeRequest(data?: {}) { + let requestOptions: DecorateRequestOptions | undefined; + + class FakeRequest extends Readable { + constructor(_requestOptions?: DecorateRequestOptions) { + super(); + requestOptions = _requestOptions; + this._read = () => { + if (data) { + this.push(data); + } + this.push(null); + }; + } + + static getRequestOptions() { + return requestOptions; + } + } + + // Return a Proxy of FakeRequest which can be instantiated + // without new. + return new Proxy(FakeRequest, { + apply(target, _, argumentsList) { + return new target(...argumentsList); + }, + }); + } + + function getFakeSuccessfulRequest(data: {}) { + // tslint:disable-next-line:variable-name + const FakeRequest = getFakeRequest(data); + + class FakeSuccessfulRequest extends FakeRequest { + constructor(req?: DecorateRequestOptions) { + super(req); + setImmediate(() => { + const stream = new FakeRequest(); + this.emit('response', stream); + }); + } + } + + // Return a Proxy of FakeSuccessfulRequest which can be instantiated + // without new. + return new Proxy(FakeSuccessfulRequest, { + apply(target, _, argumentsList) { + return new target(...argumentsList); + }, + }); + } + + function getFakeFailedRequest(error: Error) { + // tslint:disable-next-line:variable-name + const FakeRequest = getFakeRequest(); + + class FakeFailedRequest extends FakeRequest { + constructor(_req?: DecorateRequestOptions) { + super(_req); + setImmediate(() => { + this.emit('error', error); + }); + } + } + + // Return a Proxy of FakeFailedRequest which can be instantiated + // without new. + return new Proxy(FakeFailedRequest, { + apply(target, _, argumentsList) { + return new target(...argumentsList); + }, + }); + } + + beforeEach(() => { + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + const rawResponseStream = new PassThrough(); + Object.assign(rawResponseStream, { + toJSON() { + return {headers: {}}; + }, + }); + callback(null, null, rawResponseStream); + setImmediate(() => { + rawResponseStream.end(); + }); + }; + }); + + it('should throw if both a range and validation is given', () => { + assert.throws(() => { + file.createReadStream({ + validation: true, + start: 3, + end: 8, + }); + }, /Cannot use validation with file ranges \(start\/end\)\./); + + assert.throws(() => { + file.createReadStream({ + validation: true, + start: 3, + }); + }, /Cannot use validation with file ranges \(start\/end\)\./); + + assert.throws(() => { + file.createReadStream({ + validation: true, + end: 8, + }); + }, /Cannot use validation with file ranges \(start\/end\)\./); + + assert.doesNotThrow(() => { + file.createReadStream({ + start: 3, + end: 8, + }); + }); + }); + + it('should send query.generation if File has one', done => { + const versionedFile = new File(BUCKET, 'file.txt', {generation: 1}); + + versionedFile.requestStream = (rOpts: DecorateRequestOptions) => { + assert.strictEqual(rOpts.qs.generation, 1); + setImmediate(done); + return duplexify(); + }; + + versionedFile.createReadStream().resume(); + }); + + it('should send query.userProject if provided', done => { + const options = { + userProject: 'user-project-id', + }; + + file.requestStream = (rOpts: DecorateRequestOptions) => { + assert.strictEqual(rOpts.qs.userProject, options.userProject); + setImmediate(done); + return duplexify(); + }; + + file.createReadStream(options).resume(); + }); + + it('should pass the `GCCL_GCS_CMD_KEY` to `requestStream`', done => { + const expected = 'expected/value'; + + file.requestStream = (opts: DecorateRequestOptions) => { + assert.equal(opts[GCCL_GCS_CMD_KEY], expected); + + process.nextTick(() => done()); + + return duplexify(); + }; + + file + .createReadStream({ + [GCCL_GCS_CMD_KEY]: expected, + }) + .resume(); + }); + + describe('authenticating', () => { + it('should create an authenticated request', done => { + file.requestStream = (opts: DecorateRequestOptions) => { + assert.deepStrictEqual(opts, { + uri: '', + headers: { + 'Accept-Encoding': 'gzip', + 'Cache-Control': 'no-store', + }, + qs: { + alt: 'media', + }, + }); + setImmediate(() => { + done(); + }); + return duplexify(); + }; + + file.createReadStream().resume(); + }); + + describe('errors', () => { + const ERROR = new Error('Error.'); + + beforeEach(() => { + file.requestStream = () => { + const requestStream = new PassThrough(); + + setImmediate(() => { + requestStream.emit('error', ERROR); + }); + + return requestStream; + }; + }); + + it('should emit an error from authenticating', done => { + file + .createReadStream() + .once('error', (err: Error) => { + assert.strictEqual(err, ERROR); + done(); + }) + .resume(); + }); + }); + }); + + describe('requestStream', () => { + it('should get readable stream from request', done => { + file.requestStream = () => { + setImmediate(() => { + done(); + }); + + return new PassThrough(); + }; + + file.createReadStream().resume(); + }); + + it('should emit response event from request', done => { + file.requestStream = getFakeSuccessfulRequest('body'); + + file + .createReadStream({validation: false}) + .on('response', () => { + done(); + }) + .resume(); + }); + + it('should let util.handleResp handle the response', done => { + const response = {a: 'b', c: 'd'}; + + handleRespOverride = (err: Error, response_: {}, body: {}) => { + assert.strictEqual(err, null); + assert.strictEqual(response_, response); + assert.strictEqual(body, null); + done(); + }; + + file.requestStream = () => { + const rowRequestStream = new PassThrough(); + setImmediate(() => { + rowRequestStream.emit('response', response); + }); + return rowRequestStream; + }; + + file.createReadStream().resume(); + }); + + describe('errors', () => { + const ERROR = new Error('Error.'); + + beforeEach(() => { + file.requestStream = getFakeFailedRequest(ERROR); + }); + + it('should emit the error', done => { + file + .createReadStream() + .once('error', (err: Error) => { + assert.deepStrictEqual(err, ERROR); + done(); + }) + .resume(); + }); + + it('should parse a response stream for a better error', done => { + const rawResponsePayload = 'error message from body'; + const rawResponseStream = new PassThrough(); + const requestStream = new PassThrough(); + + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + callback(ERROR, null, res); + setImmediate(() => { + rawResponseStream.end(rawResponsePayload); + }); + }; + + file.requestStream = () => { + setImmediate(() => { + requestStream.emit('response', rawResponseStream); + }); + return requestStream; + }; + + file + .createReadStream() + .once('error', (err: Error) => { + assert.strictEqual(err, ERROR); + assert.strictEqual(err.message, rawResponsePayload); + done(); + }) + .resume(); + }); + + it('should emit errors from the request stream', done => { + const error = new Error('Error.'); + const rawResponseStream = new PassThrough(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (rawResponseStream as any).toJSON = () => { + return {headers: {}}; + }; + const requestStream = new PassThrough(); + + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + callback(null, null, rawResponseStream); + setImmediate(() => { + rawResponseStream.emit('error', error); + }); + }; + + file.requestStream = () => { + setImmediate(() => { + requestStream.emit('response', rawResponseStream); + }); + return requestStream; + }; + + file + .createReadStream() + .on('error', (err: Error) => { + assert.strictEqual(err, error); + done(); + }) + .resume(); + }); + + it('should not handle both error and end events', done => { + const error = new Error('Error.'); + const rawResponseStream = new PassThrough(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (rawResponseStream as any).toJSON = () => { + return {headers: {}}; + }; + const requestStream = new PassThrough(); + + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + callback(null, null, rawResponseStream); + setImmediate(() => { + rawResponseStream.emit('error', error); + }); + }; + + file.requestStream = () => { + setImmediate(() => { + requestStream.emit('response', rawResponseStream); + }); + return requestStream; + }; + + file + .createReadStream({validation: false}) + .on('error', (err: Error) => { + assert.strictEqual(err, error); + rawResponseStream.emit('end'); + setImmediate(done); + }) + .on('end', () => { + done(new Error('Should not have been called.')); + }) + .resume(); + }); + }); + }); + + describe('compression', () => { + beforeEach(() => { + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + const rawResponseStream = new PassThrough(); + Object.assign(rawResponseStream, { + toJSON() { + return { + headers: { + 'content-encoding': 'gzip', + 'x-goog-hash': `crc32c=${CRC32C_HASH_GZIP},md5=${MD5_HASH}`, + }, + }; + }, + }); + callback(null, null, rawResponseStream); + + rawResponseStream.end(GZIPPED_DATA); + }; + file.requestStream = getFakeSuccessfulRequest(GZIPPED_DATA); + }); + + it('should gunzip the response', async () => { + const collection: Buffer[] = []; + + for await (const data of file.createReadStream()) { + collection.push(data); + } + + assert.equal(Buffer.concat(collection).toString(), DATA); + }); + + it('should not gunzip the response if "decompress: false" is passed', async () => { + const collection: Buffer[] = []; + + for await (const data of file.createReadStream({decompress: false})) { + collection.push(data); + } + + assert.equal( + Buffer.compare(Buffer.concat(collection), GZIPPED_DATA), + 0 + ); + }); + + it('should emit errors from the gunzip stream', done => { + const error = new Error('Error.'); + const createGunzipStream = new PassThrough(); + createGunzipOverride = () => { + process.nextTick(() => { + createGunzipStream.emit('error', error); + }); + return createGunzipStream; + }; + file + .createReadStream() + .on('error', (err: Error) => { + assert.strictEqual(err, error); + done(); + }) + .resume(); + }); + + it('should not handle both error and end events', done => { + const error = new Error('Error.'); + const createGunzipStream = new PassThrough(); + createGunzipOverride = () => { + process.nextTick(() => { + createGunzipStream.emit('error', error); + }); + return createGunzipStream; + }; + file + .createReadStream({validation: false}) + .on('error', (err: Error) => { + assert.strictEqual(err, error); + createGunzipStream.emit('end'); + setImmediate(done); + }) + .on('end', () => { + done(new Error('Should not have been called.')); + }) + .resume(); + }); + }); + + describe('validation', () => { + let responseCRC32C = CRC32C_HASH; + let responseMD5 = MD5_HASH; + + beforeEach(() => { + responseCRC32C = CRC32C_HASH; + responseMD5 = MD5_HASH; + + file.getMetadata = async () => ({}); + + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + const rawResponseStream = new PassThrough(); + Object.assign(rawResponseStream, { + toJSON() { + return { + headers: { + 'x-goog-hash': `crc32c=${responseCRC32C},md5=${responseMD5}`, + 'x-goog-stored-content-encoding': 'identity', + }, + }; + }, + }); + callback(null, null, rawResponseStream); + setImmediate(() => { + rawResponseStream.end(DATA); + }); + }; + file.requestStream = getFakeSuccessfulRequest(DATA); + }); + + function setFileValidationToError(e: Error = new Error('test-error')) { + // Simulating broken CRC32C instance - used by the validation stream + file.crc32cGenerator = () => { + class C extends CRC32C { + update() { + throw e; + } + } + + return new C(); + }; + } + + describe('server decompression', () => { + it('should skip validation if file was stored compressed and served decompressed', done => { + file.metadata.crc32c = '.invalid.'; + file.metadata.contentEncoding = 'gzip'; + + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + const rawResponseStream = new PassThrough(); + Object.assign(rawResponseStream, { + toJSON() { + return { + headers: { + 'x-goog-hash': `crc32c=${responseCRC32C},md5=${responseMD5}`, + 'x-goog-stored-content-encoding': 'gzip', + }, + }; + }, + }); + callback(null, null, rawResponseStream); + setImmediate(() => { + rawResponseStream.end(DATA); + }); + }; + + file + .createReadStream({validation: 'crc32c'}) + .on('end', done) + .resume(); + }); + }); + + it('should perform validation if file was stored compressed and served compressed', done => { + file.metadata.crc32c = '.invalid.'; + file.metadata.contentEncoding = 'gzip'; + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + const rawResponseStream = new PassThrough(); + Object.assign(rawResponseStream, { + toJSON() { + return { + headers: { + 'x-goog-hash': `crc32c=${responseCRC32C},md5=${responseMD5}`, + 'x-goog-stored-content-encoding': 'gzip', + 'content-encoding': 'gzip', + }, + }; + }, + }); + callback(null, null, rawResponseStream); + setImmediate(() => { + rawResponseStream.end(DATA); + }); + }; + + const expectedError = new Error('test error'); + setFileValidationToError(expectedError); + + file + .createReadStream({validation: 'crc32c'}) + .on('error', (err: Error) => { + assert(err === expectedError); + done(); + }) + .resume(); + }); + + it('should emit errors from the validation stream', done => { + const expectedError = new Error('test error'); + + file.requestStream = getFakeSuccessfulRequest(DATA); + setFileValidationToError(expectedError); + + file + .createReadStream() + .on('error', (err: Error) => { + assert(err === expectedError); + + done(); + }) + .resume(); + }); + + it('should not handle both error and end events', done => { + const expectedError = new Error('test error'); + + file.requestStream = getFakeSuccessfulRequest(DATA); + setFileValidationToError(expectedError); + + file + .createReadStream() + .on('error', (err: Error) => { + assert(err === expectedError); + + setImmediate(done); + }) + .on('end', () => { + done(new Error('Should not have been called.')); + }) + .resume(); + }); + + it('should validate with crc32c', done => { + file.requestStream = getFakeSuccessfulRequest(DATA); + + file + .createReadStream({validation: 'crc32c'}) + .on('error', done) + .on('end', done) + .resume(); + }); + + it('should emit an error if crc32c validation fails', done => { + file.requestStream = getFakeSuccessfulRequest('bad-data'); + + responseCRC32C = 'bad-crc32c'; + + file + .createReadStream({validation: 'crc32c'}) + .on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'CONTENT_DOWNLOAD_MISMATCH'); + done(); + }) + .resume(); + }); + + it('should validate with md5', done => { + file.requestStream = getFakeSuccessfulRequest(DATA); + + file + .createReadStream({validation: 'md5'}) + .on('error', done) + .on('end', done) + .resume(); + }); + + it('should emit an error if md5 validation fails', done => { + file.requestStream = getFakeSuccessfulRequest('bad-data'); + + responseMD5 = 'bad-md5'; + + file + .createReadStream({validation: 'md5'}) + .on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'CONTENT_DOWNLOAD_MISMATCH'); + done(); + }) + .resume(); + }); + + it('should default to crc32c validation', done => { + file.requestStream = getFakeSuccessfulRequest('bad-data'); + + responseCRC32C = 'bad-crc32c'; + + file + .createReadStream() + .on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'CONTENT_DOWNLOAD_MISMATCH'); + done(); + }) + .resume(); + }); + + it('should ignore a data mismatch if validation: false', done => { + file.requestStream = getFakeSuccessfulRequest(DATA); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + // (fakeValidationStream as any).test = () => false; + file + .createReadStream({validation: false}) + .resume() + .on('error', done) + .on('end', done); + }); + + it('should handle x-goog-hash with only crc32c', done => { + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + const rawResponseStream = new PassThrough(); + Object.assign(rawResponseStream, { + toJSON() { + return { + headers: { + 'x-goog-hash': `crc32c=${CRC32C_HASH}`, + }, + }; + }, + }); + callback(null, null, rawResponseStream); + setImmediate(() => { + rawResponseStream.end(DATA); + }); + }; + + file.requestStream = getFakeSuccessfulRequest(DATA); + + file.createReadStream().on('error', done).on('end', done).resume(); + }); + + describe('destroying the through stream', () => { + it('should destroy after failed validation', done => { + file.requestStream = getFakeSuccessfulRequest('bad-data'); + + responseMD5 = 'bad-md5'; + + const readStream = file.createReadStream({validation: 'md5'}); + readStream.on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'CONTENT_DOWNLOAD_MISMATCH'); + done(); + }); + + readStream.resume(); + }); + + it('should destroy if MD5 is requested but absent', done => { + handleRespOverride = ( + err: Error, + res: {}, + body: {}, + callback: Function + ) => { + const rawResponseStream = new PassThrough(); + Object.assign(rawResponseStream, { + toJSON() { + return { + headers: {}, + }; + }, + }); + callback(null, null, rawResponseStream); + setImmediate(() => { + rawResponseStream.end(); + }); + }; + file.requestStream = getFakeSuccessfulRequest('bad-data'); + + const readStream = file.createReadStream({validation: 'md5'}); + + readStream.on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'MD5_NOT_AVAILABLE'); + done(); + }); + + readStream.resume(); + }); + }); + }); + + describe('range requests', () => { + it('should accept a start range', done => { + const startOffset = 100; + + file.requestStream = (opts: DecorateRequestOptions) => { + setImmediate(() => { + assert.strictEqual( + opts.headers!.Range, + 'bytes=' + startOffset + '-' + ); + done(); + }); + return duplexify(); + }; + + file.createReadStream({start: startOffset}).resume(); + }); + + it('should accept an end range and set start to 0', done => { + const endOffset = 100; + + file.requestStream = (opts: DecorateRequestOptions) => { + setImmediate(() => { + assert.strictEqual(opts.headers!.Range, 'bytes=0-' + endOffset); + done(); + }); + return duplexify(); + }; + + file.createReadStream({end: endOffset}).resume(); + }); + + it('should accept both a start and end range', done => { + const startOffset = 100; + const endOffset = 101; + + file.requestStream = (opts: DecorateRequestOptions) => { + setImmediate(() => { + const expectedRange = 'bytes=' + startOffset + '-' + endOffset; + assert.strictEqual(opts.headers!.Range, expectedRange); + done(); + }); + return duplexify(); + }; + + file.createReadStream({start: startOffset, end: endOffset}).resume(); + }); + + it('should accept range start and end as 0', done => { + const startOffset = 0; + const endOffset = 0; + + file.requestStream = (opts: DecorateRequestOptions) => { + setImmediate(() => { + const expectedRange = 'bytes=0-0'; + assert.strictEqual(opts.headers!.Range, expectedRange); + done(); + }); + return duplexify(); + }; + + file.createReadStream({start: startOffset, end: endOffset}).resume(); + }); + + it('should end the through stream', done => { + file.requestStream = getFakeSuccessfulRequest(DATA); + + const readStream = file.createReadStream({start: 100}); + readStream.on('end', done); + readStream.resume(); + }); + }); + + describe('tail requests', () => { + it('should make a request for the tail bytes', done => { + const endOffset = -10; + + file.requestStream = (opts: DecorateRequestOptions) => { + setImmediate(() => { + assert.strictEqual(opts.headers!.Range, 'bytes=' + endOffset); + done(); + }); + return duplexify(); + }; + + file.createReadStream({end: endOffset}).resume(); + }); + }); + }); + + describe('createResumableUpload', () => { + it('should not require options', done => { + resumableUploadOverride = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + createURI(opts: any, callback: Function) { + assert.strictEqual(opts.metadata, undefined); + callback(); + }, + }; + + file.createResumableUpload(done); + }); + + it('should disable autoRetry when ifMetagenerationMatch is undefined', done => { + resumableUploadOverride = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + createURI(opts: any, callback: Function) { + assert.strictEqual(opts.retryOptions.autoRetry, false); + callback(); + }, + }; + file.createResumableUpload(done); + assert.strictEqual(file.storage.retryOptions.autoRetry, true); + }); + + it('should create a resumable upload URI', done => { + const options = { + metadata: { + contentType: 'application/json', + }, + origin: '*', + predefinedAcl: 'predefined-acl', + private: 'private', + public: 'public', + userProject: 'user-project-id', + retryOptions: { + autoRetry: true, + maxRetries: 3, + maxRetryDelay: 60, + retryDelayMultiplier: 2, + totalTimeout: 600, + }, + preconditionOpts: { + ifGenerationMatch: 100, + ifMetagenerationMatch: 101, + }, + }; + + file.generation = 3; + file.encryptionKey = 'encryption-key'; + file.kmsKeyName = 'kms-key-name'; + + resumableUploadOverride = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + createURI(opts: any, callback: Function) { + const bucket = file.bucket; + const storage = bucket.storage; + + assert.strictEqual(opts.authClient, storage.authClient); + assert.strictEqual(opts.apiEndpoint, storage.apiEndpoint); + assert.strictEqual(opts.bucket, bucket.name); + assert.strictEqual(opts.file, file.name); + assert.strictEqual(opts.generation, file.generation); + assert.strictEqual(opts.key, file.encryptionKey); + assert.strictEqual(opts.kmsKeyName, file.kmsKeyName); + assert.strictEqual(opts.metadata, options.metadata); + assert.strictEqual(opts.origin, options.origin); + assert.strictEqual(opts.predefinedAcl, options.predefinedAcl); + assert.strictEqual(opts.private, options.private); + assert.strictEqual(opts.public, options.public); + assert.strictEqual(opts.userProject, options.userProject); + assert.strictEqual( + opts.retryOptions.autoRetry, + options.retryOptions.autoRetry + ); + assert.strictEqual( + opts.retryOptions.maxRetries, + options.retryOptions.maxRetries + ); + assert.strictEqual( + opts.retryOptions.maxRetryDelay, + options.retryOptions.maxRetryDelay + ); + assert.strictEqual( + opts.retryOptions.retryDelayMultiplier, + options.retryOptions.retryDelayMultiplier + ); + assert.strictEqual( + opts.retryOptions.totalTimeout, + options.retryOptions.totalTimeout + ); + assert.strictEqual(opts.params, options.preconditionOpts); + + callback(); + }, + }; + + file.createResumableUpload(options, done); + }); + + it('should create a resumable upload URI using precondition options from constructor', done => { + file = new File(BUCKET, FILE_NAME, { + preconditionOpts: { + ifGenerationMatch: 200, + ifGenerationNotMatch: 201, + ifMetagenerationMatch: 202, + ifMetagenerationNotMatch: 203, + }, + }); + const options = { + metadata: { + contentType: 'application/json', + }, + origin: '*', + predefinedAcl: 'predefined-acl', + private: 'private', + public: 'public', + userProject: 'user-project-id', + retryOptions: { + autoRetry: true, + maxRetries: 3, + maxRetryDelay: 60, + retryDelayMultiplier: 2, + totalTimeout: 600, + }, + }; + + file.generation = 3; + file.encryptionKey = 'encryption-key'; + file.kmsKeyName = 'kms-key-name'; + + resumableUploadOverride = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + createURI(opts: any, callback: Function) { + const bucket = file.bucket; + const storage = bucket.storage; + + assert.strictEqual(opts.authClient, storage.authClient); + assert.strictEqual(opts.apiEndpoint, storage.apiEndpoint); + assert.strictEqual(opts.bucket, bucket.name); + assert.strictEqual(opts.file, file.name); + assert.strictEqual(opts.generation, file.generation); + assert.strictEqual(opts.key, file.encryptionKey); + assert.strictEqual(opts.kmsKeyName, file.kmsKeyName); + assert.strictEqual(opts.metadata, options.metadata); + assert.strictEqual(opts.origin, options.origin); + assert.strictEqual(opts.predefinedAcl, options.predefinedAcl); + assert.strictEqual(opts.private, options.private); + assert.strictEqual(opts.public, options.public); + assert.strictEqual(opts.userProject, options.userProject); + assert.strictEqual( + opts.retryOptions.autoRetry, + options.retryOptions.autoRetry + ); + assert.strictEqual( + opts.retryOptions.maxRetries, + options.retryOptions.maxRetries + ); + assert.strictEqual( + opts.retryOptions.maxRetryDelay, + options.retryOptions.maxRetryDelay + ); + assert.strictEqual( + opts.retryOptions.retryDelayMultiplier, + options.retryOptions.retryDelayMultiplier + ); + assert.strictEqual( + opts.retryOptions.totalTimeout, + options.retryOptions.totalTimeout + ); + assert.strictEqual(opts.params, file.instancePreconditionOpts); + + callback(); + }, + }; + + file.createResumableUpload(options, done); + }); + }); + + describe('createWriteStream', () => { + const METADATA = {a: 'b', c: 'd'}; + + beforeEach(() => { + Object.assign(fakeFs, { + access(dir: string, check: {}, callback: Function) { + // Assume that the required config directory is writable. + callback(); + }, + }); + }); + + it('should return a stream', () => { + assert(file.createWriteStream() instanceof Stream); + }); + + it('should emit errors', done => { + const error = new Error('Error.'); + const uploadStream = new PassThrough(); + + file.startResumableUpload_ = (dup: duplexify.Duplexify) => { + dup.setWritable(uploadStream); + uploadStream.emit('error', error); + }; + + const writable = file.createWriteStream(); + + writable.on('error', (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + + writable.write('data'); + }); + + it('should emit RangeError', done => { + const error = new RangeError( + 'Cannot provide an `offset` without providing a `uri`' + ); + + const options = { + offset: 1, + isPartialUpload: true, + }; + const writable = file.createWriteStream(options); + + writable.on('error', (err: RangeError) => { + assert.deepEqual(err, error); + done(); + }); + + writable.write('data'); + }); + + it('should emit progress via resumable upload', done => { + const progress = {}; + + resumableUploadOverride = { + upload() { + const uploadStream = new PassThrough(); + setImmediate(() => { + uploadStream.emit('progress', progress); + }); + + return uploadStream; + }, + }; + + const writable = file.createWriteStream(); + + writable.on('progress', (evt: {}) => { + assert.strictEqual(evt, progress); + done(); + }); + + writable.write('data'); + }); + + it('should emit progress via simple upload', done => { + const progress = {}; + + makeWritableStreamOverride = (dup: duplexify.Duplexify) => { + const uploadStream = new PassThrough(); + uploadStream.on('progress', evt => dup.emit('progress', evt)); + + dup.setWritable(uploadStream); + setImmediate(() => { + uploadStream.emit('progress', progress); + }); + }; + + const writable = file.createWriteStream({resumable: false}); + + writable.on('progress', (evt: {}) => { + assert.strictEqual(evt, progress); + done(); + }); + + writable.write('data'); + }); + + it('should start a simple upload if specified', done => { + const options = { + metadata: METADATA, + resumable: false, + customValue: true, + }; + const writable = file.createWriteStream(options); + + file.startSimpleUpload_ = () => { + done(); + }; + + writable.write('data'); + }); + + it('should start a resumable upload if specified', done => { + const options = { + metadata: METADATA, + resumable: true, + customValue: true, + }; + const writable = file.createWriteStream(options); + + file.startResumableUpload_ = () => { + done(); + }; + + writable.write('data'); + }); + + it('should default to a resumable upload', done => { + const writable = file.createWriteStream({ + metadata: METADATA, + }); + + file.startResumableUpload_ = () => { + done(); + }; + + writable.write('data'); + }); + + it('should alias contentType to metadata object', done => { + const contentType = 'text/html'; + const writable = file.createWriteStream({contentType}); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(options.metadata.contentType, contentType); + done(); + }; + + writable.write('data'); + }); + + it('should detect contentType with contentType:auto', done => { + const writable = file.createWriteStream({contentType: 'auto'}); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(options.metadata.contentType, 'image/png'); + done(); + }; + + writable.write('data'); + }); + + it('should detect contentType if not defined', done => { + const writable = file.createWriteStream(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(options.metadata.contentType, 'image/png'); + done(); + }; + + writable.write('data'); + }); + + it('should not set a contentType if mime lookup failed', done => { + const file = new File('file-without-ext'); + const writable = file.createWriteStream(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(typeof options.metadata.contentType, 'undefined'); + done(); + }; + + writable.write('data'); + }); + + it('should set encoding with gzip:true', done => { + const writable = file.createWriteStream({gzip: true}); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(options.metadata.contentEncoding, 'gzip'); + done(); + }; + + writable.write('data'); + }); + + it('should set ifGenerationMatch with 100', done => { + const writable = file.createWriteStream({ + preconditionOpts: {ifGenerationMatch: 100}, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(options.preconditionOpts.ifGenerationMatch, 100); + done(); + }; + + writable.write('data'); + }); + + it('should set ifGenerationNotMatch with 100', done => { + const writable = file.createWriteStream({ + preconditionOpts: {ifGenerationNotMatch: 100}, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(options.preconditionOpts.ifGenerationNotMatch, 100); + done(); + }; + + writable.write('data'); + }); + + it('should set ifMetagenerationMatch with 100', done => { + const writable = file.createWriteStream({ + preconditionOpts: {ifMetagenerationMatch: 100}, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(options.preconditionOpts.ifMetagenerationMatch, 100); + done(); + }; + + writable.write('data'); + }); + + it('should set ifMetagenerationNotMatch with 100', done => { + const writable = file.createWriteStream({ + preconditionOpts: {ifMetagenerationNotMatch: 100}, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual( + options.preconditionOpts.ifMetagenerationNotMatch, + 100 + ); + done(); + }; + + writable.write('data'); + }); + + it('should set encoding with gzip:auto & compressible', done => { + const writable = file.createWriteStream({ + gzip: 'auto', + contentType: 'text/html', // (compressible) + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(options.metadata.contentEncoding, 'gzip'); + done(); + }; + + writable.write('data'); + }); + + it('should not set encoding with gzip:auto & non-compressible', done => { + const writable = file.createWriteStream({gzip: 'auto'}); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.startResumableUpload_ = (stream: {}, options: any) => { + assert.strictEqual(options.metadata.contentEncoding, undefined); + done(); + }; + writable.write('data'); + }); + + it('should re-emit response event', done => { + const writable = file.createWriteStream(); + const resp = {}; + + file.startResumableUpload_ = (stream: Duplex) => { + stream.emit('response', resp); + }; + + writable.on('response', (resp_: {}) => { + assert.strictEqual(resp_, resp); + done(); + }); + + writable.write('data'); + }); + + it('should emit `finish` only after underlying pipeline is complete', done => { + const writable = file.createWriteStream({ + resumable: false, + validation: false, + }); + + let streamFinishedCalled = false; + + writable.on('finish', () => { + try { + assert(streamFinishedCalled); + done(); + } catch (e) { + done(e); + } + }); + + file.startSimpleUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + streamFinishedCalled = true; + }); + }; + + writable.end('data'); + }); + + it('should close upstream when pipeline fails', done => { + const writable: Stream.Writable = file.createWriteStream(); + const error = new Error('My error'); + const uploadStream = new PassThrough(); + + let receivedBytes = 0; + const validateStream = new PassThrough(); + validateStream.on('data', (chunk: Buffer) => { + receivedBytes += chunk.length; + if (receivedBytes > 5) { + // this aborts the pipeline which should also close the internal pipeline within createWriteStream + pLine.destroy(error); + } + }); + + file.startResumableUpload_ = (dup: duplexify.Duplexify) => { + dup.setWritable(uploadStream); + // Emit an error so the pipeline's error-handling logic is triggered + uploadStream.emit('error', error); + // Explicitly destroy the stream so that the 'close' event is guaranteed to fire, + // even in Node v14 where autoDestroy defaults may prevent automatic closing + uploadStream.destroy(); + }; + + let closed = false; + uploadStream.on('close', () => { + closed = true; + }); + + const pLine = pipeline( + (function* () { + yield 'foo'; // write some data + yield 'foo'; // write some data + yield 'foo'; // write some data + })(), + validateStream, + writable, + (e: Error | null) => { + assert.strictEqual(e, error); + assert.strictEqual(closed, true); + done(); + } + ); + }); + + it('should error pipeline if source stream emits error before any data', done => { + const writable = file.createWriteStream(); + const error = new Error('Error before first chunk'); + pipeline( + // eslint-disable-next-line require-yield + (function* () { + throw error; + })(), + writable, + (e: Error | null) => { + assert.strictEqual(e, error); + done(); + } + ); + }); + + describe('validation', () => { + const data = 'test'; + + const fakeMetadata = { + crc32c: {crc32c: 'hqBywA=='}, + md5: {md5Hash: 'CY9rzUYh03PK3k6DJie09g=='}, + }; + + it('should validate with crc32c', done => { + const writable = file.createWriteStream({validation: 'crc32c'}); + + file.startResumableUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + file.metadata = fakeMetadata.crc32c; + }); + }; + + writable.end(data); + + writable.on('error', done).on('finish', done); + }); + + it('should emit an error if crc32c validation fails', done => { + const writable = file.createWriteStream({validation: 'crc32c'}); + + file.startResumableUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + file.metadata = fakeMetadata.crc32c; + }); + }; + + file.delete = async () => {}; + + writable.write('bad-data'); + writable.end(); + + writable.on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'FILE_NO_UPLOAD'); + done(); + }); + }); + + it('should validate with md5', done => { + const writable = file.createWriteStream({validation: 'md5'}); + + file.startResumableUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + file.metadata = fakeMetadata.md5; + }); + }; + + writable.write(data); + writable.end(); + + writable.on('error', done).on('finish', done); + }); + + it('should emit an error if md5 validation fails', done => { + const writable = file.createWriteStream({validation: 'md5'}); + + file.startResumableUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + file.metadata = fakeMetadata.md5; + }); + }; + + file.delete = async () => {}; + + writable.write('bad-data'); + writable.end(); + + writable.on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'FILE_NO_UPLOAD'); + done(); + }); + }); + + it('should default to md5 validation', done => { + const writable = file.createWriteStream(); + + file.startResumableUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + file.metadata = {md5Hash: 'bad-hash'}; + }); + }; + + file.delete = async () => {}; + + writable.write(data); + writable.end(); + + writable.on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'FILE_NO_UPLOAD'); + done(); + }); + }); + + it('should ignore a data mismatch if validation: false', done => { + const writable = file.createWriteStream({validation: false}); + + file.startResumableUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + file.metadata = {md5Hash: 'bad-hash'}; + }); + }; + + writable.write(data); + writable.end(); + + writable.on('error', done); + writable.on('finish', done); + }); + + it('should delete the file if validation fails', done => { + const writable = file.createWriteStream(); + + file.startResumableUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + file.metadata = {md5Hash: 'bad-hash'}; + }); + }; + + file.delete = async () => {}; + + writable.on('error', (e: ApiError) => { + assert.equal(e.code, 'FILE_NO_UPLOAD'); + done(); + }); + + writable.write(data); + writable.end(); + }); + + it('should emit an error if MD5 is requested but absent', done => { + const writable = file.createWriteStream({validation: 'md5'}); + + file.startResumableUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + file.metadata = {crc32c: 'not-md5'}; + }); + }; + + file.delete = async () => {}; + + writable.write(data); + writable.end(); + + writable.on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'MD5_NOT_AVAILABLE'); + done(); + }); + }); + + it('should emit a different error if delete fails', done => { + const writable = file.createWriteStream(); + + file.startResumableUpload_ = (stream: duplexify.Duplexify) => { + stream.setWritable(new PassThrough()); + stream.emit('metadata'); + + stream.on('finish', () => { + file.metadata = {md5Hash: 'bad-hash'}; + }); + }; + + const deleteErrorMessage = 'Delete error message.'; + const deleteError = new Error(deleteErrorMessage); + file.delete = async () => { + throw deleteError; + }; + + writable.write(data); + writable.end(); + + writable.on('error', (err: ApiError) => { + assert.strictEqual(err.code, 'FILE_NO_UPLOAD_DELETE'); + assert(err.message.indexOf(deleteErrorMessage) > -1); + done(); + }); + }); + }); + }); + + describe('download', () => { + let fileReadStream: Readable; + let originalSetEncryptionKey: Function; + + beforeEach(() => { + fileReadStream = new Readable(); + fileReadStream._read = util.noop; + + fileReadStream.on('end', () => { + fileReadStream.emit('complete'); + }); + + file.createReadStream = () => { + return fileReadStream; + }; + + originalSetEncryptionKey = file.setEncryptionKey; + file.setEncryptionKey = sinon.stub(); + }); + + afterEach(() => { + file.setEncryptionKey = originalSetEncryptionKey; + }); + + it('should accept just a callback', done => { + fileReadStream._read = () => { + done(); + }; + + file.download(assert.ifError); + }); + + it('should accept an options object and callback', done => { + fileReadStream._read = () => { + done(); + }; + + file.download({}, assert.ifError); + }); + + it('should not mutate options object after use', done => { + const optionsObject = {destination: './unknown.jpg'}; + fileReadStream._read = () => { + assert.strictEqual(optionsObject.destination, './unknown.jpg'); + assert.deepStrictEqual(optionsObject, {destination: './unknown.jpg'}); + done(); + }; + file.download(optionsObject, assert.ifError); + }); + + it('should pass the provided options to createReadStream', done => { + const readOptions = {start: 100, end: 200, destination: './unknown.jpg'}; + + file.createReadStream = (options: {}) => { + assert.deepStrictEqual(options, {start: 100, end: 200}); + assert.deepStrictEqual(readOptions, { + start: 100, + end: 200, + destination: './unknown.jpg', + }); + done(); + return fileReadStream; + }; + + file.download(readOptions, assert.ifError); + }); + + it('should call setEncryptionKey with the provided key and not pass it to createReadStream', done => { + const encryptionKey = Buffer.from('encryption-key'); + const downloadOptions = { + encryptionKey: encryptionKey, + userProject: 'user-project-id', + }; + + file.createReadStream = (options: {}) => { + assert.deepStrictEqual(options, {userProject: 'user-project-id'}); + return fileReadStream; + }; + + file.download(downloadOptions, (err: Error) => { + assert.ifError(err); + // Verify that setEncryptionKey was called with the correct key + assert.ok( + (file.setEncryptionKey as sinon.SinonStub).calledWith(encryptionKey) + ); + done(); + }); + + fileReadStream.push('some data'); + fileReadStream.push(null); + }); + + it('should only execute callback once', done => { + Object.assign(fileReadStream, { + _read(this: Readable) { + // Do not fire the errors immediately as this is a synchronous operation here + // and the iterator getter is also synchronous in file.getBufferFromReadable. + // this is only an issue for <= node 12. This cannot happen in practice. + process.nextTick(() => { + this.emit('error', new Error('Error.')); + this.emit('error', new Error('Error.')); + }); + }, + }); + + file.download(() => { + done(); + }); + }); + + describe('into memory', () => { + it('should buffer a file into memory if no destination', done => { + const fileContents = 'abcdefghijklmnopqrstuvwxyz'; + + Object.assign(fileReadStream, { + _read(this: Readable) { + this.push(fileContents); + this.push(null); + }, + }); + + file.download((err: Error, remoteFileContents: {}) => { + assert.ifError(err); + + assert.strictEqual(fileContents, remoteFileContents.toString()); + done(); + }); + }); + + it('should execute callback with error', done => { + const error = new Error('Error.'); + + Object.assign(fileReadStream, { + _read(this: Readable) { + // Do not fire the errors immediately as this is a synchronous operation here + // and the iterator getter is also synchronous in file.getBufferFromReadable. + // this is only an issue for <= node 12. This cannot happen in practice. + process.nextTick(() => { + this.emit('error', error); + }); + }, + }); + + file.download((err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + }); + + describe('with destination', () => { + const sandbox = sinon.createSandbox(); + + afterEach(() => { + sandbox.restore(); + }); + + it('should write the file to a destination if provided', done => { + tmp.setGracefulCleanup(); + tmp.file((err, tmpFilePath) => { + assert.ifError(err); + + const fileContents = 'abcdefghijklmnopqrstuvwxyz'; + + Object.assign(fileReadStream, { + _read(this: Readable) { + this.push(fileContents); + this.push(null); + }, + }); + + file.download({destination: tmpFilePath}, (err: Error) => { + assert.ifError(err); + + fs.readFile(tmpFilePath, (err, tmpFileContents) => { + assert.ifError(err); + + assert.strictEqual(fileContents, tmpFileContents.toString()); + done(); + }); + }); + }); + }); + + it('should process the entire stream', done => { + tmp.setGracefulCleanup(); + tmp.file(async (err, tmpFilePath) => { + assert.ifError(err); + + const fileContents = 'abcdefghijklmnopqrstuvwxyz'; + + fileReadStream.on('resume', () => { + fileReadStream.emit('data', fileContents); + fileReadStream.emit('data', fileContents); + setImmediate(() => { + fileReadStream.emit('end'); + }); + }); + + file.download({destination: tmpFilePath}, (err: Error) => { + assert.ifError(err); + fs.readFile(tmpFilePath, (err, tmpFileContents) => { + assert.ifError(err); + assert.strictEqual( + fileContents + fileContents, + tmpFileContents.toString() + ); + done(); + }); + }); + }); + }); + + it('empty file should be processed correctly', done => { + tmp.setGracefulCleanup(); + tmp.file(async (err, tmpFilePath) => { + assert.ifError(err); + + fileReadStream.on('resume', () => { + setImmediate(() => { + fileReadStream.emit('end'); + }); + }); + + file.download({destination: tmpFilePath}, (err: Error) => { + assert.ifError(err); + fs.readFile(tmpFilePath, (err, tmpFileContents) => { + assert.ifError(err); + assert.strictEqual('', tmpFileContents.toString()); + done(); + }); + }); + }); + }); + + it('file contents should remain unchanged if file nonexistent', done => { + tmp.setGracefulCleanup(); + tmp.file(async (err, tmpFilePath) => { + assert.ifError(err); + + const fileContents = 'file contents that should remain unchanged'; + fs.writeFileSync(tmpFilePath, fileContents, 'utf-8'); + + const error = new Error('Error.'); + fileReadStream.on('resume', () => { + setImmediate(() => { + fileReadStream.emit('error', error); + }); + }); + + file.download({destination: tmpFilePath}, (err: Error) => { + assert.strictEqual(err, error); + fs.readFile(tmpFilePath, (err, tmpFileContents) => { + assert.ifError(err); + assert.strictEqual(fileContents, tmpFileContents.toString()); + done(); + }); + }); + }); + }); + + it('should execute callback with error', done => { + tmp.setGracefulCleanup(); + tmp.file((err, tmpFilePath) => { + assert.ifError(err); + + const error = new Error('Error.'); + + Object.assign(fileReadStream, { + _read(this: Readable) { + this.emit('error', error); + }, + }); + + file.download({destination: tmpFilePath}, (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + }); + + it('should fail if provided destination directory does not exist', done => { + tmp.setGracefulCleanup(); + tmp.dir(async (err, tmpDirPath) => { + assert.ifError(err); + + const fileContents = 'nested-abcdefghijklmnopqrstuvwxyz'; + + Object.assign(fileReadStream, { + _read(this: Readable) { + this.push(fileContents); + this.push(null); + }, + }); + + const nestedPath = path.join(tmpDirPath, 'a', 'b', 'c', 'file.txt'); + + file.download({destination: nestedPath}, (err: Error) => { + assert.ok(err); + done(); + }); + }); + }); + }); + }); + + describe('getExpirationDate', () => { + it('should refresh metadata', done => { + file.getMetadata = () => { + done(); + }; + + file.getExpirationDate(assert.ifError); + }); + + it('should return error from getMetadata', done => { + const error = new Error('Error.'); + const apiResponse = {}; + + file.getMetadata = (callback: Function) => { + callback(error, null, apiResponse); + }; + + file.getExpirationDate( + (err: Error, expirationDate: {}, apiResponse_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(expirationDate, null); + assert.strictEqual(apiResponse_, apiResponse); + done(); + } + ); + }); + + it('should return an error if there is no expiration time', done => { + const apiResponse = {}; + + file.getMetadata = (callback: Function) => { + callback(null, {}, apiResponse); + }; + + file.getExpirationDate( + (err: Error, expirationDate: {}, apiResponse_: {}) => { + assert.strictEqual( + err.message, + FileExceptionMessages.EXPIRATION_TIME_NA + ); + assert.strictEqual(expirationDate, null); + assert.strictEqual(apiResponse_, apiResponse); + done(); + } + ); + }); + + it('should return the expiration time as a Date object', done => { + const expirationTime = new Date(); + + const apiResponse = { + retentionExpirationTime: expirationTime.toJSON(), + }; + + file.getMetadata = (callback: Function) => { + callback(null, apiResponse, apiResponse); + }; + + file.getExpirationDate( + (err: Error, expirationDate: {}, apiResponse_: {}) => { + assert.ifError(err); + assert.deepStrictEqual(expirationDate, expirationTime); + assert.strictEqual(apiResponse_, apiResponse); + done(); + } + ); + }); + }); + + describe('generateSignedPostPolicyV2', () => { + let CONFIG: GenerateSignedPostPolicyV2Options; + + beforeEach(() => { + CONFIG = { + expires: Date.now() + 2000, + }; + + BUCKET.storage.authClient = { + sign: () => { + return Promise.resolve('signature'); + }, + }; + }); + + it('should create a signed policy', done => { + BUCKET.storage.authClient.sign = (blobToSign: string) => { + const policy = Buffer.from(blobToSign, 'base64').toString(); + assert.strictEqual(typeof JSON.parse(policy), 'object'); + return Promise.resolve('signature'); + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.generateSignedPostPolicyV2( + CONFIG, + (err: Error, signedPolicy: PolicyDocument) => { + assert.ifError(err); + assert.strictEqual(typeof signedPolicy.string, 'string'); + assert.strictEqual(typeof signedPolicy.base64, 'string'); + assert.strictEqual(typeof signedPolicy.signature, 'string'); + done(); + } + ); + }); + + it('should not modify the configuration object', done => { + const originalConfig = Object.assign({}, CONFIG); + + file.generateSignedPostPolicyV2(CONFIG, (err: Error) => { + assert.ifError(err); + assert.deepStrictEqual(CONFIG, originalConfig); + done(); + }); + }); + + it('should return an error if signBlob errors', done => { + const error = new Error('Error.'); + + BUCKET.storage.authClient.sign = () => { + return Promise.reject(error); + }; + + file.generateSignedPostPolicyV2(CONFIG, (err: Error) => { + assert.strictEqual(err.name, 'SigningError'); + assert.strictEqual(err.message, error.message); + done(); + }); + }); + + it('should add key equality condition', done => { + file.generateSignedPostPolicyV2( + CONFIG, + (err: Error, signedPolicy: PolicyDocument) => { + const conditionString = '["eq","$key","' + file.name + '"]'; + assert.ifError(err); + assert(signedPolicy.string.indexOf(conditionString) > -1); + done(); + } + ); + }); + + it('should add ACL condition', done => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + acl: '', + }, + (err: Error, signedPolicy: PolicyDocument) => { + const conditionString = '{"acl":""}'; + assert.ifError(err); + assert(signedPolicy.string.indexOf(conditionString) > -1); + done(); + } + ); + }); + + it('should add success redirect', done => { + const redirectUrl = 'http://redirect'; + + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + successRedirect: redirectUrl, + }, + (err: Error, signedPolicy: PolicyDocument) => { + assert.ifError(err); + + const policy = JSON.parse(signedPolicy.string); + + assert( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + policy.conditions.some((condition: any) => { + return condition.success_action_redirect === redirectUrl; + }) + ); + + done(); + } + ); + }); + + it('should add success status', done => { + const successStatus = '200'; + + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + successStatus, + }, + (err: Error, signedPolicy: PolicyDocument) => { + assert.ifError(err); + + const policy = JSON.parse(signedPolicy.string); + + assert( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + policy.conditions.some((condition: any) => { + return condition.success_action_status === successStatus; + }) + ); + + done(); + } + ); + }); + + describe('expires', () => { + it('should accept Date objects', done => { + const expires = new Date(Date.now() + 1000 * 60); + + file.generateSignedPostPolicyV2( + { + expires, + }, + (err: Error, policy: PolicyDocument) => { + assert.ifError(err); + const expires_ = JSON.parse(policy.string).expiration; + assert.strictEqual(expires_, expires.toISOString()); + done(); + } + ); + }); + + it('should accept numbers', done => { + const expires = Date.now() + 1000 * 60; + + file.generateSignedPostPolicyV2( + { + expires, + }, + (err: Error, policy: PolicyDocument) => { + assert.ifError(err); + const expires_ = JSON.parse(policy.string).expiration; + assert.strictEqual(expires_, new Date(expires).toISOString()); + done(); + } + ); + }); + + it('should accept strings', done => { + const expires = '12-12-2099'; + + file.generateSignedPostPolicyV2( + { + expires, + }, + (err: Error, policy: PolicyDocument) => { + assert.ifError(err); + const expires_ = JSON.parse(policy.string).expiration; + assert.strictEqual(expires_, new Date(expires).toISOString()); + done(); + } + ); + }); + + it('should throw if a date is invalid', () => { + const expires = new Date('31-12-2019'); + + assert.throws(() => { + file.generateSignedPostPolicyV2( + { + expires, + }, + () => {} + ), + ExceptionMessages.EXPIRATION_DATE_INVALID; + }); + }); + + it('should throw if a date from the past is given', () => { + const expires = Date.now() - 5; + + assert.throws(() => { + file.generateSignedPostPolicyV2( + { + expires, + }, + () => {} + ), + ExceptionMessages.EXPIRATION_DATE_PAST; + }); + }); + }); + + describe('equality condition', () => { + it('should add equality conditions (array of arrays)', done => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + equals: [['$', '']], + }, + (err: Error, signedPolicy: PolicyDocument) => { + const conditionString = '["eq","$",""]'; + assert.ifError(err); + assert(signedPolicy.string.indexOf(conditionString) > -1); + done(); + } + ); + }); + + it('should add equality condition (array)', done => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + equals: ['$', ''], + }, + (err: Error, signedPolicy: PolicyDocument) => { + const conditionString = '["eq","$",""]'; + assert.ifError(err); + assert(signedPolicy.string.indexOf(conditionString) > -1); + done(); + } + ); + }); + + it('should throw if equal condition is not an array', () => { + assert.throws(() => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + equals: [{}], + }, + () => {} + ), + FileExceptionMessages.EQUALS_CONDITION_TWO_ELEMENTS; + }); + }); + + it('should throw if equal condition length is not 2', () => { + assert.throws(() => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + equals: [['1', '2', '3']], + }, + () => {} + ), + FileExceptionMessages.EQUALS_CONDITION_TWO_ELEMENTS; + }); + }); + }); + + describe('prefix conditions', () => { + it('should add prefix conditions (array of arrays)', done => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + startsWith: [['$', '']], + }, + (err: Error, signedPolicy: PolicyDocument) => { + const conditionString = '["starts-with","$",""]'; + assert.ifError(err); + assert(signedPolicy.string.indexOf(conditionString) > -1); + done(); + } + ); + }); + + it('should add prefix condition (array)', done => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + startsWith: ['$', ''], + }, + (err: Error, signedPolicy: PolicyDocument) => { + const conditionString = '["starts-with","$",""]'; + assert.ifError(err); + assert(signedPolicy.string.indexOf(conditionString) > -1); + done(); + } + ); + }); + + it('should throw if prefix condition is not an array', () => { + assert.throws(() => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + startsWith: [{}], + }, + () => {} + ), + FileExceptionMessages.STARTS_WITH_TWO_ELEMENTS; + }); + }); + + it('should throw if prefix condition length is not 2', () => { + assert.throws(() => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + startsWith: [['1', '2', '3']], + }, + () => {} + ), + FileExceptionMessages.STARTS_WITH_TWO_ELEMENTS; + }); + }); + }); + + describe('content length', () => { + it('should add content length condition', done => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + contentLengthRange: {min: 0, max: 1}, + }, + (err: Error, signedPolicy: PolicyDocument) => { + const conditionString = '["content-length-range",0,1]'; + assert.ifError(err); + assert(signedPolicy.string.indexOf(conditionString) > -1); + done(); + } + ); + }); + + it('should throw if content length has no min', () => { + assert.throws(() => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + contentLengthRange: [{max: 1}], + }, + () => {} + ), + FileExceptionMessages.CONTENT_LENGTH_RANGE_MIN_MAX; + }); + }); + + it('should throw if content length has no max', () => { + assert.throws(() => { + file.generateSignedPostPolicyV2( + { + expires: Date.now() + 2000, + contentLengthRange: [{min: 0}], + }, + () => {} + ), + FileExceptionMessages.CONTENT_LENGTH_RANGE_MIN_MAX; + }); + }); + }); + }); + + describe('generateSignedPostPolicyV4', () => { + let CONFIG: GenerateSignedPostPolicyV4Options; + + const NOW = new Date('2020-01-01'); + const CLIENT_EMAIL = 'test@domain.com'; + const SIGNATURE = 'signature'; + + let fakeTimer: sinon.SinonFakeTimers; + let sandbox: sinon.SinonSandbox; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + fakeTimer = sinon.useFakeTimers(NOW); + CONFIG = { + expires: NOW.valueOf() + 2000, + }; + + BUCKET.storage.authClient = { + sign: sandbox.stub().resolves(SIGNATURE), + getCredentials: sandbox.stub().resolves({client_email: CLIENT_EMAIL}), + }; + }); + + afterEach(() => { + sandbox.restore(); + fakeTimer.restore(); + }); + + const fieldsToConditions = (fields: object) => + Object.entries(fields).map(([k, v]) => ({[k]: v})); + + it('should create a signed policy', done => { + CONFIG.fields = { + 'x-goog-meta-foo': 'bar', + }; + + const requiredFields = { + key: file.name, + 'x-goog-date': '20200101T000000Z', + 'x-goog-credential': `${CLIENT_EMAIL}/20200101/auto/storage/goog4_request`, + 'x-goog-algorithm': 'GOOG4-RSA-SHA256', + }; + + const policy = { + conditions: [ + ...fieldsToConditions(CONFIG.fields), + {bucket: BUCKET.name}, + ...fieldsToConditions(requiredFields), + ], + expiration: formatAsUTCISO(new Date(CONFIG.expires), true, '-', ':'), + }; + + const policyString = JSON.stringify(policy); + const EXPECTED_POLICY = Buffer.from(policyString).toString('base64'); + const EXPECTED_SIGNATURE = Buffer.from(SIGNATURE, 'base64').toString( + 'hex' + ); + const EXPECTED_FIELDS = { + ...CONFIG.fields, + ...requiredFields, + 'x-goog-signature': EXPECTED_SIGNATURE, + policy: EXPECTED_POLICY, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + assert(res.url, `${STORAGE_POST_POLICY_BASE_URL}/${BUCKET.name}`); + + assert.deepStrictEqual(res.fields, EXPECTED_FIELDS); + + const signStub = BUCKET.storage.authClient.sign; + assert.deepStrictEqual( + Buffer.from(signStub.getCall(0).args[0], 'base64').toString(), + policyString + ); + + done(); + } + ); + }); + + it('should not modify the configuration object', done => { + const originalConfig = Object.assign({}, CONFIG); + + file.generateSignedPostPolicyV4(CONFIG, (err: Error) => { + assert.ifError(err); + assert.deepStrictEqual(CONFIG, originalConfig); + done(); + }); + }); + + it('should return an error if signBlob errors', done => { + const error = new Error('Error.'); + + BUCKET.storage.authClient.sign.rejects(error); + + file.generateSignedPostPolicyV4(CONFIG, (err: Error) => { + assert.strictEqual(err.name, 'SigningError'); + assert.strictEqual(err.message, error.message); + done(); + }); + }); + + it('should add key condition', done => { + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + + assert.strictEqual(res.fields['key'], file.name); + const EXPECTED_POLICY_ELEMENT = `{"key":"${file.name}"}`; + assert( + Buffer.from(res.fields.policy, 'base64') + .toString('utf-8') + .includes(EXPECTED_POLICY_ELEMENT) + ); + done(); + } + ); + }); + + it('should include fields in conditions', done => { + CONFIG = { + fields: { + 'x-goog-meta-foo': 'bar', + }, + ...CONFIG, + }; + + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + + const expectedConditionString = JSON.stringify(CONFIG.fields); + assert.strictEqual(res.fields['x-goog-meta-foo'], 'bar'); + const decodedPolicy = Buffer.from( + res.fields.policy, + 'base64' + ).toString('utf-8'); + assert(decodedPolicy.includes(expectedConditionString)); + done(); + } + ); + }); + + it('should encode special characters in policy', done => { + CONFIG = { + fields: { + 'x-goog-meta-foo': 'bår', + }, + ...CONFIG, + }; + + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + + assert.strictEqual(res.fields['x-goog-meta-foo'], 'bår'); + const decodedPolicy = Buffer.from( + res.fields.policy, + 'base64' + ).toString('utf-8'); + assert(decodedPolicy.includes('"x-goog-meta-foo":"b\\u00e5r"')); + done(); + } + ); + }); + + it('should not include fields with x-ignore- prefix in conditions', done => { + CONFIG = { + fields: { + 'x-ignore-foo': 'bar', + }, + ...CONFIG, + }; + + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + + const expectedConditionString = JSON.stringify(CONFIG.fields); + assert.strictEqual(res.fields['x-ignore-foo'], 'bar'); + const decodedPolicy = Buffer.from( + res.fields.policy, + 'base64' + ).toString('utf-8'); + assert(!decodedPolicy.includes(expectedConditionString)); + + const signStub = BUCKET.storage.authClient.sign; + assert(!signStub.getCall(0).args[0].includes('x-ignore-foo')); + done(); + } + ); + }); + + it('should accept conditions', done => { + CONFIG = { + conditions: [['starts-with', '$key', 'prefix-']], + ...CONFIG, + }; + + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + + const expectedConditionString = JSON.stringify(CONFIG.conditions); + const decodedPolicy = Buffer.from( + res.fields.policy, + 'base64' + ).toString('utf-8'); + assert(decodedPolicy.includes(expectedConditionString)); + + const signStub = BUCKET.storage.authClient.sign; + assert( + !signStub.getCall(0).args[0].includes(expectedConditionString) + ); + done(); + } + ); + }); + + it('should output url with cname', done => { + CONFIG.bucketBoundHostname = 'http://domain.tld'; + + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + assert(res.url, CONFIG.bucketBoundHostname); + done(); + } + ); + }); + + it('should output a virtualHostedStyle url', done => { + CONFIG.virtualHostedStyle = true; + + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + assert(res.url, `https://${BUCKET.name}.storage.googleapis.com/`); + done(); + } + ); + }); + + it('should prefer a customEndpoint > virtualHostedStyle, cname', done => { + const customEndpoint = 'https://my-custom-endpoint.com'; + + STORAGE.apiEndpoint = customEndpoint; + STORAGE.customEndpoint = true; + + CONFIG.virtualHostedStyle = true; + CONFIG.bucketBoundHostname = 'http://domain.tld'; + + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + assert(res.url, `https://${BUCKET.name}.storage.googleapis.com/`); + done(); + } + ); + }); + + it('should append bucket name to the URL when using the emulator', done => { + const emulatorHost = 'http://127.0.0.1:9199'; + const originalApiEndpoint = STORAGE.apiEndpoint; + const originalCustomEndpoint = STORAGE.customEndpoint; + const originalEnvHost = process.env.STORAGE_EMULATOR_HOST; + + process.env.STORAGE_EMULATOR_HOST = emulatorHost; + STORAGE.apiEndpoint = emulatorHost; + STORAGE.customEndpoint = true; + + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + STORAGE.apiEndpoint = originalApiEndpoint; + STORAGE.customEndpoint = originalCustomEndpoint; + if (originalEnvHost) { + process.env.STORAGE_EMULATOR_HOST = originalEnvHost; + } else { + delete process.env.STORAGE_EMULATOR_HOST; + } + + assert.ifError(err); + assert.strictEqual(res.url, `${emulatorHost}/${BUCKET.name}`); + done(); + } + ); + }); + + describe('expires', () => { + it('should accept Date objects', done => { + const expires = new Date(Date.now() + 1000 * 60); + + file.generateSignedPostPolicyV4( + { + expires, + }, + (err: Error, response: SignedPostPolicyV4Output) => { + assert.ifError(err); + const policy = JSON.parse( + Buffer.from(response.fields.policy, 'base64').toString() + ); + assert.strictEqual( + policy.expiration, + formatAsUTCISO(expires, true, '-', ':') + ); + done(); + } + ); + }); + + it('should accept numbers', done => { + const expires = Date.now() + 1000 * 60; + + file.generateSignedPostPolicyV4( + { + expires, + }, + (err: Error, response: SignedPostPolicyV4Output) => { + assert.ifError(err); + const policy = JSON.parse( + Buffer.from(response.fields.policy, 'base64').toString() + ); + assert.strictEqual( + policy.expiration, + formatAsUTCISO(new Date(expires), true, '-', ':') + ); + done(); + } + ); + }); + + it('should accept strings', done => { + const expires = formatAsUTCISO( + new Date(Date.now() + 2 * 24 * 60 * 60 * 1000), + false, + '-' + ); + + file.generateSignedPostPolicyV4( + { + expires, + }, + (err: Error, response: SignedPostPolicyV4Output) => { + assert.ifError(err); + const policy = JSON.parse( + Buffer.from(response.fields.policy, 'base64').toString() + ); + assert.strictEqual( + policy.expiration, + formatAsUTCISO(new Date(expires), true, '-', ':') + ); + done(); + } + ); + }); + + it('should throw if a date is invalid', () => { + const expires = new Date('31-12-2019'); + + assert.throws(() => { + file.generateSignedPostPolicyV4( + { + expires, + }, + () => {} + ), + ExceptionMessages.EXPIRATION_DATE_INVALID; + }); + }); + + it('should throw if a date from the past is given', () => { + const expires = Date.now() - 5; + + assert.throws(() => { + file.generateSignedPostPolicyV4( + { + expires, + }, + () => {} + ), + ExceptionMessages.EXPIRATION_DATE_PAST; + }); + }); + + it('should throw if a date beyond 7 days is given', () => { + const expires = Date.now() + 7.1 * 24 * 60 * 60 * 1000; + + assert.throws(() => { + file.generateSignedPostPolicyV4( + { + expires, + }, + () => {} + ), + {message: 'Max allowed expiration is seven days (604800 seconds).'}; + }); + }); + }); + }); + + describe('getSignedUrl', () => { + const EXPECTED_SIGNED_URL = 'signed-url'; + const CNAME = 'https://www.example.com'; + + let sandbox: sinon.SinonSandbox; + let signer: {getSignedUrl: Function}; + let signerGetSignedUrlStub: sinon.SinonStub; + let urlSignerStub: sinon.SinonStub; + let SIGNED_URL_CONFIG: GetSignedUrlConfig; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + + signerGetSignedUrlStub = sandbox.stub().resolves(EXPECTED_SIGNED_URL); + + signer = { + getSignedUrl: signerGetSignedUrlStub, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + urlSignerStub = (sandbox.stub as any)(fakeSigner, 'URLSigner').returns( + signer + ); + + SIGNED_URL_CONFIG = { + version: 'v4', + expires: new Date(), + action: 'read', + cname: CNAME, + }; + }); + + afterEach(() => sandbox.restore()); + + it('should construct a URLSigner and call getSignedUrl', done => { + const accessibleAtDate = new Date(); + const config = { + contentMd5: 'md5-hash', + contentType: 'application/json', + accessibleAt: accessibleAtDate, + virtualHostedStyle: true, + ...SIGNED_URL_CONFIG, + }; + // assert signer is lazily-initialized. + assert.strictEqual(file.signer, undefined); + file.getSignedUrl(config, (err: Error | null, signedUrl: string) => { + assert.ifError(err); + assert.strictEqual(file.signer, signer); + assert.strictEqual(signedUrl, EXPECTED_SIGNED_URL); + + const ctorArgs = urlSignerStub.getCall(0).args; + assert.strictEqual(ctorArgs[0], file.storage.authClient); + assert.strictEqual(ctorArgs[1], file.bucket); + assert.strictEqual(ctorArgs[2], file); + + const getSignedUrlArgs = signerGetSignedUrlStub.getCall(0).args; + assert.deepStrictEqual(getSignedUrlArgs[0], { + method: 'GET', + version: 'v4', + expires: config.expires, + accessibleAt: accessibleAtDate, + extensionHeaders: {}, + host: undefined, + queryParams: {}, + contentMd5: config.contentMd5, + contentType: config.contentType, + cname: CNAME, + virtualHostedStyle: true, + }); + done(); + }); + }); + + it('should add "x-goog-resumable: start" header if action is resumable', done => { + SIGNED_URL_CONFIG.action = 'resumable'; + SIGNED_URL_CONFIG.extensionHeaders = { + 'another-header': 'value', + }; + + file.getSignedUrl(SIGNED_URL_CONFIG, (err: Error | null) => { + assert.ifError(err); + const getSignedUrlArgs = signerGetSignedUrlStub.getCall(0).args; + assert.strictEqual(getSignedUrlArgs[0]['method'], 'POST'); + assert.deepStrictEqual(getSignedUrlArgs[0]['extensionHeaders'], { + 'another-header': 'value', + 'x-goog-resumable': 'start', + }); + done(); + }); + }); + + it('should add response-content-type query parameter', done => { + SIGNED_URL_CONFIG.responseType = 'application/json'; + file.getSignedUrl(SIGNED_URL_CONFIG, (err: Error | null) => { + assert.ifError(err); + const getSignedUrlArgs = signerGetSignedUrlStub.getCall(0).args; + assert.deepStrictEqual(getSignedUrlArgs[0]['queryParams'], { + 'response-content-type': 'application/json', + }); + done(); + }); + }); + + it('should respect promptSaveAs argument', done => { + const filename = 'fname.txt'; + SIGNED_URL_CONFIG.promptSaveAs = filename; + file.getSignedUrl(SIGNED_URL_CONFIG, (err: Error | null) => { + assert.ifError(err); + const getSignedUrlArgs = signerGetSignedUrlStub.getCall(0).args; + assert.deepStrictEqual(getSignedUrlArgs[0]['queryParams'], { + 'response-content-disposition': + 'attachment; filename="' + filename + '"', + }); + done(); + }); + }); + + it('should add response-content-disposition query parameter', done => { + const disposition = 'attachment; filename="fname.ext"'; + SIGNED_URL_CONFIG.responseDisposition = disposition; + file.getSignedUrl(SIGNED_URL_CONFIG, (err: Error | null) => { + assert.ifError(err); + const getSignedUrlArgs = signerGetSignedUrlStub.getCall(0).args; + assert.deepStrictEqual(getSignedUrlArgs[0]['queryParams'], { + 'response-content-disposition': disposition, + }); + done(); + }); + }); + + it('should ignore promptSaveAs if set', done => { + const saveAs = 'fname2.ext'; + const disposition = 'attachment; filename="fname.ext"'; + SIGNED_URL_CONFIG.promptSaveAs = saveAs; + SIGNED_URL_CONFIG.responseDisposition = disposition; + + file.getSignedUrl(SIGNED_URL_CONFIG, (err: Error | null) => { + assert.ifError(err); + const getSignedUrlArgs = signerGetSignedUrlStub.getCall(0).args; + assert.deepStrictEqual(getSignedUrlArgs[0]['queryParams'], { + 'response-content-disposition': disposition, + }); + done(); + }); + }); + + it('should add generation to query parameter', done => { + file.generation = '246680131'; + + file.getSignedUrl(SIGNED_URL_CONFIG, (err: Error | null) => { + assert.ifError(err); + const getSignedUrlArgs = signerGetSignedUrlStub.getCall(0).args; + assert.deepStrictEqual(getSignedUrlArgs[0]['queryParams'], { + generation: file.generation, + }); + done(); + }); + }); + }); + + describe('makePrivate', () => { + it('should execute callback with API response', done => { + const apiResponse = {}; + + file.setMetadata = ( + metadata: FileMetadata, + optionsOrCallback: SetMetadataOptions | MetadataCallback, + cb: MetadataCallback + ) => { + Promise.resolve([apiResponse]).then(resp => cb(null, ...resp)); + }; + + file.makePrivate((err: Error, apiResponse_: {}) => { + assert.ifError(err); + assert.strictEqual(apiResponse_, apiResponse); + + done(); + }); + }); + + it('should make the file private to project by default', done => { + file.setMetadata = (metadata: {}, query: {}) => { + assert.deepStrictEqual(metadata, {acl: null}); + assert.deepStrictEqual(query, {predefinedAcl: 'projectPrivate'}); + done(); + }; + + file.makePrivate(util.noop); + }); + + it('should make the file private to user if strict = true', done => { + file.setMetadata = (metadata: {}, query: {}) => { + assert.deepStrictEqual(query, {predefinedAcl: 'private'}); + done(); + }; + + file.makePrivate({strict: true}, util.noop); + }); + + it('should accept metadata', done => { + const options = { + metadata: {a: 'b', c: 'd'}, + }; + file.setMetadata = (metadata: {}) => { + assert.deepStrictEqual(metadata, { + acl: null, + ...options.metadata, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + assert.strictEqual(typeof (options.metadata as any).acl, 'undefined'); + done(); + }; + file.makePrivate(options, assert.ifError); + }); + + it('should accept userProject', done => { + const options = { + userProject: 'user-project-id', + }; + + file.setMetadata = (metadata: {}, query: SetFileMetadataOptions) => { + assert.strictEqual(query.userProject, options.userProject); + done(); + }; + + file.makePrivate(options, assert.ifError); + }); + }); + + describe('makePublic', () => { + it('should execute callback', done => { + file.acl.add = (options: {}, callback: Function) => { + callback(); + }; + + file.makePublic(done); + }); + + it('should make the file public', done => { + file.acl.add = (options: {}) => { + assert.deepStrictEqual(options, {entity: 'allUsers', role: 'READER'}); + done(); + }; + + file.makePublic(util.noop); + }); + }); + + describe('publicUrl', () => { + it('should return the public URL', done => { + const NAME = 'file-name'; + const file = new File(BUCKET, NAME); + assert.strictEqual( + file.publicUrl(), + `https://storage.googleapis.com/bucket-name/${encodeURIComponent(NAME)}` + ); + done(); + }); + + it('with slash in the name', done => { + const NAME = 'parent/child'; + const file = new File(BUCKET, NAME); + assert.strictEqual( + file.publicUrl(), + `https://storage.googleapis.com/bucket-name/${encodeURIComponent(NAME)}` + ); + done(); + }); + + it('with tilde in the name', done => { + const NAME = 'foo~bar'; + const file = new File(BUCKET, NAME); + assert.strictEqual( + file.publicUrl(), + `https://storage.googleapis.com/bucket-name/${encodeURIComponent(NAME)}` + ); + done(); + }); + + it('with non ascii in the name', done => { + const NAME = '\u2603'; + const file = new File(BUCKET, NAME); + assert.strictEqual( + file.publicUrl(), + `https://storage.googleapis.com/bucket-name/${encodeURIComponent(NAME)}` + ); + done(); + }); + + it('with an ampersand in the name', done => { + const NAME = '&foo'; + const file = new File(BUCKET, NAME); + assert.strictEqual( + file.publicUrl(), + `https://storage.googleapis.com/bucket-name/${encodeURIComponent(NAME)}` + ); + done(); + }); + }); + + describe('isPublic', () => { + const sandbox = sinon.createSandbox(); + + afterEach(() => sandbox.restore()); + + it('should execute callback with `true` in response', done => { + file.isPublic((err: ApiError, resp: boolean) => { + assert.ifError(err); + assert.strictEqual(resp, true); + done(); + }); + }); + + it('should execute callback with `false` in response', done => { + fakeUtil.makeRequest = function ( + reqOpts: DecorateRequestOptions, + config: object, + callback: BodyResponseCallback + ) { + const error = new ApiError('Permission Denied.'); + error.code = 403; + callback(error); + }; + file.isPublic((err: ApiError, resp: boolean) => { + assert.ifError(err); + assert.strictEqual(resp, false); + done(); + }); + }); + + it('should propagate non-403 errors to user', done => { + const error = new ApiError('400 Error.'); + error.code = 400; + fakeUtil.makeRequest = function ( + reqOpts: DecorateRequestOptions, + config: object, + callback: BodyResponseCallback + ) { + callback(error); + }; + file.isPublic((err: ApiError) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should correctly send a GET request', done => { + fakeUtil.makeRequest = function ( + reqOpts: DecorateRequestOptions, + config: object, + callback: BodyResponseCallback + ) { + assert.strictEqual(reqOpts.method, 'GET'); + callback(null); + }; + file.isPublic((err: ApiError) => { + assert.ifError(err); + done(); + }); + }); + + it('should correctly format URL in the request', done => { + file = new File(BUCKET, 'my#file$.png'); + const expectedURL = `https://storage.googleapis.com/${ + BUCKET.name + }/${encodeURIComponent(file.name)}`; + + fakeUtil.makeRequest = function ( + reqOpts: DecorateRequestOptions, + config: object, + callback: BodyResponseCallback + ) { + assert.strictEqual(reqOpts.uri, expectedURL); + callback(null); + }; + file.isPublic((err: ApiError) => { + assert.ifError(err); + done(); + }); + }); + + it('should not set any headers when there are no interceptors', done => { + fakeUtil.makeRequest = function ( + reqOpts: DecorateRequestOptions, + config: object, + callback: BodyResponseCallback + ) { + assert.deepStrictEqual(reqOpts.headers, {}); + callback(null); + }; + file.isPublic((err: ApiError) => { + assert.ifError(err); + done(); + }); + }); + + it('should set headers when an interceptor is defined', done => { + const expectedHeader = {hello: 'world'}; + file.storage.interceptors = []; + file.storage.interceptors.push({ + request: (requestConfig: DecorateRequestOptions) => { + requestConfig.headers = requestConfig.headers || {}; + Object.assign(requestConfig.headers, expectedHeader); + return requestConfig as DecorateRequestOptions; + }, + }); + + fakeUtil.makeRequest = function ( + reqOpts: DecorateRequestOptions, + config: object, + callback: BodyResponseCallback + ) { + assert.deepStrictEqual(reqOpts.headers, expectedHeader); + callback(null); + }; + file.isPublic((err: ApiError) => { + assert.ifError(err); + done(); + }); + }); + }); + + describe('moveFileAtomic', () => { + function assertmoveFileAtomic( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file: any, + expectedDestination: string, + callback: Function + ) { + file.moveFileAtomic = (destination: string) => { + assert.strictEqual(destination, expectedDestination); + callback(); + }; + } + + it('should throw if no destination is provided', () => { + assert.throws(() => { + file.moveFileAtomic(); + }, /Destination file should have a name\./); + }); + + it('should URI encode file names', done => { + const newFile = new File(BUCKET, 'nested/file.jpg'); + + const expectedPath = `/moveTo/o/${encodeURIComponent(newFile.name)}`; + + directoryFile.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, expectedPath); + done(); + }; + + directoryFile.moveFileAtomic(newFile); + }); + + it('should call moveFileAtomic with string', done => { + const newFileName = 'new-file-name.png'; + assertmoveFileAtomic(file, newFileName, done); + file.moveFileAtomic(newFileName); + }); + + it('should call moveFileAtomic with File', done => { + const newFile = new File(BUCKET, 'new-file'); + assertmoveFileAtomic(file, newFile, done); + file.moveFileAtomic(newFile); + }); + + it('should accept an options object', done => { + const newFile = new File(BUCKET, 'name'); + const options = {}; + + file.moveFileAtomic = (destination: {}, options_: {}) => { + assert.strictEqual(options_, options); + done(); + }; + + file.moveFileAtomic(newFile, options, assert.ifError); + }); + + it('should execute callback with error & API response', done => { + const error = new Error('Error.'); + const apiResponse = {}; + + const newFile = new File(BUCKET, 'new-file'); + + file.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(error, apiResponse); + }; + + file.moveFileAtomic(newFile, (err: Error, file: {}, apiResponse_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(file, null); + assert.strictEqual(apiResponse_, apiResponse); + + done(); + }); + }); + + it('should pass through userProject', done => { + const options = { + userProject: 'user-project', + }; + const originalOptions = Object.assign({}, options); + const newFile = new File(BUCKET, 'new-file'); + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.userProject, options.userProject); + assert.strictEqual(reqOpts.json.userProject, undefined); + assert.deepStrictEqual(options, originalOptions); + done(); + }; + + file.moveFileAtomic(newFile, options, assert.ifError); + }); + + it('should handle optionsOrCallback being the options', done => { + const options = { + preconditionOpts: {ifGenerationMatch: 123}, + }; + const originalOptions = Object.assign({}, options); + const newFile = new File(BUCKET, 'new-file'); + + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.qs.ifGenerationMatch, + options.preconditionOpts.ifGenerationMatch + ); + assert.strictEqual(reqOpts.json.userProject, undefined); + assert.deepStrictEqual(options, originalOptions); + done(); + }; + + file.moveFileAtomic(newFile, options, assert.ifError); + }); + + describe('destination types', () => { + function assertPathEquals( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file: any, + expectedPath: string, + callback: Function + ) { + file.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, expectedPath); + callback(); + }; + } + + it('should allow a string', done => { + const newFileName = 'new-file-name.png'; + const newFile = new File(BUCKET, newFileName); + const expectedPath = `/moveTo/o/${newFile.name}`; + assertPathEquals(file, expectedPath, done); + file.moveFileAtomic(newFileName); + }); + + it('should allow a string with leading slash.', done => { + const newFileName = '/new-file-name.png'; + const newFile = new File(BUCKET, newFileName); + const expectedPath = `/moveTo/o/${encodeURIComponent(newFile.name)}`; + assertPathEquals(file, expectedPath, done); + file.moveFileAtomic(newFileName); + }); + + it('should allow a "gs://..." string', done => { + const newFileName = 'gs://other-bucket/new-file-name.png'; + const expectedPath = '/moveTo/o/new-file-name.png'; + assertPathEquals(file, expectedPath, done); + file.moveFileAtomic(newFileName); + }); + + it('should allow a File', done => { + const newFile = new File(BUCKET, 'new-file'); + const expectedPath = `/moveTo/o/${newFile.name}`; + assertPathEquals(file, expectedPath, done); + file.moveFileAtomic(newFile); + }); + + it('should throw if a destination cannot be parsed', () => { + assert.throws(() => { + file.moveFileAtomic(() => {}); + }, /Destination file should have a name\./); + }); + }); + + describe('returned File object', () => { + beforeEach(() => { + const resp = {success: true}; + file.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, resp); + }; + }); + + it('should re-use file object if one is provided', done => { + const newFile = new File(BUCKET, 'new-file'); + file.moveFileAtomic(newFile, (err: Error, copiedFile: {}) => { + assert.ifError(err); + assert.deepStrictEqual(copiedFile, newFile); + done(); + }); + }); + + it('should create new file on the same bucket', done => { + const newFilename = 'new-filename'; + file.moveFileAtomic(newFilename, (err: Error, copiedFile: File) => { + assert.ifError(err); + assert.strictEqual(copiedFile.bucket.name, BUCKET.name); + assert.strictEqual(copiedFile.name, newFilename); + done(); + }); + }); + }); + }); + + describe('move', () => { + describe('copy to destination', () => { + function assertCopyFile( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file: any, + expectedDestination: string, + callback: Function + ) { + file.copy = (destination: string) => { + assert.strictEqual(destination, expectedDestination); + callback(); + }; + } + + it('should call copy with string', done => { + const newFileName = 'new-file-name.png'; + assertCopyFile(file, newFileName, done); + file.move(newFileName); + }); + + it('should call copy with Bucket', done => { + assertCopyFile(file, BUCKET, done); + file.move(BUCKET); + }); + + it('should call copy with File', done => { + const newFile = new File(BUCKET, 'new-file'); + assertCopyFile(file, newFile, done); + file.move(newFile); + }); + + it('should accept an options object', done => { + const newFile = new File(BUCKET, 'name'); + const options = {}; + + file.copy = (destination: {}, options_: {}) => { + assert.strictEqual(options_, options); + done(); + }; + + file.move(newFile, options, assert.ifError); + }); + + it('should fail if copy fails', done => { + const originalErrorMessage = 'Original error message.'; + const error = new Error(originalErrorMessage); + file.copy = (destination: {}, options: {}, callback: Function) => { + callback(error); + }; + file.move('new-filename', (err: Error) => { + assert.strictEqual(err, error); + assert.strictEqual( + err.message, + `file#copy failed with an error - ${originalErrorMessage}` + ); + done(); + }); + }); + }); + + describe('delete original file', () => { + it('should call the callback with destinationFile and copyApiResponse', done => { + const copyApiResponse = {}; + const newFile = new File(BUCKET, 'new-filename'); + file.copy = (destination: {}, options: {}, callback: Function) => { + callback(null, newFile, copyApiResponse); + }; + file.delete = (_: {}, callback: Function) => { + callback(); + }; + + file.move( + 'new-filename', + (err: Error, destinationFile: File, apiResponse: {}) => { + assert.ifError(err); + assert.strictEqual(destinationFile, newFile); + assert.strictEqual(apiResponse, copyApiResponse); + done(); + } + ); + }); + + it('should delete if copy is successful', done => { + const destinationFile = {bucket: {}}; + file.copy = (destination: {}, options: {}, callback: Function) => { + callback(null, destinationFile); + }; + Object.assign(file, { + delete() { + assert.strictEqual(this, file); + done(); + }, + }); + file.move('new-filename'); + }); + + it('should not delete if copy fails', done => { + let deleteCalled = false; + file.copy = (destination: {}, options: {}, callback: Function) => { + callback(new Error('Error.')); + }; + file.delete = () => { + deleteCalled = true; + }; + file.move('new-filename', () => { + assert.strictEqual(deleteCalled, false); + done(); + }); + }); + + it('should not delete the destination is same as origin', done => { + file.request = (config: {}, callback: Function) => { + callback(null, {}); + }; + const stub = sinon.stub(file, 'delete'); + // destination is same bucket as object + file.move(BUCKET, (err: Error) => { + assert.ifError(err); + // destination is same file as object + file.move(file, (err: Error) => { + assert.ifError(err); + // destination is same file name as string + file.move(file.name, (err: Error) => { + assert.ifError(err); + assert.ok(stub.notCalled); + stub.reset(); + done(); + }); + }); + }); + }); + + it('should pass options to delete', done => { + const options = {}; + const destinationFile = {bucket: {}}; + + file.copy = (destination: {}, options: {}, callback: Function) => { + callback(null, destinationFile); + }; + + file.delete = (options_: {}) => { + assert.strictEqual(options_, options); + done(); + }; + + file.move('new-filename', options, assert.ifError); + }); + + it('should fail if delete fails', done => { + const originalErrorMessage = 'Original error message.'; + const error = new Error(originalErrorMessage); + const destinationFile = {bucket: {}}; + file.copy = (destination: {}, options: {}, callback: Function) => { + callback(null, destinationFile); + }; + file.delete = (options: {}, callback: Function) => { + callback(error); + }; + file.move('new-filename', (err: Error) => { + assert.strictEqual(err, error); + assert.strictEqual( + err.message, + `file#delete failed with an error - ${originalErrorMessage}` + ); + done(); + }); + }); + }); + }); + + describe('rename', () => { + it('should correctly call File#move', done => { + const newFileName = 'renamed-file.txt'; + const options = {}; + file.move = (dest: string, opts: MoveOptions, cb: Function) => { + assert.strictEqual(dest, newFileName); + assert.strictEqual(opts, options); + assert.strictEqual(cb, done); + cb(); + }; + file.rename(newFileName, options, done); + }); + + it('should accept File object', done => { + const newFileObject = new File(BUCKET, 'renamed-file.txt'); + const options = {}; + file.move = (dest: string, opts: MoveOptions, cb: Function) => { + assert.strictEqual(dest, newFileObject); + assert.strictEqual(opts, options); + assert.strictEqual(cb, done); + cb(); + }; + file.rename(newFileObject, options, done); + }); + + it('should not require options', done => { + file.move = (dest: string, opts: MoveOptions, cb: Function) => { + assert.deepStrictEqual(opts, {}); + cb(); + }; + file.rename('new-name', done); + }); + }); + + describe('restore', () => { + it('should pass options to underlying request call', async () => { + file.parent.request = function ( + reqOpts: DecorateRequestOptions, + callback_: Function + ) { + assert.strictEqual(this, file); + assert.deepStrictEqual(reqOpts, { + method: 'POST', + uri: '/restore', + qs: {generation: 123}, + }); + assert.strictEqual(callback_, undefined); + return []; + }; + + await file.restore({generation: 123}); + }); + }); + + describe('request', () => { + it('should call the parent request function', () => { + const options = {}; + const callback = () => {}; + const expectedReturnValue = {}; + + file.parent.request = function ( + reqOpts: DecorateRequestOptions, + callback_: Function + ) { + assert.strictEqual(this, file); + assert.strictEqual(reqOpts, options); + assert.strictEqual(callback_, callback); + return expectedReturnValue; + }; + + const returnedValue = file.request(options, callback); + assert.strictEqual(returnedValue, expectedReturnValue); + }); + }); + + describe('rotateEncryptionKey', () => { + it('should create new File correctly', done => { + const options = {}; + + file.bucket.file = (id: {}, options_: {}) => { + assert.strictEqual(id, file.id); + assert.strictEqual(options_, options); + done(); + }; + + file.rotateEncryptionKey(options, assert.ifError); + }); + + it('should default to customer-supplied encryption key', done => { + const encryptionKey = 'encryption-key'; + + file.bucket.file = (id: {}, options: FileOptions) => { + assert.strictEqual(options.encryptionKey, encryptionKey); + done(); + }; + + file.rotateEncryptionKey(encryptionKey, assert.ifError); + }); + + it('should accept a Buffer for customer-supplied encryption key', done => { + const encryptionKey = crypto.randomBytes(32); + + file.bucket.file = (id: {}, options: FileOptions) => { + assert.strictEqual(options.encryptionKey, encryptionKey); + done(); + }; + + file.rotateEncryptionKey(encryptionKey, assert.ifError); + }); + + it('should call copy correctly', done => { + const newFile = {}; + + file.bucket.file = () => { + return newFile; + }; + + file.copy = ( + destination: string, + options: object, + callback: Function + ) => { + assert.strictEqual(destination, newFile); + assert.deepStrictEqual(options, {}); + callback(); // done() + }; + + file.rotateEncryptionKey({}, done); + }); + }); + + describe('save', () => { + const DATA = 'Data!'; + const BUFFER_DATA = Buffer.from(DATA, 'utf8'); + const UINT8_ARRAY_DATA = Uint8Array.from( + Array.from(DATA).map(l => l.charCodeAt(0)) + ); + + class DelayedStreamNoError extends Transform { + _transform(chunk: string | Buffer, _encoding: string, done: Function) { + this.push(chunk); + setTimeout(() => { + done(); + }, 5); + } + } + + class DelayedStream500Error extends Transform { + retryCount: number; + constructor(retryCount: number) { + super(); + this.retryCount = retryCount; + } + _transform(chunk: string | Buffer, _encoding: string, done: Function) { + this.push(chunk); + setTimeout(() => { + if (this.retryCount === 1) { + done(new HTTPError('first error', 500)); + } else { + done(); + } + }, 5); + } + } + + describe('retry multipart upload', () => { + it('should save a string with no errors', async () => { + const options = {resumable: false}; + file.createWriteStream = () => { + return new DelayedStreamNoError(); + }; + await file.save(DATA, options, assert.ifError); + }); + + it('should save a buffer with no errors', async () => { + const options = {resumable: false}; + file.createWriteStream = () => { + return new DelayedStreamNoError(); + }; + await file.save(BUFFER_DATA, options, assert.ifError); + }); + + it('should save a Uint8Array with no errors', async () => { + const options = {resumable: false}; + file.createWriteStream = () => { + return new DelayedStreamNoError(); + }; + await file.save(UINT8_ARRAY_DATA, options, assert.ifError); + }); + + it('string upload should retry on first failure', async () => { + const options = { + resumable: false, + preconditionOpts: {ifGenerationMatch: 100}, + }; + let retryCount = 0; + file.createWriteStream = () => { + retryCount++; + return new DelayedStream500Error(retryCount); + }; + await file.save(DATA, options); + assert.ok(retryCount === 2); + }); + + it('string upload should not retry if nonretryable error code', async () => { + const options = {resumable: false}; + let retryCount = 0; + file.createWriteStream = () => { + class DelayedStream403Error extends Transform { + _transform( + chunk: string | Buffer, + _encoding: string, + done: Function + ) { + this.push(chunk); + setTimeout(() => { + retryCount++; + if (retryCount === 1) { + done(new HTTPError('first error', 403)); + } else { + done(); + } + }, 5); + } + } + return new DelayedStream403Error(); + }; + try { + await file.save(DATA, options); + throw Error('unreachable'); + } catch (e) { + assert.strictEqual((e as Error).message, 'first error'); + } + }); + + it('should save a Readable with no errors (String)', done => { + const options = {resumable: false}; + file.createWriteStream = () => { + const writeStream = new PassThrough(); + writeStream.on('data', data => { + assert.strictEqual(data.toString(), DATA); + }); + writeStream.once('finish', done); + return writeStream; + }; + + const readable = new Readable({ + read() { + this.push(DATA); + this.push(null); + }, + }); + + void file.save(readable, options); + }); + + it('should save a Readable with no errors (Buffer)', done => { + const options = {resumable: false}; + file.createWriteStream = () => { + const writeStream = new PassThrough(); + writeStream.on('data', data => { + assert.strictEqual(data.toString(), DATA); + }); + writeStream.once('finish', done); + return writeStream; + }; + + const readable = new Readable({ + read() { + this.push(BUFFER_DATA); + this.push(null); + }, + }); + + void file.save(readable, options); + }); + + it('should save a Readable with no errors (Uint8Array)', done => { + const options = {resumable: false}; + file.createWriteStream = () => { + const writeStream = new PassThrough(); + writeStream.on('data', data => { + assert.strictEqual(data.toString(), DATA); + }); + writeStream.once('finish', done); + return writeStream; + }; + + const readable = new Readable({ + read() { + this.push(UINT8_ARRAY_DATA); + this.push(null); + }, + }); + + void file.save(readable, options); + }); + + it('should propagate Readable errors', done => { + const options = {resumable: false}; + file.createWriteStream = () => { + const writeStream = new PassThrough(); + let errorCalled = false; + writeStream.on('data', data => { + assert.strictEqual(data.toString(), DATA); + }); + writeStream.on('error', err => { + errorCalled = true; + assert.strictEqual(err.message, 'Error!'); + }); + writeStream.on('finish', () => { + assert.ok(errorCalled); + }); + return writeStream; + }; + + const readable = new Readable({ + read() { + setTimeout(() => { + this.push(DATA); + this.destroy(new Error('Error!')); + }, 50); + }, + }); + + file.save(readable, options, (err: Error) => { + assert.strictEqual(err.message, 'Error!'); + done(); + }); + }); + + it('Readable upload should not retry', async () => { + const options = {resumable: false}; + + let retryCount = 0; + + file.createWriteStream = () => { + retryCount++; + return new Transform({ + transform( + chunk: string | Buffer, + _encoding: string, + done: Function + ) { + this.push(chunk); + setTimeout(() => { + done(new HTTPError('retryable error', 408)); + }, 5); + }, + }); + }; + try { + const readable = new Readable({ + read() { + this.push(DATA); + this.push(null); + }, + }); + + await file.save(readable, options); + throw Error('unreachable'); + } catch (e) { + assert.strictEqual((e as Error).message, 'retryable error'); + assert.ok(retryCount === 1); + } + }); + + it('should save a generator with no error', done => { + const options = {resumable: false}; + file.createWriteStream = () => { + const writeStream = new PassThrough(); + writeStream.on('data', data => { + assert.strictEqual(data.toString(), DATA); + done(); + }); + return writeStream; + }; + + const generator = async function* (arg?: {signal?: AbortSignal}) { + await new Promise(resolve => setTimeout(resolve, 5)); + if (arg?.signal?.aborted) return; + yield DATA; + }; + + void file.save(generator, options); + }); + + it('should propagate async iterable errors', done => { + const options = {resumable: false}; + file.createWriteStream = () => { + const writeStream = new PassThrough(); + let errorCalled = false; + writeStream.on('data', data => { + assert.strictEqual(data.toString(), DATA); + }); + writeStream.on('error', err => { + errorCalled = true; + assert.strictEqual(err.message, 'Error!'); + }); + writeStream.on('finish', () => { + assert.ok(errorCalled); + }); + return writeStream; + }; + + const generator = async function* () { + yield DATA; + throw new Error('Error!'); + }; + + file.save(generator(), options, (err: Error) => { + assert.strictEqual(err.message, 'Error!'); + done(); + }); + }); + + it('buffer upload should retry on first failure', async () => { + const options = { + resumable: false, + preconditionOpts: {ifGenerationMatch: 100}, + }; + let retryCount = 0; + file.createWriteStream = () => { + retryCount++; + return new DelayedStream500Error(retryCount); + }; + await file.save(BUFFER_DATA, options); + assert.ok(retryCount === 2); + }); + + it('resumable upload should retry', async () => { + const options = { + resumable: true, + preconditionOpts: {ifGenerationMatch: 100}, + }; + let retryCount = 0; + file.createWriteStream = () => { + retryCount++; + return new DelayedStream500Error(retryCount); + }; + + await file.save(BUFFER_DATA, options); + assert.ok(retryCount === 2); + }); + + it('should not retry if ifMetagenerationMatch is undefined', async () => { + const options = { + resumable: true, + preconditionOpts: {ifGenerationMatch: 100}, + }; + let retryCount = 0; + file.createWriteStream = () => { + retryCount++; + return new DelayedStream500Error(retryCount); + }; + try { + await file.save(BUFFER_DATA, options); + } catch { + assert.strictEqual(retryCount, 1); + } + }); + }); + + it('should execute callback', async () => { + const options = {resumable: true}; + let retryCount = 0; + file.createWriteStream = () => { + retryCount++; + return new DelayedStream500Error(retryCount); + }; + + file.save(DATA, options, (err: HTTPError) => { + assert.strictEqual(err.code, 500); + }); + }); + + it('should accept an options object', done => { + const options = {}; + + file.createWriteStream = (options_: {}) => { + assert.strictEqual(options_, options); + setImmediate(done); + return new PassThrough(); + }; + + file.save(DATA, options, assert.ifError); + }); + + it('should not require options', done => { + file.createWriteStream = (options_: {}) => { + assert.deepStrictEqual(options_, {}); + setImmediate(done); + return new PassThrough(); + }; + + file.save(DATA, assert.ifError); + }); + + it('should register the error listener', done => { + file.createWriteStream = () => { + const writeStream = new PassThrough(); + writeStream.on('error', done); + setImmediate(() => { + writeStream.emit('error'); + }); + return writeStream; + }; + + file.save(DATA, assert.ifError); + }); + + it('should register the finish listener', done => { + file.createWriteStream = () => { + const writeStream = new PassThrough(); + writeStream.once('finish', done); + return writeStream; + }; + + file.save(DATA, assert.ifError); + }); + + it('should register the progress listener if onUploadProgress is passed', done => { + const onUploadProgress = util.noop; + file.createWriteStream = () => { + const writeStream = new PassThrough(); + setImmediate(() => { + const [listener] = writeStream.listeners('progress'); + assert.strictEqual(listener, onUploadProgress); + done(); + }); + return writeStream; + }; + + file.save(DATA, {onUploadProgress}, assert.ifError); + }); + + it('should write the data', done => { + file.createWriteStream = () => { + const writeStream = new PassThrough(); + writeStream.on('data', data => { + assert.strictEqual(data.toString(), DATA); + done(); + }); + return writeStream; + }; + + file.save(DATA, assert.ifError); + }); + }); + + describe('setMetadata', () => { + it('should accept overrideUnlockedRetention option and set query parameter', done => { + const newFile = new File(BUCKET, 'new-file'); + + newFile.parent.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.overrideUnlockedRetention, true); + done(); + }; + + newFile.setMetadata( + {retention: null}, + {overrideUnlockedRetention: true}, + assert.ifError + ); + }); + }); + + describe('setStorageClass', () => { + const STORAGE_CLASS = 'new_storage_class'; + + it('should make the correct copy request', done => { + file.copy = (newFile: {}, options: {}) => { + assert.strictEqual(newFile, file); + assert.deepStrictEqual(options, { + storageClass: STORAGE_CLASS.toUpperCase(), + }); + done(); + }; + + file.setStorageClass(STORAGE_CLASS, assert.ifError); + }); + + it('should accept options', done => { + const options = { + a: 'b', + c: 'd', + }; + + const expectedOptions = { + a: 'b', + c: 'd', + storageClass: STORAGE_CLASS.toUpperCase(), + }; + + file.copy = (newFile: {}, options: {}) => { + assert.deepStrictEqual(options, expectedOptions); + done(); + }; + + file.setStorageClass(STORAGE_CLASS, options, assert.ifError); + }); + + it('should convert camelCase to snake_case', done => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.copy = (newFile: {}, options: any) => { + assert.strictEqual(options.storageClass, 'CAMEL_CASE'); + done(); + }; + + file.setStorageClass('camelCase', assert.ifError); + }); + + it('should convert hyphenate to snake_case', done => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + file.copy = (newFile: {}, options: any) => { + assert.strictEqual(options.storageClass, 'HYPHENATED_CLASS'); + done(); + }; + + file.setStorageClass('hyphenated-class', assert.ifError); + }); + + describe('error', () => { + const ERROR = new Error('Error.'); + const API_RESPONSE = {}; + + beforeEach(() => { + file.copy = (newFile: {}, options: {}, callback: Function) => { + callback(ERROR, null, API_RESPONSE); + }; + }); + + it('should execute callback with error & API response', done => { + file.setStorageClass(STORAGE_CLASS, (err: Error, apiResponse: {}) => { + assert.strictEqual(err, ERROR); + assert.strictEqual(apiResponse, API_RESPONSE); + done(); + }); + }); + }); + + describe('success', () => { + const METADATA = {}; + + const COPIED_FILE = { + metadata: METADATA, + }; + + const API_RESPONSE = {}; + + beforeEach(() => { + file.copy = (newFile: {}, options: {}, callback: Function) => { + callback(null, COPIED_FILE, API_RESPONSE); + }; + }); + + it('should update the metadata on the file', done => { + file.setStorageClass(STORAGE_CLASS, (err: Error) => { + assert.ifError(err); + assert.strictEqual(file.metadata, METADATA); + done(); + }); + }); + + it('should execute callback with api response', done => { + file.setStorageClass(STORAGE_CLASS, (err: Error, apiResponse: {}) => { + assert.ifError(err); + assert.strictEqual(apiResponse, API_RESPONSE); + done(); + }); + }); + }); + }); + + describe('setEncryptionKey', () => { + const KEY = crypto.randomBytes(32); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const KEY_BASE64 = Buffer.from(KEY as any).toString('base64'); + const KEY_HASH = crypto + .createHash('sha256') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .update(KEY_BASE64, 'base64' as any) + .digest('base64'); + let _file: {}; + + beforeEach(() => { + _file = file.setEncryptionKey(KEY); + }); + + it('should localize the key', () => { + assert.strictEqual(file.encryptionKey, KEY); + }); + + it('should localize the base64 key', () => { + assert.strictEqual(file.encryptionKeyBase64, KEY_BASE64); + }); + + it('should localize the hash', () => { + assert.strictEqual(file.encryptionKeyHash, KEY_HASH); + }); + + it('should return the file instance', () => { + assert.strictEqual(_file, file); + }); + + it('should push the correct request interceptor', done => { + const expectedInterceptor = { + headers: { + 'x-goog-encryption-algorithm': 'AES256', + 'x-goog-encryption-key': KEY_BASE64, + 'x-goog-encryption-key-sha256': KEY_HASH, + }, + }; + + assert.deepStrictEqual( + file.interceptors[0].request({}), + expectedInterceptor + ); + assert.deepStrictEqual( + file.encryptionKeyInterceptor.request({}), + expectedInterceptor + ); + + done(); + }); + }); + + describe('startResumableUpload_', () => { + beforeEach(() => { + file.getRequestInterceptors = () => []; + }); + + describe('starting', () => { + it('should start a resumable upload', done => { + const options = { + metadata: {}, + offset: 1234, + public: true, + private: false, + predefinedAcl: 'allUsers', + uri: 'http://resumable-uri', + userProject: 'user-project-id', + chunkSize: 262144, // 256 KiB + }; + + file.generation = 3; + file.encryptionKey = 'key'; + file.kmsKeyName = 'kms-key-name'; + + const customRequestInterceptors = [ + (reqOpts: DecorateRequestOptions) => { + reqOpts.headers = Object.assign({}, reqOpts.headers, { + a: 'b', + }); + return reqOpts; + }, + (reqOpts: DecorateRequestOptions) => { + reqOpts.headers = Object.assign({}, reqOpts.headers, { + c: 'd', + }); + return reqOpts; + }, + ]; + file.getRequestInterceptors = () => { + return customRequestInterceptors; + }; + + resumableUploadOverride = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + upload(opts: any) { + const bucket = file.bucket; + const storage = bucket.storage; + const authClient = storage.makeAuthenticatedRequest.authClient; + + assert.strictEqual(opts.authClient, authClient); + assert.strictEqual(opts.apiEndpoint, storage.apiEndpoint); + assert.strictEqual(opts.bucket, bucket.name); + assert.deepStrictEqual(opts.customRequestOptions, { + headers: { + a: 'b', + c: 'd', + }, + }); + assert.strictEqual(opts.file, file.name); + assert.strictEqual(opts.generation, file.generation); + assert.strictEqual(opts.key, file.encryptionKey); + assert.deepStrictEqual(opts.metadata, options.metadata); + assert.strictEqual(opts.offset, options.offset); + assert.strictEqual(opts.predefinedAcl, options.predefinedAcl); + assert.strictEqual(opts.private, options.private); + assert.strictEqual(opts.public, options.public); + assert.strictEqual(opts.uri, options.uri); + assert.strictEqual(opts.userProject, options.userProject); + assert.deepStrictEqual(opts.retryOptions, { + ...storage.retryOptions, + }); + assert.strictEqual(opts.params, storage.preconditionOpts); + assert.strictEqual(opts.chunkSize, options.chunkSize); + + setImmediate(done); + return new PassThrough(); + }, + }; + + file.startResumableUpload_(duplexify(), options); + }); + + it('should emit the response', done => { + const resp = {}; + const uploadStream = new PassThrough(); + + resumableUploadOverride = { + upload() { + setImmediate(() => { + uploadStream.emit('response', resp); + }); + return uploadStream; + }, + }; + + uploadStream.on('response', resp_ => { + assert.strictEqual(resp_, resp); + done(); + }); + + file.startResumableUpload_(duplexify()); + }); + + it('should set the metadata from the metadata event', done => { + const metadata = {}; + const uploadStream = new PassThrough(); + + resumableUploadOverride = { + upload() { + setImmediate(() => { + uploadStream.emit('metadata', metadata); + + setImmediate(() => { + assert.strictEqual(file.metadata, metadata); + done(); + }); + }); + return uploadStream; + }, + }; + + file.startResumableUpload_(duplexify()); + }); + + it('should emit complete after the stream finishes', done => { + const dup = duplexify(); + + dup.on('complete', done); + + resumableUploadOverride = { + upload() { + const uploadStream = new Transform(); + setImmediate(() => { + uploadStream.end(); + }); + return uploadStream; + }, + }; + + file.startResumableUpload_(dup); + }); + + it('should set the writable stream', done => { + const dup = duplexify(); + const uploadStream = new PassThrough(); + + dup.setWritable = (stream: Duplex) => { + assert.strictEqual(stream, uploadStream); + done(); + }; + + resumableUploadOverride = { + upload() { + return uploadStream; + }, + }; + + file.startResumableUpload_(dup); + }); + + it('should emit progress event', done => { + const progress = {}; + const dup = duplexify(); + dup.on('progress', evt => { + assert.strictEqual(evt, progress); + done(); + }); + + resumableUploadOverride = { + upload() { + const uploadStream = new Transform(); + setImmediate(() => { + uploadStream.emit('progress', progress); + }); + + return uploadStream; + }, + }; + + file.startResumableUpload_(dup); + }); + + it('should set autoRetry to false when ifMetagenerationMatch is undefined', done => { + const dup = duplexify(); + const uploadStream = new PassThrough(); + + dup.setWritable = (stream: Duplex) => { + assert.strictEqual(stream, uploadStream); + done(); + }; + + resumableUploadOverride = { + upload(options_: resumableUpload.UploadConfig) { + assert.strictEqual(options_?.retryOptions?.autoRetry, false); + return uploadStream; + }, + }; + + file.startResumableUpload_(dup, {retryOptions: {autoRetry: true}}); + assert.strictEqual(file.retryOptions.autoRetry, true); + }); + }); + }); + + describe('startSimpleUpload_', () => { + it('should get a writable stream', done => { + makeWritableStreamOverride = () => { + done(); + }; + + file.startSimpleUpload_(duplexify()); + }); + + it('should pass the required arguments', done => { + const options = { + metadata: {}, + predefinedAcl: 'allUsers', + private: true, + public: true, + timeout: 99, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + makeWritableStreamOverride = (stream: {}, options_: any) => { + assert.deepStrictEqual(options_.metadata, options.metadata); + assert.deepStrictEqual(options_.request, { + [GCCL_GCS_CMD_KEY]: undefined, + qs: { + name: file.name, + predefinedAcl: options.predefinedAcl, + }, + timeout: options.timeout, + uri: + 'https://storage.googleapis.com/upload/storage/v1/b/' + + file.bucket.name + + '/o', + }); + done(); + }; + + file.startSimpleUpload_(duplexify(), options); + }); + + it('should set predefinedAcl when public: true', done => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + makeWritableStreamOverride = (stream: {}, options_: any) => { + assert.strictEqual(options_.request.qs.predefinedAcl, 'publicRead'); + done(); + }; + + file.startSimpleUpload_(duplexify(), {public: true}); + }); + + it('should set predefinedAcl when private: true', done => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + makeWritableStreamOverride = (stream: {}, options_: any) => { + assert.strictEqual(options_.request.qs.predefinedAcl, 'private'); + done(); + }; + + file.startSimpleUpload_(duplexify(), {private: true}); + }); + + it('should send query.ifGenerationMatch if File has one', done => { + const versionedFile = new File(BUCKET, 'new-file.txt', {generation: 1}); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + makeWritableStreamOverride = (stream: {}, options: any) => { + assert.strictEqual(options.request.qs.ifGenerationMatch, 1); + done(); + }; + + versionedFile.startSimpleUpload_(duplexify(), {}); + }); + + it('should send query.kmsKeyName if File has one', done => { + file.kmsKeyName = 'kms-key-name'; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + makeWritableStreamOverride = (stream: {}, options: any) => { + assert.strictEqual(options.request.qs.kmsKeyName, file.kmsKeyName); + done(); + }; + + file.startSimpleUpload_(duplexify(), {}); + }); + + it('should send userProject if set', done => { + const options = { + userProject: 'user-project-id', + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + makeWritableStreamOverride = (stream: {}, options_: any) => { + assert.strictEqual( + options_.request.qs.userProject, + options.userProject + ); + done(); + }; + + file.startSimpleUpload_(duplexify(), options); + }); + + describe('request', () => { + describe('error', () => { + const error = new Error('Error.'); + + beforeEach(() => { + file.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error); + }; + }); + + it('should destroy the stream', done => { + const stream = duplexify(); + + file.startSimpleUpload_(stream); + + stream.on('error', (err: Error) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + assert.strictEqual((stream as any).destroyed, true); + assert.strictEqual(err, error); + done(); + }); + }); + }); + + describe('success', () => { + const body = {}; + const resp = {}; + + beforeEach(() => { + file.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, body, resp); + }; + }); + + it('should set the metadata', () => { + const stream = duplexify(); + + file.startSimpleUpload_(stream); + + assert.strictEqual(file.metadata, body); + }); + + it('should emit the response', done => { + const stream = duplexify(); + + stream.on('response', resp_ => { + assert.strictEqual(resp_, resp); + done(); + }); + + file.startSimpleUpload_(stream); + }); + + it('should emit complete', done => { + const stream = duplexify(); + + stream.on('complete', done); + + file.startSimpleUpload_(stream); + }); + }); + }); + }); + + describe('setUserProject', () => { + it('should call the parent setUserProject function', done => { + const userProject = 'grape-spaceship-123'; + + file.parent.setUserProject = function (userProject_: string) { + assert.strictEqual(this, file); + assert.strictEqual(userProject_, userProject); + done(); + }; + + file.setUserProject(userProject); + }); + }); + + describe('from', () => { + it('should create a File object from a gs:// formatted URL', () => { + const gsUrl = 'gs://mybucket/myfile'; + const result = File.from(gsUrl, STORAGE); + + assert(result); + assert.strictEqual(result.bucket.name, 'mybucket'); + assert.strictEqual(result.name, 'myfile'); + }); + + it('should create a File object from a gs:// formatted URL including a folder', () => { + const gsUrl = 'gs://mybucket/myfolder/myfile'; + const result = File.from(gsUrl, STORAGE); + + assert(result); + assert.strictEqual(result.bucket.name, 'mybucket'); + assert.strictEqual(result.name, 'myfolder/myfile'); + }); + + it('should create a File object from a https:// formatted URL', () => { + const httpsUrl = 'https://storage.googleapis.com/mybucket/myfile'; + const result = File.from(httpsUrl, STORAGE); + + assert(result); + assert.strictEqual(result.bucket.name, 'mybucket'); + assert.strictEqual(result.name, 'myfile'); + }); + + it('should create a File object from a https:// formatted URL including a folder', () => { + const httpsUrl = + 'https://storage.googleapis.com/mybucket/myfolder/myfile'; + const result = File.from(httpsUrl, STORAGE); + + assert(result); + assert.strictEqual(result.bucket.name, 'mybucket'); + assert.strictEqual(result.name, 'myfolder/myfile'); + }); + + it('should throw an error when invoked with an incorrectly formatted URL', () => { + const invalidUrl = 'https://storage.com/mybucket/myfile'; + + assert.throws(() => File.from(invalidUrl, STORAGE)); + }); + }); +}); diff --git a/handwritten/storage/test/fixtures/keys.json b/handwritten/storage/test/fixtures/keys.json new file mode 100644 index 00000000000..073edb70713 --- /dev/null +++ b/handwritten/storage/test/fixtures/keys.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "project-id", + "private_key_id": "12345", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC5z21IhrvlHBj7\nifRhobA9ibn25Od7DpE5OauGmqy7B+A9LQOsk1ZujAFdHItnBPcjihSVHpiYxf1a\nLpFbM8z/hRvDvYS3Hs1pyRejmpGiznoOjCyUf6Wv3T1xKelbgn0twHHjqD1o0xzW\njyUILl7yuCbsAf8QlsV6ewS3IqO3i5A9RNHfKjeap8e6A7U3s9QBtR58RrxaMQpM\nz72gw7yOdJRfElkerQfyZTbtu/EBfE6CcskOyoMoRN3YgkjQqLMr1yVdL5/phEcQ\n5hpbDN5lrafGHN7FUtsrMge2iIuIYFfWQUTqu7HtnNXVmwj1LJNq5WeI1iInWaGz\nb7c1rUT9AgMBAAECggEAEB0FicqVX3L7qk9LsBkeItgKFnfB/eaaKsTuM7K/fqCv\njjPpzlIgprQ20g+i+dYbuytC9Fjo5tFp/SNuBji3Ha7kuil56Yoe9NOJh0M6P0zP\nQj+0W1Rj1p0bB5tDhLoLh6eEDjgNde+zioUeCFhCck4MogmHnbVVfspNnba/99oD\nl36heAioqj/KODdkbQ83+ByiH+3BzqblqJ4TR/3y0wOUXtlQvCHko1qximJFIM0z\n3TNoPiit74hTiFFOYfJyHpmRsiEJ5FUUImkmCJz2gk4fbpafKrgxxOMo1m7GqlsE\nE+ybHxyAq61HYbZOoUOO8B4md1/52QXP7DgPvV7JyQKBgQD+JS5nsR4TXRl61c9G\nNxoPW9yCMCoarIjkpyPmhh0uJ7y68cj9wHFgX6ATi1QuTnG9BzJ4z27PMgvv70N+\nAK6k74sdIT2ts8wYsD8H0UyuxDxeKiAnb2JW2f5GTcXNmELQi6rKkMNMoS8jv00d\ngzLCV7UbCbdf+ng9uRPs+Fvk9wKBgQC7KpNaeYFf5dmIYRWQhlZWBRoftdm1ROH/\n5GJsURkzlEjUH1g1y9eAigBn5I+Z9hylX2q1vHLpUHqONWwDz8oQ1L1o2iLz+tkp\nkNoaLSAb9uCl6t8tpqCG2dqUrxOmy1+xj3G8KI8XuYb+IwVSy6KK2df8fWN4d+i0\ng+TBb75MqwKBgEezwcXriKq554hqblJHFYkjx7DLWfWwm+a26UAOsojlGTA9KxG8\ni8A++nDJLHTsGNbWAv1muMKoQgntnUMdeih6lOshB7/MLFcC0qWn/VSJdOa0R+IY\nYMxUMJMxOg9pV+BypzsDYLZr+1rAjEc5TsbZ6/S25w+jIO15HBANeg+9AoGAZulz\nGkVDCLq2UJGpLM1gvW2Svqrb6RrV9UDbiVlSNRUssk4Fz5akiM3YiUeYWfyEJb4A\nS6sxt+4DZRwkpzfikDyZZQTEQUjFjWBTPB9hz16AiVpKmqxLCbrRv/1AHe8nT9di\nnyXiABaIDkatT6geWKCNbQx43C16a382EdJiXX8CgYEAqyAS2xuDi2+uoljRm1Bp\naz7Q2UBtBbcBr/CQmagEacWPXsSyCL6EySOH0e985k7ABZiW+AzWlOwKS5WMWAIb\nkncmxP0SU6WQDWl8xGbXAQ8Dw+HTu5G1n0vrl1rRO5FPwRs3pbV94ML+d5eoai6D\njHs1asOGIpdQ3OGpBpNRub0=\n-----END PRIVATE KEY-----\n", + "client_email": "some-email@example.com", + "client_id": "12345", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/el-gato%40el-gato.iam.gserviceaccount.com" +} diff --git a/handwritten/storage/test/headers.ts b/handwritten/storage/test/headers.ts new file mode 100644 index 00000000000..9ccc685814b --- /dev/null +++ b/handwritten/storage/test/headers.ts @@ -0,0 +1,96 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as assert from 'assert'; +import {describe, it} from 'mocha'; +import proxyquire from 'proxyquire'; + +const error = Error('not implemented'); + +interface Request { + headers: { + [key: string]: string; + }; +} + +describe('headers', () => { + const requests: Request[] = []; + const {Storage} = proxyquire('../src', { + 'google-auth-library': { + GoogleAuth: class { + async getProjectId() { + return 'foo-project'; + } + async getClient() { + return class { + async request() { + return {}; + } + }; + } + getCredentials() { + return {}; + } + async authorizeRequest(req: Request) { + requests.push(req); + throw error; + } + }, + '@global': true, + }, + }); + + afterEach(() => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + globalThis.Deno = undefined; + }); + + it('populates x-goog-api-client header (node)', async () => { + const storage = new Storage(); + const bucket = storage.bucket('foo-bucket'); + try { + await bucket.create(); + } catch (err) { + if (err !== error) throw err; + } + assert.ok( + /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( + requests[0].headers['x-goog-api-client'] + ) + ); + }); + + it('populates x-goog-api-client header (deno)', async () => { + const storage = new Storage(); + const bucket = storage.bucket('foo-bucket'); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + globalThis.Deno = { + version: { + deno: '0.00.0', + }, + }; + try { + await bucket.create(); + } catch (err) { + if (err !== error) throw err; + } + assert.ok( + /^gl-deno\/0.00.0 gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( + requests[1].headers['x-goog-api-client'] + ) + ); + }); +}); diff --git a/handwritten/storage/test/hmacKey.ts b/handwritten/storage/test/hmacKey.ts new file mode 100644 index 00000000000..309b988358b --- /dev/null +++ b/handwritten/storage/test/hmacKey.ts @@ -0,0 +1,109 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as sinon from 'sinon'; +import proxyquire from 'proxyquire'; +import assert from 'assert'; +import {describe, it, beforeEach, afterEach} from 'mocha'; +import {util, ServiceObject} from '../src/nodejs-common/index.js'; +import {HmacKeyMetadata, IdempotencyStrategy} from '../src/index.js'; + +const sandbox = sinon.createSandbox(); +// eslint-disable-next-line @typescript-eslint/no-explicit-any +let STORAGE: any; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +let hmacKey: any; + +const ACCESS_ID = 'fake-access-id'; + +class HTTPError extends Error { + code: number; + constructor(message: string, code: number) { + super(message); + this.code = code; + } +} + +describe('HmacKey', () => { + afterEach(() => sandbox.restore()); + + describe('initialization', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let serviceObjectSpy: sinon.SinonSpy; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let commonModule: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let HmacKey: any; + + beforeEach(() => { + commonModule = {ServiceObject}; + serviceObjectSpy = sandbox.spy(commonModule, 'ServiceObject'); + + HmacKey = proxyquire('../src/hmacKey', { + './nodejs-common': commonModule, + }).HmacKey; + + STORAGE = { + request: util.noop, + projectId: 'my-project', + retryOptions: { + autoRetry: true, + maxRetries: 3, + retryDelayMultiplier: 2, + totalTimeout: 600, + maxRetryDelay: 60, + retryableErrorFn: (err: HTTPError) => { + return err.code === 500; + }, + idempotencyStrategy: IdempotencyStrategy.RetryConditional, + }, + }; + + hmacKey = new HmacKey(STORAGE, ACCESS_ID); + }); + + it('should inherit from ServiceObject', () => { + assert(hmacKey instanceof ServiceObject); + const ctorArg = serviceObjectSpy.firstCall.args[0]; + assert(ctorArg.parent, STORAGE); + assert(ctorArg.id, ACCESS_ID); + assert(ctorArg.baseUrl, '/projects/my-project/hmacKeys'); + assert.deepStrictEqual(ctorArg.methods, { + delete: true, + get: true, + getMetadata: true, + setMetadata: { + reqOpts: { + method: 'PUT', + }, + }, + }); + }); + + it('should form baseUrl using options.projectId if given', () => { + hmacKey = new HmacKey(STORAGE, ACCESS_ID, {projectId: 'another-project'}); + const ctorArg = serviceObjectSpy.firstCall.args[0]; + assert(ctorArg.baseUrl, '/projects/another-project/hmacKeys'); + }); + + it('should correctly call setMetadata', done => { + hmacKey.setMetadata = (metadata: HmacKeyMetadata, callback: Function) => { + assert.deepStrictEqual(metadata.accessId, ACCESS_ID); + Promise.resolve([]).then(resp => callback(null, ...resp)); + }; + + hmacKey.setMetadata({accessId: ACCESS_ID}, done); + }); + }); +}); diff --git a/handwritten/storage/test/iam.ts b/handwritten/storage/test/iam.ts new file mode 100644 index 00000000000..92327daa614 --- /dev/null +++ b/handwritten/storage/test/iam.ts @@ -0,0 +1,286 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {DecorateRequestOptions, util} from '../src/nodejs-common/index.js'; +import assert from 'assert'; +import {describe, it, before, beforeEach} from 'mocha'; +import proxyquire from 'proxyquire'; +import {IAMExceptionMessages} from '../src/iam.js'; + +describe('storage/iam', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let Iam: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let iam: any; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let BUCKET_INSTANCE: any; + let promisified = false; + const fakePromisify = { + // tslint:disable-next-line:variable-name + promisifyAll(Class: Function) { + if (Class.name === 'Iam') { + promisified = true; + } + }, + }; + + before(() => { + Iam = proxyquire('../src/iam.js', { + '@google-cloud/promisify': fakePromisify, + }).Iam; + }); + + beforeEach(() => { + const id = 'bucket-id'; + BUCKET_INSTANCE = { + id, + request: util.noop, + getId: () => id, + }; + + iam = new Iam(BUCKET_INSTANCE); + }); + + describe('initialization', () => { + it('should promisify all the things', () => { + assert(promisified); + }); + + it('should localize the request function', done => { + Object.assign(BUCKET_INSTANCE, { + request(callback: Function) { + assert.strictEqual(this, BUCKET_INSTANCE); + callback(); // done() + }, + }); + + const iam = new Iam(BUCKET_INSTANCE); + iam.request_(done); + }); + + it('should localize the resource ID', () => { + assert.strictEqual(iam.resourceId_, 'buckets/' + BUCKET_INSTANCE.id); + }); + }); + + describe('getPolicy', () => { + it('should make the correct api request', done => { + iam.request_ = (reqOpts: DecorateRequestOptions, callback: Function) => { + assert.deepStrictEqual(reqOpts, { + uri: '/iam', + qs: {}, + }); + + callback(); // done() + }; + + iam.getPolicy(done); + }); + + it('should accept an options object', done => { + const options = { + userProject: 'grape-spaceship-123', + }; + + iam.request_ = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.qs, options); + done(); + }; + + iam.getPolicy(options, assert.ifError); + }); + + it('should map requestedPolicyVersion option to optionsRequestedPolicyVersion', done => { + const VERSION = 3; + const options = { + requestedPolicyVersion: VERSION, + }; + + iam.request_ = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.qs, { + optionsRequestedPolicyVersion: VERSION, + }); + done(); + }; + + iam.getPolicy(options, assert.ifError); + }); + }); + + describe('setPolicy', () => { + it('should throw an error if a policy is not supplied', () => { + assert.throws(() => { + iam.setPolicy(util.noop), IAMExceptionMessages.POLICY_OBJECT_REQUIRED; + }); + }); + + it('should make the correct API request', done => { + const policy = { + a: 'b', + }; + + iam.request_ = (reqOpts: DecorateRequestOptions, callback: Function) => { + assert.deepStrictEqual(reqOpts, { + method: 'PUT', + uri: '/iam', + maxRetries: 0, + json: Object.assign( + { + resourceId: iam.resourceId_, + }, + policy + ), + qs: {}, + }); + + callback(); // done() + }; + + iam.setPolicy(policy, done); + }); + + it('should accept an options object', done => { + const policy = { + a: 'b', + }; + + const options = { + userProject: 'grape-spaceship-123', + }; + + iam.request_ = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs, options); + done(); + }; + + iam.setPolicy(policy, options, assert.ifError); + }); + }); + + describe('testPermissions', () => { + it('should throw an error if permissions are missing', () => { + assert.throws(() => { + iam.testPermissions(util.noop), + IAMExceptionMessages.PERMISSIONS_REQUIRED; + }); + }); + + it('should make the correct API request', done => { + const permissions = 'storage.bucket.list'; + + iam.request_ = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts, { + uri: '/iam/testPermissions', + qs: { + permissions: [permissions], + }, + useQuerystring: true, + }); + + done(); + }; + + iam.testPermissions(permissions, assert.ifError); + }); + + it('should send an error back if the request fails', done => { + const permissions = ['storage.bucket.list']; + const error = new Error('Error.'); + const apiResponse = {}; + + iam.request_ = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(error, apiResponse); + }; + + iam.testPermissions( + permissions, + (err: Error, permissions: Array<{}>, apiResp: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(permissions, null); + assert.strictEqual(apiResp, apiResponse); + done(); + } + ); + }); + + it('should pass back a hash of permissions the user has', done => { + const permissions = ['storage.bucket.list', 'storage.bucket.consume']; + const apiResponse = { + permissions: ['storage.bucket.consume'], + }; + + iam.request_ = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, apiResponse); + }; + + iam.testPermissions( + permissions, + (err: Error, permissions: Array<{}>, apiResp: {}) => { + assert.ifError(err); + assert.deepStrictEqual(permissions, { + 'storage.bucket.list': false, + 'storage.bucket.consume': true, + }); + assert.strictEqual(apiResp, apiResponse); + + done(); + } + ); + }); + + it('should return false for supplied permissions if user has no permissions', done => { + const permissions = ['storage.bucket.list', 'storage.bucket.consume']; + const apiResponse = {permissions: undefined}; + + iam.request_ = (reqOpts: DecorateRequestOptions, callback: Function) => { + callback(null, apiResponse); + }; + iam.testPermissions( + permissions, + (err: Error, permissions: Array<{}>, apiResp: {}) => { + assert.ifError(err); + assert.deepStrictEqual(permissions, { + 'storage.bucket.list': false, + 'storage.bucket.consume': false, + }); + assert.strictEqual(apiResp, apiResponse); + + done(); + } + ); + }); + + it('should accept an options object', done => { + const permissions = ['storage.bucket.list']; + const options = { + userProject: 'grape-spaceship-123', + }; + + const expectedQuery = Object.assign( + { + permissions, + }, + options + ); + + iam.request_ = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.qs, expectedQuery); + done(); + }; + + iam.testPermissions(permissions, options, assert.ifError); + }); + }); +}); diff --git a/handwritten/storage/test/index.ts b/handwritten/storage/test/index.ts new file mode 100644 index 00000000000..c7fbed8467b --- /dev/null +++ b/handwritten/storage/test/index.ts @@ -0,0 +1,1550 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + ApiError, + DecorateRequestOptions, + Service, + ServiceConfig, + util, +} from '../src/nodejs-common/index.js'; +import {PromisifyAllOptions} from '@google-cloud/promisify'; +import assert from 'assert'; +import {describe, it, before, beforeEach, after, afterEach} from 'mocha'; +import proxyquire from 'proxyquire'; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import {Bucket, CRC32C_DEFAULT_VALIDATOR_GENERATOR} from '../src/index.js'; +import {GetFilesOptions} from '../src/bucket.js'; +import * as sinon from 'sinon'; +import {HmacKey} from '../src/hmacKey.js'; +import { + HmacKeyResourceResponse, + PROTOCOL_REGEX, + StorageExceptionMessages, +} from '../src/storage.js'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import {getPackageJSON} from '../src/package-json-helper.cjs'; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const hmacKeyModule = require('../src/hmacKey'); + +class FakeChannel { + calledWith_: Array<{}>; + constructor(...args: Array<{}>) { + this.calledWith_ = args; + } +} + +class FakeService extends Service { + calledWith_: Array<{}>; + constructor(...args: Array<{}>) { + super(args[0] as ServiceConfig); + this.calledWith_ = args; + } +} + +let extended = false; +const fakePaginator = { + paginator: { + // tslint:disable-next-line:variable-name + extend(Class: Function, methods: string[]) { + if (Class.name !== 'Storage') { + return; + } + + assert.strictEqual(Class.name, 'Storage'); + assert.deepStrictEqual(methods, ['getBuckets', 'getHmacKeys']); + extended = true; + }, + streamify(methodName: string) { + return methodName; + }, + }, +}; + +let promisified = false; +const fakePromisify = { + // tslint:disable-next-line:variable-name + promisifyAll(Class: Function, options: PromisifyAllOptions) { + if (Class.name !== 'Storage') { + return; + } + + promisified = true; + assert.deepStrictEqual(options.exclude, ['bucket', 'channel', 'hmacKey']); + }, +}; + +describe('Storage', () => { + const PROJECT_ID = 'project-id'; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let Storage: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let storage: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let Bucket: any; + + before(() => { + Storage = proxyquire('../src/storage', { + '@google-cloud/promisify': fakePromisify, + '@google-cloud/paginator': fakePaginator, + './nodejs-common': { + Service: FakeService, + }, + './channel.js': {Channel: FakeChannel}, + './hmacKey': hmacKeyModule, + }).Storage; + Bucket = Storage.Bucket; + }); + + beforeEach(() => { + storage = new Storage({projectId: PROJECT_ID}); + }); + + describe('instantiation', () => { + it('should extend the correct methods', () => { + assert(extended); // See `fakePaginator.extend` + }); + + it('should streamify the correct methods', () => { + assert.strictEqual(storage.getBucketsStream, 'getBuckets'); + assert.strictEqual(storage.getHmacKeysStream, 'getHmacKeys'); + }); + + it('should promisify all the things', () => { + assert(promisified); + }); + + it('should inherit from Service', () => { + // Using assert.strictEqual instead of assert to prevent + // coercing of types. + assert.strictEqual(storage instanceof Service, true); + + const calledWith = storage.calledWith_[0]; + + const baseUrl = 'https://storage.googleapis.com/storage/v1'; + assert.strictEqual(calledWith.baseUrl, baseUrl); + assert.strictEqual(calledWith.projectIdRequired, false); + assert.deepStrictEqual(calledWith.scopes, [ + 'https://www.googleapis.com/auth/iam', + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/devstorage.full_control', + ]); + assert.deepStrictEqual( + calledWith.packageJson, + // eslint-disable-next-line @typescript-eslint/no-var-requires + getPackageJSON() + ); + }); + + it('should not modify options argument', () => { + const options = { + projectId: PROJECT_ID, + }; + const expectedCalledWith = Object.assign({}, options, { + apiEndpoint: 'https://storage.googleapis.com', + }); + const storage = new Storage(options); + const calledWith = storage.calledWith_[1]; + assert.notStrictEqual(calledWith, options); + assert.notDeepStrictEqual(calledWith, options); + assert.deepStrictEqual(calledWith, expectedCalledWith); + }); + + it('should propagate the apiEndpoint option', () => { + const apiEndpoint = 'https://some.fake.endpoint'; + const storage = new Storage({ + projectId: PROJECT_ID, + apiEndpoint, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.baseUrl, `${apiEndpoint}/storage/v1`); + assert.strictEqual(calledWith.apiEndpoint, `${apiEndpoint}`); + }); + + it('should not set `customEndpoint` if `apiEndpoint` matches default', () => { + const apiEndpoint = 'https://storage.googleapis.com'; + const storage = new Storage({ + apiEndpoint, + }); + + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.apiEndpoint, apiEndpoint); + assert.strictEqual(calledWith.customEndpoint, false); + }); + + it('should not set `customEndpoint` if `apiEndpoint` matches default (w/ universe domain)', () => { + const universeDomain = 'my.universe'; + const apiEndpoint = `https://storage.${universeDomain}`; + const storage = new Storage({ + apiEndpoint, + universeDomain, + }); + + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.apiEndpoint, apiEndpoint); + assert.strictEqual(calledWith.customEndpoint, false); + }); + + it('should propagate the useAuthWithCustomEndpoint option', () => { + const useAuthWithCustomEndpoint = true; + const apiEndpoint = 'https://some.fake.endpoint'; + const storage = new Storage({ + projectId: PROJECT_ID, + useAuthWithCustomEndpoint, + apiEndpoint, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.apiEndpoint, 'https://some.fake.endpoint'); + assert.strictEqual(calledWith.customEndpoint, true); + assert.strictEqual(calledWith.useAuthWithCustomEndpoint, true); + }); + + it('should propagate autoRetry in retryOptions', () => { + const autoRetry = false; + const storage = new Storage({ + projectId: PROJECT_ID, + retryOptions: {autoRetry}, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.retryOptions.autoRetry, autoRetry); + }); + + it('should propagate retryDelayMultiplier', () => { + const retryDelayMultiplier = 4; + const storage = new Storage({ + projectId: PROJECT_ID, + retryOptions: {retryDelayMultiplier}, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual( + calledWith.retryOptions.retryDelayMultiplier, + retryDelayMultiplier + ); + }); + + it('should propagate totalTimeout', () => { + const totalTimeout = 60; + const storage = new Storage({ + projectId: PROJECT_ID, + retryOptions: {totalTimeout}, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.retryOptions.totalTimeout, totalTimeout); + }); + + it('should propagate maxRetryDelay', () => { + const maxRetryDelay = 640; + const storage = new Storage({ + projectId: PROJECT_ID, + retryOptions: {maxRetryDelay}, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.retryOptions.maxRetryDelay, maxRetryDelay); + }); + + it('should set correct defaults for retry configs', () => { + const autoRetryDefault = true; + const maxRetryDefault = 3; + const retryDelayMultiplierDefault = 2; + const totalTimeoutDefault = 600; + const maxRetryDelayDefault = 64; + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.retryOptions.autoRetry, autoRetryDefault); + assert.strictEqual(calledWith.retryOptions.maxRetries, maxRetryDefault); + assert.strictEqual( + calledWith.retryOptions.retryDelayMultiplier, + retryDelayMultiplierDefault + ); + assert.strictEqual( + calledWith.retryOptions.totalTimeout, + totalTimeoutDefault + ); + assert.strictEqual( + calledWith.retryOptions.maxRetryDelay, + maxRetryDelayDefault + ); + }); + + it('should propagate maxRetries in retryOptions', () => { + const maxRetries = 1; + const storage = new Storage({ + projectId: PROJECT_ID, + retryOptions: {maxRetries}, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.retryOptions.maxRetries, maxRetries); + }); + + it('should set retryFunction', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + assert(calledWith.retryOptions.retryableErrorFn); + }); + + it('should retry a 502 error', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + const error = new ApiError('502 Error'); + error.code = 502; + assert.strictEqual(calledWith.retryOptions.retryableErrorFn(error), true); + }); + + it('should not retry blank error', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + const error = undefined; + assert.strictEqual( + calledWith.retryOptions.retryableErrorFn(error), + false + ); + }); + + it('should retry a reset connection error', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + const error = new ApiError('Connection Reset By Peer error'); + error.errors = [ + { + reason: 'ECONNRESET', + }, + ]; + assert.strictEqual(calledWith.retryOptions.retryableErrorFn(error), true); + }); + + it('should retry a broken pipe error', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + const error = new ApiError('Broken pipe'); + error.errors = [ + { + reason: 'EPIPE', + }, + ]; + assert.strictEqual(calledWith.retryOptions.retryableErrorFn(error), true); + }); + + it('should retry a socket connection timeout', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + const error = new ApiError('Broken pipe'); + const innerError = { + /** + * @link https://nodejs.org/api/errors.html#err_socket_connection_timeout + * @link https://github.com/nodejs/node/blob/798db3c92a9b9c9f991eed59ce91e9974c052bc9/lib/internal/errors.js#L1570-L1571 + */ + reason: 'Socket connection timeout', + }; + + error.errors = [innerError]; + assert.strictEqual(calledWith.retryOptions.retryableErrorFn(error), true); + }); + + it('should not retry a 999 error', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + const error = new ApiError('999 Error'); + error.code = 0; + assert.strictEqual( + calledWith.retryOptions.retryableErrorFn(error), + false + ); + }); + + it('should return false if reason and code are both undefined', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + const error = new ApiError('error without a code'); + error.errors = [ + { + message: 'some error message', + }, + ]; + assert.strictEqual( + calledWith.retryOptions.retryableErrorFn(error), + false + ); + }); + + it('should retry a 999 error if dictated by custom function', () => { + const customRetryFunc = function (err?: ApiError) { + if (err) { + if ([999].indexOf(err.code!) !== -1) { + return true; + } + } + return false; + }; + const storage = new Storage({ + projectId: PROJECT_ID, + retryOptions: {retryableErrorFn: customRetryFunc}, + }); + const calledWith = storage.calledWith_[0]; + const error = new ApiError('999 Error'); + error.code = 999; + assert.strictEqual(calledWith.retryOptions.retryableErrorFn(error), true); + }); + + it('should set customEndpoint to true when using apiEndpoint', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + apiEndpoint: 'https://apiendpoint', + }); + + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.customEndpoint, true); + }); + + it('should prepend apiEndpoint with default protocol', () => { + const protocollessApiEndpoint = 'some.fake.endpoint'; + const storage = new Storage({ + projectId: PROJECT_ID, + apiEndpoint: protocollessApiEndpoint, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual( + calledWith.baseUrl, + `https://${protocollessApiEndpoint}/storage/v1` + ); + assert.strictEqual( + calledWith.apiEndpoint, + `https://${protocollessApiEndpoint}` + ); + }); + + it('should strip trailing slash from apiEndpoint', () => { + const apiEndpoint = 'https://some.fake.endpoint/'; + const storage = new Storage({ + projectId: PROJECT_ID, + apiEndpoint, + }); + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.baseUrl, `${apiEndpoint}storage/v1`); + assert.strictEqual(calledWith.apiEndpoint, 'https://some.fake.endpoint'); + }); + + it('should accept a `crc32cGenerator`', () => { + const crc32cGenerator = () => {}; + + const storage = new Storage({crc32cGenerator}); + assert.strictEqual(storage.crc32cGenerator, crc32cGenerator); + }); + + it('should use `CRC32C_DEFAULT_VALIDATOR_GENERATOR` by default', () => { + assert.strictEqual( + storage.crc32cGenerator, + CRC32C_DEFAULT_VALIDATOR_GENERATOR + ); + }); + + it('should accept and use a `universeDomain`', () => { + const universeDomain = 'my-universe.com'; + + const storage = new Storage({universeDomain}); + + assert.equal(storage.apiEndpoint, `https://storage.${universeDomain}`); + }); + + describe('STORAGE_EMULATOR_HOST', () => { + // Note: EMULATOR_HOST is an experimental configuration variable. Use apiEndpoint instead. + const EMULATOR_HOST = 'https://internal.benchmark.com/path'; + before(() => { + process.env.STORAGE_EMULATOR_HOST = EMULATOR_HOST; + }); + + after(() => { + delete process.env.STORAGE_EMULATOR_HOST; + }); + + it('should set baseUrl to env var STORAGE_EMULATOR_HOST', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.baseUrl, EMULATOR_HOST); + assert.strictEqual( + calledWith.apiEndpoint, + 'https://internal.benchmark.com/path' + ); + }); + + it('should be overridden by apiEndpoint', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + apiEndpoint: 'https://some.api.com', + }); + + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.baseUrl, EMULATOR_HOST); + assert.strictEqual(calledWith.apiEndpoint, 'https://some.api.com'); + }); + + it('should prepend default protocol and strip trailing slash', () => { + const EMULATOR_HOST = 'internal.benchmark.com/path/'; + process.env.STORAGE_EMULATOR_HOST = EMULATOR_HOST; + + const storage = new Storage({ + projectId: PROJECT_ID, + }); + + const calledWith = storage.calledWith_[0]; + assert.strictEqual(calledWith.baseUrl, EMULATOR_HOST); + assert.strictEqual( + calledWith.apiEndpoint, + 'https://internal.benchmark.com/path' + ); + }); + + it('should set customEndpoint to true', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + + assert.strictEqual(storage.customEndpoint, true); + }); + }); + }); + + describe('bucket', () => { + it('should throw if no name was provided', () => { + assert.throws(() => { + storage.bucket(), StorageExceptionMessages.BUCKET_NAME_REQUIRED; + }); + }); + + it('should accept a string for a name', () => { + const newBucketName = 'new-bucket-name'; + const bucket = storage.bucket(newBucketName); + assert(bucket instanceof Bucket); + assert.strictEqual(bucket.name, newBucketName); + }); + + it('should optionally accept options', () => { + const options = { + userProject: 'grape-spaceship-123', + }; + + const bucket = storage.bucket('bucket-name', options); + assert.strictEqual(bucket.userProject, options.userProject); + }); + }); + + describe('channel', () => { + const ID = 'channel-id'; + const RESOURCE_ID = 'resource-id'; + + it('should create a Channel object', () => { + const channel = storage.channel(ID, RESOURCE_ID); + + assert(channel instanceof FakeChannel); + + assert.strictEqual(channel.calledWith_[0], storage); + assert.strictEqual(channel.calledWith_[1], ID); + assert.strictEqual(channel.calledWith_[2], RESOURCE_ID); + }); + }); + + describe('hmacKey', () => { + let hmacKeyCtor: sinon.SinonSpy; + beforeEach(() => { + hmacKeyCtor = sinon.spy(hmacKeyModule, 'HmacKey'); + }); + + afterEach(() => { + hmacKeyCtor.restore(); + }); + + it('should throw if accessId is not provided', () => { + assert.throws(() => { + storage.hmacKey(), StorageExceptionMessages.HMAC_ACCESS_ID; + }); + }); + + it('should pass options object to HmacKey constructor', () => { + const options = {myOpts: 'a'}; + storage.hmacKey('access-id', options); + assert.deepStrictEqual(hmacKeyCtor.getCall(0).args, [ + storage, + 'access-id', + options, + ]); + }); + }); + + describe('createHmacKey', () => { + const SERVICE_ACCOUNT_EMAIL = 'service-account@gserviceaccount.com'; + const ACCESS_ID = 'some-access-id'; + const metadataResponse = { + accessId: ACCESS_ID, + etag: 'etag', + id: ACCESS_ID, + projectId: 'project-id', + serviceAccountEmail: SERVICE_ACCOUNT_EMAIL, + state: 'ACTIVE', + timeCreated: '20190101T00:00:00Z', + updated: '20190101T00:00:00Z', + }; + const response = { + secret: 'my-secret', + metadata: metadataResponse, + }; + const OPTIONS = { + some: 'value', + }; + + let hmacKeyCtor: sinon.SinonSpy; + beforeEach(() => { + hmacKeyCtor = sinon.spy(hmacKeyModule, 'HmacKey'); + }); + + afterEach(() => { + hmacKeyCtor.restore(); + }); + + it('should make correct API request', done => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.method, 'POST'); + assert.strictEqual( + reqOpts.uri, + `/projects/${storage.projectId}/hmacKeys` + ); + assert.strictEqual( + reqOpts.qs.serviceAccountEmail, + SERVICE_ACCOUNT_EMAIL + ); + + callback(null, response); + }; + + storage.createHmacKey(SERVICE_ACCOUNT_EMAIL, done); + }); + + it('should throw without a serviceAccountEmail', () => { + assert.throws(() => { + storage.createHmacKey(), StorageExceptionMessages.HMAC_SERVICE_ACCOUNT; + }); + }); + + it('should throw when first argument is not a string', () => { + assert.throws(() => { + storage.createHmacKey({ + userProject: 'my-project', + }), + StorageExceptionMessages.HMAC_SERVICE_ACCOUNT; + }); + }); + + it('should make request with method options as query parameter', async () => { + storage.request = sinon + .stub() + .returns((_reqOpts: {}, callback: Function) => callback()); + + await storage.createHmacKey(SERVICE_ACCOUNT_EMAIL, OPTIONS); + const reqArg = storage.request.firstCall.args[0]; + assert.deepStrictEqual(reqArg.qs, { + serviceAccountEmail: SERVICE_ACCOUNT_EMAIL, + ...OPTIONS, + }); + }); + + it('should not modify the options object', done => { + storage.request = (_reqOpts: {}, callback: Function) => { + callback(null, response); + }; + const originalOptions = Object.assign({}, OPTIONS); + + storage.createHmacKey(SERVICE_ACCOUNT_EMAIL, OPTIONS, (err: Error) => { + assert.ifError(err); + assert.deepStrictEqual(OPTIONS, originalOptions); + done(); + }); + }); + + it('should invoke callback with a secret and an HmacKey instance', done => { + storage.request = (_reqOpts: {}, callback: Function) => { + callback(null, response); + }; + + storage.createHmacKey( + SERVICE_ACCOUNT_EMAIL, + (err: Error, hmacKey: HmacKey, secret: string) => { + assert.ifError(err); + assert.strictEqual(secret, response.secret); + assert.deepStrictEqual(hmacKeyCtor.getCall(0).args, [ + storage, + response.metadata.accessId, + {projectId: response.metadata.projectId}, + ]); + assert.strictEqual(hmacKey.metadata, metadataResponse); + done(); + } + ); + }); + + it('should invoke callback with raw apiResponse', done => { + storage.request = (_reqOpts: {}, callback: Function) => { + callback(null, response); + }; + + storage.createHmacKey( + SERVICE_ACCOUNT_EMAIL, + ( + err: Error, + _hmacKey: HmacKey, + _secret: string, + apiResponse: HmacKeyResourceResponse + ) => { + assert.ifError(err); + assert.strictEqual(apiResponse, response); + done(); + } + ); + }); + + it('should execute callback with request error', done => { + const error = new Error('Request error'); + const response = {success: false}; + storage.request = (_reqOpts: {}, callback: Function) => { + callback(error, response); + }; + + storage.createHmacKey( + SERVICE_ACCOUNT_EMAIL, + (err: Error, _hmacKey: HmacKey, _secret: string, apiResponse: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(apiResponse, response); + done(); + } + ); + }); + }); + + describe('createBucket', () => { + const BUCKET_NAME = 'new-bucket-name'; + const METADATA = {a: 'b', c: {d: 'e'}}; + const BUCKET = {name: BUCKET_NAME}; + + it('should make correct API request', done => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.method, 'POST'); + assert.strictEqual(reqOpts.uri, '/b'); + assert.strictEqual(reqOpts.qs.project, storage.projectId); + assert.strictEqual(reqOpts.json.name, BUCKET_NAME); + + callback(); + }; + + storage.createBucket(BUCKET_NAME, done); + }); + + it('should accept a name, metadata, and callback', done => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.deepStrictEqual( + reqOpts.json, + Object.assign(METADATA, {name: BUCKET_NAME}) + ); + callback(null, METADATA); + }; + storage.bucket = (name: string) => { + assert.strictEqual(name, BUCKET_NAME); + return BUCKET; + }; + storage.createBucket(BUCKET_NAME, METADATA, (err: Error) => { + assert.ifError(err); + done(); + }); + }); + + it('should accept a name and callback only', done => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(); + }; + storage.createBucket(BUCKET_NAME, done); + }); + + it('should throw if no name is provided', () => { + assert.throws(() => { + storage.createBucket(), + StorageExceptionMessages.BUCKET_NAME_REQUIRED_CREATE; + }); + }); + + it('should honor the userProject option', done => { + const options = { + userProject: 'grape-spaceship-123', + }; + + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs.userProject, options.userProject); + done(); + }; + + storage.createBucket(BUCKET_NAME, options, assert.ifError); + }); + + it('should execute callback with bucket', done => { + storage.bucket = () => { + return BUCKET; + }; + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, METADATA); + }; + storage.createBucket(BUCKET_NAME, (err: Error, bucket: Bucket) => { + assert.ifError(err); + assert.deepStrictEqual(bucket, BUCKET); + assert.deepStrictEqual(bucket.metadata, METADATA); + done(); + }); + }); + + it('should execute callback on error', done => { + const error = new Error('Error.'); + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error); + }; + storage.createBucket(BUCKET_NAME, (err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should execute callback with apiResponse', done => { + const resp = {success: true}; + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, resp); + }; + storage.createBucket( + BUCKET_NAME, + (err: Error, bucket: Bucket, apiResponse: unknown) => { + assert.strictEqual(resp, apiResponse); + done(); + } + ); + }); + + it('should allow a user-specified storageClass', done => { + const storageClass = 'nearline'; + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.json.storageClass, storageClass); + callback(); // done + }; + storage.createBucket(BUCKET_NAME, {storageClass}, done); + }); + + it('should allow settings `storageClass` to same value as provided storage class name', done => { + const storageClass = 'coldline'; + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual( + reqOpts.json.storageClass, + storageClass.toUpperCase() + ); + callback(); // done + }; + + assert.doesNotThrow(() => { + storage.createBucket( + BUCKET_NAME, + {storageClass, [storageClass]: true}, + done + ); + }); + }); + + it('should allow setting rpo', done => { + const location = 'NAM4'; + const rpo = 'ASYNC_TURBO'; + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.json.location, location); + assert.strictEqual(reqOpts.json.rpo, rpo); + callback(); + }; + storage.createBucket(BUCKET_NAME, {location, rpo}, done); + }); + + it('should throw when `storageClass` is set to different value than provided storageClass name', () => { + assert.throws(() => { + storage.createBucket( + BUCKET_NAME, + { + storageClass: 'nearline', + coldline: true, + }, + assert.ifError + ); + }, /Both `coldline` and `storageClass` were provided./); + }); + + it('should allow enabling object retention', done => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.qs.enableObjectRetention, true); + callback(); + }; + storage.createBucket(BUCKET_NAME, {enableObjectRetention: true}, done); + }); + + it('should allow enabling hierarchical namespace', done => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.json.hierarchicalNamespace.enabled, true); + callback(); + }; + storage.createBucket( + BUCKET_NAME, + {hierarchicalNamespace: {enabled: true}}, + done + ); + }); + + describe('storage classes', () => { + it('should expand metadata.archive', done => { + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.json.storageClass, 'ARCHIVE'); + done(); + }; + + storage.createBucket(BUCKET_NAME, {archive: true}, assert.ifError); + }); + + it('should expand metadata.coldline', done => { + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.json.storageClass, 'COLDLINE'); + done(); + }; + + storage.createBucket(BUCKET_NAME, {coldline: true}, assert.ifError); + }); + + it('should expand metadata.dra', done => { + storage.request = (reqOpts: DecorateRequestOptions) => { + const body = reqOpts.json; + assert.strictEqual(body.storageClass, 'DURABLE_REDUCED_AVAILABILITY'); + done(); + }; + + storage.createBucket(BUCKET_NAME, {dra: true}, assert.ifError); + }); + + it('should expand metadata.multiRegional', done => { + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.json.storageClass, 'MULTI_REGIONAL'); + done(); + }; + + storage.createBucket( + BUCKET_NAME, + { + multiRegional: true, + }, + assert.ifError + ); + }); + + it('should expand metadata.nearline', done => { + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.json.storageClass, 'NEARLINE'); + done(); + }; + + storage.createBucket(BUCKET_NAME, {nearline: true}, assert.ifError); + }); + + it('should expand metadata.regional', done => { + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.json.storageClass, 'REGIONAL'); + done(); + }; + + storage.createBucket(BUCKET_NAME, {regional: true}, assert.ifError); + }); + + it('should expand metadata.standard', done => { + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.json.storageClass, 'STANDARD'); + done(); + }; + + storage.createBucket(BUCKET_NAME, {standard: true}, assert.ifError); + }); + }); + + describe('requesterPays', () => { + it('should accept requesterPays setting', done => { + const options = { + requesterPays: true, + }; + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.json.billing, options); + assert.strictEqual(reqOpts.json.requesterPays, undefined); + done(); + }; + storage.createBucket(BUCKET_NAME, options, assert.ifError); + }); + }); + }); + + describe('getBuckets', () => { + it('should get buckets without a query', done => { + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, '/b'); + assert.deepStrictEqual(reqOpts.qs, {project: storage.projectId}); + done(); + }; + storage.getBuckets(util.noop); + }); + + it('should get buckets with a query', done => { + const token = 'next-page-token'; + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.qs, { + project: storage.projectId, + maxResults: 5, + pageToken: token, + }); + done(); + }; + storage.getBuckets({maxResults: 5, pageToken: token}, util.noop); + }); + + it('should execute callback with error', done => { + const error = new Error('Error.'); + const apiResponse = {}; + + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error, apiResponse); + }; + + storage.getBuckets( + {}, + (err: Error, buckets: Bucket[], nextQuery: {}, resp: unknown) => { + assert.strictEqual(err, error); + assert.strictEqual(buckets, null); + assert.strictEqual(nextQuery, null); + assert.strictEqual(resp, apiResponse); + done(); + } + ); + }); + + it('should return nextQuery if more results exist', () => { + const token = 'next-page-token'; + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, {nextPageToken: token, items: []}); + }; + storage.getBuckets( + {maxResults: 5}, + (err: Error, results: {}, nextQuery: GetFilesOptions) => { + assert.strictEqual(nextQuery.pageToken, token); + assert.strictEqual(nextQuery.maxResults, 5); + } + ); + }); + + it('should return null nextQuery if there are no more results', () => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, {items: []}); + }; + storage.getBuckets( + {maxResults: 5}, + (err: Error, results: {}, nextQuery: {}) => { + assert.strictEqual(nextQuery, null); + } + ); + }); + + it('should return Bucket objects', done => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, {items: [{id: 'fake-bucket-name'}]}); + }; + storage.getBuckets((err: Error, buckets: Bucket[]) => { + assert.ifError(err); + assert(buckets[0] instanceof Bucket); + done(); + }); + }); + + it('should return apiResponse', done => { + const resp = {items: [{id: 'fake-bucket-name'}]}; + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, resp); + }; + storage.getBuckets( + (err: Error, buckets: Bucket[], nextQuery: {}, apiResponse: {}) => { + assert.deepStrictEqual(resp, apiResponse); + done(); + } + ); + }); + + it('should populate returned Bucket object with metadata', done => { + const bucketMetadata = { + id: 'bucketname', + contentType: 'x-zebra', + metadata: { + my: 'custom metadata', + }, + }; + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, {items: [bucketMetadata]}); + }; + storage.getBuckets((err: Error, buckets: Bucket[]) => { + assert.ifError(err); + assert.deepStrictEqual(buckets[0].metadata, bucketMetadata); + done(); + }); + }); + + it('should return unreachable when returnPartialSuccess is true', done => { + const unreachableList = ['projects/_/buckets/fail-bucket']; + const itemsList = [{id: 'fake-bucket-name'}]; + const resp = {items: itemsList, unreachable: unreachableList}; + + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.qs.returnPartialSuccess, true); + callback(null, resp); + }; + + storage.getBuckets( + {returnPartialSuccess: true}, + (err: Error, buckets: Bucket[], nextQuery: {}, apiResponse: {}) => { + assert.ifError(err); + assert.strictEqual(buckets.length, 2); + + const reachableBucket = buckets.find( + b => b.name === 'fake-bucket-name' + ); + assert.ok(reachableBucket); + assert.strictEqual(reachableBucket.unreachable, false); + + const unreachableBucket = buckets.find(b => b.name === 'fail-bucket'); + assert.ok(unreachableBucket); + assert.strictEqual(unreachableBucket.unreachable, true); + assert.deepStrictEqual(apiResponse, resp); + done(); + } + ); + }); + + it('should handle partial failure with zero reachable buckets', done => { + const unreachableList = ['projects/_/buckets/fail-bucket']; + const resp = {items: [], unreachable: unreachableList}; + + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.qs.returnPartialSuccess, true); + callback(null, resp); + }; + + storage.getBuckets( + {returnPartialSuccess: true}, + (err: Error, buckets: Bucket[]) => { + assert.ifError(err); + assert.strictEqual(buckets.length, 1); + assert.deepStrictEqual(buckets[0].name, 'fail-bucket'); + assert.strictEqual(buckets[0].unreachable, true); + assert.deepStrictEqual(buckets[0].metadata, {}); + done(); + } + ); + }); + + it('should handle API success where zero items and zero unreachable items are returned', done => { + const resp = {items: [], unreachable: []}; + + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.qs.returnPartialSuccess, true); + callback(null, resp); + }; + + storage.getBuckets( + {returnPartialSuccess: true}, + (err: Error, buckets: Bucket[], nextQuery: {}, apiResponse: {}) => { + assert.ifError(err); + assert.strictEqual(buckets.length, 0); + assert.deepStrictEqual(apiResponse, resp); + done(); + } + ); + }); + }); + + describe('getHmacKeys', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let storageRequestStub: sinon.SinonStub; + const SERVICE_ACCOUNT_EMAIL = 'service-account@gserviceaccount.com'; + const ACCESS_ID = 'some-access-id'; + const metadataResponse = { + accessId: ACCESS_ID, + etag: 'etag', + id: ACCESS_ID, + projectId: 'project-id', + serviceAccountEmail: SERVICE_ACCOUNT_EMAIL, + state: 'ACTIVE', + timeCreated: '20190101T00:00:00Z', + updated: '20190101T00:00:00Z', + }; + + beforeEach(() => { + storageRequestStub = sinon.stub(storage, 'request'); + storageRequestStub.callsFake((_opts: {}, callback: Function) => { + callback(null, {}); + }); + }); + + let hmacKeyCtor: sinon.SinonSpy; + beforeEach(() => { + hmacKeyCtor = sinon.spy(hmacKeyModule, 'HmacKey'); + }); + + afterEach(() => { + hmacKeyCtor.restore(); + }); + + it('should get HmacKeys without a query', done => { + storage.getHmacKeys(() => { + const firstArg = storage.request.firstCall.args[0]; + assert.strictEqual( + firstArg.uri, + `/projects/${storage.projectId}/hmacKeys` + ); + assert.deepStrictEqual(firstArg.qs, {}); + done(); + }); + }); + + it('should get HmacKeys with a query', done => { + const query = { + maxResults: 5, + pageToken: 'next-page-token', + serviceAccountEmail: SERVICE_ACCOUNT_EMAIL, + showDeletedKeys: false, + }; + + storage.getHmacKeys(query, () => { + const firstArg = storage.request.firstCall.args[0]; + assert.strictEqual( + firstArg.uri, + `/projects/${storage.projectId}/hmacKeys` + ); + assert.deepStrictEqual(firstArg.qs, query); + done(); + }); + }); + + it('should execute callback with error', done => { + const error = new Error('Error.'); + const apiResponse = {}; + + storageRequestStub.callsFake((_opts: {}, callback: Function) => { + callback(error, apiResponse); + }); + + storage.getHmacKeys( + {}, + (err: Error, hmacKeys: HmacKey[], nextQuery: {}, resp: unknown) => { + assert.strictEqual(err, error); + assert.strictEqual(hmacKeys, null); + assert.strictEqual(nextQuery, null); + assert.strictEqual(resp, apiResponse); + done(); + } + ); + }); + + it('should return nextQuery if more results exist', done => { + const token = 'next-page-token'; + const query = { + param1: 'a', + param2: 'b', + }; + const expectedNextQuery = Object.assign({}, query, {pageToken: token}); + storageRequestStub.callsFake((_opts: {}, callback: Function) => { + callback(null, {nextPageToken: token, items: []}); + }); + + storage.getHmacKeys( + query, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (err: Error, _hmacKeys: [], nextQuery: any) => { + assert.ifError(err); + assert.deepStrictEqual(nextQuery, expectedNextQuery); + done(); + } + ); + }); + + it('should return null nextQuery if there are no more results', done => { + storageRequestStub.callsFake((_opts: {}, callback: Function) => { + callback(null, {items: []}); + }); + + storage.getHmacKeys({}, (err: Error, _hmacKeys: [], nextQuery: {}) => { + assert.ifError(err); + assert.strictEqual(nextQuery, null); + done(); + }); + }); + + it('should return apiResponse', done => { + const resp = {items: [metadataResponse]}; + storageRequestStub.callsFake((_opts: {}, callback: Function) => { + callback(null, resp); + }); + + storage.getHmacKeys( + (err: Error, _hmacKeys: [], _nextQuery: {}, apiResponse: unknown) => { + assert.ifError(err); + assert.deepStrictEqual(resp, apiResponse); + done(); + } + ); + }); + + it('should populate returned HmacKey object with accessId and metadata', done => { + storageRequestStub.callsFake((_opts: {}, callback: Function) => { + callback(null, {items: [metadataResponse]}); + }); + + storage.getHmacKeys((err: Error, hmacKeys: HmacKey[]) => { + assert.ifError(err); + assert.deepStrictEqual(hmacKeyCtor.getCall(0).args, [ + storage, + metadataResponse.accessId, + {projectId: metadataResponse.projectId}, + ]); + assert.deepStrictEqual(hmacKeys[0].metadata, metadataResponse); + done(); + }); + }); + }); + + describe('getServiceAccount', () => { + it('should make the correct request', done => { + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.uri, + `/projects/${storage.projectId}/serviceAccount` + ); + assert.deepStrictEqual(reqOpts.qs, {}); + done(); + }; + + storage.getServiceAccount(assert.ifError); + }); + + it('should allow user options', done => { + const options = { + projectIdentifier: 'test-identifier', + userProject: 'test-user-project', + }; + + storage.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.qs, options); + done(); + }; + + storage.getServiceAccount(options, assert.ifError); + }); + + describe('error', () => { + const ERROR = new Error('Error.'); + const API_RESPONSE = {}; + + beforeEach(() => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(ERROR, API_RESPONSE); + }; + }); + + it('should return the error and apiResponse', done => { + storage.getServiceAccount( + (err: Error, serviceAccount: {}, apiResponse: unknown) => { + assert.strictEqual(err, ERROR); + assert.strictEqual(serviceAccount, null); + assert.strictEqual(apiResponse, API_RESPONSE); + done(); + } + ); + }); + }); + + describe('success', () => { + const API_RESPONSE = {}; + + beforeEach(() => { + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, API_RESPONSE); + }; + }); + + it('should convert snake_case response to camelCase', done => { + const apiResponse = { + snake_case: true, + }; + + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, apiResponse); + }; + + storage.getServiceAccount( + ( + err: Error, + serviceAccount: {[index: string]: string | undefined} + ) => { + assert.ifError(err); + assert.strictEqual( + serviceAccount.snakeCase, + apiResponse.snake_case + ); + assert.strictEqual(serviceAccount.snake_case, undefined); + done(); + } + ); + }); + + it('should return the serviceAccount and apiResponse', done => { + storage.getServiceAccount( + (err: Error, serviceAccount: {}, apiResponse: {}) => { + assert.ifError(err); + assert.deepStrictEqual(serviceAccount, {}); + assert.strictEqual(apiResponse, API_RESPONSE); + done(); + } + ); + }); + }); + }); + + describe('#sanitizeEndpoint', () => { + const USER_DEFINED_SHORT_API_ENDPOINT = 'myapi.com:8080'; + const USER_DEFINED_PROTOCOL = 'myproto'; + const USER_DEFINED_FULL_API_ENDPOINT = `${USER_DEFINED_PROTOCOL}://myapi.com:8080`; + + it('should default protocol to https', () => { + const endpoint = Storage.sanitizeEndpoint( + USER_DEFINED_SHORT_API_ENDPOINT + ); + assert.strictEqual(endpoint.match(PROTOCOL_REGEX)![1], 'https'); + }); + + it('should not override protocol', () => { + const endpoint = Storage.sanitizeEndpoint(USER_DEFINED_FULL_API_ENDPOINT); + assert.strictEqual( + endpoint.match(PROTOCOL_REGEX)![1], + USER_DEFINED_PROTOCOL + ); + }); + + it('should remove trailing slashes from URL', () => { + const endpointsWithTrailingSlashes = [ + `${USER_DEFINED_FULL_API_ENDPOINT}/`, + `${USER_DEFINED_FULL_API_ENDPOINT}//`, + ]; + for (const endpointWithTrailingSlashes of endpointsWithTrailingSlashes) { + const endpoint = Storage.sanitizeEndpoint(endpointWithTrailingSlashes); + assert.strictEqual(endpoint.endsWith('/'), false); + } + }); + }); +}); diff --git a/handwritten/storage/test/nodejs-common/index.ts b/handwritten/storage/test/nodejs-common/index.ts new file mode 100644 index 00000000000..35bfd07da25 --- /dev/null +++ b/handwritten/storage/test/nodejs-common/index.ts @@ -0,0 +1,26 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import assert from 'assert'; +import {describe, it} from 'mocha'; +import {Service, ServiceObject, util} from '../../src/nodejs-common/index.js'; + +describe('common', () => { + it('should correctly export the common modules', () => { + assert(Service); + assert(ServiceObject); + assert(util); + }); +}); diff --git a/handwritten/storage/test/nodejs-common/service-object.ts b/handwritten/storage/test/nodejs-common/service-object.ts new file mode 100644 index 00000000000..3bba5f4faad --- /dev/null +++ b/handwritten/storage/test/nodejs-common/service-object.ts @@ -0,0 +1,1234 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + promisify, + promisifyAll, + PromisifyAllOptions, +} from '@google-cloud/promisify'; +import assert from 'assert'; +import {describe, it, beforeEach, afterEach} from 'mocha'; +import proxyquire from 'proxyquire'; +import * as r from 'teeny-request'; +import * as sinon from 'sinon'; +import {Service} from '../../src/nodejs-common/index.js'; +import * as SO from '../../src/nodejs-common/service-object.js'; + +let promisified = false; +const fakePromisify = { + // tslint:disable-next-line:variable-name + promisifyAll(Class: Function, options: PromisifyAllOptions) { + if (Class.name === 'ServiceObject') { + promisified = true; + assert.deepStrictEqual(options.exclude, ['getRequestInterceptors']); + } + + return promisifyAll(Class, options); + }, +}; +const ServiceObject = proxyquire('../../src/nodejs-common/service-object', { + '@google-cloud/promisify': fakePromisify, +}).ServiceObject; + +import { + ApiError, + BodyResponseCallback, + DecorateRequestOptions, + util, +} from '../../src/nodejs-common/util.js'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type FakeServiceObject = any; +interface InternalServiceObject { + request_: ( + reqOpts: DecorateRequestOptions, + callback?: BodyResponseCallback + ) => void | r.Request; + createMethod?: Function; + methods: SO.Methods; + interceptors: SO.Interceptor[]; +} + +function asInternal( + serviceObject: SO.ServiceObject +) { + return serviceObject as {} as InternalServiceObject; +} + +describe('ServiceObject', () => { + let serviceObject: SO.ServiceObject; + const sandbox = sinon.createSandbox(); + + const CONFIG = { + baseUrl: 'base-url', + parent: {} as Service, + id: 'id', + createMethod: util.noop, + }; + + beforeEach(() => { + serviceObject = new ServiceObject(CONFIG); + serviceObject.parent.interceptors = []; + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('instantiation', () => { + it('should promisify all the things', () => { + assert(promisified); + }); + + it('should create an empty metadata object', () => { + assert.deepStrictEqual(serviceObject.metadata, {}); + }); + + it('should localize the baseUrl', () => { + assert.strictEqual(serviceObject.baseUrl, CONFIG.baseUrl); + }); + + it('should localize the parent instance', () => { + assert.strictEqual(serviceObject.parent, CONFIG.parent); + }); + + it('should localize the ID', () => { + assert.strictEqual(serviceObject.id, CONFIG.id); + }); + + it('should localize the createMethod', () => { + assert.strictEqual( + asInternal(serviceObject).createMethod, + CONFIG.createMethod + ); + }); + + it('should localize the methods', () => { + const methods = {}; + const config = {...CONFIG, methods}; + const serviceObject = new ServiceObject(config); + assert.deepStrictEqual(asInternal(serviceObject).methods, methods); + }); + + it('should default methods to an empty object', () => { + assert.deepStrictEqual(asInternal(serviceObject).methods, {}); + }); + + it('should clear out methods that are not asked for', () => { + const config = { + ...CONFIG, + methods: { + create: true, + }, + }; + const serviceObject = new ServiceObject(config); + assert.strictEqual(typeof serviceObject.create, 'function'); + assert.strictEqual(serviceObject.delete, undefined); + }); + + it('should always expose the request method', () => { + const methods = {}; + const config = {...CONFIG, methods}; + const serviceObject = new ServiceObject(config); + assert.strictEqual(typeof serviceObject.request, 'function'); + }); + + it('should always expose the getRequestInterceptors method', () => { + const methods = {}; + const config = {...CONFIG, methods}; + const serviceObject = new ServiceObject(config); + assert.strictEqual( + typeof serviceObject.getRequestInterceptors, + 'function' + ); + }); + }); + + describe('create', () => { + it('should call createMethod', done => { + const config = {...CONFIG, createMethod}; + const options = {}; + + function createMethod( + id: string, + options_: {}, + callback: (err: Error | null, a: {}, b: {}) => void + ) { + assert.strictEqual(id, config.id); + assert.strictEqual(options_, options); + callback(null, {}, {}); // calls done() + } + + const serviceObject = new ServiceObject(config); + serviceObject.create(options, done); + }); + + it('should not require options', done => { + const config = {...CONFIG, createMethod}; + + function createMethod(id: string, options: Function, callback: Function) { + assert.strictEqual(id, config.id); + assert.strictEqual(typeof options, 'function'); + assert.strictEqual(callback, undefined); + options(null, {}, {}); // calls done() + } + + const serviceObject = new ServiceObject(config); + serviceObject.create(done); + }); + + it('should update id with metadata id', done => { + const config = {...CONFIG, createMethod}; + const options = {}; + + function createMethod( + id: string, + options_: {}, + callback: (err: Error | null, a: {}, b: {}) => void + ) { + assert.strictEqual(id, config.id); + assert.strictEqual(options_, options); + callback(null, {metadata: {id: 14}}, {}); + } + + const serviceObject = new ServiceObject(config); + serviceObject.create(options); + assert.strictEqual(serviceObject.id, 14); + done(); + }); + + it('should pass error to callback', done => { + const config = {...CONFIG, createMethod}; + const options = {}; + const error = new Error('Error.'); + const apiResponse = {}; + function createMethod(id: string, options_: {}, callback: Function) { + callback(error, null, apiResponse); + } + + const serviceObject = new ServiceObject(config); + serviceObject.create( + options, + (err: Error | null, instance: {}, apiResponse_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(instance, null); + assert.strictEqual(apiResponse_, apiResponse); + done(); + } + ); + }); + + it('should return instance and apiResponse to callback', async () => { + const config = {...CONFIG, createMethod}; + const options = {}; + const apiResponse = {}; + function createMethod(id: string, options_: {}, callback: Function) { + callback(null, {}, apiResponse); + } + + const serviceObject = new ServiceObject(config); + const [instance_, apiResponse_] = await serviceObject.create(options); + assert.strictEqual(instance_, serviceObject); + assert.strictEqual(apiResponse_, apiResponse); + }); + + it('should assign metadata', async () => { + const config = {...CONFIG, createMethod}; + const options = {}; + const instance = { + metadata: {}, + }; + function createMethod(id: string, options_: {}, callback: Function) { + callback(null, instance, {}); + } + const serviceObject = new ServiceObject(config); + const [instance_] = await serviceObject.create(options); + assert.strictEqual(instance_.metadata, instance.metadata); + }); + + it('should execute callback with any amount of arguments', done => { + const config = {...CONFIG, createMethod}; + const options = {}; + + const args = ['a', 'b', 'c', 'd', 'e', 'f']; + + function createMethod(id: string, options_: {}, callback: Function) { + callback(...args); + } + + const serviceObject = new ServiceObject(config); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + serviceObject.create(options, (...args: any[]) => { + assert.deepStrictEqual([].slice.call(args), args); + done(); + }); + }); + }); + + describe('delete', () => { + it('should make the correct request', done => { + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.strictEqual(opts.method, 'DELETE'); + assert.strictEqual(opts.uri, ''); + done(); + cb(null, null, {} as r.Response); + }); + serviceObject.delete(assert.ifError); + }); + + it('should accept options', done => { + const options = {queryOptionProperty: true}; + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual(opts.qs, options); + done(); + cb(null, null, {} as r.Response); + }); + serviceObject.delete(options, assert.ifError); + }); + + it('should override method and uri field in request with methodConfig', done => { + const methodConfig = { + reqOpts: { + uri: 'v2', + method: 'PATCH', + }, + }; + + const cachedMethodConfig = {reqOpts: {...methodConfig.reqOpts}}; + + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual( + serviceObject.methods.delete, + cachedMethodConfig + ); + assert.deepStrictEqual(opts.uri, 'v2'); + assert.deepStrictEqual(opts.method, 'PATCH'); + done(); + cb(null, null, null!); + }); + + const serviceObject = new ServiceObject(CONFIG) as FakeServiceObject; + serviceObject.methods.delete = methodConfig; + serviceObject.delete(); + }); + + it('should respect ignoreNotFound option', done => { + const options = {ignoreNotFound: true}; + const error = new ApiError({code: 404, response: {} as r.Response}); + sandbox.stub(ServiceObject.prototype, 'request').callsArgWith(1, error); + serviceObject.delete(options, (err, apiResponse_) => { + assert.ifError(err); + assert.strictEqual(apiResponse_, undefined); + done(); + }); + }); + + it('should propagate other then 404 error', done => { + const options = {ignoreNotFound: true}; + const error = new ApiError({code: 406, response: {} as r.Response}); + sandbox.stub(ServiceObject.prototype, 'request').callsArgWith(1, error); + serviceObject.delete(options, (err, apiResponse_) => { + assert.strictEqual(err, error); + assert.strictEqual(apiResponse_, undefined); + done(); + }); + }); + + it('should not pass ignoreNotFound to request', done => { + const options = {ignoreNotFound: true}; + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.strictEqual(opts.qs.ignoreNotFound, undefined); + done(); + cb(null, null, {} as r.Response); + }); + serviceObject.delete(options, assert.ifError); + }); + + it('should extend the defaults with request options', done => { + const methodConfig = { + reqOpts: { + qs: { + defaultProperty: true, + thisPropertyWasOverridden: false, + }, + }, + }; + + const cachedMethodConfig = {reqOpts: {qs: {...methodConfig.reqOpts.qs}}}; + + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual( + serviceObject.methods.delete, + cachedMethodConfig + ); + assert.deepStrictEqual(opts.qs, { + defaultProperty: true, + optionalProperty: true, + thisPropertyWasOverridden: true, + }); + done(); + cb(null, null, null!); + }); + + const serviceObject = new ServiceObject(CONFIG) as FakeServiceObject; + serviceObject.methods.delete = methodConfig; + serviceObject.delete({ + optionalProperty: true, + thisPropertyWasOverridden: true, + }); + }); + + it('should not require a callback', () => { + sandbox + .stub(ServiceObject.prototype, 'request') + .callsArgWith(1, null, null, {}); + assert.doesNotThrow(() => { + serviceObject.delete(); + }); + }); + + it('should execute callback with correct arguments', done => { + const error = new Error('🦃'); + sandbox.stub(ServiceObject.prototype, 'request').callsArgWith(1, error); + const serviceObject = new ServiceObject(CONFIG); + serviceObject.delete((err: Error, apiResponse_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(apiResponse_, undefined); + done(); + }); + }); + }); + + describe('exists', () => { + it('should call get', done => { + sandbox.stub(serviceObject, 'get').callsFake(() => done()); + serviceObject.exists(() => {}); + }); + + it('should accept options', done => { + const options = {queryOptionProperty: true}; + sandbox + .stub(ServiceObject.prototype, 'get') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual(opts, options); + done(); + cb(null, null, {} as r.Response); + }); + serviceObject.exists(options, assert.ifError); + }); + + it('should execute callback with false if 404', done => { + const error = new ApiError(''); + error.code = 404; + sandbox.stub(serviceObject, 'get').callsArgWith(1, error); + serviceObject.exists((err: Error, exists: boolean) => { + assert.ifError(err); + assert.strictEqual(exists, false); + done(); + }); + }); + + it('should execute callback with error if not 404', done => { + const error = new ApiError(''); + error.code = 500; + sandbox.stub(serviceObject, 'get').callsArgWith(1, error); + serviceObject.exists((err: Error, exists: boolean) => { + assert.strictEqual(err, error); + assert.strictEqual(exists, undefined); + done(); + }); + }); + + it('should execute callback with true if no error', done => { + sandbox.stub(serviceObject, 'get').callsArgWith(1, null); + serviceObject.exists((err: Error, exists: boolean) => { + assert.ifError(err); + assert.strictEqual(exists, true); + done(); + }); + }); + }); + + describe('get', () => { + it('should get the metadata', done => { + serviceObject.getMetadata = promisify((): void => { + done(); + }); + + serviceObject.get(assert.ifError); + }); + + it('should accept options', done => { + const options = {}; + serviceObject.getMetadata = promisify( + (options_: SO.GetMetadataOptions): void => { + assert.deepStrictEqual(options, options_); + done(); + } + ); + serviceObject.exists(options, assert.ifError); + }); + + it('handles not getting a config', done => { + serviceObject.getMetadata = promisify((): void => { + done(); + }); + (serviceObject as FakeServiceObject).get(assert.ifError); + }); + + it('should execute callback with error & metadata', done => { + const error = new Error('Error.'); + const metadata = {} as SO.BaseMetadata; + + serviceObject.getMetadata = promisify( + ( + options: SO.GetMetadataOptions, + callback: SO.MetadataCallback + ) => { + callback(error, metadata); + } + ); + + serviceObject.get((err, instance, metadata_) => { + assert.strictEqual(err, error); + assert.strictEqual(instance, null); + assert.strictEqual(metadata_, metadata); + + done(); + }); + }); + + it('should execute callback with instance & metadata', done => { + const metadata = {} as SO.BaseMetadata; + + serviceObject.getMetadata = promisify( + ( + options: SO.GetMetadataOptions, + callback: SO.MetadataCallback + ) => { + callback(null, metadata); + } + ); + + serviceObject.get((err, instance, metadata_) => { + assert.ifError(err); + + assert.strictEqual(instance, serviceObject); + assert.strictEqual(metadata_, metadata); + + done(); + }); + }); + + describe('autoCreate', () => { + let AUTO_CREATE_CONFIG: {}; + + const ERROR = new ApiError('bad'); + ERROR.code = 404; + const METADATA = {} as SO.BaseMetadata; + + beforeEach(() => { + AUTO_CREATE_CONFIG = { + autoCreate: true, + }; + + serviceObject.getMetadata = promisify( + ( + options: SO.GetMetadataOptions, + callback: SO.MetadataCallback + ) => { + callback(ERROR, METADATA); + } + ); + }); + + it('should keep the original options intact', () => { + const expectedConfig = Object.assign({}, AUTO_CREATE_CONFIG); + serviceObject.get(AUTO_CREATE_CONFIG, () => {}); + assert.deepStrictEqual(AUTO_CREATE_CONFIG, expectedConfig); + }); + + it('should not auto create if there is no create method', done => { + (serviceObject as FakeServiceObject).create = undefined; + + serviceObject.get(AUTO_CREATE_CONFIG, err => { + assert.strictEqual(err, ERROR); + done(); + }); + }); + + it('should pass config to create if it was provided', done => { + const expectedConfig = {maxResults: 5} as SO.GetConfig; + const config = {...AUTO_CREATE_CONFIG, ...expectedConfig}; + + sandbox.stub(serviceObject, 'create').callsFake(config_ => { + assert.deepStrictEqual(config_, expectedConfig); + done(); + }); + serviceObject.get(config, assert.ifError); + }); + + it('should pass only a callback to create if no config', done => { + sandbox.stub(serviceObject, 'create').callsArgWith(0, null); + serviceObject.get(AUTO_CREATE_CONFIG, done); + }); + + describe('error', () => { + it('should execute callback with error & API response', done => { + const error = new Error('Error.'); + const apiResponse = {} as r.Response; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (sandbox.stub(serviceObject, 'create') as any).callsFake( + (optsOrCb: {}, cb: Function) => { + const callback = typeof optsOrCb === 'function' ? optsOrCb : cb; + sandbox.stub(serviceObject, 'get').callsFake((cfg, callback) => { + assert.deepStrictEqual(cfg, {}); + callback!(null); // done() + }); + callback!(error, null, apiResponse); + } + ); + + serviceObject.get(AUTO_CREATE_CONFIG, (err, instance, resp) => { + assert.strictEqual(err, error); + assert.strictEqual(instance, null); + assert.strictEqual(resp, apiResponse); + done(); + }); + }); + + it('should refresh the metadata after a 409', done => { + const error = new ApiError('errrr'); + error.code = 409; + sandbox.stub(serviceObject, 'create').callsFake(callback => { + sandbox.stub(serviceObject, 'get').callsFake((cfgOrCb, cb) => { + const config = typeof cfgOrCb === 'object' ? cfgOrCb : {}; + const callback = typeof cfgOrCb === 'function' ? cfgOrCb : cb; + assert.deepStrictEqual(config, {}); + callback!(null, null, {} as r.Response); // done() + }); + callback(error, null, undefined); + }); + serviceObject.get(AUTO_CREATE_CONFIG, done); + }); + }); + }); + }); + + describe('getMetadata', () => { + it('should make the correct request', done => { + sandbox.stub(ServiceObject.prototype, 'request').callsFake(function ( + this: SO.ServiceObject, + reqOpts, + callback + ) { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.strictEqual(this, serviceObject); + assert.strictEqual(opts.uri, ''); + done(); + cb(null, null, {} as r.Response); + }); + serviceObject.getMetadata(() => {}); + }); + + it('should accept options', done => { + const options = {queryOptionProperty: true}; + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual(opts.qs, options); + done(); + cb(null, null, {} as r.Response); + }); + serviceObject.getMetadata(options, assert.ifError); + }); + + it('should override uri field in request with methodConfig', done => { + const methodConfig = { + reqOpts: { + uri: 'v2', + }, + }; + + const cachedMethodConfig = {reqOpts: {...methodConfig.reqOpts}}; + + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual( + serviceObject.methods.getMetadata, + cachedMethodConfig + ); + assert.deepStrictEqual(opts.uri, 'v2'); + done(); + cb(null, null, null!); + }); + + const serviceObject = new ServiceObject(CONFIG) as FakeServiceObject; + serviceObject.methods.getMetadata = methodConfig; + serviceObject.getMetadata(); + }); + + it('should extend the defaults with request options', done => { + const methodConfig = { + reqOpts: { + qs: { + defaultProperty: true, + thisPropertyWasOverridden: false, + }, + }, + }; + + const cachedMethodConfig = {reqOpts: {qs: {...methodConfig.reqOpts.qs}}}; + + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual( + serviceObject.methods.getMetadata, + cachedMethodConfig + ); + assert.deepStrictEqual(opts.qs, { + defaultProperty: true, + optionalProperty: true, + thisPropertyWasOverridden: true, + }); + done(); + cb(null, null, null!); + }); + + const serviceObject = new ServiceObject(CONFIG) as FakeServiceObject; + serviceObject.methods.getMetadata = methodConfig; + serviceObject.getMetadata({ + optionalProperty: true, + thisPropertyWasOverridden: true, + }); + }); + + it('should execute callback with error & apiResponse', done => { + const error = new Error('ಠ_ಠ'); + sandbox.stub(ServiceObject.prototype, 'request').callsArgWith(1, error); + serviceObject.getMetadata((err: Error, metadata: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(metadata, undefined); + done(); + }); + }); + + it('should update metadata', done => { + const apiResponse = {}; + sandbox + .stub(ServiceObject.prototype, 'request') + .callsArgWith(1, null, {}, apiResponse); + serviceObject.getMetadata((err: Error) => { + assert.ifError(err); + assert.deepStrictEqual(serviceObject.metadata, apiResponse); + done(); + }); + }); + + it('should execute callback with metadata & API response', done => { + const apiResponse = {}; + const requestResponse = {body: apiResponse}; + sandbox + .stub(ServiceObject.prototype, 'request') + .callsArgWith(1, null, apiResponse, requestResponse); + serviceObject.getMetadata((err: Error, metadata: {}) => { + assert.ifError(err); + assert.strictEqual(metadata, apiResponse); + done(); + }); + }); + }); + + describe('getRequestInterceptors', () => { + it('should call the request interceptors in order', () => { + // Called first. + serviceObject.parent.interceptors.push({ + request(reqOpts: DecorateRequestOptions) { + reqOpts.uri = '1'; + return reqOpts; + }, + }); + + // Called third. + serviceObject.interceptors.push({ + request(reqOpts: DecorateRequestOptions) { + reqOpts.uri += '3'; + return reqOpts; + }, + }); + + // Called second. + serviceObject.parent.interceptors.push({ + request(reqOpts: DecorateRequestOptions) { + reqOpts.uri += '2'; + return reqOpts; + }, + }); + + // Called fourth. + serviceObject.interceptors.push({ + request(reqOpts: DecorateRequestOptions) { + reqOpts.uri += '4'; + return reqOpts; + }, + }); + + serviceObject.parent.getRequestInterceptors = () => { + return serviceObject.parent.interceptors.map( + interceptor => interceptor.request + ); + }; + + const reqOpts: DecorateRequestOptions = {uri: ''}; + const requestInterceptors = serviceObject.getRequestInterceptors(); + requestInterceptors.forEach((requestInterceptor: Function) => { + Object.assign(reqOpts, requestInterceptor(reqOpts)); + }); + assert.strictEqual(reqOpts.uri, '1234'); + }); + + it('should not affect original interceptor arrays', () => { + function request(reqOpts: DecorateRequestOptions) { + return reqOpts; + } + + serviceObject.parent.interceptors = [{request}]; + serviceObject.interceptors = [{request}]; + + const originalParentInterceptors = [].slice.call( + serviceObject.parent.interceptors + ); + const originalLocalInterceptors = [].slice.call( + serviceObject.interceptors + ); + + serviceObject.getRequestInterceptors(); + + assert.deepStrictEqual( + serviceObject.parent.interceptors, + originalParentInterceptors + ); + assert.deepStrictEqual( + serviceObject.interceptors, + originalLocalInterceptors + ); + }); + + it('should not call unrelated interceptors', () => { + (serviceObject.interceptors as object[]).push({ + anotherInterceptor() { + throw new Error('Unrelated interceptor was called.'); + }, + request(reqOpts: DecorateRequestOptions) { + return reqOpts; + }, + }); + + const requestInterceptors = serviceObject.getRequestInterceptors(); + requestInterceptors.forEach((requestInterceptor: Function) => { + requestInterceptor(); + }); + }); + }); + + describe('setMetadata', () => { + it('should make the correct request', done => { + const metadata = {metadataProperty: true}; + sandbox.stub(ServiceObject.prototype, 'request').callsFake(function ( + this: SO.ServiceObject, + reqOpts, + callback + ) { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.strictEqual(this, serviceObject); + assert.strictEqual(opts.method, 'PATCH'); + assert.strictEqual(opts.uri, ''); + assert.deepStrictEqual(opts.json, metadata); + done(); + cb(null, null, {} as r.Response); + }); + serviceObject.setMetadata(metadata, () => {}); + }); + + it('should accept options', done => { + const metadata = {}; + const options = {queryOptionProperty: true}; + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual(opts.qs, options); + done(); + cb(null, null, {} as r.Response); + }); + serviceObject.setMetadata(metadata, options, () => {}); + }); + + it('should override uri and method with methodConfig', done => { + const methodConfig = { + reqOpts: { + uri: 'v2', + method: 'PUT', + }, + }; + const cachedMethodConfig = {reqOpts: {...methodConfig.reqOpts}}; + + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual( + serviceObject.methods.setMetadata, + cachedMethodConfig + ); + assert.deepStrictEqual(opts.uri, 'v2'); + assert.deepStrictEqual(opts.method, 'PUT'); + done(); + cb(null, null, null!); + }); + + const serviceObject = new ServiceObject(CONFIG) as FakeServiceObject; + serviceObject.methods.setMetadata = methodConfig; + serviceObject.setMetadata({}); + }); + + it('should extend the defaults with request options', done => { + const methodConfig = { + reqOpts: { + qs: { + defaultProperty: true, + thisPropertyWasOverridden: false, + }, + }, + }; + const cachedMethodConfig = {reqOpts: {qs: {...methodConfig.reqOpts.qs}}}; + + sandbox + .stub(ServiceObject.prototype, 'request') + .callsFake((reqOpts, callback) => { + const opts = reqOpts as r.OptionsWithUri; + const cb = callback as BodyResponseCallback; + assert.deepStrictEqual( + serviceObject.methods.setMetadata, + cachedMethodConfig + ); + assert.deepStrictEqual(opts.qs, { + defaultProperty: true, + optionalProperty: true, + thisPropertyWasOverridden: true, + }); + done(); + cb(null, null, null!); + }); + + const serviceObject = new ServiceObject(CONFIG) as FakeServiceObject; + serviceObject.methods.setMetadata = methodConfig; + serviceObject.setMetadata( + {}, + { + optionalProperty: true, + thisPropertyWasOverridden: true, + } + ); + }); + + it('should execute callback with error & apiResponse', done => { + const error = new Error('Error.'); + sandbox.stub(ServiceObject.prototype, 'request').callsArgWith(1, error); + serviceObject.setMetadata({}, (err: Error, apiResponse_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(apiResponse_, undefined); + done(); + }); + }); + + it('should update metadata', done => { + const apiResponse = {}; + sandbox + .stub(ServiceObject.prototype, 'request') + .callsArgWith(1, undefined, apiResponse); + serviceObject.setMetadata({}, (err: Error) => { + assert.ifError(err); + assert.strictEqual(serviceObject.metadata, apiResponse); + done(); + }); + }); + + it('should execute callback with metadata & API response', done => { + const body = {}; + const apiResponse = {body}; + sandbox + .stub(ServiceObject.prototype, 'request') + .callsArgWith(1, null, body, apiResponse); + serviceObject.setMetadata({}, (err: Error, metadata: {}) => { + assert.ifError(err); + assert.strictEqual(metadata, body); + done(); + }); + }); + }); + + describe('request_', () => { + let reqOpts: DecorateRequestOptions; + beforeEach(() => { + reqOpts = { + uri: 'uri', + }; + }); + + it('should compose the correct request', done => { + const expectedUri = [ + serviceObject.baseUrl, + serviceObject.id, + reqOpts.uri, + ].join('/'); + + serviceObject.parent.request = (reqOpts_, callback) => { + assert.notStrictEqual(reqOpts_, reqOpts); + assert.strictEqual(reqOpts_.uri, expectedUri); + assert.deepStrictEqual(reqOpts_.interceptors_, []); + callback(null, null, {} as r.Response); + }; + asInternal(serviceObject).request_(reqOpts, () => done()); + }); + + it('should not require a service object ID', done => { + const expectedUri = [serviceObject.baseUrl, reqOpts.uri].join('/'); + serviceObject.parent.request = (reqOpts, callback) => { + assert.strictEqual(reqOpts.uri, expectedUri); + callback(null, null, {} as r.Response); + }; + serviceObject.id = undefined; + asInternal(serviceObject).request_(reqOpts, () => done()); + }); + + it('should support absolute uris', done => { + const expectedUri = 'http://www.google.com'; + serviceObject.parent.request = (reqOpts, callback) => { + assert.strictEqual(reqOpts.uri, expectedUri); + callback(null, null, {} as r.Response); + }; + asInternal(serviceObject).request_({uri: expectedUri}, () => { + done(); + }); + }); + + it('should remove empty components', done => { + const reqOpts = {uri: ''}; + const expectedUri = [ + serviceObject.baseUrl, + serviceObject.id, + // reqOpts.uri (reqOpts.uri is an empty string, so it should be removed) + ].join('/'); + serviceObject.parent.request = (reqOpts_, callback) => { + assert.strictEqual(reqOpts_.uri, expectedUri); + callback(null, null, {} as r.Response); + }; + asInternal(serviceObject).request_(reqOpts, () => done()); + }); + + it('should trim slashes', done => { + const reqOpts = { + uri: '//1/2//', + }; + const expectedUri = [serviceObject.baseUrl, serviceObject.id, '1/2'].join( + '/' + ); + serviceObject.parent.request = (reqOpts_, callback) => { + assert.strictEqual(reqOpts_.uri, expectedUri); + callback(null, null, {} as r.Response); + }; + asInternal(serviceObject).request_(reqOpts, () => { + done(); + }); + }); + + it('should extend interceptors from child ServiceObjects', async () => { + const parent = new ServiceObject(CONFIG) as FakeServiceObject; + parent.interceptors.push({ + request(reqOpts: DecorateRequestOptions) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (reqOpts as any).parent = true; + return reqOpts; + }, + }); + + const child = new ServiceObject({...CONFIG, parent}) as FakeServiceObject; + child.interceptors.push({ + request(reqOpts: DecorateRequestOptions) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (reqOpts as any).child = true; + return reqOpts; + }, + }); + + sandbox + .stub( + parent.parent as SO.ServiceObject, + 'request' + ) + .callsFake((reqOpts, callback) => { + assert.deepStrictEqual( + reqOpts.interceptors_![0].request({} as DecorateRequestOptions), + { + child: true, + } + ); + assert.deepStrictEqual( + reqOpts.interceptors_![1].request({} as DecorateRequestOptions), + { + parent: true, + } + ); + callback(null, null, {} as r.Response); + }); + + await child.request_({uri: ''}); + }); + + it('should pass a clone of the interceptors', done => { + asInternal(serviceObject).interceptors.push({ + request(reqOpts: DecorateRequestOptions) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (reqOpts as any).one = true; + return reqOpts; + }, + }); + + serviceObject.parent.request = (reqOpts, callback) => { + const serviceObjectInterceptors = + asInternal(serviceObject).interceptors; + assert.deepStrictEqual( + reqOpts.interceptors_, + serviceObjectInterceptors + ); + assert.notStrictEqual(reqOpts.interceptors_, serviceObjectInterceptors); + callback(null, null, {} as r.Response); + done(); + }; + asInternal(serviceObject).request_({uri: ''}, () => {}); + }); + + it('should call the parent requestStream method', () => { + const fakeObj = {}; + const expectedUri = [ + serviceObject.baseUrl, + serviceObject.id, + reqOpts.uri, + ].join('/'); + + serviceObject.parent.requestStream = reqOpts_ => { + assert.notStrictEqual(reqOpts_, reqOpts); + assert.strictEqual(reqOpts_.uri, expectedUri); + assert.deepStrictEqual(reqOpts_.interceptors_, []); + return fakeObj as r.Request; + }; + + const opts = {...reqOpts, shouldReturnStream: true}; + const res = asInternal(serviceObject).request_(opts); + assert.strictEqual(res, fakeObj); + }); + }); + + describe('request', () => { + it('should call through to request_', async () => { + const fakeOptions = {} as DecorateRequestOptions; + sandbox + .stub(asInternal(serviceObject), 'request_') + .callsFake((reqOpts, callback) => { + assert.strictEqual(reqOpts, fakeOptions); + callback!(null, null, {} as r.Response); + }); + await serviceObject.request(fakeOptions); + }); + + it('should accept a callback', done => { + const response = {body: {abc: '123'}, statusCode: 200} as r.Response; + sandbox + .stub(asInternal(serviceObject), 'request_') + .callsArgWith(1, null, response.body, response); + serviceObject.request({} as DecorateRequestOptions, (err, body, res) => { + assert.ifError(err); + assert.deepStrictEqual(res, response); + assert.deepStrictEqual(body, response.body); + done(); + }); + }); + + it('should return response with a request error and callback', done => { + const errorBody = '🤮'; + const response = {body: {error: errorBody}, statusCode: 500}; + const err = new Error(errorBody); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (err as any).response = response; + sandbox + .stub(asInternal(serviceObject), 'request_') + .callsArgWith(1, err, response.body, response); + serviceObject.request({} as DecorateRequestOptions, (err, body, res) => { + assert(err instanceof Error); + assert.deepStrictEqual(res, response); + assert.deepStrictEqual(body, response.body); + done(); + }); + }); + }); + + describe('requestStream', () => { + it('should call through to request_', async () => { + const fakeOptions = {} as DecorateRequestOptions; + const serviceObject = new ServiceObject(CONFIG); + asInternal(serviceObject).request_ = reqOpts => { + assert.deepStrictEqual(reqOpts, {shouldReturnStream: true}); + }; + serviceObject.requestStream(fakeOptions); + }); + }); +}); diff --git a/handwritten/storage/test/nodejs-common/service.ts b/handwritten/storage/test/nodejs-common/service.ts new file mode 100644 index 00000000000..502c4e5419f --- /dev/null +++ b/handwritten/storage/test/nodejs-common/service.ts @@ -0,0 +1,718 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import {describe, it, before, beforeEach, after} from 'mocha'; +import proxyquire from 'proxyquire'; +import {Request} from 'teeny-request'; +import {AuthClient, GoogleAuth, OAuth2Client} from 'google-auth-library'; + +import {Interceptor} from '../../src/nodejs-common/index.js'; +import { + DEFAULT_PROJECT_ID_TOKEN, + ServiceConfig, + ServiceOptions, +} from '../../src/nodejs-common/service.js'; +import { + BodyResponseCallback, + DecorateRequestOptions, + GCCL_GCS_CMD_KEY, + MakeAuthenticatedRequest, + MakeAuthenticatedRequestFactoryConfig, + util, + Util, +} from '../../src/nodejs-common/util.js'; +import {getUserAgentString, getModuleFormat} from '../../src/util.js'; + +proxyquire.noPreserveCache(); + +const fakeCfg = {} as ServiceConfig; + +const makeAuthRequestFactoryCache = util.makeAuthenticatedRequestFactory; +let makeAuthenticatedRequestFactoryOverride: + | null + | (( + config: MakeAuthenticatedRequestFactoryConfig + ) => MakeAuthenticatedRequest); + +util.makeAuthenticatedRequestFactory = function ( + this: Util, + config: MakeAuthenticatedRequestFactoryConfig +) { + if (makeAuthenticatedRequestFactoryOverride) { + return makeAuthenticatedRequestFactoryOverride.call(this, config); + } + return makeAuthRequestFactoryCache.call(this, config); +}; + +describe('Service', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let service: any; + const Service = proxyquire('../../src/nodejs-common/service', { + './util': util, + }).Service; + + const CONFIG = { + scopes: [], + baseUrl: 'base-url', + projectIdRequired: false, + apiEndpoint: 'common.endpoint.local', + packageJson: { + name: '@google-cloud/service', + version: '0.2.0', + }, + }; + + const OPTIONS = { + authClient: new GoogleAuth(), + credentials: {}, + keyFile: {}, + email: 'email', + projectId: 'project-id', + token: 'token', + } as ServiceOptions; + + beforeEach(() => { + makeAuthenticatedRequestFactoryOverride = null; + service = new Service(CONFIG, OPTIONS); + }); + + describe('instantiation', () => { + it('should not require options', () => { + assert.doesNotThrow(() => { + new Service(CONFIG); + }); + }); + + it('should create an authenticated request factory', () => { + const authenticatedRequest = {} as MakeAuthenticatedRequest; + + makeAuthenticatedRequestFactoryOverride = ( + config: MakeAuthenticatedRequestFactoryConfig + ) => { + const expectedConfig = { + ...CONFIG, + authClient: OPTIONS.authClient, + credentials: OPTIONS.credentials, + keyFile: OPTIONS.keyFilename, + email: OPTIONS.email, + projectIdRequired: CONFIG.projectIdRequired, + projectId: OPTIONS.projectId, + clientOptions: { + universeDomain: undefined, + }, + }; + + assert.deepStrictEqual(config, expectedConfig); + + return authenticatedRequest; + }; + + const svc = new Service(CONFIG, OPTIONS); + assert.strictEqual(svc.makeAuthenticatedRequest, authenticatedRequest); + }); + + it('should localize the authClient', () => { + const authClient = {}; + makeAuthenticatedRequestFactoryOverride = () => { + return { + authClient, + } as MakeAuthenticatedRequest; + }; + const service = new Service(CONFIG, OPTIONS); + assert.strictEqual(service.authClient, authClient); + }); + + it('should localize the provided authClient', () => { + const service = new Service(CONFIG, OPTIONS); + assert.strictEqual(service.authClient, OPTIONS.authClient); + }); + + describe('`AuthClient` support', () => { + // Using a custom `AuthClient` to ensure any `AuthClient` would work + class CustomAuthClient extends AuthClient { + async getAccessToken() { + return {token: '', res: undefined}; + } + + async getRequestHeaders() { + return {}; + } + + request = OAuth2Client.prototype.request.bind(this); + } + + it('should accept an `AuthClient` passed to config', async () => { + const authClient = new CustomAuthClient(); + const serviceObject = new Service({...CONFIG, authClient}); + + // The custom `AuthClient` should be passed to `GoogleAuth` and used internally + const client = await serviceObject.authClient.getClient(); + + assert.strictEqual(client, authClient); + }); + + it('should accept an `AuthClient` passed to options', async () => { + const authClient = new CustomAuthClient(); + const serviceObject = new Service(CONFIG, {authClient}); + + // The custom `AuthClient` should be passed to `GoogleAuth` and used internally + const client = await serviceObject.authClient.getClient(); + + assert.strictEqual(client, authClient); + }); + }); + + it('should localize the baseUrl', () => { + assert.strictEqual(service.baseUrl, CONFIG.baseUrl); + }); + + it('should localize the apiEndpoint', () => { + assert.strictEqual(service.apiEndpoint, CONFIG.apiEndpoint); + }); + + it('should default the timeout to undefined', () => { + assert.strictEqual(service.timeout, undefined); + }); + + it('should localize the timeout', () => { + const timeout = 10000; + const options = {...OPTIONS, timeout}; + const service = new Service(fakeCfg, options); + assert.strictEqual(service.timeout, timeout); + }); + + it('should default globalInterceptors to an empty array', () => { + assert.deepStrictEqual(service.globalInterceptors, []); + }); + + it('should preserve the original global interceptors', () => { + const globalInterceptors: Interceptor[] = []; + const options = {...OPTIONS}; + options.interceptors_ = globalInterceptors; + const service = new Service(fakeCfg, options); + assert.strictEqual(service.globalInterceptors, globalInterceptors); + }); + + it('should default interceptors to an empty array', () => { + assert.deepStrictEqual(service.interceptors, []); + }); + + it('should localize package.json', () => { + assert.strictEqual(service.packageJson, CONFIG.packageJson); + }); + + it('should localize the projectId', () => { + assert.strictEqual(service.projectId, OPTIONS.projectId); + }); + + it('should default projectId with placeholder', () => { + const service = new Service(fakeCfg, {}); + assert.strictEqual(service.projectId, DEFAULT_PROJECT_ID_TOKEN); + }); + + it('should localize the projectIdRequired', () => { + assert.strictEqual(service.projectIdRequired, CONFIG.projectIdRequired); + }); + + it('should default projectIdRequired to true', () => { + const service = new Service(fakeCfg, OPTIONS); + assert.strictEqual(service.projectIdRequired, true); + }); + + it('should disable forever agent for Cloud Function envs', () => { + process.env.FUNCTION_NAME = 'cloud-function-name'; + const service = new Service(CONFIG, OPTIONS); + delete process.env.FUNCTION_NAME; + + const interceptor = service.interceptors[0]; + + const modifiedReqOpts = interceptor.request({forever: true}); + assert.strictEqual(modifiedReqOpts.forever, false); + }); + }); + + describe('getRequestInterceptors', () => { + it('should call the request interceptors in order', () => { + // Called first. + service.globalInterceptors.push({ + request(reqOpts: {order: string}) { + reqOpts.order = '1'; + return reqOpts; + }, + }); + + // Called third. + service.interceptors.push({ + request(reqOpts: {order: string}) { + reqOpts.order += '3'; + return reqOpts; + }, + }); + + // Called second. + service.globalInterceptors.push({ + request(reqOpts: {order: string}) { + reqOpts.order += '2'; + return reqOpts; + }, + }); + + // Called fourth. + service.interceptors.push({ + request(reqOpts: {order: string}) { + reqOpts.order += '4'; + return reqOpts; + }, + }); + + const reqOpts: {order?: string} = {}; + const requestInterceptors = service.getRequestInterceptors(); + requestInterceptors.forEach((requestInterceptor: Function) => { + Object.assign(reqOpts, requestInterceptor(reqOpts)); + }); + assert.strictEqual(reqOpts.order, '1234'); + }); + + it('should not affect original interceptor arrays', () => { + function request(reqOpts: DecorateRequestOptions) { + return reqOpts; + } + + service.globalInterceptors = [{request}]; + service.interceptors = [{request}]; + + const originalGlobalInterceptors = [].slice.call( + service.globalInterceptors + ); + const originalLocalInterceptors = [].slice.call(service.interceptors); + + service.getRequestInterceptors(); + + assert.deepStrictEqual( + service.globalInterceptors, + originalGlobalInterceptors + ); + assert.deepStrictEqual(service.interceptors, originalLocalInterceptors); + }); + + it('should not call unrelated interceptors', () => { + service.interceptors.push({ + anotherInterceptor() { + throw new Error('Unrelated interceptor was called.'); + }, + request() { + return {}; + }, + }); + + const requestInterceptors = service.getRequestInterceptors(); + requestInterceptors.forEach((requestInterceptor: Function) => { + requestInterceptor(); + }); + }); + }); + + describe('getProjectId', () => { + it('should get the project ID from the auth client', done => { + service.authClient = { + getProjectId() { + done(); + }, + }; + + service.getProjectId(assert.ifError); + }); + + it('should return error from auth client', done => { + const error = new Error('Error.'); + + service.authClient = { + async getProjectId() { + throw error; + }, + }; + + service.getProjectId((err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should update and return the project ID if found', done => { + const service = new Service(fakeCfg, {}); + const projectId = 'detected-project-id'; + + service.authClient = { + async getProjectId() { + return projectId; + }, + }; + + service.getProjectId((err: Error, projectId_: string) => { + assert.ifError(err); + assert.strictEqual(service.projectId, projectId); + assert.strictEqual(projectId_, projectId); + done(); + }); + }); + + it('should return a promise if no callback is provided', () => { + const value = {}; + service.getProjectIdAsync = () => value; + assert.strictEqual(service.getProjectId(), value); + }); + }); + + describe('request_', () => { + let reqOpts: DecorateRequestOptions; + + beforeEach(() => { + reqOpts = { + uri: 'uri', + }; + }); + + it('should compose the correct request', done => { + const expectedUri = [service.baseUrl, reqOpts.uri].join('/'); + service.makeAuthenticatedRequest = ( + reqOpts_: DecorateRequestOptions, + callback: BodyResponseCallback + ) => { + assert.notStrictEqual(reqOpts_, reqOpts); + assert.strictEqual(reqOpts_.uri, expectedUri); + assert.strictEqual(reqOpts.interceptors_, undefined); + callback(null); // done() + }; + service.request_(reqOpts, () => done()); + }); + + it('should support absolute uris', done => { + const expectedUri = 'http://www.google.com'; + + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, expectedUri); + done(); + }; + + service.request_({uri: expectedUri}, assert.ifError); + }); + + it('should trim slashes', done => { + const reqOpts = { + uri: '//1/2//', + }; + + const expectedUri = [service.baseUrl, '1/2'].join('/'); + + service.makeAuthenticatedRequest = (reqOpts_: DecorateRequestOptions) => { + assert.strictEqual(reqOpts_.uri, expectedUri); + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + + it('should replace path/:subpath with path:subpath', done => { + const reqOpts = { + uri: ':test', + }; + + const expectedUri = service.baseUrl + reqOpts.uri; + service.makeAuthenticatedRequest = (reqOpts_: DecorateRequestOptions) => { + assert.strictEqual(reqOpts_.uri, expectedUri); + done(); + }; + service.request_(reqOpts, assert.ifError); + }); + + it('should not set timeout', done => { + service.makeAuthenticatedRequest = (reqOpts_: DecorateRequestOptions) => { + assert.strictEqual(reqOpts_.timeout, undefined); + done(); + }; + service.request_(reqOpts, assert.ifError); + }); + + it('should set reqOpt.timeout', done => { + const timeout = 10000; + const config = {...CONFIG}; + const options = {...OPTIONS, timeout}; + const service = new Service(config, options); + + service.makeAuthenticatedRequest = (reqOpts_: DecorateRequestOptions) => { + assert.strictEqual(reqOpts_.timeout, timeout); + done(); + }; + service.request_(reqOpts, assert.ifError); + }); + + it('should add the User Agent', done => { + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual( + reqOpts.headers!['User-Agent'], + getUserAgentString() + ); + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + + it('should add the api-client header', done => { + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + const pkg = service.packageJson; + const r = new RegExp( + `^gl-node/${process.versions.node} gccl/${ + pkg.version + }-${getModuleFormat()} gccl-invocation-id/(?[^W]+)$` + ); + assert.ok(r.test(reqOpts.headers!['x-goog-api-client'])); + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + + it('should add the `gccl-gcs-cmd` to the api-client header when provided', done => { + const expected = 'example.expected/value'; + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + const pkg = service.packageJson; + const r = new RegExp( + `^gl-node/${process.versions.node} gccl/${ + pkg.version + }-${getModuleFormat()} gccl-invocation-id/(?[^W]+) gccl-gcs-cmd/${expected}$` + ); + assert.ok(r.test(reqOpts.headers!['x-goog-api-client'])); + done(); + }; + + service.request_( + {...reqOpts, [GCCL_GCS_CMD_KEY]: expected}, + assert.ifError + ); + }); + + describe('projectIdRequired', () => { + describe('false', () => { + it('should include the projectId', done => { + const config = {...CONFIG, projectIdRequired: false}; + const service = new Service(config, OPTIONS); + + const expectedUri = [service.baseUrl, reqOpts.uri].join('/'); + + service.makeAuthenticatedRequest = ( + reqOpts_: DecorateRequestOptions + ) => { + assert.strictEqual(reqOpts_.uri, expectedUri); + + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + }); + + describe('true', () => { + it('should not include the projectId', done => { + const config = {...CONFIG, projectIdRequired: true}; + const service = new Service(config, OPTIONS); + + const expectedUri = [ + service.baseUrl, + 'projects', + service.projectId, + reqOpts.uri, + ].join('/'); + + service.makeAuthenticatedRequest = ( + reqOpts_: DecorateRequestOptions + ) => { + assert.strictEqual(reqOpts_.uri, expectedUri); + + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + + it('should use projectId override', done => { + const config = {...CONFIG, projectIdRequired: true}; + const service = new Service(config, OPTIONS); + const projectOverride = 'turing'; + + reqOpts.projectId = projectOverride; + + const expectedUri = [ + service.baseUrl, + 'projects', + projectOverride, + reqOpts.uri, + ].join('/'); + + service.makeAuthenticatedRequest = ( + reqOpts_: DecorateRequestOptions + ) => { + assert.strictEqual(reqOpts_.uri, expectedUri); + + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + }); + }); + + describe('request interceptors', () => { + type FakeRequestOptions = DecorateRequestOptions & {a: string; b: string}; + + it('should include request interceptors', done => { + const requestInterceptors = [ + (reqOpts: FakeRequestOptions) => { + reqOpts.a = 'a'; + return reqOpts; + }, + (reqOpts: FakeRequestOptions) => { + reqOpts.b = 'b'; + return reqOpts; + }, + ]; + + service.getRequestInterceptors = () => { + return requestInterceptors; + }; + + service.makeAuthenticatedRequest = (reqOpts: FakeRequestOptions) => { + assert.strictEqual(reqOpts.a, 'a'); + assert.strictEqual(reqOpts.b, 'b'); + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + + it('should combine reqOpts interceptors', done => { + const requestInterceptors = [ + (reqOpts: FakeRequestOptions) => { + reqOpts.a = 'a'; + return reqOpts; + }, + ]; + + service.getRequestInterceptors = () => { + return requestInterceptors; + }; + + reqOpts.interceptors_ = [ + { + request: (reqOpts: FakeRequestOptions) => { + reqOpts.b = 'b'; + return reqOpts; + }, + }, + ]; + + service.makeAuthenticatedRequest = (reqOpts: FakeRequestOptions) => { + assert.strictEqual(reqOpts.a, 'a'); + assert.strictEqual(reqOpts.b, 'b'); + assert.strictEqual(typeof reqOpts.interceptors_, 'undefined'); + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + }); + + describe('error handling', () => { + it('should re-throw any makeAuthenticatedRequest callback error', done => { + const err = new Error('🥓'); + const res = {body: undefined}; + service.makeAuthenticatedRequest = (_: void, callback: Function) => { + callback(err, res.body, res); + }; + service.request_({uri: ''}, (e: Error) => { + assert.strictEqual(e, err); + done(); + }); + }); + }); + }); + + describe('request', () => { + let request_: Request; + + before(() => { + request_ = Service.prototype.request_; + }); + + after(() => { + Service.prototype.request_ = request_; + }); + + it('should call through to _request', async () => { + const fakeOpts = {}; + Service.prototype.request_ = async (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts, fakeOpts); + return Promise.resolve({}); + }; + await service.request(fakeOpts); + }); + + it('should accept a callback', done => { + const fakeOpts = {}; + const response = {body: {abc: '123'}, statusCode: 200}; + Service.prototype.request_ = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts, fakeOpts); + callback(null, response.body, response); + }; + + service.request(fakeOpts, (err: Error, body: {}, res: {}) => { + assert.ifError(err); + assert.deepStrictEqual(res, response); + assert.deepStrictEqual(body, response.body); + done(); + }); + }); + }); + + describe('requestStream', () => { + let request_: Request; + + before(() => { + request_ = Service.prototype.request_; + }); + + after(() => { + Service.prototype.request_ = request_; + }); + + it('should return whatever _request returns', async () => { + const fakeOpts = {}; + const fakeStream = {}; + + Service.prototype.request_ = async (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts, {shouldReturnStream: true}); + return fakeStream; + }; + + const stream = await service.requestStream(fakeOpts); + assert.strictEqual(stream, fakeStream); + }); + }); +}); diff --git a/handwritten/storage/test/nodejs-common/util.ts b/handwritten/storage/test/nodejs-common/util.ts new file mode 100644 index 00000000000..3efc73d11d6 --- /dev/null +++ b/handwritten/storage/test/nodejs-common/util.ts @@ -0,0 +1,1893 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + MissingProjectIdError, + replaceProjectIdToken, +} from '@google-cloud/projectify'; +import assert from 'assert'; +import {describe, it, before, beforeEach, afterEach} from 'mocha'; +import { + AuthClient, + GoogleAuth, + GoogleAuthOptions, + OAuth2Client, +} from 'google-auth-library'; +import * as nock from 'nock'; +import proxyquire from 'proxyquire'; +import * as r from 'teeny-request'; +import retryRequest from 'retry-request'; +import * as sinon from 'sinon'; +import * as stream from 'stream'; +import {teenyRequest} from 'teeny-request'; + +import { + Abortable, + ApiError, + DecorateRequestOptions, + Duplexify, + GCCL_GCS_CMD_KEY, + GoogleErrorBody, + GoogleInnerError, + MakeAuthenticatedRequestFactoryConfig, + MakeRequestConfig, + ParsedHttpRespMessage, + Util, +} from '../../src/nodejs-common/util.js'; +import {DEFAULT_PROJECT_ID_TOKEN} from '../../src/nodejs-common/service.js'; +import duplexify from 'duplexify'; + +nock.disableNetConnect(); + +const fakeResponse = { + statusCode: 200, + body: {star: 'trek'}, +} as r.Response; + +const fakeBadResp = { + statusCode: 400, + statusMessage: 'Not Good', +} as r.Response; + +const fakeReqOpts: DecorateRequestOptions = { + uri: 'http://so-fake', + method: 'GET', +}; + +const fakeError = new Error('this error is like so fake'); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +let requestOverride: any; +function fakeRequest() { + // eslint-disable-next-line prefer-spread, prefer-rest-params + return (requestOverride || teenyRequest).apply(null, arguments); +} + +fakeRequest.defaults = (defaults: r.CoreOptions) => { + assert.ok( + /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( + defaults.headers!['x-goog-api-client'] + ) + ); + return fakeRequest; +}; + +let retryRequestOverride: Function | null; +function fakeRetryRequest() { + // eslint-disable-next-line prefer-spread, prefer-rest-params + return (retryRequestOverride || retryRequest).apply(null, arguments); +} + +let replaceProjectIdTokenOverride: Function | null; +function fakeReplaceProjectIdToken() { + // eslint-disable-next-line prefer-spread, prefer-rest-params + return (replaceProjectIdTokenOverride || replaceProjectIdToken).apply( + null, + // eslint-disable-next-line prefer-spread, prefer-rest-params + arguments + ); +} + +describe('common/util', () => { + let util: Util & {[index: string]: Function}; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function stub(method: keyof Util, meth: (...args: any[]) => any) { + return sandbox.stub(util, method).callsFake(meth); + } + + function createExpectedErrorMessage(errors: string[]): string { + if (errors.length < 2) { + return errors[0]; + } + + errors = errors.map((error, i) => ` ${i + 1}. ${error}`); + errors.unshift( + 'Multiple errors occurred during the request. Please see the `errors` array for complete details.\n' + ); + errors.push('\n'); + + return errors.join('\n'); + } + + const fakeGoogleAuth = { + // Using a custom `AuthClient` to ensure any `AuthClient` would work + AuthClient: class CustomAuthClient extends AuthClient { + async getAccessToken() { + return {token: '', res: undefined}; + } + + async getRequestHeaders() { + return {}; + } + + request = OAuth2Client.prototype.request.bind(this); + }, + GoogleAuth: class { + constructor(config?: GoogleAuthOptions) { + return new GoogleAuth(config); + } + }, + }; + + before(() => { + util = proxyquire('../../src/nodejs-common/util', { + 'google-auth-library': fakeGoogleAuth, + 'retry-request': fakeRetryRequest, + 'teeny-request': {teenyRequest: fakeRequest}, + '@google-cloud/projectify': { + replaceProjectIdToken: fakeReplaceProjectIdToken, + }, + }).util; + }); + + let sandbox: sinon.SinonSandbox; + beforeEach(() => { + sandbox = sinon.createSandbox(); + requestOverride = null; + retryRequestOverride = null; + replaceProjectIdTokenOverride = null; + }); + afterEach(() => { + sandbox.restore(); + }); + + describe('ApiError', () => { + it('should accept just a message', () => { + const expectedMessage = 'Hi, I am an error message!'; + const apiError = new ApiError(expectedMessage); + + assert.strictEqual(apiError.message, expectedMessage); + }); + + it('should use message in stack', () => { + const expectedMessage = 'Message is in the stack too!'; + const apiError = new ApiError(expectedMessage); + assert(apiError.stack?.includes(expectedMessage)); + }); + + it('should build correct ApiError', () => { + const fakeMessage = 'Formatted Error.'; + const fakeResponse = {statusCode: 200} as r.Response; + const errors = [{message: 'Hi'}, {message: 'Bye'}]; + const error = { + errors, + code: 100, + message: 'Uh oh', + response: fakeResponse, + }; + + sandbox + .stub(ApiError, 'createMultiErrorMessage') + .withArgs(error, errors) + .returns(fakeMessage); + + const apiError = new ApiError(error); + assert.strictEqual(apiError.errors, error.errors); + assert.strictEqual(apiError.code, error.code); + assert.strictEqual(apiError.response, error.response); + assert.strictEqual(apiError.message, fakeMessage); + }); + + it('should parse the response body for errors', () => { + const fakeMessage = 'Formatted Error.'; + const error = {message: 'Error.'}; + const errors = [error, error]; + + const errorBody = { + code: 123, + response: { + body: JSON.stringify({ + error: { + errors, + }, + }), + } as r.Response, + }; + + sandbox + .stub(ApiError, 'createMultiErrorMessage') + .withArgs(errorBody, errors) + .returns(fakeMessage); + + const apiError = new ApiError(errorBody); + assert.strictEqual(apiError.message, fakeMessage); + }); + + describe('createMultiErrorMessage', () => { + it('should append the custom error message', () => { + const errorMessage = 'API error message'; + const customErrorMessage = 'Custom error message'; + + const errors = [new Error(errorMessage)]; + const error = { + code: 100, + response: {} as r.Response, + message: customErrorMessage, + }; + + const expectedErrorMessage = createExpectedErrorMessage([ + customErrorMessage, + errorMessage, + ]); + const multiError = ApiError.createMultiErrorMessage(error, errors); + assert.strictEqual(multiError, expectedErrorMessage); + }); + + it('should use any inner errors', () => { + const messages = ['Hi, I am an error!', 'Me too!']; + const errors: GoogleInnerError[] = messages.map(message => ({message})); + const error: GoogleErrorBody = { + code: 100, + response: {} as r.Response, + }; + + const expectedErrorMessage = createExpectedErrorMessage(messages); + const multiError = ApiError.createMultiErrorMessage(error, errors); + assert.strictEqual(multiError, expectedErrorMessage); + }); + + it('should parse and append the decoded response body', () => { + const errorMessage = 'API error message'; + const responseBodyMsg = 'Response body message <'; + + const error = { + message: errorMessage, + code: 100, + response: { + body: Buffer.from(responseBodyMsg), + } as r.Response, + }; + + const expectedErrorMessage = createExpectedErrorMessage([ + 'API error message', + 'Response body message <', + ]); + const multiError = ApiError.createMultiErrorMessage(error); + assert.strictEqual(multiError, expectedErrorMessage); + }); + + it('should use default message if there are no errors', () => { + const fakeResponse = {statusCode: 200} as r.Response; + const expectedErrorMessage = 'A failure occurred during this request.'; + const error = { + code: 100, + response: fakeResponse, + }; + + const multiError = ApiError.createMultiErrorMessage(error); + assert.strictEqual(multiError, expectedErrorMessage); + }); + + it('should filter out duplicate errors', () => { + const expectedErrorMessage = 'Error during request.'; + const error = { + code: 100, + message: expectedErrorMessage, + response: { + body: expectedErrorMessage, + } as r.Response, + }; + + const multiError = ApiError.createMultiErrorMessage(error); + assert.strictEqual(multiError, expectedErrorMessage); + }); + }); + }); + + describe('PartialFailureError', () => { + it('should build correct PartialFailureError', () => { + const fakeMessage = 'Formatted Error.'; + const errors = [{}, {}]; + const error = { + code: 123, + errors, + response: fakeResponse, + message: 'Partial failure occurred', + }; + + sandbox + .stub(util.ApiError, 'createMultiErrorMessage') + .withArgs(error, errors) + .returns(fakeMessage); + + const partialFailureError = new util.PartialFailureError(error); + + assert.strictEqual(partialFailureError.errors, error.errors); + assert.strictEqual(partialFailureError.name, 'PartialFailureError'); + assert.strictEqual(partialFailureError.response, error.response); + assert.strictEqual(partialFailureError.message, fakeMessage); + }); + }); + + describe('handleResp', () => { + it('should handle errors', done => { + const error = new Error('Error.'); + + util.handleResp(error, fakeResponse, null, err => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('uses a no-op callback if none is sent', () => { + util.handleResp(null, fakeResponse, ''); + }); + + it('should parse response', done => { + stub('parseHttpRespMessage', resp_ => { + assert.deepStrictEqual(resp_, fakeResponse); + return { + resp: fakeResponse, + }; + }); + + stub('parseHttpRespBody', body_ => { + assert.strictEqual(body_, fakeResponse.body); + return { + body: fakeResponse.body, + }; + }); + + util.handleResp( + fakeError, + fakeResponse, + fakeResponse.body, + (err, body, resp) => { + assert.deepStrictEqual(err, fakeError); + assert.deepStrictEqual(body, fakeResponse.body); + assert.deepStrictEqual(resp, fakeResponse); + done(); + } + ); + }); + + it('should parse response for error', done => { + const error = new Error('Error.'); + + sandbox.stub(util, 'parseHttpRespMessage').callsFake(() => { + return {err: error} as ParsedHttpRespMessage; + }); + + util.handleResp(null, fakeResponse, {}, err => { + assert.deepStrictEqual(err, error); + done(); + }); + }); + + it('should parse body for error', done => { + const error = new Error('Error.'); + + stub('parseHttpRespBody', () => { + return {err: error}; + }); + + util.handleResp(null, fakeResponse, {}, err => { + assert.deepStrictEqual(err, error); + done(); + }); + }); + + it('should not parse undefined response', done => { + stub('parseHttpRespMessage', () => done()); // Will throw. + util.handleResp(null, null, null, done); + }); + + it('should not parse undefined body', done => { + stub('parseHttpRespBody', () => done()); // Will throw. + util.handleResp(null, null, null, done); + }); + + it('should handle non-JSON body', done => { + const unparsableBody = 'Unparsable body.'; + + util.handleResp(null, null, unparsableBody, (err, body) => { + assert(body.includes(unparsableBody)); + done(); + }); + }); + + it('should include the status code when the error body cannot be JSON-parsed', done => { + const unparsableBody = 'Bad gateway'; + const statusCode = 502; + + util.handleResp( + null, + {body: unparsableBody, statusCode} as r.Response, + unparsableBody, + err => { + assert(err, 'there should be an error'); + const apiError = err! as ApiError; + assert.strictEqual(apiError.code, statusCode); + + const response = apiError.response; + if (!response) { + assert.fail('there should be a response property on the error'); + } else { + assert.strictEqual(response.body, unparsableBody); + } + + done(); + } + ); + }); + }); + + describe('parseHttpRespMessage', () => { + it('should build ApiError with non-200 status and message', () => { + const res = util.parseHttpRespMessage(fakeBadResp); + const error_ = res.err!; + assert.strictEqual(error_.code, fakeBadResp.statusCode); + assert.strictEqual(error_.message, fakeBadResp.statusMessage); + assert.strictEqual(error_.response, fakeBadResp); + }); + + it('should return the original response message', () => { + const parsedHttpRespMessage = util.parseHttpRespMessage(fakeBadResp); + assert.strictEqual(parsedHttpRespMessage.resp, fakeBadResp); + }); + }); + + describe('parseHttpRespBody', () => { + it('should detect body errors', () => { + const apiErr = { + errors: [{message: 'bar'}], + code: 400, + message: 'an error occurred', + }; + + const parsedHttpRespBody = util.parseHttpRespBody({error: apiErr}); + const expectedErrorMessage = createExpectedErrorMessage([ + apiErr.message, + apiErr.errors[0].message, + ]); + + const err = parsedHttpRespBody.err as ApiError; + assert.deepStrictEqual(err.errors, apiErr.errors); + assert.strictEqual(err.code, apiErr.code); + assert.deepStrictEqual(err.message, expectedErrorMessage); + }); + + it('should try to parse JSON if body is string', () => { + const httpRespBody = '{ "foo": "bar" }'; + const parsedHttpRespBody = util.parseHttpRespBody(httpRespBody); + + assert.strictEqual(parsedHttpRespBody.body.foo, 'bar'); + }); + + it('should return the original body', () => { + const httpRespBody = {}; + const parsedHttpRespBody = util.parseHttpRespBody(httpRespBody); + assert.strictEqual(parsedHttpRespBody.body, httpRespBody); + }); + }); + + describe('makeWritableStream', () => { + it('should use defaults', done => { + const dup = duplexify(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const metadata = {a: 'b', c: 'd'} as any; + util.makeWritableStream(dup, { + metadata, + makeAuthenticatedRequest(request: DecorateRequestOptions) { + assert.strictEqual(request.method, 'POST'); + assert.strictEqual(request.qs.uploadType, 'multipart'); + assert.strictEqual(request.timeout, 0); + assert.strictEqual(request.maxRetries, 0); + assert.strictEqual(Array.isArray(request.multipart), true); + + const mp = request.multipart as r.RequestPart[]; + + assert.strictEqual( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (mp[0] as any)['Content-Type'], + 'application/json' + ); + assert.strictEqual(mp[0].body, JSON.stringify(metadata)); + + assert.strictEqual( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (mp[1] as any)['Content-Type'], + 'application/octet-stream' + ); + // (is a writable stream:) + assert.strictEqual( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + typeof (mp[1].body as any)._writableState, + 'object' + ); + + done(); + }, + }); + }); + + it('should allow overriding defaults', done => { + const dup = duplexify(); + + const req = { + uri: 'http://foo', + method: 'PUT', + qs: { + uploadType: 'media', + }, + [GCCL_GCS_CMD_KEY]: 'some.value', + } as DecorateRequestOptions; + + util.makeWritableStream(dup, { + metadata: { + contentType: 'application/json', + }, + makeAuthenticatedRequest(request) { + assert.strictEqual(request.method, req.method); + assert.deepStrictEqual(request.qs, req.qs); + assert.strictEqual(request.uri, req.uri); + assert.strictEqual(request[GCCL_GCS_CMD_KEY], req[GCCL_GCS_CMD_KEY]); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const mp = request.multipart as any[]; + assert.strictEqual(mp[1]['Content-Type'], 'application/json'); + + done(); + }, + + request: req, + }); + }); + + it('should emit an error', done => { + const error = new Error('Error.'); + + const ws = duplexify(); + ws.on('error', err => { + assert.strictEqual(err, error); + done(); + }); + + util.makeWritableStream(ws, { + makeAuthenticatedRequest(request, opts) { + opts!.onAuthenticated(error); + }, + }); + }); + + it('should set the writable stream', done => { + const dup = duplexify(); + + dup.setWritable = () => { + done(); + }; + + util.makeWritableStream(dup, {makeAuthenticatedRequest() {}}); + }); + + it('dup should emit a progress event with the bytes written', done => { + let happened = false; + + const dup = duplexify(); + dup.on('progress', () => { + happened = true; + }); + + util.makeWritableStream(dup, {makeAuthenticatedRequest() {}}, util.noop); + dup.write(Buffer.from('abcdefghijklmnopqrstuvwxyz'), 'utf-8', util.noop); + + assert.strictEqual(happened, true); + done(); + }); + + it('should emit an error if the request fails', done => { + const dup = duplexify(); + const fakeStream = new stream.Writable(); + const error = new Error('Error.'); + fakeStream.write = () => false; + dup.end = () => dup; + + stub('handleResp', (err, res, body, callback) => { + callback(error); + }); + + requestOverride = ( + reqOpts: DecorateRequestOptions, + callback: (err: Error) => void + ) => { + callback(error); + }; + + requestOverride.defaults = () => requestOverride; + + dup.on('error', err => { + assert.strictEqual(err, error); + done(); + }); + + util.makeWritableStream(dup, { + makeAuthenticatedRequest(request, opts) { + opts.onAuthenticated(null); + }, + }); + + setImmediate(() => { + fakeStream.emit('complete', {}); + }); + }); + + it('should emit the response', done => { + const dup = duplexify(); + const fakeStream = new stream.Writable(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (fakeStream as any).write = () => {}; + + stub('handleResp', (err, res, body, callback) => { + callback(); + }); + + requestOverride = ( + reqOpts: DecorateRequestOptions, + callback: (err: Error | null, res: r.Response) => void + ) => { + callback(null, fakeResponse); + }; + + requestOverride.defaults = () => requestOverride; + const options = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + makeAuthenticatedRequest(request: DecorateRequestOptions, opts: any) { + opts.onAuthenticated(); + }, + }; + + dup.on('response', resp => { + assert.strictEqual(resp, fakeResponse); + done(); + }); + + util.makeWritableStream(dup, options, util.noop); + }); + + it('should pass back the response data to the callback', done => { + const dup = duplexify(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const fakeStream: any = new stream.Writable(); + const fakeResponse = {}; + + fakeStream.write = () => {}; + + stub('handleResp', (err, res, body, callback) => { + callback(null, fakeResponse); + }); + + requestOverride = ( + reqOpts: DecorateRequestOptions, + callback: () => void + ) => { + callback(); + }; + requestOverride.defaults = () => { + return requestOverride; + }; + + const options = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + makeAuthenticatedRequest(request: DecorateRequestOptions, opts: any) { + opts.onAuthenticated(); + }, + }; + + util.makeWritableStream(dup, options, (data: {}) => { + assert.strictEqual(data, fakeResponse); + done(); + }); + + setImmediate(() => { + fakeStream.emit('complete', {}); + }); + }); + }); + + describe('makeAuthenticatedRequestFactory', () => { + const AUTH_CLIENT_PROJECT_ID = 'authclient-project-id'; + const authClient = { + getCredentials() {}, + getProjectId: () => Promise.resolve(AUTH_CLIENT_PROJECT_ID), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + it('should create an authClient', done => { + const config = {test: true} as MakeAuthenticatedRequestFactoryConfig; + + sandbox + .stub(fakeGoogleAuth, 'GoogleAuth') + .callsFake((config_: GoogleAuthOptions) => { + assert.deepStrictEqual(config_, { + ...config, + authClient: undefined, + clientOptions: undefined, + }); + setImmediate(done); + return authClient; + }); + + util.makeAuthenticatedRequestFactory(config); + }); + + it('should pass an `AuthClient` to `GoogleAuth` when provided', done => { + const customAuthClient = new fakeGoogleAuth.AuthClient(); + + const config: MakeAuthenticatedRequestFactoryConfig = { + authClient: customAuthClient, + clientOptions: undefined, + }; + + sandbox + .stub(fakeGoogleAuth, 'GoogleAuth') + .callsFake((config_: GoogleAuthOptions) => { + assert.deepStrictEqual(config_, config); + setImmediate(done); + return authClient; + }); + + util.makeAuthenticatedRequestFactory(config); + }); + + it('should not pass projectId token to google-auth-library', done => { + const config = {projectId: DEFAULT_PROJECT_ID_TOKEN}; + + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').callsFake(config_ => { + assert.strictEqual(config_.projectId, undefined); + setImmediate(done); + return authClient; + }); + + util.makeAuthenticatedRequestFactory(config); + }); + + it('should not remove projectId from config object', done => { + const config = {projectId: DEFAULT_PROJECT_ID_TOKEN}; + + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').callsFake(() => { + assert.strictEqual(config.projectId, DEFAULT_PROJECT_ID_TOKEN); + setImmediate(done); + return authClient; + }); + + util.makeAuthenticatedRequestFactory(config); + }); + + it('should return a function', () => { + assert.strictEqual( + typeof util.makeAuthenticatedRequestFactory({}), + 'function' + ); + }); + + it('should return a getCredentials method', done => { + function getCredentials() { + done(); + } + + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').callsFake(() => { + return {getCredentials}; + }); + + const makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory({}); + makeAuthenticatedRequest.getCredentials(util.noop); + }); + + it('should return the authClient', () => { + const authClient = {getCredentials() {}}; + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + const mar = util.makeAuthenticatedRequestFactory({}); + assert.strictEqual(mar.authClient, authClient); + }); + + describe('customEndpoint (no authentication attempted)', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let makeAuthenticatedRequest: any; + const config = {customEndpoint: true}; + + beforeEach(() => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory(config); + }); + + it('should decorate the request', done => { + const decoratedRequest = {}; + stub('decorateRequest', reqOpts_ => { + assert.strictEqual(reqOpts_, fakeReqOpts); + return decoratedRequest; + }); + + makeAuthenticatedRequest(fakeReqOpts, { + onAuthenticated( + err: Error, + authenticatedReqOpts: DecorateRequestOptions + ) { + assert.ifError(err); + assert.strictEqual(authenticatedReqOpts, decoratedRequest); + done(); + }, + }); + }); + + it('should return an error while decorating', done => { + const error = new Error('Error.'); + stub('decorateRequest', () => { + throw error; + }); + makeAuthenticatedRequest(fakeReqOpts, { + onAuthenticated(err: Error) { + assert.strictEqual(err, error); + done(); + }, + }); + }); + + it('should pass options back to callback', done => { + const reqOpts = {a: 'b', c: 'd'}; + makeAuthenticatedRequest(reqOpts, { + onAuthenticated( + err: Error, + authenticatedReqOpts: DecorateRequestOptions + ) { + assert.ifError(err); + assert.deepStrictEqual(reqOpts, authenticatedReqOpts); + done(); + }, + }); + }); + + it('should not authenticate requests with a custom API', done => { + const reqOpts = {a: 'b', c: 'd'}; + + stub('makeRequest', rOpts => { + assert.deepStrictEqual(rOpts, reqOpts); + done(); + }); + + makeAuthenticatedRequest(reqOpts, assert.ifError); + }); + }); + + describe('customEndpoint (authentication attempted)', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let makeAuthenticatedRequest: any; + const config = {customEndpoint: true, useAuthWithCustomEndpoint: true}; + + beforeEach(() => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory(config); + }); + + it('should authenticate requests with a custom API', done => { + const reqOpts = {a: 'b', c: 'd'}; + + stub('makeRequest', rOpts => { + assert.deepStrictEqual(rOpts, reqOpts); + done(); + }); + + authClient.authorizeRequest = async (opts: {}) => { + assert.strictEqual(opts, reqOpts); + done(); + }; + + makeAuthenticatedRequest(reqOpts, assert.ifError); + }); + }); + + describe('authentication', () => { + it('should pass correct args to authorizeRequest', done => { + const fake = { + ...authClient, + authorizeRequest: async (rOpts: {}) => { + assert.deepStrictEqual(rOpts, fakeReqOpts); + setImmediate(done); + return rOpts; + }, + }; + retryRequestOverride = () => { + return new stream.PassThrough(); + }; + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(fake); + const mar = util.makeAuthenticatedRequestFactory({}); + mar(fakeReqOpts); + }); + + it('should return a stream if callback is missing', () => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').callsFake(() => { + return { + ...authClient, + authorizeRequest: async (rOpts: {}) => { + return rOpts; + }, + }; + }); + retryRequestOverride = () => { + return new stream.PassThrough(); + }; + const mar = util.makeAuthenticatedRequestFactory({}); + const s = mar(fakeReqOpts); + assert(s instanceof stream.Stream); + }); + + describe('projectId', () => { + const reqOpts = {} as DecorateRequestOptions; + + it('should default to authClient projectId', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + stub('decorateRequest', (reqOpts, projectId) => { + assert.strictEqual(projectId, AUTH_CLIENT_PROJECT_ID); + setImmediate(done); + }); + + const makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory( + {customEndpoint: true} + ); + + makeAuthenticatedRequest(reqOpts, { + onAuthenticated: assert.ifError, + }); + }); + + it('should prefer user-provided projectId', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + + const config = { + customEndpoint: true, + projectId: 'user-provided-project-id', + }; + + stub('decorateRequest', (reqOpts, projectId) => { + assert.strictEqual(projectId, config.projectId); + setImmediate(done); + }); + + const makeAuthenticatedRequest = + util.makeAuthenticatedRequestFactory(config); + + makeAuthenticatedRequest(reqOpts, { + onAuthenticated: assert.ifError, + }); + }); + + it('should use default `projectId` and not call `authClient#getProjectId` when !`projectIdRequired`', done => { + const getProjectIdSpy = sandbox.spy(authClient, 'getProjectId'); + + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + + const config = { + customEndpoint: true, + projectIdRequired: false, + }; + + stub('decorateRequest', (reqOpts, projectId) => { + assert.strictEqual(projectId, DEFAULT_PROJECT_ID_TOKEN); + }); + + const makeAuthenticatedRequest = + util.makeAuthenticatedRequestFactory(config); + + makeAuthenticatedRequest(reqOpts, { + onAuthenticated: e => { + assert.ifError(e); + assert(getProjectIdSpy.notCalled); + done(e); + }, + }); + }); + + it('should fallback to checking for a `projectId` on when missing a `projectId` when !`projectIdRequired`', done => { + const getProjectIdSpy = sandbox.spy(authClient, 'getProjectId'); + + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + + const config = { + customEndpoint: true, + projectIdRequired: false, + }; + + const decorateRequestStub = sandbox.stub(util, 'decorateRequest'); + + decorateRequestStub.onFirstCall().callsFake(() => { + throw new MissingProjectIdError(); + }); + + decorateRequestStub.onSecondCall().callsFake((reqOpts, projectId) => { + assert.strictEqual(projectId, AUTH_CLIENT_PROJECT_ID); + return reqOpts; + }); + + const makeAuthenticatedRequest = + util.makeAuthenticatedRequestFactory(config); + + makeAuthenticatedRequest(reqOpts, { + onAuthenticated: e => { + assert.ifError(e); + assert(getProjectIdSpy.calledOnce); + done(e); + }, + }); + }); + }); + + describe('authentication errors', () => { + const error = new Error('🤮'); + + beforeEach(() => { + authClient.authorizeRequest = async () => { + throw error; + }; + }); + + it('should attempt request anyway', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + const makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory( + {} + ); + + const correctReqOpts = {} as DecorateRequestOptions; + const incorrectReqOpts = {} as DecorateRequestOptions; + + authClient.authorizeRequest = async () => { + throw new Error('Could not load the default credentials'); + }; + + makeAuthenticatedRequest(correctReqOpts, { + onAuthenticated(err, reqOpts) { + assert.ifError(err); + assert.strictEqual(reqOpts, correctReqOpts); + assert.notStrictEqual(reqOpts, incorrectReqOpts); + done(); + }, + }); + }); + + it('should block 401 API errors', done => { + const authClientError = new Error( + 'Could not load the default credentials' + ); + authClient.authorizeRequest = async () => { + throw authClientError; + }; + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + + const makeRequestArg1 = new Error('API 401 Error.') as ApiError; + makeRequestArg1.code = 401; + const makeRequestArg2 = {}; + const makeRequestArg3 = {}; + stub('makeRequest', (authenticatedReqOpts, cfg, callback) => { + callback(makeRequestArg1, makeRequestArg2, makeRequestArg3); + }); + + const makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory( + {} + ); + makeAuthenticatedRequest( + {} as DecorateRequestOptions, + (arg1, arg2, arg3) => { + assert.strictEqual(arg1, authClientError); + assert.strictEqual(arg2, makeRequestArg2); + assert.strictEqual(arg3, makeRequestArg3); + done(); + } + ); + }); + + it('should not block 401 errors if auth client succeeds', done => { + authClient.authorizeRequest = async () => { + return {}; + }; + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + + const makeRequestArg1 = new Error('API 401 Error.') as ApiError; + makeRequestArg1.code = 401; + const makeRequestArg2 = {}; + const makeRequestArg3 = {}; + stub('makeRequest', (authenticatedReqOpts, cfg, callback) => { + callback(makeRequestArg1, makeRequestArg2, makeRequestArg3); + }); + + const makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory( + {} + ); + makeAuthenticatedRequest( + {} as DecorateRequestOptions, + (arg1, arg2, arg3) => { + assert.strictEqual(arg1, makeRequestArg1); + assert.strictEqual(arg2, makeRequestArg2); + assert.strictEqual(arg3, makeRequestArg3); + done(); + } + ); + }); + + it('should block decorateRequest error', done => { + const decorateRequestError = new Error('Error.'); + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + stub('decorateRequest', () => { + throw decorateRequestError; + }); + + const makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory( + {} + ); + makeAuthenticatedRequest(fakeReqOpts, { + onAuthenticated(err) { + assert.notStrictEqual(err, decorateRequestError); + assert.strictEqual(err, error); + done(); + }, + }); + }); + + it('should invoke the callback with error', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + const mar = util.makeAuthenticatedRequestFactory({}); + mar(fakeReqOpts, err => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('should exec onAuthenticated callback with error', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + const mar = util.makeAuthenticatedRequestFactory({}); + mar(fakeReqOpts, { + onAuthenticated(err) { + assert.strictEqual(err, error); + done(); + }, + }); + }); + + it('should emit an error and end the stream', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + const mar = util.makeAuthenticatedRequestFactory({}); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const stream = mar(fakeReqOpts) as any; + stream.on('error', (err: Error) => { + assert.strictEqual(err, error); + setImmediate(() => { + assert.strictEqual(stream.destroyed, true); + done(); + }); + }); + }); + }); + + describe('authentication success', () => { + const reqOpts = fakeReqOpts; + beforeEach(() => { + authClient.authorizeRequest = async () => reqOpts; + }); + + it('should return authenticated request to callback', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + stub('decorateRequest', reqOpts_ => { + assert.deepStrictEqual(reqOpts_, reqOpts); + return reqOpts; + }); + + const mar = util.makeAuthenticatedRequestFactory({}); + mar(reqOpts, { + onAuthenticated(err, authenticatedReqOpts) { + assert.strictEqual(authenticatedReqOpts, reqOpts); + done(); + }, + }); + }); + + it('should make request with correct options', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + const config = {keyFile: 'foo'}; + stub('decorateRequest', reqOpts_ => { + assert.deepStrictEqual(reqOpts_, reqOpts); + return reqOpts; + }); + stub('makeRequest', (authenticatedReqOpts, cfg, cb) => { + assert.deepStrictEqual(authenticatedReqOpts, reqOpts); + assert.deepStrictEqual(cfg, config); + cb(); + }); + const mar = util.makeAuthenticatedRequestFactory(config); + mar(reqOpts, done); + }); + + it('should return abort() from the active request', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + const retryRequest = { + abort: done, + }; + sandbox.stub(util, 'makeRequest').returns(retryRequest); + const mar = util.makeAuthenticatedRequestFactory({}); + const req = mar(reqOpts, assert.ifError) as Abortable; + req.abort(); + }); + + it('should only abort() once', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + const retryRequest = { + abort: done, // Will throw if called more than once. + }; + stub('makeRequest', () => { + return retryRequest; + }); + + const mar = util.makeAuthenticatedRequestFactory({}); + const authenticatedRequest = mar( + reqOpts, + assert.ifError + ) as Abortable; + + authenticatedRequest.abort(); // done() + authenticatedRequest.abort(); // done() + }); + + it('should provide stream to makeRequest', done => { + sandbox.stub(fakeGoogleAuth, 'GoogleAuth').returns(authClient); + stub('makeRequest', (authenticatedReqOpts, cfg) => { + setImmediate(() => { + assert.strictEqual(cfg.stream, stream); + done(); + }); + }); + const mar = util.makeAuthenticatedRequestFactory({}); + const stream = mar(reqOpts); + }); + }); + }); + }); + + describe('shouldRetryRequest', () => { + it('should return false if there is no error', () => { + assert.strictEqual(util.shouldRetryRequest(), false); + }); + + it('should return false from generic error', () => { + const error = new ApiError('Generic error with no code'); + assert.strictEqual(util.shouldRetryRequest(error), false); + }); + + it('should return true with error code 408', () => { + const error = new ApiError('408'); + error.code = 408; + assert.strictEqual(util.shouldRetryRequest(error), true); + }); + + it('should return true with error code 429', () => { + const error = new ApiError('429'); + error.code = 429; + assert.strictEqual(util.shouldRetryRequest(error), true); + }); + + it('should return true with error code 500', () => { + const error = new ApiError('500'); + error.code = 500; + assert.strictEqual(util.shouldRetryRequest(error), true); + }); + + it('should return true with error code 502', () => { + const error = new ApiError('502'); + error.code = 502; + assert.strictEqual(util.shouldRetryRequest(error), true); + }); + + it('should return true with error code 503', () => { + const error = new ApiError('503'); + error.code = 503; + assert.strictEqual(util.shouldRetryRequest(error), true); + }); + + it('should return true with error code 504', () => { + const error = new ApiError('504'); + error.code = 504; + assert.strictEqual(util.shouldRetryRequest(error), true); + }); + + it('should detect rateLimitExceeded reason', () => { + const rateLimitError = new ApiError('Rate limit error without code.'); + rateLimitError.errors = [{reason: 'rateLimitExceeded'}]; + assert.strictEqual(util.shouldRetryRequest(rateLimitError), true); + }); + + it('should detect userRateLimitExceeded reason', () => { + const rateLimitError = new ApiError('Rate limit error without code.'); + rateLimitError.errors = [{reason: 'userRateLimitExceeded'}]; + assert.strictEqual(util.shouldRetryRequest(rateLimitError), true); + }); + + it('should retry on EAI_AGAIN error code', () => { + const eaiAgainError = new ApiError('EAI_AGAIN'); + eaiAgainError.errors = [ + {reason: 'getaddrinfo EAI_AGAIN pubsub.googleapis.com'}, + ]; + assert.strictEqual(util.shouldRetryRequest(eaiAgainError), true); + }); + }); + + describe('makeRequest', () => { + const reqOpts = { + method: 'GET', + } as DecorateRequestOptions; + + function testDefaultRetryRequestConfig(done: () => void) { + return (reqOpts_: DecorateRequestOptions, config: MakeRequestConfig) => { + assert.strictEqual(reqOpts_, reqOpts); + assert.strictEqual(config.retries, 3); + + const error = new Error('Error.'); + stub('parseHttpRespMessage', () => { + return {err: error}; + }); + stub('shouldRetryRequest', err => { + assert.strictEqual(err, error); + done(); + }); + + config.shouldRetryFn!(); + }; + } + const errorMessage = 'Error.'; + const customRetryRequestFunctionConfig = { + retryOptions: { + retryableErrorFn: function (err: ApiError) { + return err.message === errorMessage; + }, + }, + }; + function testCustomFunctionRetryRequestConfig(done: () => void) { + return (reqOpts_: DecorateRequestOptions, config: MakeRequestConfig) => { + assert.strictEqual(reqOpts_, reqOpts); + assert.strictEqual(config.retries, 3); + + const error = new Error(errorMessage); + stub('parseHttpRespMessage', () => { + return {err: error}; + }); + stub('shouldRetryRequest', err => { + assert.strictEqual(err, error); + done(); + }); + + assert.strictEqual(config.shouldRetryFn!(), true); + done(); + }; + } + + const noRetryRequestConfig = {autoRetry: false}; + function testNoRetryRequestConfig(done: () => void) { + return ( + reqOpts: DecorateRequestOptions, + config: retryRequest.Options + ) => { + assert.strictEqual(config.retries, 0); + done(); + }; + } + + const retryOptionsConfig = { + retryOptions: { + autoRetry: false, + maxRetries: 7, + retryDelayMultiplier: 3, + totalTimeout: 60, + maxRetryDelay: 640, + }, + }; + function testRetryOptions(done: () => void) { + return ( + reqOpts: DecorateRequestOptions, + config: retryRequest.Options + ) => { + assert.strictEqual( + config.retries, + 0 //autoRetry was set to false, so shouldn't retry + ); + assert.strictEqual( + config.noResponseRetries, + 0 //autoRetry was set to false, so shouldn't retry + ); + assert.strictEqual( + config.retryDelayMultiplier, + retryOptionsConfig.retryOptions.retryDelayMultiplier + ); + assert.strictEqual( + config.totalTimeout, + retryOptionsConfig.retryOptions.totalTimeout + ); + assert.strictEqual( + config.maxRetryDelay, + retryOptionsConfig.retryOptions.maxRetryDelay + ); + done(); + }; + } + + const customRetryRequestConfig = {maxRetries: 10}; + function testCustomRetryRequestConfig(done: () => void) { + return (reqOpts: DecorateRequestOptions, config: MakeRequestConfig) => { + assert.strictEqual(config.retries, customRetryRequestConfig.maxRetries); + done(); + }; + } + + describe('stream mode', () => { + it('should forward the specified events to the stream', done => { + const requestStream = duplexify(); + const userStream = duplexify(); + + const error = new Error('Error.'); + const response = {}; + const complete = {}; + + userStream + .on('error', error_ => { + assert.strictEqual(error_, error); + requestStream.emit('response', response); + }) + .on('response', response_ => { + assert.strictEqual(response_, response); + requestStream.emit('complete', complete); + }) + .on('complete', complete_ => { + assert.strictEqual(complete_, complete); + done(); + }); + + retryRequestOverride = () => { + setImmediate(() => { + requestStream.emit('error', error); + }); + + return requestStream; + }; + + util.makeRequest(reqOpts, {stream: userStream}, util.noop); + }); + + describe('GET requests', () => { + it('should use retryRequest', done => { + const userStream = duplexify(); + retryRequestOverride = (reqOpts_: DecorateRequestOptions) => { + assert.strictEqual(reqOpts_, reqOpts); + setImmediate(done); + return new stream.Stream(); + }; + util.makeRequest(reqOpts, {stream: userStream}, util.noop); + }); + + it('should set the readable stream', done => { + const userStream = duplexify(); + const retryRequestStream = new stream.Stream(); + retryRequestOverride = () => { + return retryRequestStream; + }; + userStream.setReadable = stream => { + assert.strictEqual(stream, retryRequestStream); + done(); + }; + util.makeRequest(reqOpts, {stream: userStream}, util.noop); + }); + + it('should expose the abort method from retryRequest', done => { + const userStream = duplexify() as Duplexify & Abortable; + + retryRequestOverride = () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const requestStream: any = new stream.Stream(); + requestStream.abort = done; + return requestStream; + }; + + util.makeRequest(reqOpts, {stream: userStream}, util.noop); + userStream.abort(); + }); + }); + + describe('non-GET requests', () => { + it('should not use retryRequest', done => { + const userStream = duplexify(); + const reqOpts = { + method: 'POST', + } as DecorateRequestOptions; + + retryRequestOverride = done; // will throw. + requestOverride = (reqOpts_: DecorateRequestOptions) => { + assert.strictEqual(reqOpts_, reqOpts); + setImmediate(done); + return userStream; + }; + requestOverride.defaults = () => requestOverride; + util.makeRequest(reqOpts, {stream: userStream}, util.noop); + }); + + it('should set the writable stream', done => { + const userStream = duplexify(); + const requestStream = new stream.Stream(); + requestOverride = () => requestStream; + requestOverride.defaults = () => requestOverride; + userStream.setWritable = stream => { + assert.strictEqual(stream, requestStream); + done(); + }; + util.makeRequest( + {method: 'POST'} as DecorateRequestOptions, + {stream: userStream}, + util.noop + ); + }); + + it('should expose the abort method from request', done => { + const userStream = duplexify() as Duplexify & Abortable; + + requestOverride = Object.assign( + () => { + const requestStream = duplexify() as Duplexify & Abortable; + requestStream.abort = done; + return requestStream; + }, + {defaults: () => requestOverride} + ); + + util.makeRequest(reqOpts, {stream: userStream}, util.noop); + userStream.abort(); + }); + }); + }); + + describe('callback mode', () => { + it('should pass the default options to retryRequest', done => { + retryRequestOverride = testDefaultRetryRequestConfig(done); + util.makeRequest( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + reqOpts, + {}, + assert.ifError + ); + }); + + it('should allow setting a custom retry function', done => { + retryRequestOverride = testCustomFunctionRetryRequestConfig(done); + util.makeRequest( + reqOpts, + customRetryRequestFunctionConfig, + assert.ifError + ); + }); + + it('should allow turning off retries to retryRequest', done => { + retryRequestOverride = testNoRetryRequestConfig(done); + util.makeRequest(reqOpts, noRetryRequestConfig, assert.ifError); + }); + + it('should override number of retries to retryRequest', done => { + retryRequestOverride = testCustomRetryRequestConfig(done); + util.makeRequest(reqOpts, customRetryRequestConfig, assert.ifError); + }); + + it('should use retryOptions if provided', done => { + retryRequestOverride = testRetryOptions(done); + util.makeRequest(reqOpts, retryOptionsConfig, assert.ifError); + }); + + it('should allow request options to control retry setting', done => { + retryRequestOverride = testCustomRetryRequestConfig(done); + const reqOptsWithRetrySettings = { + ...reqOpts, + ...customRetryRequestConfig, + }; + util.makeRequest( + reqOptsWithRetrySettings, + noRetryRequestConfig, + assert.ifError + ); + }); + + it('should return the instance of retryRequest', () => { + const requestInstance = {}; + retryRequestOverride = () => { + return requestInstance; + }; + const res = util.makeRequest(reqOpts, {}, assert.ifError); + assert.strictEqual(res, requestInstance); + }); + + it('should let handleResp handle the response', done => { + const error = new Error('Error.'); + const body = fakeResponse.body; + + retryRequestOverride = ( + rOpts: DecorateRequestOptions, + opts: MakeRequestConfig, + callback: r.RequestCallback + ) => { + callback(error, fakeResponse, body); + }; + + stub('handleResp', (err, resp, body_) => { + assert.strictEqual(err, error); + assert.strictEqual(resp, fakeResponse); + assert.strictEqual(body_, body); + done(); + }); + + util.makeRequest(fakeReqOpts, {}, assert.ifError); + }); + }); + }); + + describe('decorateRequest', () => { + const projectId = 'not-a-project-id'; + it('should delete qs.autoPaginate', () => { + const decoratedReqOpts = util.decorateRequest( + { + autoPaginate: true, + } as DecorateRequestOptions, + projectId + ); + + assert.strictEqual(decoratedReqOpts.autoPaginate, undefined); + }); + + it('should delete qs.autoPaginateVal', () => { + const decoratedReqOpts = util.decorateRequest( + { + autoPaginateVal: true, + } as DecorateRequestOptions, + projectId + ); + + assert.strictEqual(decoratedReqOpts.autoPaginateVal, undefined); + }); + + it('should delete objectMode', () => { + const decoratedReqOpts = util.decorateRequest( + { + objectMode: true, + } as DecorateRequestOptions, + projectId + ); + + assert.strictEqual(decoratedReqOpts.objectMode, undefined); + }); + + it('should delete qs.autoPaginate', () => { + const decoratedReqOpts = util.decorateRequest( + { + qs: { + autoPaginate: true, + }, + } as DecorateRequestOptions, + projectId + ); + + assert.strictEqual(decoratedReqOpts.qs.autoPaginate, undefined); + }); + + it('should delete qs.autoPaginateVal', () => { + const decoratedReqOpts = util.decorateRequest( + { + qs: { + autoPaginateVal: true, + }, + } as DecorateRequestOptions, + projectId + ); + + assert.strictEqual(decoratedReqOpts.qs.autoPaginateVal, undefined); + }); + + it('should delete json.autoPaginate', () => { + const decoratedReqOpts = util.decorateRequest( + { + json: { + autoPaginate: true, + }, + } as DecorateRequestOptions, + projectId + ); + + assert.strictEqual(decoratedReqOpts.json.autoPaginate, undefined); + }); + + it('should delete json.autoPaginateVal', () => { + const decoratedReqOpts = util.decorateRequest( + { + json: { + autoPaginateVal: true, + }, + } as DecorateRequestOptions, + projectId + ); + + assert.strictEqual(decoratedReqOpts.json.autoPaginateVal, undefined); + }); + + it('should replace project ID tokens for qs object', () => { + const projectId = 'project-id'; + const reqOpts = { + uri: 'http://', + qs: {}, + }; + const decoratedQs = {}; + + replaceProjectIdTokenOverride = (qs: {}, projectId_: string) => { + if (qs === reqOpts.uri) { + return; + } + assert.deepStrictEqual(qs, reqOpts.qs); + assert.strictEqual(projectId_, projectId); + return decoratedQs; + }; + + const decoratedRequest = util.decorateRequest(reqOpts, projectId); + assert.deepStrictEqual(decoratedRequest.qs, decoratedQs); + }); + + it('should replace project ID tokens for multipart array', () => { + const projectId = 'project-id'; + const reqOpts = { + uri: 'http://', + multipart: [ + { + 'Content-Type': '...', + body: '...', + }, + ], + }; + const decoratedPart = {}; + + replaceProjectIdTokenOverride = (part: {}, projectId_: string) => { + if (part === reqOpts.uri) { + return; + } + assert.deepStrictEqual(part, reqOpts.multipart[0]); + assert.strictEqual(projectId_, projectId); + return decoratedPart; + }; + + const decoratedRequest = util.decorateRequest(reqOpts, projectId); + assert.deepStrictEqual(decoratedRequest.multipart, [decoratedPart]); + }); + + it('should replace project ID tokens for json object', () => { + const projectId = 'project-id'; + const reqOpts = { + uri: 'http://', + json: {}, + }; + const decoratedJson = {}; + + replaceProjectIdTokenOverride = (json: {}, projectId_: string) => { + if (json === reqOpts.uri) { + return; + } + assert.strictEqual(reqOpts.json, json); + assert.strictEqual(projectId_, projectId); + return decoratedJson; + }; + + const decoratedRequest = util.decorateRequest(reqOpts, projectId); + assert.deepStrictEqual(decoratedRequest.json, decoratedJson); + }); + + it('should decorate the request', () => { + const projectId = 'project-id'; + const reqOpts = { + uri: 'http://', + }; + const decoratedUri = 'http://decorated'; + + replaceProjectIdTokenOverride = (uri: string, projectId_: string) => { + assert.strictEqual(uri, reqOpts.uri); + assert.strictEqual(projectId_, projectId); + return decoratedUri; + }; + + assert.deepStrictEqual(util.decorateRequest(reqOpts, projectId), { + uri: decoratedUri, + }); + }); + }); + + describe('isCustomType', () => { + class PubSub {} + + class MiddleLayer { + parent = new PubSub(); + } + + class Subscription { + parent = new MiddleLayer(); + } + + const pubsub = new PubSub(); + const subscription = new Subscription(); + + describe('Service objects', () => { + it('should match by constructor name', () => { + assert(util.isCustomType(pubsub, 'pubsub')); + }); + + it('should support any casing', () => { + assert(util.isCustomType(pubsub, 'PubSub')); + }); + + it('should not match if the wrong Service', () => { + assert(!util.isCustomType(subscription, 'BigQuery')); + }); + }); + + describe('ServiceObject objects', () => { + it('should match by constructor names', () => { + assert(util.isCustomType(subscription, 'pubsub')); + assert(util.isCustomType(subscription, 'pubsub/subscription')); + + assert(util.isCustomType(subscription, 'middlelayer')); + assert(util.isCustomType(subscription, 'middlelayer/subscription')); + }); + + it('should support any casing', () => { + assert(util.isCustomType(subscription, 'PubSub/Subscription')); + }); + + it('should not match if the wrong ServiceObject', () => { + assert(!util.isCustomType(subscription, 'pubsub/topic')); + }); + }); + }); + + describe('maybeOptionsOrCallback', () => { + it('should allow passing just a callback', () => { + const optionsOrCallback = () => {}; + const [opts, cb] = util.maybeOptionsOrCallback(optionsOrCallback); + assert.strictEqual(optionsOrCallback, cb); + assert.deepStrictEqual(opts, {}); + }); + + it('should allow passing both opts and callback', () => { + const optionsOrCallback = {}; + const callback = () => {}; + const [opts, cb] = util.maybeOptionsOrCallback( + optionsOrCallback, + callback + ); + assert.strictEqual(opts, optionsOrCallback); + assert.strictEqual(cb, callback); + }); + }); +}); diff --git a/handwritten/storage/test/notification.ts b/handwritten/storage/test/notification.ts new file mode 100644 index 00000000000..fe396dcb512 --- /dev/null +++ b/handwritten/storage/test/notification.ts @@ -0,0 +1,377 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + BaseMetadata, + DecorateRequestOptions, + ServiceObject, + ServiceObjectConfig, + util, +} from '../src/nodejs-common/index.js'; +import assert from 'assert'; +import {describe, it, before, beforeEach} from 'mocha'; +import proxyquire from 'proxyquire'; + +import {Bucket} from '../src/index.js'; + +class FakeServiceObject extends ServiceObject { + calledWith_: IArguments; + constructor(config: ServiceObjectConfig) { + super(config); + // eslint-disable-next-line prefer-rest-params + this.calledWith_ = arguments; + } +} + +describe('Notification', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let Notification: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let notification: any; + let promisified = false; + const fakeUtil = Object.assign({}, util); + const fakePromisify = { + // tslint:disable-next-line:variable-name + promisifyAll(Class: Function) { + if (Class.name === 'Notification') { + promisified = true; + } + }, + }; + + const BUCKET = { + createNotification: fakeUtil.noop, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + request(_reqOpts: DecorateRequestOptions, _callback: Function) { + return fakeUtil.noop(); + }, + }; + + const ID = '123'; + + before(() => { + Notification = proxyquire('../src/notification.js', { + '@google-cloud/promisify': fakePromisify, + './nodejs-common': { + ServiceObject: FakeServiceObject, + util: fakeUtil, + }, + }).Notification; + }); + + beforeEach(() => { + BUCKET.createNotification = fakeUtil.noop = () => {}; + BUCKET.request = fakeUtil.noop = () => {}; + notification = new Notification(BUCKET, ID); + }); + + describe('instantiation', () => { + it('should promisify all the things', () => { + assert(promisified); + }); + + it('should inherit from ServiceObject', () => { + assert(notification instanceof FakeServiceObject); + + const calledWith = notification.calledWith_[0]; + + assert.strictEqual(calledWith.parent, BUCKET); + assert.strictEqual(calledWith.baseUrl, '/notificationConfigs'); + assert.strictEqual(calledWith.id, ID); + + assert.deepStrictEqual(calledWith.methods, { + create: true, + delete: { + reqOpts: { + qs: {}, + }, + }, + get: { + reqOpts: { + qs: {}, + }, + }, + getMetadata: { + reqOpts: { + qs: {}, + }, + }, + exists: true, + }); + }); + + it('should use Bucket#createNotification for the createMethod', () => { + const bound = () => {}; + + Object.assign(BUCKET.createNotification, { + bind(context: Bucket) { + assert.strictEqual(context, BUCKET); + return bound; + }, + }); + + const notification = new Notification(BUCKET, ID); + const calledWith = notification.calledWith_[0]; + assert.strictEqual(calledWith.createMethod, bound); + }); + + it('should convert number IDs to strings', () => { + const notification = new Notification(BUCKET, 1); + const calledWith = notification.calledWith_[0]; + + assert.strictEqual(calledWith.id, '1'); + }); + }); + + describe('delete', () => { + it('should make the correct request', done => { + const options = {}; + + BUCKET.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.method, 'DELETE'); + assert.strictEqual(reqOpts.uri, 'notificationConfigs/123'); + assert.deepStrictEqual(reqOpts.qs, options); + callback(); // the done fn + }; + + notification.delete(options, done); + }); + + it('should optionally accept options', done => { + BUCKET.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.deepStrictEqual(reqOpts.qs, {}); + callback(); // the done fn + }; + + notification.delete(done); + }); + + it('should optionally accept a callback', done => { + BUCKET.request = ( + _reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(); // the done fn + }; + + notification.delete(done); + }); + }); + + describe('get', () => { + it('should get the metadata', done => { + notification.getMetadata = () => { + done(); + }; + + notification.get(assert.ifError); + }); + + it('should accept an options object', done => { + const options = {}; + + notification.getMetadata = (options_: {}) => { + assert.deepStrictEqual(options_, options); + done(); + }; + + notification.get(options, assert.ifError); + }); + + it('should execute callback with error & metadata', done => { + const error = new Error('Error.'); + const metadata = {}; + + notification.getMetadata = (_options: {}, callback: Function) => { + callback(error, metadata); + }; + + notification.get((err: Error, instance: {}, metadata_: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(instance, null); + assert.strictEqual(metadata_, metadata); + + done(); + }); + }); + + it('should execute callback with instance & metadata', done => { + const metadata = {}; + + notification.getMetadata = (_options: {}, callback: Function) => { + callback(null, metadata); + }; + + notification.get((err: Error, instance: {}, metadata_: {}) => { + assert.ifError(err); + + assert.strictEqual(instance, notification); + assert.strictEqual(metadata_, metadata); + + done(); + }); + }); + + describe('autoCreate', () => { + let AUTO_CREATE_CONFIG: {}; + + const ERROR = {code: 404}; + const METADATA = {}; + + beforeEach(() => { + AUTO_CREATE_CONFIG = { + autoCreate: true, + }; + + notification.getMetadata = (_options: {}, callback: Function) => { + callback(ERROR, METADATA); + }; + }); + + it('should pass config to create if it was provided', done => { + const config = Object.assign( + {}, + { + maxResults: 5, + } + ); + + notification.get = (config_: {}) => { + assert.deepStrictEqual(config_, config); + done(); + }; + + notification.get(config); + }); + + it('should pass only a callback to create if no config', done => { + notification.create = (callback: Function) => { + callback(); // done() + }; + + notification.get(AUTO_CREATE_CONFIG, done); + }); + + describe('error', () => { + it('should execute callback with error & API response', done => { + const error = new Error('Error.'); + const apiResponse = {}; + + notification.create = (callback: Function) => { + notification.get = (config: {}, callback: Function) => { + assert.deepStrictEqual(config, {}); + callback(); // done() + }; + + callback(error, null, apiResponse); + }; + + notification.get( + AUTO_CREATE_CONFIG, + (err: Error, instance: {}, resp: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(instance, null); + assert.strictEqual(resp, apiResponse); + done(); + } + ); + }); + + it('should refresh the metadata after a 409', done => { + const error = { + code: 409, + }; + + notification.create = (callback: Function) => { + notification.get = (config: {}, callback: Function) => { + assert.deepStrictEqual(config, {}); + callback(); // done() + }; + + callback(error); + }; + + notification.get(AUTO_CREATE_CONFIG, done); + }); + }); + }); + }); + + describe('getMetadata', () => { + it('should make the correct request', done => { + const options = {}; + + BUCKET.request = (reqOpts: DecorateRequestOptions) => { + assert.strictEqual(reqOpts.uri, 'notificationConfigs/123'); + assert.deepStrictEqual(reqOpts.qs, options); + done(); + }; + + notification.getMetadata(options, assert.ifError); + }); + + it('should optionally accept options', done => { + BUCKET.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.qs, {}); + done(); + }; + + notification.getMetadata(assert.ifError); + }); + + it('should return any errors to the callback', done => { + const error = new Error('err'); + const response = {}; + + BUCKET.request = ( + _reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(error, response, response); + }; + + notification.getMetadata((err: Error, metadata: {}, resp: {}) => { + assert.strictEqual(err, error); + assert.strictEqual(metadata, response); + assert.strictEqual(resp, response); + done(); + }); + }); + + it('should set and return the metadata', done => { + const response = {}; + + BUCKET.request = ( + _reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, response, response); + }; + + notification.getMetadata((err: Error, metadata: {}, resp: {}) => { + assert.ifError(err); + assert.strictEqual(metadata, response); + assert.strictEqual(notification.metadata, response); + assert.strictEqual(resp, response); + done(); + }); + }); + }); +}); diff --git a/handwritten/storage/test/resumable-upload.ts b/handwritten/storage/test/resumable-upload.ts new file mode 100644 index 00000000000..381044d64d9 --- /dev/null +++ b/handwritten/storage/test/resumable-upload.ts @@ -0,0 +1,3138 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import assert from 'assert'; +import {describe, it, beforeEach, before, afterEach, after} from 'mocha'; +import * as crypto from 'crypto'; +import * as mockery from 'mockery'; +import nock from 'nock'; +import * as path from 'path'; +import * as sinon from 'sinon'; +import {Readable, Writable} from 'stream'; +import { + RETRY_DELAY_MULTIPLIER_DEFAULT, + TOTAL_TIMEOUT_DEFAULT, + MAX_RETRY_DELAY_DEFAULT, + AUTO_RETRY_DEFAULT, + MAX_RETRY_DEFAULT, + RETRYABLE_ERR_FN_DEFAULT, +} from '../src/storage.js'; + +import { + ApiError, + CreateUriCallback, + PROTOCOL_REGEX, + UploadConfig, +} from '../src/resumable-upload.js'; +import {GaxiosOptions, GaxiosError, GaxiosResponse} from 'gaxios'; +import {GCCL_GCS_CMD_KEY} from '../src/nodejs-common/util.js'; +import {getDirName} from '../src/util.js'; +import {FileExceptionMessages} from '../src/file.js'; + +nock.disableNetConnect(); + +class AbortController { + aborted = false; + signal = this; + abort() { + this.aborted = true; + } +} + +const RESUMABLE_INCOMPLETE_STATUS_CODE = 308; +/** 256 KiB */ +const CHUNK_SIZE_MULTIPLE = 2 ** 18; +const queryPath = '/?userProject=user-project-id'; +const X_GOOG_API_HEADER_REGEX = + /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+) gccl-gcs-cmd\/(?[^W]+)$/; +const USER_AGENT_REGEX = /^gcloud-node-storage\/(?[^W]+)$/; +const CORRECT_CLIENT_CRC32C = 'Q2hlY2tzdW0h'; +const INCORRECT_SERVER_CRC32C = 'Q2hlY2tzdVUa'; +const CORRECT_CLIENT_MD5 = 'CorrectMD5Hash'; +const INCORRECT_SERVER_MD5 = 'IncorrectMD5Hash'; + +function mockAuthorizeRequest( + code = 200, + data: {} | string = { + access_token: 'abc123', + } +) { + return nock('https://www.googleapis.com') + .post('/oauth2/v4/token') + .reply(code, data); +} + +describe('resumable-upload', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let upload: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let up: any; + + const BUCKET = 'bucket-name'; + const CUSTOM_REQUEST_OPTIONS = {headers: {'X-My-Header': 'My custom value'}}; + const FILE = 'file-name'; + const GENERATION = Date.now(); + const METADATA = {contentLength: 1024, contentType: 'application/json'}; + const ORIGIN = '*'; + const PARAMS = {ifMetagenerationNotMatch: 3}; + const PREDEFINED_ACL = 'authenticatedRead'; + const USER_PROJECT = 'user-project-id'; + const API_ENDPOINT = 'https://fake.googleapis.com'; + const BASE_URI = `${API_ENDPOINT}/upload/storage/v1/b`; + const RETRY_OPTIONS = { + retryDelayMultiplier: RETRY_DELAY_MULTIPLIER_DEFAULT, + totalTimeout: TOTAL_TIMEOUT_DEFAULT, + maxRetryDelay: MAX_RETRY_DELAY_DEFAULT, + autoRetry: AUTO_RETRY_DEFAULT, + maxRetries: MAX_RETRY_DEFAULT, + retryableErrorFn: RETRYABLE_ERR_FN_DEFAULT, + }; + let REQ_OPTS: GaxiosOptions; + const keyFile = path.join(getDirName(), '../../../test/fixtures/keys.json'); + + before(() => { + mockery.registerMock('abort-controller', AbortController); + mockery.enable({useCleanCache: true, warnOnUnregistered: false}); + upload = require('../src/resumable-upload').upload; + }); + + beforeEach(() => { + REQ_OPTS = {url: 'http://fake.local'}; + up = upload({ + bucket: BUCKET, + file: FILE, + customRequestOptions: CUSTOM_REQUEST_OPTIONS, + generation: GENERATION, + metadata: METADATA, + origin: ORIGIN, + params: PARAMS, + predefinedAcl: PREDEFINED_ACL, + userProject: USER_PROJECT, + authConfig: {keyFile}, + apiEndpoint: API_ENDPOINT, + retryOptions: {...RETRY_OPTIONS}, + [GCCL_GCS_CMD_KEY]: 'sample.command', + clientCrc32c: CORRECT_CLIENT_CRC32C, + clientMd5Hash: CORRECT_CLIENT_MD5, + }); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + after(() => { + mockery.deregisterAll(); + mockery.disable(); + }); + + describe('ctor', () => { + it('should be a Writable', () => { + assert(up instanceof Writable); + }); + + it('should throw if a bucket or file is not given', () => { + assert.throws(() => { + upload(); + }, /A bucket and file name are required/); + }); + + it('should localize the bucket', () => { + assert.strictEqual(up.bucket, BUCKET); + }); + + it('should localize the cacheKey', () => { + assert.strictEqual(up.cacheKey, [BUCKET, FILE, GENERATION].join('/')); + }); + + it('should localize customRequestOptions', () => { + assert.strictEqual(up.customRequestOptions, CUSTOM_REQUEST_OPTIONS); + }); + + it('should default customRequestOptions to empty object', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + retryOptions: RETRY_OPTIONS, + }); + assert.deepStrictEqual(up.customRequestOptions, {}); + }); + + it('should include ZERO generation value in the cacheKey', () => { + const upWithZeroGeneration = upload({ + bucket: BUCKET, + file: FILE, + generation: 0, + metadata: METADATA, + origin: ORIGIN, + predefinedAcl: PREDEFINED_ACL, + userProject: USER_PROJECT, + authConfig: {keyFile}, + apiEndpoint: API_ENDPOINT, + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual( + upWithZeroGeneration.cacheKey, + [BUCKET, FILE, 0].join('/') + ); + }); + + it('should not include a generation in the cacheKey if it was not set', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + retryOptions: RETRY_OPTIONS, + }); + + assert.strictEqual(up.cacheKey, [BUCKET, FILE].join('/')); + }); + + it('should localize the file', () => { + assert.strictEqual(up.file, FILE); + }); + + it('should localize the generation', () => { + assert.strictEqual(up.generation, GENERATION); + }); + + it('should localize the apiEndpoint', () => { + assert.strictEqual(up.apiEndpoint, API_ENDPOINT); + assert.strictEqual(up.baseURI, BASE_URI); + }); + + it('should prepend https:// to apiEndpoint if not present', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + apiEndpoint: 'fake.googleapis.com', + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual(up.apiEndpoint, API_ENDPOINT); + assert.strictEqual(up.baseURI, BASE_URI); + }); + + it('should localize the KMS key name', () => { + const kmsKeyName = 'kms-key-name'; + const up = upload({ + bucket: 'BUCKET', + file: FILE, + kmsKeyName, + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual(up.kmsKeyName, kmsKeyName); + }); + + it('should localize metadata or default to empty object', () => { + assert.strictEqual(up.metadata, METADATA); + + const upWithoutMetadata = upload({ + bucket: BUCKET, + file: FILE, + retryOptions: RETRY_OPTIONS, + }); + assert.deepStrictEqual(upWithoutMetadata.metadata, {}); + }); + + it('should set the offset if it is provided', () => { + const offset = 10; + const up = upload({ + bucket: BUCKET, + file: FILE, + offset, + uri: 'https://example.com', + retryOptions: RETRY_OPTIONS, + }); + + assert.strictEqual(up.offset, offset); + }); + + it('should localize the origin', () => { + assert.strictEqual(up.origin, ORIGIN); + }); + + it('should localize the params', () => { + assert.strictEqual(up.params, PARAMS); + }); + + it('should localize userProject', () => { + assert.strictEqual(up.userProject, USER_PROJECT); + }); + + it('should localize an encryption object from a key', () => { + const key = crypto.randomBytes(32); + const up = upload({ + bucket: BUCKET, + file: FILE, + key, + retryOptions: RETRY_OPTIONS, + }); + const expectedKey = key.toString('base64'); + const expectedHash = crypto + .createHash('sha256') + .update(key) + .digest('base64'); + assert.deepStrictEqual(up.encryption, { + key: expectedKey, + hash: expectedHash, + }); + }); + + it('should localize the predefinedAcl', () => { + assert.strictEqual(up.predefinedAcl, PREDEFINED_ACL); + }); + + it('should set the predefinedAcl with public: true', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + public: true, + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual(up.predefinedAcl, 'publicRead'); + }); + + it('should set the predefinedAcl with private: true', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + private: true, + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual(up.predefinedAcl, 'private'); + }); + + it('should set numBytesWritten to 0', () => { + assert.strictEqual(up.numBytesWritten, 0); + }); + + it('should set numRetries to 0', () => { + assert.strictEqual(up.numRetries, 0); + }); + + it('should set the contentLength if provided', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + metadata: {contentLength: METADATA.contentLength}, + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual(up.contentLength, METADATA.contentLength); + }); + + it('should default the contentLength to *', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual(up.contentLength, '*'); + }); + + it('should localize the uri', () => { + const uri = 'http://www.blah.com/'; + const upWithUri = upload({ + bucket: BUCKET, + file: FILE, + uri, + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual(upWithUri.uriProvidedManually, true); + assert.strictEqual(upWithUri.uri, uri); + }); + + it('should not have `chunkSize` by default', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual(up.chunkSize, undefined); + }); + + it('should accept and set `chunkSize`', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + chunkSize: 123, + retryOptions: RETRY_OPTIONS, + }); + assert.strictEqual(up.chunkSize, 123); + }); + + it('should have a default `writableHighWaterMark`', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + retryOptions: RETRY_OPTIONS, + }); + + assert(up.writableHighWaterMark); + }); + + it('should accept a `highWaterMark` and set a `writableHighWaterMark`', () => { + const up = upload({ + bucket: BUCKET, + file: FILE, + retryOptions: RETRY_OPTIONS, + highWaterMark: 123, + }); + assert.strictEqual(up.writableHighWaterMark, 123); + }); + + describe('on write', () => { + let uri = ''; + + beforeEach(() => { + uri = 'uri'; + }); + + it("should emit 'writing' when piped", done => { + let read = false; + const upstreamBuffer = new Readable({ + read() { + if (!read) { + this.push(Buffer.alloc(1)); + read = true; + } + }, + }); + + up.createURI = () => {}; + up.once('writing', () => { + upstreamBuffer.push(null); + done(); + }); + upstreamBuffer.pipe(up); + }); + + it("should set `upstreamEnded` to `true` and emit 'upstreamFinished' on `#end()`", done => { + const upstreamBuffer = new Readable({ + read() { + this.push(Buffer.alloc(1)); + this.push(null); + }, + }); + + up.createURI = () => {}; + up.once('writing', () => { + up.on('upstreamFinished', () => { + assert.equal(up.upstreamEnded, true); + done(); + }); + + assert.equal(up.upstreamEnded, false); + + up.once('wroteToChunkBuffer', () => up.emit('readFromChunkBuffer')); + }); + upstreamBuffer.pipe(up); + }); + + it('should continue uploading', done => { + up.uri = uri; + up.continueUploading = done; + up.emit('writing'); + }); + + it('should create an upload', done => { + up.startUploading = done; + up.createURI = (callback: CreateUriCallback) => { + callback(null); + }; + up.emit('writing'); + }); + + it('should destroy the stream from an error', done => { + const error: ApiError = { + message: ':(', + name: ':(', + code: 123, + }; + up.destroy = (err: ApiError) => { + assert(err.message.indexOf(error.message) > -1); + assert(err.name.indexOf(error.name) > -1); + assert.strictEqual(err.code, 123); + done(); + }; + up.createURI = (callback: CreateUriCallback) => { + callback(error); + }; + up.emit('writing'); + }); + }); + }); + + describe('upstream', () => { + beforeEach(() => { + up.createURI = () => {}; + }); + + it('should handle writes to class', done => { + up.on('wroteToChunkBuffer', () => { + assert.equal(up.writeBuffers[0].byteLength, 16); + done(); + }); + + up.write(Buffer.alloc(16)); + }); + + it("should setup a 'uploadFinished' handler on 'upstreamFinished'", done => { + assert.equal(up.eventNames().includes('uploadFinished'), false); + + up.on('upstreamFinished', () => { + assert.equal(up.eventNames().includes('uploadFinished'), true); + done(); + }); + + up.end(); + }); + + it("should finish only after 'uploadFinished' is emitted", done => { + const upstreamBuffer = new Readable({ + read() { + this.push(Buffer.alloc(1)); + this.push(null); + }, + }); + + // Readable has ended + upstreamBuffer.on('end', () => { + // The data has been written to the buffer + up.on('wroteToChunkBuffer', () => { + // Allow the writer's callback be called immediately + up.emit('readFromChunkBuffer'); + + // setting up the listener now to prove it hasn't been fired before + up.on('finish', done); + process.nextTick(() => up.emit('uploadFinished')); + }); + }); + + upstreamBuffer.pipe(up); + }); + }); + + describe('#_write', () => { + beforeEach(() => { + up.createURI = () => {}; + }); + + it('should append buffer to existing `writeBuffers`', () => { + up.writeBuffers = [Buffer.from('abc')]; + up.write(Buffer.from('def')); + + assert.equal( + Buffer.compare(Buffer.concat(up.writeBuffers), Buffer.from('abcdef')), + 0 + ); + }); + + it('should convert string with encoding to Buffer and append to existing `writeBuffers`', () => { + const existing = 'a '; + const sample = '🦃'; + const concat = existing + sample; + + up.writeBuffers = [Buffer.from(existing)]; + + up.write(sample, 'utf-8', () => {}); + + for (const buf of up.writeBuffers) { + assert(Buffer.isBuffer(buf)); + } + assert.equal(Buffer.concat(up.writeBuffers), concat); + }); + + it("should callback on 'readFromChunkBuffer'", done => { + // The 'done' here is a callback from 'readFromChunkBuffer' + up.write('sample', 'utf-8', done); + up.emit('readFromChunkBuffer'); + }); + + it("should emit 'wroteToChunkBuffer' asynchronously", done => { + up.write('sample', 'utf-8', () => {}); + + // setting this here proves it's async + up.on('wroteToChunkBuffer', done); + }); + }); + + describe('#prependLocalBufferToUpstream', () => { + it('should synchronously prepend to existing buffer', () => { + up.localWriteCache = [Buffer.from('123')]; + up.localWriteCacheByteLength = up.localWriteCache[0].byteLength; + up.writeBuffers = [Buffer.from('456')]; + + up.prependLocalBufferToUpstream(); + + assert.equal(up.localWriteCache.length, 0); + // shouldn't concat any buffers, thus writeBuffers.length = 2 + assert.equal(up.writeBuffers.length, 2); + assert.equal(Buffer.concat(up.writeBuffers).toString(), '123456'); + assert.equal(up.localWriteCacheByteLength, 0); + }); + + it('should keep the desired last few bytes', () => { + up.localWriteCache = [Buffer.from('123'), Buffer.from('456')]; + up.localWriteCacheByteLength = up.localWriteCache.reduce( + (a: Buffer, b: number) => a.byteLength + b + ); + up.writeBuffers = [Buffer.from('789')]; + + up.prependLocalBufferToUpstream(2); + + assert.equal(up.localWriteCache.length, 0); + // shouldn't concat any buffers, thus writeBuffers.length = 2 + assert.equal(up.writeBuffers.length, 2); + assert.equal(Buffer.concat(up.writeBuffers).toString(), '56789'); + assert.equal(up.localWriteCacheByteLength, 0); + }); + }); + + describe('#pullFromChunkBuffer', () => { + it('should retrieve from the beginning of the `writeBuffers`', () => { + up.writeBuffers = [Buffer.from('ab')]; + + const [chunk] = [...up.pullFromChunkBuffer(1)]; + assert.equal(chunk.toString(), 'a'); + assert.equal(up.writeBuffers.length, 1); + assert.equal(up.writeBuffers[0].toString(), 'b'); + }); + + it('should retrieve no more than the limit provided', () => { + up.writeBuffers = [Buffer.from('0123456789')]; + + const chunks = [...up.pullFromChunkBuffer(4)]; + assert.equal(chunks.join('').toString(), '0123'); + + // length should be 1 + assert.equal(up.writeBuffers.length, 1); + assert.equal(up.writeBuffers[0].toString(), '456789'); + }); + + it('should retrieve less than the limit if no more data is available (single write)', () => { + up.writeBuffers = [Buffer.from('0123456789')]; + + const chunks = [...up.pullFromChunkBuffer(512)]; + assert.equal(chunks.join('').toString(), '0123456789'); + assert.equal(up.writeBuffers.length, 0); + }); + + it('should retrieve less than the limit if no more data is available (multi write)', () => { + // an array of 1-char buffers + up.writeBuffers = '0123456789'.split('').map(c => Buffer.from(c)); + + const chunks = [...up.pullFromChunkBuffer(512)]; + assert.equal(chunks.join('').toString(), '0123456789'); + assert.equal(up.writeBuffers.length, 0); + }); + + it('should retrieve a subset of part of a buffer and prepend the remainder', () => { + up.writeBuffers = [ + Buffer.from('0'), + Buffer.from('123'), + Buffer.from('456'), // this buffer should be split + Buffer.from('789'), + ]; + + const chunks = [...up.pullFromChunkBuffer(5)]; + assert.equal(chunks.join('').toString(), '01234'); + assert.equal(up.writeBuffers.length, 2); + assert.equal(up.writeBuffers[0].toString(), '56'); + assert.equal(up.writeBuffers[1].toString(), '789'); + }); + + it('should return all data if `Infinity` is provided', () => { + up.writeBuffers = [Buffer.from('012345'), Buffer.from('6789')]; + const chunks = [...up.pullFromChunkBuffer(Infinity)]; + assert.equal(chunks.join('').toString(), '0123456789'); + assert.equal(up.writeBuffers.length, 0); + }); + + it("should emit 'readFromChunkBuffer' synchronously on each iterator", () => { + up.writeBuffers = [Buffer.from('012345'), Buffer.from('6789')]; + + const iter = up.pullFromChunkBuffer(Infinity); + let count = 0; + let loop = 0; + + up.on('readFromChunkBuffer', () => count++); + + while (!iter.next().done) { + assert.equal(count, loop++); + } + }); + }); + + describe('#waitForNextChunk', () => { + it('should resolve `true` asynchronously if `writeBuffers.length` has data', async () => { + up.writeBuffers = [Buffer.from('ab')]; + + assert(await up.waitForNextChunk()); + }); + + it('should resolve `false` asynchronously if `upstreamEnded`', async () => { + up.upstreamEnded = true; + + assert.equal(await up.waitForNextChunk(), false); + }); + + it('should resolve `true` asynchronously if `writeBuffers.length` and `upstreamEnded`', async () => { + up.writeBuffers = [Buffer.from('ab')]; + up.upstreamEnded = true; + + assert(await up.waitForNextChunk()); + }); + + it('should wait for `wroteToChunkBuffer` if !`writeBuffers.length` && !`upstreamEnded`', async () => { + const result = await new Promise(resolve => { + up.waitForNextChunk().then(resolve); + up.emit('wroteToChunkBuffer'); + }); + + assert(result); + }); + + it("should wait for 'upstreamFinished' if !`writeBuffers.length` && !`upstreamEnded`", async () => { + await new Promise(resolve => { + up.waitForNextChunk().then(resolve); + up.emit('upstreamFinished'); + }); + }); + + it("should wait for 'upstreamFinished' and resolve `false` if data is not available", async () => { + const result = await new Promise(resolve => { + up.waitForNextChunk().then(resolve); + up.emit('upstreamFinished'); + }); + + assert.equal(result, false); + }); + + it("should wait for 'upstreamFinished' and resolve `true` if data is available", async () => { + const result = await new Promise(resolve => { + up.on('newListener', (event: string) => { + if (event === 'upstreamFinished') { + // Update the `writeBuffers` before emitting 'upstreamFinished' + up.writeBuffers = [Buffer.from('abc')]; + + process.nextTick(() => up.emit('upstreamFinished')); + } + }); + + up.waitForNextChunk().then(resolve); + }); + + assert.equal(result, true); + }); + + it("should wait for 'upstreamFinished' if !`writeBuffers.length` && !`upstreamEnded`", async () => { + await new Promise(resolve => { + up.waitForNextChunk().then(resolve); + up.emit('upstreamFinished'); + }); + }); + + it("should wait for 'upstreamFinished' and resolve `false` if data is not available", async () => { + const result = await new Promise(resolve => { + up.waitForNextChunk().then(resolve); + up.emit('upstreamFinished'); + }); + + assert.equal(result, false); + }); + + it("should wait for 'upstreamFinished' and resolve `true` if data is available", async () => { + const result = await new Promise(resolve => { + up.on('newListener', (event: string) => { + if (event === 'upstreamFinished') { + // Update the `writeBuffers` before emitting 'upstreamFinished' + up.writeBuffers = [Buffer.from('abc')]; + + process.nextTick(() => up.emit('upstreamFinished')); + } + }); + + up.waitForNextChunk().then(resolve); + }); + + assert.equal(result, true); + }); + + it('should remove listeners after calling back from `wroteToChunkBuffer`', async () => { + assert.equal(up.listenerCount('wroteToChunkBuffer'), 0); + assert.equal(up.listenerCount('upstreamFinished'), 0); + + await new Promise(resolve => { + up.on('newListener', (event: string) => { + if (event === 'wroteToChunkBuffer') { + process.nextTick(() => up.emit('wroteToChunkBuffer')); + } + }); + + up.waitForNextChunk().then(resolve); + }); + + assert.equal(up.listenerCount('wroteToChunkBuffer'), 0); + assert.equal(up.listenerCount('upstreamFinished'), 0); + }); + + it("should remove listeners after calling back from 'upstreamFinished'", async () => { + assert.equal(up.listenerCount('wroteToChunkBuffer'), 0); + assert.equal(up.listenerCount('upstreamFinished'), 0); + + await new Promise(resolve => { + up.on('newListener', (event: string) => { + if (event === 'upstreamFinished') { + process.nextTick(() => up.emit('upstreamFinished')); + } + }); + + up.waitForNextChunk().then(resolve); + }); + + assert.equal(up.listenerCount('wroteToChunkBuffer'), 0); + assert.equal(up.listenerCount('upstreamFinished'), 0); + }); + }); + + describe('#upstreamIterator', () => { + it('should yield all data from upstream by default', done => { + up.writeBuffers = [Buffer.alloc(1)]; + assert(up.writableHighWaterMark); + + up.pullFromChunkBuffer = (limit: number) => { + assert.equal(limit, Infinity); + done(); + }; + + const iterator = up.upstreamIterator(); + iterator.next(); + }); + + it('should yield up to limit if provided', async () => { + up.writeBuffers = [Buffer.alloc(16)]; + + let data = Buffer.alloc(0); + + for await (const chunk of up.upstreamIterator(8)) { + data = Buffer.concat([data, chunk]); + } + + assert.equal(data.byteLength, 8); + }); + + it("should yield less than the limit if that's all that's available", async () => { + up.writeBuffers = [Buffer.alloc(8)]; + up.upstreamEnded = true; + + let data = Buffer.alloc(0); + + for await (const chunk of up.upstreamIterator(16)) { + data = Buffer.concat([data, chunk]); + } + + assert.equal(data.byteLength, 8); + }); + + it('should yield many, arbitrarily sized chunks', async () => { + up.waitForNextChunk = () => true; + up.pullFromChunkBuffer = () => [Buffer.from('a')]; + + let data = Buffer.alloc(0); + let count = 0; + + for await (const chunk of up.upstreamIterator(16)) { + data = Buffer.concat([data, chunk]); + count++; + } + + assert.equal(data.toString(), 'a'.repeat(16)); + assert.equal(count, 16); + }); + }); + + describe('#createURI', () => { + it('should make the correct request', done => { + up.makeRequest = async (reqOpts: GaxiosOptions) => { + assert.strictEqual(reqOpts.method, 'POST'); + assert.strictEqual(reqOpts.url, `${BASE_URI}/${BUCKET}/o`); + assert.deepStrictEqual(reqOpts.params, { + predefinedAcl: up.predefinedAcl, + name: FILE, + uploadType: 'resumable', + ifGenerationMatch: GENERATION, + ifMetagenerationNotMatch: PARAMS.ifMetagenerationNotMatch, + }); + const metadataNoHeaders = {...up.metadata}; + delete metadataNoHeaders.contentLength; + delete metadataNoHeaders.contentType; + assert.deepStrictEqual(reqOpts.data, metadataNoHeaders); + done(); + return {headers: {location: '/foo'}}; + }; + up.createURI(); + }); + + it('should pass through the KMS key name', done => { + const kmsKeyName = 'kms-key-name'; + const up = upload({ + bucket: BUCKET, + file: FILE, + kmsKeyName, + retryOptions: RETRY_OPTIONS, + }); + + up.makeRequest = async (reqOpts: GaxiosOptions) => { + assert.strictEqual(reqOpts.params.kmsKeyName, kmsKeyName); + done(); + return {headers: {location: '/foo'}}; + }; + + up.createURI(); + }); + + it('should respect 0 as a generation', done => { + up.makeRequest = async (reqOpts: GaxiosOptions) => { + assert.strictEqual(reqOpts.params.ifGenerationMatch, 0); + done(); + return {headers: {location: '/foo'}}; + }; + up.generation = 0; + up.createURI(); + }); + + describe('error', () => { + const error = new Error(':('); + + beforeEach(() => { + up.makeRequest = async () => { + throw error; + }; + }); + + it('should exec callback with error', done => { + up.createURI((err: Error) => { + assert.strictEqual(err, error); + done(); + }); + }); + + it('currentInvocationId.uri should remain the same on error', done => { + const beforeCallInvocationId = up.currentInvocationId.uri; + up.createURI((err: Error) => { + assert(err); + assert.equal(beforeCallInvocationId, up.currentInvocationId.uri); + done(); + }); + }); + }); + + describe('success', () => { + const URI = 'uri'; + const RESP = {headers: {location: URI}}; + + beforeEach(() => { + up.makeRequest = async () => { + return RESP; + }; + }); + + it('should localize the uri', done => { + up.createURI((err: Error) => { + assert.ifError(err); + assert.strictEqual(up.uri, URI); + assert.strictEqual(up.offset, 0); + done(); + }); + }); + + it('should default the offset to 0', done => { + up.createURI((err: Error) => { + assert.ifError(err); + assert.strictEqual(up.offset, 0); + done(); + }); + }); + + it('should exec callback with URI', done => { + up.createURI((err: Error, uri: string) => { + assert.ifError(err); + assert.strictEqual(uri, URI); + done(); + }); + }); + + it('currentInvocationId.uri should be different after success', done => { + const beforeCallInvocationId = up.currentInvocationId.uri; + up.createURI(() => { + assert.notEqual(beforeCallInvocationId, up.currentInvocationId.uri); + done(); + }); + }); + }); + }); + + describe('#continueUploading', () => { + it('should start uploading if an offset was set', done => { + up.offset = 0; + up.startUploading = async () => { + done(); + }; + up.continueUploading(); + }); + + it('should get and set offset if no offset was set', done => { + up.getAndSetOffset = async () => { + done(); + }; + up.startUploading = () => Promise.resolve(); + up.continueUploading(); + }); + + it('should start uploading when done', done => { + up.startUploading = async function () { + assert.strictEqual(this, up); + done(); + }; + up.getAndSetOffset = () => Promise.resolve(); + up.continueUploading(); + }); + }); + + describe('#startUploading', () => { + beforeEach(() => { + up.makeRequestStream = async () => null; + up.writeBuffers = [Buffer.alloc(16)]; + }); + + it('should reset `numChunksReadInRequest` to 0', async () => { + up.numChunksReadInRequest = 1; + + await up.startUploading(); + + assert.equal(up.numChunksReadInRequest, 0); + }); + + it('should set `offset` to 0 when not set', async () => { + assert.equal(up.offset, undefined); + + await up.startUploading(); + + assert.equal(up.offset, 0); + }); + + it('should emit error if `offset` < `numBytesWritten`', done => { + up.numBytesWritten = 1; + + const expectedSent = up.numBytesWritten; + const expectedServer = 0; + const expectedDelta = expectedSent - expectedServer; + + up.on('error', (error: Error) => { + assert(error instanceof RangeError); + + const m = error.message; + assert(m.includes('offset is lower than the number of bytes written')); + assert(m.includes(`server has ${expectedServer} bytes`)); + assert(m.includes(`${expectedSent} bytes has been uploaded`)); + assert(m.includes(`${expectedDelta} bytes are missing`)); + done(); + }); + + up.startUploading(); + }); + + it("should 'fast-forward' upstream if `numBytesWritten` < `offset`", async () => { + up.writeBuffers = [Buffer.alloc(24)]; + + up.offset = 9; + up.numBytesWritten = 1; + + await up.startUploading(); + + // Should fast-forward (up.offset - up.numBytesWritten) bytes + assert.equal(up.offset, 9); + assert.equal(up.numBytesWritten, 9); + assert.equal(up.writeBuffers.length, 1); + assert.equal(up.writeBuffers[0].byteLength, 16); + }); + + it('should emit a progress event with the bytes written', done => { + up.writeBuffers = [Buffer.alloc(24)]; + up.upstreamEnded = true; + up.contentLength = 24; + + up.on( + 'progress', + (data: {bytesWritten: number; contentLength: number}) => { + assert.equal(data.bytesWritten, 24); + assert.equal(data.contentLength, 24); + + done(); + } + ); + + up.makeRequestStream = async (reqOpts: GaxiosOptions) => { + reqOpts.body.on('data', () => {}); + }; + + up.startUploading(); + }); + + it('should destroy the stream if the request failed', done => { + const error = new Error('Error.'); + up.on('error', (e: Error) => { + assert.strictEqual(e, error); + done(); + }); + + up.makeRequestStream = async () => { + throw error; + }; + up.startUploading(); + }); + + it('should retry retryable errors if the request failed', done => { + const error = new Error('Error.'); + + // mock as retryable + up.retryOptions.retryableErrorFn = () => true; + + up.on('error', done); + up.attemptDelayedRetry = () => done(); + + up.makeRequestStream = async () => { + throw error; + }; + + up.startUploading(); + }); + + describe('request preparation', () => { + // Simulating the amount of data written from upstream (exhaustive) + const UPSTREAM_BUFFER_SIZE = 512; + const UPSTREAM_ENDED = true; + // a convenient handle for getting the request options + let reqOpts: GaxiosOptions; + + async function getAllDataFromRequest() { + let payload = Buffer.alloc(0); + + await new Promise(resolve => { + reqOpts.body.on('data', (data: Buffer) => { + payload = Buffer.concat([payload, data]); + }); + + reqOpts.body.on('end', () => { + resolve(payload); + }); + }); + + return payload; + } + + beforeEach(() => { + reqOpts = {}; + up.makeRequestStream = async (requestOptions: GaxiosOptions) => { + assert.equal(requestOptions.method, 'PUT'); + assert.equal(requestOptions.url, up.uri); + assert.equal(typeof requestOptions.headers, 'object'); + assert(requestOptions.body instanceof Readable); + + reqOpts = requestOptions; + }; + up.writeBuffers = [Buffer.alloc(UPSTREAM_BUFFER_SIZE)]; + up.upstreamEnded = UPSTREAM_ENDED; + }); + + describe('single chunk', () => { + it('should use `contentLength` and `offset` if set', async () => { + const OFFSET = 100; + const CONTENT_LENGTH = 123; + + up.offset = OFFSET; + up.contentLength = CONTENT_LENGTH; + + await up.startUploading(); + + assert(reqOpts.headers); + assert.equal( + reqOpts.headers['Content-Range'], + `bytes ${OFFSET}-*/${CONTENT_LENGTH}` + ); + assert.ok( + X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + ); + assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + + const data = await getAllDataFromRequest(); + + assert.equal(data.byteLength, 23); + }); + + it('should prepare a valid request if `contentLength` is unknown', async () => { + up.contentLength = '*'; + + await up.startUploading(); + + assert(reqOpts.headers); + assert.equal(reqOpts.headers['Content-Range'], 'bytes 0-*/*'); + assert.ok( + X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + ); + assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + + const data = await getAllDataFromRequest(); + + assert.equal(data.byteLength, 512); + }); + }); + + describe('multiple chunk', () => { + const CHUNK_SIZE = 256; + + beforeEach(() => { + up.chunkSize = CHUNK_SIZE; + }); + + it('should use `chunkSize` if less than `contentLength`', async () => { + const OFFSET = 100; + const CONTENT_LENGTH = 512; + + up.offset = OFFSET; + up.contentLength = CONTENT_LENGTH; + + await up.startUploading(); + + const endByte = OFFSET + CHUNK_SIZE - 1; + assert(reqOpts.headers); + assert.equal(reqOpts.headers['Content-Length'], CHUNK_SIZE); + assert.equal( + reqOpts.headers['Content-Range'], + `bytes ${OFFSET}-${endByte}/${CONTENT_LENGTH}` + ); + assert.ok( + X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + ); + assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + + const data = await getAllDataFromRequest(); + + assert.equal(data.byteLength, CHUNK_SIZE); + }); + + it('should prepare a valid request if `contentLength` is unknown', async () => { + const OFFSET = 100; + const EXPECTED_STREAM_AMOUNT = Math.min( + UPSTREAM_BUFFER_SIZE - OFFSET, + CHUNK_SIZE + ); + const ENDING_BYTE = EXPECTED_STREAM_AMOUNT + OFFSET - 1; + + up.offset = OFFSET; + up.contentLength = '*'; + + await up.startUploading(); + + assert(reqOpts.headers); + assert.equal( + reqOpts.headers['Content-Length'], + EXPECTED_STREAM_AMOUNT + ); + assert.equal( + reqOpts.headers['Content-Range'], + `bytes ${OFFSET}-${ENDING_BYTE}/*` + ); + assert.ok( + X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + ); + assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + + const data = await getAllDataFromRequest(); + + assert.equal(data.byteLength, CHUNK_SIZE); + }); + + it('should prepare a valid request if the remaining data is less than `chunkSize`', async () => { + const NUM_BYTES_WRITTEN = 400; + const OFFSET = NUM_BYTES_WRITTEN; + const CONTENT_LENGTH = 512; + + up.offset = OFFSET; + up.numBytesWritten = NUM_BYTES_WRITTEN; + up.contentLength = CONTENT_LENGTH; + + await up.startUploading(); + + const endByte = CONTENT_LENGTH - NUM_BYTES_WRITTEN + OFFSET - 1; + assert(reqOpts.headers); + assert.equal( + reqOpts.headers['Content-Length'], + CONTENT_LENGTH - NUM_BYTES_WRITTEN + ); + assert.equal( + reqOpts.headers['Content-Range'], + `bytes ${OFFSET}-${endByte}/${CONTENT_LENGTH}` + ); + assert.ok( + X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + ); + assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + const data = await getAllDataFromRequest(); + + assert.equal(data.byteLength, CONTENT_LENGTH - NUM_BYTES_WRITTEN); + }); + }); + }); + + describe('X-Goog-Hash header injection', () => { + const CALCULATED_CRC32C = 'bzKmHw=='; + const CALCULATED_MD5 = 'VpBzljOcorCZvRIkX5Nt3A=='; + const DUMMY_CONTENT = Buffer.alloc(512, 'a'); + const CHUNK_SIZE = 256; + + let requestCount: number; + + /** + * Creates a mocked HashValidator object with forced getters to return + * predefined hash values, bypassing internal stream calculation logic. + */ + function createMockHashValidator( + crc32cEnabled: boolean, + md5Enabled: boolean + ) { + const mockValidator = { + crc32cEnabled: crc32cEnabled, + md5Enabled: md5Enabled, + end: () => {}, // Mock the end method + write: () => {}, + }; + + Object.defineProperty(mockValidator, 'crc32c', { + get: () => CALCULATED_CRC32C, + configurable: true, + }); + Object.defineProperty(mockValidator, 'md5Digest', { + get: () => CALCULATED_MD5, + configurable: true, + }); + return mockValidator; + } + + const MOCK_AUTH_CLIENT = { + // Mock the request method to return a dummy response + request: async (opts: GaxiosOptions) => { + return { + status: 200, + data: {}, + headers: {}, + config: opts, + statusText: 'OK', + } as GaxiosResponse; + }, + getRequestHeaders: async () => ({}), + getRequestMetadata: async () => ({}), + getRequestMetadataAsync: async () => ({}), + getClient: async () => MOCK_AUTH_CLIENT, + }; + + /** + * Sets up the `up` instance for hash injection tests. + * @param configOptions Partial UploadConfig to apply. + */ + function setupHashUploadInstance( + configOptions: Partial & {crc32c?: boolean; md5?: boolean} + ) { + up = upload({ + bucket: BUCKET, + file: FILE, + authClient: MOCK_AUTH_CLIENT, + retryOptions: {...RETRY_OPTIONS, maxRetries: 0}, + metadata: { + contentLength: DUMMY_CONTENT.byteLength, + contentType: 'text/plain', + }, + ...configOptions, + }); + + // Manually inject the mock HashStreamValidator if needed + const calculateCrc32c = + !configOptions.clientCrc32c && configOptions.crc32c; + const calculateMd5 = !configOptions.clientMd5Hash && configOptions.md5; + + if (calculateCrc32c || calculateMd5) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (up as any)['#hashValidator'] = createMockHashValidator( + !!calculateCrc32c, + !!calculateMd5 + ); + } + } + + async function performUpload( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + uploadInstance: any, + data: Buffer, + isMultiChunk: boolean, + expectedCrc32c?: string, + expectedMd5?: string + ): Promise { + const capturedReqOpts: GaxiosOptions[] = []; + requestCount = 0; + + uploadInstance.makeRequestStream = async ( + requestOptions: GaxiosOptions + ) => { + requestCount++; + capturedReqOpts.push(requestOptions); + + await new Promise(resolve => { + requestOptions.body.on('data', () => {}); + requestOptions.body.on('end', resolve); + }); + + const serverCrc32c = expectedCrc32c || CALCULATED_CRC32C; + const serverMd5 = expectedMd5 || CALCULATED_MD5; + if ( + isMultiChunk && + requestCount < Math.ceil(DUMMY_CONTENT.byteLength / CHUNK_SIZE) + ) { + const lastByteReceived = requestCount * CHUNK_SIZE - 1; + return { + data: '', + status: RESUMABLE_INCOMPLETE_STATUS_CODE, + headers: {range: `bytes=0-${lastByteReceived}`}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + } else { + return { + status: 200, + data: { + crc32c: serverCrc32c, + md5Hash: serverMd5, + name: FILE, + bucket: BUCKET, + size: DUMMY_CONTENT.byteLength.toString(), + }, + headers: {}, + config: {}, + statusText: 'OK', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + } + }; + + return new Promise((resolve, reject) => { + uploadInstance.on('error', reject); + uploadInstance.on('uploadFinished', () => { + resolve(capturedReqOpts); + }); + + const upstreamBuffer = new Readable({ + read() { + this.push(data); + this.push(null); + }, + }); + upstreamBuffer.pipe(uploadInstance); + }); + } + + describe('single chunk', () => { + it('should include X-Goog-Hash header with crc32c when crc32c is enabled (via validator)', async () => { + setupHashUploadInstance({crc32c: true}); + const reqOpts = await performUpload(up, DUMMY_CONTENT, false); + assert.strictEqual(reqOpts.length, 1); + assert.equal( + reqOpts[0].headers!['X-Goog-Hash'], + `crc32c=${CALCULATED_CRC32C}` + ); + }); + + it('should include X-Goog-Hash header with md5 when md5 is enabled (via validator)', async () => { + setupHashUploadInstance({md5: true}); + const reqOpts = await performUpload(up, DUMMY_CONTENT, false); + assert.strictEqual(reqOpts.length, 1); + assert.equal( + reqOpts[0].headers!['X-Goog-Hash'], + `md5=${CALCULATED_MD5}` + ); + }); + + it('should include both crc32c and md5 in X-Goog-Hash when both are enabled (via validator)', async () => { + setupHashUploadInstance({crc32c: true, md5: true}); + const reqOpts = await performUpload(up, DUMMY_CONTENT, false); + assert.strictEqual(reqOpts.length, 1); + const xGoogHash = reqOpts[0].headers!['X-Goog-Hash']; + assert.ok(xGoogHash); + const expectedHashes = [ + `crc32c=${CALCULATED_CRC32C}`, + `md5=${CALCULATED_MD5}`, + ]; + const actualHashes = xGoogHash + .split(',') + .map((s: string) => s.trim()); + assert.deepStrictEqual(actualHashes.sort(), expectedHashes.sort()); + }); + + it('should use clientCrc32c if provided (pre-calculated hash)', async () => { + const customCrc32c = 'CUSTOMCRC'; + setupHashUploadInstance({crc32c: true, clientCrc32c: customCrc32c}); + const reqOpts = await performUpload( + up, + DUMMY_CONTENT, + false, + customCrc32c + ); + assert.strictEqual(reqOpts.length, 1); + assert.strictEqual( + reqOpts[0].headers!['X-Goog-Hash'], + `crc32c=${customCrc32c}` + ); + }); + + it('should use clientMd5Hash if provided (pre-calculated hash)', async () => { + const customMd5 = 'CUSTOMMD5'; + setupHashUploadInstance({md5: true, clientMd5Hash: customMd5}); + const reqOpts = await performUpload( + up, + DUMMY_CONTENT, + false, + undefined, + customMd5 + ); + assert.strictEqual(reqOpts.length, 1); + assert.strictEqual( + reqOpts[0].headers!['X-Goog-Hash'], + `md5=${customMd5}` + ); + }); + + it('should not include X-Goog-Hash if neither crc32c nor md5 are enabled', async () => { + setupHashUploadInstance({}); + const reqOpts = await performUpload(up, DUMMY_CONTENT, false); + assert.strictEqual(reqOpts.length, 1); + assert.strictEqual(reqOpts[0].headers!['X-Goog-Hash'], undefined); + }); + }); + + describe('multiple chunk', () => { + beforeEach(() => { + setupHashUploadInstance({ + crc32c: true, + md5: true, + chunkSize: CHUNK_SIZE, + }); + }); + + it('should NOT include X-Goog-Hash header on intermediate multi-chunk requests', async () => { + const reqOpts = await performUpload(up, DUMMY_CONTENT, true); + assert.strictEqual(reqOpts.length, 2); + + assert.strictEqual(reqOpts[0].headers!['Content-Length'], CHUNK_SIZE); + assert.strictEqual(reqOpts[0].headers!['X-Goog-Hash'], undefined); + }); + + it('should include X-Goog-Hash header ONLY on the final multi-chunk request', async () => { + const expectedHashHeader = `crc32c=${CALCULATED_CRC32C},md5=${CALCULATED_MD5}`; + const reqOpts = await performUpload(up, DUMMY_CONTENT, true); + assert.strictEqual(reqOpts.length, 2); + + assert.strictEqual(reqOpts[1].headers!['Content-Length'], CHUNK_SIZE); + assert.equal(reqOpts[1].headers!['X-Goog-Hash'], expectedHashHeader); + }); + }); + }); + }); + + describe('#responseHandler', () => { + it('should emit the metadata', done => { + const BODY = {hi: 1}; + const RESP = {data: BODY, status: 200}; + up.on('metadata', (body: {}) => { + assert.strictEqual(body, BODY); + done(); + }); + up.upstreamEnded = true; + + up.responseHandler(RESP); + }); + + it('should return response data size as number', done => { + const metadata = { + size: '0', + }; + const RESP = {data: metadata, status: 200}; + up.on('metadata', (data: {size: number}) => { + assert.strictEqual(Number(metadata.size), data.size); + assert.strictEqual(typeof data.size, 'number'); + done(); + }); + up.upstreamEnded = true; + + up.responseHandler(RESP); + }); + + it('should destroy the stream if an error occurred', done => { + const RESP = {data: {error: new Error('Error.')}}; + up.on('metadata', done); + // metadata shouldn't be emitted... will blow up test if called + up.destroy = (err: Error) => { + assert.strictEqual(err, RESP.data.error); + done(); + }; + up.upstreamEnded = true; + up.responseHandler(RESP); + }); + + it('should destroy the stream if the status code is out of range', done => { + const RESP = {data: {}, status: 300}; + up.on('metadata', done); + // metadata shouldn't be emitted... will blow up test if called + up.destroy = (err: Error) => { + assert.strictEqual(err.message, 'Upload failed'); + done(); + }; + up.upstreamEnded = true; + up.responseHandler(RESP); + }); + + it('should destroy the stream on CRC32C checksum mismatch', done => { + const CLIENT_CRC = 'client_hash'; + const SERVER_CRC = 'server_hash'; + const RESP = { + data: { + crc32c: SERVER_CRC, + md5Hash: 'md5_match', + size: '100', + }, + status: 200, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (up as any)['#hashValidator'] = { + crc32cEnabled: true, + md5Enabled: true, + crc32c: CLIENT_CRC, + md5Digest: 'md5_match', + }; + up.upstreamEnded = true; + + up.destroy = (err: Error) => { + assert.strictEqual(err.message, FileExceptionMessages.UPLOAD_MISMATCH); + done(); + }; + + up.responseHandler(RESP); + }); + + it('should destroy the stream on MD5 checksum mismatch', done => { + const CLIENT_MD5 = 'client_md5'; + const SERVER_MD5 = 'server_md5'; + const RESP = { + data: { + crc32c: 'crc32c_match', + md5Hash: SERVER_MD5, + size: '100', + }, + status: 200, + }; + + up.md5 = true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (up as any)['#hashValidator'] = { + crc32c: 'crc32c_match', + md5Digest: CLIENT_MD5, + }; + up.upstreamEnded = true; + + up.destroy = (err: Error) => { + assert.strictEqual(err.message, FileExceptionMessages.UPLOAD_MISMATCH); + done(); + }; + + up.responseHandler(RESP); + }); + + it('should continue with multi-chunk upload when incomplete', done => { + const lastByteReceived = 9; + + const RESP = { + data: '', + status: RESUMABLE_INCOMPLETE_STATUS_CODE, + headers: { + range: `bytes=0-${lastByteReceived}`, + }, + }; + + up.chunkSize = 1; + up.writeBuffers = [Buffer.alloc(0)]; + + up.continueUploading = () => { + assert.equal(up.offset, lastByteReceived + 1); + + done(); + }; + + up.responseHandler(RESP); + }); + + it('should not continue with multi-chunk upload when incomplete if a partial upload has finished', done => { + const lastByteReceived = 9; + + const RESP = { + data: '', + status: RESUMABLE_INCOMPLETE_STATUS_CODE, + headers: { + range: `bytes=0-${lastByteReceived}`, + }, + }; + + up.chunkSize = 1; + up.upstreamEnded = true; + up.isPartialUpload = true; + + up.on('uploadFinished', done); + + up.responseHandler(RESP); + }); + + it('should error when upload is incomplete and the upstream is not a partial upload', done => { + const lastByteReceived = 9; + + const RESP = { + data: '', + status: RESUMABLE_INCOMPLETE_STATUS_CODE, + headers: { + range: `bytes=0-${lastByteReceived}`, + }, + }; + + up.chunkSize = 1; + up.upstreamEnded = true; + + up.on('error', (e: Error) => { + assert.match(e.message, /Upload failed/); + + done(); + }); + + up.responseHandler(RESP); + }); + + it('should unshift missing data if server did not receive the entire chunk', done => { + const NUM_BYTES_WRITTEN = 20; + const LAST_CHUNK_LENGTH = 256; + const UPSTREAM_BUFFER_LENGTH = 1024; + const lastByteReceived = 9; + const expectedUnshiftAmount = NUM_BYTES_WRITTEN - lastByteReceived - 1; + + const RESP = { + data: '', + status: RESUMABLE_INCOMPLETE_STATUS_CODE, + headers: { + range: `bytes=0-${lastByteReceived}`, + }, + }; + + up.chunkSize = 256; + up.numBytesWritten = NUM_BYTES_WRITTEN; + up.writeBuffers = [Buffer.alloc(UPSTREAM_BUFFER_LENGTH, 'b')]; + + up.localWriteCache = [ + Buffer.alloc(LAST_CHUNK_LENGTH, 'c'), + // different to ensure this is the data that's prepended + Buffer.alloc(expectedUnshiftAmount, 'a'), + ]; + + up.continueUploading = () => { + assert.equal(up.offset, lastByteReceived + 1); + assert.equal( + Buffer.concat(up.writeBuffers).byteLength, + UPSTREAM_BUFFER_LENGTH + expectedUnshiftAmount + ); + assert.equal( + Buffer.concat(up.writeBuffers) + .subarray(0, expectedUnshiftAmount) + .toString(), + 'a'.repeat(expectedUnshiftAmount) + ); + + // we should discard part of the last chunk, as we know what the server + // has at this point. + assert.deepEqual(up.localWriteCache, []); + + done(); + }; + + up.responseHandler(RESP); + }); + + it('currentInvocationId.chunk should be different after success', done => { + const beforeCallInvocationId = up.currentInvocationId.chunk; + const RESP = {data: '', status: 200}; + up.upstreamEnded = true; + + up.on('uploadFinished', () => { + assert.notEqual(beforeCallInvocationId, up.currentInvocationId.chunk); + done(); + }); + up.responseHandler(RESP); + }); + + it('currentInvocationId.chunk should be the same after error', done => { + const beforeCallInvocationId = up.currentInvocationId.chunk; + const RESP = {data: {error: new Error('Error.')}}; + up.destroy = () => { + assert.equal(beforeCallInvocationId, up.currentInvocationId.chunk); + done(); + }; + up.responseHandler(RESP); + }); + }); + + it('currentInvocationId.checkUploadStatus should be different after success', async () => { + const beforeCallInvocationId = up.currentInvocationId.checkUploadStatus; + up.makeRequest = () => { + return {}; + }; + await up.getAndSetOffset(); + assert.notEqual( + beforeCallInvocationId, + up.currentInvocationId.checkUploadStatus + ); + }); + + it('currentInvocationId.checkUploadStatus should be the same on error', done => { + const beforeCallInvocationId = up.currentInvocationId.checkUploadStatus; + up.destroy = () => { + assert.equal( + beforeCallInvocationId, + up.currentInvocationId.checkUploadStatus + ); + done(); + }; + up.makeRequest = () => { + throw new Error() as GaxiosError; + }; + + up.getAndSetOffset().catch(done); + }); + + describe('#getAndSetOffset', () => { + const RANGE = 123456; + const RESP = {status: 308, headers: {range: `range-${RANGE}`}}; + + it('should make the correct request', done => { + const URI = 'uri'; + up.uri = URI; + up.makeRequest = async (reqOpts: GaxiosOptions) => { + assert.strictEqual(reqOpts.method, 'PUT'); + assert.strictEqual(reqOpts.url, URI); + assert(reqOpts.headers); + assert.equal(reqOpts.headers['Content-Length'], 0); + assert.equal(reqOpts.headers['Content-Range'], 'bytes */*'); + assert.ok( + X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + ); + assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + done(); + return {}; + }; + up.getAndSetOffset(); + }); + + it('should set the offset from the range', async () => { + up.makeRequest = async () => RESP; + await up.getAndSetOffset(); + assert.strictEqual(up.offset, RANGE + 1); + }); + + it('should set the offset to 0 if no range is back from the API', async () => { + up.makeRequest = async () => { + return {}; + }; + await up.getAndSetOffset(); + assert.strictEqual(up.offset, 0); + }); + + it('should retry retryable errors if the request failed', done => { + const error = new Error('Error.'); + + // mock as retryable + up.retryOptions.retryableErrorFn = () => true; + + up.on('error', done); + up.attemptDelayedRetry = () => done(); + + up.makeRequest = async () => { + throw error; + }; + + up.getAndSetOffset(); + }); + }); + + describe('#makeRequest', () => { + it('should set encryption headers', async () => { + const key = crypto.randomBytes(32); + up = upload({ + bucket: 'BUCKET', + file: FILE, + key, + authConfig: {keyFile}, + retryOptions: RETRY_OPTIONS, + }); + const scopes = [ + mockAuthorizeRequest(), + nock(REQ_OPTS.url!).get('/').reply(200, {}), + ]; + const res = await up.makeRequest(REQ_OPTS); + scopes.forEach(x => x.done()); + const headers = res.config.headers; + assert.strictEqual(headers['x-goog-encryption-algorithm'], 'AES256'); + assert.strictEqual(headers['x-goog-encryption-key'], up.encryption.key); + assert.strictEqual( + headers['x-goog-encryption-key-sha256'], + up.encryption.hash + ); + }); + + it('should set userProject', async () => { + const scopes = [ + mockAuthorizeRequest(), + nock(REQ_OPTS.url!).get(queryPath).reply(200, {}), + ]; + const res: GaxiosResponse = await up.makeRequest(REQ_OPTS); + assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + scopes.forEach(x => x.done()); + }); + + it('should set validate status', done => { + up.authClient = { + request: (reqOpts: GaxiosOptions) => { + assert.strictEqual(reqOpts.validateStatus!(100), false); + assert.strictEqual(reqOpts.validateStatus!(199), false); + assert.strictEqual(reqOpts.validateStatus!(300), false); + assert.strictEqual(reqOpts.validateStatus!(400), false); + assert.strictEqual(reqOpts.validateStatus!(500), false); + + assert.strictEqual(reqOpts.validateStatus!(200), true); + assert.strictEqual(reqOpts.validateStatus!(299), true); + assert.strictEqual(reqOpts.validateStatus!(308), true); + + done(); + + return {}; + }, + }; + up.makeRequest(REQ_OPTS); + }); + + it('should make the correct request', async () => { + const scopes = [ + mockAuthorizeRequest(), + nock(REQ_OPTS.url!).get(queryPath).reply(200, undefined, {}), + ]; + const res = await up.makeRequest(REQ_OPTS); + scopes.forEach(x => x.done()); + assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + assert.deepStrictEqual(res.headers, {}); + }); + + it('should bypass authentication if emulator context detected', async () => { + up = upload({ + bucket: BUCKET, + file: FILE, + customRequestOptions: CUSTOM_REQUEST_OPTIONS, + generation: GENERATION, + metadata: METADATA, + origin: ORIGIN, + params: PARAMS, + predefinedAcl: PREDEFINED_ACL, + userProject: USER_PROJECT, + authConfig: {keyFile}, + apiEndpoint: 'https://fake.endpoint.com', + retryOptions: RETRY_OPTIONS, + }); + const scopes = [ + nock(REQ_OPTS.url!).get(queryPath).reply(200, undefined, {}), + ]; + const res = await up.makeRequest(REQ_OPTS); + scopes.forEach(x => x.done()); + assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + assert.deepStrictEqual(res.headers, {}); + }); + + it('should use authentication with custom endpoint when useAuthWithCustomEndpoint is true', async () => { + up = upload({ + bucket: BUCKET, + file: FILE, + customRequestOptions: CUSTOM_REQUEST_OPTIONS, + generation: GENERATION, + metadata: METADATA, + origin: ORIGIN, + params: PARAMS, + predefinedAcl: PREDEFINED_ACL, + userProject: USER_PROJECT, + authConfig: {keyFile}, + apiEndpoint: 'https://custom-proxy.example.com', + useAuthWithCustomEndpoint: true, + retryOptions: RETRY_OPTIONS, + }); + + // Mock the authorization request + mockAuthorizeRequest(); + + // Mock the actual request with auth header expectation + const scopes = [ + nock(REQ_OPTS.url!) + .matchHeader('authorization', /Bearer .+/) + .get(queryPath) + .reply(200, undefined, {}), + ]; + + const res = await up.makeRequest(REQ_OPTS); + scopes.forEach(x => x.done()); + assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + // Headers should include authorization + assert.ok(res.config.headers?.['Authorization']); + }); + + it('should bypass authentication with custom endpoint when useAuthWithCustomEndpoint is false', async () => { + up = upload({ + bucket: BUCKET, + file: FILE, + customRequestOptions: CUSTOM_REQUEST_OPTIONS, + generation: GENERATION, + metadata: METADATA, + origin: ORIGIN, + params: PARAMS, + predefinedAcl: PREDEFINED_ACL, + userProject: USER_PROJECT, + authConfig: {keyFile}, + apiEndpoint: 'https://storage-emulator.local', + useAuthWithCustomEndpoint: false, + retryOptions: RETRY_OPTIONS, + }); + + const scopes = [ + nock(REQ_OPTS.url!).get(queryPath).reply(200, undefined, {}), + ]; + const res = await up.makeRequest(REQ_OPTS); + scopes.forEach(x => x.done()); + assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + // When auth is bypassed, no auth headers should be present + assert.deepStrictEqual(res.headers, {}); + }); + + it('should bypass authentication with custom endpoint when useAuthWithCustomEndpoint is undefined (backward compatibility)', async () => { + up = upload({ + bucket: BUCKET, + file: FILE, + customRequestOptions: CUSTOM_REQUEST_OPTIONS, + generation: GENERATION, + metadata: METADATA, + origin: ORIGIN, + params: PARAMS, + predefinedAcl: PREDEFINED_ACL, + userProject: USER_PROJECT, + authConfig: {keyFile}, + apiEndpoint: 'https://storage-emulator.local', + // useAuthWithCustomEndpoint is intentionally not set + retryOptions: RETRY_OPTIONS, + }); + + const scopes = [ + nock(REQ_OPTS.url!).get(queryPath).reply(200, undefined, {}), + ]; + const res = await up.makeRequest(REQ_OPTS); + scopes.forEach(x => x.done()); + assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + // When auth is bypassed (backward compatibility), no auth headers should be present + assert.deepStrictEqual(res.headers, {}); + }); + + it('should combine customRequestOptions', done => { + const up = upload({ + bucket: BUCKET, + file: FILE, + retryOptions: RETRY_OPTIONS, + customRequestOptions: { + headers: { + 'X-My-Header': 'My custom value', + }, + }, + }); + mockAuthorizeRequest(); + up.authClient = { + request: (reqOpts: GaxiosOptions) => { + const customHeader = + reqOpts.headers && reqOpts.headers['X-My-Header']; + assert.strictEqual(customHeader, 'My custom value'); + setImmediate(done); + return {}; + }, + }; + up.makeRequest(REQ_OPTS); + }); + + it('should execute the callback with a body error & response', async () => { + const error = new GaxiosError('Error message', {}, { + config: {}, + data: {}, + status: 500, + statusText: 'sad trombone', + headers: {}, + } as GaxiosResponse); + mockAuthorizeRequest(); + const scope = nock(REQ_OPTS.url!).get(queryPath).reply(500, {error}); + await assert.rejects(up.makeRequest(REQ_OPTS), (err: GaxiosError) => { + scope.done(); + assert.strictEqual(err.status, 500); + return true; + }); + }); + + it('should execute the callback with a body error & response for non-2xx status codes', async () => { + const error = new GaxiosError('Error message', {}, { + config: {}, + data: {}, + status: 500, + statusText: 'sad trombone', + headers: {}, + } as GaxiosResponse); + mockAuthorizeRequest(); + const scope = nock(REQ_OPTS.url!).get(queryPath).reply(500, {error}); + await assert.rejects(up.makeRequest(REQ_OPTS), (err: GaxiosError) => { + scope.done(); + assert.deepStrictEqual(err.status, 500); + return true; + }); + }); + + it('should execute the callback', async () => { + const data = {red: 'tape'}; + mockAuthorizeRequest(); + up.onResponse = () => true; + const scope = nock(REQ_OPTS.url!).get(queryPath).reply(200, data); + const res = await up.makeRequest(REQ_OPTS); + scope.done(); + assert.strictEqual(res.status, 200); + assert.deepStrictEqual(res.data, data); + }); + }); + + describe('#makeRequestStream', () => { + beforeEach(() => { + up.authClient = {request: () => {}}; + up.onResponse = () => {}; + }); + + it('should pass a signal from the abort controller', done => { + up.authClient = { + request: (reqOpts: GaxiosOptions) => { + assert(reqOpts.signal instanceof AbortController); + done(); + }, + }; + up.makeRequestStream(REQ_OPTS); + }); + + it('should abort on an error', done => { + up.on('error', () => {}); + + let abortController: AbortController; + up.authClient = { + request: (reqOpts: GaxiosOptions) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + abortController = reqOpts.signal as any; + }, + }; + + up.makeRequestStream(REQ_OPTS); + up.emit('error', new Error('Error.')); + + setImmediate(() => { + assert.strictEqual(abortController.aborted, true); + done(); + }); + }); + + it('should set userProject', done => { + up.userProject = 'user-project'; + up.authClient = { + request: (reqOpts: GaxiosOptions) => { + assert.deepStrictEqual(reqOpts.params, {userProject: 'user-project'}); + done(); + }, + }; + up.makeRequestStream(REQ_OPTS); + }); + + it('should not remove existing params when userProject is set', done => { + REQ_OPTS.params = {a: 'b', c: 'd'}; + up.userProject = 'user-project'; + up.authClient = { + request: (reqOpts: GaxiosOptions) => { + assert.deepStrictEqual(reqOpts.params, { + userProject: 'user-project', + a: 'b', + c: 'd', + }); + done(); + }, + }; + up.makeRequestStream(REQ_OPTS); + }); + + it('should always validate the status', done => { + up.authClient = { + request: (reqOpts: GaxiosOptions) => { + assert.strictEqual(reqOpts.validateStatus!(0), true); + done(); + }, + }; + up.makeRequestStream(REQ_OPTS); + }); + + it('should combine customRequestOptions', done => { + const up = upload({ + bucket: BUCKET, + file: FILE, + retryOptions: RETRY_OPTIONS, + customRequestOptions: { + headers: { + 'X-My-Header': 'My custom value', + }, + }, + }); + mockAuthorizeRequest(); + up.authClient = { + request: (reqOpts: GaxiosOptions) => { + const customHeader = + reqOpts.headers && reqOpts.headers['X-My-Header']; + assert.strictEqual(customHeader, 'My custom value'); + setImmediate(done); + return {}; + }, + }; + up.makeRequestStream(REQ_OPTS); + }); + + it('should pass the response to the handler', done => { + const response = {}; + up.authClient = { + request: async () => response, + }; + up.onResponse = (res: GaxiosResponse) => { + assert.strictEqual(res, response); + done(); + }; + up.makeRequestStream(REQ_OPTS); + }); + + it('should return the response if successful', async () => { + const response = {some: 'response'}; + up.authClient = { + request: async () => response, + }; + up.onResponse = () => true; + + const stream = await up.makeRequestStream(REQ_OPTS); + assert.strictEqual(stream, response); + }); + + it('should return `null` if the response is unsuccessful', async () => { + const response = {some: 'response'}; + up.authClient = { + request: async () => response, + }; + up.onResponse = () => false; + + const stream = await up.makeRequestStream(REQ_OPTS); + assert.strictEqual(stream, null); + }); + }); + + describe('#onResponse', () => { + beforeEach(() => { + up.numRetries = 0; + up.startUploading = () => {}; + up.continueUploading = () => {}; + }); + + describe('500s', () => { + const RESP = {status: 500, data: 'error message from server'}; + + it('should increase the retry count if less than limit', () => { + up.getRetryDelay = () => 1; + assert.strictEqual(up.numRetries, 0); + assert.strictEqual(up.onResponse(RESP), false); + assert.strictEqual(up.numRetries, 1); + }); + + it('should destroy the stream if greater than limit', done => { + up.getRetryDelay = () => 1; + up.destroy = (err: Error) => { + assert.strictEqual( + err.message, + `Retry limit exceeded - ${JSON.stringify(RESP.data)}` + ); + done(); + }; + + up.onResponse(RESP); + up.onResponse(RESP); + up.onResponse(RESP); + up.onResponse(RESP); + }); + + describe('exponential back off', () => { + let clock: sinon.SinonFakeTimers; + let setTimeoutSpy: sinon.SinonSpy; + beforeEach(() => { + clock = sinon.useFakeTimers({toFake: ['setTimeout']}); + setTimeoutSpy = sinon.spy(global, 'setTimeout'); + }); + afterEach(() => { + clock.restore(); + }); + + it('should continue uploading after retry count^2 * random', done => { + up.continueUploading = function () { + assert.strictEqual(this, up); + + const minTime = Math.pow(2, up.numRetries - 1) * 1000; + const maxTime = minTime + 1000; + + const delay = setTimeoutSpy.lastCall.args[1]; + assert(delay >= minTime); + assert(delay <= maxTime); + + // make it keep retrying until the limit is reached + up.onResponse(RESP); + }; + + up.on('error', (err: Error) => { + assert.strictEqual(up.numRetries, 3); + assert.strictEqual( + err.message, + `Retry limit exceeded - ${JSON.stringify(RESP.data)}` + ); + done(); + }); + + up.onResponse(RESP); + clock.runAll(); + }); + }); + }); + + describe('all others', () => { + const RESP = {status: 200}; + + it('should emit the response on the stream', done => { + up.getRetryDelay = () => 1; + up.on('response', (resp: {}) => { + assert.strictEqual(resp, RESP); + done(); + }); + up.onResponse(RESP); + }); + + it('should return true', () => { + up.getRetryDelay = () => 1; + assert.strictEqual(up.onResponse(RESP), true); + }); + + it('should handle a custom status code when passed a retry function', () => { + up.getRetryDelay = () => 1; + const RESP = {status: 1000}; + const customHandlerFunction = (err: ApiError) => { + return err.code === 1000; + }; + up.retryOptions.retryableErrorFn = customHandlerFunction; + + assert.strictEqual(up.onResponse(RESP), false); + }); + }); + }); + + describe('#attemptDelayedRetry', () => { + beforeEach(() => { + up.startUploading = () => {}; + up.continueUploading = () => {}; + up.getRetryDelay = () => 1; + }); + + it('should increment numRetries', () => { + assert.equal(up.numRetries, 0); + + up.attemptDelayedRetry({}); + + assert.equal(up.numRetries, 1); + }); + + it('should call `startUploading` on 404 && !this.numChunksReadInRequest', done => { + up.startUploading = done; + up.continueUploading = () => done('wanted `startUploading`'); + + up.attemptDelayedRetry({status: 404}); + }); + + it('should not call `startUploading` when on 404 && this.numChunksReadInRequest != 0', done => { + up.startUploading = () => done('wanted `continueUploading`'); + up.continueUploading = done; + + up.numChunksReadInRequest = 1; + up.attemptDelayedRetry({status: 404}); + }); + + it('should not call `startUploading` when !this.numChunksReadInRequest && status != 404', done => { + up.startUploading = () => done('wanted `continueUploading`'); + up.continueUploading = done; + + up.attemptDelayedRetry({status: 400}); + }); + + it('should call `getRetryDelay` when not calling `startUploading`', done => { + up.startUploading = () => done('wanted `continueUploading`'); + up.getRetryDelay = () => { + process.nextTick(done); + return 1; + }; + + up.attemptDelayedRetry({}); + }); + + it('should unshift the write buffer, unset `offset`, and call `continueUploading` when not calling `startUploading`', done => { + up.startUploading = () => done('wanted `continueUploading`'); + up.continueUploading = () => { + assert.equal(up.numBytesWritten, 4); + assert.equal(up.localWriteCache.length, 0); + assert.equal( + Buffer.concat(up.writeBuffers).toString(), + 'a'.repeat(12) + 'b'.repeat(10) + ); + assert.equal(up.offset, undefined); + + done(); + }; + + up.numBytesWritten = 16; + up.localWriteCache = [Buffer.alloc(12, 'a')]; + up.localWriteCacheByteLength = up.localWriteCache[0].byteLength; + up.writeBuffers = [Buffer.alloc(10, 'b')]; + up.offset = 16; + + up.attemptDelayedRetry({}); + }); + + it('should destroy if retry total time limit exceeded (0)', done => { + up.getRetryDelay = () => 0; + up.on('error', (error: Error) => { + assert(error.message.match(/Retry total time limit exceeded/)); + done(); + }); + + up.attemptDelayedRetry({}); + }); + + it('should destroy if retry total time limit exceeded (< 0)', done => { + up.getRetryDelay = () => -123; + up.on('error', (error: Error) => { + assert(error.message.match(/Retry total time limit exceeded/)); + done(); + }); + + up.attemptDelayedRetry({}); + }); + + it('should destroy the object if this.numRetries > this.retryLimit', done => { + up.startUploading = () => done("shouldn't have called this"); + up.continueUploading = () => done("shouldn't have called this"); + up.getRetryDelay = () => done("shouldn't have called this"); + + up.on('error', (error: Error) => { + assert(error.message.match(/Retry limit exceeded/)); + done(); + }); + + up.numRetries = 4; + up.retryLimit = 3; + + up.attemptDelayedRetry({}); + }); + + it('should destroy the object if this.numRetries === this.retryLimit', done => { + up.startUploading = () => done("shouldn't have called this"); + up.continueUploading = () => done("shouldn't have called this"); + up.getRetryDelay = () => done("shouldn't have called this"); + + up.on('error', (error: Error) => { + assert(error.message.match(/Retry limit exceeded/)); + done(); + }); + + up.numRetries = 3; + up.retryLimit = 3; + + up.attemptDelayedRetry({}); + }); + }); + + describe('PROTOCOL_REGEX', () => { + it('should match a protocol', () => { + const urls = [ + {input: 'http://www.hi.com', match: 'http'}, + {input: 'mysite://www.hi.com', match: 'mysite'}, + {input: 'www.hi.com', match: null}, + ]; + + for (const url of urls) { + assert.strictEqual( + url.input.match(PROTOCOL_REGEX) && + url.input.match(PROTOCOL_REGEX)![1], + url.match + ); + } + }); + }); + + describe('#sanitizeEndpoint', () => { + const USER_DEFINED_SHORT_API_ENDPOINT = 'myapi.com:8080'; + const USER_DEFINED_PROTOCOL = 'myproto'; + const USER_DEFINED_FULL_API_ENDPOINT = `${USER_DEFINED_PROTOCOL}://myapi.com:8080`; + + it('should default protocol to https', () => { + const endpoint = up.sanitizeEndpoint(USER_DEFINED_SHORT_API_ENDPOINT); + assert.strictEqual(endpoint.match(PROTOCOL_REGEX)![1], 'https'); + }); + + it('should not override protocol', () => { + const endpoint = up.sanitizeEndpoint(USER_DEFINED_FULL_API_ENDPOINT); + assert.strictEqual( + endpoint.match(PROTOCOL_REGEX)![1], + USER_DEFINED_PROTOCOL + ); + }); + + it('should remove trailing slashes from URL', () => { + const endpointsWithTrailingSlashes = [ + `${USER_DEFINED_FULL_API_ENDPOINT}/`, + `${USER_DEFINED_FULL_API_ENDPOINT}//`, + ]; + for (const endpointWithTrailingSlashes of endpointsWithTrailingSlashes) { + const endpoint = up.sanitizeEndpoint(endpointWithTrailingSlashes); + assert.strictEqual(endpoint.endsWith('/'), false); + } + }); + }); + + describe('#getRetryDelay', () => { + beforeEach(() => { + up.timeOfFirstRequest = Date.now(); + }); + + it('should return exponential retry delay', () => { + const min = Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000; + const max = + Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000 + 1000; + const delayValue = up.getRetryDelay(); + + assert(delayValue >= min && delayValue <= max); + }); + + it('allows overriding the delay multiplier', () => { + [1, 2, 3].forEach(delayMultiplier => { + up.retryDelayMultiplier = delayMultiplier; + const min = Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000; + const max = + Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000 + 1000; + const delayValue = up.getRetryDelay(); + + assert(delayValue >= min && delayValue <= max); + }); + }); + + it('allows overriding the number of retries', () => { + [1, 2, 3].forEach(numRetry => { + up.retryOptions.maxRetries = numRetry; + const min = Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000; + const max = + Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000 + 1000; + const delayValue = up.getRetryDelay(); + + assert(delayValue >= min && delayValue <= max); + }); + }); + + it('returns the value of totaltimeout when calculated values are larger', () => { + up.retryOptions.totalTimeout = 1; + const delayValue = up.getRetryDelay(); + + // This is a timing-based test, there could be a few milliseconds of lag + assert(delayValue <= 1000 && delayValue >= 995); + }); + }); + + describe('upload', () => { + describe('single chunk', () => { + let uri = ''; + + beforeEach(() => { + uri = 'uri'; + + up.contentLength = CHUNK_SIZE_MULTIPLE * 8; + up.createURI = ( + callback: (error: Error | null, uri: string) => void + ) => { + up.uri = uri; + up.offset = 0; + callback(null, uri); + }; + }); + + it('should make the correct request', done => { + // For additional information: + // - https://cloud.google.com/storage/docs/performing-resumable-uploads#single-chunk-upload + + const CHUNK_SIZE = CHUNK_SIZE_MULTIPLE * 2; + const NON_CHUNK_SIZE_DIVISIBLE_AMOUNT = 2; + const CONTENT_LENGTH = CHUNK_SIZE * 8 + NON_CHUNK_SIZE_DIVISIBLE_AMOUNT; + const EXPECTED_NUM_REQUESTS = 1; + + // We want the class to be able to handle varying chunk sizes uniformly. + let wrote = 0; + let wroteChunkLargerThanChunkSize = false; + let wroteChunkEqualToChunkSize = false; + let wroteChunkLessThanChunkSize = false; + + const upstreamBuffer = new Readable({ + read() { + const remainingToWrite = CONTENT_LENGTH - wrote; + + if (!remainingToWrite) { + // signal finish + this.push(null); + } else if (remainingToWrite > CHUNK_SIZE * 3) { + // write large chunk + const LARGE_CHUNK = Buffer.alloc(CHUNK_SIZE * 2); + + wrote += LARGE_CHUNK.byteLength; + wroteChunkLargerThanChunkSize = true; + + this.push(LARGE_CHUNK); + } else if (remainingToWrite > CHUNK_SIZE) { + // write chunk-sized chunk + const EQUAL_CHUNK = Buffer.alloc(CHUNK_SIZE); + + wrote += EQUAL_CHUNK.byteLength; + wroteChunkEqualToChunkSize = true; + + this.push(EQUAL_CHUNK); + } else { + // write small chunk + const SMALL_CHUNK = Buffer.alloc(remainingToWrite); + + wrote += SMALL_CHUNK.byteLength; + wroteChunkLessThanChunkSize = true; + + this.push(SMALL_CHUNK); + } + }, + }); + + const requests: { + dataReceived: number; + opts: GaxiosOptions; + chunkWritesInRequest: number; + }[] = []; + let overallDataReceived = 0; + + up.contentLength = CONTENT_LENGTH; + + up.makeRequestStream = async (opts: GaxiosOptions) => { + let dataReceived = 0; + let chunkWritesInRequest = 0; + + const res = await new Promise(resolve => { + opts.body.on('data', (data: Buffer) => { + dataReceived += data.byteLength; + overallDataReceived += data.byteLength; + chunkWritesInRequest++; + }); + + opts.body.on('end', () => { + requests.push({dataReceived, opts, chunkWritesInRequest}); + + resolve({ + status: 200, + data: {}, + }); + + resolve(null); + }); + }); + + return res; + }; + + up.on('error', done); + + up.on('finish', () => { + // Ensure the correct number of requests and data look correct + assert.equal(requests.length, EXPECTED_NUM_REQUESTS); + assert.equal(overallDataReceived, CONTENT_LENGTH); + + // Make sure we wrote the desire mix of chunk sizes + assert(wroteChunkLargerThanChunkSize); + assert(wroteChunkEqualToChunkSize); + assert(wroteChunkLessThanChunkSize); + + // Validate the single request + const request = requests[0]; + + assert.strictEqual(request.opts.method, 'PUT'); + assert.strictEqual(request.opts.url, uri); + + // We should be writing multiple buffers down the wire + assert(request.chunkWritesInRequest > 1); + + assert.equal(request.dataReceived, CONTENT_LENGTH); + assert(request.opts.headers); + assert.equal( + request.opts.headers['Content-Range'], + `bytes 0-*/${CONTENT_LENGTH}` + ); + assert.ok( + X_GOOG_API_HEADER_REGEX.test( + request.opts.headers['x-goog-api-client'] + ) + ); + assert.ok(USER_AGENT_REGEX.test(request.opts.headers['User-Agent'])); + + done(); + }); + + // init the request + upstreamBuffer.pipe(up); + }); + }); + + describe('multiple chunk', () => { + let uri = ''; + + beforeEach(() => { + uri = 'uri'; + + up.chunkSize = CHUNK_SIZE_MULTIPLE; + up.contentLength = CHUNK_SIZE_MULTIPLE * 8; + up.createURI = ( + callback: (error: Error | null, uri: string) => void + ) => { + up.uri = uri; + up.offset = 0; + callback(null, uri); + }; + }); + + it('should make the correct requests', done => { + // For additional information: + // - https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload + // - https://cloud.google.com/storage/docs/resumable-uploads#resent-data + + const CHUNK_SIZE = CHUNK_SIZE_MULTIPLE * 2; + // This is important - we want to make sure requests + // where `CONTENT_LENGTH % CHUNK_SIZE !== 0` are fine. + const LAST_REQUEST_SIZE = 2; + const CONTENT_LENGTH = CHUNK_SIZE * 8 + LAST_REQUEST_SIZE; + const EXPECTED_NUM_REQUESTS = + Math.floor(CONTENT_LENGTH / CHUNK_SIZE) + 1; + + // We want the class to be able to handle varying chunk sizes uniformly. + let wrote = 0; + let wroteChunkLargerThanChunkSize = false; + let wroteChunkEqualToChunkSize = false; + let wroteChunkLessThanChunkSize = false; + + const upstreamBuffer = new Readable({ + read() { + const remainingToWrite = CONTENT_LENGTH - wrote; + + if (!remainingToWrite) { + // signal finish + this.push(null); + } else if (remainingToWrite > CHUNK_SIZE * 3) { + // write large chunk + const LARGE_CHUNK = Buffer.alloc(CHUNK_SIZE * 2); + + wrote += LARGE_CHUNK.byteLength; + wroteChunkLargerThanChunkSize = true; + + this.push(LARGE_CHUNK); + } else if (remainingToWrite > CHUNK_SIZE) { + // write chunk-sized chunk + const EQUAL_CHUNK = Buffer.alloc(CHUNK_SIZE); + + wrote += EQUAL_CHUNK.byteLength; + wroteChunkEqualToChunkSize = true; + + this.push(EQUAL_CHUNK); + } else { + // write small chunk + const SMALL_CHUNK = Buffer.alloc(remainingToWrite); + + wrote += SMALL_CHUNK.byteLength; + wroteChunkLessThanChunkSize = true; + + this.push(SMALL_CHUNK); + } + }, + }); + + const requests: { + dataReceived: number; + opts: GaxiosOptions; + chunkWritesInRequest: number; + }[] = []; + let overallDataReceived = 0; + + up.chunkSize = CHUNK_SIZE; + up.contentLength = CONTENT_LENGTH; + + up.makeRequestStream = async (opts: GaxiosOptions) => { + let dataReceived = 0; + let chunkWritesInRequest = 0; + + const res = await new Promise(resolve => { + opts.body.on('data', (data: Buffer) => { + dataReceived += data.byteLength; + overallDataReceived += data.byteLength; + chunkWritesInRequest++; + }); + + opts.body.on('end', () => { + requests.push({dataReceived, opts, chunkWritesInRequest}); + + if (overallDataReceived < CONTENT_LENGTH) { + const lastByteReceived = overallDataReceived + ? overallDataReceived - 1 + : 0; + + resolve({ + status: RESUMABLE_INCOMPLETE_STATUS_CODE, + headers: { + range: `bytes=0-${lastByteReceived}`, + }, + data: {}, + }); + } else { + resolve({ + status: 200, + data: {}, + }); + } + }); + }); + + return res; + }; + + up.on('error', done); + + up.on('finish', () => { + // Ensure the correct number of requests and data look correct + assert.equal(requests.length, EXPECTED_NUM_REQUESTS); + assert.equal(overallDataReceived, CONTENT_LENGTH); + + // Make sure we wrote the desire mix of chunk sizes + assert(wroteChunkLargerThanChunkSize); + assert(wroteChunkEqualToChunkSize); + assert(wroteChunkLessThanChunkSize); + + // Validate each request + for (let i = 0; i < requests.length; i++) { + const request = requests[i]; + const offset = i * CHUNK_SIZE; + + assert.strictEqual(request.opts.method, 'PUT'); + assert.strictEqual(request.opts.url, uri); + + // We should be writing N buffers down the wire, although + // the request is "1 chunk" + assert(request.chunkWritesInRequest >= 1); + + if (requests.length - i === 1) { + // The last chunk + const endByte = offset + LAST_REQUEST_SIZE - 1; + + assert.equal(request.dataReceived, LAST_REQUEST_SIZE); + assert(request.opts.headers); + assert.equal( + request.opts.headers['Content-Length'], + LAST_REQUEST_SIZE + ); + assert.equal( + request.opts.headers['Content-Range'], + `bytes ${offset}-${endByte}/${CONTENT_LENGTH}` + ); + assert.ok( + X_GOOG_API_HEADER_REGEX.test( + request.opts.headers['x-goog-api-client'] + ) + ); + assert.ok( + USER_AGENT_REGEX.test(request.opts.headers['User-Agent']) + ); + } else { + // The preceding chunks + const endByte = offset + CHUNK_SIZE - 1; + + assert.equal(request.dataReceived, CHUNK_SIZE); + assert(request.opts.headers); + assert.equal(request.opts.headers['Content-Length'], CHUNK_SIZE); + assert.equal( + request.opts.headers['Content-Range'], + `bytes ${offset}-${endByte}/${CONTENT_LENGTH}` + ); + assert.ok( + X_GOOG_API_HEADER_REGEX.test( + request.opts.headers['x-goog-api-client'] + ) + ); + assert.ok( + USER_AGENT_REGEX.test(request.opts.headers['User-Agent']) + ); + } + } + + done(); + }); + + // init the request + upstreamBuffer.pipe(up); + }); + }); + + describe('empty object', () => { + let uri = ''; + + beforeEach(() => { + uri = 'uri'; + + up.contentLength = 0; + up.createURI = ( + callback: (error: Error | null, uri: string) => void + ) => { + up.uri = uri; + up.offset = 0; + callback(null, uri); + }; + }); + + it('should support uploading empty objects', done => { + const CONTENT_LENGTH = 0; + const EXPECTED_NUM_REQUESTS = 1; + + const upstreamBuffer = new Readable({ + read() { + this.push(null); + }, + }); + + const requests: { + dataReceived: number; + opts: GaxiosOptions; + chunkWritesInRequest: number; + }[] = []; + let overallDataReceived = 0; + + up.makeRequestStream = async (opts: GaxiosOptions) => { + let dataReceived = 0; + let chunkWritesInRequest = 0; + + const res = await new Promise(resolve => { + opts.body.on('data', (data: Buffer) => { + dataReceived += data.byteLength; + overallDataReceived += data.byteLength; + chunkWritesInRequest++; + }); + + opts.body.on('end', () => { + requests.push({dataReceived, opts, chunkWritesInRequest}); + + resolve({ + status: 200, + data: {}, + }); + + resolve(null); + }); + }); + + return res; + }; + + up.on('error', done); + + up.on('finish', () => { + // Ensure the correct number of requests and data look correct + assert.equal(requests.length, EXPECTED_NUM_REQUESTS); + assert.equal(overallDataReceived, CONTENT_LENGTH); + + // Validate the single request + const request = requests[0]; + + assert.strictEqual(request.opts.method, 'PUT'); + assert.strictEqual(request.opts.url, uri); + + // No data should be written + assert(request.chunkWritesInRequest === 0); + + assert.equal(request.dataReceived, CONTENT_LENGTH); + assert(request.opts.headers); + + assert.equal( + request.opts.headers['Content-Range'], + `bytes 0-*/${CONTENT_LENGTH}` + ); + assert.ok( + X_GOOG_API_HEADER_REGEX.test( + request.opts.headers['x-goog-api-client'] + ) + ); + assert.ok(USER_AGENT_REGEX.test(request.opts.headers['User-Agent'])); + + done(); + }); + + // init the request + upstreamBuffer.pipe(up); + }); + }); + }); + + describe('Validation of Client Checksums Against Server Response', () => { + const DUMMY_CONTENT = Buffer.alloc(CHUNK_SIZE_MULTIPLE * 2); + let URI = ''; + beforeEach(() => { + up.contentLength = DUMMY_CONTENT.byteLength; + URI = 'uri'; + up.createURI = (callback: (error: Error | null, uri: string) => void) => { + up.uri = URI; + up.offset = 0; + callback(null, URI); + }; + }); + const checksumScenarios = [ + { + type: 'CRC32C', + match: true, + desc: 'successfully finish the upload if server-reported CRC32C matches client CRC32C', + serverCrc: CORRECT_CLIENT_CRC32C, + serverMd5: CORRECT_CLIENT_MD5, + }, + { + type: 'CRC32C', + match: false, + desc: 'fail and destroy the stream if server-reported CRC32C mismatches client CRC32C', + serverCrc: INCORRECT_SERVER_CRC32C, + serverMd5: CORRECT_CLIENT_MD5, + errorPart: 'CRC32C checksum mismatch.', + }, + { + type: 'MD5', + match: true, + desc: 'successfully finish the upload if server-reported MD5 matches client MD5', + serverCrc: CORRECT_CLIENT_CRC32C, + serverMd5: CORRECT_CLIENT_MD5, + }, + { + type: 'MD5', + match: false, + desc: 'fail and destroy the stream if server-reported MD5 mismatches client MD5', + serverCrc: CORRECT_CLIENT_CRC32C, + serverMd5: INCORRECT_SERVER_MD5, + errorPart: 'MD5 checksum mismatch.', + }, + ]; + + checksumScenarios.forEach(scenario => { + it(`should ${scenario.desc}`, done => { + up.makeRequestStream = async (opts: GaxiosOptions) => { + await new Promise(resolve => { + opts.body.on('data', () => {}); + opts.body.on('end', resolve); + }); + + return { + status: 200, + data: { + crc32c: scenario.serverCrc, + md5Hash: scenario.serverMd5, + name: up.file, + bucket: up.bucket, + size: DUMMY_CONTENT.byteLength.toString(), + }, + headers: {}, + config: opts, + statusText: 'OK', + }; + }; + + if (scenario.match) { + up.on('error', (err: Error) => { + done(new Error(`Upload failed unexpectedly: ${err.message}`)); + }); + up.on('finish', () => { + done(); + }); + } else { + up.on('error', (err: Error) => { + assert.strictEqual( + err.message, + FileExceptionMessages.UPLOAD_MISMATCH + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const detailError = (err as any).errors && (err as any).errors[0]; + assert.ok( + detailError && detailError.message.includes(scenario.errorPart!), + `Error message should contain: ${scenario.errorPart}` + ); + assert.strictEqual(up.uri, URI); + done(); + }); + + up.on('finish', () => { + done( + new Error( + `Upload should have failed due to ${scenario.type} mismatch, but emitted finish.` + ) + ); + }); + } + + const upstreamBuffer = new Readable({ + read() { + this.push(DUMMY_CONTENT); + this.push(null); + }, + }); + upstreamBuffer.pipe(up); + }); + }); + }); +}); diff --git a/handwritten/storage/test/signer.ts b/handwritten/storage/test/signer.ts new file mode 100644 index 00000000000..6e840ac6759 --- /dev/null +++ b/handwritten/storage/test/signer.ts @@ -0,0 +1,1074 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +import assert from 'assert'; +import * as crypto from 'crypto'; +import * as sinon from 'sinon'; +import {describe, it, beforeEach, afterEach} from 'mocha'; + +import { + URLSigner, + SigningError, + AuthClient, + BucketI, + FileI, + SignerGetSignedUrlConfig, + PATH_STYLED_HOST, + GetSignedUrlConfigInternal, + Query, + SignerExceptionMessages, +} from '../src/signer.js'; +import {encodeURI, formatAsUTCISO, qsStringify} from '../src/util.js'; +import {ExceptionMessages, Storage} from '../src/storage.js'; +import {OutgoingHttpHeaders} from 'http'; +import {GoogleAuth} from 'google-auth-library'; + +interface SignedUrlArgs { + bucket: string; + method: 'GET' | 'POST' | 'PUT' | 'DELETE'; + contentMd5?: string; + contentType?: string; + extensionHeaders?: OutgoingHttpHeaders; + expiration?: number; + file: string; +} + +describe('signer', () => { + const BUCKET_NAME = 'bucket-name'; + const FILE_NAME = 'file-name.png'; + const CLIENT_EMAIL = 'client-email'; + + let sandbox: sinon.SinonSandbox; + beforeEach(() => (sandbox = sinon.createSandbox())); + afterEach(() => sandbox.restore()); + + describe('URLSigner', () => { + let authClient: GoogleAuth | AuthClient; + let bucket: BucketI; + let file: FileI; + + const NOW = new Date('2019-03-18T00:00:00.999Z'); + let fakeTimers: sinon.SinonFakeTimers; + + beforeEach(() => (fakeTimers = sinon.useFakeTimers(NOW))); + afterEach(() => fakeTimers.restore()); + + beforeEach(() => { + authClient = { + sign: async () => 'signature', + getCredentials: async () => ({client_email: CLIENT_EMAIL}), + }; + bucket = {name: BUCKET_NAME}; + file = {name: FILE_NAME}; + }); + + describe('URLSigner constructor', () => { + let signer: URLSigner; + beforeEach(() => { + signer = new URLSigner(authClient, bucket, file); + }); + + it('should localize authClient', () => { + assert.strictEqual(signer['auth'], authClient); + }); + + it('should localize bucket', () => { + assert.strictEqual(signer['bucket'], bucket); + }); + + it('should localize file', () => { + assert.strictEqual(signer['file'], file); + }); + }); + + describe('getSignedUrl', () => { + let signer: URLSigner; + let storage: Storage; + let CONFIG: SignerGetSignedUrlConfig; + + beforeEach(() => { + storage = new Storage(); + signer = new URLSigner(authClient, bucket, file, storage); + + CONFIG = { + method: 'GET', + expires: new Date().valueOf() + 2000, + }; + }); + + describe('version', () => { + it('should default to v2 if version is not given', async () => { + const v2 = sandbox + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .stub(signer, 'getSignedUrlV2') + .resolves({}); + + await signer.getSignedUrl(CONFIG); + assert(v2.calledOnce); + }); + + it('should use v2 if set', async () => { + CONFIG = { + version: 'v2', + contentMd5: 'md5', + contentType: 'application/json', + extensionHeaders: { + key: 'value', + }, + ...CONFIG, + }; + const v2 = sandbox + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .stub(signer, 'getSignedUrlV2') + .resolves({}); + + await signer.getSignedUrl(CONFIG); + assert(v2.calledOnce); + const v2arg = v2.getCall(0).args[0] as SignedUrlArgs; + assert.strictEqual(v2arg.bucket, bucket.name); + assert.strictEqual(v2arg.method, CONFIG.method); + assert.strictEqual(v2arg.contentMd5, CONFIG.contentMd5); + assert.strictEqual(v2arg.contentType, CONFIG.contentType); + assert.deepStrictEqual( + v2arg.extensionHeaders, + CONFIG.extensionHeaders + ); + }); + + it('should use v4 if set', async () => { + CONFIG = { + version: 'v4', + contentMd5: 'md5', + contentType: 'application/json', + extensionHeaders: { + key: 'value', + }, + ...CONFIG, + }; + const v4 = sandbox + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .stub(signer, 'getSignedUrlV4') + .resolves({}); + + await signer.getSignedUrl(CONFIG); + assert(v4.calledOnce); + const v4arg = v4.getCall(0).args[0] as SignedUrlArgs; + assert.strictEqual(v4arg.bucket, bucket.name); + assert.strictEqual(v4arg.method, CONFIG.method); + assert.strictEqual(v4arg.contentMd5, CONFIG.contentMd5); + assert.strictEqual(v4arg.contentType, CONFIG.contentType); + assert.deepStrictEqual( + v4arg.extensionHeaders, + CONFIG.extensionHeaders + ); + }); + + it('should error for an invalid version', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + CONFIG.version = 'v42' as any; + + assert.throws( + () => signer.getSignedUrl(CONFIG), + /Invalid signed URL version: v42\. Supported versions are 'v2' and 'v4'\./ + ); + }); + }); + + describe('accessibleAt', () => { + const accessibleAtNumber = 1581984000000; //2020-02-17T16:00:00-08:00 + const expiresNumber = accessibleAtNumber + 86400000; //2020-02-18T16:00:00-08:00 + + it('should set correct settings if accessibleAt provided', async () => { + const authClientSign = sandbox + .stub(authClient, 'sign') + .resolves('signature'); + const accessibleAt = new Date(accessibleAtNumber); + await signer.getSignedUrl({ + version: 'v4', + method: 'GET', + accessibleAt, + expires: expiresNumber, + }); + const blobToSign = authClientSign.getCall(0).args[0]; + assert(blobToSign.includes(formatAsUTCISO(accessibleAt, true))); + }); + + it('should throw if an expiration date from the before accessibleAt date is given', () => { + const accessibleAt = accessibleAtNumber; + const expires = accessibleAt - 86400000; + + assert.throws(() => { + signer.getSignedUrl({ + version: 'v4', + method: 'GET', + accessibleAt, + expires, + }), + SignerExceptionMessages.EXPIRATION_BEFORE_ACCESSIBLE_DATE; + }); + }); + + describe('checkInputTypes', () => { + const query = { + 'X-Goog-Date': formatAsUTCISO(new Date(accessibleAtNumber), true), + }; + + it('should accept Date objects', async () => { + const accessibleAt = new Date(accessibleAtNumber); + const signedUrl = await signer.getSignedUrl({ + version: 'v4', + method: 'GET', + accessibleAt, + expires: expiresNumber, + }); + assert(signedUrl.includes(qsStringify(query))); + }); + + it('should accept numbers', async () => { + const accessibleAt = accessibleAtNumber; + const signedUrl = await signer.getSignedUrl({ + version: 'v4', + method: 'GET', + accessibleAt, + expires: expiresNumber, + }); + assert(signedUrl.includes(qsStringify(query))); + }); + + it('should accept strings', async () => { + const accessibleAt = '2020-02-17T16:00:00-08:00'; + const signedUrl = await signer.getSignedUrl({ + version: 'v4', + method: 'GET', + accessibleAt, + expires: expiresNumber, + }); + assert(signedUrl.includes(qsStringify(query))); + }); + + it('should throw if a date is invalid', () => { + const accessibleAt = new Date('31-12-2019'); + + assert.throws(() => { + signer.getSignedUrl({ + version: 'v4', + method: 'GET', + accessibleAt, + expires: expiresNumber, + }), + SignerExceptionMessages.ACCESSIBLE_DATE_INVALID; + }); + }); + }); + }); + + describe('expires', () => { + it('should parse Date object into expiration seconds', async () => { + const parseExpires = sandbox + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .spy(signer, 'parseExpires'); + + const v2 = sandbox + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .stub(signer, 'getSignedUrlV2') + .resolves({}); + + await signer.getSignedUrl(CONFIG); + assert(parseExpires.calledOnceWith(CONFIG.expires)); + const expiresInSeconds = parseExpires.getCall(0).lastArg; + + assert( + (v2.getCall(0).args[0] as SignedUrlArgs).expiration, + expiresInSeconds + ); + }); + }); + + describe('URL style', () => { + let v2: sinon.SinonStub; + beforeEach(() => { + v2 = sandbox + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .stub(signer, 'getSignedUrlV2') + .resolves({}); + }); + + it('should pass cname', async () => { + CONFIG.cname = 'http://www.example.com'; + + const url = await signer.getSignedUrl(CONFIG); + const v2arg = v2.getCall(0).args[0]; + assert.strictEqual(v2arg.cname, CONFIG.cname); + assert(url.startsWith(CONFIG.cname)); + }); + + it('should pass virtual host to cname', async () => { + CONFIG.virtualHostedStyle = true; + const expectedCname = `https://${bucket.name}.storage.googleapis.com`; + + await signer.getSignedUrl(CONFIG); + const v2arg = v2.getCall(0).args[0]; + assert.strictEqual(v2arg.cname, expectedCname); + }); + + it('should use a universe domain with the virtual host', async () => { + storage.universeDomain = 'my-universe.com'; + + CONFIG.virtualHostedStyle = true; + const expectedCname = `https://${bucket.name}.storage.my-universe.com`; + + await signer.getSignedUrl(CONFIG); + const v2arg = v2.getCall(0).args[0]; + assert.strictEqual(v2arg.cname, expectedCname); + }); + + it('should take precedence in cname if both passed', async () => { + CONFIG = { + virtualHostedStyle: true, + cname: 'http://www.example.com', + ...CONFIG, + }; + + await signer.getSignedUrl(CONFIG); + const v2arg = v2.getCall(0).args[0]; + assert.strictEqual(v2arg.cname, CONFIG.cname); + }); + + it('should not pass cname parameter', async () => { + CONFIG = { + virtualHostedStyle: false, + cname: undefined, + ...CONFIG, + }; + + await signer.getSignedUrl(CONFIG); + const v2arg = v2.getCall(0).args[0]; + assert.strictEqual(v2arg.cname, undefined); + }); + }); + + describe('composing signed URL', () => { + const query = { + GoogleAccessId: CLIENT_EMAIL, + Expires: NOW.valueOf() + 2000, + Signature: 'signature', + }; + + beforeEach(() => { + sandbox + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .stub(signer, 'getSignedUrlV2') + .resolves(query) as sinon.SinonStub; + }); + + it('should insert user-provided queryParams', async () => { + CONFIG.queryParams = {key: 'AZ!*()*%/f'}; + + const url = await signer.getSignedUrl(CONFIG); + assert( + url.includes( + qsStringify({ + ...query, + ...CONFIG.queryParams, + }) + ) + ); + }); + }); + + it('should URI encode file name with special characters', async () => { + file.name = "special/azAZ!*'()*%/file.jpg"; + const encoded = encodeURI(file.name, false); + const signedUrl = await signer.getSignedUrl(CONFIG); + + const v2 = sandbox + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .stub(signer, 'getSignedUrlV2') + .resolves({}); + + await signer.getSignedUrl(CONFIG); + const v2arg = v2.getCall(0).args[0] as SignedUrlArgs; + assert.strictEqual(v2arg.file, encoded); + assert(signedUrl.includes(encoded)); + }); + + it('should generate URL with given cname', async () => { + CONFIG.cname = 'http://www.example.com'; + const signedUrl = await signer.getSignedUrl(CONFIG); + assert(signedUrl.startsWith(CONFIG.cname)); + }); + + it('should remove trailing slashes from cname', async () => { + CONFIG.cname = 'http://www.example.com//'; + const signedUrl = await signer.getSignedUrl(CONFIG); + assert(signedUrl.startsWith(`http://www.example.com/${file.name}`)); + }); + + it('should generate virtual hosted style URL', async () => { + CONFIG.virtualHostedStyle = true; + const signedUrl = await signer.getSignedUrl(CONFIG); + assert( + signedUrl.startsWith( + `https://${bucket.name}.storage.googleapis.com/${file.name}` + ) + ); + }); + + it('should generate path styled URL', async () => { + CONFIG.virtualHostedStyle = false; + const signedUrl = await signer.getSignedUrl(CONFIG); + assert(signedUrl.startsWith(PATH_STYLED_HOST)); + }); + + it('should generate URL with returned query params appended', async () => { + const query = { + 'X-Goog-Foo': 'value', + 'X-Goog-Bar': 'azAZ!*()*%', + }; + sandbox + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .stub(signer, 'getSignedUrlV2') + .resolves(query); + + const signedUrl = await signer.getSignedUrl(CONFIG); + assert(signedUrl.includes(qsStringify(query))); + }); + }); + + describe('getSignedUrlV2', () => { + let signer: URLSigner; + let CONFIG: GetSignedUrlConfigInternal; + + beforeEach(() => { + signer = new URLSigner(authClient, bucket, file); + CONFIG = { + method: 'GET', + expiration: Math.floor((NOW.valueOf() + 2000) / 1000), + bucket: bucket.name, + file: file.name, + }; + }); + + describe('blobToSign', () => { + let authClientSign: sinon.SinonStub; + beforeEach(() => { + authClientSign = sandbox + .stub(authClient, 'sign') + .resolves('signature'); + }); + + it('should sign method', async () => { + await signer['getSignedUrlV2'](CONFIG); + + const blobToSign = authClientSign.getCall(0).args[0]; + assert(blobToSign.startsWith('GET')); + }); + + it('should sign using the `signingEndpoint` when provided', async () => { + const signingEndpoint = 'https://my-endpoint.com'; + + CONFIG = { + ...CONFIG, + signingEndpoint, + }; + + await signer['getSignedUrlV2'](CONFIG); + + const endpoint = authClientSign.getCall(0).args[1]; + assert.equal(endpoint, signingEndpoint); + }); + + it('should sign contentMd5 if given', async () => { + CONFIG.contentMd5 = 'md5-hash'; + + await signer['getSignedUrlV2'](CONFIG); + + const blobToSign = authClientSign.getCall(0).args[0]; + assert(blobToSign.includes(CONFIG.contentMd5)); + }); + + it('should sign contentType if given', async () => { + CONFIG.contentType = 'application/octet-stream'; + + await signer['getSignedUrlV2'](CONFIG); + + const blobToSign = authClientSign.getCall(0).args[0]; + assert(blobToSign.includes(CONFIG.contentType)); + }); + + it('should sign expiration', async () => { + await signer['getSignedUrlV2'](CONFIG); + + const blobToSign = authClientSign.getCall(0).args[0]; + assert(blobToSign.includes(CONFIG.expiration.toString(10))); + }); + + it('should sign canonical headers', async () => { + sandbox + .stub(signer, 'getCanonicalHeaders') + .returns('canonical-headers'); + await signer['getSignedUrlV2'](CONFIG); + + const blobToSign = authClientSign.getCall(0).args[0]; + assert(blobToSign.includes('canonical-headers')); + }); + + it('should sign resource path', async () => { + sandbox.stub(signer, 'getResourcePath').returns('/resource/path'); + await signer['getSignedUrlV2'](CONFIG); + + const blobToSign = authClientSign.getCall(0).args[0]; + assert(blobToSign.endsWith('/resource/path')); + }); + + it('should compose blobToSign without contentMd5 and contentType', async () => { + sandbox + .stub(signer, 'getCanonicalHeaders') + .returns('canonical-headers'); + sandbox.stub(signer, 'getResourcePath').returns('/resource/path'); + await signer['getSignedUrlV2'](CONFIG); + + const blobToSign = authClientSign.getCall(0).args[0]; + assert.strictEqual( + blobToSign, + [ + 'GET', + '', + '', + CONFIG.expiration, + 'canonical-headers' + '/resource/path', + ].join('\n') + ); + }); + }); + + it('should return v2 query', async () => { + const query = (await signer['getSignedUrlV2'](CONFIG)) as Query; + assert.deepStrictEqual(query, { + GoogleAccessId: CLIENT_EMAIL, + Expires: CONFIG.expiration, + Signature: 'signature', + }); + }); + + it('rejects with SigningError on signing Error', () => { + const err = new Error('my-err'); + err.stack = 'some-stack-trace'; + sandbox.stub(authClient, 'sign').rejects(err); + + assert.rejects(() => signer['getSignedUrlV2'](CONFIG), { + name: 'SigningError', + message: 'my-err', + stack: 'some-stack-trace', + }); + }); + }); + + describe('getSignedUrlV4', () => { + let signer: URLSigner; + let CONFIG: GetSignedUrlConfigInternal; + + beforeEach(() => { + signer = new URLSigner(authClient, bucket, file); + CONFIG = { + method: 'GET', + expiration: (NOW.valueOf() + 2000) / 1000, + bucket: bucket.name, + }; + }); + + it('should fail for expirations beyond 7 days', () => { + CONFIG.expiration = NOW.valueOf() + 7.1 * 24 * 60 * 60; + const SEVEN_DAYS = 7 * 24 * 60 * 60; + + assert.throws( + () => { + signer['getSignedUrlV4'](CONFIG); + }, + { + message: `Max allowed expiration is seven days (${SEVEN_DAYS} seconds).`, + } + ); + }); + + it('should not throw with expiration of exactly 7 days', async () => { + const ACCESSIBLE_AT = NOW.valueOf(); + const SEVEN_DAYS_IN_SECONDS = 7 * 24 * 60 * 60; + const SEVEN_DAYS_IN_MS = SEVEN_DAYS_IN_SECONDS * 1000; + await assert.doesNotReject( + async () => { + await signer.getSignedUrl({ + method: 'GET', + expires: ACCESSIBLE_AT + SEVEN_DAYS_IN_MS, + accessibleAt: ACCESSIBLE_AT, + version: 'v4', + }); + }, + err => { + assert(err instanceof Error); + assert.strictEqual( + err.message, + `Max allowed expiration is seven days (${SEVEN_DAYS_IN_SECONDS.toString()} seconds).` + ); + return true; + } + ); + }); + + describe('headers', () => { + it('should add path-styled host header', async () => { + const getCanonicalHeaders = sandbox + .stub(signer, 'getCanonicalHeaders') + .returns(''); + + await signer['getSignedUrlV4'](CONFIG); + const arg = getCanonicalHeaders.getCall(0).args[0]; + assert.strictEqual( + arg.host, + PATH_STYLED_HOST.replace('https://', '') + ); + }); + + it('should add cname as host header', async () => { + CONFIG.cname = 'http://www.example.com'; + const getCanonicalHeaders = sandbox + .stub(signer, 'getCanonicalHeaders') + .returns(''); + + await signer['getSignedUrlV4'](CONFIG); + const arg = getCanonicalHeaders.getCall(0).args[0]; + assert.strictEqual(arg.host, 'www.example.com'); + }); + + it('should strip trailing slashes from host', async () => { + CONFIG.cname = 'http://www.example.com//'; + const getCanonicalHeaders = sandbox + .stub(signer, 'getCanonicalHeaders') + .returns(''); + + await signer['getSignedUrlV4'](CONFIG); + const arg = getCanonicalHeaders.getCall(0).args[0]; + assert.strictEqual(arg.host, 'www.example.com'); + }); + + it('should add Content-MD5 to header', async () => { + CONFIG.contentMd5 = 'md5-hash'; + const getCanonicalHeaders = sandbox + .stub(signer, 'getCanonicalHeaders') + .returns(''); + + await signer['getSignedUrlV4'](CONFIG); + const arg = getCanonicalHeaders.getCall(0).args[0]; + assert.strictEqual(arg['content-md5'], CONFIG.contentMd5); + }); + + it('should add Content-Type to header', async () => { + CONFIG.contentType = 'application/octet-stream'; + const getCanonicalHeaders = sandbox + .stub(signer, 'getCanonicalHeaders') + .returns(''); + + await signer['getSignedUrlV4'](CONFIG); + const arg = getCanonicalHeaders.getCall(0).args[0]; + assert.strictEqual(arg['content-type'], CONFIG.contentType); + }); + + it('should merge extensionHeaders', async () => { + CONFIG = { + extensionHeaders: { + 'x-goog-content-sha256': + '76af7efae0d034d1e3335ed1b90f24b6cadf2bf1', + }, + cname: 'http://www.example.com', + contentMd5: 'md5-hash', + contentType: 'application/octet-stream', + ...CONFIG, + }; + + const getCanonicalHeaders = sandbox + .stub(signer, 'getCanonicalHeaders') + .returns(''); + + await signer['getSignedUrlV4'](CONFIG); + const arg = getCanonicalHeaders.getCall(0).args[0]; + + assert.deepStrictEqual(arg, { + ...CONFIG.extensionHeaders, + host: CONFIG.cname!.replace('http://', ''), + 'content-md5': CONFIG.contentMd5, + 'content-type': CONFIG.contentType, + }); + }); + + it('should throw if x-goog-content-sha256 header is not a hash', () => { + CONFIG = { + extensionHeaders: { + 'x-goog-content-sha256': 'not-a-hash', + }, + ...CONFIG, + }; + + assert.throws(() => { + signer['getSignedUrlV4'](CONFIG), + SignerExceptionMessages.X_GOOG_CONTENT_SHA256; + }); + }); + }); + + describe('query parameters', () => { + let getCanonicalQueryParams: sinon.SinonStub<[Query]>; + beforeEach(() => { + getCanonicalQueryParams = sandbox + .stub(signer, 'getCanonicalQueryParams') + .returns(''); + }); + + it('should populate X-Goog-Algorithm', async () => { + const query = (await signer['getSignedUrlV4'](CONFIG)) as Query; + const arg = getCanonicalQueryParams.getCall(0).args[0]; + + assert.strictEqual(arg['X-Goog-Algorithm'], 'GOOG4-RSA-SHA256'); + assert.strictEqual(query['X-Goog-Algorithm'], 'GOOG4-RSA-SHA256'); + }); + + it('should populate X-Goog-Credential', async () => { + const query = (await signer['getSignedUrlV4'](CONFIG)) as Query; + const arg = getCanonicalQueryParams.getCall(0).args[0]; + + const datestamp = formatAsUTCISO(NOW); + const credentialScope = `${datestamp}/auto/storage/goog4_request`; + const EXPECTED_CREDENTIAL = `${CLIENT_EMAIL}/${credentialScope}`; + + assert.strictEqual(arg['X-Goog-Credential'], EXPECTED_CREDENTIAL); + assert.strictEqual(query['X-Goog-Credential'], EXPECTED_CREDENTIAL); + }); + + it('should populate X-Goog-Date', async () => { + const dateISO = formatAsUTCISO(NOW, true); + + const query = (await signer['getSignedUrlV4'](CONFIG)) as Query; + const arg = getCanonicalQueryParams.getCall(0).args[0]; + + assert.strictEqual(arg['X-Goog-Date'], dateISO); + assert.strictEqual(query['X-Goog-Date'], dateISO); + }); + + it('should populate X-Goog-Expires', async () => { + const query = (await signer['getSignedUrlV4'](CONFIG)) as Query; + const arg = getCanonicalQueryParams.getCall(0).args[0]; + + assert.strictEqual(arg['X-Goog-Expires'], '2'); + assert.strictEqual(query['X-Goog-Expires'], '2'); + }); + + it('should lowercase and sort signed headers, and populate X-Goog-SignedHeaders', async () => { + CONFIG.extensionHeaders = { + 'x-foo': 'bar', + 'X-Goog-acl': 'public-read', + }; + + const query = (await signer['getSignedUrlV4'](CONFIG)) as Query; + const arg = getCanonicalQueryParams.getCall(0).args[0]; + + assert.strictEqual( + arg['X-Goog-SignedHeaders'], + 'host;x-foo;x-goog-acl' + ); + assert.strictEqual( + query['X-Goog-SignedHeaders'], + 'host;x-foo;x-goog-acl' + ); + }); + + it('should merge user-provided queryParams', async () => { + CONFIG.queryParams = { + foo: 'bar', + }; + + const query = (await signer['getSignedUrlV4'](CONFIG)) as Query; + const arg = getCanonicalQueryParams.getCall(0).args[0]; + + assert.strictEqual(arg['foo'], 'bar'); + assert.strictEqual(query['foo'], 'bar'); + }); + }); + + it('should build canonical request', async () => { + CONFIG.extensionHeaders = { + 'x-foo': 'bar', + 'x-goog-content-sha256': '76af7efae0d034d1e3335ed1b90f24b6cadf2bf1', + }; + CONFIG.file = 'file-name.png'; + sinon.stub(signer, 'getCanonicalHeaders').returns('canonical-headers'); + sinon + .stub(signer, 'getCanonicalQueryParams') + .returns('canonical-query'); + const getCanonicalRequest = sinon.spy(signer, 'getCanonicalRequest'); + + await signer['getSignedUrlV4'](CONFIG); + const args = getCanonicalRequest.getCall(0).args; + + assert.strictEqual(args[0], CONFIG.method); + assert.strictEqual(args[1], '/bucket-name/file-name.png'); + assert.strictEqual(args[2], 'canonical-query'); + assert.strictEqual(args[3], 'canonical-headers'); + assert.strictEqual(args[4], 'host;x-foo;x-goog-content-sha256'); + assert.strictEqual(args[5], '76af7efae0d034d1e3335ed1b90f24b6cadf2bf1'); + }); + + it('should compute SHA256 digest in hex on canonical request', async () => { + sinon.stub(signer, 'getCanonicalRequest').returns('canonical-request'); + const authClientSign = sinon + .stub(authClient, 'sign') + .resolves('signature'); + + await signer['getSignedUrlV4'](CONFIG); + const blobToSign = authClientSign.getCall(0).args[0]; + + const canonicalRequestHash = crypto + .createHash('sha256') + .update('canonical-request') + .digest('hex'); + + assert(blobToSign.endsWith(canonicalRequestHash)); + }); + + it('should sign using the `signingEndpoint` when provided', async () => { + const signingEndpoint = 'https://my-endpoint.com'; + + sinon.stub(signer, 'getCanonicalRequest').returns('canonical-request'); + const authClientSign = sinon + .stub(authClient, 'sign') + .resolves('signature'); + + CONFIG = { + ...CONFIG, + signingEndpoint, + }; + + await signer['getSignedUrlV4'](CONFIG); + + const endpoint = authClientSign.getCall(0).args[1]; + assert.equal(endpoint, signingEndpoint); + }); + + it('should compose blobToSign', async () => { + const datestamp = formatAsUTCISO(NOW); + const credentialScope = `${datestamp}/auto/storage/goog4_request`; + const dateISO = formatAsUTCISO(NOW, true); + + const authClientSign = sinon + .stub(authClient, 'sign') + .resolves('signature'); + + await signer['getSignedUrlV4'](CONFIG); + const blobToSign = authClientSign.getCall(0).args[0]; + + assert( + blobToSign.startsWith( + ['GOOG4-RSA-SHA256', dateISO, credentialScope].join('\n') + ) + ); + }); + + it('rejects with SigningError on signing Error', () => { + const err = new Error('my-err'); + err.stack = 'some-stack-trace'; + sinon.stub(authClient, 'sign').rejects(err); + + assert.rejects(() => signer['getSignedUrlV4'](CONFIG), { + name: 'SigningError', + message: 'my-err', + stack: 'some-stack-trace', + }); + }); + + it('should returns query params with signature', async () => { + CONFIG.queryParams = { + foo: 'bar', + }; + + const query = (await signer['getSignedUrlV4'](CONFIG)) as Query; + const signatureInHex = Buffer.from('signature', 'base64').toString( + 'hex' + ); + assert.strictEqual(query['X-Goog-Signature'], signatureInHex); + }); + }); + + describe('getCanonicalHeaders', () => { + const signer = new URLSigner(authClient, bucket, file); + + it('should accept multi-valued header as an array', () => { + const headers = { + foo: ['bar', 'pub'], + }; + + const canonical = signer.getCanonicalHeaders(headers); + assert.strictEqual(canonical, 'foo:bar,pub\n'); + }); + + it('should lowercase and then sort header names', () => { + const headers = { + B: 'foo', + a: 'bar', + }; + + const canonical = signer.getCanonicalHeaders(headers); + assert.strictEqual(canonical, 'a:bar\nb:foo\n'); + }); + + it('should trim leading and trailing space', () => { + const headers = { + foo: ' bar ', + my: '\t header ', + }; + + const canonical = signer.getCanonicalHeaders(headers); + assert.strictEqual(canonical, 'foo:bar\nmy:header\n'); + }); + + it('should convert sequential spaces into single space', () => { + const headers = { + foo: 'a\t\t\tbar pub', + }; + + const canonical = signer.getCanonicalHeaders(headers); + assert.strictEqual(canonical, 'foo:a bar pub\n'); + }); + }); + + describe('getCanonicalRequest', () => { + const signer = new URLSigner(authClient, bucket, file); + + it('should return canonical request string with unsigned-payload', () => { + const args: [string, string, string, string, string] = [ + 'DELETE', + 'path', + 'query', + 'headers', + 'signedHeaders', + ]; + + const canonical = signer.getCanonicalRequest(...args); + + const EXPECTED = [...args, 'UNSIGNED-PAYLOAD'].join('\n'); + assert.strictEqual(canonical, EXPECTED); + }); + + it('should include contentSha256 value if not undefined', () => { + const SHA = '76af7efae0d034d1e3335ed1b90f24b6cadf2bf1'; + const canonical = signer.getCanonicalRequest( + 'DELETE', + 'path', + 'query', + 'headers', + 'signedHeaders', + SHA + ); + + const EXPECTED = [ + 'DELETE', + 'path', + 'query', + 'headers', + 'signedHeaders', + SHA, + ].join('\n'); + assert.strictEqual(canonical, EXPECTED); + }); + }); + + describe('getCanonicalQueryParams', () => { + const signer = new URLSigner(authClient, bucket, file); + + it('should encode key', () => { + const key = 'AZ!*()*%/f'; + const query: Query = {}; + query[key] = 'value'; + const canonical = signer.getCanonicalQueryParams(query); + + const EXPECTED = `${encodeURI(key, true)}=value`; + assert.strictEqual(canonical, EXPECTED); + }); + + it('should encode value', () => { + const value = 'AZ!*()*%/f'; + const query = {key: value}; + const canonical = signer.getCanonicalQueryParams(query); + + const EXPECTED = `key=${encodeURI(value, true)}`; + assert.strictEqual(canonical, EXPECTED); + }); + + it('should sort by key', () => { + const query = { + B: 'bar', + A: 'foo', + }; + + const canonical = signer.getCanonicalQueryParams(query); + const EXPECTED = 'A=foo&B=bar'; + assert.strictEqual(canonical, EXPECTED); + }); + }); + + describe('getResourcePath', () => { + const signer = new URLSigner(authClient, bucket, file); + + it('should not include bucket with cname', () => { + const path = signer.getResourcePath(true, bucket.name, file.name); + assert.strictEqual(path, `/${file.name}`); + }); + + it('should include file name', () => { + const path = signer.getResourcePath(false, bucket.name, file.name); + assert.strictEqual(path, `/${bucket.name}/${file.name}`); + }); + + it('should return path with no file name', () => { + const path = signer.getResourcePath(false, bucket.name); + assert.strictEqual(path, `/${bucket.name}`); + }); + }); + + describe('parseExpires', () => { + const signer = new URLSigner(authClient, bucket, file); + + it('throws invalid date', () => { + assert.throws(() => signer.parseExpires('2019-31-12T25:60:60Z'), { + message: ExceptionMessages.EXPIRATION_DATE_INVALID, + }); + }); + + it('throws if expiration is in the past', () => { + assert.throws(() => signer.parseExpires(NOW.valueOf() - 1, NOW), { + message: ExceptionMessages.EXPIRATION_DATE_PAST, + }); + }); + + it('returns expiration date in seconds', () => { + const expires = signer.parseExpires(NOW); + assert.strictEqual(expires, Math.floor(NOW.valueOf() / 1000)); + }); + }); + }); + + describe('SigningError', () => { + it('should extend from Error', () => { + const err = new SigningError(); + assert(err instanceof Error); + assert.strictEqual(err.name, 'SigningError'); + }); + }); +}); diff --git a/handwritten/storage/test/testdata/testfile.json b/handwritten/storage/test/testdata/testfile.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/handwritten/storage/test/testdata/textfile.txt b/handwritten/storage/test/testdata/textfile.txt new file mode 100644 index 00000000000..c66d471e359 --- /dev/null +++ b/handwritten/storage/test/testdata/textfile.txt @@ -0,0 +1 @@ +This is a test file! diff --git a/handwritten/storage/test/transfer-manager.ts b/handwritten/storage/test/transfer-manager.ts new file mode 100644 index 00000000000..2582782fa7a --- /dev/null +++ b/handwritten/storage/test/transfer-manager.ts @@ -0,0 +1,734 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + ApiError, + Bucket, + File, + CRC32C, + DownloadCallback, + DownloadOptions, + IdempotencyStrategy, + MultiPartHelperGenerator, + MultiPartUploadError, + MultiPartUploadHelper, + UploadOptions, + TransferManager, + Storage, + DownloadResponse, + DownloadManyFilesOptions, +} from '../src/index.js'; +import assert from 'assert'; +import * as path from 'path'; +import {GaxiosOptions, GaxiosResponse} from 'gaxios'; +import {GCCL_GCS_CMD_KEY} from '../src/nodejs-common/util.js'; +import {AuthClient, GoogleAuth} from 'google-auth-library'; +import {tmpdir} from 'os'; +import fs from 'fs'; +import {promises as fsp, Stats} from 'fs'; + +import * as sinon from 'sinon'; + +describe('Transfer Manager', () => { + const BUCKET_NAME = 'test-bucket'; + const STORAGE = sinon.stub( + new Storage({ + retryOptions: { + autoRetry: true, + maxRetries: 3, + retryDelayMultiplier: 2, + totalTimeout: 600, + maxRetryDelay: 60, + retryableErrorFn: (err: ApiError) => { + return err.code === 500; + }, + idempotencyStrategy: IdempotencyStrategy.RetryConditional, + }, + }) + ); + let sandbox: sinon.SinonSandbox; + let transferManager: TransferManager; + let bucket: Bucket; + + before(() => { + sandbox = sinon.createSandbox(); + }); + + beforeEach(() => { + bucket = new Bucket(STORAGE, BUCKET_NAME); + transferManager = new TransferManager(bucket); + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('instantiation', () => { + it('should correctly set the bucket', () => { + assert.strictEqual(transferManager.bucket, bucket); + }); + }); + + describe('uploadManyFiles', () => { + beforeEach(() => { + sandbox.stub(fsp, 'lstat').resolves({ + isDirectory: () => { + return false; + }, + } as Stats); + }); + + it('calls upload with the provided file paths', async () => { + const paths = ['/a/b/c', '/d/e/f', '/h/i/j']; + let count = 0; + sandbox.stub(bucket, 'upload').callsFake(path => { + count++; + assert(paths.includes(path)); + }); + + await transferManager.uploadManyFiles(paths); + assert.strictEqual(count, paths.length); + }); + + it('sets ifGenerationMatch to 0 if skipIfExists is set', async () => { + const paths = ['/a/b/c']; + sandbox.stub(bucket, 'upload').callsFake((path, options) => { + assert.strictEqual( + (options as UploadOptions).preconditionOpts?.ifGenerationMatch, + 0 + ); + }); + + await transferManager.uploadManyFiles(paths, {skipIfExists: true}); + }); + + it('sets destination to prefix + filename when prefix is supplied', async () => { + const filePaths = ['a', 'b', 'foo', 'bar.txt'].join(path.sep); + const expectedDestination = [ + 'hello', + 'world', + 'a', + 'b', + 'foo', + 'bar.txt', + ].join(path.posix.sep); + sandbox.stub(bucket, 'upload').callsFake((path, options) => { + assert.strictEqual( + (options as UploadOptions).destination, + expectedDestination + ); + }); + + await transferManager.uploadManyFiles([filePaths], { + prefix: ['hello', 'world'].join(path.sep), + }); + }); + + it('returns a promise with the uploaded file if there is no callback', async () => { + const paths = [['a', 'b', 'foo', 'bar.txt'].join(path.sep)]; + sandbox.stub(bucket, 'upload').callsFake(() => { + const resp = [{name: paths[0].split(path.sep).join(path.posix.sep)}]; + return Promise.resolve(resp); + }); + + const result = await transferManager.uploadManyFiles(paths); + assert.strictEqual( + result[0][0].name, + paths[0].split(path.sep).join(path.posix.sep) + ); + }); + + it('should set the appropriate `GCCL_GCS_CMD_KEY`', async () => { + const paths = ['/a/b/foo/bar.txt']; + + sandbox.stub(bucket, 'upload').callsFake(async (_path, options) => { + assert.strictEqual( + (options as UploadOptions)[GCCL_GCS_CMD_KEY], + 'tm.upload_many' + ); + }); + + await transferManager.uploadManyFiles(paths, {prefix: 'hello/world'}); + }); + + it('replaces OS specific separator with posix separator when calling bucket.upload', async () => { + const filePath = ['a', 'b', 'c'].join(path.sep); + const expected = ['a', 'b', 'c'].join(path.posix.sep); + + sandbox.stub(bucket, 'upload').callsFake((path, options) => { + assert.strictEqual(expected, (options as UploadOptions).destination); + }); + + await transferManager.uploadManyFiles([filePath]); + }); + + it('allows the user to apply a custom destination transformation when supplied a custom function', async () => { + const paths = ['a', 'b', 'foo/bar', 'bar.txt']; + const expected = ['foo/a', 'b/bar', 'foo/foo/bar', 'bar.txt/bar']; + sandbox.stub(bucket, 'upload').callsFake((path, options) => { + const uploadOpts = options as UploadOptions; + assert(expected.includes(uploadOpts.destination as string)); + }); + + let callCount = 0; + const transformationFunc = (path: string) => { + assert.strictEqual(path, paths[callCount]); + return expected[callCount++]; + }; + await transferManager.uploadManyFiles(paths, { + customDestinationBuilder: transformationFunc, + }); + }); + }); + + describe('downloadManyFiles', () => { + beforeEach(() => { + sandbox.stub(fs, 'existsSync').returns(true); + }); + + it('calls download for each provided file', async () => { + let count = 0; + const firstFile = new File(bucket, 'first.txt'); + sandbox.stub(firstFile, 'download').callsFake(() => { + count++; + }); + const secondFile = new File(bucket, 'second.txt'); + sandbox.stub(secondFile, 'download').callsFake(() => { + count++; + }); + + const files = [firstFile, secondFile]; + await transferManager.downloadManyFiles(files); + assert.strictEqual(count, 2); + }); + + it('sets the destination correctly when provided a prefix', async () => { + const prefix = 'test-prefix'; + const filename = 'first.txt'; + const expectedDestination = path.normalize(`${prefix}/${filename}`); + + const file = new File(bucket, filename); + sandbox.stub(file, 'download').callsFake(options => { + assert.strictEqual( + (options as DownloadOptions).destination, + expectedDestination + ); + }); + await transferManager.downloadManyFiles([file], {prefix}); + }); + + it('sets the destination correctly when provided a strip prefix', async () => { + const stripPrefix = 'should-be-removed/'; + const filename = 'should-be-removed/first.txt'; + const expectedDestination = 'first.txt'; + + const file = new File(bucket, filename); + sandbox.stub(file, 'download').callsFake(options => { + assert.strictEqual( + (options as DownloadOptions).destination, + expectedDestination + ); + }); + await transferManager.downloadManyFiles([file], {stripPrefix}); + }); + + it('should set the appropriate `GCCL_GCS_CMD_KEY`', async () => { + const file = new File(bucket, 'first.txt'); + + sandbox.stub(file, 'download').callsFake(async options => { + assert.strictEqual( + (options as DownloadOptions)[GCCL_GCS_CMD_KEY], + 'tm.download_many' + ); + }); + + await transferManager.downloadManyFiles([file]); + }); + + it('sets the destination correctly when provided a passthroughOptions.destination', async () => { + const passthroughOptions = { + destination: 'test-destination', + }; + const filename = 'first.txt'; + const expectedDestination = path.normalize( + `${passthroughOptions.destination}/${filename}` + ); + const download = (optionsOrCb?: DownloadOptions | DownloadCallback) => { + if (typeof optionsOrCb === 'function') { + optionsOrCb(null, Buffer.alloc(0)); + } else if (optionsOrCb) { + assert.strictEqual(optionsOrCb.destination, expectedDestination); + } + return Promise.resolve([Buffer.alloc(0)]) as Promise; + }; + + const file = new File(bucket, filename); + file.download = download; + await transferManager.downloadManyFiles([file], {passthroughOptions}); + }); + + it('does not download files that already exist locally when skipIfExists is true', async () => { + const firstFile = new File(bucket, 'first.txt'); + sandbox.stub(firstFile, 'download').callsFake(options => { + assert.strictEqual( + (options as DownloadManyFilesOptions).skipIfExists, + 0 + ); + }); + const secondFile = new File(bucket, 'second.txt'); + sandbox.stub(secondFile, 'download').callsFake(options => { + assert.strictEqual( + (options as DownloadManyFilesOptions).skipIfExists, + 0 + ); + }); + + const files = [firstFile, secondFile]; + const options = {skipIfExists: true}; + await transferManager.downloadManyFiles(files, options); + }); + + it('does not set the destination when prefix, strip prefix and passthroughOptions.destination are not provided', async () => { + const options = {}; + const filename = 'first.txt'; + const download = (optionsOrCb?: DownloadOptions | DownloadCallback) => { + if (typeof optionsOrCb === 'function') { + optionsOrCb(null, Buffer.alloc(0)); + } else if (optionsOrCb) { + assert.strictEqual(optionsOrCb.destination, undefined); + } + return Promise.resolve([Buffer.alloc(0)]) as Promise; + }; + + const file = new File(bucket, filename); + file.download = download; + await transferManager.downloadManyFiles([file], options); + }); + + it('should recursively create directory and write file contents if destination path is nested', async () => { + const prefix = 'text-prefix'; + const folder = 'nestedFolder/'; + const file = 'first.txt'; + const filesOrFolder = [folder, path.join(folder, file)]; + const expectedFilePath = path.join(prefix, folder, file); + const expectedDir = path.join(prefix, folder); + const mkdirSpy = sandbox.spy(fsp, 'mkdir'); + const download = (optionsOrCb?: DownloadOptions | DownloadCallback) => { + if (typeof optionsOrCb === 'function') { + optionsOrCb(null, Buffer.alloc(0)); + } else if (optionsOrCb) { + assert.strictEqual(optionsOrCb.destination, expectedFilePath); + } + return Promise.resolve([Buffer.alloc(0)]) as Promise; + }; + + sandbox.stub(bucket, 'file').callsFake(filename => { + const file = new File(bucket, filename); + file.download = download; + return file; + }); + await transferManager.downloadManyFiles(filesOrFolder, { + prefix: prefix, + }); + assert.strictEqual( + mkdirSpy.calledOnceWith(expectedDir, { + recursive: true, + }), + true + ); + }); + }); + + describe('downloadFileInChunks', () => { + let file: File; + + beforeEach(() => { + sandbox.stub(fsp, 'open').resolves({ + close: () => Promise.resolve(), + write: (buffer: unknown) => Promise.resolve({buffer}), + } as fsp.FileHandle); + + file = new File(bucket, 'some-large-file'); + sandbox.stub(file, 'get').resolves([ + { + metadata: { + size: 1024, + crc32c: 'AAAAAA==', + }, + }, + ]); + }); + + it('should download a single chunk if file size is below threshold', async () => { + let downloadCallCount = 0; + sandbox.stub(file, 'download').callsFake(() => { + downloadCallCount++; + return Promise.resolve([Buffer.alloc(100)]); + }); + + await transferManager.downloadFileInChunks(file); + assert.strictEqual(downloadCallCount, 1); + }); + + it('should return downloaded data', async () => { + sandbox.stub(file, 'download').callsFake(() => { + return Promise.resolve([Buffer.alloc(100)]); + }); + + const data = await transferManager.downloadFileInChunks(file); + assert.deepStrictEqual(data, [Buffer.alloc(1024)]); + }); + + it('should not return downloaded data when noReturnData flag is set', async () => { + sandbox.stub(file, 'download').callsFake(() => { + return Promise.resolve([Buffer.alloc(100)]); + }); + + const data = await transferManager.downloadFileInChunks(file, { + noReturnData: true, + }); + assert.strictEqual(data, undefined); + }); + + it('should call fromFile when validation is set to crc32c', async () => { + let callCount = 0; + file.download = () => { + return Promise.resolve([Buffer.alloc(0)]) as Promise; + }; + CRC32C.fromFile = () => { + callCount++; + return Promise.resolve(new CRC32C(0)); + }; + + await transferManager.downloadFileInChunks(file, {validation: 'crc32c'}); + assert.strictEqual(callCount, 1); + }); + + it('should throw an error if crc32c validation fails', async () => { + file.download = () => { + return Promise.resolve([Buffer.alloc(0)]) as Promise; + }; + CRC32C.fromFile = () => { + return Promise.resolve(new CRC32C(1)); // Set non-expected initial value + }; + + await assert.rejects( + transferManager.downloadFileInChunks(file, {validation: 'crc32c'}), + { + code: 'CONTENT_DOWNLOAD_MISMATCH', + } + ); + }); + + it('should set the appropriate `GCCL_GCS_CMD_KEY`', async () => { + sandbox.stub(file, 'download').callsFake(async options => { + assert.strictEqual( + (options as DownloadOptions)[GCCL_GCS_CMD_KEY], + 'tm.download_sharded' + ); + return [Buffer.alloc(100)]; + }); + + await transferManager.downloadFileInChunks(file); + }); + }); + + describe('uploadFileInChunks', () => { + let mockGeneratorFunction: MultiPartHelperGenerator; + let fakeHelper: sinon.SinonStubbedInstance; + let readStreamSpy: sinon.SinonSpy; + let directory: string; + let filePath: string; + class FakeXMLHelper implements MultiPartUploadHelper { + bucket: Bucket; + fileName: string; + uploadId?: string | undefined; + partsMap?: Map | undefined; + constructor(bucket: Bucket, fileName: string) { + this.bucket = bucket; + this.fileName = fileName; + } + initiateUpload(): Promise { + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + uploadPart(partNumber: number, chunk: Buffer): Promise { + throw new Error('Method not implemented.'); + } + completeUpload(): Promise { + throw new Error('Method not implemented.'); + } + abortUpload(): Promise { + throw new Error('Method not implemented.'); + } + } + + before(async () => { + directory = await fsp.mkdtemp( + path.join(tmpdir(), 'tm-uploadFileInChunks-') + ); + + filePath = path.join(directory, 't.txt'); + + await fsp.writeFile(filePath, 'hello'); + }); + + beforeEach(async () => { + readStreamSpy = sandbox.spy(fs, 'createReadStream'); + mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => { + fakeHelper = sandbox.createStubInstance(FakeXMLHelper); + fakeHelper.uploadId = uploadId || ''; + fakeHelper.partsMap = partsMap || new Map(); + fakeHelper.initiateUpload.resolves(); + fakeHelper.uploadPart.resolves(); + fakeHelper.completeUpload.resolves(); + fakeHelper.abortUpload.resolves(); + return fakeHelper; + }; + }); + + after(async () => { + await fsp.rm(directory, {force: true, recursive: true}); + }); + + it('should call initiateUpload, uploadPart, and completeUpload', async () => { + await transferManager.uploadFileInChunks( + filePath, + {}, + mockGeneratorFunction + ); + assert.strictEqual(fakeHelper.initiateUpload.calledOnce, true); + assert.strictEqual(fakeHelper.uploadPart.calledOnce, true); + assert.strictEqual(fakeHelper.completeUpload.calledOnce, true); + }); + + it('should call createReadStream with a highWaterMark equal to chunkSize', async () => { + const options = {highWaterMark: 32 * 1024 * 1024, start: 0}; + + await transferManager.uploadFileInChunks( + filePath, + { + chunkSizeBytes: 32 * 1024 * 1024, + }, + mockGeneratorFunction + ); + + assert.strictEqual(readStreamSpy.calledOnceWith(filePath, options), true); + }); + + it('should set the correct start offset when called with an existing parts map', async () => { + const options = { + highWaterMark: 32 * 1024 * 1024, + start: 64 * 1024 * 1024, + }; + + await transferManager.uploadFileInChunks( + filePath, + { + uploadId: '123', + partsMap: new Map([ + [1, '123'], + [2, '321'], + ]), + chunkSizeBytes: 32 * 1024 * 1024, + }, + mockGeneratorFunction + ); + + assert.strictEqual(readStreamSpy.calledOnceWith(filePath, options), true); + }); + + it('should not call initiateUpload if an uploadId is provided', async () => { + await transferManager.uploadFileInChunks( + filePath, + { + uploadId: '123', + partsMap: new Map([ + [1, '123'], + [2, '321'], + ]), + }, + mockGeneratorFunction + ); + + assert.strictEqual(fakeHelper.uploadId, '123'); + assert.strictEqual(fakeHelper.initiateUpload.notCalled, true); + }); + + it('should reject with an error with empty uploadId and partsMap', async () => { + const expectedErr = new MultiPartUploadError( + 'Hello World', + '', + new Map() + ); + mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => { + fakeHelper = sandbox.createStubInstance(FakeXMLHelper); + fakeHelper.uploadId = uploadId || ''; + fakeHelper.partsMap = partsMap || new Map(); + fakeHelper.initiateUpload.rejects(new Error(expectedErr.message)); + fakeHelper.uploadPart.resolves(); + fakeHelper.completeUpload.resolves(); + fakeHelper.abortUpload.resolves(); + return fakeHelper; + }; + assert.rejects( + transferManager.uploadFileInChunks( + filePath, + {autoAbortFailure: false}, + mockGeneratorFunction + ), + expectedErr + ); + }); + + it('should pass through headers to initiateUpload', async () => { + const headersToAdd = { + 'Content-Type': 'foo/bar', + 'x-goog-meta-foo': 'foobar', + 'User-Agent': 'barfoo', + }; + + mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => { + fakeHelper = sandbox.createStubInstance(FakeXMLHelper); + fakeHelper.uploadId = uploadId || ''; + fakeHelper.partsMap = partsMap || new Map(); + fakeHelper.initiateUpload.callsFake(headers => { + assert.deepStrictEqual(headers, headersToAdd); + return Promise.resolve(); + }); + fakeHelper.uploadPart.resolves(); + fakeHelper.completeUpload.resolves(); + fakeHelper.abortUpload.resolves(); + return fakeHelper; + }; + + await transferManager.uploadFileInChunks( + filePath, + {headers: headersToAdd}, + mockGeneratorFunction + ); + }); + + it('should call abortUpload when a failure occurs after an uploadID is established', async () => { + const expectedErr = new MultiPartUploadError( + 'Hello World', + '', + new Map() + ); + const fakeId = '123'; + + mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => { + fakeHelper = sandbox.createStubInstance(FakeXMLHelper); + fakeHelper.uploadId = uploadId || ''; + fakeHelper.partsMap = partsMap || new Map(); + fakeHelper.initiateUpload.resolves(); + fakeHelper.uploadPart.callsFake(() => { + fakeHelper.uploadId = fakeId; + return Promise.reject(expectedErr); + }); + fakeHelper.completeUpload.resolves(); + fakeHelper.abortUpload.callsFake(() => { + assert.strictEqual(fakeHelper.uploadId, fakeId); + return Promise.resolve(); + }); + return fakeHelper; + }; + + assert.doesNotThrow(() => + transferManager.uploadFileInChunks(filePath, {}, mockGeneratorFunction) + ); + }); + + it('should set the appropriate `GCCL_GCS_CMD_KEY`', async () => { + let called = false; + class TestAuthClient extends AuthClient { + async getAccessToken() { + return {token: '', res: undefined}; + } + + async getRequestHeaders() { + return {}; + } + + async request(opts: GaxiosOptions) { + called = true; + + assert(opts.headers); + assert('x-goog-api-client' in opts.headers); + assert.match( + opts.headers['x-goog-api-client'], + /gccl-gcs-cmd\/tm.upload_sharded/ + ); + + return { + data: Buffer.from( + ` + 1 + ` + ), + headers: {}, + } as GaxiosResponse; + } + } + + transferManager.bucket.storage.authClient = new GoogleAuth({ + authClient: new TestAuthClient(), + }); + + await transferManager.uploadFileInChunks(filePath); + + assert(called); + }); + + it('should set User-Agent correctly based on package.json', async () => { + let called = false; + class TestAuthClient extends AuthClient { + async getAccessToken() { + return {token: '', res: undefined}; + } + + async getRequestHeaders() { + return {}; + } + + async request(opts: GaxiosOptions) { + called = true; + + assert(opts.headers); + assert('User-Agent' in opts.headers); + assert.match(opts.headers['User-Agent'], /gcloud-node/); + + return { + data: Buffer.from( + ` + 1 + ` + ), + headers: {}, + } as GaxiosResponse; + } + } + + transferManager.bucket.storage.authClient = new GoogleAuth({ + authClient: new TestAuthClient(), + }); + + await transferManager.uploadFileInChunks(filePath); + + assert(called); + }); + }); +}); diff --git a/handwritten/storage/tsconfig.cjs.json b/handwritten/storage/tsconfig.cjs.json new file mode 100644 index 00000000000..d0dbd70c64c --- /dev/null +++ b/handwritten/storage/tsconfig.cjs.json @@ -0,0 +1,19 @@ +{ + "extends": "./node_modules/gts/tsconfig-google.json", + "compilerOptions": { + "rootDir": ".", + "outDir": "build/cjs", + "resolveJsonModule": true, + "sourceMap": false, + "esModuleInterop": true, + }, + "include": [ + "src/*.ts", + "src/*.cjs", + "test/**/*.ts", + "system-test/*.ts", + "conformance-test/*.ts", + "conformance-test/scenarios/*.ts", + "internal-tooling/*.ts" + ] +} diff --git a/handwritten/storage/tsconfig.json b/handwritten/storage/tsconfig.json new file mode 100644 index 00000000000..bf65354d9fa --- /dev/null +++ b/handwritten/storage/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "./node_modules/gts/tsconfig-google.json", + "compilerOptions": { + "rootDir": ".", + "outDir": "build/esm", + "resolveJsonModule": true, + "module": "es2020", + "moduleResolution": "node", + "lib": ["ES2020"], + "sourceMap": false, + "esModuleInterop": true, + }, + "include": [ + "src/*.ts", + "src/*.cjs", + "internal-tooling/*.ts", + "system-test/*.ts" + ] +} \ No newline at end of file diff --git a/release-please-submodules.json b/release-please-submodules.json index 31841c0d0fd..148285687f3 100644 --- a/release-please-submodules.json +++ b/release-please-submodules.json @@ -1,13 +1,7 @@ { - "release-type": "node", - "separate-pull-requests": true, "commit-batch-size": 1, "include-component-in-tag": true, - "tag-separator": "-", "packages": { - "handwritten/logging-winston": { - "component": "logging-winston" - }, "handwritten/bigquery": { "component": "bigquery" }, @@ -19,11 +13,18 @@ }, "handwritten/logging-bunyan": { "component": "logging-bunyan" - } + }, + "handwritten/logging-winston": { + "component": "logging-winston" + }, + "handwritten/storage": {} }, "plugins": [ { "type": "sentence-case" } - ] + ], + "release-type": "node", + "separate-pull-requests": true, + "tag-separator": "-" }