diff --git a/.buildkite/commands/build-bundles.sh b/.buildkite/commands/build-bundles.sh new file mode 100755 index 0000000000..eb293abf4e --- /dev/null +++ b/.buildkite/commands/build-bundles.sh @@ -0,0 +1,26 @@ +#!/bin/bash -eu + +.buildkite/commands/install-node-dependencies.sh + +if [ -z "${BUILDKITE_TAG:-}" ]; then + echo "--- :package: Skip bundle prep work" +else + echo "--- :package: Run bundle prep work" + npm run prebundle:js +fi + +echo "--- :android: Build Android bundle" +npm run bundle:android + +echo "--- :arrow_up: Upload Android bundle and source map artifacts" +buildkite-agent artifact upload bundle/android/App.js +buildkite-agent artifact upload bundle/android/App.composed.js.map + +echo "--- :ios: Build iOS bundle" +npm run bundle:ios + +echo "--- :arrow_up: Upload iOS bundle and source map artifacts" +buildkite-agent artifact upload bundle/ios/App.js +buildkite-agent artifact upload bundle/ios/App.composed.js.map +tar -czvf ios-assets.tar.gz -C ios-xcframework/Gutenberg/Resources assets/ +buildkite-agent artifact upload ios-assets.tar.gz \ No newline at end of file diff --git a/.buildkite/commands/build-ios.sh b/.buildkite/commands/build-ios.sh index 265a7ba2e1..81a9696c21 100755 --- a/.buildkite/commands/build-ios.sh +++ b/.buildkite/commands/build-ios.sh @@ -1,10 +1,22 @@ #!/bin/bash -eu +PLATFORM=$(uname -s) +ARCHITECTURE=$(uname -m) +PODFILE_HASH=$(hash_file gutenberg/packages/react-native-editor/ios/Podfile.lock) +PODFILE_CACHEKEY="$BUILDKITE_PIPELINE_SLUG-pods-$PLATFORM-$ARCHITECTURE-$PODFILE_HASH" +PODS_PATH="gutenberg/packages/react-native-editor/ios" +PODS_FOLDER="Pods" + echo '--- :desktop_computer: Clear up some disk space' rm -rfv ~/.Trash/15.1.xip .buildkite/commands/install-node-dependencies.sh +echo "--- :cocoapods: Restore Pods if present" +pushd "$PODS_PATH" +restore_cache "$PODFILE_CACHEKEY" +popd + echo '--- :ios: Set env var for iOS E2E testing' set -x export TEST_RN_PLATFORM=ios @@ -35,3 +47,8 @@ curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" \ --form 'payload=@"./gutenberg/packages/react-native-editor/ios/GutenbergDemo.app.zip"' \ --form "name=Gutenberg-$SAUCE_FILENAME.app.zip" \ --form 'description="Gutenberg"' + +echo "--- :cocoapods: Save Pods cache if necessary" +pushd "$PODS_PATH" +save_cache "$PODS_FOLDER" "$PODFILE_CACHEKEY" +popd diff --git a/.buildkite/commands/install-node-dependencies.sh b/.buildkite/commands/install-node-dependencies.sh index d522c5f133..52507e4938 100755 --- a/.buildkite/commands/install-node-dependencies.sh +++ b/.buildkite/commands/install-node-dependencies.sh @@ -12,16 +12,59 @@ done PLATFORM=$(uname -s) ARCHITECTURE=$(uname -m) NODE_VERSION=$(node --version) +GUTENBERG_PACKAGE_VERSION=$(jq -r .version package.json) + +# npm PACKAGE_HASH=$(hash_file package-lock.json) GUTENBERG_PACKAGE_HASH=$(hash_file gutenberg/package-lock.json) +NPM_CACHEKEY="$BUILDKITE_PIPELINE_SLUG-npm-$PLATFORM-$ARCHITECTURE-node-$NODE_VERSION-$PACKAGE_HASH-$GUTENBERG_PACKAGE_HASH" +NPM_CACHE_FOLDER=".npm" + +# pnpm JETPACK_PACKAGE_HASH=$(hash_file jetpack/pnpm-lock.yaml) -CACHEKEY="$BUILDKITE_PIPELINE_SLUG-npm-$PLATFORM-$ARCHITECTURE-node$NODE_VERSION-$PACKAGE_HASH-$GUTENBERG_PACKAGE_HASH" -PNPM_CACHEKEY="$BUILDKITE_PIPELINE_SLUG-pnpm-$PLATFORM-$ARCHITECTURE-node$NODE_VERSION-$JETPACK_PACKAGE_HASH" +PNPM_CACHEKEY="$BUILDKITE_PIPELINE_SLUG-pnpm-$PLATFORM-$ARCHITECTURE-node-$NODE_VERSION-$JETPACK_PACKAGE_HASH" +PNPM_CACHE_FOLDER="store" + +# yarn +BLOCK_EXPERIMENTS_YARN_HASH=$(hash_file block-experiments/yarn.lock) +BLOCK_EXPERIMENTS_PACKAGE_HASH=$(hash_file block-experiments/package.json) +YARN_CACHEKEY="$BUILDKITE_PIPELINE_SLUG-yarn-$PLATFORM-$ARCHITECTURE-node-$NODE_VERSION-$BLOCK_EXPERIMENTS_YARN_HASH-$BLOCK_EXPERIMENTS_PACKAGE_HASH" + +# i18n +I18N_CACHEKEY="$BUILDKITE_PIPELINE_SLUG-i18n-$PLATFORM-$ARCHITECTURE-node-$NODE_VERSION-$GUTENBERG_PACKAGE_VERSION" +I18N_CACHE_FOLDER="src/i18n-cache" + +# Cache folder directories +if [ "$PLATFORM" = "Darwin" ]; then + PNPM_PATH="$HOME/Library/pnpm" + YARN_PATH="$HOME/Library/Caches" + YARN_CACHE_FOLDER="Yarn" +elif [ "$PLATFORM" = "Linux" ]; then + PNPM_PATH="$HOME/.local/share/pnpm" + YARN_PATH="$HOME/.cache" + YARN_CACHE_FOLDER="yarn" +fi echo "--- :npm: Restore cache if present" -restore_cache "$CACHEKEY" +# npm +pushd "$HOME" +restore_cache "$NPM_CACHEKEY" +popd + +# pnpm +mkdir -p "$PNPM_PATH" +pushd "$PNPM_PATH" restore_cache "$PNPM_CACHEKEY" +popd +# yarn +mkdir -p "$YARN_PATH" +pushd "$YARN_PATH" +restore_cache "$YARN_CACHEKEY" +popd + +# i18n +restore_cache "$I18N_CACHEKEY" if [[ "${RESTORE_ONLY}" == 'true' ]]; then echo 'Exiting after restoring caches as per --restore-only call parameter.' exit 0 @@ -41,7 +84,18 @@ echo "--- :npm: Save cache if necessary" # # Example: https://buildkite.com/automattic/gutenberg-mobile/builds/8857#018e37eb-7afc-4280-b736-cba76f02f1a3/524 rm -rf "$HOME/.npm/_cacache/tmp" -save_cache "$HOME/.npm" "$CACHEKEY" + +# npm +pushd "$HOME" +save_cache "$NPM_CACHE_FOLDER" "$NPM_CACHEKEY" +popd + +# i18n +if [ -d "$I18N_CACHE_FOLDER" ]; then + save_cache "$I18N_CACHE_FOLDER" "$I18N_CACHEKEY" +else + echo "Directory $I18N_CACHE_FOLDER does not exist. Skipping..." +fi # If we attempted to save the pnpm cache when npm run with '--prefix gutenberg', the command might fail. # That's because the Jetpack submodule alone uses pnpm. @@ -51,13 +105,12 @@ if echo "$@" | grep -q -- '--prefix gutenberg'; then exit 0 fi -if [ "$PLATFORM" = "Darwin" ]; then - PNPM_PATH="$HOME/Library/pnpm/store/v3" -elif [ "$PLATFORM" = "Linux" ]; then - PNPM_PATH="$HOME/.local/share/pnpm/store/v3" -else - echo "Unsupported platform: $PLATFORM." - exit 1 -fi +# pnpm +pushd "$PNPM_PATH" +save_cache "$PNPM_CACHE_FOLDER" "$PNPM_CACHEKEY" +popd -save_cache "$PNPM_PATH" "$PNPM_CACHEKEY" +# yarn +pushd "$YARN_PATH" +save_cache "$YARN_CACHE_FOLDER" "$YARN_CACHEKEY" +popd diff --git a/.buildkite/commands/lint.sh b/.buildkite/commands/lint.sh index a7913bc659..8b58afce42 100755 --- a/.buildkite/commands/lint.sh +++ b/.buildkite/commands/lint.sh @@ -1,6 +1,14 @@ #!/bin/bash -eu -.buildkite/commands/install-node-dependencies.sh +# Restore the caches, if any +.buildkite/commands/install-node-dependencies.sh --restore-only +# Set up the gutenberg-mobile dependencies without scripts (--ignore-scripts) +echo "--- :npm: Install Node dependencies" +npm ci --prefer-offline --no-progress --no-audit --ignore-scripts +# Set up building the i18n cache +npm run i18n:check-cache +# Set up the gutenberg submodule dependencies +npm ci --prefer-offline --no-progress --no-audit --prefix gutenberg --ignore-scripts echo "--- :node: Lint" CHECK_CORRECTNESS=true CHECK_TESTS=false ./bin/ci-checks-js.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 250cf2e8ac..03ac4deae4 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,14 +1,4 @@ x-common-params: - - &gb-mobile-docker-container - docker#v3.8.0: - image: 'public.ecr.aws/automattic/gb-mobile-image:latest' - environment: - - 'CI=true' - # Allow WP-CLI to be run as root, otherwise it throws an exception. - # Reference: https://git.io/J9q2S - - 'WP_CLI_ALLOW_ROOT=true' - # Increase max available memory for node - - 'NODE_OPTIONS="--max-old-space-size=4096"' - &git-partial-clone-plugin automattic/git-partial-clone#0.1.0 - &publish-android-artifacts-docker-container @@ -113,46 +103,13 @@ steps: - label: Build JS Bundles key: js-bundles - agents: - queue: android plugins: - # The following plugins are disabled temporarily until PHP is available. - # In the meantime, we'll keep using the GB-mobile docker container. - - *gb-mobile-docker-container - # - *ci_toolkit_plugin # unused? - # - *nvm_plugin - - *git-partial-clone-plugin - command: | - source /root/.bashrc - - echo "--- :node: Set up Node environment" - nvm install && nvm use - - echo "--- :npm: Install Node dependencies" - npm ci --unsafe-perm --prefer-offline --no-audit --no-progress - - if [[ -z "$BUILDKITE_TAG" ]]; then - echo "--- :package: Skip bundle prep work" - else - echo "--- :package: Run bundle prep work" - npm run prebundle:js - fi - - echo "--- :android: Build Android bundle" - npm run bundle:android - - echo "--- :arrow_up: Upload Android bundle and source map artifacts" - buildkite-agent artifact upload bundle/android/App.js - buildkite-agent artifact upload bundle/android/App.composed.js.map - - echo "--- :ios: Build iOS bundle" - npm run bundle:ios - - echo "--- :arrow_up: Upload iOS bundle and source map artifacts" - buildkite-agent artifact upload bundle/ios/App.js - buildkite-agent artifact upload bundle/ios/App.composed.js.map - tar -czvf ios-assets.tar.gz -C ios-xcframework/Gutenberg/Resources assets/ - buildkite-agent artifact upload ios-assets.tar.gz + - *ci_toolkit_plugin + - *nvm_plugin + command: .buildkite/commands/build-bundles.sh + agents: + queue: mac + env: *xcode_agent_env - label: "Build Android RN Aztec & Publish to S3" key: "publish-react-native-aztec-android" diff --git a/bin/ci-checks-js.sh b/bin/ci-checks-js.sh index 9daa3535d5..2ab82d51e7 100755 --- a/bin/ci-checks-js.sh +++ b/bin/ci-checks-js.sh @@ -38,12 +38,6 @@ if [ "$CHECK_CORRECTNESS" = true ] ; then checkDiff echo "--- :eslint: Lint" - # Need to build gutenberg packages before linting so that eslint-plugin-import can resolve those. - # See https://github.com/WordPress/gutenberg/pull/22088 for more information. - cd gutenberg - npm run build:packages || pFail - cd .. - npm run lint || pFail fi @@ -55,11 +49,11 @@ if [ "$CHECK_TESTS" = true ] ; then # we'll run the tests twice (once for each platform) if the platform env var is not set if [[ -z "${TEST_RN_PLATFORM:-}" ]] ; then echo "--- :microscope: :android: Unit tests" - TEST_RN_PLATFORM=android npm run test --maxWorkers=4 > "$LOGS_DIR/android-tests-out.log" || pFail + TEST_RN_PLATFORM=android npm run test > "$LOGS_DIR/android-tests-out.log" || pFail echo "--- :microscope: :ios: Unit tests" - TEST_RN_PLATFORM=ios npm run test --maxWorkers=4 > "$LOGS_DIR/ios-tests-out.log" || pFail + TEST_RN_PLATFORM=ios npm run test > "$LOGS_DIR/ios-tests-out.log" || pFail else echo "--- :microscope: :$TEST_RN_PLATFORM: Unit tests" - npm run test --maxWorkers=4 > "$TEST_RN_PLATFORM-tests-out.log" || pFail + npm run test > "$TEST_RN_PLATFORM-tests-out.log" || pFail fi fi diff --git a/bin/run-block-experiments-command.sh b/bin/run-block-experiments-command.sh index fb77fddad2..f8d6f6b7a0 100755 --- a/bin/run-block-experiments-command.sh +++ b/bin/run-block-experiments-command.sh @@ -1,28 +1,6 @@ #!/bin/bash set -Eeuo pipefail -local_run_script="${GBM_LOCAL_BLOCK_EXPERIMENTS_RUN_SCRIPT:-./bin/run-block-experiments-command.sh.local}" - -if [ -e "$local_run_script" ] -then - source "$local_run_script" - exit 0 -fi - -# Check if nvm is installed -[ -z "$NVM_DIR" ] && NVM_DIR="$HOME/.nvm" -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - -command -v nvm >/dev/null 2>&1 || { - echo "nvm is not installed or cannot be sourced from $NVM_DIR/nvm.sh. Please verify that "'$NVM_DIR'" points to the .nvm directory." - exit 1 -} - -pushd block-experiments - -# Set up node requirement for block-experiments -nvm install - # Check if Yarn is installed if ! command -v yarn &> /dev/null then @@ -35,4 +13,4 @@ else fi # Install only regular dependencies (excluding devDependencies) -npx --silent yarn install --production \ No newline at end of file +npx --silent yarn install --cwd block-experiments --ignore-engines --production --prefer-offline \ No newline at end of file diff --git a/gutenberg b/gutenberg index eaf2d6550b..1ee3cc7615 160000 --- a/gutenberg +++ b/gutenberg @@ -1 +1 @@ -Subproject commit eaf2d6550b39587dc8945bf2e20fc4290a7139fe +Subproject commit 1ee3cc7615773e50077bffb078904cdbe1efbe04 diff --git a/package.json b/package.json index 3c9044de78..2b68cb33e4 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "webdriverio": "8.16.20" }, "scripts": { - "postinstall": "patch-package && npm run clean:gutenberg:distclean && npm ci --prefix gutenberg && ./bin/run-block-experiments-command.sh && npm run i18n:check-cache && ./bin/run-jetpack-command.sh \"install --ignore-scripts\"", + "postinstall": "npm ci --prefix gutenberg && ./bin/run-block-experiments-command.sh && npm run i18n:check-cache && ./bin/run-jetpack-command.sh \"install --prefer-offline --ignore-scripts\"", "start": "echo \"\\x1b[33mThe start command is not available in this project. It is strongly recommended to use \\x1b[1:33mstart:reset\\x1b[0m\\x1b[33m to perform some cleanup when starting the metro bundler.\nOr you may use \\x1b[1:33mstart:quick\\x1b[0m\\x1b[33m for a quicker startup, but this may lead to unexpected javascript errors when running the app.\\x1b[0m\"", "start:reset": "npm run core clean:runtime && npm run start:quick -- --reset-cache", "start:quick": "npm run core start:quick -- -- -- --config ../../../metro.config.js", @@ -99,7 +99,7 @@ "clean:gutenberg": "cd gutenberg && npm run clean:packages", "clean:gutenberg:distclean": "cd gutenberg && npm run distclean", "clean:i18n": "rm -rf src/i18n-cache && npm run i18n:check-cache", - "lint": "eslint . --ext .js", + "lint": "eslint . --ext .js --quiet", "lint:fix": "npm run lint -- --fix", "xcframework:setup": "./bin/xcframework-setup", "xcframework:build": "npm run xcframework:setup && pushd ./ios-xcframework && ./build.sh && popd"