From 9ad2542a2993951f1a240d7400aa31951e0e60d6 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Tue, 3 Jun 2025 18:17:37 +0200 Subject: [PATCH 01/14] ci: parallelize the mdbook build process --- .github/workflows/build.sh | 13 +++++++++++++ .github/workflows/publish.yml | 8 +++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index 5569b290c228..c2d637cf694f 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -15,6 +15,19 @@ set -Eeuo pipefail book_lang=${1:?"Usage: $0 "} dest_dir=${2:?"Usage: $0 "} +# ensure source dir exists +mkdir -p source +# clean previous build artifacts +SOURCE_DIR="source/${book_lang}" +rm -rf "$SOURCE_DIR" +mkdir "$SOURCE_DIR" + +# clone the current state into the source directory +git clone . "$SOURCE_DIR" + +# now work from that new directory as the base directory +cd "$SOURCE_DIR" + if [ "$book_lang" = "en" ]; then echo "::group::Building English course" else diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index fe0ac23095e3..7d765e5e0989 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -53,12 +53,10 @@ jobs: run: .github/workflows/build.sh en book - name: Build all translations + # note: build.sh here uses the given book path relative to source/{language} run: | - for po_lang in ${{ env.LANGUAGES }}; do - .github/workflows/build.sh $po_lang book/$po_lang - mv book/$po_lang/html book/html/$po_lang - done - + parallel -i .github/workflows/build.sh {} book/{} -- ${{ env.LANGUAGES }} + parallel -i mv source/{}/book/{}/html book/html/{} -- ${{ env.LANGUAGES }} - name: Build translation report run: i18n-report report book/html/translation-report.html po/*.po From ffd64422578e29f159468d3c9735721fd3dd982a Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Tue, 3 Jun 2025 18:29:39 +0200 Subject: [PATCH 02/14] test the publish process without actually publishing by removing permissions and publish steps --- .github/workflows/publish.yml | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7d765e5e0989..7f40b18d968a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,21 +3,12 @@ name: Publish # See also TRANSLATIONS.md. on: + pull_request: push: branches: - main workflow_dispatch: -permissions: - contents: read - pages: write - id-token: write - -# Allow one concurrent deployment -concurrency: - group: pages - cancel-in-progress: true - env: CARGO_TERM_COLOR: always # Update the language picker in index.hbs to link new languages. @@ -25,9 +16,6 @@ env: jobs: publish: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest steps: - name: Checkout @@ -66,15 +54,3 @@ jobs: MDBOOK_OUTPUT='{"xgettext": {"pot-file": "messages.pot", "granularity": 0}}' mdbook build -d synced-po for file in synced-po/*.po; do msgmerge --update $file synced-po/messages.pot ; done i18n-report report book/html/synced-translation-report.html synced-po/*.po - - - name: Setup Pages - uses: actions/configure-pages@v5 - - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: book/html - - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 From de81b2fd9087e3623162b9778dfdc94a129b7be2 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Tue, 3 Jun 2025 18:41:09 +0200 Subject: [PATCH 03/14] install parallel (with moreutils) --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7f40b18d968a..0f95eb3bea73 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,6 +43,7 @@ jobs: - name: Build all translations # note: build.sh here uses the given book path relative to source/{language} run: | + sudo apt install moreutils parallel -i .github/workflows/build.sh {} book/{} -- ${{ env.LANGUAGES }} parallel -i mv source/{}/book/{}/html book/html/{} -- ${{ env.LANGUAGES }} - name: Build translation report From 2ee2402b257c4c691fc4894c799a078e55a70e53 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Tue, 3 Jun 2025 18:53:56 +0200 Subject: [PATCH 04/14] use gnu parallel that is already installed --- .github/workflows/publish.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0f95eb3bea73..700551e7490f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,9 +43,8 @@ jobs: - name: Build all translations # note: build.sh here uses the given book path relative to source/{language} run: | - sudo apt install moreutils - parallel -i .github/workflows/build.sh {} book/{} -- ${{ env.LANGUAGES }} - parallel -i mv source/{}/book/{}/html book/html/{} -- ${{ env.LANGUAGES }} + parallel .github/workflows/build.sh {} book/{} ::: ${{ env.LANGUAGES }} + parallel mv source/{}/book/{}/html book/html/{} ::: ${{ env.LANGUAGES }} - name: Build translation report run: i18n-report report book/html/translation-report.html po/*.po From 8791e59cedcbf965914a32cb607cc7a07ea9800b Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Tue, 3 Jun 2025 19:38:43 +0200 Subject: [PATCH 05/14] english translation needs to be moved to correct directory --- .github/workflows/publish.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 700551e7490f..37b5f2cbaeee 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,8 +38,9 @@ jobs: uses: ./.github/workflows/install-mdbook - name: Build course in English - run: .github/workflows/build.sh en book - + run: | + .github/workflows/build.sh en book + mv source/en/book book - name: Build all translations # note: build.sh here uses the given book path relative to source/{language} run: | From 8895c864fab15bbbc8198b00e4bcb0aacf5203db Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Fri, 6 Jun 2025 10:21:36 +0200 Subject: [PATCH 06/14] Revert "english translation needs to be moved to correct directory" This reverts commit 8791e59cedcbf965914a32cb607cc7a07ea9800b. --- .github/workflows/publish.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 37b5f2cbaeee..700551e7490f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,9 +38,8 @@ jobs: uses: ./.github/workflows/install-mdbook - name: Build course in English - run: | - .github/workflows/build.sh en book - mv source/en/book book + run: .github/workflows/build.sh en book + - name: Build all translations # note: build.sh here uses the given book path relative to source/{language} run: | From 98598e6b4a944d8a1f40718df2aa7b1bf497af09 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Fri, 6 Jun 2025 10:21:48 +0200 Subject: [PATCH 07/14] Revert "use gnu parallel that is already installed" This reverts commit 2ee2402b257c4c691fc4894c799a078e55a70e53. --- .github/workflows/publish.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 700551e7490f..0f95eb3bea73 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,8 +43,9 @@ jobs: - name: Build all translations # note: build.sh here uses the given book path relative to source/{language} run: | - parallel .github/workflows/build.sh {} book/{} ::: ${{ env.LANGUAGES }} - parallel mv source/{}/book/{}/html book/html/{} ::: ${{ env.LANGUAGES }} + sudo apt install moreutils + parallel -i .github/workflows/build.sh {} book/{} -- ${{ env.LANGUAGES }} + parallel -i mv source/{}/book/{}/html book/html/{} -- ${{ env.LANGUAGES }} - name: Build translation report run: i18n-report report book/html/translation-report.html po/*.po From fce62f98a41f16598e3e8cb92e5436e232465809 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Fri, 6 Jun 2025 10:22:05 +0200 Subject: [PATCH 08/14] Revert "install parallel (with moreutils)" This reverts commit de81b2fd9087e3623162b9778dfdc94a129b7be2. --- .github/workflows/publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0f95eb3bea73..7f40b18d968a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,7 +43,6 @@ jobs: - name: Build all translations # note: build.sh here uses the given book path relative to source/{language} run: | - sudo apt install moreutils parallel -i .github/workflows/build.sh {} book/{} -- ${{ env.LANGUAGES }} parallel -i mv source/{}/book/{}/html book/html/{} -- ${{ env.LANGUAGES }} - name: Build translation report From 20307f62a24d1561fa2fc2a92362c6e1df4f8c04 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Fri, 6 Jun 2025 10:22:22 +0200 Subject: [PATCH 09/14] Revert "ci: parallelize the mdbook build process" This reverts commit 9ad2542a2993951f1a240d7400aa31951e0e60d6. --- .github/workflows/build.sh | 13 ------------- .github/workflows/publish.yml | 8 +++++--- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index c2d637cf694f..5569b290c228 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -15,19 +15,6 @@ set -Eeuo pipefail book_lang=${1:?"Usage: $0 "} dest_dir=${2:?"Usage: $0 "} -# ensure source dir exists -mkdir -p source -# clean previous build artifacts -SOURCE_DIR="source/${book_lang}" -rm -rf "$SOURCE_DIR" -mkdir "$SOURCE_DIR" - -# clone the current state into the source directory -git clone . "$SOURCE_DIR" - -# now work from that new directory as the base directory -cd "$SOURCE_DIR" - if [ "$book_lang" = "en" ]; then echo "::group::Building English course" else diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7f40b18d968a..f6fa997833e3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -41,10 +41,12 @@ jobs: run: .github/workflows/build.sh en book - name: Build all translations - # note: build.sh here uses the given book path relative to source/{language} run: | - parallel -i .github/workflows/build.sh {} book/{} -- ${{ env.LANGUAGES }} - parallel -i mv source/{}/book/{}/html book/html/{} -- ${{ env.LANGUAGES }} + for po_lang in ${{ env.LANGUAGES }}; do + .github/workflows/build.sh $po_lang book/$po_lang + mv book/$po_lang/html book/html/$po_lang + done + - name: Build translation report run: i18n-report report book/html/translation-report.html po/*.po From 0cc11a5ba2a1562d2200408697fc6e5dbeea14b7 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Fri, 6 Jun 2025 10:37:52 +0200 Subject: [PATCH 10/14] ci: Use a job matrix for translations and combine the artifacts in the publish job --- .github/workflows/publish.yml | 40 +++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f6fa997833e3..87de7b20f48b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,10 +12,12 @@ on: env: CARGO_TERM_COLOR: always # Update the language picker in index.hbs to link new languages. - LANGUAGES: ar bn da de el es fa fr id it ja ko pl pt-BR ro ru tr uk vi zh-CN zh-TW jobs: - publish: + create-translation: + strategy: + matrix: + language: ["en", "ar", "bn", "da", "de", "el", "es", "fa", "fr", "id", "it", "ja", "ko", "pl", "pt-BR", "ro", "ru", "tr", "uk", "vi", "zh-CN", "zh-TW"] runs-on: ubuntu-latest steps: - name: Checkout @@ -38,14 +40,32 @@ jobs: uses: ./.github/workflows/install-mdbook - name: Build course in English + if: matrix.language == 'en' run: .github/workflows/build.sh en book - - name: Build all translations + - name: Build ${{ matrix.language }} translation + if: matrix.language != 'en' run: | - for po_lang in ${{ env.LANGUAGES }}; do - .github/workflows/build.sh $po_lang book/$po_lang - mv book/$po_lang/html book/html/$po_lang - done + .github/workflows/build.sh ${{ matrix.language }} book/${{ matrix.language }} + mkdir book/html + mv book/${{ matrix.language }}/html book/html/${{ matrix.language }} + + - name: Upload translation + uses: actions/upload-artifact@v4 + with: + name: comprehensive-rust-${{ matrix.language }} + path: book/ + + publish: + runs-on: ubuntu-latest + needs: create-translation + steps: + - name: Download all translations + uses: actions/download-artifact@v4 + with: + path: book/ + pattern: comprehensive-rust-* + merge-multiple: true - name: Build translation report run: i18n-report report book/html/translation-report.html po/*.po @@ -56,3 +76,9 @@ jobs: MDBOOK_OUTPUT='{"xgettext": {"pot-file": "messages.pot", "granularity": 0}}' mdbook build -d synced-po for file in synced-po/*.po; do msgmerge --update $file synced-po/messages.pot ; done i18n-report report book/html/synced-translation-report.html synced-po/*.po + + - name: Upload all translations (for verification) + uses: actions/upload-artifact@v4 + with: + name: comprehensive-rust-all + path: book/ From cd922c46b4a37ba4b185d30c5cc5c5aa07391ad7 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Tue, 10 Jun 2025 10:49:16 +0200 Subject: [PATCH 11/14] install i18n-report (via install-mdbook step) in publish job --- .github/workflows/publish.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87de7b20f48b..eea8a02ed54c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -60,6 +60,15 @@ jobs: runs-on: ubuntu-latest needs: create-translation steps: + - name: Update Rust + run: rustup update + + - name: Setup Rust cache + uses: ./.github/workflows/setup-rust-cache + + - name: Install mdbook + uses: ./.github/workflows/install-mdbook + - name: Download all translations uses: actions/download-artifact@v4 with: From 67e4a6ad77aebb20c66dac47074cf3dc1983b4ea Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Tue, 10 Jun 2025 10:59:30 +0200 Subject: [PATCH 12/14] satisfy our pushy dprint --- .github/workflows/publish.yml | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eea8a02ed54c..2af393d12275 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,31 @@ jobs: create-translation: strategy: matrix: - language: ["en", "ar", "bn", "da", "de", "el", "es", "fa", "fr", "id", "it", "ja", "ko", "pl", "pt-BR", "ro", "ru", "tr", "uk", "vi", "zh-CN", "zh-TW"] + language: + [ + "en", + "ar", + "bn", + "da", + "de", + "el", + "es", + "fa", + "fr", + "id", + "it", + "ja", + "ko", + "pl", + "pt-BR", + "ro", + "ru", + "tr", + "uk", + "vi", + "zh-CN", + "zh-TW", + ] runs-on: ubuntu-latest steps: - name: Checkout @@ -85,7 +109,7 @@ jobs: MDBOOK_OUTPUT='{"xgettext": {"pot-file": "messages.pot", "granularity": 0}}' mdbook build -d synced-po for file in synced-po/*.po; do msgmerge --update $file synced-po/messages.pot ; done i18n-report report book/html/synced-translation-report.html synced-po/*.po - + - name: Upload all translations (for verification) uses: actions/upload-artifact@v4 with: From 0b30b2859f4e34b56c1c9f64d1afe5b3f94f5a43 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Tue, 10 Jun 2025 16:40:46 +0200 Subject: [PATCH 13/14] add checkout action --- .github/workflows/publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2af393d12275..5bab20a26495 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -84,6 +84,9 @@ jobs: runs-on: ubuntu-latest needs: create-translation steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Update Rust run: rustup update From 1c62035fa2d8e1cb530456b60dbe0a4dafd43644 Mon Sep 17 00:00:00 2001 From: Michael Kerscher Date: Thu, 12 Jun 2025 15:38:33 +0200 Subject: [PATCH 14/14] install gettext for msgmerge --- .github/workflows/publish.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5bab20a26495..7799c2040244 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -93,6 +93,11 @@ jobs: - name: Setup Rust cache uses: ./.github/workflows/setup-rust-cache + - name: Install Gettext + run: | + sudo apt update + sudo apt install gettext + - name: Install mdbook uses: ./.github/workflows/install-mdbook