Merge branch 'master' of github.com:HackTricks-wiki/hacktricks #134
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Translate All | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - 'scripts/**' | |
| - '.gitignore' | |
| - '.github/**' | |
| - Dockerfile | |
| workflow_dispatch: | |
| permissions: | |
| packages: write | |
| id-token: write | |
| contents: write | |
| jobs: | |
| translate: | |
| name: Translate → ${{ matrix.name }} (${{ matrix.branch }}) | |
| runs-on: ubuntu-latest | |
| # Run N languages in parallel (tune max-parallel if needed) | |
| strategy: | |
| fail-fast: false | |
| # max-parallel: 3 #Nothing to run all in parallel | |
| matrix: | |
| include: | |
| - { name: "Afrikaans", language: "Afrikaans", branch: "af" } | |
| - { name: "German", language: "German", branch: "de" } | |
| - { name: "Greek", language: "Greek", branch: "el" } | |
| - { name: "Spanish", language: "Spanish", branch: "es" } | |
| - { name: "French", language: "French", branch: "fr" } | |
| - { name: "Hindi", language: "Hindi", branch: "hi" } | |
| - { name: "Italian", language: "Italian", branch: "it" } | |
| - { name: "Japanese", language: "Japanese", branch: "ja" } | |
| - { name: "Korean", language: "Korean", branch: "ko" } | |
| - { name: "Polish", language: "Polish", branch: "pl" } | |
| - { name: "Portuguese", language: "Portuguese", branch: "pt" } | |
| - { name: "Serbian", language: "Serbian", branch: "sr" } | |
| - { name: "Swahili", language: "Swahili", branch: "sw" } | |
| - { name: "Turkish", language: "Turkish", branch: "tr" } | |
| - { name: "Ukrainian", language: "Ukrainian", branch: "uk" } | |
| - { name: "Chinese", language: "Chinese", branch: "zh" } | |
| # Ensure only one job per branch runs at a time (even across workflow runs) | |
| concurrency: | |
| group: translate-cloud-${{ matrix.branch }} | |
| cancel-in-progress: false | |
| container: | |
| image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest | |
| env: | |
| LANGUAGE: ${{ matrix.language }} | |
| BRANCH: ${{ matrix.branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Update and download scripts | |
| run: | | |
| sudo apt-get update | |
| # Install GitHub CLI properly | |
| curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ | |
| && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ | |
| && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ | |
| && sudo apt update \ | |
| && sudo apt install gh -y \ | |
| && sudo apt-get install -y wget | |
| mkdir scripts | |
| cd scripts | |
| wget -O get_and_save_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/get_and_save_refs.py | |
| wget -O compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py | |
| wget -O translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py | |
| cd .. | |
| - name: Run get_and_save_refs.py | |
| run: | | |
| python scripts/get_and_save_refs.py | |
| - name: Download language branch & update refs | |
| run: | | |
| git config --global --add safe.directory /__w/hacktricks/hacktricks | |
| git config --global user.name 'Translator' | |
| git config --global user.email '[email protected]' | |
| git config pull.rebase false | |
| git checkout $BRANCH | |
| git pull | |
| python scripts/compare_and_fix_refs.py --files-unmatched-paths /tmp/file_paths.txt | |
| git add . | |
| git commit -m "Fix unmatched refs" || echo "No changes to commit" | |
| git push || echo "No changes to push" | |
| - name: Run translation script on changed files | |
| run: | | |
| git checkout master | |
| export OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} | |
| git diff --name-only HEAD~1 | grep -v "SUMMARY.md" | while read -r file; do | |
| if echo "$file" | grep -qE '\.md$'; then | |
| echo -n ",$file" >> /tmp/file_paths.txt | |
| fi | |
| done | |
| echo "Files to translate:" | |
| cat /tmp/file_paths.txt | |
| echo "" | |
| echo "" | |
| touch /tmp/file_paths.txt | |
| if [ -s /tmp/file_paths.txt ]; then | |
| python scripts/translator.py \ | |
| --language "$LANGUAGE" \ | |
| --branch "$BRANCH" \ | |
| --api-key "$OPENAI_API_KEY" \ | |
| -f "$(cat /tmp/file_paths.txt)" \ | |
| -t 3 | |
| else | |
| echo "No markdown files changed, skipping translation." | |
| fi | |
| - name: Build mdBook | |
| run: | | |
| git checkout "$BRANCH" | |
| git pull | |
| MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1) | |
| - name: Publish search index release asset | |
| shell: bash | |
| env: | |
| PAT_TOKEN: ${{ secrets.PAT_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| ASSET="book/searchindex.js" | |
| TAG="searchindex-${BRANCH}" | |
| TITLE="Search Index (${BRANCH})" | |
| if [ ! -f "$ASSET" ]; then | |
| echo "Expected $ASSET to exist after build" >&2 | |
| exit 1 | |
| fi | |
| TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}" | |
| if [ -z "$TOKEN" ]; then | |
| echo "No token available for GitHub CLI" >&2 | |
| exit 1 | |
| fi | |
| export GH_TOKEN="$TOKEN" | |
| # Delete the release if it exists | |
| if gh release view "$TAG" >/dev/null 2>&1; then | |
| echo "Release $TAG already exists, deleting it..." | |
| gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" | |
| fi | |
| # Create new release | |
| gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY" | |
| # Login in AWs | |
| - name: Configure AWS credentials using OIDC | |
| uses: aws-actions/configure-aws-credentials@v3 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: us-east-1 | |
| # Sync the build to S3 | |
| - name: Sync to S3 | |
| run: | | |
| echo "Current branch:" | |
| git rev-parse --abbrev-ref HEAD | |
| echo "Syncing $BRANCH to S3" | |
| aws s3 sync ./book s3://hacktricks-wiki/$BRANCH --delete | |
| echo "Sync completed" | |
| echo "Cat 3 files from the book" | |
| find . -type f -name 'index.html' -print | head -n 3 | xargs -r cat |