Repository navigation
Merge pull request #2953 from HackTricks-wiki/research_update_src_pri… #1467
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Translate All | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - '.gitignore' | |
| - Dockerfile | |
| - '.github/**' | |
| workflow_dispatch: | |
| inputs: | |
| translation_base_sha: | |
| description: "Optional base commit for a catch-up translation range" | |
| required: false | |
| type: string | |
| translation_head_sha: | |
| description: "Optional head commit for a catch-up translation range" | |
| required: false | |
| type: string | |
| permissions: | |
| packages: write | |
| id-token: write | |
| contents: write | |
| actions: read | |
| pull-requests: read | |
| jobs: | |
| translate: | |
| name: Translate → ${{ matrix.name }} (${{ matrix.branch }}) | |
| runs-on: ubuntu-latest | |
| # Run N languages in parallel (tune max-parallel if needed) | |
| strategy: | |
| fail-fast: false | |
| # max-parallel: 3 #Nothing to run all in parallel | |
| matrix: | |
| include: | |
| - { name: "Afrikaans", language: "Afrikaans", branch: "af" } | |
| - { name: "German", language: "German", branch: "de" } | |
| - { name: "Greek", language: "Greek", branch: "el" } | |
| - { name: "Spanish", language: "Spanish", branch: "es" } | |
| - { name: "French", language: "French", branch: "fr" } | |
| - { name: "Hindi", language: "Hindi", branch: "hi" } | |
| - { name: "Italian", language: "Italian", branch: "it" } | |
| - { name: "Japanese", language: "Japanese", branch: "ja" } | |
| - { name: "Korean", language: "Korean", branch: "ko" } | |
| - { name: "Polish", language: "Polish", branch: "pl" } | |
| - { name: "Portuguese", language: "Portuguese", branch: "pt" } | |
| - { name: "Serbian", language: "Serbian", branch: "sr" } | |
| - { name: "Swahili", language: "Swahili", branch: "sw" } | |
| - { name: "Turkish", language: "Turkish", branch: "tr" } | |
| - { name: "Ukrainian", language: "Ukrainian", branch: "uk" } | |
| - { name: "Chinese", language: "Chinese", branch: "zh" } | |
| # Ensure only one job per branch runs at a time (even across workflow runs) | |
| concurrency: | |
| group: translate-cloud-${{ matrix.branch }} | |
| cancel-in-progress: false | |
| container: | |
| image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest | |
| env: | |
| LANGUAGE: ${{ matrix.language }} | |
| BRANCH: ${{ matrix.branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Verify image tools and download scripts | |
| run: | | |
| gh --version | |
| wget --version | head -n 1 | |
| cp .github/build_compact_search_index.py /tmp/build_compact_search_index.py | |
| cp theme/ht_searcher.js /tmp/ht_searcher.js | |
| mkdir -p scripts | |
| cd scripts | |
| wget -O get_and_save_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/get_and_save_refs.py | |
| wget -O compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py | |
| wget -O translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py | |
| cp mark_unchanged_s3_files.py /tmp/mark_unchanged_s3_files.py | |
| cp upload_immutable_images.sh /tmp/upload_immutable_images.sh | |
| cd .. | |
| mkdir -p /tmp/immutable-images | |
| cp \ | |
| src/images/hacktricks-summer-discount-2026-v1.webp \ | |
| src/images/arte-badge-v1.webp \ | |
| src/images/grte-badge-v1.webp \ | |
| src/images/azrte-badge-v1.webp \ | |
| /tmp/immutable-images/ | |
| cp theme/discount.js /tmp/discount.js | |
| wget -O /tmp/seo_postprocess.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks/master/scripts/seo_postprocess.py | |
| - name: Run get_and_save_refs.py | |
| run: | | |
| python scripts/get_and_save_refs.py | |
| - name: Download language branch & update refs | |
| run: | | |
| git config --global --add safe.directory /__w/hacktricks/hacktricks | |
| git config --global user.name 'Translator' | |
| git config --global user.email 'github-actions@github.com' | |
| git config pull.rebase false | |
| git checkout $BRANCH | |
| git pull | |
| python scripts/compare_and_fix_refs.py --files-unmatched-paths /tmp/file_paths.txt | |
| git add . | |
| git commit -m "Fix unmatched refs" || echo "No changes to commit" | |
| git push || echo "No changes to push" | |
| - name: Run translation script on changed files | |
| env: | |
| TRANSLATION_BASE_SHA: ${{ inputs.translation_base_sha }} | |
| TRANSLATION_HEAD_SHA: ${{ inputs.translation_head_sha }} | |
| run: | | |
| git checkout master | |
| export OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} | |
| HEAD_SHA="${TRANSLATION_HEAD_SHA:-$GITHUB_SHA}" | |
| BASE_SHA="${TRANSLATION_BASE_SHA:-${HEAD_SHA}^}" | |
| git rev-parse --verify "${BASE_SHA}^{commit}" >/dev/null | |
| git rev-parse --verify "${HEAD_SHA}^{commit}" >/dev/null | |
| echo "Collecting changed files from ${BASE_SHA} to ${HEAD_SHA}" | |
| git diff --name-only "$BASE_SHA" "$HEAD_SHA" | grep -v "SUMMARY.md" | while read -r file; do | |
| if echo "$file" | grep -qE '\.md$'; then | |
| echo -n ",$file" >> /tmp/file_paths.txt | |
| fi | |
| done | |
| echo "Files to translate (`wc -l < /tmp/file_paths.txt`):" | |
| cat /tmp/file_paths.txt | |
| echo "" | |
| echo "" | |
| touch /tmp/file_paths.txt | |
| if [ -s /tmp/file_paths.txt ]; then | |
| python scripts/translator.py \ | |
| --language "$LANGUAGE" \ | |
| --branch "$BRANCH" \ | |
| --api-key "$OPENAI_API_KEY" \ | |
| -f "$(cat /tmp/file_paths.txt)" \ | |
| -t 3 | |
| else | |
| echo "No markdown files changed, skipping translation." | |
| fi | |
| - name: Build mdBook | |
| run: | | |
| git checkout "$BRANCH" | |
| git pull | |
| MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1) | |
| cp /tmp/immutable-images/* ./book/images/ | |
| cp /tmp/discount.js ./book/discount.js | |
| cp /tmp/ht_searcher.js ./book/theme/ht_searcher.js | |
| - name: Push search index to hacktricks-searchindex repo | |
| shell: bash | |
| env: | |
| PAT_TOKEN: ${{ secrets.PAT_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| ASSET="book/searchindex.js" | |
| COMPACT_ASSET="/tmp/searchindex-v2.json" | |
| TARGET_REPO="HackTricks-wiki/hacktricks-searchindex" | |
| FILENAME="searchindex-${BRANCH}.js" | |
| COMPACT_FILENAME="searchindex-v2-${BRANCH}.json" | |
| if [ ! -f "$ASSET" ]; then | |
| echo "Expected $ASSET to exist after build" >&2 | |
| exit 1 | |
| fi | |
| TOKEN="${PAT_TOKEN}" | |
| if [ -z "$TOKEN" ]; then | |
| echo "No PAT_TOKEN available" >&2 | |
| exit 1 | |
| fi | |
| # Clone the searchindex repo | |
| git clone --depth 1 https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo | |
| # Compress the searchindex file | |
| python3 /tmp/build_compact_search_index.py "$ASSET" "$COMPACT_ASSET" | |
| gzip -9 -k -f "$ASSET" | |
| gzip -9 -k -f "$COMPACT_ASSET" | |
| # Show compression stats | |
| ORIGINAL_SIZE=$(wc -c < "$ASSET") | |
| COMPRESSED_SIZE=$(wc -c < "${ASSET}.gz") | |
| RATIO=$(awk "BEGIN {printf \"%.1f\", ($COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100}") | |
| echo "Compression: ${ORIGINAL_SIZE} bytes -> ${COMPRESSED_SIZE} bytes (${RATIO}%)" | |
| # XOR encrypt the compressed file | |
| KEY='Prevent_Online_AVs_From_Flagging_HackTricks_Search_Gzip_As_Malicious_394h7gt8rf9u3rf9g' | |
| cat > /tmp/xor_encrypt.py << 'EOF' | |
| import sys | |
| key = sys.argv[1] | |
| input_file = sys.argv[2] | |
| output_file = sys.argv[3] | |
| with open(input_file, 'rb') as f: | |
| data = f.read() | |
| key_bytes = key.encode('utf-8') | |
| encrypted = bytearray(len(data)) | |
| for i in range(len(data)): | |
| encrypted[i] = data[i] ^ key_bytes[i % len(key_bytes)] | |
| with open(output_file, 'wb') as f: | |
| f.write(encrypted) | |
| print(f"Encrypted: {len(data)} bytes") | |
| EOF | |
| python3 /tmp/xor_encrypt.py "$KEY" "${ASSET}.gz" "${ASSET}.gz.enc" | |
| python3 /tmp/xor_encrypt.py "$KEY" "${COMPACT_ASSET}.gz" "${COMPACT_ASSET}.gz.enc" | |
| # Copy ONLY the encrypted .gz version to the searchindex repo (no uncompressed .js) | |
| cp "${ASSET}.gz.enc" "/tmp/searchindex-repo/${FILENAME}.gz" | |
| cp "${COMPACT_ASSET}.gz.enc" "/tmp/searchindex-repo/${COMPACT_FILENAME}.gz" | |
| # Commit and push with retry logic | |
| cd /tmp/searchindex-repo | |
| git config user.name "GitHub Actions" | |
| git config user.email "github-actions@github.com" | |
| git add "${FILENAME}.gz" "${COMPACT_FILENAME}.gz" | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| else | |
| git commit -m "Update ${FILENAME} from hacktricks-cloud build" | |
| # Retry push up to 20 times with pull --rebase between attempts | |
| MAX_RETRIES=20 | |
| RETRY_COUNT=0 | |
| while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do | |
| if git push origin master; then | |
| echo "Successfully pushed on attempt $((RETRY_COUNT + 1))" | |
| break | |
| else | |
| RETRY_COUNT=$((RETRY_COUNT + 1)) | |
| if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then | |
| echo "Push failed, attempt $RETRY_COUNT/$MAX_RETRIES. Pulling and retrying..." | |
| # Try normal rebase first | |
| if git pull --rebase origin master 2>&1 | tee /tmp/pull_output.txt; then | |
| echo "Rebase successful, retrying push..." | |
| else | |
| # If rebase fails due to divergent histories (orphan branch reset), re-clone | |
| if grep -q "unrelated histories\|refusing to merge\|fatal: invalid upstream\|couldn't find remote ref" /tmp/pull_output.txt; then | |
| echo "Detected history rewrite, re-cloning repository..." | |
| cd /tmp | |
| rm -rf searchindex-repo | |
| git clone --depth 1 https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git searchindex-repo | |
| cd searchindex-repo | |
| git config user.name "GitHub Actions" | |
| git config user.email "github-actions@github.com" | |
| # Re-copy ONLY the encrypted .gz version (no uncompressed .js) | |
| cp "${GITHUB_WORKSPACE}/${ASSET}.gz.enc" "${FILENAME}.gz" | |
| cp "${COMPACT_ASSET}.gz.enc" "${COMPACT_FILENAME}.gz" | |
| git add "${FILENAME}.gz" "${COMPACT_FILENAME}.gz" | |
| git commit -m "Update ${FILENAME}.gz from hacktricks-cloud build" | |
| echo "Re-cloned and re-committed, will retry push..." | |
| else | |
| echo "Rebase failed for unknown reason, retrying anyway..." | |
| fi | |
| fi | |
| sleep 1 | |
| else | |
| echo "Failed to push after $MAX_RETRIES attempts" | |
| exit 1 | |
| fi | |
| fi | |
| done | |
| fi | |
| # Login in AWS | |
| - name: Configure AWS credentials using OIDC | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: us-east-1 | |
| - name: Post-process SEO artifacts | |
| run: | | |
| python3 /tmp/seo_postprocess.py pages \ | |
| --book-dir ./book \ | |
| --site-url https://hacktricks.wiki \ | |
| --lang "$BRANCH" \ | |
| --default-lang en \ | |
| --site-name "HackTricks" | |
| # Sync the build to S3 | |
| - name: Sync to S3 | |
| run: | | |
| echo "Current branch:" | |
| git rev-parse --abbrev-ref HEAD | |
| echo "Syncing $BRANCH to S3" | |
| # mdBook refreshes mtimes on every build. Age byte-identical files | |
| # based on single-part S3 ETags so `s3 sync` skips redundant PUTs. | |
| aws s3api list-objects-v2 \ | |
| --bucket hacktricks-wiki \ | |
| --prefix "$BRANCH/" \ | |
| --output json > /tmp/s3-branch-manifest.json | |
| python3 /tmp/mark_unchanged_s3_files.py \ | |
| --source ./book \ | |
| --manifest /tmp/s3-branch-manifest.json \ | |
| --remote-prefix "$BRANCH/" | |
| aws s3 sync ./book s3://hacktricks-wiki/$BRANCH --delete | |
| bash /tmp/upload_immutable_images.sh hacktricks-wiki "$BRANCH" | |
| echo "Sync completed" | |
| echo "Cat 3 files from the book" | |
| find . -type f -name 'index.html' -print | head -n 3 | xargs -r cat | |
| finalize-deployment: | |
| name: Finalize translated deployment | |
| needs: translate | |
| if: ${{ always() && !cancelled() }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| environment: prod | |
| steps: | |
| - name: Checkout deployment scripts | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS credentials using OIDC | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: us-east-1 | |
| - name: Refresh root sitemap index | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| LANGS=$(aws s3api list-objects-v2 --bucket hacktricks-wiki --delimiter / --query 'CommonPrefixes[].Prefix' --output text | tr '\t' '\n' | sed 's:/$::' | grep -E '^[a-z]{2}$' | sort | paste -sd, - || true) | |
| if [ -z "$LANGS" ]; then | |
| LANGS="en" | |
| fi | |
| python3 scripts/seo_postprocess.py index \ | |
| --site-url https://hacktricks.wiki \ | |
| --languages "$LANGS" \ | |
| --output ./sitemap.xml | |
| if aws s3 cp s3://hacktricks-wiki/sitemap.xml /tmp/current-root-sitemap.xml >/dev/null 2>&1 && cmp -s /tmp/current-root-sitemap.xml ./sitemap.xml; then | |
| echo "Root sitemap unchanged; skipping its upload." | |
| else | |
| aws s3 cp ./sitemap.xml s3://hacktricks-wiki/sitemap.xml \ | |
| --content-type application/xml \ | |
| --cache-control max-age=300 | |
| fi | |
| # A push to the default branch fans out into several workflows that each | |
| # invalidate an overlapping set of paths, and CloudFront bills every path | |
| # past the first 1000 a month. Skip the ones another run already covers. | |
| - name: Check whether this invalidation is still needed | |
| id: invalidation | |
| uses: ./.github/actions/invalidation-needed | |
| with: | |
| github-token: ${{ github.token }} | |
| merge-comment-author: "carlospolop" | |
| - name: Invalidate translated HTML and SEO assets once | |
| if: steps.invalidation.outputs.needed == 'true' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| aws cloudfront create-invalidation \ | |
| --distribution-id "${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }}" \ | |
| --paths "/*" |