translate_all.yml

name: Translator All

on:
  push:
    branches:
      - master
    paths-ignore:
      - '.gitignore'
      - Dockerfile
  workflow_dispatch:

permissions:
  packages: write
  id-token: write
  contents: write

jobs:
  translate:
    name: Translate → ${{ matrix.name }} (${{ matrix.branch }})
    runs-on: ubuntu-latest
    environment: prod

    # Run N languages in parallel (tune max-parallel if needed)
    strategy:
      fail-fast: false
      # max-parallel: 3 #Nothing to run all in parallel
      matrix:
        include:
          - { name: "Afrikaans", language: "Afrikaans", branch: "af" }
          - { name: "German", language: "German", branch: "de" }
          - { name: "Greek", language: "Greek", branch: "el" }
          - { name: "Spanish", language: "Spanish", branch: "es" }
          - { name: "French",  language: "French",  branch: "fr" }
          - { name: "Hindi",  language: "Hindi",  branch: "hi" }
          - { name: "Italian", language: "Italian", branch: "it" }
          - { name: "Japanese", language: "Japanese", branch: "ja" }
          - { name: "Korean", language: "Korean", branch: "ko" }
          - { name: "Polish", language: "Polish", branch: "pl" }
          - { name: "Portuguese", language: "Portuguese", branch: "pt" }
          - { name: "Serbian", language: "Serbian", branch: "sr" }
          - { name: "Swahili", language: "Swahili", branch: "sw" }
          - { name: "Turkish", language: "Turkish", branch: "tr" }
          - { name: "Ukrainian", language: "Ukrainian", branch: "uk" }
          - { name: "Chinese", language: "Chinese", branch: "zh" }

    # Ensure only one job per branch runs at a time (even across workflow runs)
    concurrency:
      group: translate-${{ matrix.branch }}
      cancel-in-progress: false

    container:
      image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest

    env:
      LANGUAGE: ${{ matrix.language }}
      BRANCH:   ${{ matrix.branch }}

    steps:
      - name: Wait for build_master to start
        run: |
          echo "Waiting 30 seconds to let build_master.yml initialize and reset the searchindex repo..."
          sleep 30
          echo "Continuing with translation workflow"
      
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      
      - name: Update and download scripts
        run: |
          sudo apt-get update
          # Install GitHub CLI properly
          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
          && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
          && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
          && sudo apt update \
          && sudo apt install gh -y \
          && sudo apt-get install -y wget
          wget -O /tmp/get_and_save_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/get_and_save_refs.py
          wget -O /tmp/compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py
          wget -O /tmp/translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py
          wget -O /tmp/seo_postprocess.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/seo_postprocess.py

      - name: Run get_and_save_refs.py
        run: |
          python /tmp/get_and_save_refs.py

      - name: Download language branch & update refs
        run: |
          pwd
          ls -la
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git config --global user.name 'Translator'
          git config --global user.email 'github-actions@github.com'
          git config pull.rebase false
          git checkout $BRANCH
          git pull
          python /tmp/compare_and_fix_refs.py --files-unmatched-paths /tmp/file_paths.txt
          git add .
          git commit -m "Fix unmatched refs" || echo "No changes to commit"
          git push || echo "No changes to push"

      - name: Run translation script on changed files
        run: |
          git checkout master
          cp src/SUMMARY.md /tmp/master-summary.md
          export OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}
          git diff --name-only HEAD~1 | grep -v "SUMMARY.md" | while read -r file; do
            if echo "$file" | grep -qE '\.md$'; then
              echo -n ",$file" >> /tmp/file_paths.txt
            fi
          done

          echo "Files to translate (`wc -l < /tmp/file_paths.txt`):"
          cat /tmp/file_paths.txt
          echo ""
          echo ""
          touch /tmp/file_paths.txt

          if [ -s /tmp/file_paths.txt ]; then
            python /tmp/translator.py \
              --language "$LANGUAGE" \
              --branch "$BRANCH" \
              --api-key "$OPENAI_API_KEY" \
              -f "$(cat /tmp/file_paths.txt)" \
              -t 3
          else
            echo "No markdown files changed, skipping translation."
          fi

      - name: Sync SUMMARY.md from master
        run: |
          git checkout "$BRANCH"
          git pull
          if [ -f /tmp/master-summary.md ]; then
            cp /tmp/master-summary.md src/SUMMARY.md
            git add src/SUMMARY.md
            git commit -m "Sync SUMMARY.md with master" || echo "SUMMARY already up to date"
            git push || echo "No SUMMARY updates to push"
          else
            echo "master summary not exported; failing"
            exit 1
          fi

      - name: Build mdBook
        run: |
          git checkout "$BRANCH"
          git pull
          MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)

      - name: Push search index to hacktricks-searchindex repo
        shell: bash
        env:
          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
        run: |
          set -euo pipefail

          ASSET="book/searchindex.js"
          TARGET_REPO="HackTricks-wiki/hacktricks-searchindex"
          FILENAME="searchindex-cloud-${BRANCH}.js"

          if [ ! -f "$ASSET" ]; then
            echo "Expected $ASSET to exist after build" >&2
            exit 1
          fi

          TOKEN="${PAT_TOKEN}"
          if [ -z "$TOKEN" ]; then
            echo "No PAT_TOKEN available" >&2
            exit 1
          fi

          # Clone the searchindex repo
          git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo
          
          # Compress the searchindex file
          gzip -9 -k -f "$ASSET"
          
          # Show compression stats
          ORIGINAL_SIZE=$(wc -c < "$ASSET")
          COMPRESSED_SIZE=$(wc -c < "${ASSET}.gz")
          RATIO=$(awk "BEGIN {printf \"%.1f\", ($COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100}")
          echo "Compression: ${ORIGINAL_SIZE} bytes -> ${COMPRESSED_SIZE} bytes (${RATIO}%)"
          
          # XOR encrypt the compressed file
          KEY='Prevent_Online_AVs_From_Flagging_HackTricks_Search_Gzip_As_Malicious_394h7gt8rf9u3rf9g'
          cat > /tmp/xor_encrypt.py << 'EOF'
          import sys
          key = sys.argv[1]
          input_file = sys.argv[2]
          output_file = sys.argv[3]
          with open(input_file, 'rb') as f:
              data = f.read()
          key_bytes = key.encode('utf-8')
          encrypted = bytearray(len(data))
          for i in range(len(data)):
              encrypted[i] = data[i] ^ key_bytes[i % len(key_bytes)]
          with open(output_file, 'wb') as f:
              f.write(encrypted)
          print(f"Encrypted: {len(data)} bytes")
          EOF
          python3 /tmp/xor_encrypt.py "$KEY" "${ASSET}.gz" "${ASSET}.gz.enc" 
          
          # Copy ONLY the encrypted .gz version to the searchindex repo (no uncompressed .js)
          cp "${ASSET}.gz.enc" "/tmp/searchindex-repo/${FILENAME}.gz"
          
          # Commit and push with retry logic resilient to concurrent writers/history rewrites
          cd /tmp/searchindex-repo
          git config user.name "GitHub Actions"
          git config user.email "github-actions@github.com"

          MAX_RETRIES=20
          RETRY_COUNT=0
          while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
            RETRY_COUNT=$((RETRY_COUNT + 1))
            echo "Push attempt ${RETRY_COUNT}/${MAX_RETRIES}"

            # Rebuild commit from latest remote state each attempt.
            git fetch origin master
            git checkout -B master origin/master
            cp "${GITHUB_WORKSPACE}/${ASSET}.gz.enc" "${FILENAME}.gz"
            git add "${FILENAME}.gz"

            if git diff --staged --quiet; then
              echo "No changes to commit"
              break
            fi

            git commit -m "Update ${FILENAME} from hacktricks-cloud build"

            if git push origin master 2>&1 | tee /tmp/push_output.txt; then
              echo "Successfully pushed on attempt ${RETRY_COUNT}"
              break
            fi

            if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
              if grep -q "cannot lock ref 'refs/heads/master'" /tmp/push_output.txt; then
                echo "Concurrent update detected on remote master. Retrying..."
              else
                echo "Push failed. Retrying with fresh remote state..."
              fi
              sleep 1
            else
              echo "Failed to push after ${MAX_RETRIES} attempts"
              exit 1
            fi
          done
      
      # Login in AWs
      - name: Configure AWS credentials using OIDC
        uses: aws-actions/configure-aws-credentials@v3
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Post-process SEO artifacts
        run: |
          python3 /tmp/seo_postprocess.py pages \
            --book-dir ./book \
            --site-url https://cloud.hacktricks.wiki \
            --lang "$BRANCH" \
            --default-lang en \
            --site-name "HackTricks Cloud"
      
      # Sync the build to S3
      - name: Sync to S3
        run: |
          echo "Current branch:"
          git rev-parse --abbrev-ref HEAD
          echo "Syncing $BRANCH to S3"
          aws s3 sync ./book s3://hacktricks-cloud/$BRANCH --delete
          echo "Sync completed"
          echo "Cat 3 files from the book"
          find . -type f -name 'index.html' -print | head -n 3 | xargs -r cat

      - name: Refresh root sitemap index
        run: |
          LANGS=$(aws s3api list-objects-v2 --bucket hacktricks-cloud --delimiter / --query 'CommonPrefixes[].Prefix' --output text | tr '\t' '\n' | sed 's:/$::' | grep -E '^[a-z]{2}$' | sort | paste -sd, -)
          if [ -z "$LANGS" ]; then
            LANGS="en"
          fi
          python3 /tmp/seo_postprocess.py index --site-url https://cloud.hacktricks.wiki --languages "$LANGS" --output ./sitemap.xml
          aws s3 cp ./sitemap.xml s3://hacktricks-cloud/sitemap.xml --content-type application/xml --cache-control max-age=300

      - name: Invalidate CloudFront HTML and SEO assets
        run: |
          aws cloudfront create-invalidation \
            --distribution-id "${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }}" \
            --paths "/$BRANCH/*" "/robots.txt" "/sitemap.xml" "/$BRANCH/sitemap.xml"