build_master.yml

name: Build Master

on:
  push:
    branches:
      - master
    paths-ignore:
      - '.gitignore'
      - 'book/**'
  workflow_dispatch:

concurrency: build_master

permissions:
  packages: write
  id-token: write
  contents: write

jobs:
  run-translation:
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest
    environment: prod

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 #Needed to download everything to be able to access the master & language branches
      
      # Build the mdBook
      - name: Build mdBook
        run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)

      - name: Post-process SEO artifacts
        run: |
          python3 scripts/seo_postprocess.py pages \
            --book-dir ./book \
            --site-url https://cloud.hacktricks.wiki \
            --lang en \
            --default-lang en \
            --site-name "HackTricks Cloud"
      
      - name: Push search index to hacktricks-searchindex repo
        shell: bash
        env:
          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
        run: |
          set -euo pipefail

          ASSET="book/searchindex.js"
          TARGET_REPO="HackTricks-wiki/hacktricks-searchindex"
          FILENAME="searchindex-cloud-en.js"

          if [ ! -f "$ASSET" ]; then
            echo "Expected $ASSET to exist after build" >&2
            exit 1
          fi

          TOKEN="${PAT_TOKEN}"
          if [ -z "$TOKEN" ]; then
            echo "No PAT_TOKEN available" >&2
            exit 1
          fi

          # First, compress the original file (in the build directory)
          cd "${GITHUB_WORKSPACE}"
          gzip -9 -k -f "$ASSET"
          
          # Show compression stats
          ORIGINAL_SIZE=$(wc -c < "$ASSET")
          COMPRESSED_SIZE=$(wc -c < "${ASSET}.gz")
          RATIO=$(awk "BEGIN {printf \"%.1f\", ($COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100}")
          echo "Compression: ${ORIGINAL_SIZE} bytes -> ${COMPRESSED_SIZE} bytes (${RATIO}%)"
          
          # XOR encrypt the compressed file
          KEY='Prevent_Online_AVs_From_Flagging_HackTricks_Search_Gzip_As_Malicious_394h7gt8rf9u3rf9g'
          cat > /tmp/xor_encrypt.py << 'EOF'
          import sys
          key = sys.argv[1]
          input_file = sys.argv[2]
          output_file = sys.argv[3]
          with open(input_file, 'rb') as f:
              data = f.read()
          key_bytes = key.encode('utf-8')
          encrypted = bytearray(len(data))
          for i in range(len(data)):
              encrypted[i] = data[i] ^ key_bytes[i % len(key_bytes)]
          with open(output_file, 'wb') as f:
              f.write(encrypted)
          print(f"Encrypted: {len(data)} bytes")
          EOF
          python3 /tmp/xor_encrypt.py "$KEY" "${ASSET}.gz" "${ASSET}.gz.enc"
          
          # Rebuild and force-push with retries to handle concurrent updates.
          MAX_RETRIES=20
          RETRY_COUNT=0
          while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
            RETRY_COUNT=$((RETRY_COUNT + 1))
            echo "Push attempt ${RETRY_COUNT}/${MAX_RETRIES}"

            cd "${GITHUB_WORKSPACE}"
            rm -rf /tmp/searchindex-repo /tmp/searchindex-backup
            git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo

            cd /tmp/searchindex-repo
            git config user.name "GitHub Actions"
            git config user.email "github-actions@github.com"

            # Save all current files from master branch to temp directory.
            mkdir -p /tmp/searchindex-backup
            cp -r * /tmp/searchindex-backup/ 2>/dev/null || true

            # Create a fresh orphan branch (no history).
            git checkout --orphan new-main

            # Remove all files from git index (but keep working directory).
            git rm -rf . 2>/dev/null || true

            # Restore all files from backup (keeps all language files).
            cp -r /tmp/searchindex-backup/* . 2>/dev/null || true

            # Update English searchindex artifact.
            cp "${GITHUB_WORKSPACE}/${ASSET}.gz.enc" "${FILENAME}.gz"

            git add -A
            TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
            git commit -m "Update searchindex files - ${TIMESTAMP}" --allow-empty

            if git push -f origin new-main:master 2>&1 | tee /tmp/push_output.txt; then
              echo "Successfully reset repository history and pushed all searchindex files"
              break
            fi

            if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
              if grep -q "cannot lock ref 'refs/heads/master'" /tmp/push_output.txt; then
                echo "Concurrent update detected on remote master. Retrying..."
              else
                echo "Force push failed. Retrying..."
              fi
              sleep 1
            else
              echo "Failed to push after ${MAX_RETRIES} attempts"
              exit 1
            fi
          done

      # Login in AWs
      - name: Configure AWS credentials using OIDC
        uses: aws-actions/configure-aws-credentials@v3
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1
      
      # Sync the build to S3
      - name: Sync to S3
        run: aws s3 sync ./book s3://hacktricks-cloud/en --delete

      - name: Upload root sitemap index
        run: |
          LANGS=$(aws s3api list-objects-v2 --bucket hacktricks-cloud --delimiter / --query 'CommonPrefixes[].Prefix' --output text | tr '\t' '\n' | sed 's:/$::' | grep -E '^[a-z]{2}$' | sort | paste -sd, -)
          if [ -z "$LANGS" ]; then
            LANGS="en"
          fi
          python3 scripts/seo_postprocess.py index --site-url https://cloud.hacktricks.wiki --languages "$LANGS" --output ./sitemap.xml
          aws s3 cp ./sitemap.xml s3://hacktricks-cloud/sitemap.xml --content-type application/xml --cache-control max-age=300

      - name: Upload root ads.txt
        run: |
          aws s3 cp ./ads.txt s3://hacktricks-cloud/ads.txt --content-type text/plain --cache-control max-age=300
          aws s3 cp ./ads.txt s3://hacktricks-cloud/en/ads.txt --content-type text/plain --cache-control max-age=300

      - name: Upload root robots.txt
        run: |
          aws s3 cp ./src/robots.txt s3://hacktricks-cloud/robots.txt --content-type text/plain --cache-control max-age=300
          aws s3 cp ./src/robots.txt s3://hacktricks-cloud/en/robots.txt --content-type text/plain --cache-control max-age=300

      - name: Invalidate CloudFront HTML and SEO assets
        run: |
          aws cloudfront create-invalidation \
            --distribution-id "${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }}" \
            --paths "/en/*" "/robots.txt" "/en/robots.txt" "/sitemap.xml" "/en/sitemap.xml"