Skip to content

Commit 570c0f4

Browse files
committed
fix searchindex
1 parent d0ebc37 commit 570c0f4

5 files changed

Lines changed: 129 additions & 111 deletions

File tree

.github/workflows/build_master.yml

Lines changed: 43 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -35,63 +35,59 @@ jobs:
3535
- name: Build mdBook
3636
run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
3737

38-
- name: Update searchindex in repo (purge history, keep current on HEAD)
38+
- name: Install GitHub CLI
39+
run: |
40+
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
41+
&& sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
42+
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
43+
&& sudo apt update \
44+
&& sudo apt install gh -y
45+
46+
- name: Publish search index release asset
3947
shell: bash
48+
env:
49+
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
4050
run: |
4151
set -euo pipefail
4252
43-
pwd
44-
ls -la
45-
ls -la book
46-
47-
git config --global --add safe.directory /__w/hacktricks-cloud/hacktricks-cloud
48-
git config --global user.email "build@example.com"
49-
git config --global user.name "Build master"
50-
git config pull.rebase false
51-
52-
# Ensure we're on the target branch and up to date
53-
# Handle force-pushed branches by resetting to origin/master
54-
git fetch origin
55-
git reset --hard origin/master
53+
ASSET="book/searchindex.js"
54+
TAG="searchindex-en"
55+
TITLE="Search Index (en)"
5656
57-
# Choose the file to keep at HEAD:
58-
# 1) Prefer freshly built version from book/
59-
# 2) Fallback to the file currently at HEAD (if it exists)
60-
HAS_FILE=0
61-
if [ -f "book/searchindex.js" ]; then
62-
cp "book/searchindex.js" /tmp/sidx.js
63-
HAS_FILE=1
64-
elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
65-
git show "HEAD:searchindex.js" > /tmp/sidx.js
66-
HAS_FILE=1
57+
if [ ! -f "$ASSET" ]; then
58+
echo "Expected $ASSET to exist after build" >&2
59+
exit 1
6760
fi
6861
69-
# Skip if there's nothing to purge AND nothing to keep
70-
if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 HEAD -- "searchindex.js" >/dev/null 2>&1; then
71-
# Fail early if working tree is dirty (avoid confusing rewrites)
72-
git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
73-
74-
# Install git-filter-repo and ensure it's on PATH
75-
python -m pip install --quiet --user git-filter-repo
76-
export PATH="$HOME/.local/bin:$PATH"
77-
78-
# Rewrite ONLY the current branch, dropping all historical blobs of searchindex.js
79-
git filter-repo --force --path "searchindex.js" --invert-paths --refs "$(git symbolic-ref -q HEAD)"
80-
81-
# Re-add the current version on top of rewritten history (keep it in HEAD)
82-
if [ "$HAS_FILE" = "1" ]; then
83-
mv /tmp/sidx.js "searchindex.js"
84-
git add "searchindex.js"
85-
git commit -m "Update searchindex (purged history; keep current)"
86-
else
87-
echo "No current searchindex.js to re-add after purge."
88-
fi
62+
TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
63+
if [ -z "$TOKEN" ]; then
64+
echo "No token available for GitHub CLI" >&2
65+
exit 1
66+
fi
67+
export GH_TOKEN="$TOKEN"
8968
90-
# Safer force push (only updates if remote hasn't advanced)
91-
git push --force-with-lease
69+
# Delete the release if it exists
70+
echo "Checking if release $TAG exists..."
71+
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
72+
echo "Release $TAG already exists, deleting it..."
73+
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag || {
74+
echo "Failed to delete release, trying without cleanup-tag..."
75+
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" || {
76+
echo "Warning: Could not delete existing release, will try to recreate..."
77+
}
78+
}
79+
sleep 2 # Give GitHub API a moment to process the deletion
9280
else
93-
echo "Nothing to purge; skipping."
81+
echo "Release $TAG does not exist, proceeding with creation..."
9482
fi
83+
84+
# Create new release (with force flag to overwrite if deletion failed)
85+
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY" || {
86+
echo "Failed to create release, trying with force flag..."
87+
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag >/dev/null 2>&1 || true
88+
sleep 2
89+
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY"
90+
}
9591
9692
# Login in AWs
9793
- name: Configure AWS credentials using OIDC

.github/workflows/translate_all.yml

Lines changed: 29 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,13 @@ jobs:
6666
- name: Update and download scripts
6767
run: |
6868
sudo apt-get update
69-
sudo apt-get install wget -y
69+
# Install GitHub CLI properly
70+
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
71+
&& sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
72+
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
73+
&& sudo apt update \
74+
&& sudo apt install gh -y \
75+
&& sudo apt-get install -y wget
7076
wget -O /tmp/get_and_save_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/get_and_save_refs.py
7177
wget -O /tmp/compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py
7278
wget -O /tmp/translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py
@@ -138,56 +144,37 @@ jobs:
138144
git pull
139145
MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
140146
141-
- name: Update searchindex.js in repo (purge history, keep current on HEAD)
147+
- name: Publish search index release asset
142148
shell: bash
149+
env:
150+
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
143151
run: |
144152
set -euo pipefail
145153
146-
# Be explicit about workspace trust (avoids "dubious ownership")
147-
git config --global --add safe.directory "$GITHUB_WORKSPACE"
154+
ASSET="book/searchindex.js"
155+
TAG="searchindex-${BRANCH}"
156+
TITLE="Search Index (${BRANCH})"
148157
149-
git checkout "$BRANCH"
150-
git fetch origin "$BRANCH" --quiet
151-
git pull --ff-only
152-
153-
# Choose the file to keep at HEAD:
154-
# 1) Prefer freshly built version from book/
155-
# 2) Fallback to the file currently at HEAD (if it exists)
156-
HAS_FILE=0
157-
if [ -f "book/searchindex.js" ]; then
158-
cp "book/searchindex.js" /tmp/sidx.js
159-
HAS_FILE=1
160-
elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
161-
git show "HEAD:searchindex.js" > /tmp/sidx.js
162-
HAS_FILE=1
158+
if [ ! -f "$ASSET" ]; then
159+
echo "Expected $ASSET to exist after build" >&2
160+
exit 1
163161
fi
164162
165-
# Skip if there's nothing to purge AND nothing to keep
166-
if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 "$BRANCH" -- "searchindex.js" >/dev/null 2>&1; then
167-
# **Fail early if working tree is dirty** (prevents confusing filter results)
168-
git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
169-
170-
# Make sure git-filter-repo is callable via `git filter-repo`
171-
python -m pip install --quiet --user git-filter-repo
172-
export PATH="$HOME/.local/bin:$PATH"
173-
174-
# Rewrite ONLY this branch, dropping all historical blobs of searchindex.js
175-
git filter-repo --force --path "searchindex.js" --invert-paths --refs "refs/heads/$BRANCH"
176-
177-
# Re-add the current version on top of rewritten history (keep it in HEAD)
178-
if [ "$HAS_FILE" = "1" ]; then
179-
mv /tmp/sidx.js "searchindex.js"
180-
git add "searchindex.js"
181-
git commit -m "Update searchindex (purged history; keep current)"
182-
else
183-
echo "No current searchindex.js to re-add after purge."
184-
fi
163+
TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
164+
if [ -z "$TOKEN" ]; then
165+
echo "No token available for GitHub CLI" >&2
166+
exit 1
167+
fi
168+
export GH_TOKEN="$TOKEN"
185169
186-
# **Safer force push** (prevents clobbering unexpected remote updates)
187-
git push --force-with-lease origin "$BRANCH"
188-
else
189-
echo "Nothing to purge; skipping."
170+
# Delete the release if it exists
171+
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
172+
echo "Release $TAG already exists, deleting it..."
173+
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY"
190174
fi
175+
176+
# Create new release
177+
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY"
191178
192179
# Login in AWs
193180
- name: Configure AWS credentials using OIDC

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,4 @@ book
3535
book/*
3636
hacktricks-preprocessor.log
3737
hacktricks-preprocessor-error.log
38+
searchindex.js

searchindex.js

Lines changed: 0 additions & 1 deletion
This file was deleted.

theme/ht_searcher.js

Lines changed: 56 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -21,33 +21,60 @@
2121
try { importScripts('https://cdn.jsdelivr.net/npm/elasticlunr@0.9.5/elasticlunr.min.js'); }
2222
catch { importScripts(abs('/elasticlunr.min.js')); }
2323
24-
/* 2 — load a single index (remote → local) */
25-
async function loadIndex(remote, local, isCloud=false){
26-
let rawLoaded = false;
24+
/* 2 — load a single index (remote → local) */
25+
async function loadIndex(remote, local, isCloud=false){
26+
let rawLoaded = false;
27+
if(remote){
2728
try {
2829
const r = await fetch(remote,{mode:'cors'});
2930
if (!r.ok) throw new Error('HTTP '+r.status);
3031
importScripts(URL.createObjectURL(new Blob([await r.text()],{type:'application/javascript'})));
3132
rawLoaded = true;
3233
} catch(e){ console.warn('remote',remote,'failed →',e); }
33-
if(!rawLoaded){
34-
try { importScripts(abs(local)); rawLoaded = true; }
35-
catch(e){ console.error('local',local,'failed →',e); }
34+
}
35+
if(!rawLoaded && local){
36+
try { importScripts(abs(local)); rawLoaded = true; }
37+
catch(e){ console.error('local',local,'failed →',e); }
38+
}
39+
if(!rawLoaded) return null; /* give up on this index */
40+
const data = { json:self.search.index, urls:self.search.doc_urls, cloud:isCloud };
41+
delete self.search.index; delete self.search.doc_urls;
42+
return data;
43+
}
44+
45+
async function loadWithFallback(remotes, local, isCloud=false){
46+
if(remotes.length){
47+
const [primary, ...secondary] = remotes;
48+
const primaryData = await loadIndex(primary, null, isCloud);
49+
if(primaryData) return primaryData;
50+
51+
if(local){
52+
const localData = await loadIndex(null, local, isCloud);
53+
if(localData) return localData;
54+
}
55+
56+
for (const remote of secondary){
57+
const data = await loadIndex(remote, null, isCloud);
58+
if(data) return data;
3659
}
37-
if(!rawLoaded) return null; /* give up on this index */
38-
const data = { json:self.search.index, urls:self.search.doc_urls, cloud:isCloud };
39-
delete self.search.index; delete self.search.doc_urls;
40-
return data;
4160
}
42-
43-
(async () => {
44-
const MAIN_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks/refs/heads/master/searchindex.js';
45-
const CLOUD_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/refs/heads/master/searchindex.js';
46-
47-
const indices = [];
48-
const main = await loadIndex(MAIN_RAW , '/searchindex-book.js', false); if(main) indices.push(main);
49-
const cloud= await loadIndex(CLOUD_RAW, '/searchindex.js', true ); if(cloud) indices.push(cloud);
50-
61+
62+
return local ? loadIndex(null, local, isCloud) : null;
63+
} (async () => {
64+
const htmlLang = (document.documentElement.lang || 'en').toLowerCase();
65+
const lang = htmlLang.split('-')[0];
66+
const mainReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks/releases/download';
67+
const cloudReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks-cloud/releases/download';
68+
69+
const mainTags = Array.from(new Set(['searchindex-' + lang, 'searchindex-en', 'searchindex-master']));
70+
const cloudTags = Array.from(new Set(['searchindex-' + lang, 'searchindex-en', 'searchindex-master']));
71+
72+
const MAIN_REMOTE_SOURCES = mainTags.map(function(tag) { return mainReleaseBase + '/' + tag + '/searchindex.js'; });
73+
const CLOUD_REMOTE_SOURCES = cloudTags.map(function(tag) { return cloudReleaseBase + '/' + tag + '/searchindex.js'; });
74+
75+
const indices = [];
76+
const main = await loadWithFallback(MAIN_REMOTE_SOURCES , '/searchindex-book.js', false); if(main) indices.push(main);
77+
const cloud= await loadWithFallback(CLOUD_REMOTE_SOURCES, '/searchindex.js', true ); if(cloud) indices.push(cloud);
5178
if(!indices.length){ postMessage({ready:false, error:'no-index'}); return; }
5279
5380
/* build index objects */
@@ -160,8 +187,16 @@
160187
/* ───────────── worker messages ───────────── */
161188
worker.onmessage = ({data}) => {
162189
if(data && data.ready!==undefined){
163-
if(data.ready){ icon.innerHTML=READY_ICON; icon.setAttribute('aria-label','Open search (S)'); }
164-
else { icon.textContent='❌'; icon.setAttribute('aria-label','Search unavailable'); }
190+
if(data.ready){
191+
icon.innerHTML=READY_ICON;
192+
icon.setAttribute('aria-label','Open search (S)');
193+
icon.removeAttribute('title');
194+
}
195+
else {
196+
icon.textContent='❌';
197+
icon.setAttribute('aria-label','Search unavailable');
198+
icon.setAttribute('title','Search is unavailable');
199+
}
165200
return;
166201
}
167202
const docs=data, q=bar.value.trim(), terms=q.split(/\s+/).filter(Boolean);

0 commit comments

Comments
 (0)