1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
#!/usr/bin/env bash
# One-time snapshot of RTD-served 1.2 + 1.3 (spec §6). Idempotent: builds into
# snapshot.new/ and swaps it into snapshot/ only on full success (both slugs,
# all steps) — a failed re-run must never leave the committed snapshot/
# source-of-truth partially updated.
set -euo pipefail
cd "$(dirname "$0")/.."
DOCS_REPO="${DOCS_REPO:-$HOME/GitHub/vyos-documentation}"
PAGEFIND_VERSION="1.5.2" # pinned — matches vyos-documentation's docs-build.yml pagefind pin
rm -rf snapshot.new crawl && mkdir -p snapshot.new # crawl/ wiped too — stale crawl data must never mask missing RTD pages
for slug in 1.2 1.3; do
echo "== crawling /en/$slug/"
# docs.vyos.io sits behind Cloudflare. Empirically (2026-07-10): wget's default
# User-Agent ("Wget/x.y") gets a Cloudflare managed-challenge 429 regardless of
# request rate, while a browser UA passes; an unthrottled burst also risks a
# separate rate-limit challenge. --user-agent spoofs a browser UA;
# --wait/--random-wait throttle the crawl; --retry-on-http-error=429,503 +
# --tries retry any transient edge errors instead of wget treating them as fatal.
# --no-adjust-extension (documented long-option form of -E's negation; the
# `--adjust-extension=off` value form is also accepted by wget but this is
# the form wget's own --help documents) keeps mirrored filenames matching
# the URLs referenced by internal hrefs.
wget --mirror --page-requisites --no-adjust-extension --no-parent \
--directory-prefix=crawl --no-host-directories \
--user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36" \
--wait=0.6 --random-wait --tries=10 --waitretry=20 \
--retry-on-http-error=429,503 \
"https://docs.vyos.io/en/$slug/"
mkdir -p "snapshot.new/en/$slug"
cp -r "crawl/en/$slug/." "snapshot.new/en/$slug/"
echo "== scrubbing"
find "snapshot.new/en/$slug" -name '*.html' -print0 | while IFS= read -r -d '' f; do
python - "$f" "$slug" <<'EOF'
import sys
from tools.scrub import scrub_html
p, slug = sys.argv[1], sys.argv[2]
src = open(p, encoding="utf-8", errors="ignore").read()
open(p, "w", encoding="utf-8").write(scrub_html(src, slug))
EOF
done
echo "== picker bundle + PDF"
mkdir -p "snapshot.new/en/$slug/_static/js" "snapshot.new/en/$slug/_static/css"
cp "$DOCS_REPO/docs/_static/js/version-picker.js" "snapshot.new/en/$slug/_static/js/"
cp "$DOCS_REPO/docs/_static/css/version-picker.css" "snapshot.new/en/$slug/_static/css/"
# Phase-0 findings (spec §15a): 1.2 has NO PDF artifact (skip); 1.3's PDF is
# 29.2 MiB > the 25 MiB Workers per-asset cap → staged for the R2 fallback
# (uploaded to R2 + served via an apex route), NOT placed into Worker assets.
if [ "$slug" = "1.3" ]; then
mkdir -p r2-staging
curl -fL "https://docs.vyos.io/_/downloads/en/$slug/pdf/" -o "r2-staging/vyos-documentation-1.3.pdf"
header=$(head -c 5 "r2-staging/vyos-documentation-1.3.pdf")
if [ "$header" != "%PDF-" ]; then
echo "PDF content validation FAILED for $slug: file does not start with %PDF- (got: $header)"
exit 1
fi
echo "PDF content OK for $slug: %PDF- header present"
fi
echo "== parity check (hard-fail, §6 as amended by §15a — crawl-inventory link-audit; 1.2/1.3 have no sitemap.xml)"
python - "$slug" <<'EOF'
import re, sys, pathlib
# Every internal href in the mirrored tree must resolve to a mirrored file.
# The crawl graph is the source of truth (wget --mirror follows all links);
# a dangling internal link ⇒ the crawl missed a page ⇒ hard fail.
slug = sys.argv[1]
root = pathlib.Path(f"snapshot.new/en/{slug}")
pages = list(root.rglob("*.html"))
if len(pages) < 50:
print(f"PARITY FAIL: only {len(pages)} pages mirrored for {slug} — crawl incomplete")
sys.exit(1)
missing = set()
for page in pages:
html = page.read_text(errors="ignore")
for href in re.findall(r'href="([^"#?]+)"', html):
if href.startswith(("http://", "https://", "mailto:", "//")):
continue
target = (page.parent / href).resolve()
if target.suffix in {".html", ""}:
t = target if target.suffix else target / "index.html"
try:
t.relative_to(root.resolve())
except ValueError:
continue # cross-version link — other slug's tree
if not t.is_file():
missing.add(str(t))
if missing:
print("PARITY FAIL (dangling internal links):", *sorted(missing)[:20], sep="\n ")
sys.exit(1)
print(f"parity OK for {slug}: {len(pages)} pages, no dangling internal links")
EOF
echo "== generate sitemap.xml from crawl inventory (§15a: legacy builds never had one; apex sitemap-index references it)"
python - "$slug" <<'EOF'
import pathlib, sys
slug = sys.argv[1]
root = pathlib.Path(f"snapshot.new/en/{slug}")
urls = sorted(f"https://docs.vyos.io/en/{slug}/{p.relative_to(root)}" for p in root.rglob("*.html"))
body = "".join(f"<url><loc>{u}</loc></url>" for u in urls)
(root / "sitemap.xml").write_text(
f'<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{body}</urlset>')
print(f"sitemap.xml generated for {slug}: {len(urls)} URLs")
EOF
echo "== per-version Pagefind index (§6 — one per version, no cross-contamination)"
npx --yes "pagefind@${PAGEFIND_VERSION}" --site "snapshot.new/en/$slug"
done
echo "== swap: snapshot.new/ -> snapshot/ (only on full success for both slugs)"
rm -rf snapshot
mv snapshot.new snapshot
echo "snapshot complete: $(find snapshot -type f | wc -l) files"
|
