Merge pull request #1 from vyos/claude/legacy-snapshot-phase4HEADproduction

docs: legacy 1.2/1.3 static snapshot + Worker deploy pipeline

4 files changed, 196 insertions, 0 deletions

diff --git a/tools/__init__.py b/tools/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tools/__init__.py
diff --git a/tools/scrub.py b/tools/scrub.py
new file mode 100644
index 0000000..206a96c
--- /dev/null
+++ b/tools/scrub.py
@@ -0,0 +1,36 @@
+"""Scrub RTD-crawled HTML for self-hosted serving (spec §6):
+strip RTD platform JS, analytics, ad units, and the RTD flyout; localize
+absolute self/RTD URLs; inject our picker bundle."""
+from __future__ import annotations
+
+import re
+
+STRIP_SCRIPTS = re.compile(
+    r"<script[^>]*(?:readthedocs|googletagmanager|google-analytics|gtag/js|ethicalads)[^>]*>\s*</script>",
+    re.I,
+)
+STRIP_INLINE = re.compile(
+    r"<script[^>]*>(?:(?!</script>).)*?(?:READTHEDOCS_DATA|gtag\(|GoogleAnalyticsObject|ethicalads)(?:(?!</script>).)*?</script>",
+    re.I | re.S,
+)
+STRIP_DIVS = re.compile(
+    r'<div[^>]*(?:id="rtd-footer-container"|id="ethical-ad-placement"|class="[^"]*(?:rst-versions|ethical-sidebar)[^"]*")[^>]*>.*?</div>',
+    re.I | re.S,
+)
+ABS_SELF = re.compile(r'(href|src)="https?://docs\.vyos\.io(/[^"]*)"')
+ABS_RTD = re.compile(r'(href|src)="https?://[a-z0-9.-]*readthedocs\.(?:io|org)(/[^"]*)"')
+
+PICKER = ('<div id="vyos-version-picker"></div>'
+          '<link rel="stylesheet" href="/en/{slug}/_static/css/version-picker.css"/>'
+          '<script src="/en/{slug}/_static/js/version-picker.js"></script>')
+
+
+def scrub_html(html: str, slug: str) -> str:
+    out = STRIP_SCRIPTS.sub("", html)
+    out = STRIP_INLINE.sub("", out)
+    out = STRIP_DIVS.sub("", out)
+    out = ABS_SELF.sub(r'\1="\2"', out)
+    out = ABS_RTD.sub(r'\1="\2"', out)
+    if "vyos-version-picker" not in out:
+        out = out.replace("</body>", PICKER.format(slug=slug) + "</body>", 1)
+    return out
diff --git a/tools/snapshot.sh b/tools/snapshot.sh
new file mode 100755
index 0000000..687d735
--- /dev/null
+++ b/tools/snapshot.sh
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# One-time snapshot of RTD-served 1.2 + 1.3 (spec §6). Idempotent: builds into
+# snapshot.new/ and swaps it into snapshot/ only on full success (both slugs,
+# all steps) — a failed re-run must never leave the committed snapshot/
+# source-of-truth partially updated.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+DOCS_REPO="${DOCS_REPO:-$HOME/GitHub/vyos-documentation}"
+PAGEFIND_VERSION="1.5.2"   # pinned — matches vyos-documentation's docs-build.yml pagefind pin
+
+rm -rf snapshot.new crawl && mkdir -p snapshot.new   # crawl/ wiped too — stale crawl data must never mask missing RTD pages
+for slug in 1.2 1.3; do
+  echo "== crawling /en/$slug/"
+  # docs.vyos.io sits behind Cloudflare. Empirically (2026-07-10): wget's default
+  # User-Agent ("Wget/x.y") gets a Cloudflare managed-challenge 429 regardless of
+  # request rate, while a browser UA passes; an unthrottled burst also risks a
+  # separate rate-limit challenge. --user-agent spoofs a browser UA;
+  # --wait/--random-wait throttle the crawl; --retry-on-http-error=429,503 +
+  # --tries retry any transient edge errors instead of wget treating them as fatal.
+  # --no-adjust-extension (documented long-option form of -E's negation; the
+  # `--adjust-extension=off` value form is also accepted by wget but this is
+  # the form wget's own --help documents) keeps mirrored filenames matching
+  # the URLs referenced by internal hrefs.
+  wget --mirror --page-requisites --no-adjust-extension --no-parent \
+       --directory-prefix=crawl --no-host-directories \
+       --user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36" \
+       --wait=0.6 --random-wait --tries=10 --waitretry=20 \
+       --retry-on-http-error=429,503 \
+       "https://docs.vyos.io/en/$slug/"
+  mkdir -p "snapshot.new/en/$slug"
+  cp -r "crawl/en/$slug/." "snapshot.new/en/$slug/"
+
+  echo "== scrubbing"
+  find "snapshot.new/en/$slug" -name '*.html' -print0 | while IFS= read -r -d '' f; do
+    python - "$f" "$slug" <<'EOF'
+import sys
+from tools.scrub import scrub_html
+p, slug = sys.argv[1], sys.argv[2]
+src = open(p, encoding="utf-8", errors="ignore").read()
+open(p, "w", encoding="utf-8").write(scrub_html(src, slug))
+EOF
+  done
+
+  echo "== picker bundle + PDF"
+  mkdir -p "snapshot.new/en/$slug/_static/js" "snapshot.new/en/$slug/_static/css"
+  cp "$DOCS_REPO/docs/_static/js/version-picker.js"  "snapshot.new/en/$slug/_static/js/"
+  cp "$DOCS_REPO/docs/_static/css/version-picker.css" "snapshot.new/en/$slug/_static/css/"
+  # Phase-0 findings (spec §15a): 1.2 has NO PDF artifact (skip); 1.3's PDF is
+  # 29.2 MiB > the 25 MiB Workers per-asset cap → staged for the R2 fallback
+  # (uploaded to R2 + served via an apex route), NOT placed into Worker assets.
+  if [ "$slug" = "1.3" ]; then
+    mkdir -p r2-staging
+    curl -fL "https://docs.vyos.io/_/downloads/en/$slug/pdf/" -o "r2-staging/vyos-documentation-1.3.pdf"
+    header=$(head -c 5 "r2-staging/vyos-documentation-1.3.pdf")
+    if [ "$header" != "%PDF-" ]; then
+      echo "PDF content validation FAILED for $slug: file does not start with %PDF- (got: $header)"
+      exit 1
+    fi
+    echo "PDF content OK for $slug: %PDF- header present"
+  fi
+
+  echo "== parity check (hard-fail, §6 as amended by §15a — crawl-inventory link-audit; 1.2/1.3 have no sitemap.xml)"
+  python - "$slug" <<'EOF'
+import re, sys, pathlib
+# Every internal href in the mirrored tree must resolve to a mirrored file.
+# The crawl graph is the source of truth (wget --mirror follows all links);
+# a dangling internal link ⇒ the crawl missed a page ⇒ hard fail.
+slug = sys.argv[1]
+root = pathlib.Path(f"snapshot.new/en/{slug}")
+pages = list(root.rglob("*.html"))
+if len(pages) < 50:
+    print(f"PARITY FAIL: only {len(pages)} pages mirrored for {slug} — crawl incomplete")
+    sys.exit(1)
+missing = set()
+for page in pages:
+    html = page.read_text(errors="ignore")
+    for href in re.findall(r'href="([^"#?]+)"', html):
+        if href.startswith(("http://", "https://", "mailto:", "//")):
+            continue
+        target = (page.parent / href).resolve()
+        if target.suffix in {".html", ""}:
+            t = target if target.suffix else target / "index.html"
+            try:
+                t.relative_to(root.resolve())
+            except ValueError:
+                continue  # cross-version link — other slug's tree
+            if not t.is_file():
+                missing.add(str(t))
+if missing:
+    print("PARITY FAIL (dangling internal links):", *sorted(missing)[:20], sep="\n  ")
+    sys.exit(1)
+print(f"parity OK for {slug}: {len(pages)} pages, no dangling internal links")
+EOF
+
+  echo "== generate sitemap.xml from crawl inventory (§15a: legacy builds never had one; apex sitemap-index references it)"
+  python - "$slug" <<'EOF'
+import pathlib, sys
+slug = sys.argv[1]
+root = pathlib.Path(f"snapshot.new/en/{slug}")
+urls = sorted(f"https://docs.vyos.io/en/{slug}/{p.relative_to(root)}" for p in root.rglob("*.html"))
+body = "".join(f"<url><loc>{u}</loc></url>" for u in urls)
+(root / "sitemap.xml").write_text(
+    f'<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{body}</urlset>')
+print(f"sitemap.xml generated for {slug}: {len(urls)} URLs")
+EOF
+
+  echo "== per-version Pagefind index (§6 — one per version, no cross-contamination)"
+  npx --yes "pagefind@${PAGEFIND_VERSION}" --site "snapshot.new/en/$slug"
+done
+
+echo "== swap: snapshot.new/ -> snapshot/ (only on full success for both slugs)"
+rm -rf snapshot
+mv snapshot.new snapshot
+echo "snapshot complete: $(find snapshot -type f | wc -l) files"
diff --git a/tools/test_scrub.py b/tools/test_scrub.py
new file mode 100644
index 0000000..690fe86
--- /dev/null
+++ b/tools/test_scrub.py
@@ -0,0 +1,46 @@
+from tools import scrub
+
+RTD_HTML = """<html><head>
+<script async src="https://assets.readthedocs.org/static/javascript/readthedocs-addons.js"></script>
+<link rel="stylesheet" href="https://docs.vyos.io/en/1.3/_static/css/theme.css"/>
+<script>READTHEDOCS_DATA = {"project": "vyos"};</script>
+</head><body>
+<div id="rtd-footer-container"></div>
+<p>content</p>
+</body></html>"""
+
+
+def test_strips_rtd_platform_js_and_containers():
+    out = scrub.scrub_html(RTD_HTML, slug="1.3")
+    assert "readthedocs-addons" not in out
+    assert "READTHEDOCS_DATA" not in out
+    assert "rtd-footer-container" not in out
+    assert "<p>content</p>" in out
+
+
+def test_strips_analytics_ads_and_flyout(tmp_path=None):
+    html = """<html><head>
+    <script async src="https://www.googletagmanager.com/gtag/js?id=G-X"></script>
+    <script>gtag('config', 'G-X');</script>
+    <script src="https://media.ethicalads.io/media/client/ethicalads.min.js"></script>
+    </head><body>
+    <div id="ethical-ad-placement"></div>
+    <div class="rst-versions" data-toggle="rst-versions">flyout</div>
+    <p>keep</p></body></html>"""
+    out = scrub.scrub_html(html, slug="1.2")
+    assert "googletagmanager" not in out
+    assert "gtag(" not in out
+    assert "ethicalads" not in out and "ethical-ad-placement" not in out
+    assert "rst-versions" not in out
+    assert "<p>keep</p>" in out
+
+
+def test_rewrites_absolute_self_urls_to_relative():
+    out = scrub.scrub_html(RTD_HTML, slug="1.3")
+    assert 'href="/en/1.3/_static/css/theme.css"' in out
+
+
+def test_injects_picker_bundle_once():
+    out = scrub.scrub_html(RTD_HTML, slug="1.3")
+    assert out.count('js/version-picker.js') == 1
+    assert 'id="vyos-version-picker"' in out