Merge pull request #1 from vyos/claude/legacy-snapshot-phase4HEADproduction

docs: legacy 1.2/1.3 static snapshot + Worker deploy pipeline

1 files changed, 114 insertions, 0 deletions

diff --git a/tools/snapshot.sh b/tools/snapshot.sh
new file mode 100755
index 0000000..687d735
--- /dev/null
+++ b/tools/snapshot.sh
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# One-time snapshot of RTD-served 1.2 + 1.3 (spec §6). Idempotent: builds into
+# snapshot.new/ and swaps it into snapshot/ only on full success (both slugs,
+# all steps) — a failed re-run must never leave the committed snapshot/
+# source-of-truth partially updated.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+DOCS_REPO="${DOCS_REPO:-$HOME/GitHub/vyos-documentation}"
+PAGEFIND_VERSION="1.5.2"   # pinned — matches vyos-documentation's docs-build.yml pagefind pin
+
+rm -rf snapshot.new crawl && mkdir -p snapshot.new   # crawl/ wiped too — stale crawl data must never mask missing RTD pages
+for slug in 1.2 1.3; do
+  echo "== crawling /en/$slug/"
+  # docs.vyos.io sits behind Cloudflare. Empirically (2026-07-10): wget's default
+  # User-Agent ("Wget/x.y") gets a Cloudflare managed-challenge 429 regardless of
+  # request rate, while a browser UA passes; an unthrottled burst also risks a
+  # separate rate-limit challenge. --user-agent spoofs a browser UA;
+  # --wait/--random-wait throttle the crawl; --retry-on-http-error=429,503 +
+  # --tries retry any transient edge errors instead of wget treating them as fatal.
+  # --no-adjust-extension (documented long-option form of -E's negation; the
+  # `--adjust-extension=off` value form is also accepted by wget but this is
+  # the form wget's own --help documents) keeps mirrored filenames matching
+  # the URLs referenced by internal hrefs.
+  wget --mirror --page-requisites --no-adjust-extension --no-parent \
+       --directory-prefix=crawl --no-host-directories \
+       --user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36" \
+       --wait=0.6 --random-wait --tries=10 --waitretry=20 \
+       --retry-on-http-error=429,503 \
+       "https://docs.vyos.io/en/$slug/"
+  mkdir -p "snapshot.new/en/$slug"
+  cp -r "crawl/en/$slug/." "snapshot.new/en/$slug/"
+
+  echo "== scrubbing"
+  find "snapshot.new/en/$slug" -name '*.html' -print0 | while IFS= read -r -d '' f; do
+    python - "$f" "$slug" <<'EOF'
+import sys
+from tools.scrub import scrub_html
+p, slug = sys.argv[1], sys.argv[2]
+src = open(p, encoding="utf-8", errors="ignore").read()
+open(p, "w", encoding="utf-8").write(scrub_html(src, slug))
+EOF
+  done
+
+  echo "== picker bundle + PDF"
+  mkdir -p "snapshot.new/en/$slug/_static/js" "snapshot.new/en/$slug/_static/css"
+  cp "$DOCS_REPO/docs/_static/js/version-picker.js"  "snapshot.new/en/$slug/_static/js/"
+  cp "$DOCS_REPO/docs/_static/css/version-picker.css" "snapshot.new/en/$slug/_static/css/"
+  # Phase-0 findings (spec §15a): 1.2 has NO PDF artifact (skip); 1.3's PDF is
+  # 29.2 MiB > the 25 MiB Workers per-asset cap → staged for the R2 fallback
+  # (uploaded to R2 + served via an apex route), NOT placed into Worker assets.
+  if [ "$slug" = "1.3" ]; then
+    mkdir -p r2-staging
+    curl -fL "https://docs.vyos.io/_/downloads/en/$slug/pdf/" -o "r2-staging/vyos-documentation-1.3.pdf"
+    header=$(head -c 5 "r2-staging/vyos-documentation-1.3.pdf")
+    if [ "$header" != "%PDF-" ]; then
+      echo "PDF content validation FAILED for $slug: file does not start with %PDF- (got: $header)"
+      exit 1
+    fi
+    echo "PDF content OK for $slug: %PDF- header present"
+  fi
+
+  echo "== parity check (hard-fail, §6 as amended by §15a — crawl-inventory link-audit; 1.2/1.3 have no sitemap.xml)"
+  python - "$slug" <<'EOF'
+import re, sys, pathlib
+# Every internal href in the mirrored tree must resolve to a mirrored file.
+# The crawl graph is the source of truth (wget --mirror follows all links);
+# a dangling internal link ⇒ the crawl missed a page ⇒ hard fail.
+slug = sys.argv[1]
+root = pathlib.Path(f"snapshot.new/en/{slug}")
+pages = list(root.rglob("*.html"))
+if len(pages) < 50:
+    print(f"PARITY FAIL: only {len(pages)} pages mirrored for {slug} — crawl incomplete")
+    sys.exit(1)
+missing = set()
+for page in pages:
+    html = page.read_text(errors="ignore")
+    for href in re.findall(r'href="([^"#?]+)"', html):
+        if href.startswith(("http://", "https://", "mailto:", "//")):
+            continue
+        target = (page.parent / href).resolve()
+        if target.suffix in {".html", ""}:
+            t = target if target.suffix else target / "index.html"
+            try:
+                t.relative_to(root.resolve())
+            except ValueError:
+                continue  # cross-version link — other slug's tree
+            if not t.is_file():
+                missing.add(str(t))
+if missing:
+    print("PARITY FAIL (dangling internal links):", *sorted(missing)[:20], sep="\n  ")
+    sys.exit(1)
+print(f"parity OK for {slug}: {len(pages)} pages, no dangling internal links")
+EOF
+
+  echo "== generate sitemap.xml from crawl inventory (§15a: legacy builds never had one; apex sitemap-index references it)"
+  python - "$slug" <<'EOF'
+import pathlib, sys
+slug = sys.argv[1]
+root = pathlib.Path(f"snapshot.new/en/{slug}")
+urls = sorted(f"https://docs.vyos.io/en/{slug}/{p.relative_to(root)}" for p in root.rglob("*.html"))
+body = "".join(f"<url><loc>{u}</loc></url>" for u in urls)
+(root / "sitemap.xml").write_text(
+    f'<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{body}</urlset>')
+print(f"sitemap.xml generated for {slug}: {len(urls)} URLs")
+EOF
+
+  echo "== per-version Pagefind index (§6 — one per version, no cross-contamination)"
+  npx --yes "pagefind@${PAGEFIND_VERSION}" --site "snapshot.new/en/$slug"
+done
+
+echo "== swap: snapshot.new/ -> snapshot/ (only on full success for both slugs)"
+rm -rf snapshot
+mv snapshot.new snapshot
+echo "snapshot complete: $(find snapshot -type f | wc -l) files"