diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/__init__.py | 0 | ||||
| -rw-r--r-- | tools/scrub.py | 36 | ||||
| -rwxr-xr-x | tools/snapshot.sh | 114 | ||||
| -rw-r--r-- | tools/test_scrub.py | 46 |
4 files changed, 196 insertions, 0 deletions
diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/__init__.py diff --git a/tools/scrub.py b/tools/scrub.py new file mode 100644 index 0000000..206a96c --- /dev/null +++ b/tools/scrub.py @@ -0,0 +1,36 @@ +"""Scrub RTD-crawled HTML for self-hosted serving (spec §6): +strip RTD platform JS, analytics, ad units, and the RTD flyout; localize +absolute self/RTD URLs; inject our picker bundle.""" +from __future__ import annotations + +import re + +STRIP_SCRIPTS = re.compile( + r"<script[^>]*(?:readthedocs|googletagmanager|google-analytics|gtag/js|ethicalads)[^>]*>\s*</script>", + re.I, +) +STRIP_INLINE = re.compile( + r"<script[^>]*>(?:(?!</script>).)*?(?:READTHEDOCS_DATA|gtag\(|GoogleAnalyticsObject|ethicalads)(?:(?!</script>).)*?</script>", + re.I | re.S, +) +STRIP_DIVS = re.compile( + r'<div[^>]*(?:id="rtd-footer-container"|id="ethical-ad-placement"|class="[^"]*(?:rst-versions|ethical-sidebar)[^"]*")[^>]*>.*?</div>', + re.I | re.S, +) +ABS_SELF = re.compile(r'(href|src)="https?://docs\.vyos\.io(/[^"]*)"') +ABS_RTD = re.compile(r'(href|src)="https?://[a-z0-9.-]*readthedocs\.(?:io|org)(/[^"]*)"') + +PICKER = ('<div id="vyos-version-picker"></div>' + '<link rel="stylesheet" href="/en/{slug}/_static/css/version-picker.css"/>' + '<script src="/en/{slug}/_static/js/version-picker.js"></script>') + + +def scrub_html(html: str, slug: str) -> str: + out = STRIP_SCRIPTS.sub("", html) + out = STRIP_INLINE.sub("", out) + out = STRIP_DIVS.sub("", out) + out = ABS_SELF.sub(r'\1="\2"', out) + out = ABS_RTD.sub(r'\1="\2"', out) + if "vyos-version-picker" not in out: + out = out.replace("</body>", PICKER.format(slug=slug) + "</body>", 1) + return out diff --git a/tools/snapshot.sh b/tools/snapshot.sh new file mode 100755 index 0000000..687d735 --- /dev/null +++ b/tools/snapshot.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# One-time snapshot of RTD-served 1.2 + 1.3 (spec §6). Idempotent: builds into +# snapshot.new/ and swaps it into snapshot/ only on full success (both slugs, +# all steps) — a failed re-run must never leave the committed snapshot/ +# source-of-truth partially updated. +set -euo pipefail +cd "$(dirname "$0")/.." +DOCS_REPO="${DOCS_REPO:-$HOME/GitHub/vyos-documentation}" +PAGEFIND_VERSION="1.5.2" # pinned — matches vyos-documentation's docs-build.yml pagefind pin + +rm -rf snapshot.new crawl && mkdir -p snapshot.new # crawl/ wiped too — stale crawl data must never mask missing RTD pages +for slug in 1.2 1.3; do + echo "== crawling /en/$slug/" + # docs.vyos.io sits behind Cloudflare. Empirically (2026-07-10): wget's default + # User-Agent ("Wget/x.y") gets a Cloudflare managed-challenge 429 regardless of + # request rate, while a browser UA passes; an unthrottled burst also risks a + # separate rate-limit challenge. --user-agent spoofs a browser UA; + # --wait/--random-wait throttle the crawl; --retry-on-http-error=429,503 + + # --tries retry any transient edge errors instead of wget treating them as fatal. + # --no-adjust-extension (documented long-option form of -E's negation; the + # `--adjust-extension=off` value form is also accepted by wget but this is + # the form wget's own --help documents) keeps mirrored filenames matching + # the URLs referenced by internal hrefs. + wget --mirror --page-requisites --no-adjust-extension --no-parent \ + --directory-prefix=crawl --no-host-directories \ + --user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36" \ + --wait=0.6 --random-wait --tries=10 --waitretry=20 \ + --retry-on-http-error=429,503 \ + "https://docs.vyos.io/en/$slug/" + mkdir -p "snapshot.new/en/$slug" + cp -r "crawl/en/$slug/." "snapshot.new/en/$slug/" + + echo "== scrubbing" + find "snapshot.new/en/$slug" -name '*.html' -print0 | while IFS= read -r -d '' f; do + python - "$f" "$slug" <<'EOF' +import sys +from tools.scrub import scrub_html +p, slug = sys.argv[1], sys.argv[2] +src = open(p, encoding="utf-8", errors="ignore").read() +open(p, "w", encoding="utf-8").write(scrub_html(src, slug)) +EOF + done + + echo "== picker bundle + PDF" + mkdir -p "snapshot.new/en/$slug/_static/js" "snapshot.new/en/$slug/_static/css" + cp "$DOCS_REPO/docs/_static/js/version-picker.js" "snapshot.new/en/$slug/_static/js/" + cp "$DOCS_REPO/docs/_static/css/version-picker.css" "snapshot.new/en/$slug/_static/css/" + # Phase-0 findings (spec §15a): 1.2 has NO PDF artifact (skip); 1.3's PDF is + # 29.2 MiB > the 25 MiB Workers per-asset cap → staged for the R2 fallback + # (uploaded to R2 + served via an apex route), NOT placed into Worker assets. + if [ "$slug" = "1.3" ]; then + mkdir -p r2-staging + curl -fL "https://docs.vyos.io/_/downloads/en/$slug/pdf/" -o "r2-staging/vyos-documentation-1.3.pdf" + header=$(head -c 5 "r2-staging/vyos-documentation-1.3.pdf") + if [ "$header" != "%PDF-" ]; then + echo "PDF content validation FAILED for $slug: file does not start with %PDF- (got: $header)" + exit 1 + fi + echo "PDF content OK for $slug: %PDF- header present" + fi + + echo "== parity check (hard-fail, §6 as amended by §15a — crawl-inventory link-audit; 1.2/1.3 have no sitemap.xml)" + python - "$slug" <<'EOF' +import re, sys, pathlib +# Every internal href in the mirrored tree must resolve to a mirrored file. +# The crawl graph is the source of truth (wget --mirror follows all links); +# a dangling internal link ⇒ the crawl missed a page ⇒ hard fail. +slug = sys.argv[1] +root = pathlib.Path(f"snapshot.new/en/{slug}") +pages = list(root.rglob("*.html")) +if len(pages) < 50: + print(f"PARITY FAIL: only {len(pages)} pages mirrored for {slug} — crawl incomplete") + sys.exit(1) +missing = set() +for page in pages: + html = page.read_text(errors="ignore") + for href in re.findall(r'href="([^"#?]+)"', html): + if href.startswith(("http://", "https://", "mailto:", "//")): + continue + target = (page.parent / href).resolve() + if target.suffix in {".html", ""}: + t = target if target.suffix else target / "index.html" + try: + t.relative_to(root.resolve()) + except ValueError: + continue # cross-version link — other slug's tree + if not t.is_file(): + missing.add(str(t)) +if missing: + print("PARITY FAIL (dangling internal links):", *sorted(missing)[:20], sep="\n ") + sys.exit(1) +print(f"parity OK for {slug}: {len(pages)} pages, no dangling internal links") +EOF + + echo "== generate sitemap.xml from crawl inventory (§15a: legacy builds never had one; apex sitemap-index references it)" + python - "$slug" <<'EOF' +import pathlib, sys +slug = sys.argv[1] +root = pathlib.Path(f"snapshot.new/en/{slug}") +urls = sorted(f"https://docs.vyos.io/en/{slug}/{p.relative_to(root)}" for p in root.rglob("*.html")) +body = "".join(f"<url><loc>{u}</loc></url>" for u in urls) +(root / "sitemap.xml").write_text( + f'<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{body}</urlset>') +print(f"sitemap.xml generated for {slug}: {len(urls)} URLs") +EOF + + echo "== per-version Pagefind index (§6 — one per version, no cross-contamination)" + npx --yes "pagefind@${PAGEFIND_VERSION}" --site "snapshot.new/en/$slug" +done + +echo "== swap: snapshot.new/ -> snapshot/ (only on full success for both slugs)" +rm -rf snapshot +mv snapshot.new snapshot +echo "snapshot complete: $(find snapshot -type f | wc -l) files" diff --git a/tools/test_scrub.py b/tools/test_scrub.py new file mode 100644 index 0000000..690fe86 --- /dev/null +++ b/tools/test_scrub.py @@ -0,0 +1,46 @@ +from tools import scrub + +RTD_HTML = """<html><head> +<script async src="https://assets.readthedocs.org/static/javascript/readthedocs-addons.js"></script> +<link rel="stylesheet" href="https://docs.vyos.io/en/1.3/_static/css/theme.css"/> +<script>READTHEDOCS_DATA = {"project": "vyos"};</script> +</head><body> +<div id="rtd-footer-container"></div> +<p>content</p> +</body></html>""" + + +def test_strips_rtd_platform_js_and_containers(): + out = scrub.scrub_html(RTD_HTML, slug="1.3") + assert "readthedocs-addons" not in out + assert "READTHEDOCS_DATA" not in out + assert "rtd-footer-container" not in out + assert "<p>content</p>" in out + + +def test_strips_analytics_ads_and_flyout(tmp_path=None): + html = """<html><head> + <script async src="https://www.googletagmanager.com/gtag/js?id=G-X"></script> + <script>gtag('config', 'G-X');</script> + <script src="https://media.ethicalads.io/media/client/ethicalads.min.js"></script> + </head><body> + <div id="ethical-ad-placement"></div> + <div class="rst-versions" data-toggle="rst-versions">flyout</div> + <p>keep</p></body></html>""" + out = scrub.scrub_html(html, slug="1.2") + assert "googletagmanager" not in out + assert "gtag(" not in out + assert "ethicalads" not in out and "ethical-ad-placement" not in out + assert "rst-versions" not in out + assert "<p>keep</p>" in out + + +def test_rewrites_absolute_self_urls_to_relative(): + out = scrub.scrub_html(RTD_HTML, slug="1.3") + assert 'href="/en/1.3/_static/css/theme.css"' in out + + +def test_injects_picker_bundle_once(): + out = scrub.scrub_html(RTD_HTML, slug="1.3") + assert out.count('js/version-picker.js') == 1 + assert 'id="vyos-version-picker"' in out |