From 21689ef59b0eb34b1a29eda739dc10d33d25b44a Mon Sep 17 00:00:00 2001 From: Yuriy Andamasov Date: Fri, 10 Jul 2026 17:14:13 +0300 Subject: docs: Cloudflare Workers hosting pipeline (apex, content workers, CI, previews) (#2140) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs-infra: scaffold Cloudflare workers workspace (versions.json v2, matrix, toolchain) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: record full Phase-0 plan decision in workers/PLAN.md ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: shared content worker โ€” asset serving, cache classes, X-Docs-Build, canary no-store ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: run worker script before assets; test fetch entrypoint ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: apex manifest loader + dispatch map + runtime binding guard (TDD) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: apex redirects (aliases, PDF, trailing-slash) + special paths (TDD) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: PDF redirect honors pdf:null and preserves query ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: apex UA gate โ€” allowlist-wins, log-only AI crawlers, empty block list at launch ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: apex router (pipeline ยง3.2), themed 404/503, /kb seam, env configs + congruence test ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: add missing-User-Agent regression test for apex UA gate ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: R2-streaming preview worker โ€” MIME map, noindex, no-store (TDD) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: preview 404 no-store + fetch handler tests ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: bootstrap script โ€” binding-target workers must exist before apex deploys ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: apex run_worker_first, lockfile for npm ci, PDF Location from manifest ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: derive html_baseurl from DOCS_VERSION_SLUG with RTD fallback (canonical gate prereq) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: version picker + status banner + language scaffold (vanilla JS, TDD pure core) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: picker preserves query+hash across switch; valid breadcrumb markup ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: Pagefind search wrapper with runtime base-path + preview prefix handling (TDD) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: pagefind wrapper โ€” asset-failure notice + UI stylesheet load ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: gate Pagefind searchbox to CF builds (RTD keeps stock search until cutover) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: deploy sanity gates โ€” limits, critical pages, count-delta, canonical (TDD) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: hermetic gate tests via fixture versions.json ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: docs-build workflow โ€” candidate/smoke/promote two-stage deploy + registry + rollback Two-stage CF Workers pipeline: build in pinned container, assemble artifact, sanity gates, deploy candidate, scoped pre-traffic smoke via canary apex, promote (rollback-id capture, hostname purge, registry upload), post-promote probe + auto-rollback. DOCS_CF_LIVE repo variable gates every docs.vyos.io production interaction pre-cutover. scripts/docs_gates/smoke.py adds one authorized check beyond the spec: the version's index.html probe asserts the #vyos-search mount div is present in the response body, guarding CI silently forgetting DOCS_VERSION_SLUG (which would otherwise ship stock RTD search without the Pagefind gate noticing). ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: build docs image in-workflow with buildx cache (v4.1 โ€” digest pin dropped) Plan v4.1 amendment: the ghcr.io digest-pinned image does not exist (workflow would hard-fail at the first docker step on every push). Replace the BUILD_IMAGE env placeholder with an in-workflow docker build from docker/Dockerfile via docker/setup-buildx-action@v3 + docker/build-push-action@v6 (context: docker/, load: true, tags: docs-build:local, GHA cache from/to). The checked-out commit is the pin; buildx GHA cache keeps repeat builds cheap. Sphinx-build step swaps to docs-build:local; inner script unchanged. ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: apex/preview deploy workflow โ€” canary auto, production behind environment approval * docs-infra: apex-deploy concurrency guard (per-ref, cancel-in-progress) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: fork-safe PR preview pipeline โ€” approval record, R2 prefixes, label consumption, cleanup * docs-infra: nightly preview sweep โ€” pipefail + per-prefix failure isolation ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: nightly canary QA โ€” per-entry sweep + URL-parity corpus vs RTD ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: parity sweep scoped to CF-built versions; transport-error resilience ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: one-off bootstrap workflow (binding targets โ€” runs once on this push) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: remove one-off bootstrap workflow (bootstrap complete) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: one-off canary apex + preview deploy (route targets for Task 3.6 step 2c) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: remove one-off canary deploy workflow (targets live) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: address Phase-0 CodeRabbit findings (canonical gate, error caching, registry pointer, validation) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: strengthen manifest tests (full dispatch iteration, mutation-free validate) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: address GitHub CodeRabbit review (pointer-after-probe, fail-closed sweeps, block-precedence UA gate, preview hardening) ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: adversarial review fixes โ€” error no-store, probe retry, PR-list membership, preview dotted-segment ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: serve oversized legacy PDF from R2 via apex (spec ยง5 fallback) The 1.3 PDF (29.2 MiB) exceeds the 25 MiB static-asset cap and is absent from the legacy content Worker's build, so /_/downloads/en/1.3/pdf/ (and the picker's PDF link) 301'd into a dead-end 404 post-cutover. Add the R2 object fallback spec ยง5 already documented but never implemented: a DOCS_PDFS R2 bucket binding on the apex Worker, a manifest pdf_r2_key field (1.3 only), and a router step ahead of version dispatch that streams the object with its own cache class (canary/error still force no-store). ๐Ÿค– Generated by [robots](https://vyos.io) * docs-infra: PDF R2 fallback honors Range + If-None-Match, preserves ETag ๐Ÿค– Generated by [robots](https://vyos.io) --- scripts/docs_gates/parity.py | 121 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 scripts/docs_gates/parity.py (limited to 'scripts/docs_gates/parity.py') diff --git a/scripts/docs_gates/parity.py b/scripts/docs_gates/parity.py new file mode 100644 index 00000000..08225f5a --- /dev/null +++ b/scripts/docs_gates/parity.py @@ -0,0 +1,121 @@ +"""URL-parity corpus + alias assertions (spec ยง11). + +Modes: + --sitemap-host X pull per-version sitemaps from X (RTD pre-cutover) + --probe-host Y probe every URL on Y (canary via Access, or production) +Fails (exit 1) on any status mismatch (non-200 for corpus rows; wrong +Location for alias rows). +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +import urllib.request +from pathlib import Path + +SITEMAP_LOC = re.compile(r"([^<]+)") + +# Sitemap sweep covers only the versions THIS repo builds on CF (rolling/1.5/1.4). +# 1.3/1.2 have NO RTD sitemaps (spec ยง15a.5) โ€” their parity is the legacy snapshot +# repo's crawl-inventory job. Their alias/PDF redirect rows below stay in scope. +DEFAULT_SLUGS = "rolling,1.5,1.4" + +ALIASES = [("latest", "rolling"), ("stable", "1.5"), ("lts", "1.5"), + ("circinus", "1.5"), ("sagitta", "1.4"), ("equuleus", "1.3"), ("crux", "1.2")] + + +def urls_from_sitemap(xml: str) -> list[str]: + out = [] + for loc in SITEMAP_LOC.findall(xml): + path = re.sub(r"^https?://[^/]+", "", loc) + if path.startswith("/en/"): + out.append(path) + return out + + +def alias_corpus() -> list[tuple[str, int, str]]: + rows = [(f"/en/{a}/", 301, f"/en/{s}/") for a, s in ALIASES] + # 1.2 excluded: never had an RTD PDF artifact (Phase-0 finding, spec ยง15a) โ€” pdf: null + rows += [(f"/_/downloads/en/{s}/pdf/", 301, f"/en/{s}/vyos-documentation.pdf") + for s in ["rolling", "1.5", "1.4", "1.3"]] + rows.append(("/_/downloads/en/latest/pdf/", 301, "/en/rolling/vyos-documentation.pdf")) + return rows + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """The parity checker must SEE 301s, not follow them (alias assertions).""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: D401 + return None + + +_OPENER = urllib.request.build_opener(_NoRedirect) + +# Overridable in tests (monkeypatched to "http") so fetch() can be exercised end-to-end +# against a real local http.server instead of requiring TLS for a unit test. +_SCHEME = "https" + + +def fetch(host: str, path: str, access: tuple[str, str] | None, method: str = "HEAD"): + req = urllib.request.Request(f"{_SCHEME}://{host}{path}", method=method) + if access: + req.add_header("CF-Access-Client-Id", access[0]) + req.add_header("CF-Access-Client-Secret", access[1]) + try: + with _OPENER.open(req, timeout=30) as r: + return r.status, r.headers.get("Location") + except urllib.error.HTTPError as e: # 3xx land here with the no-redirect handler + return e.code, e.headers.get("Location") + except Exception: # noqa: BLE001 โ€” DNS blip/timeout fails THIS probe, not the run + return 0, None + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--sitemap-host", required=True) + ap.add_argument("--probe-host", required=True) + ap.add_argument("--slugs", default=DEFAULT_SLUGS) + ap.add_argument("--access-id") + ap.add_argument("--access-secret") + ap.add_argument("--report", type=Path, default=Path("parity-report.json")) + a = ap.parse_args() + access = (a.access_id, a.access_secret) if a.access_id else None + failures: list[dict] = [] + checked = 0 + + for slug in a.slugs.split(","): + status, _ = fetch(a.sitemap_host, f"/en/{slug}/sitemap.xml", None, "GET") + if status != 200: + failures.append({"path": f"/en/{slug}/sitemap.xml", "reason": f"sitemap {status}"}) + continue + try: + with urllib.request.urlopen(f"https://{a.sitemap_host}/en/{slug}/sitemap.xml", + timeout=30) as r: + urls = urls_from_sitemap(r.read().decode()) + except Exception as e: # noqa: BLE001 โ€” record per-slug, keep sweeping; report ALWAYS written + failures.append({"path": f"/en/{slug}/sitemap.xml", + "reason": f"sitemap fetch error: {e}"}) + continue + for path in urls: + checked += 1 + st, _ = fetch(a.probe_host, path, access) + if st != 200: + failures.append({"path": path, "reason": f"status {st}"}) + + for path, want_status, want_loc in alias_corpus(): + checked += 1 + st, loc = fetch(a.probe_host, path, access) + if st != want_status or (loc or "") != want_loc: + failures.append({"path": path, "reason": f"got {st} โ†’ {loc}, want {want_status} โ†’ {want_loc}"}) + + a.report.write_text(json.dumps({"checked": checked, "failures": failures}, indent=2)) + for f in failures: + print(f"PARITY-FAIL {f['path']}: {f['reason']}", file=sys.stderr) + print(f"checked={checked} failures={len(failures)}") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) -- cgit v1.2.3