docs-infra: CR review fixes — wget flag form, PDF validation, R2 seed short-circuit, build-then-swapclaude/legacy-snapshot-phase4

🤖 Generated by [robots](https://vyos.io)

4 files changed, 71 insertions, 22 deletions

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 9e2c25e..fd1587e 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -6,6 +6,10 @@ on:
         description: "false = candidate only; true = promote to production worker"
         type: boolean
         default: false
+      force_pdf_refresh:
+        description: "true = re-fetch + re-upload the 1.3 PDF even if it already exists in R2"
+        type: boolean
+        default: false
 permissions:
   contents: read
 # Never cancel a mid-flight deploy — queue behind it instead (fleet parity).
@@ -17,8 +21,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with: { persist-credentials: false }
       - uses: actions/checkout@v4
-        with: { repository: vyos/vyos-documentation, ref: rolling, path: docsrepo }
+        with: { repository: vyos/vyos-documentation, ref: rolling, path: docsrepo, persist-credentials: false }
       - uses: actions/setup-node@v4
         with: { node-version: 22 }
       # Runs BEFORE the Worker deploy (fail fast): if the PDF can't be fetched, validated,
@@ -31,6 +36,20 @@ jobs:
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
         run: |
           set -eu
+          cd docsrepo/workers && npm ci   # the Deploy step's own npm ci re-run is a fast no-op
+
+          R2_KEY="vyos-docs-artifacts/legacy/1.3/vyos-documentation.pdf"
+          # 1.2/1.3 are frozen (spec §15a) — once the PDF is seeded in R2 it never
+          # changes, and after RTD sunsets, docs.vyos.io/_/downloads/... is gone, so
+          # re-dispatching this workflow must not depend on re-fetching it. Probe
+          # existence first (wrangler r2 has no head-only verb; `get --pipe` is the
+          # cheapest existence check the CLI exposes) and skip fetch+upload on a hit.
+          if [ '${{ inputs.force_pdf_refresh }}' != 'true' ] && \
+             npx wrangler r2 object get "$R2_KEY" --remote --pipe >/dev/null 2>/tmp/r2-probe.log; then
+            echo "PDF already present at $R2_KEY — skipping fetch+upload (force_pdf_refresh=false)"
+            exit 0
+          fi
+
           # r2-staging/ (tools/snapshot.sh's local output) is gitignored, so CI re-fetches
           # from the pre-sunset RTD source directly. Browser-UA + retry added defensively:
           # tools/snapshot.sh's wget mirror crawl needed a browser UA to pass Cloudflare's
@@ -52,8 +71,14 @@ jobs:
           fi
           echo "PDF size OK: $size bytes"
 
-          cd docsrepo/workers && npm ci   # the Deploy step's own npm ci re-run is a fast no-op
-          npx wrangler r2 object put "vyos-docs-artifacts/legacy/1.3/vyos-documentation.pdf" \
+          header=$(head -c 5 /tmp/vyos-documentation-1.3.pdf)
+          if [ "$header" != "%PDF-" ]; then
+            echo "PDF content validation FAILED: file does not start with %PDF- (got: $header)"
+            exit 1
+          fi
+          echo "PDF content OK: %PDF- header present"
+
+          npx wrangler r2 object put "$R2_KEY" \
             --file /tmp/vyos-documentation-1.3.pdf --remote
       - name: Deploy
         env:
diff --git a/.gitignore b/.gitignore
index aadfe27..fc92d0e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 crawl/
+snapshot.new/
 r2-staging/
 .venv/
 __pycache__/
diff --git a/README.md b/README.md
index 4058ab5..d27f984 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ point this repo's `tools/` become historical record only.
 
 ## Layout
 
-```
+```text
 tools/
   snapshot.sh       # crawl -> scrub -> picker/PDF -> parity -> sitemap -> pagefind
   scrub.py          # HTML scrubbing (RTD chrome removal + URL localization + picker injection)
@@ -47,6 +47,8 @@ r2-staging/         # 1.3's PDF (29.2 MiB, over the 25 MiB Workers per-asset cap
                      # either (large binary; staged locally for the R2 upload step).
 crawl/               # scratch — raw wget mirror output, wiped and regenerated by every
                      # snapshot.sh run, NEVER committed
+snapshot.new/        # scratch — build-then-swap staging dir; moved onto snapshot/ only
+                     # on full success, NEVER committed
 ```
 
 ## Re-running the snapshot
@@ -60,14 +62,17 @@ DOCS_REPO=/path/to/vyos-documentation bash tools/snapshot.sh
 ```
 
 `DOCS_REPO` defaults to `$HOME/GitHub/vyos-documentation`. The script is
-idempotent: it wipes both `crawl/` and `snapshot/` before every run, crawls
+idempotent: it wipes `crawl/` and `snapshot.new/` before every run, crawls
 `/en/1.2/` and `/en/1.3/` from the **live** `docs.vyos.io`, scrubs every
 mirrored HTML file via `tools/scrub.py`, copies in the picker bundle,
-downloads 1.3's PDF into `r2-staging/`, hard-fails on any dangling internal
-link (parity check — the crawl graph is the source of truth since 1.2/1.3
-never had a `sitemap.xml`), generates a `sitemap.xml` per version from the
-crawl inventory, and builds a **per-version** Pagefind index (no
-cross-version search contamination).
+downloads 1.3's PDF into `r2-staging/` (validating its `%PDF-` header),
+hard-fails on any dangling internal link (parity check — the crawl graph is
+the source of truth since 1.2/1.3 never had a `sitemap.xml`), generates a
+`sitemap.xml` per version from the crawl inventory, builds a **per-version**
+Pagefind index (no cross-version search contamination), and — only once both
+`1.2` and `1.3` complete every step successfully — atomically swaps
+`snapshot.new/` onto `snapshot/`. A failed run therefore never leaves the
+committed `snapshot/` tree partially updated.
 
 Run the unit tests with:
 
diff --git a/tools/snapshot.sh b/tools/snapshot.sh
index 065b8ce..687d735 100755
--- a/tools/snapshot.sh
+++ b/tools/snapshot.sh
@@ -1,10 +1,14 @@
 #!/usr/bin/env bash
-# One-time snapshot of RTD-served 1.2 + 1.3 (spec §6). Idempotent: wipes snapshot/ first.
+# One-time snapshot of RTD-served 1.2 + 1.3 (spec §6). Idempotent: builds into
+# snapshot.new/ and swaps it into snapshot/ only on full success (both slugs,
+# all steps) — a failed re-run must never leave the committed snapshot/
+# source-of-truth partially updated.
 set -euo pipefail
 cd "$(dirname "$0")/.."
 DOCS_REPO="${DOCS_REPO:-$HOME/GitHub/vyos-documentation}"
+PAGEFIND_VERSION="1.5.2"   # pinned — matches vyos-documentation's docs-build.yml pagefind pin
 
-rm -rf snapshot crawl && mkdir -p snapshot   # crawl/ wiped too — stale crawl data must never mask missing RTD pages
+rm -rf snapshot.new crawl && mkdir -p snapshot.new   # crawl/ wiped too — stale crawl data must never mask missing RTD pages
 for slug in 1.2 1.3; do
   echo "== crawling /en/$slug/"
   # docs.vyos.io sits behind Cloudflare. Empirically (2026-07-10): wget's default
@@ -13,17 +17,21 @@ for slug in 1.2 1.3; do
   # separate rate-limit challenge. --user-agent spoofs a browser UA;
   # --wait/--random-wait throttle the crawl; --retry-on-http-error=429,503 +
   # --tries retry any transient edge errors instead of wget treating them as fatal.
-  wget --mirror --page-requisites --adjust-extension=off --no-parent \
+  # --no-adjust-extension (documented long-option form of -E's negation; the
+  # `--adjust-extension=off` value form is also accepted by wget but this is
+  # the form wget's own --help documents) keeps mirrored filenames matching
+  # the URLs referenced by internal hrefs.
+  wget --mirror --page-requisites --no-adjust-extension --no-parent \
        --directory-prefix=crawl --no-host-directories \
        --user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36" \
        --wait=0.6 --random-wait --tries=10 --waitretry=20 \
        --retry-on-http-error=429,503 \
        "https://docs.vyos.io/en/$slug/"
-  mkdir -p "snapshot/en/$slug"
-  cp -r "crawl/en/$slug/." "snapshot/en/$slug/"
+  mkdir -p "snapshot.new/en/$slug"
+  cp -r "crawl/en/$slug/." "snapshot.new/en/$slug/"
 
   echo "== scrubbing"
-  find "snapshot/en/$slug" -name '*.html' -print0 | while IFS= read -r -d '' f; do
+  find "snapshot.new/en/$slug" -name '*.html' -print0 | while IFS= read -r -d '' f; do
     python - "$f" "$slug" <<'EOF'
 import sys
 from tools.scrub import scrub_html
@@ -34,15 +42,21 @@ EOF
   done
 
   echo "== picker bundle + PDF"
-  mkdir -p "snapshot/en/$slug/_static/js" "snapshot/en/$slug/_static/css"
-  cp "$DOCS_REPO/docs/_static/js/version-picker.js"  "snapshot/en/$slug/_static/js/"
-  cp "$DOCS_REPO/docs/_static/css/version-picker.css" "snapshot/en/$slug/_static/css/"
+  mkdir -p "snapshot.new/en/$slug/_static/js" "snapshot.new/en/$slug/_static/css"
+  cp "$DOCS_REPO/docs/_static/js/version-picker.js"  "snapshot.new/en/$slug/_static/js/"
+  cp "$DOCS_REPO/docs/_static/css/version-picker.css" "snapshot.new/en/$slug/_static/css/"
   # Phase-0 findings (spec §15a): 1.2 has NO PDF artifact (skip); 1.3's PDF is
   # 29.2 MiB > the 25 MiB Workers per-asset cap → staged for the R2 fallback
   # (uploaded to R2 + served via an apex route), NOT placed into Worker assets.
   if [ "$slug" = "1.3" ]; then
     mkdir -p r2-staging
     curl -fL "https://docs.vyos.io/_/downloads/en/$slug/pdf/" -o "r2-staging/vyos-documentation-1.3.pdf"
+    header=$(head -c 5 "r2-staging/vyos-documentation-1.3.pdf")
+    if [ "$header" != "%PDF-" ]; then
+      echo "PDF content validation FAILED for $slug: file does not start with %PDF- (got: $header)"
+      exit 1
+    fi
+    echo "PDF content OK for $slug: %PDF- header present"
   fi
 
   echo "== parity check (hard-fail, §6 as amended by §15a — crawl-inventory link-audit; 1.2/1.3 have no sitemap.xml)"
@@ -52,7 +66,7 @@ import re, sys, pathlib
 # The crawl graph is the source of truth (wget --mirror follows all links);
 # a dangling internal link ⇒ the crawl missed a page ⇒ hard fail.
 slug = sys.argv[1]
-root = pathlib.Path(f"snapshot/en/{slug}")
+root = pathlib.Path(f"snapshot.new/en/{slug}")
 pages = list(root.rglob("*.html"))
 if len(pages) < 50:
     print(f"PARITY FAIL: only {len(pages)} pages mirrored for {slug} — crawl incomplete")
@@ -82,7 +96,7 @@ EOF
   python - "$slug" <<'EOF'
 import pathlib, sys
 slug = sys.argv[1]
-root = pathlib.Path(f"snapshot/en/{slug}")
+root = pathlib.Path(f"snapshot.new/en/{slug}")
 urls = sorted(f"https://docs.vyos.io/en/{slug}/{p.relative_to(root)}" for p in root.rglob("*.html"))
 body = "".join(f"<url><loc>{u}</loc></url>" for u in urls)
 (root / "sitemap.xml").write_text(
@@ -91,6 +105,10 @@ print(f"sitemap.xml generated for {slug}: {len(urls)} URLs")
 EOF
 
   echo "== per-version Pagefind index (§6 — one per version, no cross-contamination)"
-  npx --yes pagefind --site "snapshot/en/$slug"
+  npx --yes "pagefind@${PAGEFIND_VERSION}" --site "snapshot.new/en/$slug"
 done
+
+echo "== swap: snapshot.new/ -> snapshot/ (only on full success for both slugs)"
+rm -rf snapshot
+mv snapshot.new snapshot
 echo "snapshot complete: $(find snapshot -type f | wc -l) files"