#!/usr/bin/env bash
# One-time snapshot of RTD-served 1.2 + 1.3 (spec §6). Idempotent: builds into
# snapshot.new/ and swaps it into snapshot/ only on full success (both slugs,
# all steps) — a failed re-run must never leave the committed snapshot/
# source-of-truth partially updated.
set -euo pipefail
cd "$(dirname "$0")/.."
DOCS_REPO="${DOCS_REPO:-$HOME/GitHub/vyos-documentation}"
PAGEFIND_VERSION="1.5.2"   # pinned — matches vyos-documentation's docs-build.yml pagefind pin

rm -rf snapshot.new crawl && mkdir -p snapshot.new   # crawl/ wiped too — stale crawl data must never mask missing RTD pages
for slug in 1.2 1.3; do
  echo "== crawling /en/$slug/"
  # docs.vyos.io sits behind Cloudflare. Empirically (2026-07-10): wget's default
  # User-Agent ("Wget/x.y") gets a Cloudflare managed-challenge 429 regardless of
  # request rate, while a browser UA passes; an unthrottled burst also risks a
  # separate rate-limit challenge. --user-agent spoofs a browser UA;
  # --wait/--random-wait throttle the crawl; --retry-on-http-error=429,503 +
  # --tries retry any transient edge errors instead of wget treating them as fatal.
  # --no-adjust-extension (documented long-option form of -E's negation; the
  # `--adjust-extension=off` value form is also accepted by wget but this is
  # the form wget's own --help documents) keeps mirrored filenames matching
  # the URLs referenced by internal hrefs.
  wget --mirror --page-requisites --no-adjust-extension --no-parent \
       --directory-prefix=crawl --no-host-directories \
       --user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36" \
       --wait=0.6 --random-wait --tries=10 --waitretry=20 \
       --retry-on-http-error=429,503 \
       "https://docs.vyos.io/en/$slug/"
  mkdir -p "snapshot.new/en/$slug"
  cp -r "crawl/en/$slug/." "snapshot.new/en/$slug/"

  echo "== scrubbing"
  find "snapshot.new/en/$slug" -name '*.html' -print0 | while IFS= read -r -d '' f; do
    python - "$f" "$slug" <<'EOF'
import sys
from tools.scrub import scrub_html
p, slug = sys.argv[1], sys.argv[2]
src = open(p, encoding="utf-8", errors="ignore").read()
open(p, "w", encoding="utf-8").write(scrub_html(src, slug))
EOF
  done

  echo "== picker bundle + PDF"
  mkdir -p "snapshot.new/en/$slug/_static/js" "snapshot.new/en/$slug/_static/css"
  cp "$DOCS_REPO/docs/_static/js/version-picker.js"  "snapshot.new/en/$slug/_static/js/"
  cp "$DOCS_REPO/docs/_static/css/version-picker.css" "snapshot.new/en/$slug/_static/css/"
  # Phase-0 findings (spec §15a): 1.2 has NO PDF artifact (skip); 1.3's PDF is
  # 29.2 MiB > the 25 MiB Workers per-asset cap → staged for the R2 fallback
  # (uploaded to R2 + served via an apex route), NOT placed into Worker assets.
  if [ "$slug" = "1.3" ]; then
    mkdir -p r2-staging
    curl -fL "https://docs.vyos.io/_/downloads/en/$slug/pdf/" -o "r2-staging/vyos-documentation-1.3.pdf"
    header=$(head -c 5 "r2-staging/vyos-documentation-1.3.pdf")
    if [ "$header" != "%PDF-" ]; then
      echo "PDF content validation FAILED for $slug: file does not start with %PDF- (got: $header)"
      exit 1
    fi
    echo "PDF content OK for $slug: %PDF- header present"
  fi

  echo "== parity check (hard-fail, §6 as amended by §15a — crawl-inventory link-audit; 1.2/1.3 have no sitemap.xml)"
  python - "$slug" <<'EOF'
import re, sys, pathlib
# Every internal href in the mirrored tree must resolve to a mirrored file.
# The crawl graph is the source of truth (wget --mirror follows all links);
# a dangling internal link ⇒ the crawl missed a page ⇒ hard fail.
slug = sys.argv[1]
root = pathlib.Path(f"snapshot.new/en/{slug}")
pages = list(root.rglob("*.html"))
if len(pages) < 50:
    print(f"PARITY FAIL: only {len(pages)} pages mirrored for {slug} — crawl incomplete")
    sys.exit(1)
missing = set()
for page in pages:
    html = page.read_text(errors="ignore")
    for href in re.findall(r'href="([^"#?]+)"', html):
        if href.startswith(("http://", "https://", "mailto:", "//")):
            continue
        target = (page.parent / href).resolve()
        if target.suffix in {".html", ""}:
            t = target if target.suffix else target / "index.html"
            try:
                t.relative_to(root.resolve())
            except ValueError:
                continue  # cross-version link — other slug's tree
            if not t.is_file():
                missing.add(str(t))
if missing:
    print("PARITY FAIL (dangling internal links):", *sorted(missing)[:20], sep="\n  ")
    sys.exit(1)
print(f"parity OK for {slug}: {len(pages)} pages, no dangling internal links")
EOF

  echo "== generate sitemap.xml from crawl inventory (§15a: legacy builds never had one; apex sitemap-index references it)"
  python - "$slug" <<'EOF'
import pathlib, sys
slug = sys.argv[1]
root = pathlib.Path(f"snapshot.new/en/{slug}")
urls = sorted(f"https://docs.vyos.io/en/{slug}/{p.relative_to(root)}" for p in root.rglob("*.html"))
body = "".join(f"<url><loc>{u}</loc></url>" for u in urls)
(root / "sitemap.xml").write_text(
    f'<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{body}</urlset>')
print(f"sitemap.xml generated for {slug}: {len(urls)} URLs")
EOF

  echo "== per-version Pagefind index (§6 — one per version, no cross-contamination)"
  npx --yes "pagefind@${PAGEFIND_VERSION}" --site "snapshot.new/en/$slug"
done

echo "== swap: snapshot.new/ -> snapshot/ (only on full success for both slugs)"
rm -rf snapshot
mv snapshot.new snapshot
echo "snapshot complete: $(find snapshot -type f | wc -l) files"