docs-gates: smoke per-probe retry + explicit UA; workers: broaden asset-ext classificationclaude/smoke-hardening

smoke.py — BIC independence: probe requests now send an explicit User-Agent
(vyos-docs-smoke/1.0) so the gate no longer depends on a Cloudflare Browser
Integrity Check UA-skip rule surviving. The default Python-urllib UA was blocked
by BIC until that exemption was added; a silent dependency on it is a latent gate
failure the moment the rule is touched.

smoke.py — propagation-race tolerance: each probe now retries up to 3 attempts
(20s apart; MAX_ATTEMPTS + RETRY_SLEEP_SECONDS are module-level so tests can shrink
them) and only fails after the final attempt. A freshly deployed worker version
loses a brief propagation race in which a single probe is served by the PREVIOUS
version (observed: status 307 + stale X-Docs-Build minutes after deploy), which
previously failed the entire gate. Intermediate attempts log SMOKE-RETRY; only
exhaustion logs SMOKE-FAIL and counts a failure. Retry fires only on a not-ok
outcome (wrong status, wrong/missing build header, missing search mount, or a
transport exception); a legitimately-expected 404 passes on the first attempt.

workers/branch — broaden asset classification (CodeRabbit post-merge nit): fold
.pdf into the case-insensitive ASSET_EXT_RE and add webp + otf, so uppercase .PDF
and modern image/font assets get the longer asset cache class. /_static/ and
/_images/ path checks unchanged.

🤖 Generated by [robots](https://vyos.io)

4 files changed, 160 insertions, 27 deletions

diff --git a/scripts/docs_gates/smoke.py b/scripts/docs_gates/smoke.py
index d25e9a1f..5500b728 100644
--- a/scripts/docs_gates/smoke.py
+++ b/scripts/docs_gates/smoke.py
@@ -16,11 +16,23 @@ import argparse
 import dataclasses
 import json
 import sys
+import time
 import urllib.request
 
 APEX_PATHS = ["/versions.json", "/healthz", "/robots.txt", "/sitemap.xml"]
 SEARCH_MOUNT_MARKER = 'id="vyos-search"'
 
+# Explicit UA so the gate never depends on a Cloudflare edge exemption for the default
+# Python-urllib UA. The Browser Integrity Check blocked that UA until a skip rule was added;
+# the gate must not silently rely on that rule surviving.
+USER_AGENT = "vyos-docs-smoke/1.0 (+https://github.com/vyos/vyos-documentation)"
+
+# Per-probe retry (module-level so tests can shrink them). A freshly-deployed worker version
+# can lose a propagation race: for a few minutes a single probe may be served by the PREVIOUS
+# version, returning the wrong status / a stale X-Docs-Build. Retry the probe, not the gate.
+MAX_ATTEMPTS = 3
+RETRY_SLEEP_SECONDS = 20
+
 
 class _NoRedirect(urllib.request.HTTPRedirectHandler):
     """Probes assert an EXACT status per-path (200 or 404) — following a 3xx would
@@ -66,33 +78,58 @@ def search_mount_present(html: str) -> bool:
     return SEARCH_MOUNT_MARKER in html
 
 
+def _probe_once(host: str, probe: Probe, expect_sha: str, access_id: str,
+                access_secret: str) -> tuple[bool, int | None, str | None, str | None]:
+    """One probe attempt. Returns (ok, status, docs_build, error): a transport exception
+    yields (False, None, None, <msg>); status/docs_build are surfaced for retry logging."""
+    req = urllib.request.Request(f"https://{host}{probe.path}", method="GET")
+    req.add_header("CF-Access-Client-Id", access_id)
+    req.add_header("CF-Access-Client-Secret", access_secret)
+    req.add_header("User-Agent", USER_AGENT)
+    try:
+        with _OPENER.open(req, timeout=30) as resp:
+            status, headers, body = resp.status, resp.headers, resp.read()
+    except urllib.error.HTTPError as e:  # non-2xx still carries headers
+        status, headers, body = e.code, e.headers, e.read()
+    except Exception as e:  # noqa: BLE001 — any transport error fails the probe
+        return False, None, None, str(e)
+    ok = status == probe.expect_status
+    if probe.assert_docs_build and not docs_build_ok(headers.get("X-Docs-Build"), expect_sha):
+        ok = False
+    if probe.assert_apex_build and not headers.get("X-Apex-Build"):
+        ok = False
+    if probe.assert_search_mount and not search_mount_present(
+            body.decode("utf-8", errors="replace")):
+        ok = False
+    return ok, status, headers.get("X-Docs-Build"), None
+
+
+def _probe_with_retries(host: str, probe: Probe, expect_sha: str, access_id: str,
+                        access_secret: str) -> bool:
+    """Up to MAX_ATTEMPTS attempts, RETRY_SLEEP_SECONDS between. Passes if ANY attempt is ok;
+    intermediate failures log SMOKE-RETRY and only the final failed attempt emits SMOKE-FAIL,
+    so a single propagation blip served by the previous worker version cannot fail the gate."""
+    for attempt in range(1, MAX_ATTEMPTS + 1):
+        ok, status, docs_build, error = _probe_once(
+            host, probe, expect_sha, access_id, access_secret)
+        if ok:
+            return True
+        detail = f"status={status} docs-build={docs_build}"
+        if error is not None:
+            detail += f" error={error}"
+        if attempt < MAX_ATTEMPTS:
+            print(f"SMOKE-RETRY {probe.path}: attempt {attempt} {detail}", file=sys.stderr)
+            time.sleep(RETRY_SLEEP_SECONDS)
+        else:
+            print(f"SMOKE-FAIL {probe.path}: {detail}", file=sys.stderr)
+    return False
+
+
 def run(host: str, slug: str, expect_sha: str, access_id: str, access_secret: str,
         pdf: str | None, critical: list[str]) -> int:
     failures = 0
     for probe in probe_plan(slug, pdf, critical):
-        req = urllib.request.Request(f"https://{host}{probe.path}", method="GET")
-        req.add_header("CF-Access-Client-Id", access_id)
-        req.add_header("CF-Access-Client-Secret", access_secret)
-        try:
-            with _OPENER.open(req, timeout=30) as resp:
-                status, headers, body = resp.status, resp.headers, resp.read()
-        except urllib.error.HTTPError as e:  # non-2xx still carries headers
-            status, headers, body = e.code, e.headers, e.read()
-        except Exception as e:  # noqa: BLE001 — any transport error fails the probe
-            print(f"SMOKE-FAIL {probe.path}: {e}", file=sys.stderr)
-            failures += 1
-            continue
-        ok = status == probe.expect_status
-        if probe.assert_docs_build and not docs_build_ok(headers.get("X-Docs-Build"), expect_sha):
-            ok = False
-        if probe.assert_apex_build and not headers.get("X-Apex-Build"):
-            ok = False
-        if probe.assert_search_mount and not search_mount_present(
-                body.decode("utf-8", errors="replace")):
-            ok = False
-        if not ok:
-            print(f"SMOKE-FAIL {probe.path}: status={status} "
-                  f"docs-build={headers.get('X-Docs-Build')}", file=sys.stderr)
+        if not _probe_with_retries(host, probe, expect_sha, access_id, access_secret):
             failures += 1
     print(json.dumps({"failures": failures}))
     return 1 if failures else 0
diff --git a/scripts/docs_gates/test_smoke.py b/scripts/docs_gates/test_smoke.py
index 595e6a28..f5967be7 100644
--- a/scripts/docs_gates/test_smoke.py
+++ b/scripts/docs_gates/test_smoke.py
@@ -1,5 +1,9 @@
+from __future__ import annotations
+
+import io
 import urllib.error
 import urllib.request
+from email.message import Message
 
 from scripts.docs_gates import smoke
 from scripts.docs_gates.conftest import REDIRECT_LOCATION, REDIRECT_PATH
@@ -68,3 +72,93 @@ def test_opener_observes_redirect_directly_not_followed(redirect_http_server):
 def test_search_mount_present():
     assert smoke.search_mount_present('<div id="vyos-search" role="search"></div>') is True
     assert smoke.search_mount_present('<html><body>no search here</body></html>') is False
+
+
+# --- Hardening (this change): explicit UA + per-probe retry. Mock at the _OPENER boundary
+# (the exact object _probe_once() opens through, mirroring the redirect test above which drives
+# smoke._OPENER directly), and shrink RETRY_SLEEP_SECONDS to 0 so retries don't wall-clock. ---
+
+
+class _FakeResp:
+    """Stand-in for what _OPENER.open() yields: a context manager exposing .status /
+    .headers / .read()."""
+
+    def __init__(self, status: int, headers: dict[str, str], body: bytes = b""):
+        self.status = status
+        self.headers = headers
+        self._body = body
+
+    def read(self) -> bytes:
+        return self._body
+
+    def __enter__(self) -> "_FakeResp":
+        return self
+
+    def __exit__(self, *exc: object) -> bool:
+        return False
+
+
+class _FakeOpener:
+    """Yields queued responses in order; an Exception item is raised (models a transport
+    error, or a non-2xx delivered as HTTPError). Records each opened Request for assertions."""
+
+    def __init__(self, responses: list[object]):
+        self._responses = list(responses)
+        self.calls: list[urllib.request.Request] = []
+
+    def open(self, req: urllib.request.Request, timeout: float | None = None) -> object:
+        self.calls.append(req)
+        item = self._responses.pop(0)
+        if isinstance(item, Exception):
+            raise item
+        return item
+
+
+def _http_error(code: int, headers: dict[str, str]) -> urllib.error.HTTPError:
+    hdrs = Message()
+    for k, v in headers.items():
+        hdrs[k] = v
+    return urllib.error.HTTPError("https://host.invalid/x", code, "msg", hdrs, io.BytesIO(b""))
+
+
+def test_probe_sends_explicit_user_agent(monkeypatch):
+    opener = _FakeOpener([_FakeResp(200, {"X-Docs-Build": "sha1"})])
+    monkeypatch.setattr(smoke, "_OPENER", opener)
+    probe = smoke.Probe("/en/1.5/index.html", 200, assert_docs_build=True, assert_apex_build=False)
+    ok, *_ = smoke._probe_once("host.example", probe, "sha1", "cf-id", "cf-secret")
+    assert ok is True
+    req = opener.calls[0]
+    assert req.get_header("User-agent") == smoke.USER_AGENT   # urllib capitalizes the key
+    assert req.get_header("Cf-access-client-id") == "cf-id"   # CF-Access headers still sent
+
+
+def test_retry_passes_when_second_attempt_ok(monkeypatch):
+    monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0)
+    opener = _FakeOpener([
+        _FakeResp(307, {"X-Docs-Build": "stale"}),    # attempt 1: previous-version blip
+        _FakeResp(200, {"X-Docs-Build": "goodsha"}),  # attempt 2: propagation settled
+    ])
+    monkeypatch.setattr(smoke, "_OPENER", opener)
+    probe = smoke.Probe("/en/1.5/cli.html", 200, assert_docs_build=True, assert_apex_build=False)
+    assert smoke._probe_with_retries("host", probe, "goodsha", "id", "sec") is True
+    assert len(opener.calls) == 2
+
+
+def test_retry_fails_after_exhausting_attempts(monkeypatch):
+    monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0)
+    opener = _FakeOpener([_FakeResp(307, {"X-Docs-Build": "stale"})
+                          for _ in range(smoke.MAX_ATTEMPTS)])
+    monkeypatch.setattr(smoke, "_OPENER", opener)
+    probe = smoke.Probe("/en/1.5/index.html", 200, assert_docs_build=True, assert_apex_build=False)
+    assert smoke._probe_with_retries("host", probe, "goodsha", "id", "sec") is False
+    assert len(opener.calls) == smoke.MAX_ATTEMPTS   # tried the full budget
+
+
+def test_expected_404_passes_first_attempt_without_retry(monkeypatch):
+    monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0)
+    opener = _FakeOpener([_http_error(404, {})])  # 404 delivered as HTTPError, like urllib
+    monkeypatch.setattr(smoke, "_OPENER", opener)
+    probe = smoke.Probe("/en/1.5/definitely-missing.html", 404,
+                        assert_docs_build=False, assert_apex_build=False)
+    assert smoke._probe_with_retries("host", probe, "sha", "id", "sec") is True
+    assert len(opener.calls) == 1   # a legitimately-expected 404 must NOT burn retries
diff --git a/workers/branch/src/index.ts b/workers/branch/src/index.ts
index 7cdaaf14..47ef3ad6 100644
--- a/workers/branch/src/index.ts
+++ b/workers/branch/src/index.ts
@@ -6,13 +6,12 @@ export interface Env {
 
 export type CacheClass = "page" | "asset";
 
-// Binary/media assets get the longer asset cache class, alongside .pdf and the Sphinx
-// /_static/ (theme) + /_images/ (figure) trees.
-const ASSET_EXT_RE = /\.(png|jpe?g|svg|gif|ico|woff2?|ttf|eot)$/i;
+// Binary/media asset extensions (case-insensitive, so ".PDF" also matches) get the longer
+// asset cache class, alongside the Sphinx /_static/ (theme) + /_images/ (figure) trees.
+const ASSET_EXT_RE = /\.(pdf|png|jpe?g|webp|svg|gif|ico|woff2?|ttf|otf|eot)$/i;
 
 export function classifyPath(path: string): CacheClass {
   if (
-    path.endsWith(".pdf") ||
     path.includes("/_static/") ||
     path.includes("/_images/") ||
     ASSET_EXT_RE.test(path)
diff --git a/workers/branch/test/content.test.ts b/workers/branch/test/content.test.ts
index 94a0d5a6..6d04c35b 100644
--- a/workers/branch/test/content.test.ts
+++ b/workers/branch/test/content.test.ts
@@ -243,8 +243,11 @@ describe("asset cache-class classification (§3.3, extended)", () => {
       "/en/rolling/_static/fonts/roboto.woff2",
       "/en/rolling/logo.svg",
       "/en/rolling/photo.jpeg",
+      "/en/rolling/hero.webp",               // webp added to ASSET_EXT_RE (standalone path)
+      "/en/rolling/font.otf",                // otf added to ASSET_EXT_RE (standalone path)
       "/en/rolling/icon.ico",
       "/en/rolling/vyos-documentation.pdf",
+      "/en/rolling/vyos-documentation.PDF",  // .pdf folded into the case-insensitive regex
     ]) {
       expect(classifyPath(p)).toBe("asset");
     }