diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/docs_gates/smoke.py | 83 | ||||
| -rw-r--r-- | scripts/docs_gates/test_smoke.py | 94 |
2 files changed, 154 insertions, 23 deletions
diff --git a/scripts/docs_gates/smoke.py b/scripts/docs_gates/smoke.py index d25e9a1f..5500b728 100644 --- a/scripts/docs_gates/smoke.py +++ b/scripts/docs_gates/smoke.py @@ -16,11 +16,23 @@ import argparse import dataclasses import json import sys +import time import urllib.request APEX_PATHS = ["/versions.json", "/healthz", "/robots.txt", "/sitemap.xml"] SEARCH_MOUNT_MARKER = 'id="vyos-search"' +# Explicit UA so the gate never depends on a Cloudflare edge exemption for the default +# Python-urllib UA. The Browser Integrity Check blocked that UA until a skip rule was added; +# the gate must not silently rely on that rule surviving. +USER_AGENT = "vyos-docs-smoke/1.0 (+https://github.com/vyos/vyos-documentation)" + +# Per-probe retry (module-level so tests can shrink them). A freshly-deployed worker version +# can lose a propagation race: for a few minutes a single probe may be served by the PREVIOUS +# version, returning the wrong status / a stale X-Docs-Build. Retry the probe, not the gate. +MAX_ATTEMPTS = 3 +RETRY_SLEEP_SECONDS = 20 + class _NoRedirect(urllib.request.HTTPRedirectHandler): """Probes assert an EXACT status per-path (200 or 404) — following a 3xx would @@ -66,33 +78,58 @@ def search_mount_present(html: str) -> bool: return SEARCH_MOUNT_MARKER in html +def _probe_once(host: str, probe: Probe, expect_sha: str, access_id: str, + access_secret: str) -> tuple[bool, int | None, str | None, str | None]: + """One probe attempt. Returns (ok, status, docs_build, error): a transport exception + yields (False, None, None, <msg>); status/docs_build are surfaced for retry logging.""" + req = urllib.request.Request(f"https://{host}{probe.path}", method="GET") + req.add_header("CF-Access-Client-Id", access_id) + req.add_header("CF-Access-Client-Secret", access_secret) + req.add_header("User-Agent", USER_AGENT) + try: + with _OPENER.open(req, timeout=30) as resp: + status, headers, body = resp.status, resp.headers, resp.read() + except urllib.error.HTTPError as e: # non-2xx still carries headers + status, headers, body = e.code, e.headers, e.read() + except Exception as e: # noqa: BLE001 — any transport error fails the probe + return False, None, None, str(e) + ok = status == probe.expect_status + if probe.assert_docs_build and not docs_build_ok(headers.get("X-Docs-Build"), expect_sha): + ok = False + if probe.assert_apex_build and not headers.get("X-Apex-Build"): + ok = False + if probe.assert_search_mount and not search_mount_present( + body.decode("utf-8", errors="replace")): + ok = False + return ok, status, headers.get("X-Docs-Build"), None + + +def _probe_with_retries(host: str, probe: Probe, expect_sha: str, access_id: str, + access_secret: str) -> bool: + """Up to MAX_ATTEMPTS attempts, RETRY_SLEEP_SECONDS between. Passes if ANY attempt is ok; + intermediate failures log SMOKE-RETRY and only the final failed attempt emits SMOKE-FAIL, + so a single propagation blip served by the previous worker version cannot fail the gate.""" + for attempt in range(1, MAX_ATTEMPTS + 1): + ok, status, docs_build, error = _probe_once( + host, probe, expect_sha, access_id, access_secret) + if ok: + return True + detail = f"status={status} docs-build={docs_build}" + if error is not None: + detail += f" error={error}" + if attempt < MAX_ATTEMPTS: + print(f"SMOKE-RETRY {probe.path}: attempt {attempt} {detail}", file=sys.stderr) + time.sleep(RETRY_SLEEP_SECONDS) + else: + print(f"SMOKE-FAIL {probe.path}: {detail}", file=sys.stderr) + return False + + def run(host: str, slug: str, expect_sha: str, access_id: str, access_secret: str, pdf: str | None, critical: list[str]) -> int: failures = 0 for probe in probe_plan(slug, pdf, critical): - req = urllib.request.Request(f"https://{host}{probe.path}", method="GET") - req.add_header("CF-Access-Client-Id", access_id) - req.add_header("CF-Access-Client-Secret", access_secret) - try: - with _OPENER.open(req, timeout=30) as resp: - status, headers, body = resp.status, resp.headers, resp.read() - except urllib.error.HTTPError as e: # non-2xx still carries headers - status, headers, body = e.code, e.headers, e.read() - except Exception as e: # noqa: BLE001 — any transport error fails the probe - print(f"SMOKE-FAIL {probe.path}: {e}", file=sys.stderr) - failures += 1 - continue - ok = status == probe.expect_status - if probe.assert_docs_build and not docs_build_ok(headers.get("X-Docs-Build"), expect_sha): - ok = False - if probe.assert_apex_build and not headers.get("X-Apex-Build"): - ok = False - if probe.assert_search_mount and not search_mount_present( - body.decode("utf-8", errors="replace")): - ok = False - if not ok: - print(f"SMOKE-FAIL {probe.path}: status={status} " - f"docs-build={headers.get('X-Docs-Build')}", file=sys.stderr) + if not _probe_with_retries(host, probe, expect_sha, access_id, access_secret): failures += 1 print(json.dumps({"failures": failures})) return 1 if failures else 0 diff --git a/scripts/docs_gates/test_smoke.py b/scripts/docs_gates/test_smoke.py index 595e6a28..f5967be7 100644 --- a/scripts/docs_gates/test_smoke.py +++ b/scripts/docs_gates/test_smoke.py @@ -1,5 +1,9 @@ +from __future__ import annotations + +import io import urllib.error import urllib.request +from email.message import Message from scripts.docs_gates import smoke from scripts.docs_gates.conftest import REDIRECT_LOCATION, REDIRECT_PATH @@ -68,3 +72,93 @@ def test_opener_observes_redirect_directly_not_followed(redirect_http_server): def test_search_mount_present(): assert smoke.search_mount_present('<div id="vyos-search" role="search"></div>') is True assert smoke.search_mount_present('<html><body>no search here</body></html>') is False + + +# --- Hardening (this change): explicit UA + per-probe retry. Mock at the _OPENER boundary +# (the exact object _probe_once() opens through, mirroring the redirect test above which drives +# smoke._OPENER directly), and shrink RETRY_SLEEP_SECONDS to 0 so retries don't wall-clock. --- + + +class _FakeResp: + """Stand-in for what _OPENER.open() yields: a context manager exposing .status / + .headers / .read().""" + + def __init__(self, status: int, headers: dict[str, str], body: bytes = b""): + self.status = status + self.headers = headers + self._body = body + + def read(self) -> bytes: + return self._body + + def __enter__(self) -> "_FakeResp": + return self + + def __exit__(self, *exc: object) -> bool: + return False + + +class _FakeOpener: + """Yields queued responses in order; an Exception item is raised (models a transport + error, or a non-2xx delivered as HTTPError). Records each opened Request for assertions.""" + + def __init__(self, responses: list[object]): + self._responses = list(responses) + self.calls: list[urllib.request.Request] = [] + + def open(self, req: urllib.request.Request, timeout: float | None = None) -> object: + self.calls.append(req) + item = self._responses.pop(0) + if isinstance(item, Exception): + raise item + return item + + +def _http_error(code: int, headers: dict[str, str]) -> urllib.error.HTTPError: + hdrs = Message() + for k, v in headers.items(): + hdrs[k] = v + return urllib.error.HTTPError("https://host.invalid/x", code, "msg", hdrs, io.BytesIO(b"")) + + +def test_probe_sends_explicit_user_agent(monkeypatch): + opener = _FakeOpener([_FakeResp(200, {"X-Docs-Build": "sha1"})]) + monkeypatch.setattr(smoke, "_OPENER", opener) + probe = smoke.Probe("/en/1.5/index.html", 200, assert_docs_build=True, assert_apex_build=False) + ok, *_ = smoke._probe_once("host.example", probe, "sha1", "cf-id", "cf-secret") + assert ok is True + req = opener.calls[0] + assert req.get_header("User-agent") == smoke.USER_AGENT # urllib capitalizes the key + assert req.get_header("Cf-access-client-id") == "cf-id" # CF-Access headers still sent + + +def test_retry_passes_when_second_attempt_ok(monkeypatch): + monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0) + opener = _FakeOpener([ + _FakeResp(307, {"X-Docs-Build": "stale"}), # attempt 1: previous-version blip + _FakeResp(200, {"X-Docs-Build": "goodsha"}), # attempt 2: propagation settled + ]) + monkeypatch.setattr(smoke, "_OPENER", opener) + probe = smoke.Probe("/en/1.5/cli.html", 200, assert_docs_build=True, assert_apex_build=False) + assert smoke._probe_with_retries("host", probe, "goodsha", "id", "sec") is True + assert len(opener.calls) == 2 + + +def test_retry_fails_after_exhausting_attempts(monkeypatch): + monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0) + opener = _FakeOpener([_FakeResp(307, {"X-Docs-Build": "stale"}) + for _ in range(smoke.MAX_ATTEMPTS)]) + monkeypatch.setattr(smoke, "_OPENER", opener) + probe = smoke.Probe("/en/1.5/index.html", 200, assert_docs_build=True, assert_apex_build=False) + assert smoke._probe_with_retries("host", probe, "goodsha", "id", "sec") is False + assert len(opener.calls) == smoke.MAX_ATTEMPTS # tried the full budget + + +def test_expected_404_passes_first_attempt_without_retry(monkeypatch): + monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0) + opener = _FakeOpener([_http_error(404, {})]) # 404 delivered as HTTPError, like urllib + monkeypatch.setattr(smoke, "_OPENER", opener) + probe = smoke.Probe("/en/1.5/definitely-missing.html", 404, + assert_docs_build=False, assert_apex_build=False) + assert smoke._probe_with_retries("host", probe, "sha", "id", "sec") is True + assert len(opener.calls) == 1 # a legitimately-expected 404 must NOT burn retries |