diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/docs_gates/smoke.py | 107 | ||||
| -rw-r--r-- | scripts/docs_gates/test_smoke.py | 155 |
2 files changed, 199 insertions, 63 deletions
diff --git a/scripts/docs_gates/smoke.py b/scripts/docs_gates/smoke.py index 5500b728..2dab82ff 100644 --- a/scripts/docs_gates/smoke.py +++ b/scripts/docs_gates/smoke.py @@ -27,11 +27,15 @@ SEARCH_MOUNT_MARKER = 'id="vyos-search"' # the gate must not silently rely on that rule surviving. USER_AGENT = "vyos-docs-smoke/1.0 (+https://github.com/vyos/vyos-documentation)" -# Per-probe retry (module-level so tests can shrink them). A freshly-deployed worker version -# can lose a propagation race: for a few minutes a single probe may be served by the PREVIOUS -# version, returning the wrong status / a stale X-Docs-Build. Retry the probe, not the gate. -MAX_ATTEMPTS = 3 +# Round-based retry (module-level so tests can shrink them). A freshly-deployed worker version +# can lose a propagation race: for a few minutes a probe may be served by the PREVIOUS version +# (wrong status / stale X-Docs-Build). Rather than fail-fast, each round re-probes ONLY the +# still-failing probes — this preserves the full per-probe failure enumeration (diagnostic +# value) while adding at most (MAX_ROUNDS - 1) inter-round sleeps. DEADLINE_SECONDS caps total +# wall-clock so a pile-up of slow / timing-out probes cannot run unbounded. +MAX_ROUNDS = 3 RETRY_SLEEP_SECONDS = 20 +DEADLINE_SECONDS = 480 class _NoRedirect(urllib.request.HTTPRedirectHandler): @@ -57,6 +61,9 @@ class Probe: def probe_plan(slug: str, pdf: str | None, critical: list[str]) -> list[Probe]: + # `critical` may itself list "index.html" (it does in critical-pages.txt); drop it so the + # index page is probed exactly once — as plan[0], the sole search-mount probe below. + critical = [rel for rel in critical if rel != "index.html"] plan = [Probe(f"/en/{slug}/{rel}", 200, True, False) for rel in ["index.html", *critical]] plan.append(Probe(f"/en/{slug}/pagefind/pagefind.js", 200, True, False)) if pdf: @@ -80,18 +87,21 @@ def search_mount_present(html: str) -> bool: def _probe_once(host: str, probe: Probe, expect_sha: str, access_id: str, access_secret: str) -> tuple[bool, int | None, str | None, str | None]: - """One probe attempt. Returns (ok, status, docs_build, error): a transport exception - yields (False, None, None, <msg>); status/docs_build are surfaced for retry logging.""" + """One probe attempt. Returns (ok, status, docs_build, error). ANY exception in the open + OR body-read path — including a transport error DURING HTTPError.read() — is contained and + yields (False, None, None, <msg>): a retryable failure, never a traceback that crashes the + gate. status/docs_build are surfaced for retry logging.""" req = urllib.request.Request(f"https://{host}{probe.path}", method="GET") req.add_header("CF-Access-Client-Id", access_id) req.add_header("CF-Access-Client-Secret", access_secret) req.add_header("User-Agent", USER_AGENT) try: - with _OPENER.open(req, timeout=30) as resp: - status, headers, body = resp.status, resp.headers, resp.read() - except urllib.error.HTTPError as e: # non-2xx still carries headers - status, headers, body = e.code, e.headers, e.read() - except Exception as e: # noqa: BLE001 — any transport error fails the probe + try: + with _OPENER.open(req, timeout=30) as resp: + status, headers, body = resp.status, resp.headers, resp.read() + except urllib.error.HTTPError as e: # non-2xx still carries headers/body + status, headers, body = e.code, e.headers, e.read() + except Exception as e: # noqa: BLE001 — open OR read failure → retryable probe result return False, None, None, str(e) ok = status == probe.expect_status if probe.assert_docs_build and not docs_build_ok(headers.get("X-Docs-Build"), expect_sha): @@ -104,33 +114,58 @@ def _probe_once(host: str, probe: Probe, expect_sha: str, access_id: str, return ok, status, headers.get("X-Docs-Build"), None -def _probe_with_retries(host: str, probe: Probe, expect_sha: str, access_id: str, - access_secret: str) -> bool: - """Up to MAX_ATTEMPTS attempts, RETRY_SLEEP_SECONDS between. Passes if ANY attempt is ok; - intermediate failures log SMOKE-RETRY and only the final failed attempt emits SMOKE-FAIL, - so a single propagation blip served by the previous worker version cannot fail the gate.""" - for attempt in range(1, MAX_ATTEMPTS + 1): - ok, status, docs_build, error = _probe_once( - host, probe, expect_sha, access_id, access_secret) - if ok: - return True - detail = f"status={status} docs-build={docs_build}" - if error is not None: - detail += f" error={error}" - if attempt < MAX_ATTEMPTS: - print(f"SMOKE-RETRY {probe.path}: attempt {attempt} {detail}", file=sys.stderr) - time.sleep(RETRY_SLEEP_SECONDS) - else: - print(f"SMOKE-FAIL {probe.path}: {detail}", file=sys.stderr) - return False - - def run(host: str, slug: str, expect_sha: str, access_id: str, access_secret: str, pdf: str | None, critical: list[str]) -> int: - failures = 0 - for probe in probe_plan(slug, pdf, critical): - if not _probe_with_retries(host, probe, expect_sha, access_id, access_secret): - failures += 1 + """Probe the whole plan, then re-probe ONLY the still-failing probes each round (up to + MAX_ROUNDS, one RETRY_SLEEP_SECONDS between rounds). A probe passing in ANY round passes; + a single propagation blip served by the previous worker version cannot fail the gate. + DEADLINE_SECONDS bounds total wall-clock — on breach, unresolved probes count as failed.""" + plan = probe_plan(slug, pdf, critical) + start = time.monotonic() + pending = list(plan) # probes not yet passed + detail_by_path: dict[str, str] = {} # last failure detail per path, for logging + deadline_hit = False + + def _past_deadline() -> bool: + return time.monotonic() - start >= DEADLINE_SECONDS + + for round_num in range(1, MAX_ROUNDS + 1): + if not pending: + break + still_failing: list[Probe] = [] + unprobed: list[Probe] = [] + for i, probe in enumerate(pending): + if _past_deadline(): # checked before each probe + deadline_hit = True + unprobed = pending[i:] # not reached this round → still unresolved + break + ok, status, docs_build, error = _probe_once( + host, probe, expect_sha, access_id, access_secret) + if ok: + continue + still_failing.append(probe) + detail = f"status={status} docs-build={docs_build}" + if error is not None: + detail += f" error={error}" + detail_by_path[probe.path] = detail + pending = still_failing + unprobed + if deadline_hit or not pending or round_num == MAX_ROUNDS: + break + if _past_deadline(): # checked before the inter-round sleep + deadline_hit = True + break + for probe in pending: + print(f"SMOKE-RETRY {probe.path}: round {round_num} " + f"{detail_by_path[probe.path]}", file=sys.stderr) + time.sleep(RETRY_SLEEP_SECONDS) + + if deadline_hit: + print("SMOKE-DEADLINE: overall deadline reached — remaining probes counted as failed", + file=sys.stderr) + for probe in pending: + print(f"SMOKE-FAIL {probe.path}: {detail_by_path.get(probe.path, 'unresolved')}", + file=sys.stderr) + failures = len(pending) print(json.dumps({"failures": failures})) return 1 if failures else 0 diff --git a/scripts/docs_gates/test_smoke.py b/scripts/docs_gates/test_smoke.py index f5967be7..063b3a7a 100644 --- a/scripts/docs_gates/test_smoke.py +++ b/scripts/docs_gates/test_smoke.py @@ -4,6 +4,7 @@ import io import urllib.error import urllib.request from email.message import Message +from urllib.parse import urlsplit from scripts.docs_gates import smoke from scripts.docs_gates.conftest import REDIRECT_LOCATION, REDIRECT_PATH @@ -74,9 +75,10 @@ def test_search_mount_present(): assert smoke.search_mount_present('<html><body>no search here</body></html>') is False -# --- Hardening (this change): explicit UA + per-probe retry. Mock at the _OPENER boundary -# (the exact object _probe_once() opens through, mirroring the redirect test above which drives -# smoke._OPENER directly), and shrink RETRY_SLEEP_SECONDS to 0 so retries don't wall-clock. --- +# --- Hardening: explicit UA + round-based retry with an overall deadline. Mock at the _OPENER +# boundary (the object _probe_once() opens through, mirroring the redirect test above which +# drives smoke._OPENER directly). run()-level round tests inject a small plan via probe_plan and +# a path-keyed opener; sleeps are spied (or constants shrunk) so nothing wall-clocks. --- class _FakeResp: @@ -121,6 +123,42 @@ def _http_error(code: int, headers: dict[str, str]) -> urllib.error.HTTPError: return urllib.error.HTTPError("https://host.invalid/x", code, "msg", hdrs, io.BytesIO(b"")) +class _ReadBoom(io.BytesIO): + """A body whose .read() raises — models a transport error DURING HTTPError.read(). + urllib's addbase binds the instance's read to fp.read, so overriding it on the fp is the + reliable way to make e.read() blow up.""" + + def read(self, *args: object) -> bytes: # noqa: D401 + raise OSError("reset during body read") + + +def _http_error_read_boom(code: int) -> urllib.error.HTTPError: + return urllib.error.HTTPError("https://host.invalid/x", code, "msg", Message(), _ReadBoom()) + + +class _PathOpener: + """Opener keyed by request path: each path maps to a queue consumed one item per probe of + that path (item = _FakeResp, or an Exception that is raised). Records probed paths in order, + so round scoping / retry counts are assertable across rounds that re-probe only failures.""" + + def __init__(self, by_path: dict[str, list[object]]): + self._by_path = {p: list(v) for p, v in by_path.items()} + self.calls: list[str] = [] + + def open(self, req: urllib.request.Request, timeout: float | None = None) -> object: + path = urlsplit(req.full_url).path + self.calls.append(path) + item = self._by_path[path].pop(0) + if isinstance(item, Exception): + raise item + return item + + +def _plan(paths: list[str]) -> list[smoke.Probe]: + """Minimal status-only plan (no build/search assertions) for run() round tests.""" + return [smoke.Probe(p, 200, assert_docs_build=False, assert_apex_build=False) for p in paths] + + def test_probe_sends_explicit_user_agent(monkeypatch): opener = _FakeOpener([_FakeResp(200, {"X-Docs-Build": "sha1"})]) monkeypatch.setattr(smoke, "_OPENER", opener) @@ -132,33 +170,96 @@ def test_probe_sends_explicit_user_agent(monkeypatch): assert req.get_header("Cf-access-client-id") == "cf-id" # CF-Access headers still sent -def test_retry_passes_when_second_attempt_ok(monkeypatch): +def test_transport_error_recovers_in_second_round(monkeypatch): + monkeypatch.setattr(smoke, "probe_plan", + lambda slug, pdf, critical: _plan(["/en/rolling/index.html"])) monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0) - opener = _FakeOpener([ - _FakeResp(307, {"X-Docs-Build": "stale"}), # attempt 1: previous-version blip - _FakeResp(200, {"X-Docs-Build": "goodsha"}), # attempt 2: propagation settled - ]) + opener = _PathOpener({ + "/en/rolling/index.html": [OSError("dns hiccup"), _FakeResp(200, {})], + }) monkeypatch.setattr(smoke, "_OPENER", opener) - probe = smoke.Probe("/en/1.5/cli.html", 200, assert_docs_build=True, assert_apex_build=False) - assert smoke._probe_with_retries("host", probe, "goodsha", "id", "sec") is True - assert len(opener.calls) == 2 + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 0 + # round 1 transport error, round 2 success — the same path is re-probed + assert opener.calls == ["/en/rolling/index.html", "/en/rolling/index.html"] -def test_retry_fails_after_exhausting_attempts(monkeypatch): - monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0) - opener = _FakeOpener([_FakeResp(307, {"X-Docs-Build": "stale"}) - for _ in range(smoke.MAX_ATTEMPTS)]) +def test_httperror_read_crash_is_contained_as_failure(monkeypatch): + # agy-critical: a transport error DURING e.read() must not escape as a traceback. + monkeypatch.setattr(smoke, "_OPENER", _FakeOpener([_http_error_read_boom(502)])) + probe = smoke.Probe("/en/rolling/index.html", 200, + assert_docs_build=False, assert_apex_build=False) + ok, status, docs_build, error = smoke._probe_once("host", probe, "sha", "id", "sec") + assert ok is False + assert status is None and docs_build is None + assert error is not None and "reset during body read" in error + + +def test_sleeps_once_per_inter_round_gap_not_per_probe(monkeypatch): + monkeypatch.setattr(smoke, "probe_plan", + lambda slug, pdf, critical: _plan(["/a", "/b", "/c"])) + sleeps: list[float] = [] + monkeypatch.setattr(smoke.time, "sleep", lambda s: sleeps.append(s)) + # /a and /b fail round 1 then pass round 2; /c passes round 1 + opener = _PathOpener({ + "/a": [_FakeResp(500, {}), _FakeResp(200, {})], + "/b": [_FakeResp(500, {}), _FakeResp(200, {})], + "/c": [_FakeResp(200, {})], + }) monkeypatch.setattr(smoke, "_OPENER", opener) - probe = smoke.Probe("/en/1.5/index.html", 200, assert_docs_build=True, assert_apex_build=False) - assert smoke._probe_with_retries("host", probe, "goodsha", "id", "sec") is False - assert len(opener.calls) == smoke.MAX_ATTEMPTS # tried the full budget - - -def test_expected_404_passes_first_attempt_without_retry(monkeypatch): - monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0) - opener = _FakeOpener([_http_error(404, {})]) # 404 delivered as HTTPError, like urllib + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 0 + assert sleeps == [smoke.RETRY_SLEEP_SECONDS] # ONE inter-round sleep despite 2 failing probes + + +def test_second_round_reprobes_only_the_failed_path(monkeypatch): + monkeypatch.setattr(smoke, "probe_plan", + lambda slug, pdf, critical: _plan(["/a", "/b", "/c"])) + monkeypatch.setattr(smoke.time, "sleep", lambda s: None) + opener = _PathOpener({ + "/a": [_FakeResp(200, {})], # passes round 1 + "/b": [_FakeResp(500, {}), _FakeResp(200, {})], # fails r1, passes r2 + "/c": [_FakeResp(200, {})], # passes round 1 + }) monkeypatch.setattr(smoke, "_OPENER", opener) - probe = smoke.Probe("/en/1.5/definitely-missing.html", 404, - assert_docs_build=False, assert_apex_build=False) - assert smoke._probe_with_retries("host", probe, "sha", "id", "sec") is True - assert len(opener.calls) == 1 # a legitimately-expected 404 must NOT burn retries + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 0 + assert opener.calls == ["/a", "/b", "/c", "/b"] # only the failed path is re-probed + + +def test_run_reports_failure_count_and_nonzero_exit(monkeypatch, capsys): + monkeypatch.setattr(smoke, "probe_plan", lambda slug, pdf, critical: _plan(["/a", "/b"])) + monkeypatch.setattr(smoke, "MAX_ROUNDS", 1) # single round, no retries + monkeypatch.setattr(smoke, "_OPENER", _PathOpener({ + "/a": [_FakeResp(200, {})], + "/b": [_FakeResp(500, {})], + })) + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 1 + assert '{"failures": 1}' in capsys.readouterr().out + + +def test_run_reports_clean_and_zero_exit(monkeypatch, capsys): + monkeypatch.setattr(smoke, "probe_plan", lambda slug, pdf, critical: _plan(["/a", "/b"])) + monkeypatch.setattr(smoke, "_OPENER", _PathOpener({ + "/a": [_FakeResp(200, {})], + "/b": [_FakeResp(200, {})], + })) + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 0 + assert '{"failures": 0}' in capsys.readouterr().out + + +def test_deadline_counts_unresolved_as_failed(monkeypatch, capsys): + monkeypatch.setattr(smoke, "probe_plan", lambda slug, pdf, critical: _plan(["/a", "/b"])) + monkeypatch.setattr(smoke, "DEADLINE_SECONDS", 0) # trips before the first probe + opener = _PathOpener({"/a": [_FakeResp(200, {})], "/b": [_FakeResp(200, {})]}) + monkeypatch.setattr(smoke, "_OPENER", opener) + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 1 + captured = capsys.readouterr() + assert "SMOKE-DEADLINE" in captured.err + assert '{"failures": 2}' in captured.out + assert opener.calls == [] # deadline reached before any probe ran + + +def test_probe_plan_dedups_index_html(): + plan = smoke.probe_plan("rolling", None, ["index.html", "cli.html"]) + index_probes = [p for p in plan if p.path == "/en/rolling/index.html"] + assert len(index_probes) == 1 # not duplicated by the critical list + assert index_probes[0].assert_search_mount is True # still the single search-mount probe + assert sum(1 for p in plan if p.path == "/en/rolling/cli.html") == 1 # cli.html preserved |