diff options
| author | Yuriy Andamasov <yuriy@vyos.io> | 2026-07-22 17:27:20 +0300 |
|---|---|---|
| committer | Yuriy Andamasov <yuriy@vyos.io> | 2026-07-22 17:27:20 +0300 |
| commit | f1dcd5ac53decfb3b11c7eb281ddfb3ea4c94d12 (patch) | |
| tree | f9487f63167d25738e30b6636d6bc3bd6a9dbd90 /scripts/docs_gates/test_smoke.py | |
| parent | 2c696df25168c654cf6af657e4407ac7fae27c75 (diff) | |
| download | vyos-documentation-f1dcd5ac53decfb3b11c7eb281ddfb3ea4c94d12.tar.gz vyos-documentation-f1dcd5ac53decfb3b11c7eb281ddfb3ea4c94d12.zip | |
docs-gates: round-based smoke retries + deadline; contain HTTPError read crash; dedup index probe (review round 1)
Adversarial round (Codex + agy, both REQUEST CHANGES) on the per-probe retry
model shipped in the prior commit — reworked:
Round-based retries (both providers' critical): probe the whole plan once, then
re-probe ONLY the still-failing probes each round (up to MAX_ROUNDS=3, one
RETRY_SLEEP_SECONDS=20 gap between rounds). A probe passing in any round passes.
This keeps the full per-probe failure enumeration (diagnostic value) that a
fail-fast retry would lose, while bounding added time to at most 2 sleeps.
DEADLINE_SECONDS=480 (time.monotonic from run() start, checked before each probe
AND before each inter-round sleep) caps total wall-clock; on breach a single
SMOKE-DEADLINE line is logged and every unresolved probe counts as failed.
Intermediate not-ok logs "SMOKE-RETRY <path>: round <n> ..."; the JSON
{"failures": n} summary and exit contract are unchanged.
Contain HTTPError read crash (agy critical): a transport error DURING e.read()
inside the HTTPError branch previously escaped the outer catch and crashed the
gate. _probe_once now nests the open/HTTPError handling so ANY exception on the
open OR body-read path yields a retryable transport-error result, never a
traceback.
Dedup index probe (agy): critical-pages.txt lists index.html, so
/en/<slug>/index.html was probed twice. probe_plan now filters index.html out of
the critical list; plan[0] stays the single index (and sole search-mount) probe.
ua-policy.json intentionally left unchanged (pushback recorded: fail-open plus
block-precedence make an allow entry non-protective).
Tests reworked for round semantics: transport-error recovery across rounds,
HTTPError-read containment, one-sleep-per-inter-round-gap spy, round scoping
(only the failed path re-probed), run() JSON + exit contract, zero-deadline
path, and index-probe dedup.
🤖 Generated by [robots](https://vyos.io)
Diffstat (limited to 'scripts/docs_gates/test_smoke.py')
| -rw-r--r-- | scripts/docs_gates/test_smoke.py | 155 |
1 files changed, 128 insertions, 27 deletions
diff --git a/scripts/docs_gates/test_smoke.py b/scripts/docs_gates/test_smoke.py index f5967be7..063b3a7a 100644 --- a/scripts/docs_gates/test_smoke.py +++ b/scripts/docs_gates/test_smoke.py @@ -4,6 +4,7 @@ import io import urllib.error import urllib.request from email.message import Message +from urllib.parse import urlsplit from scripts.docs_gates import smoke from scripts.docs_gates.conftest import REDIRECT_LOCATION, REDIRECT_PATH @@ -74,9 +75,10 @@ def test_search_mount_present(): assert smoke.search_mount_present('<html><body>no search here</body></html>') is False -# --- Hardening (this change): explicit UA + per-probe retry. Mock at the _OPENER boundary -# (the exact object _probe_once() opens through, mirroring the redirect test above which drives -# smoke._OPENER directly), and shrink RETRY_SLEEP_SECONDS to 0 so retries don't wall-clock. --- +# --- Hardening: explicit UA + round-based retry with an overall deadline. Mock at the _OPENER +# boundary (the object _probe_once() opens through, mirroring the redirect test above which +# drives smoke._OPENER directly). run()-level round tests inject a small plan via probe_plan and +# a path-keyed opener; sleeps are spied (or constants shrunk) so nothing wall-clocks. --- class _FakeResp: @@ -121,6 +123,42 @@ def _http_error(code: int, headers: dict[str, str]) -> urllib.error.HTTPError: return urllib.error.HTTPError("https://host.invalid/x", code, "msg", hdrs, io.BytesIO(b"")) +class _ReadBoom(io.BytesIO): + """A body whose .read() raises — models a transport error DURING HTTPError.read(). + urllib's addbase binds the instance's read to fp.read, so overriding it on the fp is the + reliable way to make e.read() blow up.""" + + def read(self, *args: object) -> bytes: # noqa: D401 + raise OSError("reset during body read") + + +def _http_error_read_boom(code: int) -> urllib.error.HTTPError: + return urllib.error.HTTPError("https://host.invalid/x", code, "msg", Message(), _ReadBoom()) + + +class _PathOpener: + """Opener keyed by request path: each path maps to a queue consumed one item per probe of + that path (item = _FakeResp, or an Exception that is raised). Records probed paths in order, + so round scoping / retry counts are assertable across rounds that re-probe only failures.""" + + def __init__(self, by_path: dict[str, list[object]]): + self._by_path = {p: list(v) for p, v in by_path.items()} + self.calls: list[str] = [] + + def open(self, req: urllib.request.Request, timeout: float | None = None) -> object: + path = urlsplit(req.full_url).path + self.calls.append(path) + item = self._by_path[path].pop(0) + if isinstance(item, Exception): + raise item + return item + + +def _plan(paths: list[str]) -> list[smoke.Probe]: + """Minimal status-only plan (no build/search assertions) for run() round tests.""" + return [smoke.Probe(p, 200, assert_docs_build=False, assert_apex_build=False) for p in paths] + + def test_probe_sends_explicit_user_agent(monkeypatch): opener = _FakeOpener([_FakeResp(200, {"X-Docs-Build": "sha1"})]) monkeypatch.setattr(smoke, "_OPENER", opener) @@ -132,33 +170,96 @@ def test_probe_sends_explicit_user_agent(monkeypatch): assert req.get_header("Cf-access-client-id") == "cf-id" # CF-Access headers still sent -def test_retry_passes_when_second_attempt_ok(monkeypatch): +def test_transport_error_recovers_in_second_round(monkeypatch): + monkeypatch.setattr(smoke, "probe_plan", + lambda slug, pdf, critical: _plan(["/en/rolling/index.html"])) monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0) - opener = _FakeOpener([ - _FakeResp(307, {"X-Docs-Build": "stale"}), # attempt 1: previous-version blip - _FakeResp(200, {"X-Docs-Build": "goodsha"}), # attempt 2: propagation settled - ]) + opener = _PathOpener({ + "/en/rolling/index.html": [OSError("dns hiccup"), _FakeResp(200, {})], + }) monkeypatch.setattr(smoke, "_OPENER", opener) - probe = smoke.Probe("/en/1.5/cli.html", 200, assert_docs_build=True, assert_apex_build=False) - assert smoke._probe_with_retries("host", probe, "goodsha", "id", "sec") is True - assert len(opener.calls) == 2 + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 0 + # round 1 transport error, round 2 success — the same path is re-probed + assert opener.calls == ["/en/rolling/index.html", "/en/rolling/index.html"] -def test_retry_fails_after_exhausting_attempts(monkeypatch): - monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0) - opener = _FakeOpener([_FakeResp(307, {"X-Docs-Build": "stale"}) - for _ in range(smoke.MAX_ATTEMPTS)]) +def test_httperror_read_crash_is_contained_as_failure(monkeypatch): + # agy-critical: a transport error DURING e.read() must not escape as a traceback. + monkeypatch.setattr(smoke, "_OPENER", _FakeOpener([_http_error_read_boom(502)])) + probe = smoke.Probe("/en/rolling/index.html", 200, + assert_docs_build=False, assert_apex_build=False) + ok, status, docs_build, error = smoke._probe_once("host", probe, "sha", "id", "sec") + assert ok is False + assert status is None and docs_build is None + assert error is not None and "reset during body read" in error + + +def test_sleeps_once_per_inter_round_gap_not_per_probe(monkeypatch): + monkeypatch.setattr(smoke, "probe_plan", + lambda slug, pdf, critical: _plan(["/a", "/b", "/c"])) + sleeps: list[float] = [] + monkeypatch.setattr(smoke.time, "sleep", lambda s: sleeps.append(s)) + # /a and /b fail round 1 then pass round 2; /c passes round 1 + opener = _PathOpener({ + "/a": [_FakeResp(500, {}), _FakeResp(200, {})], + "/b": [_FakeResp(500, {}), _FakeResp(200, {})], + "/c": [_FakeResp(200, {})], + }) monkeypatch.setattr(smoke, "_OPENER", opener) - probe = smoke.Probe("/en/1.5/index.html", 200, assert_docs_build=True, assert_apex_build=False) - assert smoke._probe_with_retries("host", probe, "goodsha", "id", "sec") is False - assert len(opener.calls) == smoke.MAX_ATTEMPTS # tried the full budget - - -def test_expected_404_passes_first_attempt_without_retry(monkeypatch): - monkeypatch.setattr(smoke, "RETRY_SLEEP_SECONDS", 0) - opener = _FakeOpener([_http_error(404, {})]) # 404 delivered as HTTPError, like urllib + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 0 + assert sleeps == [smoke.RETRY_SLEEP_SECONDS] # ONE inter-round sleep despite 2 failing probes + + +def test_second_round_reprobes_only_the_failed_path(monkeypatch): + monkeypatch.setattr(smoke, "probe_plan", + lambda slug, pdf, critical: _plan(["/a", "/b", "/c"])) + monkeypatch.setattr(smoke.time, "sleep", lambda s: None) + opener = _PathOpener({ + "/a": [_FakeResp(200, {})], # passes round 1 + "/b": [_FakeResp(500, {}), _FakeResp(200, {})], # fails r1, passes r2 + "/c": [_FakeResp(200, {})], # passes round 1 + }) monkeypatch.setattr(smoke, "_OPENER", opener) - probe = smoke.Probe("/en/1.5/definitely-missing.html", 404, - assert_docs_build=False, assert_apex_build=False) - assert smoke._probe_with_retries("host", probe, "sha", "id", "sec") is True - assert len(opener.calls) == 1 # a legitimately-expected 404 must NOT burn retries + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 0 + assert opener.calls == ["/a", "/b", "/c", "/b"] # only the failed path is re-probed + + +def test_run_reports_failure_count_and_nonzero_exit(monkeypatch, capsys): + monkeypatch.setattr(smoke, "probe_plan", lambda slug, pdf, critical: _plan(["/a", "/b"])) + monkeypatch.setattr(smoke, "MAX_ROUNDS", 1) # single round, no retries + monkeypatch.setattr(smoke, "_OPENER", _PathOpener({ + "/a": [_FakeResp(200, {})], + "/b": [_FakeResp(500, {})], + })) + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 1 + assert '{"failures": 1}' in capsys.readouterr().out + + +def test_run_reports_clean_and_zero_exit(monkeypatch, capsys): + monkeypatch.setattr(smoke, "probe_plan", lambda slug, pdf, critical: _plan(["/a", "/b"])) + monkeypatch.setattr(smoke, "_OPENER", _PathOpener({ + "/a": [_FakeResp(200, {})], + "/b": [_FakeResp(200, {})], + })) + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 0 + assert '{"failures": 0}' in capsys.readouterr().out + + +def test_deadline_counts_unresolved_as_failed(monkeypatch, capsys): + monkeypatch.setattr(smoke, "probe_plan", lambda slug, pdf, critical: _plan(["/a", "/b"])) + monkeypatch.setattr(smoke, "DEADLINE_SECONDS", 0) # trips before the first probe + opener = _PathOpener({"/a": [_FakeResp(200, {})], "/b": [_FakeResp(200, {})]}) + monkeypatch.setattr(smoke, "_OPENER", opener) + assert smoke.run("host", "rolling", "sha", "id", "sec", None, []) == 1 + captured = capsys.readouterr() + assert "SMOKE-DEADLINE" in captured.err + assert '{"failures": 2}' in captured.out + assert opener.calls == [] # deadline reached before any probe ran + + +def test_probe_plan_dedups_index_html(): + plan = smoke.probe_plan("rolling", None, ["index.html", "cli.html"]) + index_probes = [p for p in plan if p.path == "/en/rolling/index.html"] + assert len(index_probes) == 1 # not duplicated by the critical list + assert index_probes[0].assert_search_mount is True # still the single search-mount probe + assert sum(1 for p in plan if p.path == "/en/rolling/cli.html") == 1 # cli.html preserved |