From 7a3aa29fc010c2604b84d6437ed87053947f3f27 Mon Sep 17 00:00:00 2001 From: Yuriy Andamasov Date: Sun, 10 May 2026 19:11:24 +0300 Subject: ci: extend doc-linter to MyST Markdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Active docs are now MyST `.md`; the linter previously only inspected `.rst` and `.txt`, so ~250 active pages were unchecked for IP usage and line length on every PR. scripts/doc-linter.py: - Add `.md` to the extension filter (use `endswith` for correctness; the prior 4-char slice silently skipped `.md` files). - Track MyST/Markdown fenced code blocks (```` ``` ```` and `:::`) for line-length exemption — same semantics as `.. code-block::` for RST. - Recognize both suppression marker forms: `.. stop_vyoslinter` / `.. start_vyoslinter` (RST and `.txt` includes) and `% stop_vyoslinter` / `% start_vyoslinter` (MyST). Both work in either context; pick the form that matches the surrounding parser. - Replace the brittle `try/finally: fp.close()` with a `with` block — the previous form raised `UnboundLocalError` if `open()` itself failed. - Fix typo `forgett` → `forget`. .github/instructions/rst-linter.instructions.md → doc-linter.instructions.md: - Broaden `applyTo` from `**/*.rst` to `**/*.md,**/*.rst,**/*.txt`. - Document MyST suppression syntax and fenced-code line-length exemption. - Note the parser-form rule for `{eval-rst}` blocks. No regression on `.txt` includes: identical lint output verified against the origin/rolling baseline on a sample of files. Pre-existing IP violations exist in 14 `.md` files (e.g. `configexamples/lac-lns.md` line 95 — a `8.8.8.8` already wrapped in `% stop_vyoslinter`/`% start_vyoslinter`, correctly suppressed). PRs touching unsuppressed violations will start failing CI; this is the intent of enabling the check. 🤖 Generated by [robots](https://vyos.io) --- scripts/doc-linter.py | 133 +++++++++++++++++++++++++++----------------------- 1 file changed, 73 insertions(+), 60 deletions(-) (limited to 'scripts') diff --git a/scripts/doc-linter.py b/scripts/doc-linter.py index 3dc7c2fc..ec00472c 100644 --- a/scripts/doc-linter.py +++ b/scripts/doc-linter.py @@ -27,6 +27,20 @@ MAC = r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})' NUMBER = r"([\s']\d+[\s'])" +SUPPORTED_EXTS = ('.md', '.rst', '.txt') + +# MyST / Markdown fenced code block: leading whitespace + 3+ backticks or 3+ colons. +# Same character and length-or-greater closes. +MD_FENCE_RE = re.compile(r'^(\s*)(`{3,}|:{3,})') + + +def is_suppression_marker(line, kind): + """Detect `.. stop_vyoslinter` (RST/txt) or `% stop_vyoslinter` (MyST).""" + token = f"{kind}_vyoslinter" + if token not in line: + return False + return f".. {token}" in line or f"% {token}" in line + def lint_mac(cnt, line): mac = re.search(MAC, line, re.I) @@ -79,65 +93,64 @@ def lint_linelen(cnt, line): def handle_file_action(filepath): errors = [] - try: - with open(filepath) as fp: - line = fp.readline() - cnt = 1 - test_line_lenght = True - start_vyoslinter = True - indentation = 0 - while line: - # search for ignore linter comments in lines - if ".. stop_vyoslinter" in line: - start_vyoslinter = False - if ".. start_vyoslinter" in line: - start_vyoslinter = True - if start_vyoslinter: - # ignore every '.. code-block::' for line lenght - # rst code-block have its own style in html the format in rst - # and the build page must be the same - if test_line_lenght is False: - if len(line) > indentation: - #print(f"'{line}'") - #print(indentation) - if line[indentation].isspace() is False: - test_line_lenght = True - - if ".. code-block::" in line: - test_line_lenght = False - indentation = 0 - for i in line: - if i.isspace(): - indentation = indentation + 1 - else: - break - - err_mac = lint_mac(cnt, line.strip()) - # disable mac detection for the moment, too many false positives - err_mac = None - err_ip4 = lint_ipv4(cnt, line.strip()) - err_ip6 = lint_ipv6(cnt, line.strip()) - if test_line_lenght: - err_len = lint_linelen(cnt, line) + in_md_fence = False + md_fence_char = None + md_fence_min_len = 0 + in_rst_codeblock = False + rst_codeblock_indent = 0 + start_vyoslinter = True + cnt = 0 + + with open(filepath) as fp: + for cnt, line in enumerate(fp, start=1): + if is_suppression_marker(line, 'stop'): + start_vyoslinter = False + if is_suppression_marker(line, 'start'): + start_vyoslinter = True + + if not start_vyoslinter: + continue + + # MD/MyST fenced code block tracking (``` or :::) + fence_match = MD_FENCE_RE.match(line) + if fence_match: + fence = fence_match.group(2) + fence_char = fence[0] + fence_len = len(fence) + if not in_md_fence: + in_md_fence = True + md_fence_char = fence_char + md_fence_min_len = fence_len + elif fence_char == md_fence_char and fence_len >= md_fence_min_len: + in_md_fence = False + + # RST `.. code-block::` tracking (existing semantics for .rst/.txt). + if in_rst_codeblock: + if len(line) > rst_codeblock_indent and not line[rst_codeblock_indent].isspace(): + in_rst_codeblock = False + if not in_rst_codeblock and ".. code-block::" in line: + in_rst_codeblock = True + rst_codeblock_indent = 0 + for ch in line: + if ch.isspace(): + rst_codeblock_indent += 1 else: - err_len = None - if err_mac: - errors.append(err_mac) - if err_ip4: - errors.append(err_ip4) - if err_ip6: - errors.append(err_ip6) - if err_len: - errors.append(err_len) - - line = fp.readline() - cnt += 1 - - # ensure linter was not stop on top and forgot to tun on again - if start_vyoslinter == False: - errors.append((f"Don't forgett to turn linter back on", cnt, 'error')) - finally: - fp.close() + break + + test_line_length = not (in_md_fence or in_rst_codeblock) + + err_mac = lint_mac(cnt, line.strip()) + # disable mac detection for the moment, too many false positives + err_mac = None + err_ip4 = lint_ipv4(cnt, line.strip()) + err_ip6 = lint_ipv6(cnt, line.strip()) + err_len = lint_linelen(cnt, line) if test_line_length else None + for e in (err_mac, err_ip4, err_ip6, err_len): + if e: + errors.append(e) + + if not start_vyoslinter: + errors.append(("Don't forget to turn linter back on", cnt, 'error')) if len(errors) > 0: ''' @@ -156,14 +169,14 @@ def main(): try: files = ast.literal_eval(sys.argv[1]) for file in files: - if file[-4:] in [".rst", ".txt"] and "_build" not in file: + if file.endswith(SUPPORTED_EXTS) and "_build" not in file: if handle_file_action(file) is False: bool_error = False except Exception as e: for root, dirs, files in os.walk("docs"): path = root.split(os.sep) for file in files: - if file[-4:] in [".rst", ".txt"] and "_build" not in path: + if file.endswith(SUPPORTED_EXTS) and "_build" not in path: fpath = '/'.join(path) filepath = f"{fpath}/{file}" if handle_file_action(filepath) is False: -- cgit v1.2.3 From d2814d1ea147f89081e15d656bc7172018ee537b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 10 May 2026 16:15:47 +0000 Subject: fix: scope vyoslinter markers to real parser contexts Agent-Logs-Url: https://github.com/vyos/vyos-documentation/sessions/5d679560-8a77-4735-b585-74c09293eea5 Co-authored-by: andamasov <12631358+andamasov@users.noreply.github.com> --- .github/instructions/doc-linter.instructions.md | 46 +++++++++++++++----- scripts/doc-linter.py | 57 +++++++++++++++++-------- 2 files changed, 75 insertions(+), 28 deletions(-) (limited to 'scripts') diff --git a/.github/instructions/doc-linter.instructions.md b/.github/instructions/doc-linter.instructions.md index 5e7d359e..cc56f8fe 100644 --- a/.github/instructions/doc-linter.instructions.md +++ b/.github/instructions/doc-linter.instructions.md @@ -4,7 +4,10 @@ applyTo: "**/*.md,**/*.rst,**/*.txt" ## Doc Linter Rules -Every pull request is automatically linted by `scripts/doc-linter.py`. It checks **only changed files** for two things: IP address usage and line length. MyST Markdown (`.md`), legacy RST (`.rst`), and `.txt` includes are all linted. +Every pull request is automatically linted by `scripts/doc-linter.py`. +It checks **only changed files** for two things: IP address usage +and line length. MyST Markdown (`.md`), legacy RST (`.rst`), and +`.txt` includes are all linted. ### IP Address Rules @@ -33,23 +36,29 @@ The linter rejects public IP addresses that are not reserved for documentation. | Example | Why suppression is needed | |---------|--------------------------| -| `8.8.8.8` (Google DNS) | Real public IP, not documentation-reserved | +| Google Public DNS | Real public IP, not documentation-reserved | | `64:ff9b::/96` (NAT64) | Well-known prefix, not in doc ranges | | Real provider IPs in examples | Authenticity matters for the example | ### Line Length Rules -Maximum 80 characters per line. The line-length check is **skipped** inside fenced code: +Maximum 80 characters per line. The line-length check is +**skipped** inside fenced code: -- MyST / Markdown: ` ```...``` ` and `:::...:::` fenced blocks (any info string, including ` ```{eval-rst} `, ` ```{cfgcmd} `, etc.). +- MyST / Markdown: ` ```...``` ` and `:::...:::` fenced blocks + (any info string, including ` ```{eval-rst} `, ` ```{cfgcmd} `, + etc.). - RST: lines indented under `.. code-block::` directives. ### Suppression Syntax -When real public IPs or long lines are unavoidable, wrap the block. The marker form depends on the surrounding parser: +When real public IPs or long lines are unavoidable, wrap the block. +The marker form depends on the surrounding parser: **MyST Markdown (`.md`)** — use the MyST comment form: +% stop_vyoslinter + ````md % stop_vyoslinter @@ -60,8 +69,12 @@ set system name-server '8.8.8.8' % start_vyoslinter ```` +% start_vyoslinter + **RST (`.rst`) and `.txt` includes** — use the RST comment form: +% stop_vyoslinter + ```rst .. stop_vyoslinter @@ -72,19 +85,30 @@ set system name-server '8.8.8.8' .. start_vyoslinter ``` -**Inside a `{eval-rst}` block in MyST** — the embedded content is parsed as RST, so the RST form (`.. stop_vyoslinter`) matches the surrounding parser. The linter recognizes both forms regardless of context, but use the form that fits the parser of the enclosing block. +% start_vyoslinter + +**Inside a `{eval-rst}` block in MyST** — the embedded content is parsed +as RST, so the RST form (`.. stop_vyoslinter`) matches the surrounding +parser. The linter recognizes marker lines only in the parser context +where they apply, including RST markers inside `{eval-rst}` blocks. **Rules for suppression markers:** - Place the stop marker on its own line, immediately before the content. - Place the start marker on its own line, immediately after the content. -- Always re-enable the linter — never leave it stopped for the rest of the file. +- Always re-enable the linter — never leave it stopped for the rest + of the file. - Suppress the smallest possible region. -- Do NOT use suppression for content that can use documentation-reserved addresses instead. +- Do NOT use suppression for content that can use + documentation-reserved addresses instead. ### Common Mistakes -- Removing `stop/start_vyoslinter` markers without fixing the underlying issue (exposes the line to the linter and fails CI). +- Removing `stop/start_vyoslinter` markers without fixing the + underlying issue (exposes the line to the linter and fails CI). - Using real public IPs when documentation addresses would work just as well. -- Adding suppression markers around content that only has private (RFC 1918) addresses or `0.0.0.0/0` — these are allowed and don't need suppression. -- Mixing marker forms (using `% stop_vyoslinter` inside an `{eval-rst}` block, or `.. stop_vyoslinter` in plain MyST prose). +- Adding suppression markers around content that only has private + (RFC 1918) addresses or `0.0.0.0/0` — these are allowed and don't + need suppression. +- Mixing marker forms (using `% stop_vyoslinter` inside an + `{eval-rst}` block, or `.. stop_vyoslinter` in plain MyST prose). diff --git a/scripts/doc-linter.py b/scripts/doc-linter.py index ec00472c..81c9baca 100644 --- a/scripts/doc-linter.py +++ b/scripts/doc-linter.py @@ -31,15 +31,31 @@ SUPPORTED_EXTS = ('.md', '.rst', '.txt') # MyST / Markdown fenced code block: leading whitespace + 3+ backticks or 3+ colons. # Same character and length-or-greater closes. -MD_FENCE_RE = re.compile(r'^(\s*)(`{3,}|:{3,})') - - -def is_suppression_marker(line, kind): - """Detect `.. stop_vyoslinter` (RST/txt) or `% stop_vyoslinter` (MyST).""" - token = f"{kind}_vyoslinter" - if token not in line: +MD_FENCE_RE = re.compile(r'^(\s*)(`{3,}|:{3,})(.*)$') + +SUPPRESSION_MARKER_RE = { + 'rst': { + 'stop': re.compile(r'^\s*\.\.\s+stop_vyoslinter\s*$'), + 'start': re.compile(r'^\s*\.\.\s+start_vyoslinter\s*$'), + }, + 'md': { + 'stop': re.compile(r'^\s*%\s+stop_vyoslinter\s*$'), + 'start': re.compile(r'^\s*%\s+start_vyoslinter\s*$'), + }, +} + + +def is_suppression_marker(line, kind, in_md_fence, in_rst_codeblock, md_fence_is_eval_rst): + """Detect valid stop/start markers in the parser context where they apply.""" + if SUPPRESSION_MARKER_RE['md'][kind].match(line): + return not in_md_fence + if not SUPPRESSION_MARKER_RE['rst'][kind].match(line): return False - return f".. {token}" in line or f"% {token}" in line + if in_rst_codeblock: + return False + if in_md_fence: + return md_fence_is_eval_rst + return True def lint_mac(cnt, line): @@ -94,6 +110,7 @@ def lint_linelen(cnt, line): def handle_file_action(filepath): errors = [] in_md_fence = False + md_fence_is_eval_rst = False md_fence_char = None md_fence_min_len = 0 in_rst_codeblock = False @@ -103,15 +120,7 @@ def handle_file_action(filepath): with open(filepath) as fp: for cnt, line in enumerate(fp, start=1): - if is_suppression_marker(line, 'stop'): - start_vyoslinter = False - if is_suppression_marker(line, 'start'): - start_vyoslinter = True - - if not start_vyoslinter: - continue - - # MD/MyST fenced code block tracking (``` or :::) + # MD/MyST fenced code block tracking (``` or :::). fence_match = MD_FENCE_RE.match(line) if fence_match: fence = fence_match.group(2) @@ -121,8 +130,10 @@ def handle_file_action(filepath): in_md_fence = True md_fence_char = fence_char md_fence_min_len = fence_len + md_fence_is_eval_rst = fence_match.group(3).strip().startswith('{eval-rst}') elif fence_char == md_fence_char and fence_len >= md_fence_min_len: in_md_fence = False + md_fence_is_eval_rst = False # RST `.. code-block::` tracking (existing semantics for .rst/.txt). if in_rst_codeblock: @@ -137,6 +148,18 @@ def handle_file_action(filepath): else: break + if is_suppression_marker( + line, 'stop', in_md_fence, in_rst_codeblock, md_fence_is_eval_rst + ): + start_vyoslinter = False + if is_suppression_marker( + line, 'start', in_md_fence, in_rst_codeblock, md_fence_is_eval_rst + ): + start_vyoslinter = True + + if not start_vyoslinter: + continue + test_line_length = not (in_md_fence or in_rst_codeblock) err_mac = lint_mac(cnt, line.strip()) -- cgit v1.2.3 From 4ef7ffeb2b337b14699e7547b43ba2fe59a00422 Mon Sep 17 00:00:00 2001 From: Yuriy Andamasov Date: Sun, 10 May 2026 21:45:26 +0300 Subject: ci: stack-based fence tracking + file-ext-aware suppression markers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues from PR review: 1. MD/MyST fence tracking treated any longer same-char fence as a closer, which would close `:::{note}` (3 cols) when seeing a nested `::::{code-block}` (4 cols) opener inside it. Real bug in `docs/configuration/interfaces/wireless.md:198–209` (currently unobservable because inner code lines are <80 chars). The "opener has info string / closer has none" heuristic is not sufficient on its own: there are 2,826 bare-fence opens in the tree, so info-string presence cannot distinguish opener from closer. Fix: stack-based tracking. A fence is treated as a closer only when (a) the stack is non-empty, (b) char and length match the top, AND (c) no info string follows. Anything else opens a new (possibly nested) fence. The outermost fence's info string still determines the `md_fence_is_eval_rst` flag. 2. `is_suppression_marker()` accepted `% stop_vyoslinter` in any file outside an MD fence. Per AGENTS.md and the doc-linter instructions, MyST `% ...` markers are only valid in `.md` files; a stray `% stop_vyoslinter` in `.rst`/`.txt` should not silently disable linting. Pass `file_ext` and gate the marker forms accordingly: `% ...` only in `.md` outside fences; `.. ...` in `.rst`/`.txt` outside RST code-blocks, or in `.md` inside an `{eval-rst}` fence. 3. Drop the `not in_rst_codeblock` guard on `.. code-block::` detection. Each occurrence resets the tracked indent (matches `origin/rolling` baseline). Without this, code-block-inside- code-block kept the outer indent and broke dedent detection (verified regression: `_rst_legacy/configuration/system/ rst-syslog.rst:216` long-line warning was lost; restored). Verified: - All 7 original synthetic fixtures pass. - New fixture `nested.md` (3-col outer wraps 4-col inner with long line in between fences) produces exactly one warning at the line outside both fences. - New fixture `wrongmarker.rst` (`%` in `.rst`) — IP error fires (marker correctly ignored). - Full-tree run vs origin/rolling baseline: zero regressions on pre-existing `.rst`/`.txt` warnings; all new output is `.md`. 🤖 Generated by [robots](https://vyos.io) --- scripts/doc-linter.py | 64 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 18 deletions(-) (limited to 'scripts') diff --git a/scripts/doc-linter.py b/scripts/doc-linter.py index 81c9baca..5ccff488 100644 --- a/scripts/doc-linter.py +++ b/scripts/doc-linter.py @@ -45,17 +45,25 @@ SUPPRESSION_MARKER_RE = { } -def is_suppression_marker(line, kind, in_md_fence, in_rst_codeblock, md_fence_is_eval_rst): - """Detect valid stop/start markers in the parser context where they apply.""" +def is_suppression_marker(line, kind, in_md_fence, in_rst_codeblock, + md_fence_is_eval_rst, file_ext): + """Detect valid stop/start markers in the parser context where they apply. + + `% ...` is only valid in MyST Markdown (`.md`) at top level (outside any fence). + `.. ...` is valid in `.rst`/`.txt` at top level (outside RST code-block) and + in `.md` only when inside an `{eval-rst}` fence. + """ if SUPPRESSION_MARKER_RE['md'][kind].match(line): - return not in_md_fence + return file_ext == '.md' and not in_md_fence if not SUPPRESSION_MARKER_RE['rst'][kind].match(line): return False if in_rst_codeblock: return False + if file_ext in ('.rst', '.txt'): + return True if in_md_fence: return md_fence_is_eval_rst - return True + return False def lint_mac(cnt, line): @@ -109,10 +117,13 @@ def lint_linelen(cnt, line): def handle_file_action(filepath): errors = [] - in_md_fence = False + file_ext = os.path.splitext(filepath)[1].lower() + # Stack of open MD/MyST fences: (char, min_len). Supports nesting like + # `:::{note}` containing `::::{code-block}`, where the inner opener does + # not close the outer note (CommonMark closing rule: matching closer must + # have no info string after the fence chars). + md_fence_stack = [] md_fence_is_eval_rst = False - md_fence_char = None - md_fence_min_len = 0 in_rst_codeblock = False rst_codeblock_indent = 0 start_vyoslinter = True @@ -126,20 +137,35 @@ def handle_file_action(filepath): fence = fence_match.group(2) fence_char = fence[0] fence_len = len(fence) - if not in_md_fence: - in_md_fence = True - md_fence_char = fence_char - md_fence_min_len = fence_len - md_fence_is_eval_rst = fence_match.group(3).strip().startswith('{eval-rst}') - elif fence_char == md_fence_char and fence_len >= md_fence_min_len: - in_md_fence = False - md_fence_is_eval_rst = False + info = fence_match.group(3).strip() + # A closer matches the top of the stack (same char, len >= + # opener) and has no info string. Anything else opens a new + # (possibly nested) fence. + is_closer = ( + md_fence_stack + and not info + and fence_char == md_fence_stack[-1][0] + and fence_len >= md_fence_stack[-1][1] + ) + if is_closer: + md_fence_stack.pop() + if not md_fence_stack: + md_fence_is_eval_rst = False + else: + if not md_fence_stack: + md_fence_is_eval_rst = info.startswith('{eval-rst}') + md_fence_stack.append((fence_char, fence_len)) + + in_md_fence = bool(md_fence_stack) # RST `.. code-block::` tracking (existing semantics for .rst/.txt). + # Each `.. code-block::` directive resets the tracked indent so a + # later dedent past that column exits the block — even when the + # directive itself appears inside an already-open outer block. if in_rst_codeblock: if len(line) > rst_codeblock_indent and not line[rst_codeblock_indent].isspace(): in_rst_codeblock = False - if not in_rst_codeblock and ".. code-block::" in line: + if ".. code-block::" in line: in_rst_codeblock = True rst_codeblock_indent = 0 for ch in line: @@ -149,11 +175,13 @@ def handle_file_action(filepath): break if is_suppression_marker( - line, 'stop', in_md_fence, in_rst_codeblock, md_fence_is_eval_rst + line, 'stop', in_md_fence, in_rst_codeblock, + md_fence_is_eval_rst, file_ext, ): start_vyoslinter = False if is_suppression_marker( - line, 'start', in_md_fence, in_rst_codeblock, md_fence_is_eval_rst + line, 'start', in_md_fence, in_rst_codeblock, + md_fence_is_eval_rst, file_ext, ): start_vyoslinter = True -- cgit v1.2.3