From b73f17d61c79475c2e741f62455bb5e90ad1e287 Mon Sep 17 00:00:00 2001 From: Yuriy Andamasov Date: Mon, 27 Apr 2026 04:39:27 +0300 Subject: feat: add swap_sources.py for incremental RST-to-MyST migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-build swap/restore script that renames md-{name}.md → {name}.md before Sphinx builds and restores after. Includes state tracking, exclude file generation, collision detection, and partial-failure rollback. 10 tests cover all specified behaviors plus rollback path. Co-Authored-By: Claude Sonnet 4.6 --- scripts/swap_sources.py | 310 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 scripts/swap_sources.py (limited to 'scripts') diff --git a/scripts/swap_sources.py b/scripts/swap_sources.py new file mode 100644 index 00000000..f983dbfa --- /dev/null +++ b/scripts/swap_sources.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""swap_sources.py — Pre-build swap/restore for incremental RST-to-MyST migration. + +Usage +----- + python scripts/swap_sources.py --swap # rename md-{name}.md → {name}.md + python scripts/swap_sources.py --restore # undo renames + python scripts/swap_sources.py --dry-run # show what would be swapped + python scripts/swap_sources.py --status # show current swap state + python scripts/swap_sources.py --swap --docs-dir /path/to/docs + +Swap list +--------- + docs/_swap.txt — one stem per line, relative to docs/, comments (#) skipped. + Example: + # migrated pages + configuration/firewall/zone + quick-start + +State files (in docs/_build/, gitignored) +----------------------------------------- + _swap_state.json — records completed renames + _swap_exclude.txt — RST paths to pass to Sphinx exclude_patterns +""" + +import argparse +import json +import os +import sys +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +STATE_FILE = "_swap_state.json" +EXCLUDE_FILE = "_swap_exclude.txt" +SWAP_LIST_FILE = "_swap.txt" +STATE_VERSION = 1 + + +# --------------------------------------------------------------------------- +# Core helpers +# --------------------------------------------------------------------------- + +def parse_swap_list(path: Path) -> list: + """Read *path* and return a list of stems (comments/blanks skipped). + + Returns [] if the file does not exist. + """ + if not path.exists(): + return [] + stems = [] + for line in path.read_text().splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + stems.append(stripped) + return stems + + +def _build_dir(docs_dir: Path) -> Path: + build = docs_dir / "_build" + build.mkdir(parents=True, exist_ok=True) + return build + + +def _state_path(docs_dir: Path) -> Path: + return docs_dir / "_build" / STATE_FILE + + +def _exclude_path(docs_dir: Path) -> Path: + return docs_dir / "_build" / EXCLUDE_FILE + + +def _resolve_stem_paths(docs_dir: Path, stem: str): + """Return (md_from, md_to, rst_path) as absolute Paths for a given stem.""" + p = Path(stem) + name = p.name + parent = p.parent # may be PosixPath('.') + if str(parent) == ".": + target_dir = docs_dir + else: + target_dir = docs_dir / parent + + md_from = target_dir / f"md-{name}.md" # source: md-prefixed + md_to = target_dir / f"{name}.md" # destination: plain .md + rst_path = target_dir / f"{name}.rst" # RST that will be excluded + return md_from, md_to, rst_path + + +# --------------------------------------------------------------------------- +# do_swap +# --------------------------------------------------------------------------- + +def do_swap(docs_dir: Path) -> None: + """Rename md-{name}.md → {name}.md for each stem in _swap.txt. + + Validates collisions, records state, writes exclude file. + Rolls back all completed renames on any failure. + """ + docs_dir = Path(docs_dir) + swap_list = parse_swap_list(docs_dir / SWAP_LIST_FILE) + if not swap_list: + print("swap_sources: no stems in swap list, nothing to do.", file=sys.stderr) + return + + # Stale state: warn and auto-restore first + state_path = _state_path(docs_dir) + if state_path.exists(): + print( + "swap_sources: WARNING — stale swap state detected. Auto-restoring before new swap.", + file=sys.stderr, + ) + do_restore(docs_dir) + + # Validate all stems before touching the filesystem + planned = [] # list of (md_from, md_to, rst_path, stem) + for stem in swap_list: + md_from, md_to, rst_path = _resolve_stem_paths(docs_dir, stem) + + if not md_from.exists(): + print( + f"swap_sources: skipping {stem!r} — md- source file not found: {md_from}", + file=sys.stderr, + ) + continue + + if not rst_path.exists(): + print( + f"swap_sources: skipping {stem!r} — RST file not found: {rst_path}", + file=sys.stderr, + ) + continue + + if md_to.exists(): + raise RuntimeError( + f"collision: both RST and unprefixed .md exist for stem {stem!r}. " + f"Remove {md_to} before swapping." + ) + + planned.append((md_from, md_to, rst_path, stem)) + + # Execute renames with rollback on failure + completed = [] # list of (md_from, md_to) that succeeded + try: + for md_from, md_to, rst_path, stem in planned: + os.rename(md_from, md_to) + completed.append((md_from, md_to)) + except Exception as exc: + # Rollback completed renames in reverse order + for from_path, to_path in reversed(completed): + try: + os.rename(to_path, from_path) + except Exception as rollback_exc: + print( + f"swap_sources: ROLLBACK ERROR for {to_path}: {rollback_exc}", + file=sys.stderr, + ) + raise RuntimeError(f"swap_sources: rename failed, rolled back. Cause: {exc}") from exc + + if not completed: + return + + # Build state and exclude data using relative paths (relative to docs_dir) + _build_dir(docs_dir) + swaps = [] + exclude_lines = [] + for md_from, md_to, rst_path, stem in planned: + rel_from = md_from.relative_to(docs_dir) + rel_to = md_to.relative_to(docs_dir) + rel_rst = rst_path.relative_to(docs_dir) + swaps.append({ + "stem": stem, + "md_from": str(rel_from), + "md_to": str(rel_to), + "rst_excluded": str(rel_rst), + }) + exclude_lines.append(str(rel_rst)) + + state = {"version": STATE_VERSION, "swaps": swaps} + state_path.write_text(json.dumps(state, indent=2)) + _exclude_path(docs_dir).write_text("\n".join(exclude_lines) + "\n") + + print(f"swap_sources: swapped {len(completed)} file(s).", file=sys.stderr) + + +# --------------------------------------------------------------------------- +# do_restore +# --------------------------------------------------------------------------- + +def do_restore(docs_dir: Path) -> None: + """Undo all renames recorded in _swap_state.json (reverse order).""" + docs_dir = Path(docs_dir) + state_path = _state_path(docs_dir) + + if not state_path.exists(): + return # no-op + + state = json.loads(state_path.read_text()) + swaps = state.get("swaps", []) + + for entry in reversed(swaps): + md_from = docs_dir / entry["md_from"] # original source (md-prefixed) + md_to = docs_dir / entry["md_to"] # current location (plain .md) + if md_to.exists(): + os.rename(md_to, md_from) + else: + print( + f"swap_sources: WARNING — expected swapped file not found: {md_to}", + file=sys.stderr, + ) + + # Clean up state and exclude files + state_path.unlink(missing_ok=True) + exclude_path = _exclude_path(docs_dir) + exclude_path.unlink(missing_ok=True) + + print(f"swap_sources: restored {len(swaps)} file(s).", file=sys.stderr) + + +# --------------------------------------------------------------------------- +# do_dry_run / do_status +# --------------------------------------------------------------------------- + +def do_dry_run(docs_dir: Path) -> None: + """Print what would be swapped without making any changes.""" + docs_dir = Path(docs_dir) + swap_list = parse_swap_list(docs_dir / SWAP_LIST_FILE) + if not swap_list: + print("(dry-run) No stems in swap list.") + return + + for stem in swap_list: + md_from, md_to, rst_path = _resolve_stem_paths(docs_dir, stem) + issues = [] + if not md_from.exists(): + issues.append("md- source missing") + if not rst_path.exists(): + issues.append("RST missing") + if md_to.exists(): + issues.append("COLLISION: plain .md already exists") + if issues: + print(f" SKIP {stem}: {', '.join(issues)}") + else: + print(f" SWAP {md_from.name} → {md_to.name} (in {md_from.parent.relative_to(docs_dir) if md_from.parent != docs_dir else '.'})") + + +def do_status(docs_dir: Path) -> None: + """Show current swap state.""" + docs_dir = Path(docs_dir) + state_path = _state_path(docs_dir) + if not state_path.exists(): + print("swap_sources: no active swap state.") + return + + state = json.loads(state_path.read_text()) + swaps = state.get("swaps", []) + print(f"swap_sources: {len(swaps)} file(s) currently swapped:") + for entry in swaps: + print(f" {entry['md_from']} → {entry['md_to']} (excludes {entry['rst_excluded']})") + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Swap md-prefixed MyST files in place of RST files before Sphinx builds." + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--swap", action="store_true", help="Perform swap") + group.add_argument("--restore", action="store_true", help="Restore original files") + group.add_argument("--dry-run", action="store_true", help="Show what would be swapped") + group.add_argument("--status", action="store_true", help="Show current swap state") + + parser.add_argument( + "--docs-dir", + type=Path, + default=None, + help="Path to docs/ directory (default: auto-detected from script location)", + ) + + args = parser.parse_args(argv) + + if args.docs_dir is None: + # Default: repo_root/docs where repo_root = parent of scripts/ + repo_root = Path(__file__).resolve().parent.parent + docs_dir = repo_root / "docs" + else: + docs_dir = args.docs_dir.resolve() + + if not docs_dir.is_dir(): + parser.error(f"docs directory not found: {docs_dir}") + + if args.swap: + import atexit + atexit.register(do_restore, docs_dir) + do_swap(docs_dir) + elif args.restore: + do_restore(docs_dir) + elif args.dry_run: + do_dry_run(docs_dir) + elif args.status: + do_status(docs_dir) + + +if __name__ == "__main__": + main() -- cgit v1.2.3 From 1396d89137d5a6e6702a5696f9ea9fe63b5a8366 Mon Sep 17 00:00:00 2001 From: Yuriy Andamasov Date: Mon, 27 Apr 2026 04:56:04 +0300 Subject: feat: add import_myst.py for importing MyST files from myst/* branches Adds scripts/import_myst.py with import_page, git_show, list_myst_files, list_rst_files, and do_import. Imported files are written as md-{name}.md alongside existing RST files; importing is decoupled from swap activation. Adds tests/test_import_myst.py covering single-page write, identical-skip, warn-on-different-without-force, force-overwrite, and nested-path creation. All 5 tests pass on Python 3.9. Co-Authored-By: Claude Sonnet 4.6 --- scripts/import_myst.py | 246 ++++++++++++++++++++++++++++++++++++++++++++++ tests/test_import_myst.py | 101 +++++++++++++++++++ 2 files changed, 347 insertions(+) create mode 100644 scripts/import_myst.py create mode 100644 tests/test_import_myst.py (limited to 'scripts') diff --git a/scripts/import_myst.py b/scripts/import_myst.py new file mode 100644 index 00000000..88ba7006 --- /dev/null +++ b/scripts/import_myst.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""import_myst.py — Import converted MD files from myst/* branches. + +Usage +----- + python scripts/import_myst.py --source myst/current --all + python scripts/import_myst.py --source myst/current --pages quick-start configuration/firewall/zone + python scripts/import_myst.py --source myst/current --all --force + python scripts/import_myst.py --source myst/current --all --dry-run + +Each imported file is written as md-{name}.md alongside the existing RST +file. Importing does not activate a page — use swap_sources.py for that. +""" + +import argparse +import subprocess +import sys +import warnings +from pathlib import Path +from typing import Optional + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parent.parent +DOCS_DIR = REPO_ROOT / "docs" + + +# --------------------------------------------------------------------------- +# Core helpers +# --------------------------------------------------------------------------- + +def git_show(ref: str, path: str) -> Optional[bytes]: + """Run `git show {ref}:{path}` and return raw bytes, or None on error.""" + result = subprocess.run( + ["git", "show", f"{ref}:{path}"], + capture_output=True, + cwd=REPO_ROOT, + ) + if result.returncode != 0: + return None + return result.stdout + + +def list_myst_files(source: str) -> list[str]: + """Return list of MD stems from `git ls-tree -r --name-only {source} -- docs/`. + + Stems are relative to docs/ with the .md extension stripped. + Only .md files are returned. + """ + result = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", source, "--", "docs/"], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ) + if result.returncode != 0: + return [] + + stems = [] + for line in result.stdout.splitlines(): + line = line.strip() + if not line.endswith(".md"): + continue + # Strip leading "docs/" prefix and .md extension + rel = line[len("docs/"):] if line.startswith("docs/") else line + stem = rel[:-3] # strip ".md" + stems.append(stem) + return stems + + +def list_rst_files(docs_dir: Path) -> set[str]: + """Return set of stems (relative to docs_dir, no extension) for all *.rst files.""" + stems = set() + for rst in docs_dir.rglob("*.rst"): + rel = rst.relative_to(docs_dir) + stem = str(rel)[:-4] # strip ".rst" + stems.add(stem) + return stems + + +def import_page(stem: str, md_content: bytes, docs_dir: Path, force: bool) -> bool: + """Write md-{name}.md for the given stem. + + - Creates parent dirs as needed. + - Skips (returns False) if identical content already exists. + - Without --force: warns and skips (returns False) if different content exists. + - With --force: overwrites (returns True). + - Returns True on successful write. + """ + p = Path(stem) + name = p.name + parent = p.parent + if str(parent) == ".": + target_dir = docs_dir + else: + target_dir = docs_dir / parent + + dest = target_dir / f"md-{name}.md" + + if dest.exists(): + existing = dest.read_bytes() + if existing == md_content: + return False # identical — skip silently + if not force: + warnings.warn( + f"import_myst: {dest} exists with different content; " + f"use --force to overwrite", + stacklevel=2, + ) + return False + # force=True — fall through to write + + target_dir.mkdir(parents=True, exist_ok=True) + dest.write_bytes(md_content) + return True + + +# --------------------------------------------------------------------------- +# do_import +# --------------------------------------------------------------------------- + +def do_import( + source: str, + pages: Optional[list], + all_pages: bool, + force: bool, + dry_run: bool, + docs_dir: Path, +) -> None: + """Orchestrate import from source branch. + + If --all: list MD files from source, intersect with RST stems in docs_dir, + import only those that have an RST counterpart. + If --pages: import only the listed stems. + """ + docs_dir = Path(docs_dir) + + if all_pages: + myst_stems = list_myst_files(source) + rst_stems = list_rst_files(docs_dir) + candidate_stems = [s for s in myst_stems if s in rst_stems] + else: + candidate_stems = list(pages or []) + + imported = 0 + skipped = 0 + + for stem in candidate_stems: + # Path inside the branch: docs/{stem}.md + ref_path = f"docs/{stem}.md" + content = git_show(source, ref_path) + if content is None: + print( + f"import_myst: WARNING — could not fetch {source}:{ref_path}", + file=sys.stderr, + ) + skipped += 1 + continue + + if dry_run: + print(f" (dry-run) would import {stem}") + imported += 1 + continue + + wrote = import_page(stem, content, docs_dir, force) + if wrote: + imported += 1 + else: + skipped += 1 + + print( + f"import_myst: imported {imported}, skipped {skipped}.", + file=sys.stderr, + ) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Import MyST MD files from a myst/* branch with md- prefix." + ) + parser.add_argument( + "--source", + required=True, + help="Git ref to import from (e.g. myst/current)", + ) + + pages_group = parser.add_mutually_exclusive_group(required=True) + pages_group.add_argument( + "--pages", + nargs="+", + metavar="STEM", + help="Specific page stems to import (e.g. quick-start configuration/firewall/zone)", + ) + pages_group.add_argument( + "--all", + action="store_true", + dest="all_pages", + help="Import all MD files that have a matching RST counterpart", + ) + + parser.add_argument( + "--force", + action="store_true", + help="Overwrite existing md- files even if content differs", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print what would be imported without writing files", + ) + parser.add_argument( + "--docs-dir", + type=Path, + default=None, + help="Path to docs/ directory (default: auto-detected from script location)", + ) + + args = parser.parse_args(argv) + + if args.docs_dir is None: + docs_dir = DOCS_DIR + else: + docs_dir = args.docs_dir.resolve() + + if not docs_dir.is_dir(): + parser.error(f"docs directory not found: {docs_dir}") + + do_import( + source=args.source, + pages=args.pages, + all_pages=args.all_pages, + force=args.force, + dry_run=args.dry_run, + docs_dir=docs_dir, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_import_myst.py b/tests/test_import_myst.py new file mode 100644 index 00000000..1d975b45 --- /dev/null +++ b/tests/test_import_myst.py @@ -0,0 +1,101 @@ +"""Tests for import_myst.py — run with: + cd tests && PYTHONPATH=../scripts python3 -m pytest test_import_myst.py -v +""" +import warnings +from pathlib import Path + +import pytest + + +# --------------------------------------------------------------------------- +# import_page tests (no git dependency) +# --------------------------------------------------------------------------- + +def test_import_single_page(tmp_path): + """Creates md-{name}.md with the given content.""" + from import_myst import import_page + + docs = tmp_path / "docs" + docs.mkdir() + content = b"# quick-start\nHello MyST\n" + + result = import_page("quick-start", content, docs, force=False) + + assert result is True + dest = docs / "md-quick-start.md" + assert dest.exists() + assert dest.read_bytes() == content + + +def test_import_skip_identical(tmp_path): + """Returns False and does not modify the file when content is identical.""" + from import_myst import import_page + + docs = tmp_path / "docs" + docs.mkdir() + content = b"# quick-start\nIdentical content\n" + dest = docs / "md-quick-start.md" + dest.write_bytes(content) + mtime_before = dest.stat().st_mtime_ns + + result = import_page("quick-start", content, docs, force=False) + + assert result is False + assert dest.read_bytes() == content + assert dest.stat().st_mtime_ns == mtime_before + + +def test_import_warn_different_without_force(tmp_path): + """Returns False and keeps old content when content differs and force=False.""" + from import_myst import import_page + + docs = tmp_path / "docs" + docs.mkdir() + old_content = b"# old content\n" + new_content = b"# new content\n" + dest = docs / "md-quick-start.md" + dest.write_bytes(old_content) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = import_page("quick-start", new_content, docs, force=False) + + assert result is False + assert dest.read_bytes() == old_content + assert len(caught) == 1 + assert "force" in str(caught[0].message).lower() + + +def test_import_overwrite_with_force(tmp_path): + """Returns True and writes new content when force=True.""" + from import_myst import import_page + + docs = tmp_path / "docs" + docs.mkdir() + old_content = b"# old content\n" + new_content = b"# new content\n" + dest = docs / "md-quick-start.md" + dest.write_bytes(old_content) + + result = import_page("quick-start", new_content, docs, force=True) + + assert result is True + assert dest.read_bytes() == new_content + + +def test_import_nested_path(tmp_path): + """Creates parent dirs and writes md-{name}.md in the correct nested location.""" + from import_myst import import_page + + docs = tmp_path / "docs" + docs.mkdir() + content = b"# zone\nFirewall zone docs\n" + + result = import_page("configuration/firewall/zone", content, docs, force=False) + + assert result is True + dest = docs / "configuration" / "firewall" / "md-zone.md" + assert dest.exists() + assert dest.read_bytes() == content + # Parent dirs were created + assert (docs / "configuration" / "firewall").is_dir() -- cgit v1.2.3 From 112a2811be8cbaff3dcf64353a13aabd9876bd4c Mon Sep 17 00:00:00 2001 From: Yuriy Andamasov Date: Mon, 27 Apr 2026 06:12:19 +0300 Subject: feat: add empty _swap.txt, remove atexit from swap script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The atexit handler in --swap mode caused immediate restore on process exit, breaking standalone usage. Makefile trap and RTD post_build handle restore reliably. 🤖 Generated by [robots](https://vyos.io) --- docs/_swap.txt | 7 +++++++ scripts/swap_sources.py | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 docs/_swap.txt (limited to 'scripts') diff --git a/docs/_swap.txt b/docs/_swap.txt new file mode 100644 index 00000000..a731b9a2 --- /dev/null +++ b/docs/_swap.txt @@ -0,0 +1,7 @@ +# Incremental MyST swap list +# One page stem per line (relative to docs/, no extension) +# Pages listed here will render from MD instead of RST at build time. +# +# Example: +# quick-start +# configuration/firewall/zone diff --git a/scripts/swap_sources.py b/scripts/swap_sources.py index f983dbfa..7e82e876 100644 --- a/scripts/swap_sources.py +++ b/scripts/swap_sources.py @@ -295,8 +295,6 @@ def main(argv=None): parser.error(f"docs directory not found: {docs_dir}") if args.swap: - import atexit - atexit.register(do_restore, docs_dir) do_swap(docs_dir) elif args.restore: do_restore(docs_dir) -- cgit v1.2.3 From 38ff65941d6cd771700480fd2e6c88dad6a01c24 Mon Sep 17 00:00:00 2001 From: Yuriy Andamasov Date: Sat, 2 May 2026 18:50:07 +0300 Subject: fix(swap): address Copilot review feedback on swap infrastructure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Category D — drop obsolete canary mechanism settings: - conf.py: remove '**/md-*.md' from exclude_patterns (no canaries left) - Makefile: replace malformed '*/_build/*' with '$(BUILDDIR)/**' and drop the '*/md-*' ignore (canary files no longer exist) Category C — script robustness: - import_myst.py: * list_myst_files() now raises SystemExit on git ls-tree failure instead of silently returning [] (would have masked typo'd --source refs) * list_rst_files() skips _build/ when scanning for .rst stems * import_page() rejects stems containing '..' or absolute paths and re-checks that the resolved destination stays under docs_dir * --dry-run uses a separate "would_import" counter; summary line now distinguishes dry-run from actual imports - swap_sources.py: * parse_swap_list() reads with explicit encoding='utf-8' * do_restore() validates state file version + entry shape before renaming files; raises with actionable message on corruption * State file reads/writes use explicit encoding='utf-8' throughout _swap.txt: - Wrap long comment line to satisfy 80-character doc-linter limit 🤖 Generated by [robots](https://vyos.io) --- docs/Makefile | 3 +-- docs/_swap.txt | 3 ++- docs/conf.py | 2 +- scripts/import_myst.py | 36 ++++++++++++++++++++++++++++++------ scripts/swap_sources.py | 29 ++++++++++++++++++++++++----- 5 files changed, 58 insertions(+), 15 deletions(-) (limited to 'scripts') diff --git a/docs/Makefile b/docs/Makefile index a2305e15..29ef3906 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -41,8 +41,7 @@ pdf: swap livehtml: swap @trap '$(SWAP) --restore' EXIT; \ sphinx-autobuild --host $(AUTOHOST) --port $(AUTOPORT) \ - --ignore '*/_build/*' \ - --ignore '*/md-*' \ + --ignore '$(BUILDDIR)/**' \ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) defaultvalue: export VYOS_DEFAULT=True diff --git a/docs/_swap.txt b/docs/_swap.txt index cee6841e..43e32cc5 100644 --- a/docs/_swap.txt +++ b/docs/_swap.txt @@ -113,7 +113,8 @@ installation/virtual/eve-ng configexamples/site-2-site-cisco contributing/upstream-packages -# Newly added 2026-04-29 (post cfgcmd/opcmd single-line body fix, BackstopJS <5%) +# Newly added 2026-04-29 (post cfgcmd/opcmd single-line body fix, +# BackstopJS <5%) configuration/policy/as-path-list configuration/policy/community-list configuration/policy/extcommunity-list diff --git a/docs/conf.py b/docs/conf.py index 62982333..c13397a9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -88,7 +88,7 @@ gettext_uuid = False # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store', '_include/vyos-1x', '**/md-*.md'] +exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store', '_include/vyos-1x'] import pathlib _build = pathlib.Path(__file__).parent / '_build' diff --git a/scripts/import_myst.py b/scripts/import_myst.py index 88ba7006..b9dd7898 100644 --- a/scripts/import_myst.py +++ b/scripts/import_myst.py @@ -57,7 +57,10 @@ def list_myst_files(source: str) -> list[str]: cwd=REPO_ROOT, ) if result.returncode != 0: - return [] + raise SystemExit( + f"import_myst: git ls-tree {source!r} failed " + f"(exit {result.returncode}): {result.stderr.strip()}" + ) stems = [] for line in result.stdout.splitlines(): @@ -73,8 +76,14 @@ def list_myst_files(source: str) -> list[str]: def list_rst_files(docs_dir: Path) -> set[str]: """Return set of stems (relative to docs_dir, no extension) for all *.rst files.""" + build_dir = (docs_dir / "_build").resolve() stems = set() for rst in docs_dir.rglob("*.rst"): + try: + rst.resolve().relative_to(build_dir) + continue # inside _build/, skip + except ValueError: + pass rel = rst.relative_to(docs_dir) stem = str(rel)[:-4] # strip ".rst" stems.add(stem) @@ -89,8 +98,12 @@ def import_page(stem: str, md_content: bytes, docs_dir: Path, force: bool) -> bo - Without --force: warns and skips (returns False) if different content exists. - With --force: overwrites (returns True). - Returns True on successful write. + + Refuses (raises ValueError) if *stem* would resolve outside *docs_dir*. """ p = Path(stem) + if p.is_absolute() or ".." in p.parts: + raise ValueError(f"import_myst: refusing unsafe stem {stem!r}") name = p.name parent = p.parent if str(parent) == ".": @@ -99,6 +112,9 @@ def import_page(stem: str, md_content: bytes, docs_dir: Path, force: bool) -> bo target_dir = docs_dir / parent dest = target_dir / f"md-{name}.md" + docs_root = docs_dir.resolve() + if not dest.resolve().parent.is_relative_to(docs_root): + raise ValueError(f"import_myst: refusing stem {stem!r} — escapes docs/") if dest.exists(): existing = dest.read_bytes() @@ -147,6 +163,7 @@ def do_import( imported = 0 skipped = 0 + would_import = 0 for stem in candidate_stems: # Path inside the branch: docs/{stem}.md @@ -162,7 +179,7 @@ def do_import( if dry_run: print(f" (dry-run) would import {stem}") - imported += 1 + would_import += 1 continue wrote = import_page(stem, content, docs_dir, force) @@ -171,10 +188,17 @@ def do_import( else: skipped += 1 - print( - f"import_myst: imported {imported}, skipped {skipped}.", - file=sys.stderr, - ) + if dry_run: + print( + f"import_myst: (dry-run) would import {would_import}, " + f"skipped {skipped}.", + file=sys.stderr, + ) + else: + print( + f"import_myst: imported {imported}, skipped {skipped}.", + file=sys.stderr, + ) # --------------------------------------------------------------------------- diff --git a/scripts/swap_sources.py b/scripts/swap_sources.py index 7e82e876..683d5ef9 100644 --- a/scripts/swap_sources.py +++ b/scripts/swap_sources.py @@ -52,7 +52,7 @@ def parse_swap_list(path: Path) -> list: if not path.exists(): return [] stems = [] - for line in path.read_text().splitlines(): + for line in path.read_text(encoding="utf-8").splitlines(): stripped = line.strip() if stripped and not stripped.startswith("#"): stems.append(stripped) @@ -179,8 +179,8 @@ def do_swap(docs_dir: Path) -> None: exclude_lines.append(str(rel_rst)) state = {"version": STATE_VERSION, "swaps": swaps} - state_path.write_text(json.dumps(state, indent=2)) - _exclude_path(docs_dir).write_text("\n".join(exclude_lines) + "\n") + state_path.write_text(json.dumps(state, indent=2), encoding="utf-8") + _exclude_path(docs_dir).write_text("\n".join(exclude_lines) + "\n", encoding="utf-8") print(f"swap_sources: swapped {len(completed)} file(s).", file=sys.stderr) @@ -197,10 +197,29 @@ def do_restore(docs_dir: Path) -> None: if not state_path.exists(): return # no-op - state = json.loads(state_path.read_text()) + try: + state = json.loads(state_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + raise RuntimeError( + f"swap_sources: cannot read state file {state_path}: {exc}. " + f"Inspect or delete it manually." + ) from exc + + version = state.get("version") + if version != STATE_VERSION: + raise RuntimeError( + f"swap_sources: state file version {version!r} does not match " + f"expected {STATE_VERSION!r}. Inspect {state_path} manually." + ) + swaps = state.get("swaps", []) for entry in reversed(swaps): + if not isinstance(entry, dict) or "md_from" not in entry or "md_to" not in entry: + raise RuntimeError( + f"swap_sources: malformed entry in {state_path}: {entry!r}. " + f"Inspect manually." + ) md_from = docs_dir / entry["md_from"] # original source (md-prefixed) md_to = docs_dir / entry["md_to"] # current location (plain .md) if md_to.exists(): @@ -254,7 +273,7 @@ def do_status(docs_dir: Path) -> None: print("swap_sources: no active swap state.") return - state = json.loads(state_path.read_text()) + state = json.loads(state_path.read_text(encoding="utf-8")) swaps = state.get("swaps", []) print(f"swap_sources: {len(swaps)} file(s) currently swapped:") for entry in swaps: -- cgit v1.2.3