Merge pull request #1876 from vyos/feat/docs-llms-txt-sagitta

docs: add llms.txt + llms-full.txt generation (sagitta)

2 files changed, 115 insertions, 0 deletions

diff --git a/docs/_templates/llms.txt.j2 b/docs/_templates/llms.txt.j2
new file mode 100644
index 00000000..dd112ae4
--- /dev/null
+++ b/docs/_templates/llms.txt.j2
@@ -0,0 +1,58 @@
+# VyOS
+
+> VyOS is a free, open-source network operating system based on Debian GNU/Linux.
+> It provides routing, firewall, and VPN functionality via a unified CLI using
+> `set`/`delete`/`show` command hierarchy. This documentation covers {{ release }}.
+
+VyOS configuration follows a tree structure. All configuration commands start with
+`set` (to add/change) or `delete` (to remove). Changes are staged and applied with
+`commit`. The CLI hierarchy maps directly to the documentation structure.
+
+## Quick Start
+
+- [Quick Start Guide]({{ baseurl }}quick-start.md): Minimal setup walkthrough
+{# cli.html (not cli.md) — sagitta's `docs/cli.rst` hasn't been
+   migrated to MyST (rst-to-myst chokes on the heading hierarchy in
+   that file), so no `.md` source is copied to the HTML root by
+   `_copy_md_sources`. Pin this one entry to `.html` until cli.rst is
+   migrated; tracked as a separate cleanup PR. -#}
+- [CLI Overview]({{ baseurl }}cli.html): Command-line interface usage
+
+## Configuration
+
+- [Firewall]({{ baseurl }}configuration/firewall/index.md): Zone-based firewall, rules, groups
+- [Interfaces]({{ baseurl }}configuration/interfaces/index.md): Ethernet, bonding, bridge, VLAN, tunnel, wireless
+- [Protocols]({{ baseurl }}configuration/protocols/index.md): BGP, OSPF, IS-IS, static routing, MPLS
+- [VPN]({{ baseurl }}configuration/vpn/index.md): IPsec, OpenVPN, WireGuard, L2TP, PPTP
+- [NAT]({{ baseurl }}configuration/nat/index.md): Source NAT, destination NAT, NAT66
+- [System]({{ baseurl }}configuration/system/index.md): DNS, NTP, syslog, users, task scheduler
+- [High Availability]({{ baseurl }}configuration/highavailability/index.md): VRRP
+- [Load Balancing]({{ baseurl }}configuration/loadbalancing/index.md): WAN and reverse proxy
+- [Containers]({{ baseurl }}configuration/container/index.md): Podman-based container support
+- [PKI]({{ baseurl }}configuration/pki/index.md): Certificate management
+- [Policy]({{ baseurl }}configuration/policy/index.md): Route maps, prefix lists, access lists
+- [Traffic Policy]({{ baseurl }}configuration/trafficpolicy/index.md): QoS and shaping
+- [Service]({{ baseurl }}configuration/service/index.md): DHCP, DNS forwarding, SNMP, SSH, HTTPS API
+- [VRF]({{ baseurl }}configuration/vrf/index.md): Virtual routing and forwarding
+
+## Operations
+
+- [Operational Commands]({{ baseurl }}operation/index.md): Show, monitor, restart commands
+
+## Installation
+
+- [Installation Guide]({{ baseurl }}installation/index.md): Bare metal, virtual, cloud deployments
+
+## Automation
+
+- [Automation]({{ baseurl }}automation/index.md): Ansible, Terraform, HTTP API, NETCONF
+
+## Configuration Examples
+
+- [Blueprints]({{ baseurl }}configexamples/index.md): Real-world topology examples
+
+## Optional
+
+- [Build VyOS]({{ baseurl }}contributing/build-vyos.md): Building images from source
+- [Development]({{ baseurl }}contributing/development.md): Development workflow
+- [Debugging]({{ baseurl }}contributing/debugging.md): Debugging VyOS
diff --git a/docs/conf.py b/docs/conf.py
index feee8d86..4a31d081 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -50,6 +50,8 @@ extensions = ['sphinx.ext.intersphinx',
               'myst_parser',
               'sphinx_design',
               'vyos',
+              'sphinx_llms_txt',
+              'sphinx_sitemap',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -110,6 +112,15 @@ todo_include_todos = True
 #
 html_theme = "sphinx_rtd_theme"
 
+html_baseurl = 'https://docs.vyos.io/en/1.4/'
+
+# sphinx-sitemap: baseurl already includes /en/1.4/, so skip lang+version
+sitemap_url_scheme = '{link}'
+
+# sphinx-llms-txt: disable auto-generated llms.txt, keep curated render via setup
+# hook; llms-full.txt is still auto-generated
+llms_txt_file = False
+
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
@@ -240,6 +251,52 @@ def _copy_md_sources(app, exception):
         dest.parent.mkdir(parents=True, exist_ok=True)
         shutil.copy2(path, dest)
 
+def _write_llms_txt(app, exception):
+    # Skip dirhtml: production publishes via the `html` / `readthedocs`
+    # builders only. The `.md` links in the curated template do
+    # actually resolve under `dirhtml` (`_copy_md_sources` puts `.md`
+    # files at their source-relative paths regardless of builder), but
+    # we still don't render llms.txt for builds we don't ship — local
+    # `make dirhtml` is a developer convenience, not a publish target.
+    if exception is not None or app.builder.name not in (
+            'html', 'readthedocs'):
+        return
+    if not app.config.html_baseurl:
+        # Fail loudly rather than rendering /quick-start.md etc. as a
+        # silently-broken root-relative URL — every supported branch
+        # sets html_baseurl, so a missing value is a regression.
+        raise RuntimeError(
+            'html_baseurl must be set to render llms.txt')
+    from pathlib import Path
+    from jinja2 import Environment, FileSystemLoader, StrictUndefined
+    tpl_dir = Path(app.srcdir) / '_templates'
+    out_path = Path(app.outdir) / 'llms.txt'
+    baseurl = app.config.html_baseurl.rstrip('/') + '/'
+    # FileSystemLoader + get_template (rather than from_string) makes
+    # Jinja tracebacks reference the real template filename and line
+    # number — useful when StrictUndefined trips on a typo in
+    # llms.txt.j2. StrictUndefined: missing template variables raise
+    # rather than silently render as empty strings, so a typo in
+    # llms.txt.j2 fails the build instead of shipping a half-blank
+    # llms.txt.
+    env = Environment(
+        loader=FileSystemLoader(str(tpl_dir)),
+        undefined=StrictUndefined,
+        keep_trailing_newline=True,
+        # Plain-text template (not HTML), so HTML autoescape is not
+        # appropriate. Setting autoescape=False explicitly to silence
+        # bandit/ruff S701 and document the intent.
+        autoescape=False,
+    )
+    template = env.get_template('llms.txt.j2')
+    rendered = template.render(
+        baseurl=baseurl,
+        release=app.config.release,
+    )
+    out_path.write_text(rendered, encoding='utf-8')
+
+
 def setup(app):
     app.connect('builder-inited', _prefer_webp)
     app.connect('build-finished', _copy_md_sources)
+    app.connect('build-finished', _write_llms_txt)