From f0402b1a08c393c6f12896e2d27c339030f030b2 Mon Sep 17 00:00:00 2001 From: Yuriy Andamasov Date: Tue, 14 Apr 2026 13:19:06 +0300 Subject: feat: LLM documentation adaptation (llms.txt, sitemap, robots.txt) (#1832) * feat: add sphinx-llms-txt and sphinx-sitemap dependencies Co-Authored-By: Claude Opus 4.6 (1M context) * feat: register sphinx-llms-txt and sphinx-sitemap extensions Co-Authored-By: Claude Opus 4.6 (1M context) * feat: add curated llms.txt for LLM discovery Co-Authored-By: Claude Opus 4.6 (1M context) * feat: update robots.txt with explicit AI bot allowances Co-Authored-By: Claude Opus 4.6 (1M context) * fix: configure sphinx-llms-txt and sphinx-sitemap options Disable auto-generated llms.txt (curated version served from _html_extra instead). Set sitemap_url_scheme to avoid doubled lang/version path segments since html_baseurl already includes /en/latest/. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- docs/_html_extra/llms.txt | 52 +++++++++++++++++++++++++++++++++++++++++++++ docs/_html_extra/robots.txt | 39 +++++++++++++++++++++++++++++----- docs/conf.py | 13 +++++++++++- 3 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 docs/_html_extra/llms.txt (limited to 'docs') diff --git a/docs/_html_extra/llms.txt b/docs/_html_extra/llms.txt new file mode 100644 index 00000000..a8f5b002 --- /dev/null +++ b/docs/_html_extra/llms.txt @@ -0,0 +1,52 @@ +# VyOS + +> VyOS is a free, open-source network operating system based on Debian GNU/Linux. +> It provides routing, firewall, and VPN functionality via a unified CLI using +> `set`/`delete`/`show` command hierarchy. Current rolling release is 1.5.x (circinus). + +VyOS configuration follows a tree structure. All configuration commands start with +`set` (to add/change) or `delete` (to remove). Changes are staged and applied with +`commit`. The CLI hierarchy maps directly to the documentation structure. + +## Quick Start + +- [Quick Start Guide](https://docs.vyos.io/en/latest/quick-start.html): Minimal setup walkthrough +- [CLI Overview](https://docs.vyos.io/en/latest/cli.html): Command-line interface usage + +## Configuration + +- [Firewall](https://docs.vyos.io/en/latest/configuration/firewall/index.html): Zone-based firewall, rules, groups +- [Interfaces](https://docs.vyos.io/en/latest/configuration/interfaces/index.html): Ethernet, bonding, bridge, VLAN, tunnel, wireless +- [Protocols](https://docs.vyos.io/en/latest/configuration/protocols/index.html): BGP, OSPF, IS-IS, static routing, MPLS +- [VPN](https://docs.vyos.io/en/latest/configuration/vpn/index.html): IPsec, OpenVPN, WireGuard, L2TP, PPTP +- [NAT](https://docs.vyos.io/en/latest/configuration/nat/index.html): Source NAT, destination NAT, NAT66 +- [System](https://docs.vyos.io/en/latest/configuration/system/index.html): DNS, NTP, syslog, users, task scheduler +- [High Availability](https://docs.vyos.io/en/latest/configuration/highavailability/index.html): VRRP +- [Load Balancing](https://docs.vyos.io/en/latest/configuration/loadbalancing/index.html): WAN and reverse proxy +- [Containers](https://docs.vyos.io/en/latest/configuration/container/index.html): Podman-based container support +- [PKI](https://docs.vyos.io/en/latest/configuration/pki/index.html): Certificate management +- [Policy](https://docs.vyos.io/en/latest/configuration/policy/index.html): Route maps, prefix lists, access lists +- [Traffic Policy](https://docs.vyos.io/en/latest/configuration/trafficpolicy/index.html): QoS and shaping +- [Service](https://docs.vyos.io/en/latest/configuration/service/index.html): DHCP, DNS forwarding, SNMP, SSH, HTTPS API +- [VRF](https://docs.vyos.io/en/latest/configuration/vrf/index.html): Virtual routing and forwarding + +## Operations + +- [Operational Commands](https://docs.vyos.io/en/latest/operation/index.html): Show, monitor, restart commands + +## Installation + +- [Installation Guide](https://docs.vyos.io/en/latest/installation/index.html): Bare metal, virtual, cloud deployments + +## Automation + +- [Automation](https://docs.vyos.io/en/latest/automation/index.html): Ansible, Terraform, HTTP API, NETCONF + +## Configuration Examples + +- [Blueprints](https://docs.vyos.io/en/latest/configexamples/index.html): Real-world topology examples + +## Optional + +- [Contributing](https://docs.vyos.io/en/latest/contributing/index.html): Development workflow +- [VPP](https://docs.vyos.io/en/latest/vpp/index.html): Vector Packet Processing integration diff --git a/docs/_html_extra/robots.txt b/docs/_html_extra/robots.txt index 627cb18e..71a6990b 100644 --- a/docs/_html_extra/robots.txt +++ b/docs/_html_extra/robots.txt @@ -1,7 +1,36 @@ -User-agent: atlassian-bot -Disallow: - User-agent: * -Disallow: # Allow everything +Allow: / + +# AI Training Crawlers +User-agent: GPTBot +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: CCBot +Allow: / + +User-agent: PerplexityBot +Allow: / + +# AI Search/Retrieval +User-agent: ChatGPT-User +Allow: / + +User-agent: Claude-SearchBot +Allow: / + +User-agent: Claude-User +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: Perplexity-User +Allow: / -Sitemap: https://docs.vyos.io/sitemap.xml \ No newline at end of file +Sitemap: https://docs.vyos.io/sitemap.xml diff --git a/docs/conf.py b/docs/conf.py index e87def82..4d8f9f16 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,7 +48,9 @@ extensions = ['sphinx.ext.intersphinx', 'autosectionlabel', 'myst_parser', 'sphinx_design', - 'vyos' + 'vyos', + 'sphinx_llms_txt', + 'sphinx_sitemap', ] # Add any paths that contain templates here, relative to this directory. @@ -111,6 +113,15 @@ html_static_path = ['_static'] html_extra_path = ['_html_extra'] +html_baseurl = 'https://docs.vyos.io/en/latest/' + +# sphinx-sitemap: baseurl already includes /en/latest/, so skip lang+version +sitemap_url_scheme = '{link}' + +# sphinx-llms-txt: disable auto-generated llms.txt, keep curated one from +# _html_extra; llms-full.txt is still auto-generated +llms_txt_file = False + # Custom sidebar templates, must be a dictionary that maps document names # to template names. # -- cgit v1.2.3