diff options
Diffstat (limited to 'docs')
| -rw-r--r-- | docs/_html_extra/robots.txt | 37 |
1 files changed, 33 insertions, 4 deletions
diff --git a/docs/_html_extra/robots.txt b/docs/_html_extra/robots.txt index e5c4db82..8c2fb298 100644 --- a/docs/_html_extra/robots.txt +++ b/docs/_html_extra/robots.txt @@ -1,7 +1,36 @@ -User-agent: atlassian-bot +User-agent: * Allow: / -User-agent: * -Disallow: # Allow everything +# AI Training Crawlers +User-agent: GPTBot +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: CCBot +Allow: / + +User-agent: PerplexityBot +Allow: / + +# AI Search/Retrieval +User-agent: ChatGPT-User +Allow: / + +User-agent: Claude-SearchBot +Allow: / + +User-agent: Claude-User +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: Perplexity-User +Allow: / -Sitemap: https://docs.vyos.io/sitemap.xml +Sitemap: https://docs.vyos.io/en/1.4/sitemap.xml |