###############################################################################
# Welcome to NVIDIA Documentation Hub
# AI Optimized robots.txt
# Purpose: guide LLM/AI crawlers (ChatGPT, Claude, Perplexity, etc.) to the
# Clean Markdown versions of our docs and to productlevel llms.txt
###############################################################################
#
# Last updated 1 April 2026 by RR

# BSP Sitemap from BSP CMS, Hosted on BSP
Sitemap: https://docs.nvidia.com/bsp-sitemap.xml

# Cumulus Server Sitemap, Hosted on BSP
Sitemap: https://docs.nvidia.com/cumulus-networking-ethernet-software-sitemap.xml

# NVIDIA Auto-Generated Sitemap from BSP Sitemap REST API, Hosted on S3
Sitemap: https://docs.nvidia.com/bsp_sitemap_index.xml

# S3 Auto-Generated Sitemap, Hosted on S3
Sitemap: https://docs.nvidia.com/s3-sitemap-index.xml     

# Dynamo Sitemap, Hosted on Fern
Sitemap: https://docs.nvidia.com/dynamo/sitemap.xml

# HeavyAI Sitemap, Hosted on Fern
Sitemap: https://docs.nvidia.com/heavyai/sitemap.xml

# Brev Sitemap, Hosted on Fern
Sitemap: https://docs.nvidia.com/brev/sitemap.xml

# NeMo Curator Sitemap, Hosted on Fern
Sitemap: https://docs.nvidia.com/nemo/curator/sitemap.xml

# OpenShell Sitemap, Hosted on Fern
Sitemap: https://docs.nvidia.com/openshell/sitemap.xml

# AIStore Sitemap, Hosted on Fern
Sitemap: https://docs.nvidia.com/aistore/sitemap.xml

# AIPerf Sitemap, Hosted on Fern
Sitemap: https://docs.nvidia.com/aiperf/sitemap.xml

# AI / LLM crawlers & fetchers

User-agent: ai2bot
User-agent: anthropic-ai
User-agent: Amazonbot
User-agent: Amazon-kendra
User-agent: Applebot-extended
User-agent: AutoGPT
User-agent: BedrockBot
User-agent: Bytespider
User-agent: CCBot
User-agent: Chatglm-spider
User-agent: ChatGPT
User-agent: ChatGPT-User
User-agent: Claudebot
User-agent: Claude-user
User-agent: Claude-search
User-agent: Claude-web
User-agent: Cohere-ai
User-agent: Cotoyogi
User-agent: Deepseek
User-agent: Deepseek Chat
User-agent: Deepseekbot
User-agent: DuckAssistBot
User-agent: DuckDuckBot
User-agent: FacebookBot
User-agent: Geminiios
User-agent: Google-Extended
User-agent: Googleagent-manier
User-agent: GPTBot
User-agent: Grok
User-agent: ia-archiver
User-agent: iaskbot
User-agent: iaskspider
User-agent: ICC-crawler
User-agent: Linerbot
User-agent: Metaaib
User-agent: Meta-externalagent
User-agent: Meta-externalfetcher
User-agent: Meta-llama
User-agent: Mistralai-user
User-agent: NovaAct
User-agent: OAI-Searchbot
User-agent: Pangubot
User-agent: Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-user
User-agent: Phindbot
User-agent: PoeBot
User-agent: Qwen
User-agent: Sogou
User-agent: Timpibot
User-agent: Tongyi
User-agent: Wenxiaobai
User-agent: Xinghuo
User-agent: Youbot
User-agent: Yuanbaobot
                         
Allow: /*.llms.txt$
Allow: /*.md$
Allow: https://docs.nvidia.com/llms.txt

User-Agent: *
Content-Signal: /vgpu/13.0/ ai-train=yes, search=no, ai-input=no
Allow: /vgpu/13.0/

User-agent: Googlebot
User-agent: Googlebot Smartphone
User-agent: Google AdsBot
User-agent: Bingbot


User-agent: *
Disallow: /*?ncid*
Disallow: /*&ncid*

Disallow: /*?nvid*
Disallow: /*&nvid*

Disallow: /*?utm*
Disallow: /*&utm*

Disallow: /*?highlight*
Disallow: /*&highlight*