User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: *
Allow: /

Sitemap: https://www.ansiblebyexample.com/sitemap.xml
Sitemap: https://www.ansiblebyexample.com/video-sitemap.xml

# LLM crawler guidance (detailed policy in llms.txt)
# For LLM crawlers, see: https://www.ansiblebyexample.com/llms.txt

Disallow: /api/
Disallow: /_next/
Disallow: /admin/

# Premium content - restricted for training/reuse without permission
Disallow: /tutorials/

# AI indexing: explicitly allow known AI crawlers to index the site
# This permits model builders / large-scale crawlers to fetch content. Remove
# or restrict these entries if you want to block specific providers.
# Known AI crawlers (explicitly allowed):
User-agent: GPTBot
Allow: /

User-agent: OpenAI
Allow: /

User-agent: Anthropic
Allow: /

User-agent: LlamaIndex
Allow: /

# Generic 'AI' crawler tokens
User-agent: ai
Allow: /
User-agent: AI
Allow: /