# SageTube.ai — Generative Engine policy
# Format: robots.txt-style directives for LLM and answer-engine crawlers.
# Last reviewed: 2026-05-07
#
# Public marketing, blog, explore, creator profiles, and public Experts: allow.
# Dashboard, billing, admin, API: disallow.
#
# Bot identifiers verified against vendor documentation:
#   OpenAI:    https://platform.openai.com/docs/bots
#   Google:    https://blog.google/technology/ai/an-update-on-web-publisher-controls/
#   Anthropic: https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web
#   Microsoft: https://www.bing.com/webmasters/help/microsoft-extended-crawl-1d825b66

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /
Allow: /blog
Allow: /blog/
Allow: /explore
Allow: /explore/
Allow: /creators/
Allow: /s/
Allow: /contact
Allow: /privacy-policy
Allow: /terms
Disallow: /dashboard
Disallow: /search
Disallow: /channel
Disallow: /video
Disallow: /experts
Disallow: /knowledge
Disallow: /thoughts
Disallow: /assistants
Disallow: /support
Disallow: /admin
Disallow: /profile
Disallow: /billing
Disallow: /settings
Disallow: /api/
Disallow: /login
Disallow: /register

User-agent: PerplexityBot
Allow: /
Allow: /blog
Allow: /blog/
Allow: /explore
Allow: /explore/
Allow: /creators/
Allow: /s/
Allow: /contact
Allow: /privacy-policy
Allow: /terms
Disallow: /dashboard
Disallow: /search
Disallow: /channel
Disallow: /video
Disallow: /experts
Disallow: /knowledge
Disallow: /thoughts
Disallow: /assistants
Disallow: /support
Disallow: /admin
Disallow: /profile
Disallow: /billing
Disallow: /settings
Disallow: /api/
Disallow: /login
Disallow: /register

User-agent: ClaudeBot
Allow: /
Allow: /blog
Allow: /blog/
Allow: /explore
Allow: /explore/
Allow: /creators/
Allow: /s/
Allow: /contact
Allow: /privacy-policy
Allow: /terms
Disallow: /dashboard
Disallow: /search
Disallow: /channel
Disallow: /video
Disallow: /experts
Disallow: /knowledge
Disallow: /thoughts
Disallow: /assistants
Disallow: /support
Disallow: /admin
Disallow: /profile
Disallow: /billing
Disallow: /settings
Disallow: /api/
Disallow: /login
Disallow: /register

User-agent: Google-Extended
User-agent: Microsoft-Extended
Allow: /
Allow: /blog
Allow: /blog/
Allow: /explore
Allow: /explore/
Allow: /creators/
Allow: /s/
Allow: /contact
Allow: /privacy-policy
Allow: /terms
Disallow: /dashboard
Disallow: /search
Disallow: /channel
Disallow: /video
Disallow: /experts
Disallow: /knowledge
Disallow: /thoughts
Disallow: /assistants
Disallow: /support
Disallow: /admin
Disallow: /profile
Disallow: /billing
Disallow: /settings
Disallow: /api/
Disallow: /login
Disallow: /register

# Default deny for unknown LLM crawlers — opt-in only.
User-agent: *
Disallow: /

Sitemap: https://sagetube.ai/sitemap.xml