# robots.txt for Awesome Video (https://awesome.video)

User-agent: *
Allow: /

# Priority crawling for primary sections
Allow: /category/
Allow: /subcategory/
Allow: /sub-subcategory/
Allow: /resource/
Allow: /journey/
Allow: /journeys
Allow: /advanced
Allow: /about

# Block API + private/auth routes
Disallow: /api/
Disallow: /admin
Disallow: /profile
Disallow: /bookmarks
Disallow: /settings/

# Sitemap (absolute URL — required by spec)
Sitemap: https://awesome.video/sitemap.xml

Crawl-delay: 1

# A crawler obeys only the most specific group that names it (RFC 9309), so
# every named group below repeats the default group's Disallow lines.

# Social media crawlers: all public pages (needed for OG previews)
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: Slackbot-LinkExpanding
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: Discordbot
User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /profile
Disallow: /bookmarks
Disallow: /settings/

# Search engines
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /profile
Disallow: /bookmarks
Disallow: /settings/
Crawl-delay: 1

# AI search & assistant crawlers: explicitly allowed so AI answer engines
# (ChatGPT, Perplexity, Claude, Gemini, Copilot, Meta AI) can crawl and cite
# this site. Same private-route exclusions as the default group.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /profile
Disallow: /bookmarks
Disallow: /settings/