# https://url2mockup.com/robots.txt
#
# Public pages (home, use cases, FAQ, compare guides, legal) are open to search
# and AI crawlers. App, admin, and sign-in routes stay disallowed for everyone.

# -----------------------------------------------------------------------------
# Default-all crawlers
# -----------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# OpenAI (training, browsing, search)
# -----------------------------------------------------------------------------
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Google (AI / Gemini training & grounding-not the same as Googlebot)
# -----------------------------------------------------------------------------
User-agent: Google-Extended
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Anthropic / Claude
# -----------------------------------------------------------------------------
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: claude-web
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Perplexity
# -----------------------------------------------------------------------------
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Apple (Apple Intelligence / services)
# -----------------------------------------------------------------------------
User-agent: Applebot-Extended
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Meta
# -----------------------------------------------------------------------------
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Amazon (Alexa / AI services)
# -----------------------------------------------------------------------------
User-agent: Amazonbot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# ByteDance / TikTok
# -----------------------------------------------------------------------------
User-agent: Bytespider
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Common Crawl (widely used for research & model training datasets)
# -----------------------------------------------------------------------------
User-agent: CCBot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Structured extraction / enterprise crawlers often used in AI pipelines
# -----------------------------------------------------------------------------
User-agent: Diffbot
User-agent: omgili
User-agent: Omgilibot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Other research / AI ecosystem crawlers
# -----------------------------------------------------------------------------
User-agent: Ai2Bot
User-agent: Ai2Bot-Dolma
User-agent: cohere-ai
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin
Disallow: /login

# -----------------------------------------------------------------------------
# Sitemap
# -----------------------------------------------------------------------------
Sitemap: https://url2mockup.com/sitemap.xml