# Robots.txt for WebTwizz
# https://webtwizz.com

# Default , search engines + scrapers
User-agent: *
Allow: /
Disallow: /api/

# AI training & retrieval crawlers , explicitly allowed
# (default would already allow them via the wildcard, but some bots
# look for their specific UA first; explicit signal gets cleaner indexing)

User-agent: GPTBot
Allow: /
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /api/

User-agent: Amazonbot
Allow: /
Disallow: /api/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/

User-agent: cohere-ai
Allow: /
Disallow: /api/

User-agent: DuckAssistBot
Allow: /
Disallow: /api/

User-agent: MistralAI-User
Allow: /
Disallow: /api/

User-agent: YouBot
Allow: /
Disallow: /api/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /api/

User-agent: Applebot
Allow: /
Disallow: /api/

# Sitemap + LLM map
Sitemap: https://webtwizz.com/sitemap.xml

# AI crawlers can also discover the LLM-readable site index at
#   /llms.txt and /llms-full.txt (https://llmstxt.org).
# These are referenced as comments rather than directives because
# the robots.txt spec only recognises User-agent / Allow / Disallow /
# Crawl-delay / Sitemap, and Lighthouse's strict parser flags any
# custom directive as an error.