# fidwen.com — crawler policy (23 Sep 2026)
# Search engines and AI *search / answer* assistants are welcome, so Fidwen can be found and cited.
# Crawlers that collect content to *train* AI models are not. Use of this site's content is also
# governed by our Website Terms of Use: https://fidwen.com/terms.html

# --- AI model-training crawlers: blocked -------------------------------------------------------
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Bytespider
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: omgili
User-agent: Timpibot
User-agent: ImagesiftBot
Disallow: /

# --- Everyone else (search engines, OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User,
#     PerplexityBot, Perplexity-User, …): the public site, but not the API or the private portal ---
User-agent: *
Disallow: /api/
Disallow: /command/
Allow: /

Sitemap: https://fidwen.com/sitemap.xml