Crawl-delay: 10
# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:

Sitemap: https://www.scurri.com/sitemap_index.xml
Sitemap: https://www.scurri.com/de/sitemap_index.xml
Sitemap: https://www.scurri.com/postal-and-logistics/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK

# ================================
# Allow major AI crawlers
# ================================

# OpenAI (ChatGPT Search + GPTBot)
User-agent: GPTBot
Allow: /

# OpenAI ChatGPT user agent for browsing tools
User-agent: ChatGPT-User
Allow: /

# Google: Allow for Gemini & AI training
# (Google uses Googlebot for general crawling and Google-Extended for AI training)
User-agent: Googlebot
Allow: /
User-agent: Google-Extended
Allow: /

# Anthropic Claude
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /

# Perplexity AI crawler
User-agent: PerplexityBot
Allow: /

# Apple’s Applebot (used in Apple Intelligence)
User-agent: Applebot
Allow: /

# Bing (used by Bing AI / Copilot)
User-agent: Bingbot
Allow: /

# ================================
# Default rule for everything else
# ================================
User-agent: *
Allow: /