# Default: block everything
User-agent: *
Disallow: /

# -----------------------------
# Search engines (allow)
# -----------------------------
User-agent: Googlebot
User-agent: Google-InspectionTool
User-agent: GoogleOther
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: Mediapartners-Google
User-agent: FeedFetcher-Google
User-agent: Google-Read-Aloud
User-agent: Bingbot
User-agent: adidxbot            # Microsoft Advertising crawler
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Amazonbot
User-agent: PetalBot
Allow: /

# Bing crawl pacing (must be in the same group as the Allow above to take effect)
User-agent: Bingbot
Crawl-delay: 10

# -----------------------------
# Social / link preview bots (allow)
# -----------------------------
User-agent: FacebookExternalHit
User-agent: facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: WhatsApp
User-agent: Discordbot
User-agent: Pinterestbot
# Optional extras:
# User-agent: TelegramBot
# User-agent: SkypeUriPreview
Allow: /

# -----------------------------
# AI / data crawlers
# -----------------------------
# ✅ Allow AI search engines that cite sources
User-agent: OAI-SearchBot
User-agent: PerplexityBot
Allow: /

# ❌ Block model-training crawlers without attribution
User-agent: GPTBot
User-agent: CCBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Bytespider         # TikTok
User-agent: FacebookBot        # Meta AI training (precautionary)
User-agent: Applebot-Extended  # Apple training
User-agent: Amazonbot-ModelTraining
Disallow: /

# -----------------------------
# Archives (optional)
# -----------------------------
User-agent: ia_archiver
Allow: /

Sitemap: https://www.hotelcecil.dk/sitemap.xml