# KI-Training und Daten-Sammlung (Bulk-Scraper fuer Modell-Entwicklung)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Google-SearchCentral
User-agent: facebookexternalhit
User-agent: FacebookBot
User-agent: CCBot
User-agent: cohere-training-data-crawler
User-agent: cohere-ai
User-agent: cohere-bot
User-agent: Diffbot
User-agent: DiffBot
User-agent: ImagesiftBot
User-agent: Bytespider
User-agent: Perplexity-rich-context
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Applebot-Extended
User-agent: iaskbot
User-agent: KangarooBot
User-agent: LightDeskBot
User-agent: FriendlyCrawler
Disallow: /

# Kommerzielle Scraper, Aggregatoren und Marken-Spiders
User-agent: Omgilibot
User-agent: omgili
User-agent: QlyzeBot
User-agent: DataForSeoBot
User-agent: GrapeshotCrawler
User-agent: Webzio-Extended
User-agent: Amazonbot
User-agent: TikTokSpider
User-agent: Scrapy
Disallow: /

# Aggressive SEO-Tools und Audit-Bots
User-agent: MJ12bot
User-agent: AhrefsBot
User-agent: SEMrushBot
User-agent: DotBot
User-agent: BLEXBot
User-agent: Rogerbot
User-agent: SEOkicks
User-agent: serpstatbot
User-agent: DomainStatsBot
User-agent: Screaming Frog SEO Spider
User-agent: Sitebulb
User-agent: ZoominfoBot
User-agent: Barkrowler
User-agent: SERankingBacklinksBot
Disallow: /

# Offline-Browser und Site-Ripper (Extremer Bandbreiten-Verbrauch)
User-agent: webzip
User-agent: webwhacker
User-agent: websauger
User-agent: webcapture
User-agent: teleport
User-agent: sitesnagger
User-agent: net attache
User-agent: httrack
Disallow: /

# Internationale Suchmaschinen-Crawler (Asien/Russland)
User-agent: Yandex
User-agent: Baiduspider
User-agent: Sogou web spider
User-agent: PetalBot
User-agent: PanguBot
Disallow: /

# Kleinere oder veraltete Projekte
User-agent: MauiBot
User-agent: adscanner
User-agent: Seekbot
Disallow: /

# KI-Suchassistenten, User-Anfragen und Echtzeit-Suchbots (Sinnvoll, aber drosseln)
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: claude-web
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ChatGPT-User/2.0
User-agent: Applebot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: YouBot
User-agent: DuckAssistBot
User-agent: Grok
User-agent: Gemini-Search
Crawl-delay: 3