# robots.txt fuer foerder-kompass.de (Hauptmarke Foerder-Kompass)
# Stand: 2026-04-17 | Strategie: Multi-Programm-Sichtbarkeit + LLM/AI-Crawler
# Rollout-Hinweis: via Rank Math SEO -> Allgemeine Einstellungen -> Bearbeiten der robots.txt
# ---------------------------------------------------------------

# Standard Search Engines - alles erlauben ausser Admin
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /*?replytocom=
Disallow: /readme.html
Allow: /wp-admin/admin-ajax.php

# Unnoetigen Crawl-Druck reduzieren
Disallow: /*?print=
Disallow: /*?utm_
Disallow: /*?gclid=
Disallow: /*?fbclid=

# -------- AI / LLM Crawler: EXPLIZIT ERLAUBT --------
# Ziel: In LLM-Antworten ueber Foerdermittelberatung, FZulG, ZIM sichtbar sein.

# OpenAI (ChatGPT Browsing, Search, Training)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude Search, Training)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity (Search + Answer Engine)
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google Gemini / Bard
User-agent: Google-Extended
Allow: /

# Meta (Llama Training)
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# Cohere / Microsoft
User-agent: cohere-ai
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

User-agent: Applebot
Allow: /

# Common Crawl (Basisdatensatz vieler LLMs)
User-agent: CCBot
Allow: /

# Andere AI-Search-Engines
User-agent: YouBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Amazonbot
Allow: /

# -------- Sitemaps --------
Sitemap: https://foerder-kompass.de/sitemap_index.xml