# Robots.txt for www.agentifizierung.de
#
# NOTE ON THE REPEATED Disallow LINES BELOW -- do not "clean this up".
# A crawler obeys exactly ONE group: the most specific User-agent match, applied
# exclusively. It never falls through to the "User-agent: *" group. So a named
# group containing only "Allow: /" gives that crawler no Disallow rules at all.
# Every group that needs the restrictions must repeat them.
#
# Disallow lines come BEFORE "Allow: /" on purpose. Google resolves conflicts by
# longest matching path, but simpler crawlers take the first matching rule. This
# ordering gives the same answer under both.
#
# /_next/ is deliberately NOT disallowed: it serves the CSS and JS required to
# render the site, and blocking it breaks rendering for any crawler that obeys it.

User-agent: Googlebot
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /

User-agent: Bingbot
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

# ChatGPT Search indexing — official search crawler
# See: https://developers.openai.com/api/docs/bots
User-agent: OAI-SearchBot
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

# Model-training crawler — permitted by current site policy
User-agent: GPTBot
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

# User-triggered page access in ChatGPT; not the Search indexing crawler
User-agent: ChatGPT-User
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

User-agent: ClaudeBot
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

User-agent: Claude-Web
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

User-agent: anthropic-ai
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

User-agent: PerplexityBot
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

User-agent: Google-Extended
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 2

# Social media crawlers fetch single URLs for link previews; no restrictions needed.
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Block SEO crawlers
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Every crawler without its own group above, including future AI crawlers.
User-agent: *
Disallow: /api/
Disallow: /404
Disallow: /500
Allow: /
Crawl-delay: 10

Sitemap: https://www.agentifizierung.de/sitemap.xml