# ============================================================
# robots.txt — https://scighera.com
# AI Crawler Management with differential Crawl-delay
# Updated: 2026-07-28
# Registry: /Storage01/App01/anth/config/ai-crawler-registry.json
# ============================================================

# === GENERAL CRAWLERS ===
User-agent: *
Allow: /
Crawl-delay: 1

# WordPress optimization
Allow: /wp-content/uploads/
Allow: /wp-content/cache/
Allow: /wp-admin/admin-ajax.php

# Security — block admin and sensitive areas
Disallow: /wp-admin/
Disallow: /?s=
Disallow: /*?s=*
Disallow: /author/
Disallow: /tag/
Disallow: */feed/
Disallow: */trackback/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /wp-config.php
Disallow: /.git/
Disallow: /.env

# === SEO CRAWLERS — FULL ACCESS ===
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# === AI CRAWLERS — WELCOME WITH CRAWL-DELAY ===
# Crawl-delay comunica al crawler quanto aspettare tra le richieste
# Rate limiting nginx attivo come backstop (429 + Retry-After)

# OpenAI GPTBot — training data (medium aggressiveness)
User-agent: GPTBot
Allow: /
Crawl-delay: 10

# OpenAI ChatGPT-User — user fetch (high aggressiveness)
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 15

# OpenAI OAI-SearchBot — search results
User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 10

# Anthropic ClaudeBot — training data (medium aggressiveness)
User-agent: ClaudeBot
Allow: /
Crawl-delay: 10

User-agent: anthropic-ai
Allow: /
Crawl-delay: 10

User-agent: claude-web
Allow: /
Crawl-delay: 10

# Google AI (Bard/Gemini)
User-agent: Google-Extended
Allow: /

# ByteDance Bytespider — very high aggressiveness
User-agent: Bytespider
Allow: /
Crawl-delay: 20

# Huawei PetalBot — medium aggressiveness
User-agent: PetalBot
Allow: /
Crawl-delay: 10

# CocCoc search — low aggressiveness
User-agent: coccocbot
Allow: /
Crawl-delay: 5

# Majestic MJ12bot — very high aggressiveness (distributed)
User-agent: MJ12bot
Allow: /
Crawl-delay: 20

# Other AI/search crawlers
User-agent: PerplexityBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# SEO analysis crawlers (owner-confirmed S45-21-05)
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: DotBot
Allow: /

User-agent: BLEXBot
Allow: /

# === SITEMAPS ===
Sitemap: https://scighera.com/sitemap_index.xml