# robots.txt for natlex.it (canonical host: www.natlex.it)
# Policy: marketing site that WANTS to be discovered by classic search
# engines AND major AI assistants (training + retrieval bots).
# Last updated: 2026-06-04

# ── Classic search ───────────────────────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Applebot
Allow: /

# ── OpenAI ──────────────────────────────────────────────────────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# ── Anthropic (Claude) ───────────────────────────────────────────
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# ── Google AI (Gemini, AI Overviews) ─────────────────────────────
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# ── Perplexity ───────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ── Microsoft Bing AI / Copilot ──────────────────────────────────
User-agent: MSNBot
Allow: /

# ── Apple Intelligence ───────────────────────────────────────────
User-agent: Applebot-Extended
Allow: /

# ── Amazon ───────────────────────────────────────────────────────
User-agent: Amazonbot
Allow: /

# ── ByteDance (Doubao / TikTok AI) ───────────────────────────────
User-agent: Bytespider
Allow: /

# ── Meta AI ──────────────────────────────────────────────────────
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# ── DuckDuckGo AI ────────────────────────────────────────────────
User-agent: DuckAssistBot
Allow: /

# ── You.com ──────────────────────────────────────────────────────
User-agent: YouBot
Allow: /

# ── Mistral ──────────────────────────────────────────────────────
User-agent: MistralAI-User
Allow: /

# ── Cohere ───────────────────────────────────────────────────────
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# ── Common Crawl (feeds many open LLMs) ──────────────────────────
User-agent: CCBot
Allow: /

# ── Diffbot ──────────────────────────────────────────────────────
User-agent: Diffbot
Allow: /

# ── Kagi ─────────────────────────────────────────────────────────
User-agent: KagiBot
Allow: /

# ── Catch-all: open by default, except admin/api ─────────────────
User-agent: *
Allow: /
Disallow: /admin.html
Disallow: /api/

Sitemap: https://www.natlex.it/sitemap.xml