# ============================================================
# robots.txt for paulroberts.jp
# Policy: Allow search/retrieval; block model-training crawlers
# Last reviewed: 2026-11-03
# ============================================================

# --- Default: allow standard crawlers ---
User-agent: *
Disallow: /?job_listing_type=
Allow: /

# ============================================================
# OPENAI
# ============================================================

# GPTBot: model training / improvement
User-agent: GPTBot
Disallow: /

# OAI-SearchBot: ChatGPT Search
User-agent: OAI-SearchBot
Allow: /

# ChatGPT-User: user-directed retrieval
User-agent: ChatGPT-User
Allow: /

# ============================================================
# ANTHROPIC
# ============================================================

# ClaudeBot: model development / training data collection
User-agent: ClaudeBot
Disallow: /

# anthropic-ai: legacy Anthropic token
User-agent: anthropic-ai
Disallow: /

# Claude-User: user-directed retrieval
User-agent: Claude-User
Allow: /

# Claude-SearchBot: Claude search indexing
User-agent: Claude-SearchBot
Allow: /

# ============================================================
# GOOGLE
# ============================================================

# Google-Extended:
# Blocked to opt out of Gemini model training and
# Google-Extended-controlled grounding.
User-agent: Google-Extended
Disallow: /

# Current NotebookLM crawler
User-agent: Google-GeminiNotebook
Allow: /

# Legacy NotebookLM crawler name (kept for compatibility)
User-agent: Google-NotebookLM
Allow: /

# ============================================================
# PERPLEXITY
# ============================================================

# PerplexityBot: Perplexity Search
# Perplexity states this crawler is not used for foundation-model training.
User-agent: PerplexityBot
Allow: /

# ============================================================
# APPLE
# ============================================================

# Applebot: search / Siri / Spotlight / Apple services
User-agent: Applebot
Allow: /

# Applebot-Extended: Apple foundation-model training
User-agent: Applebot-Extended
Disallow: /

# ============================================================
# META
# ============================================================

# Keep blocked unless Meta AI/agent retrieval is desired.
User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# ============================================================
# BYTEDANCE / TIKTOK
# ============================================================

# No specific RAG benefit identified
User-agent: Bytespider
Disallow: /

# ============================================================
# DEEPSEEK
# ============================================================

# DeepSeekBot: reported model-training crawler
User-agent: DeepSeekBot
Disallow: /

# ============================================================
# DATA BROKERS / AGGREGATORS
# ============================================================

User-agent: CCBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: webzio-extended
Disallow: /

User-agent: Diffbot
Disallow: /

# ============================================================
# X / xAI
# ============================================================

# Keep blocked unless you specifically want Grok crawling.
User-agent: xAI-Bot
Disallow: /

# ============================================================
# SITEMAP
# ============================================================

Sitemap: https://paulroberts.jp/sitemap_index.xml