# robots.txt for sanctionlistcheck.com
#
# Goal: let classic search engines and AI answer engines (ChatGPT,
# Claude, Perplexity, Gemini) crawl and cite the marketing pages
# (/ and the language pages; legal pages may be crawled but carry
# noindex, name privacy 2026-10-05), while keeping the authenticated app and
# the API off-limits to every crawler. The API is also protected at
# the edge/auth layer (see src/api/*) -- this file is a crawling
# courtesy, not a security control.
#
# Default rule: every agent may crawl marketing pages, none may crawl
# /app (the paid upload/checkout UI) or /api/ (the backend).
User-agent: *
Disallow: /app
Disallow: /api/
Allow: /

# --- Classic search crawlers ---
User-agent: Googlebot
Allow: /
Disallow: /app
Disallow: /api/

User-agent: Bingbot
Allow: /
Disallow: /app
Disallow: /api/

# --- AI answer engines: search/answer crawlers ---
# These fetch pages to answer user questions and can cite/link back to
# this site. We want them crawling the marketing pages.

# OpenAI: indexes content for ChatGPT search/answers.
User-agent: OAI-SearchBot
Allow: /
Disallow: /app
Disallow: /api/

# OpenAI: fetches a page live in response to a user's ChatGPT request.
User-agent: ChatGPT-User
Allow: /
Disallow: /app
Disallow: /api/

# OpenAI: bulk crawler, also used for model training.
User-agent: GPTBot
Allow: /
Disallow: /app
Disallow: /api/

# Anthropic: bulk crawler for Claude's index/training.
User-agent: ClaudeBot
Allow: /
Disallow: /app
Disallow: /api/

# Anthropic: fetches a page live in response to a user's Claude request.
User-agent: Claude-User
Allow: /
Disallow: /app
Disallow: /api/

# Perplexity: bulk crawler for its index.
User-agent: PerplexityBot
Allow: /
Disallow: /app
Disallow: /api/

# Perplexity: fetches a page live in response to a user's question.
User-agent: Perplexity-User
Allow: /
Disallow: /app
Disallow: /api/

# Google: powers Gemini and other Google AI answers/training.
User-agent: Google-Extended
Allow: /
Disallow: /app
Disallow: /api/

# Common Crawl: open dataset many AI labs train on.
User-agent: CCBot
Allow: /
Disallow: /app
Disallow: /api/

Sitemap: https://sanctionlistcheck.com/sitemap.xml