# robots.txt for glyph.sh
#
# Terms of use: https://glyph.sh/terms/
# Unauthorized scraping violates the Terms of Service and applicable copyright law.
#
# The crawlers disallowed below are also refused at the CDN edge with
# HTTP 403 Forbidden (robots.txt and /terms/ stay reachable).

# Default: Allow well-behaved bots
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://glyph.sh/sitemap.xml

# ============================================================
# AI SEARCH AND USER-REQUEST AGENTS - ALLOWED
# (index, cite, or fetch a page a user asked about; not training)
# ============================================================

# OpenAI search and ChatGPT user requests - ALLOWED
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /

# Anthropic search and Claude user requests - ALLOWED
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
Allow: /

# ============================================================
# AI TRAINING CRAWLERS - BLOCKED
# ============================================================

# OpenAI training crawler
User-agent: GPTBot
Disallow: /

# Anthropic training crawler (anthropic-ai is its deprecated token)
User-agent: ClaudeBot
User-agent: anthropic-ai
Disallow: /

# ByteDance (Doubao), Zhipu AI (ChatGLM), Huawei (Pangu)
User-agent: Bytespider
User-agent: ChatGLM-Spider
User-agent: PanguBot
Disallow: /

# Common Crawl (AI training corpus)
User-agent: CCBot
Disallow: /

# Perplexity AI
User-agent: PerplexityBot
User-agent: Perplexity-User
Disallow: /

# Amazon Alexa AI
User-agent: Amazonbot
Disallow: /

# Other AI scrapers
User-agent: QuillBot
User-agent: Diffbot
User-agent: Omgilibot
Disallow: /

# Google AI Training (blocked, but Googlebot allowed for search)
User-agent: Google-Extended
Disallow: /

# ============================================================
# SEARCH ENGINES - ALLOWED
# ============================================================
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
Allow: /

# ============================================================
# COPYRIGHT NOTICE
# ============================================================
# All content © 2026 glyph.sh
# Security research, scripts, and documentation are NOT licensed
# for AI/ML training purposes without explicit written permission.