# NJ Civil Service Navigator - robots.txt
# https://njcsnavigator.com
#
# This site welcomes AI crawlers and provides machine-readable
# data at /data/ and AI-specific guidance at /llms.txt

# Default: Allow all crawlers
User-agent: *
Allow: /
Crawl-delay: 1

# Prevent crawling ephemeral search/filter URLs (user-generated queries)
Disallow: /search?*
Disallow: /glossary?*
Disallow: /?q=*

# Benchmark pages are supplementary views of spec pages (spec pages are canonical)
Disallow: /benchmark/

# AI Search Crawlers - Explicitly welcomed
# These crawlers index content for AI-powered search results

# OpenAI crawlers
User-agent: GPTBot
Allow: /
Allow: /data/
Allow: /llms.txt
Crawl-delay: 2

User-agent: OAI-SearchBot
Allow: /
Allow: /data/
Allow: /llms.txt
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Allow: /data/

# Anthropic crawlers
User-agent: ClaudeBot
Allow: /
Allow: /data/
Allow: /llms.txt
Crawl-delay: 2

User-agent: Claude-SearchBot
Allow: /
Allow: /data/
Crawl-delay: 2

User-agent: Claude-User
Allow: /
Allow: /data/

User-agent: anthropic-ai
Allow: /
Allow: /data/
Allow: /llms.txt
Crawl-delay: 2

# Perplexity crawlers
User-agent: PerplexityBot
Allow: /
Allow: /data/
Allow: /llms.txt
Crawl-delay: 2

User-agent: Perplexity-User
Allow: /
Allow: /data/

# Google AI crawlers
User-agent: GoogleOther
Allow: /
Allow: /data/
Crawl-delay: 2

User-agent: Google-Extended
Allow: /
Allow: /data/
Crawl-delay: 2

# Google ads crawlers (AdSense / AdsBot) - needed for ad serving & review
User-agent: Mediapartners-Google
Allow: /
Allow: /data/

User-agent: AdsBot-Google
Allow: /
Allow: /data/

User-agent: AdsBot-Google-Mobile
Allow: /
Allow: /data/

# Amazon AI crawler
User-agent: Amazonbot
Allow: /
Allow: /data/
Crawl-delay: 2

# Meta AI crawler
User-agent: Meta-ExternalAgent
Allow: /
Allow: /data/
Crawl-delay: 2

# Cohere AI crawler
User-agent: cohere-ai
Allow: /
Allow: /data/
Crawl-delay: 2

# Apple AI crawler
User-agent: Applebot-Extended
Allow: /
Allow: /data/
Crawl-delay: 2

# Microsoft/Bing AI crawler
User-agent: Bingbot
Allow: /
Allow: /data/
Crawl-delay: 2

# Common Crawl (used for AI training datasets)
User-agent: CCBot
Allow: /
Allow: /data/
Crawl-delay: 2

# Sitemap location
Sitemap: https://njcsnavigator.com/sitemap-index.xml
Sitemap: https://njcsnavigator.com/sitemap.xml
Sitemap: https://njcsnavigator.com/sitemap-ai.xml

# AI-specific content guidance (emerging standard)
# See: https://llmstxt.org/
# Machine-readable guide: https://njcsnavigator.com/llms.txt
# AI/LLM discovery sitemap: https://njcsnavigator.com/sitemap-ai.xml

# Data license: Open Data Commons Attribution License (ODC-BY 1.0)
# https://opendatacommons.org/licenses/by/1-0/
# Content derived from public NJ Civil Service Commission records; attribution appreciated.