# Codygon.com robots.txt
# https://codygon.com/robots.txt
# Last updated: October 2025

# Default rule for all crawlers
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /internal/
Disallow: /test/
Disallow: /*.pdf$

# Allow important query parameters for tracking and features
Allow: /*?product=*
Allow: /*?plan=*
Allow: /*?utm_*
Allow: /*?ref=*
Allow: /*?source=*
Allow: /*?campaign=*

# Major search engines - full access
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

# AI Search & Chat Engines - ALLOW for better discoverability
# These bots help AI assistants cite and reference our content
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: cohere-ai
Allow: /

# Block only training/scraping bots that don't provide search/citation value
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Sitemap location
Sitemap: https://codygon.com/sitemap-index.xml

# Crawl-delay for politeness (applies to non-major crawlers)
Crawl-delay: 1

# Host directive
Host: https://codygon.com