# Robots.txt for Sophia AI Knowledge Management Platform
# https://sophia.volantis.io
# Company: Volantis Technology

# Allow all search engines and AI crawlers
User-agent: *
Allow: /

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2

# AI/LLM Bot specific rules
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: YouBot
Allow: /
Crawl-delay: 1

User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 1

# Disallow admin and internal paths (if any exist in future)
Disallow: /admin/
Disallow: /api/internal/
Disallow: /_app/
Disallow: /node_modules/

# Allow important pages explicitly
Allow: /
Allow: /about
Allow: /pricing
Allow: /login
Allow: /register
Allow: /privacy
Allow: /terms
Allow: /cookies

# Sitemap locations
Sitemap: https://sophia.volantis.io/sitemap.xml
Sitemap: https://sophia.volantis.io/sitemap.txt

# LLM-specific content for better understanding
# Structured data for AI comprehension
Allow: /llm-content/
Allow: /llm-content/homepage.md
Allow: /llm-content/about.md
Allow: /llm-content/pricing.md
Allow: /llm-content/features.md
Allow: /llm-content/company-info.md
Allow: /llm-content/legal-info.md
Allow: /llm-content/article-knowledge-lost.md
Allow: /llm-content/article-hidden-cost.md
Allow: /llm-content/article-silos-synergy.md
Allow: /llm-content/article-ai-workplace.md
Allow: /llm-content/article-future-proof.md

# Preferred crawl times (UTC)
# Peak hours: 02:00-06:00 UTC (off-peak for most users)
Request-rate: 1/10s

# Cache policy hints
Cache-delay: 86400

# Contact information for crawlers
# Contact: webmaster@volantis.io