# Mage Legal - robots.txt
# AI-powered M&A legal diligence platform

User-agent: *
Allow: /

# Block authenticated/internal areas
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# Allow marketing pages
Allow: /blog
Allow: /tabular
Allow: /memo
Allow: /redline
Allow: /questionnaire
Allow: /schedules

# ============================================
# AI CRAWLER DIRECTIVES
# ============================================

# OpenAI GPT Crawler
User-agent: GPTBot
Allow: /
Allow: /blog/
Allow: /tabular
Allow: /memo
Allow: /redline
Allow: /questionnaire
Allow: /schedules
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# Anthropic Claude Crawler
User-agent: ClaudeBot
Allow: /
Allow: /blog/
Allow: /tabular
Allow: /memo
Allow: /redline
Allow: /questionnaire
Allow: /schedules
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# Perplexity AI Crawler
User-agent: PerplexityBot
Allow: /
Allow: /blog/
Allow: /tabular
Allow: /memo
Allow: /redline
Allow: /questionnaire
Allow: /schedules
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# Google AI Training (Google-Extended)
User-agent: Google-Extended
Allow: /
Allow: /blog/
Allow: /tabular
Allow: /memo
Allow: /redline
Allow: /questionnaire
Allow: /schedules
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# Common Crawl (used for AI training)
User-agent: CCBot
Allow: /
Allow: /blog/
Allow: /tabular
Allow: /memo
Allow: /redline
Allow: /questionnaire
Allow: /schedules
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# Cohere AI Crawler
User-agent: cohere-ai
Allow: /
Allow: /blog/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# You.com AI Crawler
User-agent: YouBot
Allow: /
Allow: /blog/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# ByteDance/TikTok AI Crawler
User-agent: Bytespider
Allow: /
Allow: /blog/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# Meta AI Crawler
User-agent: meta-externalagent
Allow: /
Allow: /blog/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard/
Disallow: /demo/
Disallow: /auth/
Disallow: /api/

# ============================================
# SITEMAPS
# ============================================
Sitemap: https://magelegal.com/sitemap.xml

# ============================================
# LLM/AI INFORMATION FILES
# ============================================
# For AI systems and LLMs seeking information about Mage:
# Quick reference: https://magelegal.com/llms.txt
# Comprehensive documentation: https://magelegal.com/llms-full.txt