User-agent: *
Allow: /
Allow: /analisis/
Allow: /es/fraude/
Allow: /en/scam/

# Block access to private/authenticated areas
Disallow: /dashboard
Disallow: /history
Disallow: /reset-password
Disallow: /verify-email
Disallow: /forgot-password

# Block JSON / data endpoints — pages are the canonical content
Disallow: /api/

# AI crawlers — explicitly allowed
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

# Commercial SEO crawlers — allowed but throttled to reduce load.
# These bots respect Crawl-delay (Google does not, but Google doesn't need throttling).
User-agent: SemrushBot
Crawl-delay: 10

User-agent: SemrushBot-BA
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

Sitemap: https://desenmascara.me/sitemap-index.xml