User-agent: *
Allow: /

# Block legal pages from crawlers — thin content
Disallow: /legal/

# Block tag pages — handled by sitemap priority instead
Disallow: /blog/tags/

Sitemap: https://frontmatter.tech/sitemap-index.xml

# AI crawlers — allow all
User-agent: GPTBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Googlebot
Allow: /