# LLM/AI crawler access policy for Magento shop
# Last updated: Sept 2025
# Purpose: Allow AI/LLM crawlers to index product and content pages,
# but block sensitive Magento system/customer paths.

# ✅ Allow AI crawlers
User-agent: GPTBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: xAI-Grok
Allow: /

User-agent: cohere-ai
Allow: /

# ✅ General rules (applies to all bots including AI & search engines)
User-agent: *
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
Disallow: /catalogsearch/
Disallow: /catalog/product_compare/
Disallow: /report/
Disallow: /var/
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pub/
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /nginx.conf.sample
Disallow: /php.ini.sample
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /RELEASE_NOTES.txt
Disallow: /*SID=
Disallow: /*?dir*
Disallow: /*?limit=all
Disallow: /*?mode*

# ✅ Everything else (including products, categories, CMS pages) is allowed