############################################
# Robots.txt – All Websites
# Optimized for SEO, Security & Cloudflare
############################################

User-agent: *
# --- Allow search engines to index public pages ---
Allow: /

# --- Core Magento folders to keep private ---
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/

# --- Admin / backend (update if custom path) ---
Disallow: /admin/
Disallow: /admin_*
Disallow: /index.php/admin/
Disallow: /index.php/backend/
Disallow: /cp2020admin/

# --- Sensitive system/config files ---
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /auth.json
Disallow: /package.xml
Disallow: /php.ini
Disallow: /.user.ini
Disallow: /CHANGELOG.txt
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.md
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt

# --- Technical & server files ---
Disallow: /errors/
Disallow: /includes/
Disallow: /downloader/
Disallow: /pkginfo/
Disallow: /shell/
Disallow: /cgi-bin/
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php
Disallow: /error_log

# --- Dynamic & customer pages (not for indexing) ---
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
Disallow: /sales/
Disallow: /wishlist/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /newsletter/
Disallow: /contact/
Disallow: /ajax/
Disallow: /rest/
Disallow: /graphql/
Disallow: /catalogsearch/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/

# --- Non-SEO CMS or policy pages ---
Disallow: /terms-of-sale/
Disallow: /terms-of-use/
Disallow: /privacy-policy/
Disallow: /sales/order/
Disallow: /storecredit/info/
Disallow: /invitation/

# --- Query-parameter junk (sorting, filters, tracking) ---
Disallow: /*?SID=
Disallow: /*?dir=
Disallow: /*?limit=
Disallow: /*?mode=
Disallow: /*?price=
Disallow: /*?cat=
Disallow: /*?color=
Disallow: /*?size=
Disallow: /*?manufacturer=
Disallow: /*?q=
Disallow: /*?p=
Disallow: /*?product_list_order=
Disallow: /*?product_list_mode=
Disallow: /*?product_list_limit=
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*

# --- Prevent crawling of duplicate/index.php URLs ---
Disallow: */index.php/

# --- Development / IDE / dump files ---
Disallow: /*.svn$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.tgz$
Disallow: /CVS

# --- Allow static & media for rendering ---
Allow: /static/
Allow: /media/
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$

# --- Cloudflare CDN compatibility ---
Allow: /cdn-cgi/
Allow: /cdn-cgi/image/
Allow: /cdn-cgi/l/email-protection



############################################
# Notes:
# • Keep sitemap generation enabled in Magento admin.
# • Verify in Search Console → robots.txt tester.
# • Do NOT cache this file in Cloudflare (set “Bypass cache” rule).
# • If you add store views/domains, list extra sitemaps below.
############################################