# NOTE: /wp-content/plugins/ and /wp-content/cache/ are deliberately NOT blocked.
# The homepage alone loads 11 stylesheets/scripts from wp-content/plugins/ and 7
# Autoptimize bundles from wp-content/cache/. Blocking them stopped Googlebot
# fetching render-critical CSS and JS, so it rendered the site unstyled and could
# not judge mobile usability. Removed 2026-09-06. Do not re-add them.

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /?s=
Disallow: /search/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /mano-paskyra/
Disallow: /issaugoti/
Disallow: /krepselis/
Disallow: /atsiskaitymas/
Allow: /wp-admin/admin-ajax.php

# WooCommerce filter / sort / cart query parameters — block crawl
Disallow: /*?orderby=
Disallow: /*?filter_*
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?add-to-cart=
Disallow: /*?add_to_wishlist=
Disallow: /*?remove_item=
Disallow: /*?wc-ajax=
Disallow: /*?wc_session_id=
Disallow: /*?attribute_*
Disallow: /*&orderby=
Disallow: /*&filter_*
Disallow: /*?vtf_*
Disallow: /*&vtf_*

# llms.txt: https://variantai.lt/llms.txt
Sitemap: https://variantai.lt/sitemap_index.xml

# ---------------------------------------------------------------------------
# AI crawlers (GPTBot, OAI-SearchBot, ClaudeBot, PerplexityBot, Google-Extended,
# Applebot-Extended, CCBot, Bytespider, Amazonbot, Meta-ExternalAgent, cohere-ai,
# Diffbot …) are deliberately NOT given their own groups.
#
# A robots.txt crawler obeys ONLY the most specific group that matches it — groups
# are never merged. Giving each AI bot a group containing just "Allow: /" therefore
# REPLACED the "User-agent: *" rules for those bots, exempting them from every
# Disallow above, including /wp-admin/, the cart/checkout URLs and the WooCommerce
# filter parameters. That was the opposite of the intent.
#
# With no group of their own they fall through to "User-agent: *", which already
# allows the entire public site (nothing here blocks content pages) while keeping
# the same sensible exclusions. AI access is preserved; the holes are closed.
# Do not re-add "Allow: /"-only groups for these agents.
# ---------------------------------------------------------------------------

# SEO tool crawlers: same exclusions as everyone else, plus a crawl delay.
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /?s=
Disallow: /search/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /mano-paskyra/
Disallow: /issaugoti/
Disallow: /krepselis/
Disallow: /atsiskaitymas/
Disallow: /*?orderby=
Disallow: /*?filter_*
Disallow: /*?add-to-cart=
Disallow: /*?attribute_*
Allow: /wp-admin/admin-ajax.php

User-agent: SemrushBot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /?s=
Disallow: /search/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /mano-paskyra/
Disallow: /issaugoti/
Disallow: /krepselis/
Disallow: /atsiskaitymas/
Disallow: /*?orderby=
Disallow: /*?filter_*
Disallow: /*?add-to-cart=
Disallow: /*?attribute_*
Allow: /wp-admin/admin-ajax.php