# Lockpickwebwinkel.nl Robots.txt - FIXED FOR SEMRUSH
Sitemap: https://lockpickwebwinkel.nl/sitemap_index.xml

# SEARCH ENGINES - NO DELAY
User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Disallow: /cdn-cgi/

User-agent: Bingbot  
Allow: /
Disallow: /wp-admin/
Disallow: /cdn-cgi/

# SEO TOOLS - GEEN CRAWL DELAY VOOR RESOURCES
User-agent: SemrushBot
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: *.css$
Allow: *.js$ 
Allow: *.jpg$
Allow: *.jpeg$
Allow: *.png$
Allow: *.webp$
Allow: *.svg$
Disallow: /wp-admin/
Disallow: /cdn-cgi/
Crawl-delay: 20

User-agent: AhrefsBot
Crawl-delay: 5  # Verlaagd van 10
Allow: /wp-content/
Disallow: /wp-admin/

# WOOCOMMERCE FILTERS - BEHOUD
Disallow: /*?add-to-cart=
Disallow: /*?min_price=
Disallow: /*?*filter_*&filter_
Disallow: /*?orderby=
Disallow: /*?filter_12-merk=
Disallow: /*?filter_1-categorie=

# STANDARD BOTS
User-agent: *
Crawl-delay: 1
Allow: /
Disallow: /wp-admin/
Disallow: /winkelwagen/
Disallow: /afrekenen/
Disallow: /mijn-account/


# AI Search Crawlers - ALLOW
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /product/
Allow: /category/
Allow: /blog/
Crawl-delay: 2

User-agent: ClaudeBot
User-agent: Claude-Web
Allow: /product/
Allow: /category/
Allow: /blog/
Crawl-delay: 2

User-agent: PerplexityBot
Allow: /

User-agent: GoogleOther
User-agent: Google-Extended
Allow: /