Sitemap: https://warbooks.co.za/sitemap.xml
User-agent: *
Crawl-delay: 2

# Block technical pages that should not appear in the search results
Disallow: /cdn-cgi/
Disallow: /search/
Disallow: /checkout/
Disallow: /account/
Disallow: /ajax/
Disallow: /api/

# Block filters
Disallow: /*page=
Disallow: /*s=

# AI bots: allow search and user-requested bots, block model training bots
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

# Block bad bots causing performance issues
User-agent: Midjourney
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SemrushBot-CT
Disallow: /

User-agent: SemrushBot-BM
Disallow: /

User-agent: SemrushBot-SEOAB
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: Keybot Translation-Search-Machine
Disallow: /

User-agent: okhttp
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Scrapy
Disallow: /