# =========================================================
# General crawlers
# =========================================================

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cgi-bin/

# Internal search / utility / parameter URLs
Disallow: /*?topic=
Disallow: /*?wordfence=
Disallow: /*?s=
Disallow: /*?src=
Disallow: /*?apage=
Disallow: /*?cnt=
Disallow: /*?ct=
Disallow: /*?event_category=
Disallow: /*?file=
Disallow: /*?hash=
Disallow: /*?hr=
Disallow: /*?l=
Disallow: /*?lang=
Disallow: /*?ics=


# =========================================================
# AI search crawlers
# =========================================================

User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Claude-SearchBot
User-agent: Claude-User
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cgi-bin/

Disallow: /*?topic=
Disallow: /*?wordfence=
Disallow: /*?s=
Disallow: /*?src=
Disallow: /*?apage=
Disallow: /*?cnt=
Disallow: /*?ct=
Disallow: /*?event_category=
Disallow: /*?file=
Disallow: /*?hash=
Disallow: /*?hr=
Disallow: /*?l=
Disallow: /*?lang=
Disallow: /*?ics=

Crawl-delay: 10


# =========================================================
# Social media preview crawlers
# =========================================================

User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Meta-ExternalFetcher
User-agent: Twitterbot
User-agent: LinkedInBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cgi-bin/


# =========================================================
# AI training / data collection crawlers
# =========================================================

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: TikTokSpider
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FriendlyCrawler
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: NewsNow
Disallow: /

User-agent: news-please
Disallow: /

User-agent: Quora-Bot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: TurnitinBot
Disallow: /


# =========================================================
# Sitemap
# =========================================================

Sitemap: https://www.klarpartners.com/en/wp-sitemap.xml