# WordPress robots.txt - Enhanced Security Configuration
# Last updated: January 2025

# Global Crawl Rate Control
Crawl-delay: 30

# Sitemaps
Sitemap: https://kutubee.com/sitemap_index.xml

# Default Behavior for Good Bots
User-agent: *
Allow: /
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-json/
Disallow: /feed/
Disallow: /?feed=
Disallow: /?s=
Disallow: /search/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /cgi-bin/
Disallow: /*?s=
Disallow: /*/trackback/
Disallow: /*/feed/
Disallow: /*/comments/
Disallow: /author/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /.git/
Disallow: /.env
Disallow: /.htaccess
Disallow: /readme.html
Disallow: /license.txt
Disallow: /*?p=
Disallow: /*&p=
Disallow: /refer/
Disallow: /go/
Disallow: /recommend/
Disallow: /wp-content/uploads/wpo-plugins-tables-list.json

# Block AI and ML Training Bots
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: omgilibot
Disallow: /
User-agent: omgili
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /

# Block SEO Tools & Analytics Bots
User-agent: AhrefsBot
Disallow: /
User-agent: AhrefsSiteAudit
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-BA
Disallow: /
User-agent: SemrushBot-BM
Disallow: /
User-agent: SemrushBot-CT
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: MegaIndex
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: AspiegelBot
Disallow: /
User-agent: ZoominfoBot
Disallow: /

# Block Site Analysis & Technology Detection Tools
User-agent: BuiltWith
Disallow: /
User-agent: builtwith
Disallow: /
User-agent: BuiltWithCrawler
Disallow: /
User-agent: WhatWebBot
Disallow: /
User-agent: Wappalyzer
Disallow: /
User-agent: WhatCMS
Disallow: /
User-agent: WebMeUp
Disallow: /
User-agent: TechnoratiBot
Disallow: /
User-agent: w3c-checklink
Disallow: /
User-agent: W3C-checklink
Disallow: /
User-agent: W3C_Validator
Disallow: /

# Block Web Archive & Caching Tools
User-agent: archive.org_bot
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: EmailWolf
Disallow: /
User-agent: archive.today
Disallow: /
User-agent: ArchiveBox
Disallow: /
User-agent: wget
Disallow: /
User-agent: curl
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: LinqiaScrapeBot
Disallow: /
User-agent: WebCacheArchive
Disallow: /

# Block International Search Engines
User-agent: Baiduspider
Disallow: /
User-agent: Baiduspider-image
Disallow: /
User-agent: Baiduspider-video
Disallow: /
User-agent: Baiduspider-news
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: YandexImages
Disallow: /
User-agent: YandexVideo
Disallow: /
User-agent: YandexMedia
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: Sogou inst spider
Disallow: /
User-agent: Sogou spider2
Disallow: /

# Block Monitoring & Scraping Tools
User-agent: UptimeRobot
Disallow: /
User-agent: QuerySeekerSpider
Disallow: /
User-agent: TkSpider
Disallow: /
User-agent: HeadlessChrome
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: PiplBot
Disallow: /