# ----------------------------
# PhoenixFM robots.txt - Optimized for reducing bot load
# ----------------------------
# Block aggressive third-party SEO crawlers
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: BLEXBot
Disallow: /
# Block known media scrapers
User-agent: Yandex
Disallow: /
User-agent: Baiduspider
Disallow: /
# ----------------------------
# Block AI crawlers
# ----------------------------
User-agent: GPTBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: Applebot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: YouBot
Disallow: /
# ----------------------------
# Block Amazon's aggressive bot
# ----------------------------
User-agent: Amzn-SearchBot
Disallow: /wp-json/
# ----------------------------
# Allow main search engines (restricted paths)
# ----------------------------
User-agent: Googlebot
Disallow: /wp-login.php
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/wpcf7_captcha/
Disallow: /podcasts/
Allow: /wp-admin/admin-ajax.php
User-agent: Bingbot
Disallow: /wp-login.php
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/wpcf7_captcha/
Disallow: /podcasts/
Allow: /wp-admin/admin-ajax.php
User-agent: Slurp
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
User-agent: DuckDuckBot
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
# ----------------------------
# General WordPress rules (all other bots)
# ----------------------------
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/wpcf7_captcha/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /podcasts/
Disallow: /?s=
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 1
# ----------------------------
# Sitemap
# ----------------------------
Sitemap: https://phoenixfm.com/sitemap_index.xml

# Block Sogou from crawling paginated archives (causes database lock every Tuesday)
User-agent: Sogou web spider
Disallow: /author/
Disallow: /category/
Disallow: /?paged=