# ============================================================
# robots.txt — AK Network Solutions
# https://aknetworksolutions.com
# Updated: 2026
# Standards: Google, Bing, AI crawlers (GPTBot, ClaudeBot, etc.)
# ============================================================

# ── Default: all compliant crawlers ─────────────────────────
User-agent: *
Allow: /

# WordPress system paths
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-cron.php
Disallow: /wp-config.php
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-trackback.php

# Allow wp-admin/admin-ajax.php (required for some frontend features)
Allow: /wp-admin/admin-ajax.php

# WordPress query parameters that create duplicate content
Disallow: /*?s=
Disallow: /*?p=
Disallow: /*?replytocom=
Disallow: /*?unapproved=
Disallow: /*?moderation-hash=
Disallow: /*?doing_wp_cron=
Disallow: /*?preview=
Disallow: /*?preview_id=
Disallow: /*?preview_nonce=
Disallow: /*?ver=
Disallow: /*&s=

# WordPress archive/utility pages
Disallow: /search/
Disallow: /author/
Disallow: /trackback/
Disallow: /comments/feed/
Disallow: /feed/
Disallow: /wp-json/
Allow: /wp-json/akns/

# Upload folders — allow assets, block temp/backup files
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/akns-*/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/backup*/
Disallow: /wp-content/upgrade/
Disallow: /wp-content/debug.log

# Tag and date archives (low SEO value, paginates into duplicates)
Disallow: /tag/
Disallow: */page/2$
Disallow: */page/3$
Disallow: */page/[0-9]*/

# ── Googlebot (inherits * rules, explicit allow for rendering) ──
User-agent: Googlebot
Allow: /
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.woff2$
Allow: /*.woff$
Disallow: /wp-admin/
Disallow: /wp-login.php

# ── Googlebot-Image ─────────────────────────────────────────
User-agent: Googlebot-Image
Allow: /wp-content/uploads/

# ── Bingbot ─────────────────────────────────────────────────
User-agent: bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /*?s=
Disallow: /author/
Crawl-delay: 5

# ── AI Crawlers — Allow (good for AI search discovery) ──────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeznamBot
Crawl-delay: 20

# ── Block malicious / scrapers ────────────────────────────────
User-agent: EmailCollector
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: HTTrack
Disallow: /

# ── Block RSS/Atom feeds (crawl budget waste) ────────────────
Disallow: /feed/
Disallow: /*/feed/
Disallow: /*/feed$
Disallow: /comments/feed/

# ── Block wp-includes JS files (not pages) ───────────────────
Disallow: /wp-includes/js/
Disallow: /wp-includes/css/

# ── Block Elementor preview/template params ──────────────────
Disallow: /*?e-page-
Disallow: /*&e-page-
Disallow: /tf_header_footer/

# ── Block KML and non-HTML resources ─────────────────────────
Disallow: /locations.kml
Disallow: /langs/

# ── Block .well-known from indexing (allow for verification) ─
Disallow: /.well-known/security.txt

# ── Sitemaps ─────────────────────────────────────────────────
Sitemap: https://aknetworksolutions.com/sitemap_index.xml
Sitemap: https://aknetworksolutions.com/wp-sitemap.xml

# ── LLMs.txt — AI Discovery ──────────────────────────────────
# LLMs: https://aknetworksolutions.com/llms.txt