# robots.txt for theledgerlabs.com
# Last updated: 2026-10-08 (parameter-copy Disallows lifted for Googlebot/Bingbot/Applebot only; 2026-09-19 file otherwise unchanged)
#
# Structure rule: a crawler obeys ONLY the group that names it. Every named group therefore
# repeats the hygiene rules; a named group with a bare "Allow: /" inherits none of them.
# Training-crawler policy: ALLOWED — Gary, 2026-09-17 midday (DECISIONS.md ruling 12: "Allow for now";
# revisit 2026-10-17). The Content-Signal line below records that choice for crawlers that read it.

# ========================================
# Default rules — every crawler not named below
# ========================================
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js
Allow: /wp-content/plugins/*.png
Allow: /wp-content/plugins/*.jpg
Allow: /wp-content/plugins/*.jpeg
Allow: /wp-content/plugins/*.gif
Allow: /wp-content/plugins/*.svg
Allow: /wp-content/plugins/*.webp
Allow: /wp-content/plugins/*.woff
Allow: /wp-content/plugins/*.woff2
Allow: /wp-includes/*.css
Allow: /wp-includes/*.js
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /readme.html
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /?elementor_library=
Disallow: /*?elementor_library=
Disallow: /sample-*/
# Parameter copies of the booking page (utm_* / cta_* tails on internal CTAs): keep the clean
# URL crawlable, drop the 365 parameter variants from the crawl frontier (interim — the real fix
# is stripping the parameters at the header/footer/CTA templates, ISS-002 phase B).
Disallow: /schedule-a-meeting/?*
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?cta_location=
Disallow: /*&cta_location=

# ========================================
# Major search engines — explicitly allowed, same hygiene rules
# ========================================
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
# Parameter copies (utm_* / cta_*) are deliberately NOT disallowed for these three crawlers (2026-10-08):
# every copy serves rel=canonical to the clean URL, and a blocked URL cannot show Google that canonical,
# which left already-indexed copies stuck as "Indexed, though blocked by robots.txt". All other groups keep the block.
Allow: /
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js
Allow: /wp-content/plugins/*.png
Allow: /wp-content/plugins/*.jpg
Allow: /wp-content/plugins/*.jpeg
Allow: /wp-content/plugins/*.gif
Allow: /wp-content/plugins/*.svg
Allow: /wp-content/plugins/*.webp
Allow: /wp-content/plugins/*.woff
Allow: /wp-content/plugins/*.woff2
Allow: /wp-includes/*.css
Allow: /wp-includes/*.js
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /readme.html
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /?elementor_library=
Disallow: /*?elementor_library=
Disallow: /sample-*/

# ========================================
# AI search / retrieval bots — allowed (these fetch pages to answer and cite)
# ========================================
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: MicrosoftAIBot
Allow: /
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /readme.html
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /?elementor_library=
Disallow: /*?elementor_library=
Disallow: /sample-*/
Disallow: /schedule-a-meeting/?*
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?cta_location=
Disallow: /*&cta_location=

# ========================================
# AI training crawlers — allowed, pending Gary's recorded decision
# (retired tokens "anthropic-ai" and "Claude-Web" removed; Anthropic's crawlers are ClaudeBot,
#  Claude-User and Claude-SearchBot)
# ========================================
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: CCBot
User-agent: Meta-ExternalAgent
Allow: /
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /readme.html
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /?elementor_library=
Disallow: /*?elementor_library=
Disallow: /sample-*/
Disallow: /schedule-a-meeting/?*
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?cta_location=
Disallow: /*&cta_location=

# ========================================
# Aggressive scrapers — blocked (unchanged)
# ========================================
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Diffbot
User-agent: Omgili
User-agent: ImagesiftBot
User-agent: PetalBot
Disallow: /

# ========================================
# Purpose signal — recorded 2026-09-19 in the default group above (Content-Signal: search=yes, ai-input=yes, ai-train=yes).
# If training is later disallowed: change it to ai-train=no and move GPTBot, ClaudeBot, Google-Extended,
#   Applebot-Extended, cohere-ai, CCBot, Meta-ExternalAgent to "Disallow: /".
# ========================================

Sitemap: https://theledgerlabs.com/sitemap_index.xml