# ===========================================================================
# aiowiki.com - reviewed and consolidated 2026-08-03
#
# Structure matters here: all directives for "*" must live in ONE group.
# Two separate "User-agent: *" records is ambiguous - Google merges them, but
# many crawlers honour only the first match, which silently drops the rest.
# ===========================================================================

# Content namespaces only (main, News, Interview, Bible_Verse) - 10,917 URLs.
# Schedule: (11,354 daily listings) and Category: (28,365) are deliberately
# excluded: near-duplicate, low search value, and the bulk of wasted crawl.
# Regenerated nightly by cron.
Sitemap: https://www.aiowiki.com/sitemap/sitemap-index-aiowiki.xml

User-agent: *

# AI training is permitted deliberately. Decided 2026-08-03: the eight
# AI-crawler blocks that used to sit below this line were removed, because they
# contradicted this signal and, measured over a full day of access logs, were
# turning away a grand total of ONE request. Whatever those blocks were worth,
# it was not crawl load.
Content-Signal: search=yes,ai-train=yes

Allow: /

# The uncacheable machinery. Article paths (/wiki/...) stay fully crawlable;
# only these cost a page parse per request.
#
# Note /w/load.php is deliberately NOT blocked: it already carries
# "public, max-age=300, s-maxage=300", so blocking it saves nothing, and
# Google needs CSS and JS to render a page for indexing.
Disallow: /w/index.php?
Disallow: /w/api.php
Disallow: /w/rest.php
Disallow: /w/opensearch_desc.php

# Special pages, including the URL-encoded and German-alias forms. Search
# results are included on purpose - crawling them is expensive and yields
# nothing indexable.
Disallow: /wiki/Special:
Disallow: /wiki/Special%3A
Disallow: /wiki/Spezial:
Disallow: /w/index.php/Special:

# Query patterns that produce duplicate or machine-only views.
Disallow: /*action=edit
Disallow: /*action=history
Disallow: /*action=raw
Disallow: /*action=info
Disallow: /*action=purge
Disallow: /*action=delete
Disallow: /*action=watch
Disallow: /*diff=
Disallow: /*oldid=
Disallow: /*printable=
Disallow: /*returnto=
Disallow: /*veaction=

# Ignored by Google; honoured by Bing and Yandex. Cloudflare rate limiting is
# the actual enforcement - this only helps the well-behaved crawlers.
Crawl-delay: 10

# ---------------------------------------------------------------------------
# Bytespider (ByteDance) stays blocked, on abuse grounds rather than training
# ones. Treat this group as documentation rather than enforcement: Bytespider
# is widely reported to disregard robots.txt, so what actually stops it is the
# Cloudflare rule, not this file.
# ---------------------------------------------------------------------------
User-agent: Bytespider
Disallow: /

# ---------------------------------------------------------------------------
# Deliberately NOT listed here: Baiduspider. It made 12,574 requests in a
# single day - 26% of all traffic, more than Google, Bing, Apple, Yandex and
# every AI crawler combined - and 10,120 of those went to /w/index.php, which
# the "*" group above already disallows and which cannot be edge-cached. It
# ignores this file completely, so it is handled at Cloudflare instead. Adding
# a group for it here would only create the impression of a fix.
#
# Removed 2026-08-03: Amazonbot, Applebot-Extended, CCBot, ClaudeBot,
# Google-Extended, GPTBot, meta-externalagent. Worth knowing if this is ever
# revisited:
#   Google-Extended and Applebot-Extended were TRAINING-ONLY tokens. Removing
#   them has no effect whatsoever on Google or Apple search ranking, which use
#   Googlebot and Applebot - neither of which was ever blocked.
#   Blocking GPTBot never affected ChatGPT search citations either: those come
#   from OAI-SearchBot and ChatGPT-User, which were never in this file.
# ---------------------------------------------------------------------------