# Search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

# AEO — live-retrieval answer engines we allow so AI answers can CITE + link to us.
# Policy: cite us, do not train on us. Bulk/training crawlers (GPTBot,
# Google-Extended, ClaudeBot) stay blocked below and at the Cloudflare edge.
#
# Each of these is named explicitly rather than left to fall through to
# "User-agent: *". They were allowed only by accident of the wildcard before, so a
# future edit to the wildcard group would have silently cut off every engine that
# can cite us, with nothing failing visibly.
#
# Content-Signal declares the same policy in the machine-readable form Cloudflare's
# managed block deliberately leaves unset: yes to search and to being used as input
# for an answer, no to training.
User-agent: ChatGPT-User
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

User-agent: PerplexityBot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

User-agent: Claude-User
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

# Social previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# Block training harvesters and aggressive crawlers
#
# GPTBot, ClaudeBot, Google-Extended and Applebot-Extended were named in the
# comment above as "blocked below" and were not below. They were not anywhere:
# the only occurrence of those strings in the entire codebase was that comment,
# so all four fell through to "User-agent: *", which says Allow. The file
# asserted the opposite of the policy it documented, and had since the AEO
# section was written. Found 2026-09-06 while adding crawler-hit logging.
#
# Google-Extended and Applebot-Extended are training-permission TOKENS, not
# crawlers — Google and Apple crawl as Googlebot and Applebot and consult these
# to decide whether what they fetched may be used for training. Disallowing the
# token is how "cite us, do not train on us" is expressed for those two, and it
# does not affect their search crawling, which stays Allow above.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /api/

User-agent: AhrefsBot
Disallow: /api/

# Default: allow public pages, block app internals + API
#
# /api/public/ is the exception, and the Allow has to sit ABOVE the Disallow it
# narrows. Those endpoints are the JSON form of the calculators at
# /library/tool/ — published content that happens to be addressed under /api —
# and an answer engine that obeys "Disallow: /api/" would be refused the very
# thing this site is asking it to call. A longer, more specific Allow wins over
# a shorter Disallow for every major crawler.
User-agent: *
Allow: /
Allow: /blog
Allow: /blog/
Allow: /app/artists
Allow: /app/login
Allow: /app/register
Allow: /app/cards/
Allow: /api/public/
Disallow: /app/
Disallow: /api/

Sitemap: https://totallytarot.net/sitemap.xml
Sitemap: https://totallytarot.net/sitemap-images.xml

# /llms.txt describes what this site holds and how it is organised, in the format
# language models are increasingly read with. There is no standard robots directive
# for it, so it is named here because this file is where an agent looks first.
# LLM-Content: https://totallytarot.net/llms.txt