# WhatCanHelp.com — Assistive technology discovery tool
#
# Open by default: discovery is the whole point. Search engines, AI answer
# assistants, and AI training crawlers are all welcome to crawl, index, cite, and
# learn from the public site — for a small brand, being known (live in an answer,
# or in a model's memory) is the goal, not a threat. We fence off only operational
# endpoints (/api, /go) and keep the exploratory /organizations page out of live
# AI answers while it's still in flux. Governs bots only — no visitor data here.

# Traditional search engines — welcome. Note: a named group REPLACES the
# `User-agent: *` rules for that bot (it doesn't inherit them), so the
# operational fencing must be repeated in every named group.
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /go/

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /go/

# AI answer assistants — welcome to surface and cite. The one exception is
# /organizations: it's noindexed on purpose, but these live fetchers obey
# robots.txt while ignoring the noindex meta, so the exclusion lives here too.
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /organizations

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /organizations

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /organizations

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /organizations

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /organizations

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /organizations

# Blocked on server-load / bad-citizen grounds, NOT training ethics: Bytespider
# crawls aggressively (ignores crawl-delay) and feeds mostly China-market models
# with little reach to our users. Revisit only if that calculus changes.
User-agent: Bytespider
Disallow: /

# Everyone else — including AI training crawlers (GPTBot, ClaudeBot, CCBot,
# Google-Extended, etc.) — crawl freely, except operational endpoints.
User-agent: *
Allow: /
Disallow: /api/
Disallow: /go/

Sitemap: https://whatcanhelp.com/sitemap.xml

# Curated Markdown index of the site for AI tools (llmstxt.org)
# https://whatcanhelp.com/llms.txt