# AppBoard — product discovery for people and AI.
#
# Public pages (products, creators, leaderboard, category/use-case/platform
# hubs) are open to ALL crawlers — search engines, AI answer engines, and AI
# training bots alike. We WANT to be discovered, indexed, and cited. Only
# private/account and action pages are closed, and they are closed to everyone.
#
# AI systems: a curated, machine-friendly map of this site lives at /llms.txt
#
# NOTE: this static file is the canonical robots.txt served at the site origin.
# The API serves an equivalent /robots.txt for same-origin/local testing.

# ---------------------------------------------------------------------------
# Everyone — search engines (Googlebot, Bingbot) and any bot without its own
# group below. Gemini's live grounding also crawls as Googlebot, so it's here.
# ---------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /auth/
Disallow: /dashboard
Disallow: /my-products
Disallow: /my-votes
Disallow: /my-comments
Disallow: /activity
Disallow: /submit
Disallow: /settings
Disallow: /billing
Disallow: /unsubscribe
Disallow: /embed/
Disallow: /product/*/edit
Disallow: /product/*/analytics
Disallow: /product/*/boost
Disallow: /r/
Disallow: /my-sponsored
Disallow: /sponsor/new
Disallow: /sponsor/checkout
# Locale variants of the paths above — these resolve too (see /de/…, /es/…),
# so they must be closed in every language, not just at the root.
Disallow: /de/auth/
Disallow: /de/dashboard
Disallow: /de/my-products
Disallow: /de/my-votes
Disallow: /de/my-comments
Disallow: /de/activity
Disallow: /de/submit
Disallow: /de/settings
Disallow: /de/billing
Disallow: /de/unsubscribe
Disallow: /de/embed/
Disallow: /de/product/*/edit
Disallow: /de/product/*/analytics
Disallow: /de/product/*/boost
Disallow: /de/r/
Disallow: /de/my-sponsored
Disallow: /de/sponsor/new
Disallow: /de/sponsor/checkout
Disallow: /es/auth/
Disallow: /es/dashboard
Disallow: /es/my-products
Disallow: /es/my-votes
Disallow: /es/my-comments
Disallow: /es/activity
Disallow: /es/submit
Disallow: /es/settings
Disallow: /es/billing
Disallow: /es/unsubscribe
Disallow: /es/embed/
Disallow: /es/product/*/edit
Disallow: /es/product/*/analytics
Disallow: /es/product/*/boost
Disallow: /es/r/
Disallow: /es/my-sponsored
Disallow: /es/sponsor/new
Disallow: /es/sponsor/checkout

# ---------------------------------------------------------------------------
# AI assistants, answer engines & training crawlers — explicitly welcomed.
#
# Naming them changes nothing technically: the "User-agent: *" group above
# already says "Allow: /", so every one of these is permitted whether it is
# listed here or not. The list is a statement of intent — an operator reading
# this file can see which systems the site expects to hear from, and a bot that
# looks for its own token finds an explicit Allow rather than inferring one.
#
# Grouped by operator, because that is how it gets audited against the
# verified-bot counts behind /api/stats/ai-crawlers. Not a wish list: ShapBot
# (Parallel) and PetalBot are already among the heaviest readers of this site.
#
# Same private-path rules apply — being an AI bot doesn't open account pages.
#
# KEEP IN STEP with robotsAIAgents in the API's api/handlers/seo_handler — this
# static file is canonical at the site origin, that one covers same-origin and
# local testing, and they must not drift.
# ---------------------------------------------------------------------------
# OpenAI
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: OAI-AdsBot
User-agent: ChatGPT-User
User-agent: ChatGPT Agent
# Anthropic
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: Claude-Code
User-agent: anthropic-ai
# Google
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: Google-NotebookLM
User-agent: Gemini-Deep-Research
User-agent: Google-Gemini-CLI
User-agent: GoogleAgent-URLContext
# Microsoft & Amazon
User-agent: AzureAI-SearchBot
User-agent: bedrockbot
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
# Meta
User-agent: meta-externalagent
User-agent: Meta-ExternalFetcher
User-agent: meta-webindexer
# Apple
User-agent: Applebot
User-agent: Applebot-Extended
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# DuckDuckGo
User-agent: DuckAssistBot
# Mistral
User-agent: MistralAI-Index
User-agent: MistralAI-User
# Parallel Web Systems
User-agent: ShapBot
User-agent: Shap-User
# DeepSeek, Moonshot, Alibaba, ByteDance, Huawei
User-agent: DeepSeekBot
User-agent: Kimi-SearchBot
User-agent: Kimi-User
User-agent: KimiBot
User-agent: QwenBot
User-agent: TongyiBot
User-agent: DoubaoBot
User-agent: Bytespider
User-agent: PetalBot
User-agent: PanguBot
# Answer engines & retrieval APIs
User-agent: ExaBot
User-agent: ExaSearchBot
User-agent: TavilyBot
User-agent: LinkupBot
User-agent: FirecrawlAgent
User-agent: kagi-fetcher
User-agent: YouBot
User-agent: PhindBot
User-agent: LinerBot
User-agent: Andibot
User-agent: iAskBot
# Open corpora that feed many models
User-agent: CCBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: cohere-ai
User-agent: Timpibot
User-agent: Webzio-Extended
User-agent: omgili
User-agent: omgilibot
User-agent: Diffbot
User-agent: Diffbot-User
Allow: /
Disallow: /auth/
Disallow: /dashboard
Disallow: /my-products
Disallow: /my-votes
Disallow: /my-comments
Disallow: /activity
Disallow: /submit
Disallow: /settings
Disallow: /billing
Disallow: /unsubscribe
Disallow: /embed/
Disallow: /product/*/edit
Disallow: /product/*/analytics
Disallow: /product/*/boost
Disallow: /r/
Disallow: /my-sponsored
Disallow: /sponsor/new
Disallow: /sponsor/checkout
# Locale variants of the paths above — these resolve too (see /de/…, /es/…),
# so they must be closed in every language, not just at the root.
Disallow: /de/auth/
Disallow: /de/dashboard
Disallow: /de/my-products
Disallow: /de/my-votes
Disallow: /de/my-comments
Disallow: /de/activity
Disallow: /de/submit
Disallow: /de/settings
Disallow: /de/billing
Disallow: /de/unsubscribe
Disallow: /de/embed/
Disallow: /de/product/*/edit
Disallow: /de/product/*/analytics
Disallow: /de/product/*/boost
Disallow: /de/r/
Disallow: /de/my-sponsored
Disallow: /de/sponsor/new
Disallow: /de/sponsor/checkout
Disallow: /es/auth/
Disallow: /es/dashboard
Disallow: /es/my-products
Disallow: /es/my-votes
Disallow: /es/my-comments
Disallow: /es/activity
Disallow: /es/submit
Disallow: /es/settings
Disallow: /es/billing
Disallow: /es/unsubscribe
Disallow: /es/embed/
Disallow: /es/product/*/edit
Disallow: /es/product/*/analytics
Disallow: /es/product/*/boost
Disallow: /es/r/
Disallow: /es/my-sponsored
Disallow: /es/sponsor/new
Disallow: /es/sponsor/checkout

Sitemap: https://appboard.pro/sitemap.xml