# Metrix.bio robots.txt
#
# Policy (2026-07): AI SEARCH indexers and AI AGENT fetchers are welcome —
# they cite/link back and fetch pages live when users ask (referral traffic).
# AI TRAINING crawlers are blocked — content may not be used to train or
# fine-tune models. Machine-readable version: the Content-Signal lines below
# (https://contentsignals.org). Enforced at the edge via Cloudflare AI Crawl
# Control; deceptive crawlers are handled by our own bot mitigation.
# Search engines (full access to public pages)
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: YandexBot
Disallow: /admin/
Disallow: /api/
# /go/ is deliberately NOT disallowed here. Those are outbound redirects that
# answer X-Robots-Tag: noindex, but a URL a crawler may not fetch can still be
# indexed from links alone — and was: five /go/ URLs took 13% of this site's
# impressions in the 28 days to 2026-09-28, at positions 1.0-5.5, showing a
# redirect where the seller page should be. Letting them be fetched is what
# lets the noindex be read, after which they drop out. The links themselves
# carry rel="nofollow", so this does not invite a crawl of all of them.
# The AI groups below keep the Disallow: an indexer that honours noindex gains
# nothing by fetching a redirect.
Disallow: /seller-dashboard/
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /verify-email/
Disallow: /resend-verification
Disallow: /forgot-password
Disallow: /reset-password/
Disallow: /user-profile
Disallow: /claim-seller/
Disallow: /analytics/view/
Disallow: /unsubscribe/
# Noindex'd filter/search variants - don't waste crawl budget
Disallow: /research?
Disallow: /news?
Disallow: /videos?
Disallow: /sellers?
Disallow: /products?
# But DO allow ?page= so Googlebot can reach paginated seller/product
# detail links (referenced in sitemap.xml). Longest-match wins, so these
# specific Allows override the broader Disallows above.
#
# /videos?page= is NOT among them (owner, 2026-09-28). It was the one tap into
# 34,428 video pages, each of them a YouTube transcript we republish, and every
# one now answers noindex. Advertising the pagination invited a crawl of all of
# them to read 34,428 noindex tags — the /research misallocation twenty times
# over, at a moment when Google is indexing 3 of this site's 21.2K known URLs.
# The hub's first page and the detail pages stay fetchable by path; we just
# stop pointing at the rest. Videos are for visitors here, not for the index.
Allow: /sellers?page=
Allow: /products?page=
Allow: /research?page=
Allow: /news?page=
Allow: /
# AI answer engines, split by what the bot actually DOES with the page.
#
# INDEXERS build an index ahead of time, so a page we mark noindex is one they
# can never surface — fetching it is pure cost with no possible upside. They
# get the same query-string exclusions Googlebot has. Measured 2026-08-19:
# 76% of everything Claude-SearchBot fetched from us (7,728 of 10,180 req/24h,
# 245MB) was query-string listing pages, EVERY one of which serves
# — /research?q= (3,995), the /sellers?
# letter x product facet grid (3,636), /videos? and /news?. That crawl was the
# main reason the site felt slow to real visitors.
#
# Nothing citable is withheld: every /research/ study page (7,421 of
# them), product, seller and chart page stays fully crawlable and is listed in
# sitemap.xml. Only the noindex'd filter/search permutations are off limits.
User-agent: Claude-SearchBot
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Meta-WebIndexer
Disallow: /admin/
Disallow: /api/
Disallow: /go/
Disallow: /seller-dashboard/
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /verify-email/
Disallow: /resend-verification
Disallow: /forgot-password
Disallow: /reset-password/
Disallow: /user-profile
Disallow: /claim-seller/
Disallow: /analytics/view/
Disallow: /unsubscribe/
# Noindex'd filter/search variants — an unbounded facet space they can never
# index. Deliberately NO ?page= carve-out: pagination exists to reach detail
# pages, and every detail page is already in sitemap.xml.
Disallow: /research?
Disallow: /news?
Disallow: /videos?
Disallow: /sellers?
Disallow: /products?
# Pacing hint for the indexers that honor it. Measured 2026-09-05:
# PerplexityBot fetched 100–160 seller pages inside one minute and 24% of its
# week came back 502 while the origin was busy. Google ignores Crawl-delay,
# Perplexity's docs are silent on it, so this is a courtesy; the enforced
# limit is nginx (one budget per crawler family, 429 + Retry-After).
Crawl-delay: 5
Allow: /
# LIVE AGENT FETCHERS are not indexing — they read a page to answer one
# person's question at that moment, which makes noindex irrelevant to them and
# makes each fetch real referral traffic. No query-string restrictions: if a
# user asks about a compound, the agent should be free to read the filtered
# view. In 24h these fetched ZERO query-string pages between them (ChatGPT-User
# made 101 requests, none of them a query URL), so this costs nothing today —
# it just refuses to shut a door we actually want open.
# DuckAssistBot sits here deliberately: it is arguably an indexer, but it made
# 4 requests in 24h, and misfiling a live agent as a crawler is the expensive
# mistake of the two.
User-agent: Claude-User
User-agent: ChatGPT-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
User-agent: DuckAssistBot
Disallow: /admin/
Disallow: /api/
Disallow: /go/
Disallow: /seller-dashboard/
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /verify-email/
Disallow: /resend-verification
Disallow: /forgot-password
Disallow: /reset-password/
Disallow: /user-profile
Disallow: /claim-seller/
Disallow: /analytics/view/
Disallow: /unsubscribe/
Allow: /
# MetrixBot (our own crawler for link validation and price checking)
User-agent: MetrixBot
Disallow: /admin/
Disallow: /api/
Disallow: /seller-dashboard/
Allow: /
# AI training bots (block - no referral value)
User-agent: GPTBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: Meta-ExternalAgent
Disallow: /
# Ad-targeting crawler (not a search indexer - no referral value)
User-agent: Meta-ExternalAds
Disallow: /
User-agent: Diffbot
Disallow: /
# Training opt-out tokens (not crawlers): keep Google Search + Applebot/Siri
# crawling while opting out of Gemini / Apple Intelligence model training.
User-agent: Google-Extended
Disallow: /
User-agent: Applebot-Extended
Disallow: /
# SEO / backlink crawlers (blocked at the edge anyway — listed here so the
# well-behaved ones stop rather than bouncing off the WAF. SemrushBot alone was
# 3,539 edge blocks/day, ~24% of all blocks, purely because it was never told.)
User-agent: SemrushBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: BacklinksExtendedBot
Disallow: /
User-agent: Barkrowler
Disallow: /
# Generic scrapers and AI crawlers
User-agent: Scrapy
Disallow: /
User-agent: img2dataset
Disallow: /
# Default: allow everything public, block private
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /admin/
Disallow: /api/
Disallow: /go/
Disallow: /seller-dashboard/
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /verify-email/
Disallow: /resend-verification
Disallow: /forgot-password
Disallow: /reset-password/
Disallow: /user-profile
Disallow: /claim-seller/
Disallow: /analytics/view/
Disallow: /unsubscribe/
Disallow: /research?
Disallow: /news?
Disallow: /videos?
Disallow: /sellers?
Disallow: /products?
Allow: /sellers?page=
Allow: /products?page=
Allow: /research?page=
Allow: /news?page=
# /videos?page= deliberately absent here too — see the search-engine group.
# Honeypot decoys — do not crawl. Compliant bots skip these; only bad actors
# that probe Disallow-ed paths will hit them and get challenged.
Disallow: /internal/data/catalog
Disallow: /api/v1/sellers-export
Allow: /
# Sitemap
Sitemap: https://metrix.bio/sitemap.xml