# Private / no-SEO-value areas (also noindexed via meta robots)
User-agent: *
Disallow: /dashboard
Disallow: /payment
Disallow: /login
Disallow: /sign-up
Disallow: /recover-account
Disallow: /reset-password
Disallow: /confirm-email
Disallow: /status-tasks
Disallow: /task
Disallow: /csv/
Disallow: /subscribe
Crawl-delay: 10

# ---------------------------------------------------------------------------
# Crawl budget on the distance pair pages.
#
# /distance/from/<A>/to/<B> accepts any two places in place_directory, so the
# addressable space is ~163k^2 (~26 billion URLs) and every one returns 200.
# The sitemap advertises 28,726 pairs; crawlers reached 1,965,026 unique pair
# URLs in a single day (68x the sitemap) by walking the on-page links. Because
# ~96% of requests are first visits, the page cache never hits and each one
# costs a live Nominatim call.
#
# The hubs (/distance/from/<City>, no /to/ segment) stay open everywhere --
# they are gated to curated cities and match the sitemap set exactly.
#
# Split by what the crawler gives back: search crawlers that send referral
# traffic keep the pair pages, training-only crawlers do not.
# ---------------------------------------------------------------------------

# Search crawlers -- pair pages stay crawlable, just slower.
User-agent: Googlebot
Allow: /
Crawl-delay: 10

User-agent: Applebot
Allow: /
Crawl-delay: 10

User-agent: bingbot
Allow: /
Crawl-delay: 10

# AI search -- these surface citations and send referrals, so keep them.
User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 10

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /
Crawl-delay: 10

# Training-only crawlers -- still welcome on the rest of the site, but not on
# the combinatorial pair space. ClaudeBot alone was 492,591 requests in one day.
User-agent: ClaudeBot
Allow: /
Disallow: /distance/from/*/to/
Crawl-delay: 30

User-agent: GPTBot
Allow: /
Disallow: /distance/from/*/to/
Crawl-delay: 30

User-agent: meta-externalagent
Allow: /
Disallow: /distance/from/*/to/
Crawl-delay: 30

User-agent: Amazonbot
Allow: /
Disallow: /distance/from/*/to/
Crawl-delay: 30

User-agent: Applebot-Extended
Allow: /
Disallow: /distance/from/*/to/

User-agent: Google-Extended
Allow: /
Disallow: /distance/from/*/to/

User-agent: CCBot
Allow: /
Disallow: /distance/from/*/to/
Crawl-delay: 30

User-agent: Bytespider
Disallow: /distance/from/*/to/
Crawl-delay: 30

Sitemap: https://geloky.com/sitemaps/geo_xml_index.xml