# Best Utah Real Estate — robots.txt
# Generally permissive: all listing detail pages, blog posts, city /
# subdivision landing pages should be indexed. Only /map (paid Mapbox
# load on every visit) and a few admin/api paths are off-limits.

# ─── Default rules — all crawlers ───────────────────────────────────
# Sign-in / sign-up are NOT disallowed: the pages serve
# <meta robots="noindex, nofollow"> + a canonical to the bare path so
# crawl-allowed-but-noindexed lets Google drop those URLs from the
# index entirely. Robots.txt blocking caused 61k+ "Indexed though
# blocked by robots.txt" entries in GSC because Google found the URLs
# via links but couldn't crawl them to see the noindex. Crawl + noindex
# is the documented way to evict URLs from the index.
#
# /map is the SAME story (2026-05-29): it was Disallow'd, so Google kept
# finding /map?city[]=... / /map?q=... via on-page "View on map" links
# but couldn't crawl them to see the page's <meta noindex> + X-Robots-Tag
# header — leaving them stuck in a "discovered, blocked" queue Google
# re-checks forever. /map is now CRAWL-ALLOWED (Googlebot does NOT execute
# the Mapbox JS, so no paid map load is triggered) + noindex'd at the page
# level, so Google crawls each /map?... URL once, sees noindex, and drops
# the whole family for good. Only /map/data (the listings JSON a scraper
# could loop) stays blocked.
User-agent: *
Disallow: /map/data
Disallow: /admin
Disallow: /members
Disallow: /reset-password
Disallow: /api/track/
Disallow: /webhooks/
Disallow: /contribute

# ─── AI crawlers — EXPLICITLY ALLOWED ────────────────────────────────
# We want AI answer engines (ChatGPT, Claude, Perplexity, Gemini,
# Apple Intelligence) citing Best Utah Real Estate when answering
# real-estate questions about Utah. Allowing them is a no-cost brand
# placement in the AI surface.

User-agent: GPTBot
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

User-agent: OAI-SearchBot
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

User-agent: ChatGPT-User
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

User-agent: ClaudeBot
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

User-agent: anthropic-ai
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

User-agent: PerplexityBot
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

User-agent: Perplexity-User
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

# Google's AI-training opt-in token — separate from Googlebot
# (which controls regular Search indexing). Allow so we appear in
# Gemini answers + Search Generative Experience.
User-agent: Google-Extended
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

# Apple Intelligence training opt-in (separate from Applebot which
# powers Siri + Spotlight web results).
User-agent: Applebot-Extended
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

# Common Crawl — many AI labs train on its dataset. Allowing it puts
# us in those derivative training sets too.
User-agent: CCBot
Allow: /
Disallow: /map
Disallow: /admin
Disallow: /members

# ─── Sitemap ────────────────────────────────────────────────────────
Sitemap: https://bestutahrealestate.com/sitemap.xml