# robots.txt for https://propertylist.es
# Search engines and AI answer tools may read public pages.
# AI training crawlers are asked not to collect listing data (Terms 7.3 bans automated scraping and mining; the API Terms of Use, section 4, ban training on listings).

User-agent: *
Allow: /
Disallow: /user/
Disallow: /es/user/
Disallow: /admin/
Disallow: /leads/
Disallow: /problem_reports/
Disallow: /es/problem_reports/
Disallow: /*?search*
Disallow: /*?*&search%5*
Disallow: /*?utm_*

# AI training crawlers: blocked
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: CCBot
User-agent: Bytespider
User-agent: MistralAI-Training
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Coherebot
User-agent: Diffbot
Disallow: /

# AI search, assistant grounding and user-requested fetches: allowed, with the same private paths as above
# (Google-Extended stays allowed: it also controls Gemini app grounding, so blocking it would stop Gemini citing us)
User-agent: Google-Extended
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-WebIndexer
User-agent: Meta-ExternalFetcher
User-agent: Google-NotebookLM
User-agent: Google-GeminiNotebook
User-agent: MistralAI-User
User-agent: MistralAI-Index
User-agent: DuckAssistBot
User-agent: Diffbot-User
User-agent: Applebot
Allow: /
Disallow: /user/
Disallow: /es/user/
Disallow: /admin/
Disallow: /leads/
Disallow: /problem_reports/
Disallow: /es/problem_reports/
Disallow: /*?search*
Disallow: /*?*&search%5*
Disallow: /*?utm_*

Sitemap: https://propertylist.es/sitemap.xml