# robots.txt — Sun Life Real Estate (sunlife.realestate)
# Updated: 2026-07-06

# ============================================================
# DEFAULT: All crawlers
# ============================================================
User-agent: *
Allow: /

# --- Block private/admin sections ---
Disallow: /admin
Disallow: /admin/
Disallow: /login
Disallow: /client-portal
Disallow: /my-dossiers
Disallow: /my-account

# --- Block parameterized URLs that create duplicate content ---
Disallow: /*?search=
Disallow: /*?filter=
Disallow: /*?sort=
Disallow: /*?tab=
Disallow: /*?preview=
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*&search=
Disallow: /*&filter=
Disallow: /*&sort=

# --- Block staging/draft/test content ---
Disallow: /preview/
Disallow: /draft/
Disallow: /test
Disallow: /test/

# ============================================================
# SEARCH ENGINES — Full crawl access to public content
# ============================================================
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /login
Disallow: /client-portal
Disallow: /my-dossiers

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /login

User-agent: Slurp
Allow: /
Disallow: /admin

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: AmazonBot
Allow: /

# ============================================================
# AI DISCOVERY CRAWLERS — Full access to enable AI citations
# ============================================================

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Google Gemini / AI Mode / Vertex
User-agent: Google-Extended
Allow: /

User-agent: Googlebot-AI
Allow: /

# xAI / Grok
User-agent: Grok
Allow: /

User-agent: xAI
Allow: /

# Meta AI
User-agent: facebookexternalhit
Allow: /

User-agent: meta-externalagent
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Common Crawl (foundation for many AI training datasets)
User-agent: CCBot
Allow: /

# ByteDance / TikTok
User-agent: Bytespider
Allow: /

# Microsoft Copilot / Bing AI
User-agent: bingbot
Allow: /

User-agent: adidxbot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Mistral AI
User-agent: MistralAI-User
Allow: /

# Diffbot (structured content extraction)
User-agent: DiffBot
Allow: /

# Timpi
User-agent: Timpibot
Allow: /

# ============================================================
# AI-READABLE DISCOVERY ENDPOINTS
# ============================================================
# llms.txt  (structured AI summary):      https://sunlife.realestate/llms.txt
# llms-full.txt (complete AI content):    https://sunlife.realestate/llms-full.txt
# Sitemap index:                          https://sunlife.realestate/sitemap.xml
# Listings sitemap:                       https://sunlife.realestate/sitemap-listings.xml
# Image sitemap:                          https://sunlife.realestate/sitemap-images.xml
# Blog sitemap:                           https://sunlife.realestate/sitemap-blog.xml

# ============================================================
# SITEMAPS
# ============================================================
Sitemap: https://sunlife.realestate/sitemap.xml

# ============================================================
# CRAWL RATE
# ============================================================
Crawl-delay: 1