# robots.txt for leadarray.ai
# Hosted on Vercel | Last updated: 2026-03-09
#
# NOTE: This file covers the marketing site (www.leadarray.ai).
# The app (app.leadarray.ai) needs its own robots.txt if desired.

# =============================================
# Default: Allow all legitimate crawlers
# =============================================
User-agent: *
Allow: /

# Vercel internals (no SEO value)
Disallow: /_next/
Disallow: /_vercel/

# API routes (Next.js serverless functions)
Disallow: /api/

# Internal-only pages (not for public indexing)
Disallow: /public/checklist

# Auth and account pages (thin/duplicate content, no SEO value)
Disallow: /login
Disallow: /signup
Disallow: /auth/
Disallow: /reset-password
Disallow: /verify
Disallow: /callback

# Prevent indexing of tracking/session URLs
Disallow: /*?ref=*
Disallow: /*?utm_*
Disallow: /*?session=*
Disallow: /*?token=*

# =============================================
# Google (maximize crawl of marketing pages)
# =============================================
User-agent: Googlebot
Allow: /
Allow: /TOS
Allow: /privacy
Allow: /company
Allow: /roi-calculator
Disallow: /_next/
Disallow: /_vercel/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/

# =============================================
# Bing
# =============================================
User-agent: Bingbot
Allow: /
Disallow: /_next/
Disallow: /_vercel/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth/

# =============================================
# AI browsing/search crawlers — ALLOWED
# These serve live user queries (AISEO visibility)
# llms.txt at https://www.leadarray.ai/llms.txt
# =============================================
User-agent: GPTBot
Allow: /
Disallow: /api/

User-agent: ChatGPT-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# =============================================
# AI training corpus scrapers — BLOCKED
# Content not licensed for LLM training data
# =============================================
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

# =============================================
# Block aggressive SEO scrapers
# These burn Vercel bandwidth and expose your
# keyword/content strategy to competitors
# =============================================
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# =============================================
# Sitemap (update www if that's your canonical)
# =============================================
Sitemap: https://www.leadarray.ai/sitemap.xml