User-agent: *
Allow: /

# AI Crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

# High-priority content for search engines and AI
Allow: /research-reports
Allow: /research/*
Allow: /public-research-listings
Allow: /intern-overview
Allow: /cv-writing-guide
Allow: /internship-guide
Allow: /features
Allow: /about
Allow: /contact
Allow: /pricing

# Block admin and user-specific pages
Disallow: /admin*
Disallow: /dashboard*
Disallow: /profile*
Disallow: /analytics*
Disallow: /user-management*
Disallow: /system-settings*

# Block API endpoints
Disallow: /api/
Disallow: /_*

# Block search result pages with parameters
Disallow: /*?*
Allow: /research-reports?*

# Allow specific file types
Allow: *.css
Allow: *.js
Allow: *.png
Allow: *.jpg
Allow: *.jpeg
Allow: *.gif
Allow: *.svg
Allow: *.webp
Allow: *.pdf

# Crawl delay for respectful crawling
Crawl-delay: 1

# Sitemap locations
Sitemap: https://trainee.in/sitemap-index.xml
Sitemap: https://trainee.in/sitemap.xml
Sitemap: https://trainee.in/sitemap-static.xml
Sitemap: https://trainee.in/sitemap-research.xml
Sitemap: https://trainee.in/image-sitemap.xml

# AI crawler instructions
# See https://trainee.in/llms.txt for detailed AI crawling guidelines