# AI Bots - Training & Scraping
User-agent: Meta-ExternalAgent
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: Amazonbot
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: ChatGPT-User
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: Bytespider
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: ClaudeBot
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: OAI-SearchBot
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: GPTBot
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: PerplexityBot
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: CCBot
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: PetalBot
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: Perplexity-User
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: DuckAssistBot
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: MistralAI-User
Crawl-delay: 60
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

# SEO & Scraper Bots
User-agent: AhrefsBot
Crawl-delay: 30
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: SemrushBot
Crawl-delay: 30
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: SiteAuditBot
Crawl-delay: 30
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: dotbot
Crawl-delay: 30
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: rogerbot
Crawl-delay: 30
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

# Search Engines
User-agent: Googlebot
Crawl-delay: 5
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: Bingbot
Crawl-delay: 5
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

User-agent: Applebot
Crawl-delay: 5
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

# All Other Bots
User-agent: *
Crawl-delay: 10
Allow: /
Disallow: /api/
Disallow: /graphql
Disallow: /search/
Disallow: /health

# Sitemap
Sitemap: https://laidlawscholars.network/sitemap.xml.gz