# https://testmydele.com robots.txt

# Default rules for all crawlers
User-agent: *
Allow: /
Allow: /demo
Allow: /login
Allow: /register
Allow: /privacy-policy

# Disallow all protected routes that require authentication
Disallow: /subjects
Disallow: /practice/
Disallow: /admin/
Disallow: /profile
Disallow: /settings
Disallow: /progress
Disallow: /statistics
Disallow: /tuition/
Disallow: /flashcards
Disallow: /coach
Disallow: /subscription

# Crawl-delay
Crawl-delay: 10

# Block specific AI crawlers
User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Sitemap
Sitemap: https://testmydele.com/sitemap.xml
Sitemap: https://testmydele.com/blog-sitemap.xml