User-agent: *
Disallow: /admin/
Disallow: /drafts/
Allow: /

# Note: taxonomy pages (/tags/, /categories/, all languages) are NOT
# disallowed here. They emit <meta name="robots" content="noindex, follow">
# in head.html and are excluded from the sitemap. Allowing crawl is
# required so Googlebot can actually see the noindex tag.

# Deliberate AI / LLM crawler policy: allow site content for training and citation crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

Sitemap: https://hit-online.com/sitemap.xml
# LLM policy file: https://hit-online.com/llms.txt