User-agent: *
Allow: /
Allow: /blog/
Allow: /team/

# Internal / admin paths
Disallow: /admin/
Disallow: /api/
Disallow: /config/
Disallow: /db/
Disallow: /scripts/
Disallow: /PHPMailer-master/
Disallow: /libs/
Disallow: /analyzer/
Disallow: /video-audit
Disallow: /analyzer-php/

# Private user pages
Disallow: /views/
Disallow: /orders/
Disallow: /chat-system/
Disallow: /status/
Disallow: /Client/

# Auth & utility pages
# NOTE: set-lang.php is intentionally NOT blocked here — it already sends
# `X-Robots-Tag: noindex` + a 302 redirect. Blocking it in robots.txt would
# stop Googlebot from ever seeing that noindex, leaving the URLs stuck in the
# "Blocked by robots.txt" report. Letting it be crawled clears them cleanly.
Disallow: /Register.php
Disallow: /log-in.php
Disallow: /reset-password.php
Disallow: /404.php
Disallow: /500.php

# Blog internal files (not public pages)
Disallow: /Blog/data/
Disallow: /Blog/redirect_old
Disallow: /Blog/save_blog
Disallow: /Blog/delete_blog
Disallow: /Blog/update_blog
Disallow: /Blog/create_blog
Disallow: /Blog/edit_blog
Disallow: /Blog/track_views
Disallow: /Blog/blog_php_header
Disallow: /Blog/blog_page_template

Sitemap: https://nexthorizon.art/sitemap_dynamic.php

# AI Crawlers — welcome
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /config/
Disallow: /views/
Disallow: /orders/
Disallow: /chat-system/

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Bytespider
Allow: /