# ============================================================
# VD HOSTS (VDHOSTS INTERNET PRIVATE LIMITED)
# Search engine and AI crawler policy
#
# Public pages are open to search engines and AI assistants.
# Private, signed-in and form-processing areas are closed.
#
# Summary for AI assistants: https://vdhosts.com/llms.txt
# Full text for AI assistants: https://vdhosts.com/llms-full.txt
# ============================================================

# ------------------------------------------------------------
# DEFAULT POLICY (every other crawler)
# ------------------------------------------------------------

User-agent: *
Allow: /

# Private, signed-in and non-search areas
Disallow: /admin/
Disallow: /account
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /portal/
Disallow: /vd-talk/
Disallow: /captcha/
Disallow: /sanctum/
Disallow: /country
Disallow: /search
# The same pages under each country (doc 37: /in/login, /us/account...)
Disallow: /*/account
Disallow: /*/login
Disallow: /*/register
Disallow: /*/forgot-password
Disallow: /*/reset-password
Disallow: /*/search
Disallow: /*/privacy-choices
Disallow: /newsletter/
Disallow: /webhooks/
Disallow: /payment/
Disallow: /api/
Disallow: /ajax/
Disallow: /cron/
Disallow: /_ops/
Disallow: /logs/
Disallow: /tmp/
Disallow: /cache/
Disallow: /vendor/
Disallow: /config/
Disallow: /private/
Disallow: /internal/
Disallow: /storage/

# Session, preview and tracking parameters (duplicate URLs)
Disallow: /*?*session=
Disallow: /*?*sessionid=
Disallow: /*?*sid=
Disallow: /*?*token=
Disallow: /*?*auth=
Disallow: /*?*preview=
Disallow: /*?*debug=
Disallow: /*?*internal=
Disallow: /*?*admin=

# Public files: styles, scripts, images, fonts and media stay readable
Allow: /build/
Allow: /images/
Allow: /fonts/
Allow: /favicon
Allow: /storage/blog/
Allow: /storage/seo/
Allow: /storage/about/
Allow: /storage/site/
Allow: /storage/videos/

# ------------------------------------------------------------
# SEARCH ENGINES
# Google, Bing (also powers Microsoft Copilot and Yahoo),
# DuckDuckGo, Apple (Siri and Spotlight), Yandex, Baidu, Naver
# ------------------------------------------------------------

User-agent: Googlebot
User-agent: Bingbot
User-agent: msnbot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Yeti
Allow: /

# Private, signed-in and non-search areas
Disallow: /admin/
Disallow: /account
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /portal/
Disallow: /vd-talk/
Disallow: /captcha/
Disallow: /sanctum/
Disallow: /country
Disallow: /search
# The same pages under each country (doc 37: /in/login, /us/account...)
Disallow: /*/account
Disallow: /*/login
Disallow: /*/register
Disallow: /*/forgot-password
Disallow: /*/reset-password
Disallow: /*/search
Disallow: /*/privacy-choices
Disallow: /newsletter/
Disallow: /webhooks/
Disallow: /payment/
Disallow: /api/
Disallow: /ajax/
Disallow: /cron/
Disallow: /_ops/
Disallow: /logs/
Disallow: /tmp/
Disallow: /cache/
Disallow: /vendor/
Disallow: /config/
Disallow: /private/
Disallow: /internal/
Disallow: /storage/

# Session, preview and tracking parameters (duplicate URLs)
Disallow: /*?*session=
Disallow: /*?*sessionid=
Disallow: /*?*sid=
Disallow: /*?*token=
Disallow: /*?*auth=
Disallow: /*?*preview=
Disallow: /*?*debug=
Disallow: /*?*internal=
Disallow: /*?*admin=

# Public files: styles, scripts, images, fonts and media stay readable
Allow: /build/
Allow: /images/
Allow: /fonts/
Allow: /favicon
Allow: /storage/blog/
Allow: /storage/seo/
Allow: /storage/about/
Allow: /storage/site/
Allow: /storage/videos/

# ------------------------------------------------------------
# IMAGE, VIDEO AND LINK-PREVIEW CRAWLERS
# ------------------------------------------------------------

User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Storebot-Google
User-agent: BingPreview
User-agent: msnbot-media
User-agent: LinkedInBot
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: WhatsApp
User-agent: Slackbot
Allow: /

# Private, signed-in and non-search areas
Disallow: /admin/
Disallow: /account
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /portal/
Disallow: /vd-talk/
Disallow: /captcha/
Disallow: /sanctum/
Disallow: /country
Disallow: /search
# The same pages under each country (doc 37: /in/login, /us/account...)
Disallow: /*/account
Disallow: /*/login
Disallow: /*/register
Disallow: /*/forgot-password
Disallow: /*/reset-password
Disallow: /*/search
Disallow: /*/privacy-choices
Disallow: /newsletter/
Disallow: /webhooks/
Disallow: /payment/
Disallow: /api/
Disallow: /ajax/
Disallow: /cron/
Disallow: /_ops/
Disallow: /logs/
Disallow: /tmp/
Disallow: /cache/
Disallow: /vendor/
Disallow: /config/
Disallow: /private/
Disallow: /internal/
Disallow: /storage/

# Session, preview and tracking parameters (duplicate URLs)
Disallow: /*?*session=
Disallow: /*?*sessionid=
Disallow: /*?*sid=
Disallow: /*?*token=
Disallow: /*?*auth=
Disallow: /*?*preview=
Disallow: /*?*debug=
Disallow: /*?*internal=
Disallow: /*?*admin=

# Public files: styles, scripts, images, fonts and media stay readable
Allow: /build/
Allow: /images/
Allow: /fonts/
Allow: /favicon
Allow: /storage/blog/
Allow: /storage/seo/
Allow: /storage/about/
Allow: /storage/site/
Allow: /storage/videos/

# ------------------------------------------------------------
# AI SEARCH AND AI ASSISTANTS
# OpenAI (ChatGPT search, ChatGPT, GPTBot), Anthropic (Claude),
# Perplexity, Meta AI, Mistral (Le Chat), Amazon, DuckAssist,
# Common Crawl (used by many AI models), Cohere, You.com
# ------------------------------------------------------------

User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: MistralAI-User
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: CCBot
User-agent: cohere-ai
User-agent: YouBot
Allow: /

# Private, signed-in and non-search areas
Disallow: /admin/
Disallow: /account
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /portal/
Disallow: /vd-talk/
Disallow: /captcha/
Disallow: /sanctum/
Disallow: /country
Disallow: /search
# The same pages under each country (doc 37: /in/login, /us/account...)
Disallow: /*/account
Disallow: /*/login
Disallow: /*/register
Disallow: /*/forgot-password
Disallow: /*/reset-password
Disallow: /*/search
Disallow: /*/privacy-choices
Disallow: /newsletter/
Disallow: /webhooks/
Disallow: /payment/
Disallow: /api/
Disallow: /ajax/
Disallow: /cron/
Disallow: /_ops/
Disallow: /logs/
Disallow: /tmp/
Disallow: /cache/
Disallow: /vendor/
Disallow: /config/
Disallow: /private/
Disallow: /internal/
Disallow: /storage/

# Session, preview and tracking parameters (duplicate URLs)
Disallow: /*?*session=
Disallow: /*?*sessionid=
Disallow: /*?*sid=
Disallow: /*?*token=
Disallow: /*?*auth=
Disallow: /*?*preview=
Disallow: /*?*debug=
Disallow: /*?*internal=
Disallow: /*?*admin=

# Public files: styles, scripts, images, fonts and media stay readable
Allow: /build/
Allow: /images/
Allow: /fonts/
Allow: /favicon
Allow: /storage/blog/
Allow: /storage/seo/
Allow: /storage/about/
Allow: /storage/site/
Allow: /storage/videos/

# ------------------------------------------------------------
# AI USE CONTROLS (not crawlers; they allow AI products to use
# public pages crawled by Googlebot and Applebot)
# Google-Extended: Gemini and Vertex AI grounding
# Applebot-Extended: Apple Intelligence
# ------------------------------------------------------------

User-agent: Google-Extended
User-agent: Applebot-Extended
Allow: /

# Private, signed-in and non-search areas
Disallow: /admin/
Disallow: /account
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /portal/
Disallow: /vd-talk/
Disallow: /captcha/
Disallow: /sanctum/
Disallow: /country
Disallow: /search
# The same pages under each country (doc 37: /in/login, /us/account...)
Disallow: /*/account
Disallow: /*/login
Disallow: /*/register
Disallow: /*/forgot-password
Disallow: /*/reset-password
Disallow: /*/search
Disallow: /*/privacy-choices
Disallow: /newsletter/
Disallow: /webhooks/
Disallow: /payment/
Disallow: /api/
Disallow: /ajax/
Disallow: /cron/
Disallow: /_ops/
Disallow: /logs/
Disallow: /tmp/
Disallow: /cache/
Disallow: /vendor/
Disallow: /config/
Disallow: /private/
Disallow: /internal/
Disallow: /storage/

# Session, preview and tracking parameters (duplicate URLs)
Disallow: /*?*session=
Disallow: /*?*sessionid=
Disallow: /*?*sid=
Disallow: /*?*token=
Disallow: /*?*auth=
Disallow: /*?*preview=
Disallow: /*?*debug=
Disallow: /*?*internal=
Disallow: /*?*admin=

# Public files: styles, scripts, images, fonts and media stay readable
Allow: /build/
Allow: /images/
Allow: /fonts/
Allow: /favicon
Allow: /storage/blog/
Allow: /storage/seo/
Allow: /storage/about/
Allow: /storage/site/
Allow: /storage/videos/

# ------------------------------------------------------------
# SITEMAPS
# ------------------------------------------------------------

Sitemap: https://vdhosts.com/sitemap.xml
Sitemap: https://vdhosts.com/news-sitemap.xml