User-agent: *
# WordPress Admin
Disallow: /blog/wp-admin/
Allow: /blog/wp-admin/admin-ajax.php
Disallow: /blog/wp-login.php
Disallow: /blog/wp-register.php
# Thin Content Archives
Disallow: /blog/tag/
Disallow: /blog/author/
# Search Pages
Disallow: /blog/?s=
# Feeds
Disallow: */feed/
# Sitemaps
Sitemap: https://devtechnosys.ae/sitemap.xml
Sitemap: https://devtechnosys.ae/blog/sitemap_index.xml

# ------------------------------------------------------------
# AI Crawlers - explicitly allowed for GEO/AEO visibility
# ------------------------------------------------------------
# OpenAI - ChatGPT training
User-agent: GPTBot
Allow: /
# OpenAI - ChatGPT real-time browsing
User-agent: ChatGPT-User
Allow: /
# Anthropic - Claude
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /
# Perplexity AI
User-agent: PerplexityBot
Allow: /
# Apple - Siri, Spotlight
User-agent: Applebot
Allow: /
# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
User-agent: Meta-ExternalFetcher
Allow: /
# ByteDance / TikTok AI
User-agent: Bytespider
Allow: /
# You.com
User-agent: YouBot
Allow: /
# Cohere AI
User-agent: cohere-ai
Allow: /
# Amazon Alexa
User-agent: Amazonbot
Allow: /
# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /

# ------------------------------------------------------------
# Google - Image crawler
# ------------------------------------------------------------
User-agent: Googlebot-Image
Allow: /
# ------------------------------------------------------------
# Google - Video crawler
# ------------------------------------------------------------
User-agent: Googlebot-Video
Allow: /
# ------------------------------------------------------------
# Google - AI training (Gemini, Vertex AI)
# ------------------------------------------------------------
User-agent: Google-Extended
Allow: /
User-agent: GoogleOther
Allow: /
User-agent: GoogleOther-Image
Allow: /
# ------------------------------------------------------------
# Bing / Microsoft
# ------------------------------------------------------------
User-agent: Bingbot
Allow: /
User-agent: MicrosoftEdge
Allow: /