# robots.txt for HidenCloud
# https://www.hidencloud.com

# ============================================
# AI CRAWLERS - Specific rules for AI bots
# ============================================

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

# OpenAI ChatGPT browsing
User-agent: ChatGPT-User
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*

# Google AI (Bard/Gemini training)
User-agent: Google-Extended
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

# Meta AI
User-agent: FacebookBot
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

# Cohere AI
User-agent: cohere-ai
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

# Amazon Alexa
User-agent: Amazonbot
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

# ByteDance
User-agent: Bytespider
Allow: /
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/
Disallow: /*.php
Disallow: /.*
Crawl-delay: 1

# ============================================
# STANDARD SEARCH ENGINE CRAWLERS
# ============================================

User-agent: *
Crawl-delay: 10

# Allow public pages
Allow: /
Allow: /service/
Allow: /about-us
Allow: /contact-us
Allow: /terms-of-service
Allow: /privacy-policy
Allow: /cookies-policy
Allow: /free-discord-hosting
Allow: /free-minecraft-hosting
Allow: /discord-hosting
Allow: /minecraft-hosting
Allow: /python-hosting
Allow: /node-hosting
Allow: /game-servers
Allow: /free-tier

# Allow comparison pages (SEO)
Allow: /compare/

# Allow keyword landing pages (SEO)
Allow: /cheap-*
Allow: /best-*
Allow: /affordable-*
Allow: /premium-*
Allow: /fast-*

# Allow public assets
Allow: /assets/
Allow: /sitemap.xml
Allow: /sitemap.xsl
Allow: /ads.txt
Allow: /ai.txt
Allow: /manifest.json

# Disallow internal directories
Disallow: /vendor/
Disallow: /src/
Disallow: /lang/
Disallow: /hidencloudstorage/

# Disallow configuration and log files
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /helpers.php
Disallow: /switch-lang.php
Disallow: /php_errorlog
Disallow: /*.log

# Disallow direct PHP file access (except routed URLs)
Disallow: /*.php

# Disallow hidden files
Disallow: /.*

# Sitemap location
Sitemap: https://www.hidencloud.com/sitemap.xml

# Crawl-delay (optional, be gentle with the server)
Crawl-delay: 1