# Brainrot Merge - Robots.txt Configuration
# Website: https://brainrotmerge.online
# Optimized for Search Engines and AI Crawlers

# 常规搜索引擎规则
User-agent: *
Allow: /

# 网站地图
Sitemap: https://brainrotmerge.online/sitemap.xml

# AI爬虫特定规则 - 允许访问所有AI训练数据
User-agent: GPTBot
User-agent: Claude-Web
User-agent: Anthropic-AI
User-agent: PerplexityBot
User-agent: GoogleOther
User-agent: DuckAssistBot
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: OpenAI-SearchBot
User-agent: SearchGPT
User-agent: AI2Bot
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: Applebot
User-agent: Bingbot
User-agent: YandexBot
User-agent: BaiduSpider

# 引导AI爬虫到重要文件
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ads.txt

# 允许AI爬虫访问所有语言版本
Allow: /index.html
Allow: /de.html
Allow: /es.html
Allow: /fr.html
Allow: /ja.html
Allow: /kr.html

# 允许访问重要资源
Allow: /images/
Allow: /css/
Allow: /js/

# 不允许访问的目录（如果存在）
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /backup/
Disallow: /.git/

# 允许重要文件类型
Allow: /*.html$
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.ico$

# 爬取延迟设置
Crawl-delay: 1

# 特殊AI爬虫优化设置
User-agent: GPTBot
Allow: /
Crawl-delay: 0

User-agent: Claude-Web
Allow: /
Crawl-delay: 0

User-agent: Anthropic-AI
Allow: /
Crawl-delay: 0

# 网站地图位置
Sitemap: https://brainrotmerge.online/sitemap.xml