# =============================================
# OSKeep.com - 操作系统下载站 robots.txt
# 最后更新: 2024-01-16
# 模式: 纯Slug模式
# =============================================

User-agent: *

# ==== 允许的重要页面和目录 ====
Allow: /
Allow: /browse/
Allow: /download/
Allow: /search/
Allow: /about/
Allow: /contact/
Allow: /privacy/
Allow: /sitemap.xml
Allow: /sitemap.html
Allow: /robots.txt
Allow: /favicon.ico

# ==== 静态资源允许 ====
Allow: /assets/
Allow: /images/
Allow: /uploads/
Allow: /css/
Allow: /js/
Allow: /fonts/

# ==== 禁止爬取的敏感目录 ====
Disallow: /admin/
Disallow: /config/
Disallow: /includes/
Disallow: /backup/
Disallow: /tmp/
Disallow: /logs/
Disallow: /phpmyadmin/

# ==== 禁止爬取系统文件 ====
Disallow: /*.inc$
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.bak$
Disallow: /*.env$
Disallow: /*.htaccess$

# ==== 允许伪静态分页 ====
Allow: /browse/*/page/
Allow: /search/*/page/

# ==== 允许伪静态URL参数（纯Slug模式） ====
Allow: /browse/*/
Allow: /download/*/
Allow: /search/*/

# ==== 禁止传统查询参数URL（避免重复内容） ====
Disallow: /*?id=*
Disallow: /*?page=*
Disallow: /*?action=*
Disallow: /*?ajax=*
Disallow: /*?format=*

# ==== 禁止特定爬虫敏感路径 ====
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /package.json
Disallow: /.git/
Disallow: /node_modules/
Disallow: /vendor/

# ==== Sitemap 声明 ====
Sitemap: https://oskeep.com/sitemap.xml

# =============================================
# 特定搜索引擎优化规则
# =============================================

# ==== Google 优化 ====
User-agent: Googlebot
Allow: /
Allow: /browse/
Allow: /download/
Allow: /search/
Allow: /sitemap.xml
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /assets/images/
Allow: /uploads/
Crawl-delay: 1

# ==== 百度 优化 ====
User-agent: Baiduspider
Allow: /
Allow: /browse/
Allow: /download/
Allow: /search/
Allow: /sitemap.xml
Crawl-delay: 3

User-agent: Baiduspider-image
Allow: /assets/images/
Allow: /uploads/
Crawl-delay: 3

# ==== Bing 优化 ====
User-agent: bingbot
Allow: /
Allow: /browse/
Allow: /download/
Allow: /search/
Allow: /sitemap.xml
Crawl-delay: 2

# ==== 国内搜索引擎优化 ====
User-agent: Sogou web spider
Allow: /
Crawl-delay: 5

User-agent: 360Spider
Allow: /
Crawl-delay: 5

User-agent: YisouSpider
Allow: /
Crawl-delay: 5

# =============================================
# 友好爬虫和工具
# =============================================

User-agent: Applebot
Allow: /
Crawl-delay: 2

User-agent: Twitterbot
Allow: /
Crawl-delay: 2

User-agent: facebookexternalhit
Allow: /
Crawl-delay: 2

User-agent: LinkedInBot
Allow: /
Crawl-delay: 2

User-agent: WeChat
Allow: /
Crawl-delay: 2

# =============================================
# 限制商业和恶意爬虫
# =============================================

# 商业SEO工具 - 允许但限速
User-agent: AhrefsBot
Disallow: /admin/
Disallow: /config/
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /admin/
Disallow: /config/
Crawl-delay: 10

User-agent: SemrushBot
Disallow: /admin/
Disallow: /config/
Crawl-delay: 10

User-agent: DotBot
Disallow: /admin/
Disallow: /config/
Crawl-delay: 10

# AI爬虫 - 允许但限速
User-agent: GPTBot
Disallow: /admin/
Crawl-delay: 5

User-agent: CCBot
Disallow: /admin/
Crawl-delay: 5

# 数据收集爬虫 - 限制
User-agent: MauiBot
Disallow: /
Crawl-delay: 10

User-agent: zoominfobot
Disallow: /
Crawl-delay: 10

# =============================================
# 简化版本（针对其他所有爬虫）
# =============================================

# 通用爬取延迟设置
Crawl-delay: 2

Sitemap: https://www.oskeep.com/sitemap.xml