# Block specific file types

User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /backup/
Disallow: /tmp/
Disallow: /test/
Disallow: /cgi-bin/
Disallow: /config/
Disallow: /database/
Disallow: /logs/
Disallow: /uploads/
Disallow: /cache/
Disallow: /secure/
Disallow: /*.sql$
Disallow: /*.bak$
Disallow: /*.log$
Disallow: /*.txt$
Disallow: /*.htm$
Disallow: /*.html$
Disallow: /*.shtml$
Disallow: /*.asp$
Disallow: /*.jsp$
Disallow: /*.json$
Disallow: /*.php$
Disallow: /*.phtm$
Disallow: /*.phtml$
Disallow: /*.env$
Disallow: /*.ini$
Disallow: /*.conf$
Disallow: /*.xml$
Disallow: /*.git$
Disallow: /*.DS_Store$

# Block known bad bots
User-agent: AhrefsBot
Disallow: /
User-agent: SEMrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: spbot
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: MegaIndex
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: NetEstate NE Crawler
Disallow: /

# Block all unknown bots except search engines
User-agent: *
Crawl-delay: 10