# SLUB digital.slub-dresden.de robots.txt
User-agent: Amazonbot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: meta-externalagent
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: Qwantbot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: PerplexityBot
Disallow: /

User-agent: *
Disallow: /fileadmin/_temp_/
Disallow: /typo3conf/
Allow: /typo3conf/ext/*/Resources/Public/*
Disallow: /typo3temp/
# allow Stylesheets for Google-Mobile-Check
Allow: /typo3temp/*/*.css
Allow: /typo3temp/*.css*
Allow: /typo3temp/*.js*
Crawl-delay: 6
Request-rate: 10/1m

User-agent: AwarioRssBot
Disallow: /
User-agent: AwarioSmartBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: OpenindexSpider
Disallow: /
User-agent: Amzn-SearchBot
Disallow: /
User-agent: Claude-SearchBot
Disallow: /

# http://www.grapeshot.co.uk/crawler.php
User-agent: grapeshot
Disallow: /

User-agent: ImagesiftBot
Disallow: /