# robots.txt

User-Agent: Googlebot
Allow: /
Disallow: /CSS
Disallow: /admin
Disallow: /mod_perl
Disallow: /logs
Disallow: /site_scripts

User-Agent: Adsbot-Google
Allow: /
Disallow: /CSS
Disallow: /admin
Disallow: /mod_perl
Disallow: /logs
Disallow: /site_scripts

User-agent: Mediapartners-Google 
Allow: /
Disallow: /CSS
Disallow: /admin
Disallow: /mod_perl
Disallow: /logs
Disallow: /site_scripts

User-Agent: baiduspider
Allow: /
Disallow: /CSS
Disallow: /admin
Disallow: /mod_perl
Disallow: /logs
Disallow: /site_scripts
Crawl-delay: 10

User-Agent: MJ12bot
Disallow: /

User-Agent: AhrefsBot
Disallow: /

User-Agent: PetalBot
Disallow: /

User-Agent: AspiegelBot
Disallow: /

User-Agent: SEMrushBot
Disallow: /

User-Agent: DotBot
Disallow: /

User-Agent: MauiBot
Disallow: /

User-agent: *
Disallow: /