#
# robots.txt for jobbankerne
#

User-agent: *
Host-load: 2
Disallow: /gfx/
Disallow: /link/
Disallow: /redirect/
Disallow: /profil/
Disallow: /job-arkiv/
Disallow: /prefetch_net/
Disallow: /ajax/
Disallow: /pf/
Allow: /job/

User-agent: 008
Disallow: /
Allow: /job/

User-agent: AhrefsBot
Disallow: /
Allow: /job/

User-agent: OwlinBot
Disallow: /
Allow: /job/

# disallow gsa-crawler (AU, AAU internal)
User-agent: gsa-crawler
Crawl-delay: 10
Host-load: 1
Allow: /job/

User-agent: dotbot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: Amazonbot               # Amazon's user agent
Crawl-delay: 5