# Disallow all crawlers access to certain pages.
# Crawling the site is allowed every 120 seconds
# MJ12bot is banned since 46% of the traffic was generated by this spider

User-agent: MJ12bot
Disallow: /

User-agent: iecrawler
Disallow: /

User-agent: ie_crawler
Disallow: /

User-agent: e-SocietyRobot
Disallow: /

User-agent: *
Crawl-delay: 120
Disallow: /cgi-bin/twiki/attach
Disallow: /cgi-bin/twiki/changes
Disallow: /cgi-bin/twiki/edit
Disallow: /cgi-bin/twiki/geturl
Disallow: /cgi-bin/twiki/installpasswd
Disallow: /cgi-bin/twiki/logon
Disallow: /cgi-bin/twiki/mailnotify
Disallow: /cgi-bin/twiki/manage
Disallow: /cgi-bin/twiki/oops
Disallow: /cgi-bin/twiki/passwd
Disallow: /cgi-bin/twiki/preview
Disallow: /cgi-bin/twiki/rdiff
Disallow: /cgi-bin/twiki/register
Disallow: /cgi-bin/twiki/rename
Disallow: /cgi-bin/twiki/save
Disallow: /cgi-bin/twiki/search
Disallow: /cgi-bin/twiki/statistics
Disallow: /cgi-bin/twiki/upload
Disallow: /cgi-bin/twiki/viewauth
Disallow: /cgi-bin/twiki/viewfile
Disallow: /cgi-bin/twiki/view/Sandbox
Disallow: /cgi-bin/twiki/view/*?

Disallow: /*/rdiff*
Disallow: /*/setlib.cfg*
Disallow: /*/statistics*
Disallow: /*/testenv*
Disallow: /*/Sandbox*
Disallow: /*/TWiki*
Disallow: /*/Trash*

Disallow: /*skin=print*
Disallow: /*rev=*
Disallow: /*raw=on*
Disallow: /*CGISESSID=*
Disallow: /*CGISE=*