User-agent: * 
Disallow: /_borders/
Disallow: /_private/
Disallow: /_vti_cnf/
Disallow: /_vti_log/
Disallow: /_vti_pvt/
Disallow: /_vti_script/
Disallow: /_vti_txt/
Disallow: /_cusudi/
Disallow: /_derived/
Disallow: /_kbas/
Disallow: /_reqdis/
Disallow: /_vti_inf.html
Disallow: /_disc1/
Disallow: /_fpclass/
Disallow: /_overlay/
Disallow: /_themes/
Disallow: /_vti_inf.html
Disallow: /aspnet_client/
Disallow: /banner/
Disallow: /catalog/
Disallow: /cgi-bin/
Disallow: /classifieds/
Disallow: /ftp/
Disallow: /image/
Disallow: /images/
Disallow: /img/
Disallow: /imgstore/
Disallow: /jukebox/
Disallow: /karaoke/
Disallow: /karaoke66/
Disallow: /mp3/
Disallow: /music/
Disallow: /singers/
Disallow: /store/
Disallow: /txt/
Disallow: /video/
Disallow: /asfroot/
Disallow: /Backups/
Disallow: /cgi-bin/
Disallow: /karaoke/
Disallow: /larry/
Disallow: /trucking/
Disallow: /cusuaftr.htm
Disallow: /cusuahdr.htm
Disallow: /cusucfrm.htm
Disallow: /cusufoot.htm
Disallow: /cusuhead.htm
Disallow: /cusupost.htm
Disallow: /cususrch.htm
Disallow: /cusutoc.htm
Disallow: /disc1_cfrm.htm
Disallow: /disc1_frm.htm
Disallow: /disc1_post.htm
Disallow: /disc1_srch.htm
Disallow: /disc1_tocf.htm
Disallow: /disc1_welc.htm
Disallow: /phpinfo.php
Disallow: /postinfo.html
Disallow: /iisstart.asp
Disallow: /localstart.asp
Disallow: /index.htm
Disallow: /localstart.asp
Disallow: /membership.htm
Disallow: /memcache.php
Disallow: /phpinfo.php
Disallow: /postinfo.html
Disallow: /prx1.php
Disallow: /test.pl
Disallow: /test.cgi
Disallow: /animate.js
Disallow: /global.asa


# Disallow Collectors and Spam

User-agent: cherrypicker
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: CherryPickerElite/1.0
Disallow: /

User-agent: CherryPickerSE/1.0
Disallow: /

User-agent: Collage
Disallow: /

User-agent: Download Wonder
Disallow: /

User-agent: DSurf
Disallow: /

User-agent: EliteSys Entry
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ESIRover
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: GetBot
Disallow: /

User-agent: GetUrl
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: HavIndex
Disallow: /

User-agent: httrack
Disallow: /

User-agent: InfoSpiders
Disallow: /

User-agent: Internet Explore
Disallow: /

User-agent: Jeeves
Disallow: /

User-agent: KDD Exploror
Disallow: /

User-agent: LightningDownload
Disallow: /

User-agent: Mail Sweeper
Disallow: /

User-agent: Mozilla/2.0 (compatible; MS FrontPage 4.0)
Disallow: /

User-agent: munky
Disallow: /

User-agent: NICErsPRO
Disallow: /

User-agent: PBWF
Disallow: /

User-agent: Roverbot
Disallow: /

User-agent: Shai'Hulud
Disallow: /

User-agent: SpiderBot
Disallow: /

User-agent: ssearcher100
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: Templeton
Disallow: /

User-agent: Titan 
Disallow: /

User-agent: vobsub
Disallow: /

User-agent: w3mir
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: WebCatcher
Disallow: /

User-agent: webcopier
Disallow: /

User-agent: WebCopy
Disallow: /

User-agent: WebEmailExtrac
Disallow: /

User-agent: WebFetcher
Disallow: /

User-agent: WebMirror
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: webvac
Disallow: /

User-agent: WebWalk
Disallow: /

User-agent: wGet
Disallow: /

User-agent: xGet
Disallow: /