User-agent: *
Disallow: /images/
Disallow: /blog/category/
Disallow: /printmedia.php
Disallow: /viewauthor.php
Disallow: /cart.php
Disallow: /addtocart.php
Disallow: /page.php/prmID/13

# Bar all spiders from indexing the three pieces so often found by people looking for child
# pornography and prostitutes.
Disallow: /articles/1992/winter/lefer-little-virgins/
Disallow: /articles/2006/winter/turner-nightgirls/
Disallow: /articles/2006/spring/somerswillett-first-sex/

# Prevent indexing of republishing URLs
Disallow: */feed/
Disallow: /blog/?s=*
Disallow: /search/

# Prevent indexing of any Synopsa-style URLs
Disallow: /page.php
Disallow: *prmID*

# Bar useless, bandwidth-devouring spiders
User-agent: BecomeBot
Disallow: /

User-agent: Googlebot-Image
Disallow: /

User-agent: CazoodleBot
Disallow: /

User-agent: HolmesBot
Disallow: /

User-agent: Sosospider
Disallow: /

User-agent: Snapbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: larbin
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: LexxeBot
Disallow: /

User-agent: ShopWiki
Disallow: /

User-agent: Gigabot
Disallow: /

# The only purpose of these bots is to copy an entire website, something that does
# not interest us.
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: WebReaper
Disallow: /

# CSC's bot totally overwhelms web servers with parallel requests.
User-agent: NPBot
Disallow: /