User-agent: ia_archiver
Disallow: /

User-agent: *
# Directories
Disallow: /database/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /sites/
Disallow: /themes/
Disallow: /scripts/
Disallow: /updates/
Disallow: /profiles/
# Files
Disallow: /xmlrpc.php
Disallow: /cron.php
Disallow: /update.php
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /CHANGELOG.txt
Disallow: /MAINTAINERS.txt
Disallow: /LICENSE.txt
Disallow: /UPGRADE.txt
# Paths (clean URLs)
Disallow: /admin
Disallow: /aggregator/
Disallow: /comment/
Disallow: /contact
Disallow: /logout
Disallow: /search
# Paths (no clean URLs)
Disallow: /?q=
# Additional
Disallow: /tracker
Disallow: /comment/reply
Disallow: /node
Disallow: /user
Disallow: /search/
Disallow: /book/print
Disallow: /book/export/
Disallow: /forward/
Disallow: /polls/
# commented the following rule 19may07
# Supplemental: all but 91 of 979 indexed
# Disallow: /*?*
# update: Supplemental: all but 354 of 1490 indexed -- 2jun07
# Disallow: /*/feed  -- removed mar 28, 2010
# Stop sorted tables from being indexed
Disallow: /*sort=
# Stop thumbnails from being indexed
Disallow: /*size=
Disallow: /abuse
Disallow: /subscriptions
Disallow: /link-to-us
Disallow: /userpoints

# Redirects
Disallow: /r/

# Newsletters
Disallow: /newsletters/

# removed block on /files/ on 27jul07

Disallow: /efl-esl-games/adult-efl-esl-games?page=
Disallow: /efl-esl-games/high-school-efl-esl-games?page=
Disallow: /efl-esl-games/middle-school-efl-esl-games?page=
Disallow: /efl-esl-games/elementary-school-efl-esl-games?page=
Disallow: /efl-esl-games/efl-esl-alphabet-games?page=
Disallow: /efl-esl-games/efl-esl-grammar-games?page=
Disallow: /efl-esl-games/efl-esl-listening-games?page=
Disallow: /efl-esl-games/efl-esl-reading-games?page=
Disallow: /efl-esl-games/efl-esl-speaking-games?page=
Disallow: /efl-esl-games/efl-esl-spelling-games?page=
Disallow: /efl-esl-games/efl-esl-vocabulary-games?page=
Disallow: /efl-esl-games/efl-esl-writing-games?page=

Disallow: /files/fdl.txt
Disallow: /*sid=

# Temp block -- removed 14jul08
# Disallow: /elementary-school-english-3
# Disallow: /elementary-school-english-1
# Disallow: /busan$
# Disallow: /busan-0$
# Disallow: /chungbuk
# Disallow: /chungnam
# Disallow: /daejeon
# Disallow: /gangwon
# Disallow: /gwangju
# Disallow: /gyeonggi
# Disallow: /incheon
# Disallow: /jeju
# Disallow: /jeongbuk
# Disallow: /jeongnam
# Disallow: /ulsan

Disallow: /asiablog/10-foods-taiwan-1125.html?page=1
Disallow: /advertising
Disallow: /t_f
Disallow: /allcontent/
Disallow: /*utm_source=feedburner