User-agent: Googlebot
Disallow: /nospam
Disallow: /nospam/
Disallow: /nomirror
Disallow: /nomirror/
Disallow: /admin/admin.php
Disallow: /submit
Disallow: /comments/8008135
Disallow: /confirm
Disallow: /unsub
Disallow: /passwordreset
Disallow: /ajax
Disallow: /login
Disallow: /archives/index-*
Disallow: /archives/index.1*
Disallow: /archives/index.2*
Disallow: /*/archives/index-*
Disallow: /*/archives/index.1*
Disallow: /*/archives/index.2*
Allow: /users
Allow: /cgi/users.pl
Allow: /cgi/fark/users.pl
Allow: /comments
Allow: /cgi/comments.pl
Allow: /cgi/fark/comments.pl
Disallow: /cgi/
User-agent: Mediapartners-Google
Disallow: /nospam
Disallow: /nospam/
Disallow: /nomirror
Disallow: /nomirror/
Disallow: /admin/admin.php
Disallow: /comments/8008135
Disallow: /passwordreset
Disallow: /ajax
Disallow: /login
Disallow: /archives/index-*
Disallow: /archives/index.1*
Disallow: /archives/index.2*
Disallow: /*/archives/index-*
Disallow: /*/archives/index.1*
Disallow: /*/archives/index.2*
Allow: /users
Allow: /cgi/users.pl
Allow: /cgi/fark/users.pl
Allow: /comments
Allow: /cgi/comments.pl
Allow: /cgi/fark/comments.pl
Allow: /confirm
Allow: /unsub
Allow: /submit
Allow: /cgi/submit.pl
Allow: /cgi/feedback.pl 
Allow: /cgi/forgotpassword.pl
Allow: /cgi/newuser.pl
Disallow: /cgi/
User-agent: GPTBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: *
Crawl-Delay: 5
Disallow: /nospam
Disallow: /nospam/
Disallow: /nomirror
Disallow: /nomirror/
Disallow: /admin/admin.php
Disallow: /submit
Disallow: /users
Disallow: /comments/8008135
Disallow: /confirm
Disallow: /unsub
Disallow: /passwordreset
Allow: /ajax/headlines
Disallow: /ajax
Disallow: /login
Disallow: /archives/index-*
Disallow: /archives/index.1*
Disallow: /archives/index.2*
Disallow: /*/archives/index-*
Disallow: /*/archives/index.1*
Disallow: /*/archives/index.2*
Allow: /comments
Allow: /cgi/comments.pl
Allow: /cgi/fark/comments.pl
Disallow: /cgi/
#
# IMPORTANT NOTE:
# Fark user profiles have a meta tag on the page to tell search engines to NOT
# index them.  But to read the meta tag, the engines have to be able to crawl
# the page.  A disallow means "don't crawl", NOT "don't index" -- if some other
# site has a link to a URL we have in our disallow list, search engines may
# still index it anyway.  So, counterintuitively, the reason we allow
# Googlebot to crawl user profiles is so that they WON'T index them.
# Our intent is that Fark.com user profiles NOT appear in search engines.
# (So all you SEO spammers are wasting your time -- not that you'll ever read this)
# This is the same reason we allow /go and /goto through here, as well as
# /api and /ajax.  We block all these from indexing with either the meta tag
# or the X-Robots-Tag header.
#
# Duke Sucks