# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# TODO look into blocks for non matches of IPs http://www.iplists.com/
User-agent: *
Disallow: /search
Crawl-delay: 5

Sitemap: http://bayarea.abtain.com/sitemap.xml

User-agent: Googlebot
Disallow: /posts/new/*
Allow: *

User-agent: googlebot-image
Disallow: /posts/new/*
Allow: *

User-agent: googlebot-mobile
Disallow: /posts/new/*
Allow: *

User-agent: MSNBot
Disallow: /posts/new/*
Allow: *

User-agent: Slurp
Disallow: /posts/new/*
Allow: *

User-agent: Teoma
Disallow: /posts/new/*
Allow: *

User-agent: twiceler
Disallow: /posts/new/*
Allow: *

User-agent: Gigabot
Disallow: /posts/new/*
Allow: *

User-agent: ia_archiver
Disallow: /posts/new/*
Allow: *

User-agent: yahoo-mmcrawler
Disallow: /posts/new/*
Allow: *

User-agent: psbot
Disallow: /posts/new/*
Allow: *

User-agent: asterias
Disallow: /posts/new/*
Allow: *

User-agent: yahoo-blogs/v3.9
Disallow: /posts/new/*
Allow: *