# -----------------------------------------------------------------------------
# This Robots.txt file is for http://www.os2site.com/
# see http://www.robotstxt.org/wc/exclusion.html
# -----------------------------------------------------------------------------

User-agent: *
Disallow: /

User-agent: Teoma
Crawl-delay: 50
SITEMAP: http://www.os2site.com/sitemap.xml
Disallow: /sw/incoming/
Disallow: /sw/new/
Disallow: /sw/unknown/
Disallow: /cgi-bin/
Allow: /sitemap.xml$
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Disallow: /*.*$

User-agent: ia_archiver
Disallow: /cgi-bin/
Disallow: /dec/
Disallow: /sw/

User-agent: archive.org_bot
Disallow: /cgi-bin/
Disallow: /dec/
Disallow: /sw/

User-agent: msnbot
Disallow: /
#Crawl-delay: 100
#Disallow: /sw/incoming/
#Disallow: /sw/new/
#Disallow: /sw/unknown/
#Disallow: /cgi-bin/
#Disallow: /*.mpg$
#Disallow: /*.zip$
#Disallow: /*.lzh$
#Disallow: /*.jar$
#Disallow: /*.exe$
#Disallow: /*.wpi$
#Disallow: /*.pdf$
#Disallow: /*.arc$
#Allow: /sitemap.xml$
#Allow: /*.html$
#Allow: /*.htm$
#Allow: /*.link$
#Disallow: /*.*$

User-agent: Slurp
Disallow: /
#Crawl-delay: 200
#Disallow: /sw/incoming/
#Disallow: /sw/new/
#Disallow: /sw/unknown/
#Disallow: /cgi-bin/
#Allow: /sitemap.xml$
#Allow: /*.html$
#Allow: /*.htm$
#Allow: /*.txt$
#Allow: /*.link$
#Disallow: /*.*$

User-agent: Googlebot
Disallow: /sw/incoming/
Disallow: /sw/new/
Disallow: /sw/unknown/
Disallow: /cgi-bin/
Allow: /sitemap.xml$
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Allow: /*.*.link$
Disallow: /*.*$

User-agent: Mediapartners-Google
Disallow: /cgi-bin/
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Allow: /*.*.link$
Disallow: /*.*$

User-agent: Ocelli
Crawl-delay: 50
Disallow: /sw/incoming/
Disallow: /sw/new/
Disallow: /sw/unknown/
Disallow: /cgi-bin/
Allow: /sitemap.xml$
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Allow: /*.*.link$
Disallow: /*.*$

User-agent: http://www.almaden.ibm.com/cs/crawler
Disallow: /cgi-bin/
Disallow: /sw/incoming/
Disallow: /sw/new/

User-agent: Adsbot-Google
Disallow: /cgi-bin/