# robots.txt for http//www.unesco.org/whc/
# User-agent Mozilla/3.0 (compatible; Muscat/1.03)
# Disallow 

User-agent: * # directed to all spiders, not just Scooter
Disallow: /web/whc/archive/out/ # Directory contains early versions of documents already available.
Disallow: /web/whc/conf201/ # Merida meeting materials only (temporary)
Disallow: /web/whc/fr/archive/out  # French versions of early materials already available
Disallow: /web/whc/nwhc/ # frames and non-substantive frame content
Disallow: /web/whc/nwhc.fr/ # frames and non-substantive frame content, French pages

Disallow: /sites/ # added 2010-09-22frames and non-substantive frame content, French pages

# 15 June 1997
# updated 3 September 1998, 21 March 1999, 10 November 1999
# (also placed in whc/archive/)

Disallow: /temp/ # temp folders

# added 2010-09-22
User-agent: *
Crawl-delay: 3
Request-rate: 1/3         # maximum rate is one page every 5 seconds
# Visit-time: 0600-0845     
# only visit between 06:00 and 08:45 UTC (GMT)


User-agent: msnbot
Crawl-delay: 5
Request-rate: 1/3