# robots.txt for http//www.unesco.org/whc/
# User-agent Mozilla/3.0 (compatible; Muscat/1.03)
# Disallow 

User-agent: * # directed to all spiders, not just Scooter
Disallow: /web/whc/archive/out/ # Directory contains early versions of documents already available.
Disallow: /web/whc/conf201/ # Merida meeting materials only (temporary)
Disallow: /web/whc/fr/archive/out  # French versions of early materials already available
Disallow: /web/whc/nwhc/ # frames and non-substantive frame content
Disallow: /web/whc/nwhc.fr/ # frames and non-substantive frame content, French pages


# 15 June 1997
# updated 3 September 1998, 21 March 1999, 10 November 1999
# (also placed in whc/archive/)

Disallow: /temp/ # temp folders

User-agent: Slurp
Crawl-delay: 2