# All robots will spider the domain User-agent: * Disallow: # Disallow directory /secure/ User-agent: * Disallow: /secure/ Disallow: /*/product.aspx Disallow: /*/*.htm Allow: /*/*/Casablanca-Fans*.htm Disallow: /ProductSheet.aspx Crawl-delay: 60 User-agent: Googlebot Disallow: /more-info.aspx User-agent: msnbot Crawl-delay: 60 User-agent: Slurp Crawl-delay: 60 User-agent: BecomeBot Crawl-Delay: 60 User-agent: ShopWiki Crawl-Delay: 60 User-agent: Googlebot Crawl-Delay: 60 User-agent: Googlebot-Mobile Crawl-Delay: 60 User-agent: Googlebot-Image Crawl-Delay: 60