# Disallow spiders by default User-agent: * Disallow: / # Add Crawl-delay parameter for those crawlers that support it User-agent: * Crawl-delay: 5 #Allow freindly spiders # "Disallow:" means don't disallow anything, so all can be crawled. Same as "Allow: /" but better supported User-agent: Googlebot Disallow:/acphysci/ User-agent: Mediapartners-Google Disallow:/acphysci/ User-agent: Adsbot-Google Disallow:/acphysci/ User-agent: Googlebot-Image Disallow:/acphysci/ User-agent: Googlebot-Mobile Disallow:/acphysci/ User-agent: MSNBot Disallow:/acphysci/ #Yahoo User-agent: Slurp Disallow:/acphysci/ #Google China User-agent: baiduspider Disallow:/acphysci/ User-agent: PicoSearch/1.0 Disallow:/acphysci/ #ask.com User-agent: Teoma Disallow:/acphysci/ #gigablast.com User-agent: Gigabot Disallow:/acphysci/ #scrub the web User-agent: Scrubby Disallow:/acphysci/ #DMOZ User-agent: Robozilla Disallow:/acphysci/ # PRS defined crawler User-agent: gsa-crawler Disallow:/acphysci/