User-agent: *
Allow: /
Crawl-delay: 5

User-agent: *
Disallow: /cgi-bin
Disallow: /css
Disallow: /img
Disallow: /js