User-agent: * 

# Disallow all crawlers to certain folders.

Disallow: /admin/

Disallow: /cgi-bin/

Disallow: /db/



# Disallow all crawlers to certain files. 

Disallow: /.htaccess