User-agent: * #allows all search-bots to crawl the site
Disallow: /cgi-bin/ #no acces to cgi-bin