# Allow all robots to spider the domain User-agent: * Disallow: # Disallow all robots to spider the following folders User-agent: * Disallow: /controls/ Disallow: /css/ Disallow: /Data/ Disallow: /scripts/ Disallow: /tiny_mce/ Disallow: terms.aspx Disallow: confirmed.aspx Disallow: cancelled.aspx Disallow: error404.aspx Disallow: loginpage.aspx Disallow: submitfailed.aspx Disallow: submitsuccessful.aspx