#user agent is the robot ie:google, *=all robots User-agent: * #Disallow - every robots.txt must have a dissalow or every page is crawled /=disallow all docs and folders Disallow: /include Disallow: /officers Disallow:/logon Disallow:/css Disallow:/user