# The following block of code DISALLOWS the crawling of all files and directories to the following spiders/robots: yacybot
User-agent: yacybot
Disallow: /

#The following block of code ALLOWS the crawling of all files and directories to all spiders/robots.
User-agent: *
Disallow: