# robots.txt for https://www.CodeNewsFast.com/
#

# https://blogs.bing.com/webmaster/2009/08/10/crawl-delay-and-the-bing-crawler-msnbot
# Slow down bingbot
User-agent: msnbot
Crawl-delay: 10

User-agent: *
Allow: /robots.txt


# too much traffic from these in September 2017
User-agent: MJ12bot
Disallow: /cnf
User-agent: Baiduspider
Disallow: /cnf
User-agent: SemrushBot
Disallow: /cnf
User-agent: AhrefsBot
Disallow: /cnf
# too much traffic from this in March 2018
User-agent: BUbiNG
Disallow: /cnf



User-agent: *
Allow: /
Allow: /cnf/quickhelp
Allow: /cnf/about
Allow: /cnf/search
Allow: /cnf/newsgroupservers