# robots.txt for aarf.ai. thanks to /www.seoconsultants.com/robots-text-file/
# Whitelist: allow these good bots:
User-agent: googlebot
User-agent: slurp
User-agent: msnbot
User-agent: teoma
User-agent: User-agent: WDG_SiteValidator
User-agent: W3C-checklink
User-agent: W3C_Validator
User-agent: W3CLineMode
User-agent: Jigsaw
User-agent: W3C-mobileOK
# To the above bots: do NOT index error and web stats directories (and my temporary redirector).
Disallow: /errors/
Disallow: /error files permissions/
Disallow: /redirecting_index file/
Disallow: /WEBALIZER_REPORTS/
# to all other user agents: disallow everything from the root directory on down. Take a hike all you other bots! Put this last so the good bots see their instructions
#User-agent: *
#Disallow: /