User-Agent: *
Disallow: /application/
Disallow: /userfiles/
Disallow: /system/
#Disallow: /assets/
Disallow: /partners/
Disallow: /mobile/
Disallow: /*?*

#To shut down site completely from bots use
#Disallow: /


User-Agent: Googlebot
Allow: /

User-Agent: Adsbot-Google
Allow: /

User-Agent: rogerbot
Allow: /

User-Agent: Slurp
Allow: /

User-Agent: msnbot
Allow: /

#Allowing SEO Tools to Crawl the Website

User-agent: SemrushBot
Allow: /

User-agent: SemrushBot-SA
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: rogerbot
Allow: /


#Block out things that are secure or login oriented
#These are based on not using any rewrite rules
#Disallow: /account.php



# IF YOU DO NOT WISH TO HAVE THE GOOGLE IMAGE BOT SCAN YOUR DOMAIN FOR IMAGES
# THEN YOU CAN INCLUDE THE FOLLOWING IN YOUR ROBOTS FILE.
# I FOUND THAT MY BANDWIDTH USAGE DROPPED BY A MASSIVE AMOUNT AFTER I GOT RID
# OF THE GOOGLE IMAGE BOT. ALL I HAD WAS IMAGE HUNTERS STEALING PRODUCT SHOTS
# AND NOT EVEN BROWSING THE SITE.

User-agent: Unknown robot
Disallow: * 

User-agent: Unknown robot (identified by empty user agent string) 
Disallow: *

User-agent: Unknown robot (identified by '*bot')
Disallow: *

User-agent: Unknown robot (identified by 'bot' followed by a space or one of the following characters _+:,.;/\-)
Disallow: *

User-agent: YaBrowser
Disallow: *

User-agent: CCBot/2.0
Disallow: *

User-Agent: *
Disallow: /