User-agent: *
Disallow: /~/media/
Disallow: /*/~/media/
Disallow: /-/media/
Disallow: /*/-/media/
Disallow: /cpdemo
Disallow: /home
Disallow: /Local%20Components/
Disallow: /services/abc/
Disallow: /abc settings/
Disallow: /Global Components/
Disallow: /Coveo/
Disallow: /_apps/
Disallow: /sitecore%20modules/
Disallow: /members/request-access*
Disallow: /daily-briefing/dailybriefingprint*
Disallow: /*.asp$ 
Disallow: /*.axd$ 
Crawl-Delay: 5

User-agent: Twitterbot
Disallow: 

# SLOW BINGBOT DOWN
User-agent: msnbot
Crawl-delay: 10

# SLOW BINGBOT DOWN
User-agent: Bingbot
Crawl-delay: 10

# Block MJ12bot from crawling site
User-agent: MJ12bot
Disallow: /

# Block AhrefsBot from crawling site
User-agent: AhrefsBot
Disallow: /

# Block BLEXBot Crawler from crawling site
User-agent: BLEXBot Crawler
Disallow: /

# Block BlexBot
User-agent: BLEXBot
Disallow: /

# BLEXBot
User-agent: BLEXBot
Disallow: /

# Block TinEye from crawling site
User-agent: TinEye
Disallow: /

# Block SemrushBot from crawling site
User-agent: SemrushBot
Disallow: /

# Block DotBot from crawling site
User-agent: DotBot
Disallow: /

# Block Nutch from crawling site
User-agent: Nutch
Disallow: /

# Block Yandex from crawling site
User-agent: Yandex
Disallow: /

# Block YandexBot
User-agent: Yandex
Disallow: /

# Block Yandex
User-agent: YandexDirect
Disallow: /

# Block YandexDirectDyn
User-agent: YandexDirectDyn
Disallow: /

# Block YandexMedia
User-agent: YandexMedia
Disallow: /

# Block YandexImages
User-agent: YandexImages
Disallow: /

# Block YaDirectFetcher
User-agent: YaDirectFetcher
Disallow: /

# Block YandexBlogs
User-agent: YandexBlogs
Disallow: /

# Block YandexNews
User-agent: YandexNews
Disallow: /

# Block YandexPagechecker
User-agent: YandexPagechecker
Disallow: /

# Block YandexMetrika
User-agent: YandexMetrika
Disallow: /

# Block YandexCalendar
User-agent: YandexCalendar
Disallow: /

# Block MegaIndex.ru
User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: megaIndex.ru
Disallow: /

User-agent: spbot
Disallow: /

# http://cliqz.com/company/cliqzbot
User-agent: Cliqzbot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: IstellaBot
Disallow: /

User-agent: Domain Re-Animator Bot
Disallow: /

User-agent: aiHitBot
Disallow: /

User-agent: Vegi bot
Disallow: /

User-agent: deepcrawl
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /


User-agent: Heritrix
Disallow: /