# Default robots.txt file
# Last updated: 12.10.2020
# Generated by: Bloghosts + MartyR

# The below file is in place by default to block bad robots
# from indexing your site while allowing good robots to browse
# freely. Remove the below entries at your own risk.
# 2020-04-30 added AhrefsBot by Martin

User-agent: digout4u
User-agent: extractorpro
User-agent: GetRight
User-agent: go-ahead-got-it
User-agent: grub
User-agent: HTTPClient
User-agent: LinkWalker
User-agent: nearsite
User-agent: netattache
User-agent: NEWT ActiveX
User-agent: sitesnagger
User-agent: teleport
User-agent: TovekTools Web Indexer
User-agent: UbiCrawler
User-agent: Web Downloader
User-agent: WebTrends
User-agent: webwhacker
User-agent: webzip
User-agent: ltx71
User-agent: AhrefsBot
User-agent: BLEXBot
User-agent: DotBot
User-agent: Barkrowler
User-agent: Applebot
User-agent: PetalBot
Disallow: /

# Open Link Profiler
# Mozilla/5.0+(compatible;+spbot/4.4.2;++http://OpenLinkProfiler.org/bot+)
# http://OpenLinkProfiler.org/bot
User-agent: spbot
Disallow: /

# http://mj12bot.com - MAJESTIC
# Crawl-Delay should be an integer number and it signifies number of seconds of wait between requests.
User-Agent: MJ12bot
# Crawl-Delay: 5
Disallow: /

# bingbot
User-agent: bingbot
Crawl-delay: 5

# blocks access to whole site for All Yandex bots like YandexBot, YandexDirect, YandexDirectDyn, YandexMedia, YandexImages ....
User-agent: Yandex
Disallow: /

# block SEMrushBot
# To block SEMrushBot from crawling your site for a webgraph of links:
User-agent: SemrushBot
Disallow: /
# To block SEMrushBot from crawling your site for different SEO and technical issues:
User-agent: SemrushBot-SA
Disallow: /
# To block SEMrushBot from crawling your site for Backlink Audit tool:
User-agent: SemrushBot-BA
Disallow: /
# To block SEMrushBot from crawling your site for On Page SEO Checker tool and similar tools:
User-agent: SemrushBot-SI
Disallow: /
# To block SEMrushBot from checking URLs your site for SWA tool:
User-agent: SemrushBot-SWA
Disallow: /
# To block SEMrushBot from crawling your site for Content Analyzer and Post Tracking tools:
User-agent: SemrushBot-CT
Disallow: /
# To block SEMrushBot from crawling your site for Brand Monitoring:
User-agent: SemrushBot-BM
Disallow: /

#
User-agent: *
Disallow: /App_Data/
Disallow: /log/
Disallow: /bin/
#