# If you have subdomains, you need to place a robots.txt file on each subdomain. Otherwise, SEMrushBot will consider that it is allowed to crawl everything on your subdomain.

# Below shows that Francis3Robot is not allowed to go any where

User-agent: Francis3Robot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Uptimebot
Disallow: /

# If you have subdomains, you need to place a robots.txt file on each subdomain. Otherwise, SEMrushBot will consider that it is allowed to crawl everything on your subdomain.
# The robots.txt file must always return an HTTP 200 status code. If a 4xx status code is returned, SEMrushBot will assume that no robots.txt exists and there are no crawl restrictions. 
# Returning a 5xx status code for your robots.txt file will prevent SEMrushBot from crawling your entire site. Our crawler can handle robots.txt files with a 3xx status code.

# To block SEMrushBot from crawling your site for a webgraph of links:
User-agent: SemrushBot
Disallow: /

# To block SEMrushBot from crawling your site for different SEO and technical issues:
User-agent: SemrushBot-SA
Disallow: /
# To block SEMrushBot from crawling your site for Backlink Audit tool:
User-agent: SemrushBot-BA
Disallow: /
# To block SEMrushBot from crawling your site for On Page SEO Checker tool and similar tools:
User-agent: SemrushBot-SI
Disallow: /
# To block SEMrushBot from checking URLs your site for SWA tool:
User-agent: SemrushBot-SWA
Disallow: /
# To block SEMrushBot from crawling your site for Content Analyzer and Post Tracking tools:
User-agent: SemrushBot-CT
Disallow: /
# To block SEMrushBot from crawling your site for Brand Monitoring:
User-agent: SemrushBot-BM
Disallow: /





# No robots should visit any URL listed, each page delay 5 seconds

User-agent: *
Crawl-delay: 5
Disallow: /App_Code/
Disallow: /App_Data/
Disallow: /ApppDatabase_File/
Disallow: /aspnet_client/
Disallow: /Bin/
Disallow: /Controls/
Disallow: /css/
Disallow: /emailOnline/
Disallow: /Image-Idle/
Disallow: /images/
Disallow: /mobile_App/
Disallow: /S/
Disallow: /tinymce/
Disallow: /UpLoadPic/
Disallow: /_0PicTransfer.aspx
Disallow: /0ReversePaas.aspx
Disallow: /Activate.aspx
Disallow: /addCategory.aspx
Disallow: /ApplicationError.aspx
Disallow: /AskQuestion.aspx
Disallow: /CampaignEmail.aspx
Disallow: /Category2.aspx
Disallow: /ChatMessageItem.aspx
Disallow: /ChatRoom.aspx
Disallow: /ChatSellerRoom.aspx
Disallow: /Check1Buy.aspx
Disallow: /Check2Buy.aspx
Disallow: /ChineseList.aspx
Disallow: /ClickDefault.aspx
Disallow: /ClickEmail.aspx
Disallow: /confirmPurchase.aspx
Disallow: /Contact.aspx
Disallow: /Default.aspx
Disallow: /Default-static.aspx
Disallow: /EmailCampaign.aspx
Disallow: /EmailCollect.aspx
Disallow: /EnterDataPacket.aspx
Disallow: /EnterScrapingItem.aspx
Disallow: /EnterSell.aspx
Disallow: /Feedback.aspx
Disallow: /Feedback13.aspx
Disallow: /ForgetPassword.aspx
Disallow: /HTMLconvert_LinkAU.aspx
Disallow: /HTMLconvert_LinkEU.aspx
Disallow: /HTMLconvert_LinkUK.aspx
Disallow: /HTMLconvert_LinkUS.aspx
Disallow: /HTMLconvert_noLinkAU.aspx
Disallow: /HTMLconvert_noLinkEU.aspx
Disallow: /HTMLconvert_noLinkUK.aspx
Disallow: /HTMLconvert_noLinkUS.aspx
Disallow: /IPnumberCalculat.aspx
Disallow: /ListAnotherItem.aspx
Disallow: /ListAnotherItem2.aspx
Disallow: /ListGroupBuy.aspx
Disallow: /ListGroupBuy2.aspx
Disallow: /ListItem.aspx
Disallow: /ListItem2.aspx
Disallow: /ListPreview.aspx
Disallow: /ListPreview2.aspx
Disallow: /Login.aspx
Disallow: /MyAccount.aspx
Disallow: /NoChangeCategory.aspx
Disallow: /NoSuchItemExist.aspx
Disallow: /OrderPrint.aspx
Disallow: /Payment.aspx
Disallow: /PaypalNotify_url.aspx
Disallow: /PictureUpload.aspx
Disallow: /PictureUpload2.aspx
Disallow: /PicUploadMore.aspx
Disallow: /ReadComment.aspx
Disallow: /RegIndiShop.aspx
Disallow: /SearchKeyW_Link.aspx
Disallow: /SearchKeyW_noLink.aspx
Disallow: /SharedModules.aspx
Disallow: /SharedUploadShop.aspx
Disallow: /ShopPicUpload.aspx
Disallow: /ShoppingCart.aspx
Disallow: /ViewBuying.aspx
Disallow: /WantaBuy.aspx
Disallow: /ZeroBuy.aspx
Disallow: /m/SearchKeyW_Link.aspx
Disallow: /m/SearchKeyW_noLink.aspx





# Below shows that Francis2Mapper is allowed but with delay

User-agent: Francis2Mapper
Disallow:
Crawl-delay: 10