User-agent: * Disallow: /wp-admin Disallow: /search/page Disallow: /tools/virtual-private-server-vps-for-forex-trading/ Disallow: /es-int/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /ko/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /ru/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /fr/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /tr/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /de/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /it/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /ar/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /cn/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /mn/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /pk/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /ae/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /pt-int/tools/virtual-private-server-vps-for-forex-trading/ Disallow: /pt-br/tools/virtual-private-server-vps-for-forex-trading/ User-agent: MJ12bot User-agent: SEOkicks-Robot User-agent: ia_archiver User-agent: LinkChecker User-agent: archive.org_bot Disallow: / #User-agent: AhrefsBot #Disallow: / User-agent: SemrushBot Crawl-delay: 10 Disallow: /wp-admin Disallow: /search/page Disallow: /platforms/* Disallow: /fr/* Disallow: /tr/* Disallow: /de/* Disallow: /it/* Disallow: /es/* Disallow: /ru/* Disallow: /br/* Disallow: /id/* Disallow: /my/* Disallow: /ar/* Disallow: /cn/* Disallow: /th/* Disallow: /vn/* User-agent: dotbot Disallow: / User-agent: BLEXBot Disallow: / #User-agent: msnbot #User-agent: bingbot #User-Agent: Mail.Ru #User-agent: Yandex #Crawl-delay: 10 # Some bots are known to be trouble, particularly those designed to copy # entire sites. Please obey robots.txt. User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / User-agent: asterias Disallow: / User-agent: BackDoorBot/1.0 Disallow: / User-agent: Black Hole Disallow: / User-agent: BlowFish/1.0 Disallow: / User-agent: BotALot Disallow: / User-agent: BuiltBotTough Disallow: / User-agent: Bullseye/1.0 Disallow: / User-agent: BunnySlippers Disallow: / User-agent: Cegbfeieh Disallow: / User-agent: CheeseBot Disallow: / User-agent: CherryPicker Disallow: / User-agent: CherryPickerElite/1.0 Disallow: / User-agent: CherryPickerSE/1.0 Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: cosmos Disallow: / User-agent: Crescent Disallow: / User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: / User-agent: DittoSpyder Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: EroCrawler Disallow: / User-agent: ExtractorPro Disallow: / User-agent: Foobot Disallow: / User-agent: Harvest/1.5 Disallow: / User-agent: hloader Disallow: / User-agent: httplib Disallow: / # Misbehaving: requests much too fast: User-agent: fast Disallow: / # # Hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: WebReaper Disallow: /