User-agent:* Disallow:/cgi-bin Disallow:/users User-agent:Exalead NG/MimeLive Client Disallow:/ User-agent:NG Disallow:/ User-agent:Syndic8 Disallow:/ User-agent:pompos Disallow:/ User-agent: msnbot Crawl-delay: 10 User-agent: bingbot Crawl-delay: 10 # Disallow: SEOkicks-Robot User-agent: SEOkicks-Robot Disallow: / # http://www.opensiteexplorer.org/dotbot User-agent: DotBot Disallow: / User-agent: dotbot Disallow: / # http://www.searchmetrics.com User-agent: SearchmetricsBot Disallow: / # http://www.majestic12.co.uk/projects/dsearch/mj12bot.php User-agent: MJ12bot Disallow: / # http://www.domaintools.com/webmasters/surveybot.php User-agent: SurveyBot Disallow: / # http://www.seodiver.com/bot user-agent: SEOdiver disallow: / # http://openlinkprofiler.org/bot User-agent: spbot Disallow: / # http://www.wotbox.com/bot/ User-agent: wotbox Disallow: / User-agent: Cliqzbot/3.0 Disallow: / User-agent: Cliqzbot Disallow: / User-agent: Seekport Crawler Disallow: / User-agent: Paracrawl Disallow: / User-agent: Scrapy/1.5.0 Disallow: / User-agent: Scrapy Disallow: / User-agent: VelenPublicWebCrawler (velen.io) Disallow: / User-agent: VelenPublicWebCrawler Disallow: / User-agent: SemrushBot/2~bl Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: AspiegelBot Disallow: / User-agent: YandexBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: Verity Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Timpibot Disallow: / User-agent: OAI-SearchBot Disallow: / User-agent: meta-externalagent Disallow: / User-agent: GrapeshotCrawler Disallow: / User-agent: Amazonbot Disallow: / User-agent: facebookexternalhit Disallow: / User-agent: BLEXBot Disallow: / User-agent: Barkrowler Disallow: / User-agent: proximic Disallow: / User-agent: ias-va Disallow: / User-agent: ias Disallow: / User-agent: ias-or Disallow: / User-agent: weborama-fetcher Disallow: / User-agent: CriteoBot Disallow: / User-agent: SEOkicks Disallow: / User-agent: TTD-Content Disallow: / User-agent: AwarioBot Disallow: / User-agent: ias-va Disallow: / User-agent: ias-or Disallow: / User-agent: ias-ie Disallow: / User-agent: Qwantbot Disallow: / User-agent: GPTBot Disallow: / User-agent: Verity Disallow: / User-agent: Bytespider Disallow: / User-agent: SirdataBot Disallow: /