User-agent: * Disallow: /cgi-bin/ # Block bad bots User-agent: adbeat_bot User-agent: AhrefsBot User-agent: AITCSRobot User-agent: Alexibot User-agent: BLEXBot User-agent: Cliqzbot User-agent: dotbot User-agent: Exabot User-agent: expo9 User-agent: Huaweisymantecspider User-agent: InfluenceBot User-agent: ltx71 - (http://ltx71.com/) User-agent: ltx71+-+(http://ltx71.com/) User-agent: MaxPointCrawler User-agent: MJ12bot User-agent: Offline Explorer User-agent: rogerbot User-agent: SemrushBot User-agent: SemrushBot-SA User-agent: SiteSnagger User-agent: SurveyBot User-agent: TeleportPro User-agent: WebCopier User-agent: WebReaper User-agent: WebStripper User-agent: WebZIP User-agent: Xaldon_WebSpider User-agent: Xenu’s User-agent: Xenu’s Link Sleuth 1.1c User-agent: WordPress/4.3.1;+http://www.vagabondingheights.com User-agent: bitlybot/2.0 User-agent: NerdyBot User-agent: woopingbot/1.1 User-agent: WordPress/4.3.1;+http://www.londonhotelsinformation.com User-agent: TinEye-bot/0.51+(see+http://www.tineye.com/crawler.html) User-agent: YisouSpider User-agent: SecurityResearch.bot Disallow: / # Slow some bots down User-agent: Applebot User-agent: coccoc User-agent: ia_archiver User-agent: istellabot User-agent: grapeshot Crawl-delay: 15