# Generic robots file by robert@befidi.com # Disallow all robots User-agent: * Disallow: /cgi-bin Disallow: /dev/ Disallow: /gcnl/ Disallow: /images Disallow: /investor/ Disallow: /team/ Disallow: /gordian # Make exceptions for the following robots User-agent: AbachoBOT Disallow: /gordian User-agent: Acoon Robot Disallow: /gordian User-agent: AESOP_com_SpiderMan Disallow: /gordian User-agent: Agent-admin Disallow: /gordian User-agent: ah-ha.com crawler Disallow: /gordian User-agent: AlkalineBOT Disallow: /gordian User-agent: AltaVista-Intranet Disallow: /gordian User-agent: appie Disallow: /gordian User-agent: Arachnoidea Disallow: /gordian User-agent: Aranha Disallow: /gordian User-agent: ArchitectSpider Disallow: /gordian User-agent: ArchitextSpider Disallow: /gordian User-agent: asterias Disallow: /gordian User-agent: Atomz Disallow: /gordian User-agent: C-PBWF-ip3000.com-crawler Disallow: /gordian User-agent: Crawler Disallow: /gordian User-agent: CrawlerBoy Pinpoint.com Disallow: /gordian User-agent: DIIbot Disallow: /gordian User-agent: dloader Disallow: /gordian User-agent: dumrobo Disallow: /gordian User-agent: ESISmartSpider Disallow: /gordian User-agent: EZResult Disallow: /gordian User-agent: Fast PartnerSite Crawler Disallow: /gordian User-agent: FAST-WebCrawler Disallow: /gordian User-agent: Fluffy the spider Disallow: /gordian User-agent: gazz/1.0 Disallow: /gordian User-agent: geckobot Disallow: /gordian User-agent: GenCrawler Disallow: /gordian User-agent: Googlebot Disallow: /gordian User-agent: griffon Disallow: /gordian User-agent: Gulliver Disallow: /gordian User-agent: HenryTheMiragoRobot Disallow: /gordian User-agent: Hubater Disallow: /gordian User-agent: ia_archiver Disallow: /gordian User-agent: IncyWincy Disallow: /gordian User-agent: InfoSeek Sidewinder Disallow: /gordian User-agent: ip3000.com-crawler Disallow: /gordian User-agent: Iron33 Disallow: /gordian User-agent: Jack Disallow: /gordian User-agent: KIT-Fireball Disallow: /gordian User-agent: Kototoi/0.1 Disallow: /gordian User-agent: libwww-perl Disallow: /gordian User-agent: LNSpiderguy Disallow: /gordian User-agent: Lycos_Spider_(T-Rex) Disallow: /gordian User-agent: MantraAgent Disallow: /gordian User-agent: marvin/infoseek Disallow: /gordian User-agent: Mercator Disallow: /gordian User-agent: moget/2.0 Disallow: /gordian User-agent: MP3Bot Disallow: /gordian User-agent: MuscatFerret Disallow: /gordian User-agent: NationalDirectory-SuperSpid Disallow: /gordian User-agent: Nazilla Disallow: /gordian User-agent: NetResearchServer Disallow: /gordian User-agent: nttdirectory_robot Disallow: /gordian User-agent: Openfind piranha,Shark Disallow: /gordian User-agent: PJspider Disallow: /gordian User-agent: psbot Disallow: /gordian User-agent: RaBot Disallow: /gordian User-agent: roach.smo.av.com-1.0 Disallow: /gordian User-agent: Robot@SuperSnooper.Com Disallow: /gordian User-agent: robot-response@openfind.com Disallow: /gordian User-agent: Scooter Disallow: /gordian User-agent: Scooter2_Mercator_3-1.0 Disallow: /gordian User-agent: Slurp.so/1.0 Disallow: /gordian User-agent: Slurp/2.0j Disallow: /gordian User-agent: Slurp/2.0-KiteHourly Disallow: /gordian User-agent: Slurp/2.0-OwlWeekly Disallow: /gordian User-agent: Slurp/3.0-AU Disallow: /gordian User-agent: speedfind ramBot xtreme Disallow: /gordian User-agent: spider.batsch.com Disallow: /gordian User-agent: Spider/maxbot.com Disallow: /gordian User-agent: spider@aeneid.com Disallow: /gordian User-agent: super-robot@super.navi.ocn. Disallow: /gordian User-agent: Surfnomore Spider v1.1 Disallow: /gordian User-agent: teoma_admin@hawkholdings.co Disallow: /gordian User-agent: teoma_agent1 Disallow: /gordian User-agent: Toutatis 2.5-2 Disallow: /gordian User-agent: Tv_Merc_resh_26_1_D-1.0 Disallow: /gordian User-agent: UK Searcher Spider Disallow: /gordian User-agent: UltraSeek Disallow: /gordian User-agent: user.ip3000.com Disallow: /gordian User-agent: Wget Disallow: /gordian User-agent: WhizBang! Lab Disallow: /gordian User-agent: WIRE WebRefiner Disallow: /gordian User-agent: WSCbot Disallow: /gordian User-agent: Yandex Disallow: /gordian User-agent: Yellopet-Spider Disallow: /gordian User-agent: ZyBorg Disallow: /gordian User-agent: atomz Disallow: /gordian User-agent: architext Disallow: /gordian User-agent: google Disallow: /gordian User-agent: infoseek Disallow: /gordian User-agent: jeeves Disallow: /gordian User-agent: lycos Disallow: /gordian User-agent: directhit Disallow: /gordian User-agent: scooter Disallow: /gordian User-agent: slurp Disallow: /gordian