屏蔽掉所有垃圾蜘蛛大全
# robots.txtgenerated User-agent:larbinDisallow: /User-agent:b2w/0.1Disallow: /
User-agent:CopernicDisallow: /
User-agent:psbotDisallow: /
User-agent:Python-urllibDisallow: /
User-agent:URL_Spider_ProDisallow: /
User-agent:CherryPickerDisallow: /
User-agent:EmailCollectorDisallow: /
User-agent:EmailSiphonDisallow: /
User-agent:WebBanditDisallow: /
User-agent:EmailWolfDisallow: /
User-agent:ExtractorProDisallow: /
User-agent:CopyRightCheckDisallow: /
User-agent:CrescentDisallow: /
User-agent:SiteSnaggerDisallow: /
User-agent:ProWebWalkerDisallow: /
User-agent:CheeseBotDisallow: /
User-agent:LNSpiderguyDisallow: /
User-agent:AlexibotDisallow: /
User-agent:TeleportDisallow: /
User-agent:TeleportProDisallow: /
User-agent:MIIxpcDisallow: /
User-agent:TelesoftDisallow: /
User-agent:Website QuesterDisallow: /
User-agent:WebZipDisallow: /
User-agent:moget/2.1Disallow: /
User-agent:WebZip/4.0Disallow: /
User-agent:WebStripperDisallow: /
User-agent:WebSaugerDisallow: /
User-agent:WebCopierDisallow: /
User-agent:NetAntsDisallow: /
User-agent:Mister PiXDisallow: /
User-agent:WebAutoDisallow: /
User-agent:TheNomadDisallow: /
User-agent:WWW-Collector-EDisallow: /
User-agent: RMADisallow: /
User-agent:libWeb/clsHTTPDisallow: /
User-agent:asteriasDisallow: /
User-agent:httplibDisallow: /
User-agent:turingosDisallow: /
User-agent:spannerDisallow: /
User-agent:InfoNaviRobotDisallow: /
User-agent:Harvest/1.5Disallow: /
User-agent:Bullseye/1.0Disallow: /
User-agent:Mozilla/4.0 (compatible; BullsEye; Windows 95)Disallow: /
User-agent:Crescent Internet ToolPak HTTP OLE Control v.1.0Disallow: /
User-agent:CherryPickerSE/1.0Disallow: /
User-agent:CherryPickerElite/1.0Disallow: /
User-agent:WebBandit/3.50Disallow: /
User-agent:NICErsPRODisallow: /
User-agent:Microsoft URL Control – 5.01.4511Disallow: /
User-agent: NotYour Business!Disallow: /
User-agent:Hidden-ReferrerDisallow: /
User-agent:DittoSpyderDisallow: /
User-agent:FoobotDisallow: /
User-agent:WebmasterWorldForumBotDisallow: /
User-agent:SpankBotDisallow: /
User-agent:BotALotDisallow: /
User-agent:lwp-trivial/1.34Disallow: /
User-agent:lwp-trivialDisallow: /
User-agent:BunnySlippersDisallow: /
User-agent:Microsoft URL Control – 6.00.8169Disallow: /
User-agent: URLyWarningDisallow: /
User-agent:Wget/1.6Disallow: /
User-agent:Wget/1.5.3Disallow: /
User-agent: WgetDisallow: /
User-agent:LinkWalkerDisallow: /
User-agent:cosmosDisallow: /
User-agent:mogetDisallow: /
User-agent:hloaderDisallow: /
User-agent:humanlinksDisallow: /
User-agent:LinkextractorProDisallow: /
User-agent:Offline ExplorerDisallow: /
User-agent: MataHariDisallow: /
User-agent:LexiBotDisallow: /
User-agent: WebImage CollectorDisallow: /
User-agent: TheIntraformantDisallow: /
User-agent:True_Robot/1.0Disallow: /
User-agent:True_RobotDisallow: /
User-agent:BlowFish/1.0Disallow: /
User-agent:JennyBotDisallow: /
User-agent: MIIxpc/4.2Disallow: /
User-agent:BuiltBotToughDisallow: /
User-agent:ProPowerBot/2.14Disallow: /
User-agent:BackDoorBot/1.0Disallow: /
User-agent:toCrawl/UrlDispatcherDisallow: /
User-agent:WebEnhancerDisallow: /
User-agent:suzuranDisallow: /
User-agent:TightTwatBotDisallow: /
User-agent: VCIWebViewer VCI WebViewer Win32Disallow: /
User-agent: VCIDisallow: /
User-agent:Szukacz/1.4Disallow: /
User-agent:QueryN MetasearchDisallow: /
User-agent:Openfind data gathereDisallow: /
User-agent:OpenfindDisallow: /
User-agent:Xenu’s Link Sleuth 1.1cDisallow: /
User-agent:Xenu’sDisallow: /
User-agent: ZeusDisallow: /
User-agent:RepoMonkey Bait & Tackle/v1.01Disallow: /
User-agent:RepoMonkeyDisallow: /
User-agent:Microsoft URL ControlDisallow: /
User-agent:OpenbotDisallow: /
User-agent: URLControlDisallow: /
User-agent: ZeusLink ScoutDisallow: /
User-agent: Zeus32297 Webster Pro V2.9 Win32Disallow: /
User-agent:Webster ProDisallow: /
User-agent:EroCrawlerDisallow: /
User-agent:LinkScan/8.1a UnixDisallow: /
User-agent:Keyword Density/0.9Disallow: /
User-agent:Kenjin SpiderDisallow: /
User-agent:Iron33/1.0.2Disallow: /
User-agent:Bookmark search toolDisallow: /
User-agent:GetRight/4.2Disallow: /
User-agent:FairAd ClientDisallow: /
User-agent:GaisbotDisallow: /
User-agent:Aqua_ProductsDisallow: /
User-agent:Radiation Retriever 1.1Disallow: /
User-agent:WebmasterWorld ExtractorDisallow: /
User-agent:Flaming AttackBotDisallow: /
User-agent:Oracle Ultra SearchDisallow: /
User-agent:MSIECrawlerDisallow: /
User-agent:PerManDisallow: /
User-agent:searchpreviewDisallow: /
User-agent: *Disallow:Disallow: /bin/Sitemap:/sitemap.php
页:
[1]