svnno****@sourc*****
svnno****@sourc*****
2008年 7月 1日 (火) 15:36:20 JST
SRADRevision: 682
http://svn.sourceforge.jp/cgi-bin/viewcvs.cgi?root=slashdotjp&view=rev&rev=682
Author: tach
Date: 2008-07-01 15:36:20 +0900 (Tue, 01 Jul 2008)
Log Message:
-----------
simplify robots.txt rule
Modified Paths:
--------------
images/robots.txt
-------------- next part --------------
Modified: images/robots.txt
===================================================================
--- images/robots.txt 2008-07-01 05:58:10 UTC (rev 681)
+++ images/robots.txt 2008-07-01 06:36:20 UTC (rev 682)
@@ -7,79 +7,24 @@
User-agent: Mediapartners-Google
Disallow:
-User-agent: Googlebot
-Crawl-delay: 100
-Disallow: /firehose.pl
-Disallow: /submit.pl
-Disallow: /users.pl
-Disallow: /zoo.pl
-Disallow: firehose.pl
-Disallow: submit.pl
-Disallow: users.pl
-Disallow: zoo.pl
-
-User-agent: Slurp
-Crawl-delay: 100
-Disallow: /firehose.pl
-Disallow: /submit.pl
-Disallow: /users.pl
-Disallow: /zoo.pl
-Disallow: firehose.pl
-Disallow: submit.pl
-Disallow: users.pl
-Disallow: zoo.pl
-
-User-agent: Yahoo-NewsCrawler
-Disallow: /firehose.pl
-Disallow: /submit.pl
-Disallow: /users.pl
-Disallow: /zoo.pl
-Disallow: firehose.pl
-Disallow: submit.pl
-Disallow: users.pl
-Disallow: zoo.pl
-
-User-Agent: msnbot
-Crawl-delay: 100
-Disallow: /firehose.pl
-Disallow: /submit.pl
-Disallow: /users.pl
-Disallow: /zoo.pl
-Disallow: firehose.pl
-Disallow: submit.pl
-Disallow: users.pl
-Disallow: zoo.pl
-
User-agent: *
-Crawl-delay: 100
-Disallow: /authors.pl
-Disallow: /index.pl
-Disallow: /article.pl
-Disallow: /comments.pl
+Crawl-delay: 5
+Disallow: /404.pl
+Disallow: /admin.pl
+Disallow: /ajax.pl
+Disallow: /blob.pl
+Disallow: /bookmark.pl
+Disallow: /fileadmin.pl
Disallow: /firehose.pl
-Disallow: /journal.pl
+Disallow: /hof.pl
+Disallow: /login.pl
Disallow: /messages.pl
Disallow: /metamod.pl
-Disallow: /users.pl
-Disallow: /search.pl
-Disallow: /submit.pl
-Disallow: /pollBooth.pl
Disallow: /pubkey.pl
-Disallow: /topics.pl
+Disallow: /relocate.pl
+Disallow: /slashdot-it.pl
+Disallow: /stats.pl
+Disallow: /submit.pl
+Disallow: /tags.pl
+Disallow: /users.pl
Disallow: /zoo.pl
-Disallow: authors.pl
-Disallow: index.pl
-Disallow: article.pl
-Disallow: comments.pl
-Disallow: firehose.pl
-Disallow: journal.pl
-Disallow: messages.pl
-Disallow: metamod.pl
-Disallow: users.pl
-Disallow: search.pl
-Disallow: submit.pl
-Disallow: pollBooth.pl
-Disallow: pubkey.pl
-Disallow: topics.pl
-Disallow: zoo.pl
-Disallow: /i