From o2on-svn @ lists.sourceforge.jp Thu Jul 30 15:58:24 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Thu, 30 Jul 2009 15:58:24 +0900 Subject: [o2on-svn] =?utf-8?b?WzE0MF0gRklYOiAg44OY44OD44OA44OV44Kj44O844Or?= =?utf-8?b?44OJ5YaF44Gr44Gm44CBIDog44Gn5Yy65YiH44KJ44KM44Gm44GE44Gq44GE?= =?utf-8?b?44Go44GN44Gr55Ww5bi444Go44Gq44KL54++6LGh44Gu5L+u5q2j?= Message-ID: <1248937104.636826.27254.nullmailer@users.sourceforge.jp> Revision: 140 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=140 Author: osa_p Date: 2009-07-30 15:58:24 +0900 (Thu, 30 Jul 2009) Log Message: ----------- FIX: ???????c??????????:?у?????????????????医幻?????憠院??信罩? Modified Paths: -------------- trunk/o2on/src.o2on/httpheader.cpp Modified: trunk/o2on/src.o2on/httpheader.cpp =================================================================== --- trunk/o2on/src.o2on/httpheader.cpp 2009-06-26 02:21:58 UTC (rev 139) +++ trunk/o2on/src.o2on/httpheader.cpp 2009-07-30 06:58:24 UTC (rev 140) @@ -271,8 +271,10 @@ // ex) // Host: www.foobar.com // Content-Length: 12345 - // - // + // Hoge: fuga + // fuga + // fuga + // // //???????????? @@ -289,7 +291,9 @@ char *eol = strstr(p1, "\r\n"); size_t valpos = strspn(p1, " \t"); if (valpos == 0) { + // ???field-name???? if (!name.empty()) { + // field?????? fields.insert(strmap::value_type(name, value)); } name.erase(); @@ -298,6 +302,11 @@ //field-name if ((p2 = strchr(p1, ':')) == NULL) return (0); + if ( eol < p2 ) { + // 1????:????????+ return (0); + } + // field-name???? name.assign(p1, p2); //std::transform(name.begin(), name.end(), name.begin(), tolower); if (*(p2+1) == ' ') @@ -305,12 +314,18 @@ else p1 = p2 + 1; } - else + else { + if (name.empty()) { + // ???????????????? + return (0); + } + // ????? p1 += valpos; + } - //field-value + //field-value???????????????????? value.append(p1, eol); - p1 = eol + 2; + p1 = eol + 2; // ??? } if (!name.empty() && !value.empty()) { From o2on-svn @ lists.sourceforge.jp Thu Jul 30 16:26:30 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Thu, 30 Jul 2009 16:26:30 +0900 Subject: [o2on-svn] =?utf-8?b?WzE0MV0gRklYOiAgWE1M44Gu6ZaJ44GY44K/44Kw44KS?= =?utf-8?b?5L+u5q2j?= Message-ID: <1248938790.256184.28924.nullmailer@users.sourceforge.jp> Revision: 141 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=141 Author: osa_p Date: 2009-07-30 16:26:30 +0900 (Thu, 30 Jul 2009) Log Message: ----------- FIX: XMLの閉じタグを修正 Modified Paths: -------------- trunk/o2on/src.o2on/O2Boards.cpp Modified: trunk/o2on/src.o2on/O2Boards.cpp =================================================================== --- trunk/o2on/src.o2on/O2Boards.cpp 2009-07-30 06:58:24 UTC (rev 140) +++ trunk/o2on/src.o2on/O2Boards.cpp 2009-07-30 07:26:30 UTC (rev 141) @@ -542,7 +542,7 @@ } } ExLock.Unlock(); - xml += L""EOL; + xml += L""EOL; unicode2ascii(xml, out); } From o2on-svn @ lists.sourceforge.jp Sat Aug 8 04:50:14 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 04:50:14 +0900 Subject: [o2on-svn] [143] Fix to record profile properly. Message-ID: <1249674614.312443.9289.nullmailer@users.sourceforge.jp> Revision: 143 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=143 Author: nawota Date: 2009-08-08 04:50:14 +0900 (Sat, 08 Aug 2009) Log Message: ----------- Fix to record profile properly. Modified Paths: -------------- trunk/opy2on/lib/o2on_job.py trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py 2009-08-07 19:13:53 UTC (rev 142) +++ trunk/opy2on/lib/o2on_job.py 2009-08-07 19:50:14 UTC (rev 143) @@ -40,11 +40,14 @@ def run(self): try: if o2on_config.RecordProfile: - if not os.path.exists(os.path.dirname(o2on_config.ProfileDir)): - os.makedirs(os.path.dirname(o2on_config.ProfileDir)) - profname = os.path.join(o2on_config.ProfileDir, - "o2on_"+"_".join(self.name.split(" "))+".prof") - cProfile.runctx('self.dummy()', None, {'self':self,}, profname) + if not os.path.exists(o2on_config.ProfileDir): + try: + os.makedirs(o2on_config.ProfileDir) + except OSError, inst: + if inst.errno != 17: raise inst + profname = os.path.join(o2on_config.ProfileDir, + "o2on_"+"_".join(self.name.split(" "))+".prof") + cProfile.runctx('self.dummy()', None, {'self':self,}, profname) else: self.dummy() except Exception,inst: if o2on_config.OutputErrorFile: Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-07 19:13:53 UTC (rev 142) +++ trunk/opy2on/lib/o2on_server.py 2009-08-07 19:50:14 UTC (rev 143) @@ -394,14 +394,15 @@ if not category: category = 'dat' if category == 'dat': data = self.rfile.read(l) - dom = xml.dom.minidom.parseString(data) - top = dom.getElementsByTagName("keys") - if len(top): - for k in top[0].getElementsByTagName("key"): - key = o2on_key.Key() - key.from_xml_node(k) - self.server.glob.keydb.add(key) - dom.unlink() + if data: + dom = xml.dom.minidom.parseString(data) + top = dom.getElementsByTagName("keys") + if len(top): + for k in top[0].getElementsByTagName("key"): + key = o2on_key.Key() + key.from_xml_node(k) + self.server.glob.keydb.add(key) + dom.unlink() else: self.server.glob.logger.log("P2PSERVER","Unknown Category %s" % category) def do_im(self, node): headers = common_header.copy() From o2on-svn @ lists.sourceforge.jp Sat Aug 8 05:12:31 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 05:12:31 +0900 Subject: [o2on-svn] [144] Ignore errors when shuting down servers. Message-ID: <1249675951.216172.11684.nullmailer@users.sourceforge.jp> Revision: 144 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=144 Author: nawota Date: 2009-08-08 05:12:31 +0900 (Sat, 08 Aug 2009) Log Message: ----------- Ignore errors when shuting down servers. Modified Paths: -------------- trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-07 19:50:14 UTC (rev 143) +++ trunk/opy2on/lib/o2on_server.py 2009-08-07 20:12:31 UTC (rev 144) @@ -36,8 +36,11 @@ self.requests = [] def shutdown(self): for r in self.requests: - r.shutdown(socket.SHUT_RDWR) - r.close() + try: + r.shutdown(socket.SHUT_RDWR) + r.close() + except Exception: + pass BaseHTTPServer.HTTPServer.shutdown(self) def finish_request(self, request, client_address): self.requests.append(request) From o2on-svn @ lists.sourceforge.jp Sat Aug 8 05:33:48 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 05:33:48 +0900 Subject: [o2on-svn] [145] Ignore timeout error. Message-ID: <1249677228.196489.11182.nullmailer@users.sourceforge.jp> Revision: 145 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=145 Author: nawota Date: 2009-08-08 05:33:48 +0900 (Sat, 08 Aug 2009) Log Message: ----------- Ignore timeout error. Modified Paths: -------------- trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-07 20:12:31 UTC (rev 144) +++ trunk/opy2on/lib/o2on_server.py 2009-08-07 20:33:48 UTC (rev 145) @@ -46,6 +46,8 @@ self.requests.append(request) try: BaseHTTPServer.HTTPServer.finish_request(self, request, client_address) + except socket.timeout: + pass except Exception,inst: if isinstance(inst, socket.error) and inst.errno in (104, 32): pass From o2on-svn @ lists.sourceforge.jp Sat Aug 8 05:40:49 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 05:40:49 +0900 Subject: [o2on-svn] [146] Fix the case when dat query fail. Message-ID: <1249677649.675613.20649.nullmailer@users.sourceforge.jp> Revision: 146 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=146 Author: nawota Date: 2009-08-08 05:40:49 +0900 (Sat, 08 Aug 2009) Log Message: ----------- Fix the case when dat query fail. Modified Paths: -------------- trunk/opy2on/lib/o2on_job.py Modified: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py 2009-08-07 20:33:48 UTC (rev 145) +++ trunk/opy2on/lib/o2on_job.py 2009-08-07 20:40:49 UTC (rev 146) @@ -370,11 +370,12 @@ except socket.error, inst: logger.log("DATQUERY", inst) else: - logger.popup("DATQUERY", "Got queried dat %s" % dat.path()) + if dat: + logger.popup("DATQUERY", "Got queried dat %s" % dat.path()) + datdb.add_dat(dat) + datdb.save() nodedb.add_node(node) - datdb.add_dat(dat) nodedb.save() - datdb.save() self.glob.keyquery.save() # Server thread From o2on-svn @ lists.sourceforge.jp Sat Aug 8 06:37:22 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 06:37:22 +0900 Subject: [o2on-svn] [147] Fix dat query's bug. Message-ID: <1249681042.870983.20968.nullmailer@users.sourceforge.jp> Revision: 147 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=147 Author: nawota Date: 2009-08-08 06:37:22 +0900 (Sat, 08 Aug 2009) Log Message: ----------- Fix dat query's bug. Ignore XML Parse error. Modified Paths: -------------- trunk/opy2on/lib/o2on_job.py trunk/opy2on/lib/o2on_node.py trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py 2009-08-07 20:40:49 UTC (rev 146) +++ trunk/opy2on/lib/o2on_job.py 2009-08-07 21:37:22 UTC (rev 147) @@ -359,11 +359,11 @@ if not node: node = o2on_node.Node(k.nodeid, k.ip, k.port) logger.log("DATQUERY","dat query %s to %s" % (hexlify(k.hash),hexlify(node.id))) try: - dat = node.dat(k.hash, self.glob) + dat = node.dat(k.hash, None, self.glob) except o2on_node.NodeRemovable: nodedb.remove(node) nodedb.save() - self.glob.keydb.remove_bynodeid(node.id) + self.glob.keydb.remove_bynodeid(node.id) self.glob.keydb.save() except o2on_node.NodeRefused: pass Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2009-08-07 20:40:49 UTC (rev 146) +++ trunk/opy2on/lib/o2on_node.py 2009-08-07 21:37:22 UTC (rev 147) @@ -246,7 +246,7 @@ if datdb and board: omiyage = datdb.getRandomInBoard(board) if(dathash): - headers['X-O2-Target-Key'] = dathash + headers['X-O2-Target-Key'] = hexlify(dathash) elif(board): headers['X-O2-Target-Board'] = board if omiyage: Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-07 20:40:49 UTC (rev 146) +++ trunk/opy2on/lib/o2on_server.py 2009-08-07 21:37:22 UTC (rev 147) @@ -400,7 +400,10 @@ if category == 'dat': data = self.rfile.read(l) if data: - dom = xml.dom.minidom.parseString(data) + try: + dom = xml.dom.minidom.parseString(data) + except xml.dom.minidom.ExpatError: + return top = dom.getElementsByTagName("keys") if len(top): for k in top[0].getElementsByTagName("key"): From o2on-svn @ lists.sourceforge.jp Sat Aug 8 06:55:34 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 06:55:34 +0900 Subject: [o2on-svn] [148] Fix response 400, 404. Message-ID: <1249682134.010758.9591.nullmailer@users.sourceforge.jp> Revision: 148 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=148 Author: nawota Date: 2009-08-08 06:55:33 +0900 (Sat, 08 Aug 2009) Log Message: ----------- Fix response 400, 404. Modified Paths: -------------- trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-07 21:37:22 UTC (rev 147) +++ trunk/opy2on/lib/o2on_server.py 2009-08-07 21:55:33 UTC (rev 148) @@ -489,14 +489,14 @@ logger.log("P2PSERVER", "\theader was %s" % self.headers) header = common_header.copy() self.wfile.write("HTTP/1.0 400 Bad Request\r\n") - for h in header: self.wfile.write("%s: %s" % (h,header[h])) + for h in header: self.wfile.write("%s: %s\r\n" % (h,header[h])) self.wfile.write("\r\n") self.wfile.close() def response_404(self): #print "p2p server response 404 %s" % self.client_address[0] header = common_header.copy() self.wfile.write("HTTP/1.0 404 Not Found\r\n") - for h in header: self.wfile.write("%s: %s" % (h,header[h])) + for h in header: self.wfile.write("%s: %s\r\n" % (h,header[h])) self.wfile.write("\r\n") self.wfile.close() def get_requested_header(self): From o2on-svn @ lists.sourceforge.jp Sat Aug 8 06:57:27 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 06:57:27 +0900 Subject: [o2on-svn] [149] import ExpatError Message-ID: <1249682247.812529.11988.nullmailer@users.sourceforge.jp> Revision: 149 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=149 Author: nawota Date: 2009-08-08 06:57:27 +0900 (Sat, 08 Aug 2009) Log Message: ----------- import ExpatError Modified Paths: -------------- trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-07 21:55:33 UTC (rev 148) +++ trunk/opy2on/lib/o2on_server.py 2009-08-07 21:57:27 UTC (rev 149) @@ -18,6 +18,7 @@ import time import traceback import sys +from xml.parsers.expat import ExpatError import o2on_config from o2on_const import regHosts, ProtocolVer, AppName @@ -402,7 +403,7 @@ if data: try: dom = xml.dom.minidom.parseString(data) - except xml.dom.minidom.ExpatError: + except ExpatError: return top = dom.getElementsByTagName("keys") if len(top): From o2on-svn @ lists.sourceforge.jp Sat Aug 8 07:04:34 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 07:04:34 +0900 Subject: [o2on-svn] [150] key.xml() doesn't need argument anymore. Message-ID: <1249682674.497192.20219.nullmailer@users.sourceforge.jp> Revision: 150 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=150 Author: nawota Date: 2009-08-08 07:04:34 +0900 (Sat, 08 Aug 2009) Log Message: ----------- key.xml() doesn't need argument anymore. Modified Paths: -------------- trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-07 21:57:27 UTC (rev 149) +++ trunk/opy2on/lib/o2on_server.py 2009-08-07 22:04:34 UTC (rev 150) @@ -444,7 +444,7 @@ if key: xml_data = "\r\n" xml_data += "\r\n" - xml_data += key.xml(self.server.glob) + xml_data += key.xml() xml_data += "\r\n" else: neighbors = self.server.glob.nodedb.neighbors_nodes(target, True) From o2on-svn @ lists.sourceforge.jp Sat Aug 8 04:13:53 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 04:13:53 +0900 Subject: [o2on-svn] [142] add opy2on files. Message-ID: <1249672433.153340.18901.nullmailer@users.sourceforge.jp> Revision: 142 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=142 Author: nawota Date: 2009-08-08 04:13:53 +0900 (Sat, 08 Aug 2009) Log Message: ----------- add opy2on files. Added Paths: ----------- trunk/opy2on/ trunk/opy2on/README trunk/opy2on/lib/ trunk/opy2on/lib/o2on_const.py trunk/opy2on/lib/o2on_dat.py trunk/opy2on/lib/o2on_im.py trunk/opy2on/lib/o2on_job.py trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_node.py trunk/opy2on/lib/o2on_profile.py trunk/opy2on/lib/o2on_server.py trunk/opy2on/lib/o2on_util.py trunk/opy2on/o2on_config.py.sample trunk/opy2on/opy2on.py Property changes on: trunk/opy2on ___________________________________________________________________ Added: svn:ignore + db o2on_config.py profile util dat error-*.txt Added: trunk/opy2on/README =================================================================== --- trunk/opy2on/README (rev 0) +++ trunk/opy2on/README 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,110 @@ +これはなに? + +o2on(http://o2on.sourceforge.jp/) の Python での(不完全な)クローン +opy2on (http://pc12.2ch.net/test/read.cgi/tech/1231570128/ 356さん命名) + +※ 本家 o2on の動作と完全に一致するわけではないです。詳しくは「特徴」の + 部分を読んでください。 + +必要なもの + +- python 2.6 +- pycrypto +- D-Bus を使うならば python-dbus + +起動 + +- o2on_config.py.sample を見ながら好みにあわせて修正し、 o2on_config.py + を作る。 + - debug に協力してくださる方は、 OutputErrorFile を True にしてくださ + い。エラー時に大変役に立ちます。 + - Xが動いている時に起動するつもりの方は、 UseDBus = True の時の動きを + 使用WindowManagerとともに教えてください。 + - 重いぞ!という方は RecordProfile を True にしてください。ProfileDir + 下にプロファイルが出力されます。 +- ポートを開けたり閉めたりする。 +- ./opy2on で起動。 + +操作 + +- 以下のコマンドを認識します。 + - datq 検索中のdat一覧表示 + - exit プログラムの終了 + - exportnode ノードをexportnodes.xmlに出力 + - help ヘルプを表示 + - keys datキーを表示 + - PAGER を設定しておいてください + - myid 自ノードのIDを表示 + - mynode 自ノードのハッシュを表示 + - mypubkey 自ノードの公開鍵を表示 + - nodes ノード情報を表示 + - stat ノード数、検索中dat数、所有dat数、datキー数を表示 +- 全ての表示はUTF-8で出力されます。 +- adminサーバも動いています。 + - http:///shutdown/really でも終了します。 + +特徴 (というかほとんど直すべきとこの列挙) + +- dat を gzip で保存可能 +- Ping をしなおす時間を設定できる +- プロキシに dat をリクエストした時の反応が違う -> 下記参照 +- ping にたいして localなIPを返してくるノードがいたのでそのIPは無視している +- ノードリストが0になると Web などのノードリストから再読みこみします。 +- D-Bus を使ってポップアップします。 +- 処理されないエラーがおきると自動的に終了します。 + +- ●は対応してないです +- プロキシで range が指定されてもキャッシュから読んだ時は全体で返答 + - 複数台からプロキシに接続することをかんがえてないような気がします…。 +- オリジナルの url を記憶していないので dat をあげる時にはいつも + DAT-Path で送信します +- フラグを解釈していないです +- 大量な dat を持った環境でのテストができていない (メモリ食いすぎるかも) + - sqlite化とかするべきなのだろうな… +- GUI が動いてないです。 +- saku キーに対応してないです +- ノード名に日本語をうまく使えないです + - 一応エラーが出ないようにはなったけれど POST した時に日本語部分が消えます +- UPnP は対応してないです +- IPフィルタがないです + - IPフィルタとかは iptables なんかを使えばいいと思ってます。 + - 帯域制限も iproute とか使えばいいと思います。 + +対応状況 + +- P2P client + - dat o + - collection o + - im o + - broadcast x + - profile x + - ping o + - store o + - findnode o + - findvalue o + +- P2P server + - dat o + - collection o + - im o + - broadcast x + - profile x + - ping o + - store o + - findnode o + - findvalue o + +proxy server キャッシュへの保存の方法 +- 持ってない dat 全体が取得された -> キャッシュに保存 +- 持ってる dat の差分が取得された -> キャッシュに追記 +- 持ってない dat がリクエストされた + - RequestNonExistDat が True の時はこっそり取得 + - こっそり取得するから巡回してるとこなら dat をインポートしなくてもいいかも + +FIXME + +- HTTPヘッダは大文字・小文字区別しない -> 本家 o2on が区別してるから困る… + - とりあえずうけつける分だけでも区別しないようにしよう。 +- リロードバーボンにならないように 2chへの接続は適度に遅延をつけよう。 (navi2ch を参考に) +- エンコードまわりうまくいってるのだろうか…? +- CPU食いすぎな気がする Added: trunk/opy2on/lib/o2on_const.py =================================================================== --- trunk/opy2on/lib/o2on_const.py (rev 0) +++ trunk/opy2on/lib/o2on_const.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,21 @@ +#!/usr/bin/python +import os.path + +regHosts = r'(2ch\.net|bbspink\.com|machibbs\.com|machi\.to)' + +DBDir = "db" +DatQueryFile = os.path.join(DBDir, "datq.pkl") +KeyQueryFile = os.path.join(DBDir, "keyq.pkl") +DatDBFile = os.path.join(DBDir, "datdb.pkl") +KeyDBFile = os.path.join(DBDir, "keydb.pkl") +NodeDBFile = os.path.join(DBDir, "nodedb.pkl") +ProfileFile = os.path.join(DBDir, "profile.pkl") +IMDBFile = os.path.join(DBDir, "imdb.pkl") + +ProtocolName = "O2" +ProtocolVer = 0.2 +AppName = "opy2on" +AppMajorVer = 0 +AppMinorVer = 0 +AppBuildNo = 1 + Added: trunk/opy2on/lib/o2on_dat.py =================================================================== --- trunk/opy2on/lib/o2on_dat.py (rev 0) +++ trunk/opy2on/lib/o2on_dat.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,237 @@ +#!/usr/bin/python + +import threading +import cPickle +import os.path +import re +import gzip +import random +import time + +from o2on_const import regHosts, DatQueryFile, DatDBFile +import o2on_config +import o2on_util + +class Dat: + def __init__(self, path=None): + self.reset() + self.published = 0 + if path: self.setpath(path) + def __cmp__(self,x): + return cmp(self.hash(), x.hash()) + def reset(self): + self.domain = None + self.board = None + self.datnum = None + def setpath(self,path): + self.reset() + regDatPath = re.compile(r'^'+regHosts+'/([^/]+)/(\d+)$') + regDatUrl = re.compile(r'^http://[^.]+\.'+regHosts+'/test/read\.cgi/([^/]+)/(\d+)/$') + m = regDatPath.match(path) + if not m: m = regDatUrl.match(path) + if not m: return False + d = m.group(1) + if d == 'machi.to': d = 'machibbs.com' + self.domain = d + self.board = m.group(2) + self.datnum = m.group(3) + return True + def path(self): + if self.valid(): + return "/".join((self.domain, self.board, self.datnum)) + def fullboard(self): + if self.domain and self.board: + return self.domain+":"+self.board + raise Exception + def valid(self): + return ((self.domain and self.board and self.datnum) != None) + def datpath(self): + if self.valid(): + return os.path.join(o2on_config.DatDir, self.domain, self.board, + self.datnum[:4],self.datnum+".dat") + raise Exception + def data(self): + dp = self.datpath() + if os.path.exists(dp): f=open(dp) + elif os.path.exists(dp+".gz"): f=gzip.GzipFile(dp+".gz") + else: f=None + if f: + res=f.read() + f.close() + return res + return None + def hash(self): + return o2on_util.datkeyhash(self.path()) + def save(self,data, start=0): + if not self.valid(): return + bl = o2on_config.DatCollectionBoardList + if bl != None: + if not self.fullboard() in bl: return False + if(len(data)<2): return False + if(data[-1] != "\n"): return False + if(data[-2] == "\r"): return False + if start == 0: + m = re.compile(r'^([^\n]*)\n').match(data) + first = m.group(1) + if not re.compile(r'^.*<>.*<>.*<>.*<>.*$').match(first): return False + output = self.datpath() + if os.path.exists(output): + if o2on_config.DatSaveAsGzip: + f = open(output,'rb') + g = gzip.GzipFile(output+".gz",'w') + g.write(f.read()) + g.close() + f.close() + os.remove(output) + else: + if(os.path.getsize(output)0: + f=open(output,'ab') + f.write(data[(len(saved)-start):]) + f.close() + return True + return True + if os.path.exists(output+".gz"): + f=gzip.GzipFile(output+".gz",'rb') + saved = f.read() + f.close() + if(len(saved)0: + f=gzip.GzipFile(output+".gz",'ab') + f.write(data[(len(saved)-start):]) + f.close() + return True + return True + else: + if start>0: return True + if not os.path.exists(os.path.dirname(output)): + os.makedirs(os.path.dirname(output)) + if o2on_config.DatSaveAsGzip: + f = gzip.GzipFile(output+".gz",'w') + else: + f = open(output, 'wb') + f.write(data) + f.close() + return True + +class DatDB: + def __init__(self,g): + self.glob = g + self.lock = threading.Lock() + with self.lock: + self.hashmap = {} + self.boardmap = {} + self.publishmap = {} + self.load() + if len(self.hashmap) == 0: + g.logger.log("DATDB", "Generating DatDB") + self.generate() + self.save() + g.logger.log("DATDB","Generated DatDB") + def __len__(self): + with self.lock: + return len(self.hashmap) + def getRandomInBoard(self,board): + if board in self.boardmap: + h = random.choice(self.boardmap[board]) + return self.hashmap[h] + return None + def choice(self): + return self.hashmap[random.choice(self.hashmap.keys())] + def get(self,x): + with self.lock: + return self.hashmap.get(x) + def has_key(self,key): + with self.lock: + return o2on_util.datkeyhash(key) in self.hashmap + def add_dat(self, dat): + with self.lock: + befdat = self.hashmap.get(dat.hash()) + self.hashmap[dat.hash()] = dat + if not dat.fullboard() in self.boardmap: + self.boardmap[dat.fullboard()] = [] + self.boardmap[dat.fullboard()].append(dat.hash()) + if not befdat: + dat.published = int(time.time()) + if dat.published not in self.publishmap: + self.publishmap[dat.published]=[] + self.publishmap[dat.published].append(dat.hash()) + else: + dat.published = befdat.published + def add(self, path, data, start=0): + dat = Dat(path) + if dat.save(data, start): self.add_dat(dat) + def published(self, datid, publish_time): + if len(datid) != 20: raise Exception + with self.lock: + if datid not in self.hashmap: raise Exception + dat = self.hashmap[datid] + self.publishmap[dat.published].remove(datid) + dat.published = publish_time + if publish_time not in self.publishmap: self.publishmap[publish_time]=[] + self.publishmap[publish_time].append(datid) + def dat_to_publish(self, last_published_before, limit): + res = [] + if limit == 0: return res + for x in sorted(self.publishmap.keys()): + for y in self.publishmap[x]: + res.append(self.hashmap[y]) + limit -= 1 + if limit == 0: return res + return res + def generate(self): + regdat = re.compile('^(\d+)\.dat(?:\.gz)?$') + sep = re.escape(os.sep) + regdatdir = re.compile(regHosts+sep+'(.+)'+sep+'\d{4}$') + with self.lock: + self.hashmap = {} + self.boardmap = {} + self.publishmap = {0:[]} + for root, dirs, files in os.walk(o2on_config.DatDir): + for f in files: + m1 = regdat.match(f) + if not m1: continue + m2 = regdatdir.search(root) + if not m2: continue + path = m2.group(1)+"/"+m2.group(2)+"/"+m1.group(1) + d = Dat(path) + with self.lock: + self.hashmap[d.hash()] = d + if not d.fullboard() in self.boardmap: + self.boardmap[d.fullboard()] = [] + self.boardmap[d.fullboard()].append(d.hash()) + self.publishmap[0].append(d.hash()) + self.glob.logger.log("DATDB", "added %s" % path) + def load(self): + if(os.path.isfile(DatDBFile)): + pkl_file = open(DatDBFile,"rb") + with self.lock: + self.hashmap = cPickle.load(pkl_file) + self.boardmap = cPickle.load(pkl_file) + self.publishmap = cPickle.load(pkl_file) + pkl_file.close() + def save(self): + pkl_file = open(DatDBFile,"wb") + with self.lock: + cPickle.dump(self.hashmap, pkl_file,-1) + cPickle.dump(self.boardmap, pkl_file,-1) + cPickle.dump(self.publishmap, pkl_file,-1) + pkl_file.close() + Added: trunk/opy2on/lib/o2on_im.py =================================================================== --- trunk/opy2on/lib/o2on_im.py (rev 0) +++ trunk/opy2on/lib/o2on_im.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,81 @@ +#!/usr/bin/python + +import threading +import os.path +import cPickle +import datetime + +from o2on_const import IMDBFile +from o2on_node import e2ip, ip2e +from binascii import hexlify, unhexlify + +class IMessage: + def __init__(self): + self.ip = None + self.port = None + self.id = None + self.pubkey = None + self.name = "" + self.date = 0 + self.msg = None + self.key = None + self.mine = False + self.paths = [] + self.broadcast = False + def __eq__(self, x): + if self.broadcast != x.broadcast: return False + if self.broadcast: + return self.ip == x.ip and self.port == x.port and self.msg == x.msg + else: + return self.key == x.key + def from_node(self, n): + self.ip = n.ip + self.port = n.port + self.id = n.id + self.pubkey = n.pubkey + self.name = n.name + def from_xml_node(self, xml_node): + res = {} + for c in ('ip','port','id','pubkey','name','msg',): + try: res[c] = xml_node.getElementsByTagName(c)[0].childNodes[0].data + except IndexError: continue + if res.get('ip'): self.ip = e2ip(res.get('ip')) + if res.get('port'): self.port = int(res.get('port')) + if res.get('id'): self.id = unhexlify(res.get('id')) + if res.get('pubkey'): self.pubkey = unhexlify(res.get('pubkey')) + if res.get('name'): self.name = res.get('name').encode('utf-8') + if res.get('msg'): self.msg = res.get('msg').encode('utf-8') + +class IMDB: + def __init__(self, g): + self.lock = threading.Lock() + with self.lock: + self.ims = [] + self.glob = g + self.load() + def im_list(self): + res = [] + with self.lock: + for i in self.ims: + x = (i.mine, str(datetime.datetime.fromtimestamp(int(i.date))), + hexlify(i.id), ip2e(i.ip), i.port, i.name.decode('utf-8'), + i.msg.decode('utf-8')) + res.append(x) + return res + def add(self,im): + with self.lock: + self.ims.append(im) + def save(self): + self.expire() + f = open(IMDBFile,'wb') + with self.lock: + cPickle.dump(self.ims, f, -1) + f.close() + def load(self): + if os.path.isfile(IMDBFile): + f = open(IMDBFile,'rb') + with self.lock: + self.ims = cPickle.load(f) + f.close() + def expire(self): + pass Added: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py (rev 0) +++ trunk/opy2on/lib/o2on_job.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,416 @@ +#!/usr/bin/python +# -*- coding: utf-8 + +from binascii import hexlify, unhexlify +import threading +import time +import socket +import BaseHTTPServer +import gzip +import random +import re +import time +import socket +import os +import httplib +import traceback +import sys + +import o2on_server +import o2on_config +import o2on_util +import o2on_node +import o2on_key + +if o2on_config.RecordProfile: + import cProfile + +class JobThread(threading.Thread): + def __init__(self, name, s, g): + threading.Thread.__init__(self) + self.finish = False + self.awake = False + self.name = name + self.glob = g + self.sec = s + def stop(self): + self.finish = True + def wakeup(self): + self.awake = True + def run(self): + try: + if o2on_config.RecordProfile: + if not os.path.exists(os.path.dirname(o2on_config.ProfileDir)): + os.makedirs(os.path.dirname(o2on_config.ProfileDir)) + profname = os.path.join(o2on_config.ProfileDir, + "o2on_"+"_".join(self.name.split(" "))+".prof") + cProfile.runctx('self.dummy()', None, {'self':self,}, profname) + else: self.dummy() + except Exception,inst: + if o2on_config.OutputErrorFile: + f = open('error-'+str(int(time.time()))+'.txt', 'w') + traceback.print_exc(file=f) + f.close() + self.glob.logger.popup("ERROR", str(inst)) + self.glob.shutdown.set() + def dummy(self): + self.glob.logger.log("JOBMANAGER", "job %s started" % self.name) + while not self.finish: + #t = time.time() + self.dojob(self.glob.nodedb, self.glob.logger, self.glob.prof, self.glob.datdb, + self.glob.datquery) + if self.finish: break + diff = int(self.sec) #int(self.sec - (time.time()-t)) + if 00: + logger.popup("NODECOLLECTOR","load node from web") + nodes.node_from_web() + target = o2on_util.randomid() + for x in nodes.neighbors_nodes(target, False): + if self.finish: break + logger.log("NODECOLLECTOR", "findnode to %s" % (hexlify(x.id))) + try: + newnodes = x.findnode(target) + except o2on_node.NodeRemovable: + nodes.remove(x) + nodes.save() + self.glob.keydb.remove_bynodeid(x.id) + self.glob.keydb.save() + except o2on_node.NodeRefused: + pass + except socket.error, inst: + self.glob.logger.log("NODECOLLECTOR", inst) + else: + for n in newnodes: + logger.log("NODECOLLECTOR","\tadd node %s" % (hexlify(n.id))) + nodes.add_node(n) + nodes.add_node(x) + nodes.save() + time.sleep(1) + if len(nodes)<30: self.sec = 30 + else: self.sec = 180 + +class DatCollectorThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"dat collector",60,g) + def dojob(self, nodes, logger, prof, datdb, datq): + board = nodes.get_random_board() + if not board: return + for n in nodes.get_nodes_for_board(board): + if self.finish: break + logger.log("DATCOLLECTOR","dat (%s) to %s" % (board,hexlify(n.id))) + try: + dat = n.dat(None, board, datdb) + except o2on_node.NodeRemovable: + nodes.remove(n) + nodes.save() + self.glob.keydb.remove_bynodeid(n.id) + self.glob.keydb.save() + except o2on_node.NodeRefused: + pass + except socket.error, inst: + self.glob.logger.log("DATCOLLECTOR", inst) + else: + if dat: + logger.log("DATCOLLECTOR","\tGot dat %s" % dat.path()) + nodes.add_node(n) + datdb.add_dat(dat) + nodes.save() + datdb.save() + break + time.sleep(1) + +class GetIPThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"get global IP",60,g) + def dojob(self, nodes, logger, prof, datdb, datq): + regLocalIP = re.compile(r'^(?:10\.|192\.168\.|172\.(?:1[6-9]|2[0-9]|3[01])\.)') + for n in nodes.neighbors_nodes(prof.mynode.id, False, 100): + if self.finish: break + logger.log("GETIP","getIP to %s" % hexlify(n.id)) + try: + r = n.ping(True) + except o2on_node.NodeRemovable: + nodes.remove(n) + nodes.save() + self.glob.keydb.remove_bynodeid(n.id) + self.glob.keydb.save() + except o2on_node.NodeRefused: + pass + except socket.error, inst: + self.glob.logger.log("GETIP", inst) + else: + if r: + if not prof.mynode.ip: + ip = o2on_node.e2ip(r[:8]) + if not regLocalIP.match(ip): + prof.mynode.ip = ip + self.finish = True + logger.popup("GETIP","Got Global IP %s" % ip) + nodes.add_node(n) + break + else: break + +class AskNodeCollectionThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"ask node collection",60,g) + def dojob(self, nodedb, logger, prof, datdb, datq): + for n in nodedb.neighbors_nodes(o2on_util.randomid(), False): + if self.finish: break + logger.log("ASKNODECOLLECTION", "node collection to %s" % (hexlify(n.id))) + try: + colboards = n.collection(self.glob) + except o2on_node.NodeRemovable: + nodedb.remove(n) + nodedb.save() + self.glob.keydb.remove_bynodeid(n.id) + self.glob.keydb.save() + except o2on_node.NodeRefused: + pass + except socket.error, inst: + logger.log("ASKNODECOLLECTION", inst) + else: + logger.log("ASKNODECOLLECTION", + "\tadd collection for %s" % (hexlify(n.id))) + nodedb.reset_collection_for_node(n) + for b in colboards: + nodedb.add_collection(b,n) + nodedb.add_node(n) + nodedb.save() + +class PublishOrigThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"publish original",60,g) + def dojob(self, nodedb, logger, prof, datdb, datq): + mynode = prof.mynode + if not mynode.ip or mynode.port == 0: return + if len(nodedb)<10: return + dats = datdb.dat_to_publish(time.time()-(3*60*60), 500) + keys = [] + for d in dats: + if self.finish: return + k = o2on_key.Key() + k.from_dat(d) + k.from_node(prof.mynode) + keys.append(k) + publish_nodes = {} + for k in keys: + if self.finish: return + for n in nodedb.neighbors_nodes(k.hash, False): + if self.finish: return + if n.id not in publish_nodes: publish_nodes[n.id] = [] + publish_nodes[n.id].append(k) + for n in publish_nodes: + if self.finish: return + node = nodedb[n] + if not node: continue + logger.log("PUBLISHORIGINAL","publish original to %s" % (hexlify(n))) + try: + node.store("dat", publish_nodes[n]) + except o2on_node.NodeRemovable: + nodedb.remove(node) + nodedb.save() + self.glob.keydb.remove_bynodeid(node.id) + self.glob.keydb.save() + except o2on_node.NodeRefused: + pass + except socket.error, inst: + logger.log("PUBLISHORIGINAL", inst) + else: + nodedb.add_node(node) + pubtime = int(time.time()) + for k in publish_nodes[n]: + datdb.published(k.hash,pubtime) + self.glob.keydb.save() + datdb.save() + logger.log("PUBLISHORIGINAL","\tpublished original") + nodedb.save() + time.sleep(1) + +class PublishKeyThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"publish key",120,g) + def dojob(self, nodedb, logger, prof, datdb, datq): + mynode = prof.mynode + if not mynode.ip or mynode.port == 0: + return + publish_nodes = {} + #t = time.time() + keys = self.glob.keydb.keys_to_publish(int(time.time()-30*60)) + #logger.log("get keys to publush %d" % (time.time()-t)) + for k in keys: + if self.finish: return + for n in nodedb.neighbors_nodes(k.hash, False): + if self.finish: return + if n.id not in publish_nodes: publish_nodes[n.id] = [] + publish_nodes[n.id].append(k) + for n in publish_nodes: + if self.finish: return + node = nodedb[n] + if not node: continue + logger.log("PUBLISHKEY","publish key to %s" % (hexlify(n))) + try: + node.store("dat", publish_nodes[n]) + except o2on_node.NodeRemovable: + nodedb.remove(node) + nodedb.save() + self.glob.keydb.remove_bynodeid(node.id) + self.glob.keydb.save() + except o2on_node.NodeRefused: + pass + except socket.error, inst: + logger.log("PUBLISHKEY", inst) + else: + nodedb.add_node(node) + pubtime = int(time.time()) + for k in publish_nodes[n]: + self.glob.keydb.published(k.hash,pubtime) + self.glob.keydb.save() + logger.log("PUBLISHKEY","\tpublished key") + nodedb.save() + time.sleep(1) + +class SearchThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"search",60,g) + def stop(self): + JobThread.stop(self) + self.glob.datquery.semap.release() + def dojob(self, nodedb, logger, prof, datdb, datq): + d = datq.pop() + if self.finish: return + if datdb.has_key(d): + datq.save() + return + kid = o2on_util.datkeyhash(d) + datq.add(d) + datq.save() + reckey = [] + next = nodedb.neighbors_nodes(kid,False,5) + sent = [] + key = self.glob.keydb.get(kid) + if key: reckey.append(key) + while True: + if self.finish: return + neighbors = next + if len(neighbors)==0: break + next = [] + for node in neighbors: + if self.finish: return + logger.log("SEARCH","findvalue to %s for %s" % (hexlify(node.id),d)) + sent.append(node.id) + try: + res = node.findvalue(kid) + except o2on_node.NodeRemovable: + nodedb.remove(node) + nodedb.save() + self.glob.keydb.remove_bynodeid(node.id) + self.glob.keydb.save() + except o2on_node.NodeRefused: + pass + except socket.error, inst: + logger.log("SEARCH", inst) + else: + nodedb.add_node(node) + if not res: res = [] + for x in res: + if isinstance(x, o2on_node.Node): + nodedb.add_node(x) + if x.id != prof.mynode.id and x not in next and x.id not in sent: + logger.log("SEARCH","\tadd new neighbors %s" % hexlify(x.id)) + next.append(x) + elif isinstance(x, o2on_key.Key): + logger.log("SEARCH","\tadd new key") + if x not in reckey: reckey.append(x) + nodedb.save() + if len(reckey) == 0: logger.log("SEARCH","\tfailed to get key for %s" % d) + for key in reckey: + self.glob.keydb.add(key) + self.glob.keyquery.add(key) + self.glob.keydb.save() + +class DatQueryThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"dat query",60,g) + def stop(self): + JobThread.stop(self) + self.glob.keyquery.semap.release() + def dojob(self, nodedb, logger, prof, datdb, datq): + k = self.glob.keyquery.pop() + if self.finish: return + node = nodedb[k.nodeid] + if not node: node = o2on_node.Node(k.nodeid, k.ip, k.port) + logger.log("DATQUERY","dat query %s to %s" % (hexlify(k.hash),hexlify(node.id))) + try: + dat = node.dat(k.hash, self.glob) + except o2on_node.NodeRemovable: + nodedb.remove(node) + nodedb.save() + self.glob.keydb.remove_bynodeid(node.id) + self.glob.keydb.save() + except o2on_node.NodeRefused: + pass + except socket.error, inst: + logger.log("DATQUERY", inst) + else: + logger.popup("DATQUERY", "Got queried dat %s" % dat.path()) + nodedb.add_node(node) + datdb.add_dat(dat) + nodedb.save() + datdb.save() + self.glob.keyquery.save() + +# Server thread + +class ProxyServerThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"proxy server",0,g) + self.serv = o2on_server.O2ONServer(o2on_server.ProxyServerHandler, + o2on_config.ProxyPort, g) + def stop(self): + JobThread.stop(self) + self.serv.shutdown() + def dojob(self, nodes, logger, prof, datdb, datq): + self.serv.serve_forever() + +class P2PServerThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"p2p server",0,g) + if g.prof.mynode.port == 0: + self.serv = None + else: + self.serv = o2on_server.O2ONServer(o2on_server.P2PServerHandler, + g.prof.mynode.port, g) + def stop(self): + JobThread.stop(self) + if self.serv: self.serv.shutdown() + def dojob(self, nodes, logger, prof, datdb, datq): + if not self.serv: + self.finish = True + return + self.serv.serve_forever() + +class AdminServerThread(JobThread): + def __init__(self, g): + JobThread.__init__(self,"admin server",0,g) + self.serv = o2on_server.O2ONServer(o2on_server.AdminServerHandler, + o2on_config.AdminPort, g) + def stop(self): + JobThread.stop(self) + self.serv.shutdown() + def dojob(self, nodes, logger, prof, datdb, datq): + self.serv.serve_forever() Added: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py (rev 0) +++ trunk/opy2on/lib/o2on_key.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,236 @@ +#!/usr/bin/python +# -*- coding: utf-8 + +import os.path +import re +import cPickle +import os.path +from binascii import unhexlify, hexlify +import xml.dom.minidom +import threading +import subprocess +import os +import datetime +import hashlib +import time + +import sys + +from o2on_const import KeyDBFile, regHosts +import o2on_config +from o2on_util import hash_xor_bitlength +import o2on_dat +import o2on_node + +class Key: + def __init__(self): + self.hash = None + self.nodeid = None + self.ip = None + self.port = None + self.size = None + self.url = "" + self.title = "" + self.note = "" + self.published = 0 + self.ikhash = None + def __cmp__(self,x): + return cmp(self.idkeyhash(), x.idkeyhash()) + def idkeyhash(self): + if not self.ikhash: self.ikhash = hashlib.sha1(self.hash + self.nodeid).digest() + return self.ikhash + def from_key(self, key): + if self.idkeyhash() == key.idkeyhash(): + self.ip = key.ip + self.port = key.port + if self.size < key.size: self.size = key.size + if key.url!="": self.url = key.url + if key.title!="":self.title = key.title + if key.note!="":self.note = key.note + def from_dat(self, dat): + self.hash = dat.hash() + data = dat.data() + self.size = len(data) + first = data.split("\n",1)[0] + try: + first = first.decode('cp932').encode('utf-8') + except UnicodeDecodeError, inst: + try: + first = first.decode('euc_jp').encode('utf-8') + except UnicodeDecodeError, inst: raise inst + m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) + if m: self.title = m.group(1) + #reg = re.compile(r'^(?:2ch\.net|machibbs\.com)/(?:cyugoku|hokkaidou|k(?:an(?:a|to)|inki|(?:ousinet|yusy)u)|o(?:(?:kinaw|sak)a)|sikoku|t(?:a(?:(?:m|war)a)|o(?:kyo|u(?:hoku|kai))))') + #if re.compile("^\s*$").match(self.title) and \ + # not reg.match(dat.path()): print dat.datpath() + self.url = "http://xxx.%s/test/read.cgi/%s/%s/" % (dat.domain, + dat.board, + dat.datnum) + def from_node(self, node): + self.nodeid = node.id + self.ip = node.ip + self.port = node.port + def xml(self): + if not self.valid(): return "" + board_xml = "\r\n" + board_xml += "%s\r\n" % hexlify(self.hash) + board_xml += "%s\r\n" % hexlify(self.nodeid) + board_xml += "%s\r\n" % o2on_node.ip2e(self.ip) + board_xml += "%d\r\n" % (self.port or 0) + board_xml += "%d\r\n" % self.size + board_xml += "%s\r\n" % self.url + title = self.title.decode('utf-8') + if len(title)>32: title = title[:31]+"…".decode('utf-8') + board_xml += "<![CDATA[%s]]>\r\n" % title + note = self.note.decode('utf-8') + if len(note)>32: note = note[:31]+"…".decode('utf-8') + board_xml += "\r\n" % note + board_xml += "\r\n" + return board_xml + def valid(self): + return self.hash and self.nodeid and self.ip and self.port + def from_xml_node(self, xml_node): + res = {} + for c in ('hash', 'nodeid', 'ip', 'port', 'size', 'url', 'title', 'note'): + try: res[c] = xml_node.getElementsByTagName(c)[0].childNodes[0].data + except IndexError: continue + if res.get('hash'): self.hash = unhexlify(res.get('hash')) + if res.get('nodeid'): self.nodeid = unhexlify(res.get('nodeid')) + if res.get('ip'): self.ip = o2on_node.e2ip(res.get('ip')) + self.port = res.get('port', self.port) + if self.port: self.port = int(self.port) + self.size = res.get('size', self.size) + if self.size: self.size = int(self.size) + self.url = res.get('url', self.url) + if res.get('title'): self.title = res.get('title').encode('utf-8') + if res.get('note'): self.note = res.get('note').encode('utf-8') + +class KeyDB: + def __init__(self, g): + self.lock = threading.Lock() + with self.lock: + self.keys = {} + self.lenmap = {} + self.publishmap = {} + self.glob = g + self.load() + def __len__(self): + with self.lock: + return len(self.keys) + def show(self): + pager = os.environ.get('PAGER') + if not pager: self.glob.logger.log("KEYDB", "PAGER env must be set") + proc = subprocess.Popen(pager, shell=True, stdin=subprocess.PIPE) + pipe = proc.stdin + try: + with self.lock: + for l in sorted(self.lenmap.keys()): + for h in self.lenmap[l]: + key = self.keys[h] + s = "%3d %s:%d\t%s\t%s\t%s\t%s\t%dB\t%s\n" % \ + (l, o2on_node.ip2e(key.ip), + key.port, key.url, + key.title.decode('utf-8'), + key.note.decode('utf-8'), + str(datetime.datetime.fromtimestamp(int(key.published))), + key.size, + hexlify(key.hash)) + pipe.write(s.encode('utf-8')) + pipe.close() + proc.wait() + except IOError, inst: + if inst.errno == 32:pass # Broken pipe + else: raise inst + self.glob.logger.log("KEYDB", "Finished to show keys") + def key_list(self): + res = [] + with self.lock: + for l in sorted(self.lenmap.keys()): + for h in self.lenmap[l]: + key = self.keys[h] + x = (l, o2on_node.ip2e(key.ip), + key.port, key.url, + key.title.decode('utf-8'), + key.note.decode('utf-8'), + str(datetime.datetime.fromtimestamp(int(key.published))), + key.size, + hexlify(key.hash)) + res.append(x) + return res + def keys_to_publish(self, last_published_before): + res = [] + for x in sorted(self.publishmap.keys()): + for y in self.publishmap[x]: + res.append(self.keys[y]) + return res + def published(self, idkeyhash, publish_time): + if len(idkeyhash) != 20: raise Exception + with self.lock: + if idkeyhash not in self.keys: return + key = self.keys[idkeyhash] + self.publishmap[key.published].remove(idkeyhash) + key.published = publish_time + if publish_time not in self.publishmap: self.publishmap[publish_time]=[] + self.publishmap[publish_time].append(idkeyhash) + def get(self, target): + with self.lock: + return self.keys.get(target) + def remove_bynodeid(self, nid): + if len(nid) != 20:raise Exception + removes = [] + with self.lock: + for k in self.keys.values(): + if k.nodeid == nid: removes.append(k) + for k in removes: + self.remove(k) + def remove(self,k): + with self.lock: + del self.keys[k.idkeyhash()] + l = hash_xor_bitlength(self.glob.prof.mynode.id, k.hash) + self.lenmap[l].remove(k.idkeyhash()) + self.publishmap[k.published].remove(k.idkeyhash()) + if len(self.lenmap[l]) == 0: del self.lenmap[l] + if len(self.publishmap[k.published])==0: del self.publishmap[k.published] + def add(self, k): + if not k.valid(): return + k.published = int(time.time()) + if k.idkeyhash() in self.keys: + self.keys[k.idkeyhash()].from_key(k) + return + bl = hash_xor_bitlength(self.glob.prof.mynode.id, k.hash) + with self.lock: + if len(self.keys) < 3000: + pass + else: + maxlen = max(self.lenmap.keys()) + if bl <= maxlen: + maxkey = self.keys[self.lenmap[maxlen][0]] + del self.keys[self.lenmap[maxlen][0]] + del self.lenmap[maxlen][0] + if len(self.lenmap[maxlen]) == 0: + del self.lenmap[maxlen] + self.publishmap[maxkey.published].remove(maxkey.idkeyhash()) + if len(self.publishmap[maxkey.published]) == 0: + del self.publishmap[maxkey.published] + else: return + self.keys[k.idkeyhash()] = k + if bl not in self.lenmap: self.lenmap[bl] = [] + self.lenmap[bl].append(k.idkeyhash()) + if k.published not in self.publishmap: + self.publishmap[k.published] = [] + self.publishmap[k.published].append(k.idkeyhash()) + def save(self): + f = open(KeyDBFile,'wb') + with self.lock: + cPickle.dump(self.keys, f, -1) + cPickle.dump(self.lenmap, f, -1) + cPickle.dump(self.publishmap,f,-1) + f.close() + def load(self): + if os.path.isfile(KeyDBFile): + f = open(KeyDBFile,'rb') + with self.lock: + self.keys = cPickle.load(f) + self.lenmap = cPickle.load(f) + self.publishmap = cPickle.load(f) + f.close() Added: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py (rev 0) +++ trunk/opy2on/lib/o2on_node.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,564 @@ +#!/usr/bin/python +# -*- coding: utf-8 + +import os +from Crypto.Cipher import AES # pycrypto +import xml.dom.minidom +import re +import urllib2 +import cPickle +from struct import unpack, pack +from binascii import unhexlify, hexlify +import httplib +import socket +import threading +import random +import time + +import o2on_config +from o2on_util import hash_xor_bitlength +import o2on_const +import o2on_util +import o2on_dat +import o2on_key + +class NodeRemovable: + pass +class NodeRefused: + pass + +AESKEY = "0000000000000000" + +def aescounter(): + return "\x00" * 16 + +def e2port(s): + aesobj = AES.new(AESKEY, AES.MODE_CTR, counter=aescounter) + return unpack("> (pos%8)) & 0x01 != 0)) + +common_header = {} +def build_common_header(prof): + global common_header + if not prof.mynode.id: raise Exception("My ID is NULL") + if prof.mynode.port==None: raise Exception("My Port is NULL") + if not prof.mynode.name: prof.mynode.name="" + if not prof.mynode.pubkey: raise Exception("My pubkey is NULL") + common_header = {'Connection': "close", + 'X-O2-Node-ID': hexlify(prof.mynode.id), + 'X-O2-Port': str(prof.mynode.port), + 'X-O2-Node-Name': prof.mynode.name, + 'X-O2-Node-Flags':'--D', + 'User-Agent': prof.mynode.ua, + 'X-O2-RSA-Public-Key': hexlify(prof.mynode.pubkey)} + +class Node: + def __init__(self, node_id=None, i=None, p=None): + self.id = node_id + self.ip = i + self.port = p + self.name = None + self.pubkey = None + self.ua = "" + self.lock = threading.Lock() + self.lastping = None + self.removable = False + self.flag = "???" + self.flag_running = False + self.flag_history = False + self.flag_dat = False + def __getstate__(self): + return (self.id, self.ip,self.port, self.name, self.pubkey,self.ua,self.flag) + def __setstate__(self,x): + self.id, self.ip, self.port, self.name, self.pubkey,self.ua,self.flag = x[:7] + self.setflag(self.flag) + self.lock = threading.Lock() + self.lastping = None + self.removable = False + def __cmp__(self,x): + return cmp(self.id, x.id) + def from_node(self, n): + if self.id == n.id: + self.ip = n.ip + self.port = n.port + self.pubkey = n.pubkey + # TODO pubkey の変更には気をつけたほうがいい + if n.ua !="": self.ua = n.ua + if n.name !="": self.name = n.name + if n.flag !="": self.setflag(n.flag) + def from_xml(self, xmlnode): + try: + self.ip = e2ip(xmlnode.getElementsByTagName("ip")[0].childNodes[0].data) + self.port = int(xmlnode.getElementsByTagName("port")[0].childNodes[0].data) + self.id = unhexlify(xmlnode.getElementsByTagName("id")[0].childNodes[0].data) + except KeyError: return + names = xmlnode.getElementsByTagName("name") + if len(names)>0 and len(names[0].childNodes)>0: + self.name = names[0].childNodes[0].data.encode('utf-8') + pubkeys = xmlnode.getElementsByTagName("pubkey") + if len(pubkeys)>0 and len(pubkeys[0].childNodes)>0: + self.pubkey = unhexlify(pubkeys[0].childNodes[0].data) + def xml(self): + data = "\r\n" + data += "%s\r\n" % hexlify(self.id) + data += "%s\r\n" % ip2e(self.ip) + data += "%s\r\n" % self.port + if self.name: + data += "\r\n" % self.name.decode('utf-8') + if self.pubkey: + data += "%s\r\n" % hexlify(self.pubkey) + data += "\r\n" + return data + def setflag(self,flag): + self.flag = flag + self.flag_running = "r" in flag + self.flag_history = "t" in flag + self.flag_dat = "D" in flag + def request(self, method, path, body, addheaders): + if self.removable: raise NodeRemovable + headers = common_header.copy() + if method == 'POST': + headers['X-O2-Node-Name'] = \ + headers['X-O2-Node-Name'].decode('utf-8').encode('ascii','replace') + for x in addheaders: headers[x] = addheaders[x] + with self.lock: + conn = httplib.HTTPConnection(self.ip, self.port) + try: + socket.setdefaulttimeout(o2on_config.SocketTimeout) + conn.connect() + socket.setdefaulttimeout(None) + conn.request(method,path,body, headers) + r = conn.getresponse() + conn.close() + except socket.timeout: + socket.setdefaulttimeout(None) + self.removable = True + raise NodeRemovable + except socket.error, inst: + socket.setdefaulttimeout(None) + if inst.errno in (113, 111): raise NodeRemovable + if inst.errno in (110, 104): raise NodeRefused + else: raise inst + except httplib.BadStatusLine: + socket.setdefaulttimeout(None) + raise NodeRefused + else: + name = r.getheader('X-O2-Node-Name') + pubkey = r.getheader('X-O2-RSA-Public-Key') + ua = r.getheader('Server') + flag = r.getheader('X-O2-Node-Flags') + if not name or not pubkey or not ua: raise NodeRemovable + self.name = name + self.pubkey = unhexlify(pubkey) + self.ua = ua + if flag: self.setflag(flag) + return r + def im(self, mynode, msg): + headers = {} + im_xml = "\r\n" + im_xml += "\r\n" + im_xml += "\r\n" + mynode_xml = mynode.xml() + mynode_xml = mynode_xml[len("\r\n"):] + mynode_xml = mynode_xml[:-len("\r\n")] + im_xml += mynode_xml + im_xml += "\r\n" % msg + im_xml += "\r\n" + im_xml += "\r\n" + im_xml = im_xml.encode('utf-8') + headers['Content-Type'] = 'text/xml; charset=utf-8' + headers['Content-Length'] = str(len(im_xml)) + r = self.request("POST", "/im", im_xml, headers) + if r.status != 200: raise Exception("im status %d" % r.status) + return True + def store(self,category, keys): + headers={"X-O2-Key-Category": category} + board_xml = "\r\n" + board_xml += "\r\n" + for key in keys: board_xml += key.xml() + board_xml += "\r\n" + board_xml = board_xml.encode('utf-8') + headers['Content-Type'] = 'text/xml; charset=utf-8' + headers['Content-Length'] = str(len(board_xml)) + r = self.request("POST", "/store", board_xml, headers) + if r.status != 200: raise Exception("store status %d" % r.status) + return + def findnode(self, targetid): + headers = {"X-O2-Target-Key" : hexlify(targetid)} + r = self.request("GET","/findnode",None,headers) + data = None + result = [] + try: + if r.status == 200: + l = r.getheader('Content-Length') + if l: l = int(l) + else: return [] + data = r.read(l) + elif r.status == 404: pass + else: raise Exception("status error %d" % r.status) + except socket.error: + pass + else: + if data: + dom = xml.dom.minidom.parseString(data) + nn = dom.getElementsByTagName("nodes") + if len(nn): + for n in nn[0].getElementsByTagName("node"): + node = Node() + node.from_xml(n) + if node.ip and node.port and node.id: + result.append(node) + dom.unlink() + return result + def dat(self, dathash, board, datdb=None): + headers = {} + omiyage = None + if datdb and board: + omiyage = datdb.getRandomInBoard(board) + if(dathash): + headers['X-O2-Target-Key'] = dathash + elif(board): + headers['X-O2-Target-Board'] = board + if omiyage: + data = omiyage.data() + headers['X-O2-DAT-Path'] = omiyage.path() + headers['Content-Type'] = 'text/plain; charset=shift_jis' + headers['Content-Length'] = str(len(data)) + if omiyage: + r = self.request("POST","/dat", data,headers) + else: + r = self.request("GET","/dat",None,headers) + stat = r.status + data = r.read() + if stat == 200: + path = None + path = r.getheader("X-O2-Original-DAT-URL") + if not path: path = r.getheader("X-O2-DAT-Path") + if not path: return None + if ".." in path: return None + dat = o2on_dat.Dat(path) + if dat.save(data): return dat + elif stat == 404: pass + elif stat == 400: pass + else: raise Exception("dat status %d" % stat) + return None + def ping(self, force=False): + if not force and self.lastping and \ + (time.time() - self.lastping < o2on_config.RePingSec): + return True + r = self.request("GET","/ping",None,{}) + if r.status == 200: + self.lastping = int(time.time()) + l = r.getheader('Content-Length') + if l: l = int(l) + else: return False + return r.read(l) + else: raise Exception("ping status %d" % r.status) + return False + def collection(self, glob): + board_xml = o2on_util.xml_collecting_boards(glob) + headers = {'Content-Type':'text/xml; charset=utf-8', + 'Content-Length':len(board_xml)} + r = self.request("POST","/collection",board_xml,headers) + result = [] + if r.status == 200: + data = r.read() + # 本家o2on の bug 対策 + if data.rfind("") == -1: + index = data.rfind("") + data = data[:index] + "" + data[index+len(""):] + if len(data): + dom = xml.dom.minidom.parseString(data) + nn = dom.getElementsByTagName("boards") + if len(nn): + for b in nn[0].getElementsByTagName("board"): + result.append(b.childNodes[0].data) + dom.unlink() + else: raise Exception("collection status %d" % r.status) + return result + def findvalue(self, kid): + headers = {"X-O2-Target-Key":hexlify(kid)} + r = self.request("GET","/findvalue",None,headers) + if r.status == 200: + res = [] + dom = xml.dom.minidom.parseString(r.read()) + nodes = dom.getElementsByTagName("nodes") + keys = dom.getElementsByTagName("keys") + if len(nodes): + for n in nodes[0].getElementsByTagName("node"): + node = Node() + node.from_xml(n) + if node.ip and node.port and node.id: + res.append(node) + elif len(keys): + for k in keys[0].getElementsByTagName("key"): + key = o2on_key.Key() + key.from_xml_node(k) + res.append(key) + return res + elif r.status == 404: pass + else: raise Exception("findvalue status %d" % r.status) +class NodeDB: + def __init__(self, glob): + self.glob = glob + self.KBuckets = [] + self.port0nodes = [] + self.lock = threading.Lock() + for x in range(0,160): self.KBuckets.append([]) + self.nodes = dict() + self.boardmap = dict() + self.load() + if len(self.nodes) == 0: self.node_from_web() + def __len__(self): + with self.lock: + return len(self.nodes) + def __getitem__(self,x): + if x == self.glob.prof.mynode.id: + return self.glob.prof.mynode + with self.lock: + return self.nodes.get(x) + def show(self): + self.glob.logger.begin() + with self.lock: + if len(self.nodes) == 0: + self.glob.logger.log("NODEDB", "No nodes") + self.glob.logger.end() + return + for l in range(0,160): + for x in self.KBuckets[l]: + node = self.nodes[x] + if node.name: name = node.name.decode('utf-8') + else: name = "名無しさん".decode('utf-8') + s = "%3d %-8s\t%s %s:%d\t%24s\t%s" % (l+1, name, + node.flag, + ip2e(node.ip), node.port, + node.ua, hexlify(node.id)) + s = s.encode('utf-8') + self.glob.logger.log("NODEDB", s) + self.glob.logger.end() + def node_list(self): + with self.lock: + res = [] + if len(self.nodes) == 0: return res + for l in range(0,160): + for x in self.KBuckets[l]: + node = self.nodes[x] + if node.name: name = node.name.decode('utf-8') + else: name = "名無しさん".decode('utf-8') + x = (l+1, hexlify(node.id), ip2e(node.ip), node.port,name, + node.flag, ip2e(node.ip), node.port, + node.ua, hexlify(node.id)) + res.append(x) + return res + def exportnode(self): + with self.lock: + f = open('exportnodes.xml','w') + f.write("\n") + for x in self.nodes: + n = self.nodes[x] + f.write("%s%s%s\n" % (hexlify(n.id), ip2e(n.ip), port2e(n.port))) + f.write("\n") + f.close() + def get_nodes_for_board(self, board): + with self.lock: + if not board in self.boardmap: return [] + return map(lambda x: self.nodes[x], self.boardmap[board]) + def get_random_board(self): + with self.lock: + if len(self.boardmap) == 0: return None + return random.choice(self.boardmap.keys()) + def reset_collection_for_node(self,n): + with self.lock: + zeroboard = [] + for x in self.boardmap: + if n.id in self.boardmap[x]: + self.boardmap[x].remove(n.id) + if len(self.boardmap[x]) == 0: + zeroboard.append(x) + for z in zeroboard: del self.boardmap[z] + def add_collection(self, board, n): + if board not in self.glob.allboards: return + r = None + with self.lock: + if not board in self.boardmap: + self.boardmap[board] = [n.id] + elif len(self.boardmap[board])<10: + self.boardmap[board].append(n.id) + else: + nt = self.nodes.get(self.boardmap[board][0]) + if not nt: + raise Exception + del self.boardmap[board][0] + self.boardmap[board].append(n.id) + else: + r = nt.ping() + if r: + self.boardmap[board].append(self.boardmap[board][0]) + del self.boardmap[board][0] + else: + del self.boardmap[board][0] + self.boardmap[board].append(n.id) + if n.id not in self.nodes: self.add_node(n) + if r: self.add_node(nt) + def choice(self): + with self.lock: + return self.nodes[random.choice(self.nodes.keys())] + def remove(self, x): + if x.port == 0: + with self.lock: + if x in self.port0nodes: + self.port0nodes.remove(x) + return + with self.lock: + if x.id in self.nodes: + del self.nodes[x.id] + self.KBuckets[hash_xor_bitlength( + self.glob.prof.mynode.id, x.id)-1].remove(x.id) + zeroboard = [] + for b in self.boardmap.keys(): + if x.id in self.boardmap[b]: + self.boardmap[b].remove(x.id) + if len(self.boardmap[b]) == 0: + zeroboard.append(b) + for b in zeroboard: self.boardmap[b] + def save(self): + with self.lock: + pkl_file = open(o2on_const.NodeDBFile, 'wb') + cPickle.dump(self.nodes, pkl_file,-1) + cPickle.dump(self.boardmap, pkl_file,-1) + cPickle.dump(self.port0nodes, pkl_file,-1) + pkl_file.close() + def load(self): + if(os.path.isfile(o2on_const.NodeDBFile)): + pkl_file = open(o2on_const.NodeDBFile, 'rb') + try: + tmp = cPickle.load(pkl_file) + except EOFError: + tmp = dict() + with self.lock: + try: self.boardmap = cPickle.load(pkl_file) + except EOFError: tmp2 = dict() + with self.lock: + try: self.port0nodes = cPickle.load(pkl_file) + except: self.port0nodes = [] + pkl_file.close() + for x in tmp: self.add_node(tmp[x]) + def add_node(self,node): + if node.port == 0: + self.glob.logger.log("NODEDB", "Added port0 node %s" % hexlify(node.id)) + with self.lock: + if node in self.port0nodes: + self.port0nodes.remove(node) + self.port0nodes.append(node) + return + bitlen = hash_xor_bitlength(self.glob.prof.mynode.id, node.id)-1 + if(bitlen<0): return + with self.lock: + if node.id in self.KBuckets[bitlen]: + n = self.nodes[node.id] + n.from_node(node) + self.KBuckets[bitlen].remove(node.id) + self.KBuckets[bitlen].append(node.id) + elif len(self.KBuckets[bitlen]) < max(20, bitlen/2): + self.KBuckets[bitlen].append(node.id) + self.nodes[node.id] = node + else: + n = self.nodes[self.KBuckets[bitlen][0]] + r = n.ping() + if r: + del self.KBuckets[bitlen][0] + self.KBuckets[bitlen].append(n.id) + else: + del self.nodes[n.id] + del self.KBuckets[bitlen][0] + zeroboard = [] + for b in self.boardmap.keys(): + if n.id in self.boardmap[b]: + self.boardmap[b].remove(n.id) + if len(self.boardmap[b]) == 0: + zeroboard.append(b) + for b in zeroboard: del self.boardmap[b] + self.KBuckets[bitlen].append(node.id) + self.nodes[node.id] = node + def neighbors_nodes(self,target, includeself, cnt=3): + myid = self.glob.prof.mynode.id + d = hash_xor(target, myid) + bitlen = hash_xor_bitlength(target, myid) - 1 + result = [] + if(bitlen>=0): + with self.lock: + if(len(self.KBuckets[bitlen])): + for x in sorted(map(lambda x: hash_xor(x,target), + self.KBuckets[bitlen])): + result.append(self.nodes[hash_xor(x,target)]) + if(len(result)>=cnt): return result + for l in range(bitlen-1,-1,-1): + if(hash_bittest(d,l) and len(self.KBuckets[l])): + for x in sorted(map(lambda x: hash_xor(x,target), + self.KBuckets[l])): + result.append(self.nodes[hash_xor(x,target)]) + if(len(result)>=cnt): return result + if(includeself and self.glob.prof.mynode.ip and self.glob.prof.mynode.port != 0): + result.append(self.glob.prof.mynode) + if(len(result)>=cnt): return result + if(bitlen>=0): + for l in range(0,bitlen): + with self.lock: + if(not hash_bittest(d,l) and len(self.KBuckets[l])): + for x in sorted(map(lambda x: hash_xor(x,target), + self.KBuckets[l])): + result.append(self.nodes[hash_xor(x,target)]) + if(len(result)>=cnt): return result + for l in range(bitlen+1,160): + with self.lock: + if(len(self.KBuckets[l])): + for x in sorted(map(lambda x: hash_xor(x,target), + self.KBuckets[l])): + result.append(self.nodes[hash_xor(x,target)]) + if(len(result)>=cnt): return result + return result + def node_from_web(self): + nodeurls = o2on_config.Node_List_URLs + regNode = re.compile("([0-9a-f]{40})([0-9a-f]{8})([0-9a-f]{4})") + for url in nodeurls: + xmldata = urllib2.urlopen(url) + dom = xml.dom.minidom.parseString(xmldata.read()) + n = dom.getElementsByTagName("nodes") + if len(n): + for y in n[0].getElementsByTagName("str"): + if len(y.childNodes) == 1: + m = regNode.search(y.childNodes[0].data) + self.add_node(Node(unhexlify(m.group(1)), + e2ip(m.group(2)), + e2port(m.group(3)))) + dom.unlink() Added: trunk/opy2on/lib/o2on_profile.py =================================================================== --- trunk/opy2on/lib/o2on_profile.py (rev 0) +++ trunk/opy2on/lib/o2on_profile.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,67 @@ +#!/usr/bin/python + +from Crypto.PublicKey import RSA # pycrypto +import os +import cPickle +from binascii import unhexlify + +import o2on_util +import o2on_config +import o2on_node +from o2on_const import * + +class MyRSA: + def __init__(self, rsa): + self.rsa = rsa + def hex(self): + s = hex(self.rsa.n)[2:-1] + return "0" * (320-len(s)) + s + def pubkey(self): + return unhexlify(self.hex()) + +def uname(logger): + if os.name == 'posix': + try: + return os.uname()[0]+" "+os.uname()[4] + except AttributeError: + logger.popup("Unknown POSIX %s.\nPlease report it." % os.name) + return "Unknown POSIX" + elif os.name == 'nt': + return "Win" + try: + uname = os.uname()[0]+" "+os.uname()[4] + except AttributeError: + uname = "" + logger.popup("GLOBAL","Unknown OS %s (%s).\nPlease report it." % (os.name, uname)) + return "Unknown" + +class Profile: + def __init__(self, l): + self.logger = l + self.mynode = o2on_node.Node() + self.mynode.id = o2on_util.randomid() + self.mynode.ip = None + self.mynode.port = o2on_config.P2PPort + self.mynode.name = o2on_config.NodeName[:8].encode('utf-8') + self.mynode.pubkey = None + self.rsa = None + self.mynode.ua = "%s/%.1f (%s/%1d.%02d.%04d; %s)" % \ + (ProtocolName, ProtocolVer, AppName, AppMajorVer, AppMinorVer, AppBuildNo, + uname(l)) + self.load() + if not self.rsa: + self.logger.log("PROFILE","generating RSA key") + self.rsa = MyRSA(RSA.generate(160*8,os.urandom)) + self.save() + self.mynode.pubkey = self.rsa.pubkey() + def save(self): + pkl_file = open(ProfileFile,"wb") + cPickle.dump(self.rsa, pkl_file,-1) + cPickle.dump(self.mynode.id, pkl_file,-1) + pkl_file.close() + def load(self): + if(os.path.isfile(ProfileFile)): + pkl_file = open(ProfileFile,"rb") + self.rsa = cPickle.load(pkl_file) + self.mynode.id = cPickle.load(pkl_file) + pkl_file.close() Added: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py (rev 0) +++ trunk/opy2on/lib/o2on_server.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,859 @@ +#!/usr/bin/python +# -*- coding: utf-8 + +import BaseHTTPServer +import re +import socket +from urlparse import urlparse +import httplib +import os.path +import gzip +import base64 +import zlib +import StringIO +from binascii import hexlify, unhexlify +import xml.dom.minidom +import urllib +import cgi +import time +import traceback +import sys + +import o2on_config +from o2on_const import regHosts, ProtocolVer, AppName +import o2on_node +import o2on_dat +from o2on_node import ip2e, port2e, e2ip +import o2on_key +import o2on_im + +class O2ONServer(BaseHTTPServer.HTTPServer): + def __init__(self, handler, port, g): + BaseHTTPServer.HTTPServer.__init__(self, + ('', port), + handler) + self.glob = g + self.requests = [] + def shutdown(self): + for r in self.requests: + r.shutdown(socket.SHUT_RDWR) + r.close() + BaseHTTPServer.HTTPServer.shutdown(self) + def finish_request(self, request, client_address): + self.requests.append(request) + try: + BaseHTTPServer.HTTPServer.finish_request(self, request, client_address) + except Exception,inst: + if isinstance(inst, socket.error) and inst.errno in (104, 32): + pass + else: + if o2on_config.OutputErrorFile: + f = open('error-'+str(int(time.time()))+'.txt', 'w') + f.write(str(inst)+"\n") + traceback.print_exc(file=f) + f.close() + self.glob.logger.popup("ERROR", str(inst)) + self.glob.shutdown.set() + self.requests.remove(request) + +class ProxyServerHandler(BaseHTTPServer.BaseHTTPRequestHandler): + URLTYPE_NORMAL = 0 + URLTYPE_DAT = 1 + URLTYPE_KAKO_DAT = 2 + URLTYPE_KAKO_GZ = 3 + URLTYPE_OFFLAW = 4 + URLTYPE_MACHI = 5 + URLTYPE_UNKNOWN = 6 + regs = (re.compile( + r'^http://[^.]+\.'+regHosts+r'(?::\d+)?/test/read.cgi/[^/]+/\d+/$'), + re.compile( + r'^http://[^.]+\.'+regHosts+r'(?::\d+)?/([^/]+)/dat/(\d+)\.dat$'), + re.compile( + r'^http://[^.]+\.'+regHosts+r'(?::\d+)?/([^/]+)/kako/\d+/\d+/(\d+)\.dat$'), + re.compile( + r'^http://[^.]+\.'+regHosts+r'(?::\d+)?/([^/]+)/kako/\d+/\d+/(\d+)\.dat\.gz$'), + re.compile( + r'^http://[^.]+\.'+regHosts+r'(?::\d+)?/test/offlaw.cgi/[^/]+/\d+/\?raw='), + re.compile( + r'^http://[^.]+\.'+regHosts+r'(?::\d+)?/[^/]+/read.pl\?BBS=[^&]+&KEY=\d+'),) + def __init__(self,request, client_address, server): + BaseHTTPServer.BaseHTTPRequestHandler.__init__(self,request, client_address, server) + def urltype(self): + x = 0 + for r in self.regs: + if r.match(self.path): return x + x += 1 + return x + def get_requested_header(self): + h = self.headers.headers + hr = {} + for x in h: + f = x.split(": ",1) + hr[f[0]] = f[1][:-2] + return hr + def get_connection(self, remove=[]): + h = self.headers.headers + hr = {} + for x in h: + f = x.split(": ",1) + if f[0] != "Proxy-Connection": + hr[f[0]] = f[1][:-2] + hr["Connection"] = "close" + p = urlparse(self.path) + if "@" in p.netloc: + x = p.netloc.split("@",1) + loc = x[1] + hr["Authorization"] = "Basic "+base64.b64encode(x[0]) + else: + loc = p.netloc + for r in remove: + if r in hr: del hr[r] + conn = httplib.HTTPConnection(loc) + conn.request("GET",p.path, None, hr) + return conn + def msg(self,r): + res = '' + for x in r.getheaders(): + if x[0] in ("transfer-encoding",): + pass + else: res+=x[0]+': '+x[1]+'\r\n' + return res + def normal_proxy(self): + try: + conn = self.get_connection() + r= conn.getresponse() + conn.close() + except socket.timeout: + return + self.wfile.write("HTTP/%d.%d %d %s\r\n" % + (r.version/10,r.version%10,r.status,r.reason)) + self.wfile.write(self.msg(r)) + self.wfile.write("\r\n") + self.wfile.write(r.read()) + self.wfile.close() + def datpath(self): + m = self.regs[self.URLTYPE_DAT].match(self.path) + if not m: m = self.regs[self.URLTYPE_KAKO_DAT].match(self.path) + if not m: m = self.regs[self.URLTYPE_KAKO_GZ].match(self.path) + if not m: return None + return os.path.join(o2on_config.DatDir, m.group(1), m.group(2), + m.group(3)[:4],m.group(3)+".dat") + def datkey(self): + m = self.regs[self.URLTYPE_DAT].match(self.path) + if not m: m = self.regs[self.URLTYPE_KAKO_DAT].match(self.path) + if not m: m = self.regs[self.URLTYPE_KAKO_GZ].match(self.path) + if not m: return None + return "/".join((m.group(1), m.group(2), m.group(3))) + def do_GET(self): + logger = self.server.glob.logger + logger.log("PROXY", "proxy requested %s" % self.path) + ut = self.urltype() + if ut in (self.URLTYPE_UNKNOWN, self.URLTYPE_NORMAL, self.URLTYPE_MACHI): + self.normal_proxy() + return + try: + conn = self.get_connection() + r= conn.getresponse() + conn.close() + except socket.timeout: + r = None + data = None + header = None + if r: + logger.log("PROXY", "\tresponse %s" % r.status) + if ut != self.URLTYPE_OFFLAW and r.status in (200,206,304): + logger.log("PROXY", "\tgot response from server") + data = r.read() + if r.getheader("content-encoding") == "gzip": + sf = StringIO.StringIO(data) + dec = gzip.GzipFile(fileobj=sf) + datdata = dec.read() + else: + datdata = data + self.wfile.write("HTTP/%d.%d %d %s\r\n" % + (r.version/10,r.version%10,r.status,r.reason)) + self.wfile.write(self.msg(r)) + self.wfile.write("\r\n") + self.wfile.write(data) + self.wfile.close() + dk = self.datkey() + dp = self.datpath() + if r.status == 200: + if not self.server.glob.datdb.has_key(dk): + # 持ってない dat が取得された + logger.log("PROXY", "\tsave responsed dat for myself") + self.server.glob.datdb.add(dk, datdata) + else: + if self.server.glob.datdb.has_key(dk): + if r.status == 206: + # 持ってる dat の差分 + rg = r.getheader('Content-Range') + start = 0 + if rg: + m=re.compile(r'bytes (\d+)-').search(rg) + start = int(m.group(1)) + logger.log("PROXY", "\tsave diff dat for myself (%d-)" % start) + self.server.glob.datdb.add(dk, datdata, start) + elif o2on_config.RequestNonExistDat: + # 持ってない dat がリクエストされた -> こっそり取得 + logger.log("PROXY", "\trequest whole dat for myself :-)") + try: + conn = self.get_connection(['If-Modified-Since', 'Range']) + r2= conn.getresponse() + conn.close() + except socket.timeout: + r2 = None + if r2 and r2.status == 200: + data = r2.read() + if r.getheader("content-encoding") == "gzip": + data = zlib.decompress(data) + self.server.glob.datdb.add(dk, data) + elif ut in (self.URLTYPE_DAT, self.URLTYPE_KAKO_DAT, self.URLTYPE_KAKO_GZ): + logger.log("PROXY", "\ttry to read dat from cache") + dp = self.datpath() + wdata = None + if ut == self.URLTYPE_KAKO_GZ: + if os.path.exists(dp): + f=open(dp) + wdata=zlib.compress(f.read()) + elif os.path.exists(dp+".gz"): + f=open(dp+".gz",'r') + wdata=f.read() + else: f= None + if f: f.close() + else: + if os.path.exists(dp): + f=open(dp) + wdata=f.read() + elif os.path.exists(dp+".gz"): + f=gzip.GzipFile(dp+".gz",'r') + wdata=f.read() + else: f= None + if f: f.close() + if wdata: # FIXME range, gzip + logger.log("PROXY", "\tfound cached dat") + #reqheader = self.get_requested_header() + self.wfile.write("HTTP/1.0 200 OK\r\n") + self.wfile.write("Content-Type: text/plain\r\n") + self.wfile.write("\r\n") + self.wfile.write(wdata) + self.wfile.close() + f.close() + # gzip で書き直す + if os.path.exists(dp) and o2on_config.DatSaveAsGzip: + f = open(dp) + g = gzip.GzipFile(dp+".gz",'w') + g.write(f.read()) + g.close() + f.close() + os.remove(dp) + else: + logger.popup("PROXY", "no cached dat. query for the dat.\n%s" % self.datkey()) + self.wfile.write("HTTP/%d.%d %d %s\r\n" % + (r.version/10,r.version%10,r.status,r.reason)) + self.wfile.write(self.msg(r)) + self.wfile.write("\r\n") + self.wfile.write(r.read()) + self.wfile.close() + self.server.glob.datquery.add(self.datkey()) + +common_header = {} +def build_common_header(prof): + global common_header + if not prof.mynode.id: raise Exception("My ID is NULL") + if prof.mynode.port==None: raise Exception("My Port is NULL") + if not prof.mynode.name: prof.mynode.name="" + if not prof.mynode.pubkey: raise Exception("My pubkey is NULL") + common_header = {'Connection': "close", + 'X-O2-Node-ID': hexlify(prof.mynode.id), + 'X-O2-Port': str(prof.mynode.port), + 'X-O2-Node-Name': prof.mynode.name, + 'X-O2-Node-Flags':'--D', + 'Server': prof.mynode.ua, + 'X-O2-RSA-Public-Key': hexlify(prof.mynode.pubkey)} + +class P2PServerHandler(BaseHTTPServer.BaseHTTPRequestHandler): + regPath = re.compile(r'^(?:http://[^/]+)?/([^/]+)') + def do_dat(self, node): + logger = self.server.glob.logger + if self.command == 'POST' and self.headers.getheader('Content-Length'): + l = int(self.headers.getheader('Content-Length')) + logger.log("P2PSERVER", + "Client gave me omiyage dat %s" % hexlify(node.id)) + data = self.rfile.read(l) + daturl = self.headers.get('X-O2-Original-DAT-URL') + dat = o2on_dat.Dat() + if daturl: + if ".." in daturl: return self.response_400("DAT URL include ..") + if not dat.setpath(daturl): return self.response_400("invalid dat url") + else: + datpath = self.headers.get('X-O2-DAT-Path') + if datpath: + if ".." in datpath: return self.response_400("datpath include ..") + if not dat.setpath(datpath): return self.response_400("invalid datpath") + if not dat.save(data): + logger.log("P2PSERVER", + "I don't like this omiyage dat %s" % self.client_address[0]) + return self.response_400("invalid omiyage") + else: + self.server.glob.datdb.add_dat(dat) + # give dat + targetkey = self.headers.get('X-O2-Target-Key') + targetboard = self.headers.get('X-O2-Target-Board') + if targetkey: + dat = self.server.glob.datdb.get(targetkey) + elif targetboard: + dat = self.server.glob.datdb.getRandomInBoard(targetboard) + else: dat = self.server.glob.datdb.choice(targetboard) + if not dat: return self.response_404() + headers = common_header.copy() + data = dat.data() + headers['X-O2-DAT-Path'] = dat.path() + headers['Content-Type'] = 'text/plain; charset=shift_jis' + headers['Content-Length'] = str(len(data)) + self.wfile.write("HTTP/1.0 200 OK\r\n") + for h in headers: self.wfile.write("%s: %s\r\n" % (h,headers[h])) + self.wfile.write("\r\n") + self.wfile.write(data) + self.wfile.close() + def do_collection(self, node): + boards = o2on_config.DatCollectionBoardList + if boards == None: + boards = self.server.glob.allboards + data = "\r\n" + data += "\r\n" + for b in boards: + data += "%s\r\n" % b + data += "\r\n" + headers = common_header.copy() + headers['Content-Type'] = 'text/xml; charset=utf-8' + headers['Content-Length'] = str(len(data)) + self.wfile.write("HTTP/1.0 200 OK\r\n") + for h in headers: self.wfile.write("%s: %s\r\n" % (h, headers[h])) + self.wfile.write("\r\n") + self.wfile.write(data) + self.wfile.close() + self.server.glob.logger.log("P2PSERVER", "gave collection %s" % hexlify(node.id)) + if self.command == 'POST': + l = int(self.headers.getheader('Content-Length')) + data = self.rfile.read(l) + # o2on の bug 対策 + if data.rfind("") == -1: + index = data.rfind("") + data = data[:index] + "" + data[index+len(""):] + if len(data): + dom = xml.dom.minidom.parseString(data) + nn = dom.getElementsByTagName("boards") + result = [] + if len(nn): + for b in nn[0].getElementsByTagName("board"): + result.append(b.childNodes[0].data) + dom.unlink() + self.server.glob.logger.log("P2PSERVER", "got collection %s" % hexlify(node.id)) + self.server.glob.nodedb.reset_collection_for_node(node) + for b in result: self.server.glob.nodedb.add_collection(b,node) + def do_ping(self, node): + headers = common_header.copy() + headers['Content-Type'] = 'text/plain' + headers['Content-Length'] = "8" + self.wfile.write("HTTP/1.0 200 OK\r\n") + for h in headers: self.wfile.write("%s: %s\r\n" % (h,headers[h])) + self.wfile.write("\r\n") + self.wfile.write(ip2e(node.ip)) + self.wfile.close() + self.server.glob.logger.log("P2PSERVER", "respond to ping %s" % hexlify(node.id)) + def do_findnode(self, node): + target = self.headers.get('X-O2-Target-Key') + if not target: return self.response_400("no target key to findnode") + target = unhexlify(target) + neighbors = self.server.glob.nodedb.neighbors_nodes(target, True) + if neighbors: + data = "\r\n" + data += "\r\n" + for node in neighbors: data += node.xml() + data += "\r\n" + data = data.encode('utf-8') + headers = common_header.copy() + headers['Content-Type'] = 'text/xml; charset=utf-8' + headers['Content-Length'] = str(len(data)) + self.wfile.write("HTTP/1.0 200 OK\r\n") + for h in headers: self.wfile.write("%s: %s\r\n" % (h,headers[h])) + self.wfile.write("\r\n") + self.wfile.write(data) + self.wfile.close() + else: return self.response_404() + def do_store(self, node): + headers = common_header.copy() + self.wfile.write("HTTP/1.0 200 OK\r\n") + for h in headers: self.wfile.write("%s: %s\r\n" % (h,headers[h])) + self.wfile.write("\r\n") + self.wfile.close() + if self.headers.getheader('Content-Length'): + l = int(self.headers.getheader('Content-Length')) + category = self.headers.get('X-O2-Key-Category') + if not category: category = 'dat' + if category == 'dat': + data = self.rfile.read(l) + dom = xml.dom.minidom.parseString(data) + top = dom.getElementsByTagName("keys") + if len(top): + for k in top[0].getElementsByTagName("key"): + key = o2on_key.Key() + key.from_xml_node(k) + self.server.glob.keydb.add(key) + dom.unlink() + else: self.server.glob.logger.log("P2PSERVER","Unknown Category %s" % category) + def do_im(self, node): + headers = common_header.copy() + self.wfile.write("HTTP/1.0 200 OK\r\n") + for h in headers: self.wfile.write("%s: %s\r\n" % (h,headers[h])) + self.wfile.write("\r\n") + self.wfile.close() + if self.headers.getheader('Content-Length'): + l = int(self.headers.getheader('Content-Length')) + data = self.rfile.read(l) + dom = xml.dom.minidom.parseString(data) + top = dom.getElementsByTagName("messages") + if len(top): + for n in top[0].getElementsByTagName("message"): + im = o2on_im.IMessage() + im.from_xml_node(n) + im.date = int(time.time()) + self.server.glob.imdb.add(im) + self.server.glob.logger.popup("IM", "Received Message!") + self.server.glob.imdb.save() + dom.unlink() + def do_findvalue(self,node): + target = self.headers.get('X-O2-Target-Key') + if not target: return self.response_400("no target key to findvalue") + self.server.glob.logger.log("P2PSERVER", + "\tfindvalue from %s for %s" % (hexlify(node.id), target)) + target = unhexlify(target) + xml_data = None + key = self.server.glob.keydb.get(target) + if key: + xml_data = "\r\n" + xml_data += "\r\n" + xml_data += key.xml(self.server.glob) + xml_data += "\r\n" + else: + neighbors = self.server.glob.nodedb.neighbors_nodes(target, True) + if len(neighbors)>0: + xml_data = "\r\n" + xml_data += "\r\n" + for node in neighbors: + if node.ip and node.port: + xml_data += "\r\n" + xml_data += "%s\r\n" % hexlify(node.id) + xml_data += "%s\r\n" % ip2e(node.ip) + xml_data += "%s\r\n" % node.port + if node.name: + xml_data += "\r\n" % \ + node.name.decode('utf-8') + if node.pubkey: + xml_data += "%s\r\n" % hexlify(node.pubkey) + xml_data += "\r\n" + xml_data += "\r\n" + if xml_data: + xml_data = xml_data.encode('utf-8') + headers = common_header.copy() + headers['Content-Type'] = 'text/xml; charset=utf-8' + headers['Content-Length'] = str(len(xml_data)) + self.wfile.write("HTTP/1.0 200 OK\r\n") + for h in headers: self.wfile.write("%s: %s\r\n" % (h,headers[h])) + self.wfile.write("\r\n") + self.wfile.write(xml_data) + self.wfile.close() + else: return self.response_404() + job = {'dat':do_dat, + 'collection': do_collection, + 'ping': do_ping, + 'findnode':do_findnode, + 'store':do_store, + 'findvalue':do_findvalue, + 'im':do_im,} + def response_400(self, reason=""): + logger = self.server.glob.logger + logger.log("P2PSERVER", + "response 400 %s (%s)" % (self.client_address[0], reason)) + logger.log("P2PSERVER", "\tpath was %s" % self.path) + logger.log("P2PSERVER", "\theader was %s" % self.headers) + header = common_header.copy() + self.wfile.write("HTTP/1.0 400 Bad Request\r\n") + for h in header: self.wfile.write("%s: %s" % (h,header[h])) + self.wfile.write("\r\n") + self.wfile.close() + def response_404(self): + #print "p2p server response 404 %s" % self.client_address[0] + header = common_header.copy() + self.wfile.write("HTTP/1.0 404 Not Found\r\n") + for h in header: self.wfile.write("%s: %s" % (h,header[h])) + self.wfile.write("\r\n") + self.wfile.close() + def get_requested_header(self): + h = self.headers.headers + hr = {} + for x in h: + f = x.split(": ",1) + hr[f[0]] = f[1][:-2] + return hr + def do_POST(self): + self.do_GET() + def do_GET(self): + self.server.glob.logger.log("P2PSERVER", "connection came %s" % (self.path)) + + nid = self.headers.getheader('X-O2-Node-ID') + if not nid: return self.response_400("No NodeID") + port = self.headers.getheader('X-O2-Port') + if not port: return self.response_400("No Port") + port = int(port) + node = o2on_node.Node(unhexlify(nid), self.client_address[0], port) + + if not self.headers.getheader('X-O2-RSA-Public-Key'): + return self.response_400("No public key") + node.pubkey = unhexlify(self.headers.getheader('X-O2-RSA-Public-Key')) + + name = self.headers.getheader('X-O2-Node-Name') + if name: node.name = name.decode('utf-8').encode('utf-8') + flag = self.headers.getheader('X-O2-Node-Flags') + if flag: node.setflag(flag) + + ua = self.headers.getheader('User-Agent') + if not ua: return self.response_400("No UA") + if len(ua)<6: return self.response_403() + m = re.compile(r'O2/(\d+(?:\.\d+)?)').match(ua) + if not m: return self.response_403() + if float(m.group(1)) < ProtocolVer: return self.response_403() + node.ua = ua + + self.server.glob.nodedb.add_node(node) + m = self.regPath.match(self.path) + if m: + func = self.job.get(m.group(1)) + if func: return func(self, node) + return self.response_404() + + +class AdminServerHandler(BaseHTTPServer.BaseHTTPRequestHandler): + default = "status" + regPath = re.compile(r'^(?:http://[^/]+)?/([^/]+)(?:/?(.*?)/?)?$') + html_header = """\ + + + + %s - Admin %s + + + +""" + html_footer = """\ + + +""" + pages = (("status", "状態"), + ("nodes", "ノード"), + ("keys", "datキー"), + ("dats", "所有dat"), + ("datq", "dat検索"), + ("im", "IM"), + ("shutdown", "シャットダウン"),) + def send_nav(self, cur): + self.wfile.write("
    \n") + for x in self.pages: + if x[0] == cur: + self.wfile.write( + "
  • %s
  • \n" % x) + else: + self.wfile.write("
  • %s
  • \n" % x) + self.wfile.write("
\n") + def send_common(self, cur, curname): + self.send_response(200) + self.send_header('Content-Type', 'text/html; charset=utf-8') + self.send_header('Connection', 'close') + self.end_headers() + self.wfile.write(self.html_header % (AppName, curname)) + self.send_nav(cur) + def im_send(self,args): + if not re.compile(r'^[0-9a-f]{40}$').match(args[1]) or \ + not re.compile(r'^[0-9a-f]{8}$').match(args[2]) or \ + not re.compile(r'^\d+$').match(args[3]) or \ + not self.server.glob.prof.mynode.ip: + self.send_common("im", "Instant Messenger Send") + self.wfile.write("""\ +
+

IM送信エラー

+
+

グローバルIPが確定していないか、送信先がおかしいです。

+
+
+""") + self.wfile.write(self.html_footer) + return + nodedb = self.server.glob.nodedb + nid = unhexlify(args[1]) + ip = e2ip(args[2]) + port = int(args[3]) + node = nodedb[nid] or o2on_node.Node(nid,ip,port) + if node.name: name = "%s (ID: %s)" % (node.name.decode('utf-8'), args[1]) + else: name = "ID: %s" % args[1] + l = self.headers.get('Content-Length') + + if self.command == "GET" or not l: + self.send_common("im", "Instant Messenger Send") + self.wfile.write(("""\ +
+

IM送信

+
+

%sにIMを送信。

+
+
+ +
+
+
+""".decode('utf-8') % (name, args[1], args[2], args[3])).encode('utf-8')) + self.wfile.write(self.html_footer) + else: + l = int(l) + data = self.rfile.read(l) + m=re.compile(r'^immsg=(.*)$').match(data) + data = urllib.unquote_plus(m.group(1)).decode('utf-8') + + result = "失敗" + try: + node.im(self.server.glob.prof.mynode, data) + except o2on_node.NodeRemovable: + nodedb.remove(node) + nodedb.save() + self.server.glob.keydb.remove_bynodeid(node.id) + self.server.glob.keydb.save() + except o2on_node.NodeRefused: + pass + else: + result = "成功" + nodedb.add_node(node) + nodedb.save() + im = o2on_im.IMessage() + im.from_node(self.server.glob.prof.mynode) + im.msg = data.encode('utf-8') + im.date = int(time.time()) + im.mine = True + self.server.glob.imdb.add(im) + self.server.glob.imdb.save() + self.send_common("im", "Instant Messenger Sent") + self.wfile.write(("""\ +
+

IMを送信しました。

+
+

%sに以下のIMを送信し、%sしました。

+

%s

+

もどる

+
+
+""".decode('utf-8') % (name,result.decode('utf-8'),data)).encode('utf-8')) + self.wfile.write(self.html_footer) + def im(self,args): + if len(args)==4 and args[0] == "send": return self.im_send(args) + imdb = self.server.glob.imdb + self.send_common("im", "Instant Messenger") + self.wfile.write("""\ +
+

IM

+
+ + +""") + for x in imdb.im_list(): + if x[0]: + self.wfile.write((""\ + "" % x[1:]).encode('utf-8')) + else: + self.wfile.write((""\ + "" % x[1:]).encode('utf-8')) + self.wfile.write("""\ +
日時名前メッセージ
%s"\ + "%s%s
%s"\ + "%s%s
+
+
+""") + self.wfile.write(self.html_footer) + def keys(self,args): + keydb = self.server.glob.keydb + self.send_common("keys", "Key") + self.wfile.write("""\ +
+

キー情報

+
+

キー数 %d

+ + + + + +""" % (len(keydb))) + for x in keydb.key_list(): + self.wfile.write(("""\ + + + """ % x).encode('utf-8')) + self.wfile.write("""\ +
dIPPortURLtitlenotedatesizehash
%d%s%d%s%s%s%s%d%s
+
+
+""") + self.wfile.write(self.html_footer) + def nodes(self,args): + nodedb = self.server.glob.nodedb + self.send_common("nodes", "Nodes") + self.wfile.write("""\ +
+

ノード情報

+
+

ノード数 %d

+ + +""" % (len(nodedb))) + for x in nodedb.node_list(): + self.wfile.write( + (""\ + "" % x).encode('utf-8')) + self.wfile.write("""\ +
dNameflgIPPortUAID
%d%s%s%s%d%s%s
+
+
+""") + self.wfile.write(self.html_footer) + def status(self,args): + self.send_common("status", "Status Summary") + glob = self.server.glob + prof = glob.prof + if prof.mynode.ip: + ip = "%s (%s)" % (prof.mynode.ip, ip2e(prof.mynode.ip)) + else: + ip = "未取得" + name = prof.mynode.name or "なし" + if prof.mynode.ip: + nodehash = hexlify(glob.prof.mynode.id)+ip2e(glob.prof.mynode.ip)+\ + port2e(glob.prof.mynode.port) + else: nodehash = "IP未取得" + self.wfile.write("""\ +

+

自ノード情報

+
+ + + +
IDIPポートノード名UAハッシュ
%s%s%s%s%s%s
+
+

+""" % (hexlify(prof.mynode.id), ip, prof.mynode.port, name, prof.mynode.ua, nodehash)) + self.wfile.write("""\ +

+

概要

+ +

+""" % (len(glob.nodedb), len(glob.datdb), len(glob.keydb), len(glob.datquery))) + self.wfile.write(self.html_footer) + def shutdown(self, args): + if len(args) == 1 and args[0] == "really": + self.send_common("shutdown", "Shutdown") + self.wfile.write("""\ +

+

シャットダウン

+
+

opy2onにシャットダウンコマンドを送信しました。

+
+

+""") + self.wfile.write(self.html_footer) + self.server.glob.shutdown.set() + else: + self.send_common("shutdown", "Shutdown") + self.wfile.write("""\ +

+

シャットダウン

+
+

opy2onをシャットダウンしますか?

+

はい / いいえ

+
+

+""") + self.wfile.write(self.html_footer) + def do_POST(self): + self.do_GET() + def do_GET(self): + m = self.regPath.match(self.path) + if m: + path = m.group(1) + if m.group(2) != "": args = m.group(2).split("/") + else: args = [] + else: + path = self.default + args = [] + if not hasattr(self, path): + self.send_error(404) + return + method = getattr(self, path) + method(args) + # BaseHTTPServer の log を抑制 + def log_message(self, format, *args): + pass Added: trunk/opy2on/lib/o2on_util.py =================================================================== --- trunk/opy2on/lib/o2on_util.py (rev 0) +++ trunk/opy2on/lib/o2on_util.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,109 @@ +#!/usr/bin/python + +import random +from struct import pack +import threading +import hashlib +import re +import os.path +import cPickle + +import o2on_config +from o2on_const import DatQueryFile, KeyQueryFile, AppName + +if o2on_config.UseDBus: import dbus + +def randomid(): + return "".join(map(lambda x: pack("B",random.randint(0,0xff)),range(0,20))) + +def datkeyhash(key): + return hashlib.sha1(key.encode('utf_16_le')).digest() +def datfullboard(key): + m=re.compile(r'^([^/]+)/([^/]+)').match(key) + return m.group(1)+":"+m.group(2) + +class Logger: + def __init__(self): + self.lock = threading.RLock() + self.bus = None + def begin(self): self.lock.acquire() + def end(self): self.lock.release() + def log(self, categ, s): + if o2on_config.NoLog: return + categ = categ[:10] + categ += " " * (10-len(categ)) + if isinstance(s, str): pass #s=s.encode('utf-8','replace') + elif isinstance(s, unicode): pass + else: s = str(s) + with self.lock: + for l in map(lambda x: "[%s] %s" % (categ, s), s.split("\n")): + print l + def popup(self, categ, s): + if o2on_config.UseDBus: + if not self.bus: self.bus = dbus.SessionBus() + obj = self.bus.get_object("org.freedesktop.Notifications", + "/org/freedesktop/Notifications") + obj.Notify(AppName, 0, '', AppName+" "+categ,s, [], {}, -1, + dbus_interface="org.freedesktop.Notifications") + else: + self.log(categ, s) + +def hash_xor_bitlength(a,b): + if len(a) != 20 or len(b) != 20: raise Exception + for i in range(len(a)-1,-1,-1): + xored = ord(a[i]) ^ ord(b[i]) + for j in range(7,-1,-1): + if (xored & (1<\r\n" + board_xml += "\r\n" + for b in boards: board_xml += "%s\r\n" % b + board_xml += "\r\n" + return board_xml + +class Query: + def __init__(self, filename): + self.lock = threading.Lock() + self.file = filename + with self.lock: + self.list = [] + self.load() + with self.lock: + self.semap = threading.Semaphore(len(self.list)) + def __len__(self): + with self.lock: return len(self.list) + def __str__(self): + with self.lock: + return "\n".join(map(str, self.list)) + def save(self): + pkl_file = open(self.file,"wb") + with self.lock: + cPickle.dump(self.list, pkl_file,-1) + pkl_file.close() + def load(self): + if(os.path.isfile(self.file)): + pkl_file = open(self.file,"rb") + with self.lock: + self.list = cPickle.load(pkl_file) + pkl_file.close() + def add(self,x): + with self.lock: + if not x in self.list: + self.list.append(x) + self.semap.release() + def pop(self): + self.semap.acquire() + with self.lock: + if len(self.list): + return self.list.pop(0) + +class DatQuery(Query): + def __init__(self): + Query.__init__(self, DatQueryFile) +class KeyQuery(Query): + def __init__(self): + Query.__init__(self, KeyQueryFile) Added: trunk/opy2on/o2on_config.py.sample =================================================================== --- trunk/opy2on/o2on_config.py.sample (rev 0) +++ trunk/opy2on/o2on_config.py.sample 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,70 @@ +#!/usr/bin/python +# -*- coding: utf-8 + +# P2Pサーバのポート +P2PPort = 0 + +# プロキシのポート +ProxyPort = 8000 + +# Adminサーバのポート +AdminPort = 9999 + +# ノードをインポートするURL http, ftp, file などが使える +Node_List_URLs = ["http://o2on.jf.land.to/node/node.xml", + "http://o2on.sourceforge.jp/nodes/"] + +# dat保存ディレトクリ +DatDir = "dat" + +# 集めるdatのリスト None にしておくと全て +DatCollectionBoardList = [] +# DatCollectionBoardList = None +# DatCollectionBoardList = ["2ch.net:tech", "2ch.net:unix", "2ch.net:linux"] + +# dat を gzip で保存するかどうか +DatSaveAsGzip = False +# DatSaveAsGzip = True + +# 接続タイムアウト (単位: 秒) +SocketTimeout = 20 + +# このノードの名前 今のところ日本語(ascii以外)はうまく動かない。 +NodeName = "" + +# 持ってない dat がプロキシにリクエストされたら取得しておくかどうか +RequestNonExistDat = False +# RequestNonExistDat = True + +# 2channel.brd のパス +Path2channel_brd = "2channel.brd" + +# ping 後にふたたびpingを行なう時間 (単位: 秒) +RePingSec = 300 + +# メッセージの通知にD-Busを使うかどうか +# True にしておくといくつかのメッセージがpopupされる +UseDBus = False +# UseDBus = True + +# ログを標準出力に出力するかどうか +NoLog = False +# NoLog = True + +############################ +# 以下は基本的にデバッグ用 # +############################ + +# pingでローカルなIPアドレスを報告されても無視する +IgnoreLocalIP = True + +# プロファイルを取得する。どこで時間をとっているのかを計測します。 +RecordProfile = False + +# プロファイルの出力先 +ProfileDir = "profile" + +# エラーが起きた時に error-<時間>.txt にエラーメッセージを出力するかど +# うか。 +OutputErrorFile = False +# OutputError = True Added: trunk/opy2on/opy2on.py =================================================================== --- trunk/opy2on/opy2on.py (rev 0) +++ trunk/opy2on/opy2on.py 2009-08-07 19:13:53 UTC (rev 142) @@ -0,0 +1,200 @@ +#!/usr/bin/python +# -*- coding: utf-8 + +from binascii import hexlify +import socket +import sys +import os.path +import os +import re +import traceback +import time +import threading + +sys.path.append("lib") + +import o2on_profile +import o2on_node +import o2on_util +import o2on_config +import o2on_server +import o2on_dat +import o2on_const +import o2on_key +import o2on_job +import o2on_im + +def showstat(args): + glob.logger.begin() + glob.logger.log("GLOBAL", "nodes %d" % len(glob.nodedb)) + glob.logger.log("GLOBAL", "datquery %d" % len(glob.datquery)) + glob.logger.log("GLOBAL", "dat %d" % len(glob.datdb)) + glob.logger.log("GLOBAL", "key %d" % len(glob.keydb)) + glob.logger.end() + +def showmynode(args): + if glob.prof.mynode.ip: + glob.logger.log("GLOBAL", + "my node is %s%s%s" % (hexlify(glob.prof.mynode.id), + o2on_node.ip2e(glob.prof.mynode.ip), + o2on_node.port2e(glob.prof.mynode.port))) + else: + glob.logger.log("GLOBAL", "Didn't get global IP") + +def read_2channel_brd(): + res = [] + regBoard = re.compile(r'^\s+[^.]+\.'+o2on_const.regHosts+r'\s+([a-z0-9]+)\s') + if os.path.isfile(o2on_config.Path2channel_brd): + f = open(o2on_config.Path2channel_brd) + while True: + line = f.readline() + if line == '':break + m = regBoard.match(line) + if m: + res.append(m.group(1)+":"+m.group(2)) + f.close() + return res + +def show_myid(x): + glob.logger.log("GLOBAL", "my ID is %s" % hexlify(glob.prof.mynode.id)) + +def show_mypubkey(x): + glob.logger.log("GLOBAL", "my pubkey is %s" % hexlify(glob.prof.mynode.pubkey)) + +def exportnode(x): + glob.nodedb.exportnode() + +def showdatquery(x): + glob.logger.begin() + glob.logger.log("GLOBAL", "-"*80) + for x in str(glob.datquery).split("\n"): + glob.logger.log("GLOBAL", x) + glob.logger.log("GLOBAL", "-"*80) + glob.logger.end() + +def showhelp(x): + glob.logger.begin() + glob.logger.log("GLOBAL", "datq: show searching dat") + glob.logger.log("GLOBAL", "exit: finish program") + glob.logger.log("GLOBAL", "exportnode: export nodes to 'exportnodes.xml'") + glob.logger.log("GLOBAL", "help: show this help") + glob.logger.log("GLOBAL", "keys: show keys") + glob.logger.log("GLOBAL", "myid: show my ID") + glob.logger.log("GLOBAL", "mynode: show my node info") + glob.logger.log("GLOBAL", "mypubkey: show my RSA public key") + glob.logger.log("GLOBAL", "nodes: show nodes") + glob.logger.log("GLOBAL", + "stat: show the numbers of nodes, searching dats, owning dats, keys") + glob.logger.end() + +def readcommand(glob): + try: + regexit = re.compile(r'^exit\s*$') + while True: + foo = sys.stdin.readline() + if regexit.match(foo): break + if foo == '': break + args = re.compile("\s+").split(foo) + procd = False + for c in commands.keys(): + if c == args[0]: + commands[c](args[1:]) + procd = True + break + if not procd: glob.logger.log("GLOBAL", "No such command: %s" % args[0]) + except Exception,inst: + if o2on_config.OutputErrorFile: + f = open('error-'+str(int(time.time()))+'.txt', 'w') + traceback.print_exc(file=f) + f.close() + self.glob.logger.popup("ERROR", str(inst)) + glob.shutdown.set() + +class dammy: pass + +#socket.setdefaulttimeout(o2on_config.SocketTimeout) + +if not os.path.exists(o2on_const.DBDir): + os.makedirs(o2on_const.DBDir) + +glob = dammy() +glob.logger = o2on_util.Logger() +glob.prof = o2on_profile.Profile(glob.logger) + +glob.nodedb = o2on_node.NodeDB(glob) +glob.datdb = o2on_dat.DatDB(glob) +glob.keydb = o2on_key.KeyDB(glob) +glob.imdb = o2on_im.IMDB(glob) + +glob.datquery = o2on_util.DatQuery() +glob.keyquery = o2on_util.KeyQuery() +glob.allboards = read_2channel_brd() + +glob.shutdown = threading.Event() + +o2on_node.build_common_header(glob.prof) +o2on_server.build_common_header(glob.prof) + +th = threading.Thread(target=readcommand, args=(glob,)) +th.setDaemon(True) +th.start() + +show_myid(None) +show_mypubkey(None) + +jobs = ( + o2on_job.GetIPThread(glob), + o2on_job.ProxyServerThread(glob), + o2on_job.AdminServerThread(glob), + o2on_job.NodeCollectorThread(glob), + o2on_job.DatCollectorThread(glob), + o2on_job.AskNodeCollectionThread(glob), + o2on_job.PublishOrigThread(glob), + o2on_job.PublishKeyThread(glob), + o2on_job.SearchThread(glob), + o2on_job.DatQueryThread(glob), + o2on_job.P2PServerThread(glob), + ) + +for j in jobs: j.start() + +commands = { + "datq": showdatquery, + 'exportnode': exportnode, + "help": showhelp, + "keys": (lambda x: glob.keydb.show()), + "myid": show_myid, + "mynode" : showmynode, + "mypubkey": show_mypubkey, + "nodes": (lambda x: glob.nodedb.show()), + "stat": showstat, +} + +try: + glob.shutdown.wait() +except KeyboardInterrupt: + pass +except Exception,inst: + if o2on_config.OutputErrorFile: + f = open('error-'+str(int(time.time()))+'.txt', 'w') + traceback.print_exc(file=f) + f.close() + glob.logger.popup("ERROR", str(inst)) + +glob.logger.log("GLOBAL", "Finish Jobs") +for j in jobs: j.stop() +glob.logger.popup("GLOBAL", "Waiting for Jobs to stop") +n = len(jobs) +c = 0 +for j in jobs: + j.join(1) + while j.isAlive(): + glob.logger.popup("GLOBAL", "Waiting for %s" % j.name) + j.join(7) + c += 1 + glob.logger.log("GLOBAL", "Finished %d/%d" % (c, n)) +glob.imdb.save() +glob.keydb.save() +glob.datdb.save() +glob.nodedb.save() +glob.logger.popup("GLOBAL", "Finished Completely") Property changes on: trunk/opy2on/opy2on.py ___________________________________________________________________ Added: svn:executable + * From o2on-svn @ lists.sourceforge.jp Sat Aug 8 16:49:21 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 16:49:21 +0900 Subject: [o2on-svn] [151] Support python 2.5 Message-ID: <1249717761.267413.24116.nullmailer@users.sourceforge.jp> Revision: 151 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=151 Author: nawota Date: 2009-08-08 16:49:21 +0900 (Sat, 08 Aug 2009) Log Message: ----------- Support python 2.5 Modified Paths: -------------- trunk/opy2on/README trunk/opy2on/lib/o2on_dat.py trunk/opy2on/lib/o2on_im.py trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_node.py trunk/opy2on/lib/o2on_server.py trunk/opy2on/lib/o2on_util.py Modified: trunk/opy2on/README =================================================================== --- trunk/opy2on/README 2009-08-07 22:04:34 UTC (rev 150) +++ trunk/opy2on/README 2009-08-08 07:49:21 UTC (rev 151) @@ -8,9 +8,9 @@ 必要なもの -- python 2.6 +- python 2.5 (できれば 2.6) - pycrypto -- D-Bus を使うならば python-dbus +- D-Bus を使うならば dbus-python 起動 Modified: trunk/opy2on/lib/o2on_dat.py =================================================================== --- trunk/opy2on/lib/o2on_dat.py 2009-08-07 22:04:34 UTC (rev 150) +++ trunk/opy2on/lib/o2on_dat.py 2009-08-08 07:49:21 UTC (rev 151) @@ -1,5 +1,7 @@ #!/usr/bin/python +from __future__ import with_statement + import threading import cPickle import os.path Modified: trunk/opy2on/lib/o2on_im.py =================================================================== --- trunk/opy2on/lib/o2on_im.py 2009-08-07 22:04:34 UTC (rev 150) +++ trunk/opy2on/lib/o2on_im.py 2009-08-08 07:49:21 UTC (rev 151) @@ -1,5 +1,7 @@ #!/usr/bin/python +from __future__ import with_statement + import threading import os.path import cPickle Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-08-07 22:04:34 UTC (rev 150) +++ trunk/opy2on/lib/o2on_key.py 2009-08-08 07:49:21 UTC (rev 151) @@ -1,6 +1,8 @@ #!/usr/bin/python # -*- coding: utf-8 +from __future__ import with_statement + import os.path import re import cPickle @@ -14,8 +16,6 @@ import hashlib import time -import sys - from o2on_const import KeyDBFile, regHosts import o2on_config from o2on_util import hash_xor_bitlength Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2009-08-07 22:04:34 UTC (rev 150) +++ trunk/opy2on/lib/o2on_node.py 2009-08-08 07:49:21 UTC (rev 151) @@ -1,6 +1,8 @@ #!/usr/bin/python # -*- coding: utf-8 +from __future__ import with_statement + import os from Crypto.Cipher import AES # pycrypto import xml.dom.minidom @@ -166,8 +168,11 @@ raise NodeRemovable except socket.error, inst: socket.setdefaulttimeout(None) - if inst.errno in (113, 111): raise NodeRemovable - if inst.errno in (110, 104): raise NodeRefused + errno = None + if hasattr(inst, 'errno'): errno = inst.errno + else: errno = inst[0] + if errno in (113, 111): raise NodeRemovable + if errno in (110, 104): raise NodeRefused else: raise inst except httplib.BadStatusLine: socket.setdefaulttimeout(None) Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-07 22:04:34 UTC (rev 150) +++ trunk/opy2on/lib/o2on_server.py 2009-08-08 07:49:21 UTC (rev 151) @@ -19,6 +19,8 @@ import traceback import sys from xml.parsers.expat import ExpatError +import threading +import select import o2on_config from o2on_const import regHosts, ProtocolVer, AppName @@ -35,6 +37,32 @@ handler) self.glob = g self.requests = [] + self.__is_shut_down = threading.Event() + self.__serving = False + def serve_forever(self, poll_interval=0.5): + #hasattr(BaseHTTPServer.HTTPServer, '_handle_request_noblock'): + if sys.hexversion >= 0x020600f0: + BaseHTTPServer.HTTPServer.serve_forever(self, poll_interval) # 2.6 + else: + self._serve_forever(poll_interval) # 2.5 + def _serve_forever(self, poll_interval=0.5): + """Handle one request at a time until shutdown. + + Polls for shutdown every poll_interval seconds. Ignores + self.timeout. If you need to do periodic tasks, do them in + another thread. + """ + self.__serving = True + self.__is_shut_down.clear() + while self.__serving: + # XXX: Consider using another file descriptor or + # connecting to the socket to wake this up instead of + # polling. Polling reduces our responsiveness to a + # shutdown request and wastes cpu at all other times. + r, w, e = select.select([self], [], [], poll_interval) + if r: + self.handle_request() + self.__is_shut_down.set() def shutdown(self): for r in self.requests: try: @@ -42,7 +70,11 @@ r.close() except Exception: pass - BaseHTTPServer.HTTPServer.shutdown(self) + if hasattr(BaseHTTPServer.HTTPServer, 'shutdown'): + BaseHTTPServer.HTTPServer.shutdown(self) + else: + self.__serving = False + self.__is_shut_down.wait() def finish_request(self, request, client_address): self.requests.append(request) try: @@ -50,7 +82,11 @@ except socket.timeout: pass except Exception,inst: - if isinstance(inst, socket.error) and inst.errno in (104, 32): + errno = None + if isinstance(inst, socket.error): # 2.6 + if hasattr(inst, 'errno'): errno = inst.errno + else: errno = inst[0] + if errno in (104, 32): # 2.5 pass else: if o2on_config.OutputErrorFile: Modified: trunk/opy2on/lib/o2on_util.py =================================================================== --- trunk/opy2on/lib/o2on_util.py 2009-08-07 22:04:34 UTC (rev 150) +++ trunk/opy2on/lib/o2on_util.py 2009-08-08 07:49:21 UTC (rev 151) @@ -1,5 +1,7 @@ #!/usr/bin/python +from __future__ import with_statement + import random from struct import pack import threading From o2on-svn @ lists.sourceforge.jp Sat Aug 8 22:31:35 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 22:31:35 +0900 Subject: [o2on-svn] [152] Add 2channel.brd; Message-ID: <1249738295.157058.22307.nullmailer@users.sourceforge.jp> Revision: 152 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=152 Author: nawota Date: 2009-08-08 22:31:35 +0900 (Sat, 08 Aug 2009) Log Message: ----------- Add 2channel.brd; Fix typo in opy2on.py Modified Paths: -------------- trunk/opy2on/lib/o2on_job.py trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_node.py trunk/opy2on/lib/o2on_server.py trunk/opy2on/o2on_config.py.sample trunk/opy2on/opy2on.py Added Paths: ----------- trunk/opy2on/2channel.brd Added: trunk/opy2on/2channel.brd =================================================================== --- trunk/opy2on/2channel.brd (rev 0) +++ trunk/opy2on/2channel.brd 2009-08-08 13:31:35 UTC (rev 152) @@ -0,0 +1,829 @@ +0,0 +?? 0 + headline.2ch.net bbynamazu ??headline + news24.2ch.net namazuplus ???? + live24.2ch.net eq ???? + live23.2ch.net eqplus ????+ +be 0 + changi.2ch.net be ????news + etc7.2ch.net nandemo ????ソ? + etc7.2ch.net argue ?? +???? 0 + headline.2ch.net bbynews ??headline + news24.2ch.net bizplus ????news+ + mamono.2ch.net newsplus ??????+ + news24.2ch.net wildplus ??????+ + news24.2ch.net moeplus ??????+ + news24.2ch.net mnewsplus ?????+ + news24.2ch.net femnewsplus ??????+ + news24.2ch.net dqnplus ??????+ + news24.2ch.net scienceplus ??????+ + news24.2ch.net owabiplus ???+ + news24.2ch.net liveplus ????タ?+ + namidame.2ch.net news ?????? + news24.2ch.net trafficinfo ????+ music8.2ch.net musicnews ?????? + anime3.2ch.net comicnews ??????? + news24.2ch.net gamenews ????? + pc11.2ch.net pcnews PC???? + news24.2ch.net news7 ?????? + bubble6.2ch.net archives ??????? + news24.2ch.net news2 ?????? + tmp7.2ch.net asia ?????? + tmp7.2ch.net bakanews ?????? + money6.2ch.net editorial ミ? +????0 + society6.2ch.net kokusai ????+ news24.2ch.net news4plus ????news+ + news24.2ch.net news5 ??????+ sports2.2ch.net iraq ?????+ news24.2ch.net news5plus ??????+ + bubble6.2ch.net dejima dejima +?? 0 + changi.2ch.net entrance ???? + sports2.2ch.net entrance2 彝歇涕彈? + etc7.2ch.net qa ?Sメ?ソ? + pc11.2ch.net pcqa PC?Sメ + info.2ch.net wiki 2ch?????? + etc7.2ch.net goods ?????? + society6.2ch.net gline ?????? + etc7.2ch.net event ?????? + bubble6.2ch.net 2chse 2ch?????+ info.2ch.net rank ??????? + etc7.2ch.net dataroom ??コ + changi.2ch.net vote ???+?? 0 + qb5.2ch.net operate 2ch????+ sports2.2ch.net operatex ??????+ qb5.2ch.net sec2ch 2ch????+ qb5.2ch.net sec2chd 2ch???? + qb5.2ch.net saku2ch ??v? + qb5.2ch.net saku ????+ qb5.2ch.net sakud ??c? + qb5.2ch.net sakukb ??m?? +???? 0 + changi.2ch.net intro ゥ???+ human7.2ch.net honobono ???? + changi.2ch.net yume ????? + sports11.2ch.net offmatrix ???OFF + sports11.2ch.net offreg ??OFF + sports11.2ch.net offevent ??OFF +AA 0 + love6.2ch.net aasaloon AA??? + love6.2ch.net mona ??? + love6.2ch.net nida ??? + love6.2ch.net aastory AA?? + love6.2ch.net kao ??? +ミ? 0 + society6.2ch.net mass ???? + tmp7.2ch.net youth ?N?? + science6.2ch.net disaster ゥ??? + love6.2ch.net gender ?????_ + society6.2ch.net giin ????? + news24.2ch.net manifesto ?。??? + society6.2ch.net police ?? + society6.2ch.net court ????? + society6.2ch.net soc ミ???? + society6.2ch.net atom ????? + society6.2ch.net river ?????? + society6.2ch.net traf ????? + money6.2ch.net recruit ?E + changi.2ch.net job ?? + society6.2ch.net volunteer ?????? + society6.2ch.net welfare ????? + society6.2ch.net mayor ??ゥ。?? + money6.2ch.net ftax ?????? + society6.2ch.net jsdf ゥ?? + money6.2ch.net nenga ????? + school7.2ch.net lifework ????+ society6.2ch.net regulate ???? +?ミ??? 0 + money6.2ch.net venture ????? + money6.2ch.net manage ??? + money6.2ch.net management ???? + money6.2ch.net estate ????? + society6.2ch.net koumu ??? + school7.2ch.net shikaku ?????? + school7.2ch.net lic ???? + money6.2ch.net haken ???? + society6.2ch.net hoken ???? + money6.2ch.net tax ?????? + school7.2ch.net exam ?????? + society6.2ch.net hosp ????メ + society6.2ch.net bio ???? + society6.2ch.net hikari ??? + money6.2ch.net dtp DTP??? + changi.2ch.net part ????? + society6.2ch.net koukoku ???? + society6.2ch.net agri ????? + money6.2ch.net build ?????E + etc7.2ch.net peko ??????? +?ミ? 0 + tmp7.2ch.net company ??????+ life9.2ch.net bouhan ??????? + pc11.2ch.net antispam ?????spam + tmp7.2ch.net ihou ???? + tmp7.2ch.net ihan ?????? +?? 0 + love6.2ch.net expo ?????? + human7.2ch.net subcal ???? + love6.2ch.net bun ???? + love6.2ch.net poem ????? + tv11.2ch.net movie ?????8mm + tv11.2ch.net cinema ?????? + bubble6.2ch.net rmovie ????? + bubble6.2ch.net kinema ????? + hobby11.2ch.net occult ???? + etc7.2ch.net esp ??? + tv11.2ch.net sfx ??? + bubble6.2ch.net rsfx ?a?? + hobby11.2ch.net drama ??????メ + hobby11.2ch.net siki ????? + hobby11.2ch.net fortune ?? + hobby11.2ch.net uranai ???_タ? + love6.2ch.net kyoto ?ミ??? + academy6.2ch.net gallery ????+ hobby11.2ch.net rakugo ???? + society6.2ch.net ruins ???? +????? 0 + science6.2ch.net rikei ???? + science6.2ch.net sci ?? + science6.2ch.net life ?? + science6.2ch.net bake ?? + science6.2ch.net kikai ????? + science6.2ch.net denki ????? + science6.2ch.net robot ??????+ science6.2ch.net infosys ??V??? + science6.2ch.net informatics ??w + science6.2ch.net sim ?????? + science6.2ch.net nougaku ?? + science6.2ch.net sky ?????+ school7.2ch.net doctor ????? + science6.2ch.net kampo ???? + science6.2ch.net math ?? + science6.2ch.net doboku ????? + science6.2ch.net material ???? + love6.2ch.net space ????? + science6.2ch.net future ????+ science6.2ch.net wild ???? + science6.2ch.net earth ???? +????? 0 + academy6.2ch.net psycho ??? + academy6.2ch.net gengo ??? + academy6.2ch.net pedagogy ??? + academy6.2ch.net sociology ミ?? + academy6.2ch.net economics ??? + love6.2ch.net book ?? + love6.2ch.net poetics ??? + academy6.2ch.net history ??? + academy6.2ch.net history2 ????? + academy6.2ch.net whis ??? + academy6.2ch.net archeology ??? + academy6.2ch.net min ?????? + academy6.2ch.net kobun ????? + academy6.2ch.net english ENGLISH + society6.2ch.net korea ???? + academy6.2ch.net china ?? + academy6.2ch.net taiwan ?? + academy6.2ch.net geo ?????? + love6.2ch.net chiri ????ゥ? + academy6.2ch.net gogaku ??? + academy6.2ch.net art ??f??? + academy6.2ch.net philo ?? + academy6.2ch.net jurisp ?? + changi.2ch.net shihou ???? +???? 0 + hobby11.2ch.net kaden ???? + bubble6.2ch.net wm ?????AV + bubble6.2ch.net vcamera ?????? + bubble6.2ch.net bakery ???? + bubble6.2ch.net toilet ??????? + hobby11.2ch.net sony ??? + hobby11.2ch.net phs ?????? + hobby11.2ch.net keitai ???? + hobby11.2ch.net chakumelo ??????? + hobby11.2ch.net appli ?????? + hobby11.2ch.net dgoods ?????? + hobby11.2ch.net camera ??? + hobby11.2ch.net dcamera ???? + hobby11.2ch.net av AV?? + hobby11.2ch.net pav ???AU +?。?? 0 + money6.2ch.net seiji ?。 + society6.2ch.net diplomacy ???? + society6.2ch.net trafficpolicy ???? + money6.2ch.net eco ?? + changi.2ch.net stock ?ョ + live27.2ch.net stockb ????? + changi.2ch.net market ???? + mamono.2ch.net livemarket1 ??1 + changi.2ch.net livemarket2 ??2 + money6.2ch.net deal ?? + society6.2ch.net koumei ????? + money6.2ch.net kyousan ??? + tmp7.2ch.net sisou ?。?? + tmp7.2ch.net kova ?????? + money6.2ch.net money ?? +??? 0 + food8.2ch.net food ??? + food8.2ch.net candy ??? + food8.2ch.net juice ??????? + food8.2ch.net pot ???獻琥 + food8.2ch.net cook ?? + food8.2ch.net salt ??? + food8.2ch.net ramen ???? + food8.2ch.net nissin ??????? + food8.2ch.net jnoodle ?????? + food8.2ch.net sushi ??? + food8.2ch.net don ? + food8.2ch.net curry ??? + food8.2ch.net bread ?? + food8.2ch.net pasta ?????? + food8.2ch.net kbbq ??+ food8.2ch.net konamono ?????+ food8.2ch.net toba ?? + food8.2ch.net gurume ????? + food8.2ch.net famires ????? + food8.2ch.net jfoods ????? + food8.2ch.net bento ????? + food8.2ch.net sake ???Bar + food8.2ch.net wine ??? + food8.2ch.net drunk ??? + food8.2ch.net recipe ??? + food8.2ch.net patissier ?????? + food8.2ch.net supplement ??????? +?? 0 + life9.2ch.net lifesaloon ????? + changi.2ch.net kankon ???? + life9.2ch.net okiraku ?????? + life9.2ch.net homealone ????? + life9.2ch.net countrylife ?ノ??? + life9.2ch.net debt リ??? + life9.2ch.net inpatient ???? + life9.2ch.net sportsclub ??????? + hobby11.2ch.net bath ?????? + life9.2ch.net anniversary ??? + life9.2ch.net sousai ???? + life9.2ch.net baby ?? + life9.2ch.net kagu ?? + hobby11.2ch.net diy DIY + money6.2ch.net shop ?????? + life9.2ch.net trend ?? + etc7.2ch.net ticketplus Walker+ + life9.2ch.net model ??? + mamono.2ch.net fashion ?????? + life9.2ch.net shoes ? + life9.2ch.net female ??+ changi.2ch.net diet ?? + life9.2ch.net seikei ???? + life9.2ch.net shapeup ????? + life9.2ch.net world ?????? + life9.2ch.net northa ?????? + life9.2ch.net credit ????? + hobby11.2ch.net point ???????? + bubble6.2ch.net cafe30 ??? + bubble6.2ch.net cafe40 ??? + bubble6.2ch.net cafe50 ?????+ life9.2ch.net live ?? + life9.2ch.net souji ??S? + life9.2ch.net goki ??????+ money6.2ch.net kechi2 ??? + life9.2ch.net chance ??+ life9.2ch.net cigaret ??? + life9.2ch.net megane ??? + life9.2ch.net yuusen ????? + life9.2ch.net conv ???? + life9.2ch.net sale ???? + hobby11.2ch.net stationery ??? + life9.2ch.net class ???+???? 0 + mamono.2ch.net shar ?????? + anime3.2ch.net x3 ?????? + etc7.2ch.net denpa ?????? + hobby11.2ch.net owarai ?????+ anime3.2ch.net 2chbook ?????? + changi.2ch.net uwasa ?? + etc7.2ch.net charaneta ????? + etc7.2ch.net charaneta2 ?????? + etc7.2ch.net mascot ???????? + bubble6.2ch.net senji ?? +?????? 0 + changi.2ch.net lovesaloon ????? + love6.2ch.net ex ???? + life9.2ch.net x1 ?? + changi.2ch.net gaysaloon ?????? + human7.2ch.net nohodame ?????? + human7.2ch.net dame ????? + life9.2ch.net loser ??? + mamono.2ch.net hikky ???? + changi.2ch.net mental ??????? + bubble6.2ch.net single ???? + human7.2ch.net wom ??+ human7.2ch.net sfe ??????+ human7.2ch.net wmotenai ?????+ changi.2ch.net ms ????+ changi.2ch.net male ???? + bubble6.2ch.net motetai ????? + changi.2ch.net motenai ?????? + life9.2ch.net alone ????? + human7.2ch.net tomorrow ???? + money6.2ch.net employee ???? + ex24.2ch.net campus ???? + changi.2ch.net student ??????? + anime3.2ch.net otaku ??? + bubble6.2ch.net nendai ??? + bubble6.2ch.net sepia ??? + game14.2ch.net gag ??? + game14.2ch.net 575 ???? + game14.2ch.net tanka ?オ???? + changi.2ch.net 4649 ????? +タ?ch 0 + headline.2ch.net bbylive タ?headline + epg.2ch.net tv2chwiki ?????? + live24.2ch.net livesaturn ????タ?S + live24.2ch.net livevenus ????タ?V + live23.2ch.net livejupiter ????タ?J + live27.2ch.net liveuranus ????タ?U + live24.2ch.net endless タ?ch + live24.2ch.net weekly ??ch + live24.2ch.net livewkwest ??ch(???) + live23.2ch.net livenhk ??ch(NHK) + live23.2ch.net liveetv ??ch(??) + live23.2ch.net liventv ??ch(NTV) + live23.2ch.net livetbs ??ch(TBS) + live23.2ch.net livecx ??ch(??) + live23.2ch.net liveanb ??ch(??) + live23.2ch.net livetx ??ch(TX) + live24.2ch.net livebs BSタ?(NHK) + live24.2ch.net livewowow BSタ?(??) + live24.2ch.net liveskyp ????タ? + live24.2ch.net liveradio ???タ? + live24.2ch.net dome ????ch + live24.2ch.net livebase ??ch + live24.2ch.net livefoot ????ch + live24.2ch.net oonna ??タ?(? + live24.2ch.net ootoko ??タ?(?) + live24.2ch.net dancesite ??ch + live24.2ch.net festival ???ch +????? 0 + school7.2ch.net edu ????? + changi.2ch.net jsaloon ??????? + namidame.2ch.net kouri ???? + school7.2ch.net juku ??m???? + school7.2ch.net ojyuken ??? + school7.2ch.net senmon ???? + school7.2ch.net design ??n?? + school7.2ch.net musicology ????? + school7.2ch.net govexam ????? +?? 0 + hobby11.2ch.net hobby ???? + hobby11.2ch.net magic ????? + hobby11.2ch.net card ???? + hobby11.2ch.net puzzle ??? + hobby11.2ch.net craft ??????? + hobby11.2ch.net toy ???? + hobby11.2ch.net zoid ??? + hobby11.2ch.net watch ?????+ hobby11.2ch.net smoking ??????? + hobby11.2ch.net knife ?? + hobby11.2ch.net doll ??? + hobby11.2ch.net engei ?? + hobby11.2ch.net dog ????? + hobby11.2ch.net pet ?????? + hobby11.2ch.net aquarium ?????? + hobby11.2ch.net goldenfish ?????? + hobby11.2ch.net insect ?? + tmp7.2ch.net cat ????? + hobby11.2ch.net bike ??? + hobby11.2ch.net car ヤ + hobby11.2ch.net kcar ?ゥ?ヤ + hobby11.2ch.net auto ヤ?????? + hobby11.2ch.net usedcar ??ヤ + hobby11.2ch.net truck ?????ヤ? + hobby11.2ch.net army ?? + hobby11.2ch.net radio ?? + hobby11.2ch.net train ???? + hobby11.2ch.net rail ?????ヤ? + hobby11.2ch.net ice ??(??) + hobby11.2ch.net gage ???? + hobby11.2ch.net bus ??????? + hobby11.2ch.net airline ????? + hobby11.2ch.net mokei ?????? + hobby11.2ch.net radiocontrol RC?????) + hobby11.2ch.net gun ???? + hobby11.2ch.net fireworks ?? + ex24.2ch.net warhis ?????? + hobby11.2ch.net chinahero ???? + hobby11.2ch.net sengoku ???? + etc7.2ch.net nanminhis ???? + hobby11.2ch.net dance ??? + hobby11.2ch.net bird ???? + hobby11.2ch.net collect ?????? + hobby11.2ch.net photo ハ??? +?????? 0 + sports11.2ch.net sposaloon ??????? + sports11.2ch.net sports ???? + sports11.2ch.net rsports ??????? + sports11.2ch.net stadium ?????? + sports11.2ch.net athletics ???Z + sports11.2ch.net gymnastics ?????? + sports11.2ch.net muscle ?????? + sports11.2ch.net noroma ???? + sports11.2ch.net wsports ????? + sports11.2ch.net ski ?????? + yutori.2ch.net skate ???? + sports11.2ch.net swim ?? + sports11.2ch.net msports ??????? + sports11.2ch.net boat ????? + sports11.2ch.net birdman ????? + sports11.2ch.net fish ?? + sports11.2ch.net bass ???? + sports11.2ch.net bicycle ゥ?ヤ + sports11.2ch.net equestrian ?n???+ ex24.2ch.net f1 唹整塾澎?+ sports11.2ch.net olympic ?????? + sports11.2ch.net bullseye ????? + sports11.2ch.net parksports ?????? + sports11.2ch.net amespo ???? + sports11.2ch.net cheerleading ?? + sports11.2ch.net xsports xsports +?? 0 + ex24.2ch.net base ???? + sports11.2ch.net npb ?????? + bubble6.2ch.net meikyu ???? + sports11.2ch.net mlb ???? + sports11.2ch.net hsb ???? + sports11.2ch.net kyozin ????? + ex24.2ch.net soccer ?????? + sports11.2ch.net eleven ??????+ sports2.2ch.net wc ??????? + sports11.2ch.net football ?????? + sports11.2ch.net basket ????? + sports11.2ch.net tennis ??? + sports11.2ch.net volley ?????? + sports11.2ch.net ovalball ???? + sports11.2ch.net pingpong ?? + sports11.2ch.net gutter ????? + sports11.2ch.net golf ??? + sports11.2ch.net billiards ????? +??? 0 + ex24.2ch.net k1 ??? + sports11.2ch.net wres ???? + sports11.2ch.net budou ????? + sports11.2ch.net boxing ????? + sports11.2ch.net sumou ?? + sports11.2ch.net jyudo ????+?????0 + love6.2ch.net oversea ???? + society6.2ch.net 21oversea ????? + love6.2ch.net travel ???? + love6.2ch.net hotel ??????+ food8.2ch.net localfoods ??????? + love6.2ch.net tropical ????? + love6.2ch.net onsen ?? + love6.2ch.net park ??? + love6.2ch.net zoo ??????? + love6.2ch.net museum ???????+ love6.2ch.net out ?????? +???? 0 + tv11.2ch.net tvsaloon ?????? + sports2.2ch.net kouhaku ?????? + mamono.2ch.net tv ????? + bubble6.2ch.net natsutv ?????? + mamono.2ch.net tvd ?????? + live27.2ch.net nhkdrama ????? + bubble6.2ch.net natsudora ?????? + tv11.2ch.net kin ??? + tv11.2ch.net am ????? + bubble6.2ch.net rradio ?????? + tv11.2ch.net tv2 ????? + tv11.2ch.net cs ?????? + tv11.2ch.net skyp ???? + tv11.2ch.net bs ?????? + tv11.2ch.net nhk NHK + tv11.2ch.net cm ????? +?? 0 + tv11.2ch.net geino ?? + tv11.2ch.net celebrity ????? + tv11.2ch.net kyon2 ?????? + tv11.2ch.net actor ???? + tv11.2ch.net actress ?D + tv11.2ch.net geinoj U-15???? + changi.2ch.net geinin ???|? + ex24.2ch.net ana ?????? + tv11.2ch.net ami ????? + tv11.2ch.net apple ???? + tv11.2ch.net ainotane ????? + tv11.2ch.net zurui ????? + music8.2ch.net mendol ?????? + changi.2ch.net idol ??A??? + changi.2ch.net akb ?????? + tv11.2ch.net jan ????? + tv11.2ch.net smap ???? + tv11.2ch.net jr ?????? + tv11.2ch.net jr2 ?????Jr +????? 0 + money6.2ch.net mj ???? + money6.2ch.net pachi ??????? + money6.2ch.net pachij ???????+ money6.2ch.net pachik ??????? + money6.2ch.net slot ??????? + money6.2ch.net slotj ???????+ money6.2ch.net slotk ?????? + mamono.2ch.net keiba ?? + hobby11.2ch.net uma ??? + money6.2ch.net keirin ?? + money6.2ch.net kyotei ?? + money6.2ch.net autorace ?????? + money6.2ch.net gamble ????? + money6.2ch.net loto ??? +??? 0 + game13.2ch.net gsaloon ?????? + game13.2ch.net gameover ????? + game13.2ch.net goveract ??ACT?? + game13.2ch.net goverrpg ??RPG?? + game14.2ch.net gamerpg ???RPG + game13.2ch.net ff FF????? + game14.2ch.net gamesrpg ???SRPG + game13.2ch.net gamerobo ?????? + game14.2ch.net gal ????? + game14.2ch.net gboy ??Q??? + game14.2ch.net ggirl ??Q??? + game13.2ch.net gamespo ?????RACE + game13.2ch.net gamehis ????? + game13.2ch.net otoge ??? + game13.2ch.net gamefight ????? + game13.2ch.net gamestg ??????? + game14.2ch.net gamef PC????? + game14.2ch.net fly ?????? + game13.2ch.net famicom ?????? + game14.2ch.net retro ?????? + game14.2ch.net retro2 ?????? + game14.2ch.net game90 90????? + game13.2ch.net arc ????? + game13.2ch.net amusement ???????? + game13.2ch.net gecen ???? + game13.2ch.net game PC??? + game13.2ch.net gameama ????? + game14.2ch.net gameswf ??????? + game14.2ch.net cgame ??Q?? + game14.2ch.net tcg TCG + game14.2ch.net bgame ??E??? + game14.2ch.net gamestones ?????? + game14.2ch.net quiz ????? + namidame.2ch.net ghard ?????? + game14.2ch.net gameurawaza ????? + game13.2ch.net gamechara ?????? + game14.2ch.net gamemusic ????? +?????? 0 + game13.2ch.net handygame ??????? + game14.2ch.net handygover ?????? + game14.2ch.net handygrpg ????RPG + game13.2ch.net poke ???? + game13.2ch.net wifi Wi-Fi + game14.2ch.net rhandyg ??????? + game13.2ch.net pokechara ??????? +?????? 0 + live27.2ch.net mmonews ????? + live27.2ch.net mmoqa ???ソ? + changi.2ch.net ogame ???タ? + ex24.2ch.net ogame2 ???タ?2 + changi.2ch.net ogame3 ???タ?3 + game13.2ch.net mmosaloon ?????? + game13.2ch.net netgame ?????? + game13.2ch.net mmo ???MMO + game13.2ch.net mmominor ?K?MMO +??????0 + changi.2ch.net asaloon ?????? + changi.2ch.net anime4vip ??????ex + changi.2ch.net anime ??? + changi.2ch.net anime2 ???? + anime3.2ch.net anime3 ???????+ anime3.2ch.net ranime ????a + anime3.2ch.net ranimeh ????? + anime3.2ch.net animovie ????? + anime3.2ch.net anichara ??????? + changi.2ch.net anichara2 ??????? + anime3.2ch.net cosp ???? + anime3.2ch.net voice ???? + ex24.2ch.net voiceactor ???? + anime3.2ch.net doujin ?? + sports2.2ch.net comiket ?????? + changi.2ch.net csaloon ????? + changi.2ch.net comic ?? + anime3.2ch.net rcomic ????? + anime3.2ch.net ymag ?N?? + ex24.2ch.net wcomic ???N?? + anime3.2ch.net gcomic ????+ anime3.2ch.net 4koma ????? + anime3.2ch.net cchara ????? + anime3.2ch.net sakura CC??? + anime3.2ch.net eva ??? + anime3.2ch.net cartoon ??????? + anime3.2ch.net iga ??????? + love6.2ch.net bookall ????T?? + love6.2ch.net magazin ?????? + love6.2ch.net mystery ????? + love6.2ch.net sf SF?FT???? + love6.2ch.net zassi ?? + love6.2ch.net books ????+ love6.2ch.net ehon ?? + love6.2ch.net juvenile ???+ love6.2ch.net illustrator ???????? +?? 0 + music8.2ch.net msaloon ????? + music8.2ch.net mjsaloon ????? + music8.2ch.net musicj ?? + music8.2ch.net musicjm ?????? + music8.2ch.net musicjf ????\? + music8.2ch.net musicjg ?????? + bubble6.2ch.net natsumeloj ????? + music8.2ch.net enka ?? + music8.2ch.net mesaloon ????? + music8.2ch.net musice ?? + bubble6.2ch.net natsumeloe ????? + music8.2ch.net music ???? + bubble6.2ch.net beatles ????? + music8.2ch.net visual ??????? + music8.2ch.net visualb ??????? + music8.2ch.net dj ?????? + music8.2ch.net disco ???? + music8.2ch.net randb R&B?SOUL + music8.2ch.net punk ??? + music8.2ch.net hrhm HR?HM + music8.2ch.net hiphop HIPHOP + music8.2ch.net techno TECHNO + music8.2ch.net progre ???? + bubble6.2ch.net healmusic ??????? + music8.2ch.net wmusic ?????? + music8.2ch.net classic ??? + bubble6.2ch.net fusion ?????? + music8.2ch.net classical ????? + music8.2ch.net contemporary ???? + music8.2ch.net nika ??????? + music8.2ch.net suisou ??? + bubble6.2ch.net chorus ??+ music8.2ch.net doyo ?????+ anime3.2ch.net asong ????? + bubble6.2ch.net soundtrack ???? + music8.2ch.net karaok ???? + music8.2ch.net legend ??? + music8.2ch.net minor ?????? + bubble6.2ch.net band ??? + music8.2ch.net compose ????? + bubble6.2ch.net piano ???? +???? 0 + life9.2ch.net healing ?? + life9.2ch.net jinsei ???? + life9.2ch.net psy ????+ life9.2ch.net body ????? + human7.2ch.net handicap ???????? + etc7.2ch.net infection ?????+ love6.2ch.net hiv HIV??? + life9.2ch.net atopi ???? + life9.2ch.net allergy ????? + life9.2ch.net hage ????? + love6.2ch.net pure ????+ love6.2ch.net furin ????? + love6.2ch.net gay ??? + life9.2ch.net utu ??????? + love6.2ch.net break ク? +??? 0 + ex24.2ch.net pc2nanmin PC??? + pc11.2ch.net win Windows + pc11.2ch.net jobs ??mac + pc11.2ch.net mac ??mac + pc11.2ch.net os OS + pc11.2ch.net desktop ?????? + pc11.2ch.net pc ?????? + pc11.2ch.net notepc ???PC + pc11.2ch.net jisaku ゥ?PC + pc11.2ch.net printer ???? + pc11.2ch.net hard ?????? + pc11.2ch.net cdr CD-R,DVD + pc11.2ch.net software ?????? + pc11.2ch.net mobile ???? + pc11.2ch.net bsoft ????soft + pc11.2ch.net unix UNIX + pc11.2ch.net db ?????? + pc11.2ch.net linux Linux + pc11.2ch.net prog ?????? + pc11.2ch.net tech ????? + pc11.2ch.net cg ?? + pc11.2ch.net dtm DTM + pc11.2ch.net avi DTV + pc11.2ch.net swf FLASH + pc11.2ch.net gamedev ?????+ bubble6.2ch.net i4004 ??PC +????? 0 + pc11.2ch.net internet ??????? + changi.2ch.net download Download + pc11.2ch.net hp Web?? + pc11.2ch.net affiliate Web?? + pc11.2ch.net hosting ????? + pc11.2ch.net mysv ゥ???? + pc11.2ch.net php WebProg + pc11.2ch.net hack ?????? + pc11.2ch.net sec ?????? + pc11.2ch.net network ????+ ipv6.2ch.net ipv6 IPv6 + pc11.2ch.net friend ????irc + pc11.2ch.net isp ?????? + pc11.2ch.net netspot ?????? + pc11.2ch.net nifty Nifty + pc11.2ch.net mmag ???? + changi.2ch.net nanmin ?? + ex24.2ch.net ad ???ヲ? + pc11.2ch.net esite ??????? + pc11.2ch.net streaming YouTube + pc11.2ch.net mstreaming ???? + music8.2ch.net mdis ???? + pc11.2ch.net blog ??? + pc11.2ch.net sns ???????? + pc11.2ch.net net ???watch + pc11.2ch.net yahoo ?????? + pc11.2ch.net nntp nntp +???? 0 + etc7.2ch.net bobby ?????? + tmp7.2ch.net lobby ??? + etc7.2ch.net maru ? + changi.2ch.net mog2 ?????? + bubble6.2ch.net mukashi ? + tmp7.2ch.net kitchen ??? + tmp7.2ch.net tubo ?? + tmp7.2ch.net joke ?? + society6.2ch.net shugi ????? + tmp7.2ch.net rights ???? + ex24.2ch.net accuse 2ch???? + changi.2ch.net morningcoffee ????? + ex24.2ch.net ranking ??? + etc7.2ch.net siberia ??????? + yutori.2ch.net news4vip ????VIP + yutori.2ch.net news4viptasu ????VIP+ + namidame.2ch.net poverty ????(??) + yutori.2ch.net heaven4vip ?? + yutori.2ch.net neet4vip ??? +BBSPINK 0 + headline.bbspink.com bbypink PINKheadline + babiru.bbspink.com hnews ??????? + babiru.bbspink.com pinkqa pink?Sメ + yomi.bbspink.com sureh ??H????? + babiru.bbspink.com erolive ???タ? + venus.bbspink.com hneta ????? + yomi.bbspink.com pinkcafe PINK????? + set.bbspink.com eromog2 PINK????? + set.bbspink.com ogefin ??? + babiru.bbspink.com pinknanmin ????? + set.bbspink.com erobbs pink???? + babiru.bbspink.com housekeeping PINK????+ venus.bbspink.com ccc PINK???? + babiru.bbspink.com 21oversea2 ????? + qiufen.bbspink.com hgame ???? + qiufen.bbspink.com hgame2 ??????? + set.bbspink.com erog ????? + set.bbspink.com leaf Leaf?key + set.bbspink.com adultsite ??????? + yomi.bbspink.com webmaster ??????? + set.bbspink.com avideo AV?? + set.bbspink.com avideo2 AV?D + babiru.bbspink.com nude ??????? + yomi.bbspink.com eroanime ?????? + set.bbspink.com erocomic ??????+ yomi.bbspink.com erodoujin ???? + set.bbspink.com natuero ????? + yomi.bbspink.com kgirls ???? + set.bbspink.com erocosp ?????? + set.bbspink.com eroacademy PINK??? + set.bbspink.com mcheck ????(??) + babiru.bbspink.com couple ???? + yomi.bbspink.com kageki ????? + babiru.bbspink.com kageki2 ????? + babiru.bbspink.com onatech ???? + babiru.bbspink.com loveho ????? + babiru.bbspink.com adultgoods ??????? + babiru.bbspink.com adultaccessory ?????+ set.bbspink.com sm ?? + set.bbspink.com feti ??? + babiru.bbspink.com mature ??+ yomi.bbspink.com okama ??????? + babiru.bbspink.com gaypink ?????? + babiru.bbspink.com lesbian ??????? + babiru.bbspink.com eroaa ??AA + babiru.bbspink.com erochara ?????? + yomi.bbspink.com erochara2 ????? + yomi.bbspink.com 801 ??? + set.bbspink.com erocg ??????? + yomi.bbspink.com eroparo ???? + venus.bbspink.com ascii ????? + yomi.bbspink.com ascii2d ????? + qiufen.bbspink.com ascii2kana ???? + set.bbspink.com girls ?????? + set.bbspink.com sportgirls ?????? + qiufen.bbspink.com club ???? + qiufen.bbspink.com pub ?????n + babiru.bbspink.com host ?????? + qiufen.bbspink.com nuki ??? + qiufen.bbspink.com soap ??? + yomi.bbspink.com neet4pink ???(pink) + venus.bbspink.com cherryboy ?? + venus.bbspink.com megami ?_ Modified: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py 2009-08-08 07:49:21 UTC (rev 151) +++ trunk/opy2on/lib/o2on_job.py 2009-08-08 13:31:35 UTC (rev 152) @@ -112,6 +112,7 @@ JobThread.__init__(self,"dat collector",60,g) def dojob(self, nodes, logger, prof, datdb, datq): board = nodes.get_random_board() + logger.log("DATCOLLECTOR","Try to get dat in %s" % board) if not board: return for n in nodes.get_nodes_for_board(board): if self.finish: break @@ -190,8 +191,7 @@ logger.log("ASKNODECOLLECTION", "\tadd collection for %s" % (hexlify(n.id))) nodedb.reset_collection_for_node(n) - for b in colboards: - nodedb.add_collection(b,n) + for b in colboards: nodedb.add_collection(b,n) nodedb.add_node(n) nodedb.save() Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-08-08 07:49:21 UTC (rev 151) +++ trunk/opy2on/lib/o2on_key.py 2009-08-08 13:31:35 UTC (rev 152) @@ -119,7 +119,9 @@ return len(self.keys) def show(self): pager = os.environ.get('PAGER') - if not pager: self.glob.logger.log("KEYDB", "PAGER env must be set") + if not pager: + self.glob.logger.log("KEYDB", "PAGER env must be set") + return proc = subprocess.Popen(pager, shell=True, stdin=subprocess.PIPE) pipe = proc.stdin try: Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2009-08-08 07:49:21 UTC (rev 151) +++ trunk/opy2on/lib/o2on_node.py 2009-08-08 13:31:35 UTC (rev 152) @@ -397,6 +397,9 @@ def get_nodes_for_board(self, board): with self.lock: if not board in self.boardmap: return [] + if len(self.boardmap[board])==0: + del self.boardmap[board] + return [] return map(lambda x: self.nodes[x], self.boardmap[board]) def get_random_board(self): with self.lock: Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-08 07:49:21 UTC (rev 151) +++ trunk/opy2on/lib/o2on_server.py 2009-08-08 13:31:35 UTC (rev 152) @@ -83,10 +83,10 @@ pass except Exception,inst: errno = None - if isinstance(inst, socket.error): # 2.6 - if hasattr(inst, 'errno'): errno = inst.errno - else: errno = inst[0] - if errno in (104, 32): # 2.5 + if isinstance(inst, socket.error): + if hasattr(inst, 'errno'): errno = inst.errno # 2.6 + else: errno = inst[0] # 2.5 + if errno in (104, 32, 110): pass else: if o2on_config.OutputErrorFile: Modified: trunk/opy2on/o2on_config.py.sample =================================================================== --- trunk/opy2on/o2on_config.py.sample 2009-08-08 07:49:21 UTC (rev 151) +++ trunk/opy2on/o2on_config.py.sample 2009-08-08 13:31:35 UTC (rev 152) @@ -67,4 +67,4 @@ # ????若?莎激??????error-.txt ???????<??祉??吾??阪??????? # ????? OutputErrorFile = False -# OutputError = True +# OutputErrorFile = True Modified: trunk/opy2on/opy2on.py =================================================================== --- trunk/opy2on/opy2on.py 2009-08-08 07:49:21 UTC (rev 151) +++ trunk/opy2on/opy2on.py 2009-08-08 13:31:35 UTC (rev 152) @@ -45,7 +45,7 @@ res = [] regBoard = re.compile(r'^\s+[^.]+\.'+o2on_const.regHosts+r'\s+([a-z0-9]+)\s') if os.path.isfile(o2on_config.Path2channel_brd): - f = open(o2on_config.Path2channel_brd) + f = open(o2on_config.Path2channel_brd,'r') while True: line = f.readline() if line == '':break @@ -53,6 +53,7 @@ if m: res.append(m.group(1)+":"+m.group(2)) f.close() + else: glob.logger.popup("GLOBAL", "No 2channel.brd file") return res def show_myid(x): @@ -107,17 +108,17 @@ f = open('error-'+str(int(time.time()))+'.txt', 'w') traceback.print_exc(file=f) f.close() - self.glob.logger.popup("ERROR", str(inst)) + glob.logger.popup("ERROR", str(inst)) glob.shutdown.set() -class dammy: pass +class dummy: pass #socket.setdefaulttimeout(o2on_config.SocketTimeout) if not os.path.exists(o2on_const.DBDir): os.makedirs(o2on_const.DBDir) -glob = dammy() +glob = dummy() glob.logger = o2on_util.Logger() glob.prof = o2on_profile.Profile(glob.logger) From o2on-svn @ lists.sourceforge.jp Sat Aug 8 22:38:17 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 08 Aug 2009 22:38:17 +0900 Subject: [o2on-svn] [153] Start command reading thread after commands is set. Message-ID: <1249738697.593528.3191.nullmailer@users.sourceforge.jp> Revision: 153 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=153 Author: nawota Date: 2009-08-08 22:38:17 +0900 (Sat, 08 Aug 2009) Log Message: ----------- Start command reading thread after commands is set. Modified Paths: -------------- trunk/opy2on/opy2on.py Modified: trunk/opy2on/opy2on.py =================================================================== --- trunk/opy2on/opy2on.py 2009-08-08 13:31:35 UTC (rev 152) +++ trunk/opy2on/opy2on.py 2009-08-08 13:38:17 UTC (rev 153) @@ -136,10 +136,6 @@ o2on_node.build_common_header(glob.prof) o2on_server.build_common_header(glob.prof) -th = threading.Thread(target=readcommand, args=(glob,)) -th.setDaemon(True) -th.start() - show_myid(None) show_mypubkey(None) @@ -171,6 +167,10 @@ "stat": showstat, } +th = threading.Thread(target=readcommand, args=(glob,)) +th.setDaemon(True) +th.start() + try: glob.shutdown.wait() except KeyboardInterrupt: From o2on-svn @ lists.sourceforge.jp Sun Aug 9 09:05:14 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 09 Aug 2009 09:05:14 +0900 Subject: [o2on-svn] [154] Fix dat query's bug. Message-ID: <1249776314.350190.1072.nullmailer@users.sourceforge.jp> Revision: 154 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=154 Author: nawota Date: 2009-08-09 09:05:14 +0900 (Sun, 09 Aug 2009) Log Message: ----------- Fix dat query's bug. make proxy server to try to get dat's title. Modified Paths: -------------- trunk/opy2on/lib/o2on_dat.py trunk/opy2on/lib/o2on_job.py trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_node.py trunk/opy2on/lib/o2on_server.py trunk/opy2on/lib/o2on_util.py trunk/opy2on/opy2on.py Modified: trunk/opy2on/lib/o2on_dat.py =================================================================== --- trunk/opy2on/lib/o2on_dat.py 2009-08-08 13:38:17 UTC (rev 153) +++ trunk/opy2on/lib/o2on_dat.py 2009-08-09 00:05:14 UTC (rev 154) @@ -160,9 +160,9 @@ def get(self,x): with self.lock: return self.hashmap.get(x) - def has_key(self,key): + def has_keyhash(self,key): with self.lock: - return o2on_util.datkeyhash(key) in self.hashmap + return key in self.hashmap def add_dat(self, dat): with self.lock: befdat = self.hashmap.get(dat.hash()) Modified: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py 2009-08-08 13:38:17 UTC (rev 153) +++ trunk/opy2on/lib/o2on_job.py 2009-08-09 00:05:14 UTC (rev 154) @@ -295,18 +295,18 @@ self.glob.datquery.semap.release() def dojob(self, nodedb, logger, prof, datdb, datq): d = datq.pop() - if self.finish: return - if datdb.has_key(d): + if datdb.has_keyhash(d.hash): datq.save() return - kid = o2on_util.datkeyhash(d) + d.published = int(time.time()) datq.add(d) datq.save() + if self.finish: return reckey = [] - next = nodedb.neighbors_nodes(kid,False,5) + next = nodedb.neighbors_nodes(d.hash,False,5) sent = [] - key = self.glob.keydb.get(kid) - if key: reckey.append(key) + keys = self.glob.keydb.get_bydatkey(d.hash) + if keys: reckey.extend(keys) while True: if self.finish: return neighbors = next @@ -314,10 +314,10 @@ next = [] for node in neighbors: if self.finish: return - logger.log("SEARCH","findvalue to %s for %s" % (hexlify(node.id),d)) + logger.log("SEARCH","findvalue to %s for %s" % (hexlify(node.id),d.url)) sent.append(node.id) try: - res = node.findvalue(kid) + res = node.findvalue(d.hash) except o2on_node.NodeRemovable: nodedb.remove(node) nodedb.save() @@ -340,7 +340,7 @@ logger.log("SEARCH","\tadd new key") if x not in reckey: reckey.append(x) nodedb.save() - if len(reckey) == 0: logger.log("SEARCH","\tfailed to get key for %s" % d) + if len(reckey) == 0: logger.log("SEARCH","\tfailed to get key for %s" % d.url) for key in reckey: self.glob.keydb.add(key) self.glob.keyquery.add(key) Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-08-08 13:38:17 UTC (rev 153) +++ trunk/opy2on/lib/o2on_key.py 2009-08-09 00:05:14 UTC (rev 154) @@ -37,7 +37,9 @@ def __cmp__(self,x): return cmp(self.idkeyhash(), x.idkeyhash()) def idkeyhash(self): - if not self.ikhash: self.ikhash = hashlib.sha1(self.hash + self.nodeid).digest() + if not self.ikhash: + if self.nodeid: self.ikhash = hashlib.sha1(self.hash + self.nodeid).digest() + else: self.ikhash = self.hash return self.ikhash def from_key(self, key): if self.idkeyhash() == key.idkeyhash(): @@ -112,6 +114,7 @@ self.keys = {} self.lenmap = {} self.publishmap = {} + self.datkeymap = {} self.glob = g self.load() def __len__(self): @@ -174,9 +177,9 @@ key.published = publish_time if publish_time not in self.publishmap: self.publishmap[publish_time]=[] self.publishmap[publish_time].append(idkeyhash) - def get(self, target): + def get_bydatkey(self, target): with self.lock: - return self.keys.get(target) + return map(lambda x: self.keys.get(x), self.datkeymap.get(target,[])) def remove_bynodeid(self, nid): if len(nid) != 20:raise Exception removes = [] @@ -191,9 +194,11 @@ l = hash_xor_bitlength(self.glob.prof.mynode.id, k.hash) self.lenmap[l].remove(k.idkeyhash()) self.publishmap[k.published].remove(k.idkeyhash()) + self.datkeymap[k.hash].remove(k.idkeyhash()) if len(self.lenmap[l]) == 0: del self.lenmap[l] if len(self.publishmap[k.published])==0: del self.publishmap[k.published] - def add(self, k): + if len(self.datkeymap[k.hash])==0: del self.datkeymap[k.hash] + def add(self, k): if not k.valid(): return k.published = int(time.time()) if k.idkeyhash() in self.keys: @@ -214,6 +219,9 @@ self.publishmap[maxkey.published].remove(maxkey.idkeyhash()) if len(self.publishmap[maxkey.published]) == 0: del self.publishmap[maxkey.published] + self.datkeymap[maxkey.hash].remove(maxkey.idkeyhash()) + if len(self.datkeymap[maxkey.hash]) == 0: + del self.datkeymap[maxkey.hash] else: return self.keys[k.idkeyhash()] = k if bl not in self.lenmap: self.lenmap[bl] = [] @@ -221,12 +229,16 @@ if k.published not in self.publishmap: self.publishmap[k.published] = [] self.publishmap[k.published].append(k.idkeyhash()) + if k.hash not in self.datkeymap: + self.datkeymap[k.hash] = [] + self.datkeymap[k.hash].append(k.idkeyhash()) def save(self): f = open(KeyDBFile,'wb') with self.lock: cPickle.dump(self.keys, f, -1) cPickle.dump(self.lenmap, f, -1) cPickle.dump(self.publishmap,f,-1) + cPickle.dump(self.datkeymap,f,-1) f.close() def load(self): if os.path.isfile(KeyDBFile): @@ -235,4 +247,15 @@ self.keys = cPickle.load(f) self.lenmap = cPickle.load(f) self.publishmap = cPickle.load(f) + try: + self.datkeymap = cPickle.load(f) + except EOFError: + rebuild = True + else: rebuild = False f.close() + if rebuild: + keys = self.keys.values() + self.keys.clear() + self.lenmap.clear() + self.publishmap.clear() + for k in keys: self.add(k) Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2009-08-08 13:38:17 UTC (rev 153) +++ trunk/opy2on/lib/o2on_node.py 2009-08-09 00:05:14 UTC (rev 154) @@ -429,7 +429,10 @@ del self.boardmap[board][0] self.boardmap[board].append(n.id) else: - r = nt.ping() + try: + r = nt.ping() + except NodeRemovable, NodeRefused: + r = False if r: self.boardmap[board].append(self.boardmap[board][0]) del self.boardmap[board][0] @@ -502,7 +505,10 @@ self.nodes[node.id] = node else: n = self.nodes[self.KBuckets[bitlen][0]] - r = n.ping() + try: + r = n.ping() + except NodeRemovable, NodeRefused: + r = False if r: del self.KBuckets[bitlen][0] self.KBuckets[bitlen].append(n.id) Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-08 13:38:17 UTC (rev 153) +++ trunk/opy2on/lib/o2on_server.py 2009-08-09 00:05:14 UTC (rev 154) @@ -29,6 +29,7 @@ from o2on_node import ip2e, port2e, e2ip import o2on_key import o2on_im +import o2on_util class O2ONServer(BaseHTTPServer.HTTPServer): def __init__(self, handler, port, g): @@ -64,7 +65,7 @@ self.handle_request() self.__is_shut_down.set() def shutdown(self): - for r in self.requests: + for r in []:#self.requests: try: r.shutdown(socket.SHUT_RDWR) r.close() @@ -186,6 +187,22 @@ if not m: m = self.regs[self.URLTYPE_KAKO_GZ].match(self.path) if not m: return None return "/".join((m.group(1), m.group(2), m.group(3))) + def readcgi_url(self): + + return None + def dattitle(self,data): + first = data.split("\n",1)[0] + m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) + if not m: return "" + try: + first = first.decode('cp932').encode('utf-8') + except UnicodeDecodeError, inst: + try: + first = first.decode('euc_jp').encode('utf-8') + except UnicodeDecodeError, inst: raise inst + m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) + if not m: return "" + return m.group(1) def do_GET(self): logger = self.server.glob.logger logger.log("PROXY", "proxy requested %s" % self.path) @@ -219,14 +236,15 @@ self.wfile.write(data) self.wfile.close() dk = self.datkey() + dkh = o2on_util.datkeyhash(dk) dp = self.datpath() if r.status == 200: - if not self.server.glob.datdb.has_key(dk): + if not self.server.glob.datdb.has_keyhash(dkh): # 持ってない dat が取得された logger.log("PROXY", "\tsave responsed dat for myself") self.server.glob.datdb.add(dk, datdata) else: - if self.server.glob.datdb.has_key(dk): + if self.server.glob.datdb.has_keyhash(dkh): if r.status == 206: # 持ってる dat の差分 rg = r.getheader('Content-Range') @@ -248,7 +266,9 @@ if r2 and r2.status == 200: data = r2.read() if r.getheader("content-encoding") == "gzip": - data = zlib.decompress(data) + sf = StringIO.StringIO(data) + dec = gzip.GzipFile(fileobj=sf) + data = dec.read() self.server.glob.datdb.add(dk, data) elif ut in (self.URLTYPE_DAT, self.URLTYPE_KAKO_DAT, self.URLTYPE_KAKO_GZ): logger.log("PROXY", "\ttry to read dat from cache") @@ -290,14 +310,33 @@ f.close() os.remove(dp) else: - logger.popup("PROXY", "no cached dat. query for the dat.\n%s" % self.datkey()) + logger.popup("PROXY", "no cached dat. query for the dat.\n%s" % \ + self.datkey()) + data = r.read() self.wfile.write("HTTP/%d.%d %d %s\r\n" % (r.version/10,r.version%10,r.status,r.reason)) self.wfile.write(self.msg(r)) self.wfile.write("\r\n") - self.wfile.write(r.read()) + self.wfile.write(data) self.wfile.close() - self.server.glob.datquery.add(self.datkey()) + try: + conn = self.get_connection(['If-Modified-Since', 'Range', + 'User-Agent']) + r2= conn.getresponse() + conn.close() + except socket.timeout: + r2 = None + if r2 and r2.status == 203: + data = r2.read() + if r2.getheader("content-encoding") == "gzip": + sf = StringIO.StringIO(data) + dec = gzip.GzipFile(fileobj=sf) + data = dec.read() + title = self.dattitle(data) + else: title = "" + self.server.glob.datquery.add_bydatkey(self.datkey(), + None, title, True) + self.server.glob.datquery.save() common_header = {} def build_common_header(prof): @@ -476,11 +515,11 @@ "\tfindvalue from %s for %s" % (hexlify(node.id), target)) target = unhexlify(target) xml_data = None - key = self.server.glob.keydb.get(target) - if key: + keys = self.server.glob.keydb.get_bydatkey(target) + if keys: xml_data = "\r\n" xml_data += "\r\n" - xml_data += key.xml() + for key in keys: xml_data += key.xml() xml_data += "\r\n" else: neighbors = self.server.glob.nodedb.neighbors_nodes(target, True) @@ -668,6 +707,27 @@ self.end_headers() self.wfile.write(self.html_header % (AppName, curname)) self.send_nav(cur) + def datq(self, args): + datq = self.server.glob.datquery + self.send_common("im", "Searching Dats") + self.wfile.write("""\ +
+

検索中dat

+
+

検索中dat数 %d

+ + +""" % (len(datq))) + for x in datq.datq_list(): + self.wfile.write( + ("\n" \ + % (x[0],x[0],x[1],x[2])).encode('utf-8')) + self.wfile.write("""\ +
URLTitle最終検索日時
%s%s%s
+
+
+""") + self.wfile.write(self.html_footer) def im_send(self,args): if not re.compile(r'^[0-9a-f]{40}$').match(args[1]) or \ not re.compile(r'^[0-9a-f]{8}$').match(args[2]) or \ @@ -761,11 +821,11 @@ for x in imdb.im_list(): if x[0]: self.wfile.write(("%s"\ - "%s"\ + "%s\n"\ "%s" % x[1:]).encode('utf-8')) else: self.wfile.write(("%s"\ - "%s"\ + "%s\n"\ "%s" % x[1:]).encode('utf-8')) self.wfile.write("""\ @@ -812,7 +872,7 @@ for x in nodedb.node_list(): self.wfile.write( ("%d%s%s"\ - "%s%d%s%s" % x).encode('utf-8')) + "%s%d%s%s\n" % x).encode('utf-8')) self.wfile.write("""\ Modified: trunk/opy2on/lib/o2on_util.py =================================================================== --- trunk/opy2on/lib/o2on_util.py 2009-08-08 13:38:17 UTC (rev 153) +++ trunk/opy2on/lib/o2on_util.py 2009-08-09 00:05:14 UTC (rev 154) @@ -1,4 +1,5 @@ #!/usr/bin/python +# -*- coding: utf-8 from __future__ import with_statement @@ -9,6 +10,7 @@ import re import os.path import cPickle +import datetime import o2on_config from o2on_const import DatQueryFile, KeyQueryFile, AppName @@ -92,10 +94,11 @@ with self.lock: self.list = cPickle.load(pkl_file) pkl_file.close() - def add(self,x): + def add(self,x, front=False): with self.lock: if not x in self.list: - self.list.append(x) + if front: self.list.insert(0,x) + else: self.list.append(x) self.semap.release() def pop(self): self.semap.acquire() @@ -103,9 +106,45 @@ if len(self.list): return self.list.pop(0) +import o2on_key + class DatQuery(Query): + regDatKey = re.compile("^([^/]+)/([^/]+)/(\d+)$") def __init__(self): Query.__init__(self, DatQueryFile) + def load(self): + Query.load(self) + with self.lock: + if len(self.list)>0 and isinstance(self.list[0], str): + copy = list(self.list) + self.list = [] + for x in copy: + m = self.regDatKey.match(x) + key = o2on_key.Key() + key.hash = datkeyhash(x) + key.url = "http://xxx.%s/test/read.cgi/%s/%s/" % (m.group(1), + m.group(2), + m.group(3)) + self.list.append(key) + def add_bydatkey(self, x, url=None, title ="", front=False): + key = o2on_key.Key() + key.hash = datkeyhash(x) + if url: key.url = url + else: + m = self.regDatKey.match(x) + key.url = "http://xxx.%s/test/read.cgi/%s/%s/" % (m.group(1), + m.group(2), + m.group(3)) + key.title = title + self.add(key, front) + def datq_list(self): + with self.lock: + result = [] + for x in self.list: + if x.published == 0: t = "まだ検索されてない".decode('utf-8') + else: t = str(datetime.datetime.fromtimestamp(int(x.published))) + result.append((x.url, x.title.decode('utf-8'), t)) + return result class KeyQuery(Query): def __init__(self): Query.__init__(self, KeyQueryFile) Modified: trunk/opy2on/opy2on.py =================================================================== --- trunk/opy2on/opy2on.py 2009-08-08 13:38:17 UTC (rev 153) +++ trunk/opy2on/opy2on.py 2009-08-09 00:05:14 UTC (rev 154) @@ -198,4 +198,6 @@ glob.keydb.save() glob.datdb.save() glob.nodedb.save() +glob.datquery.save() +glob.keyquery.save() glob.logger.popup("GLOBAL", "Finished Completely") From o2on-svn @ lists.sourceforge.jp Sun Aug 9 09:31:10 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 09 Aug 2009 09:31:10 +0900 Subject: [o2on-svn] [155] throw Exception which contain dat filename. Message-ID: <1249777870.402739.31677.nullmailer@users.sourceforge.jp> Revision: 155 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=155 Author: nawota Date: 2009-08-09 09:31:10 +0900 (Sun, 09 Aug 2009) Log Message: ----------- throw Exception which contain dat filename. Modified Paths: -------------- trunk/opy2on/lib/o2on_key.py Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-08-09 00:05:14 UTC (rev 154) +++ trunk/opy2on/lib/o2on_key.py 2009-08-09 00:31:10 UTC (rev 155) @@ -59,7 +59,8 @@ except UnicodeDecodeError, inst: try: first = first.decode('euc_jp').encode('utf-8') - except UnicodeDecodeError, inst: raise inst + except UnicodeDecodeError, inst: + raise Exception("Couldn't decode first line %s" % dat.datpath()) m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) if m: self.title = m.group(1) #reg = re.compile(r'^(?:2ch\.net|machibbs\.com)/(?:cyugoku|hokkaidou|k(?:an(?:a|to)|inki|(?:ousinet|yusy)u)|o(?:(?:kinaw|sak)a)|sikoku|t(?:a(?:(?:m|war)a)|o(?:kyo|u(?:hoku|kai))))') From o2on-svn @ lists.sourceforge.jp Sun Aug 9 09:54:55 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 09 Aug 2009 09:54:55 +0900 Subject: [o2on-svn] [156] Ignore "Connection reset by peer" error. Message-ID: <1249779295.981722.28314.nullmailer@users.sourceforge.jp> Revision: 156 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=156 Author: nawota Date: 2009-08-09 09:54:55 +0900 (Sun, 09 Aug 2009) Log Message: ----------- Ignore "Connection reset by peer" error. Modified Paths: -------------- trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-09 00:31:10 UTC (rev 155) +++ trunk/opy2on/lib/o2on_server.py 2009-08-09 00:54:55 UTC (rev 156) @@ -87,7 +87,7 @@ if isinstance(inst, socket.error): if hasattr(inst, 'errno'): errno = inst.errno # 2.6 else: errno = inst[0] # 2.5 - if errno in (104, 32, 110): + if errno in (104, 32, 110, 54): pass else: if o2on_config.OutputErrorFile: From o2on-svn @ lists.sourceforge.jp Sun Aug 9 13:52:40 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 09 Aug 2009 13:52:40 +0900 Subject: [o2on-svn] [157] Fix finish time error. Message-ID: <1249793560.620852.12688.nullmailer@users.sourceforge.jp> Revision: 157 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=157 Author: nawota Date: 2009-08-09 13:52:40 +0900 (Sun, 09 Aug 2009) Log Message: ----------- Fix finish time error. Modified Paths: -------------- trunk/opy2on/lib/o2on_job.py Modified: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py 2009-08-09 00:54:55 UTC (rev 156) +++ trunk/opy2on/lib/o2on_job.py 2009-08-09 04:52:40 UTC (rev 157) @@ -295,6 +295,7 @@ self.glob.datquery.semap.release() def dojob(self, nodedb, logger, prof, datdb, datq): d = datq.pop() + if not d: return if datdb.has_keyhash(d.hash): datq.save() return @@ -354,7 +355,7 @@ self.glob.keyquery.semap.release() def dojob(self, nodedb, logger, prof, datdb, datq): k = self.glob.keyquery.pop() - if self.finish: return + if not k: return node = nodedb[k.nodeid] if not node: node = o2on_node.Node(k.nodeid, k.ip, k.port) logger.log("DATQUERY","dat query %s to %s" % (hexlify(k.hash),hexlify(node.id))) From o2on-svn @ lists.sourceforge.jp Sun Aug 9 14:06:20 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 09 Aug 2009 14:06:20 +0900 Subject: [o2on-svn] [158] Fix typo in README. Message-ID: <1249794380.751848.1215.nullmailer@users.sourceforge.jp> Revision: 158 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=158 Author: nawota Date: 2009-08-09 14:06:20 +0900 (Sun, 09 Aug 2009) Log Message: ----------- Fix typo in README. Modified Paths: -------------- trunk/opy2on/README Modified: trunk/opy2on/README =================================================================== --- trunk/opy2on/README 2009-08-09 04:52:40 UTC (rev 157) +++ trunk/opy2on/README 2009-08-09 05:06:20 UTC (rev 158) @@ -23,7 +23,7 @@ - 重いぞ!という方は RecordProfile を True にしてください。ProfileDir 下にプロファイルが出力されます。 - ポートを開けたり閉めたりする。 -- ./opy2on で起動。 +- ./opy2on.py で起動。 操作 From o2on-svn @ lists.sourceforge.jp Sun Aug 9 15:09:13 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 09 Aug 2009 15:09:13 +0900 Subject: [o2on-svn] [159] Use errno pacakge instead of direct number. Message-ID: <1249798153.349392.20822.nullmailer@users.sourceforge.jp> Revision: 159 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=159 Author: nawota Date: 2009-08-09 15:09:13 +0900 (Sun, 09 Aug 2009) Log Message: ----------- Use errno pacakge instead of direct number. Modified Paths: -------------- trunk/opy2on/lib/o2on_job.py trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_node.py trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py 2009-08-09 05:06:20 UTC (rev 158) +++ trunk/opy2on/lib/o2on_job.py 2009-08-09 06:09:13 UTC (rev 159) @@ -15,6 +15,7 @@ import httplib import traceback import sys +import errno import o2on_server import o2on_config @@ -44,7 +45,7 @@ try: os.makedirs(o2on_config.ProfileDir) except OSError, inst: - if inst.errno != 17: raise inst + if inst.errno != errno.EEXIST: raise inst profname = os.path.join(o2on_config.ProfileDir, "o2on_"+"_".join(self.name.split(" "))+".prof") cProfile.runctx('self.dummy()', None, {'self':self,}, profname) Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-08-09 05:06:20 UTC (rev 158) +++ trunk/opy2on/lib/o2on_key.py 2009-08-09 06:09:13 UTC (rev 159) @@ -15,6 +15,7 @@ import datetime import hashlib import time +import errno from o2on_const import KeyDBFile, regHosts import o2on_config @@ -145,7 +146,7 @@ pipe.close() proc.wait() except IOError, inst: - if inst.errno == 32:pass # Broken pipe + if inst.errno == errno.EPIPE:pass # Broken pipe else: raise inst self.glob.logger.log("KEYDB", "Finished to show keys") def key_list(self): Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2009-08-09 05:06:20 UTC (rev 158) +++ trunk/opy2on/lib/o2on_node.py 2009-08-09 06:09:13 UTC (rev 159) @@ -16,6 +16,7 @@ import threading import random import time +from errno import EHOSTUNREACH, ECONNREFUSED, ETIMEDOUT, ECONNRESET import o2on_config from o2on_util import hash_xor_bitlength @@ -171,8 +172,8 @@ errno = None if hasattr(inst, 'errno'): errno = inst.errno else: errno = inst[0] - if errno in (113, 111): raise NodeRemovable - if errno in (110, 104): raise NodeRefused + if errno in (EHOSTUNREACH, ETIMEDOUT): raise NodeRemovable + if errno in (ECONNREFUSED, ECONNRESET): raise NodeRefused else: raise inst except httplib.BadStatusLine: socket.setdefaulttimeout(None) Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-09 05:06:20 UTC (rev 158) +++ trunk/opy2on/lib/o2on_server.py 2009-08-09 06:09:13 UTC (rev 159) @@ -21,6 +21,7 @@ from xml.parsers.expat import ExpatError import threading import select +from errno import ECONNRESET, EPIPE, ETIMEDOUT import o2on_config from o2on_const import regHosts, ProtocolVer, AppName @@ -65,7 +66,7 @@ self.handle_request() self.__is_shut_down.set() def shutdown(self): - for r in []:#self.requests: + for r in self.requests: try: r.shutdown(socket.SHUT_RDWR) r.close() @@ -87,7 +88,7 @@ if isinstance(inst, socket.error): if hasattr(inst, 'errno'): errno = inst.errno # 2.6 else: errno = inst[0] # 2.5 - if errno in (104, 32, 110, 54): + if errno in (ECONNRESET, EPIPE, ETIMEDOUT): pass else: if o2on_config.OutputErrorFile: @@ -709,7 +710,7 @@ self.send_nav(cur) def datq(self, args): datq = self.server.glob.datquery - self.send_common("im", "Searching Dats") + self.send_common("datq", "Searching Dats") self.wfile.write("""\

検索中dat

From o2on-svn @ lists.sourceforge.jp Sun Aug 9 15:17:31 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 09 Aug 2009 15:17:31 +0900 Subject: [o2on-svn] [160] Replace unsupported SJIS charcter. Message-ID: <1249798651.903559.29565.nullmailer@users.sourceforge.jp> Revision: 160 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=160 Author: nawota Date: 2009-08-09 15:17:31 +0900 (Sun, 09 Aug 2009) Log Message: ----------- Replace unsupported SJIS charcter. Modified Paths: -------------- trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-08-09 06:09:13 UTC (rev 159) +++ trunk/opy2on/lib/o2on_key.py 2009-08-09 06:17:31 UTC (rev 160) @@ -56,7 +56,7 @@ self.size = len(data) first = data.split("\n",1)[0] try: - first = first.decode('cp932').encode('utf-8') + first = first.replace("\x86\xa6", "\x81E").decode('cp932').encode('utf-8') except UnicodeDecodeError, inst: try: first = first.decode('euc_jp').encode('utf-8') Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-09 06:09:13 UTC (rev 159) +++ trunk/opy2on/lib/o2on_server.py 2009-08-09 06:17:31 UTC (rev 160) @@ -196,7 +196,7 @@ m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) if not m: return "" try: - first = first.decode('cp932').encode('utf-8') + first = first.replace("\x86\xa6", "\x81E").decode('cp932').encode('utf-8') except UnicodeDecodeError, inst: try: first = first.decode('euc_jp').encode('utf-8') From o2on-svn @ lists.sourceforge.jp Sun Aug 9 21:15:14 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 09 Aug 2009 21:15:14 +0900 Subject: [o2on-svn] [161] Fix Exception catching error. Message-ID: <1249820114.105619.13675.nullmailer@users.sourceforge.jp> Revision: 161 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=161 Author: nawota Date: 2009-08-09 21:15:14 +0900 (Sun, 09 Aug 2009) Log Message: ----------- Fix Exception catching error. KeyDB's lock is now RLock Change Timeout style. Modified Paths: -------------- trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_node.py trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-08-09 06:17:31 UTC (rev 160) +++ trunk/opy2on/lib/o2on_key.py 2009-08-09 12:15:14 UTC (rev 161) @@ -64,9 +64,6 @@ raise Exception("Couldn't decode first line %s" % dat.datpath()) m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) if m: self.title = m.group(1) - #reg = re.compile(r'^(?:2ch\.net|machibbs\.com)/(?:cyugoku|hokkaidou|k(?:an(?:a|to)|inki|(?:ousinet|yusy)u)|o(?:(?:kinaw|sak)a)|sikoku|t(?:a(?:(?:m|war)a)|o(?:kyo|u(?:hoku|kai))))') - #if re.compile("^\s*$").match(self.title) and \ - # not reg.match(dat.path()): print dat.datpath() self.url = "http://xxx.%s/test/read.cgi/%s/%s/" % (dat.domain, dat.board, dat.datnum) @@ -111,7 +108,7 @@ class KeyDB: def __init__(self, g): - self.lock = threading.Lock() + self.lock = threading.RLock() with self.lock: self.keys = {} self.lenmap = {} @@ -188,8 +185,8 @@ with self.lock: for k in self.keys.values(): if k.nodeid == nid: removes.append(k) - for k in removes: - self.remove(k) + for k in removes: + self.remove(k) def remove(self,k): with self.lock: del self.keys[k.idkeyhash()] Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2009-08-09 06:17:31 UTC (rev 160) +++ trunk/opy2on/lib/o2on_node.py 2009-08-09 12:15:14 UTC (rev 161) @@ -157,9 +157,10 @@ with self.lock: conn = httplib.HTTPConnection(self.ip, self.port) try: - socket.setdefaulttimeout(o2on_config.SocketTimeout) + socket.setdefaulttimeout(15) conn.connect() socket.setdefaulttimeout(None) + conn.sock.settimeout(o2on_config.SocketTimeout) conn.request(method,path,body, headers) r = conn.getresponse() conn.close() @@ -432,7 +433,7 @@ else: try: r = nt.ping() - except NodeRemovable, NodeRefused: + except (NodeRemovable, NodeRefused): r = False if r: self.boardmap[board].append(self.boardmap[board][0]) @@ -508,7 +509,7 @@ n = self.nodes[self.KBuckets[bitlen][0]] try: r = n.ping() - except NodeRemovable, NodeRefused: + except (NodeRemovable, NodeRefused): r = False if r: del self.KBuckets[bitlen][0] Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-09 06:17:31 UTC (rev 160) +++ trunk/opy2on/lib/o2on_server.py 2009-08-09 12:15:14 UTC (rev 161) @@ -68,7 +68,7 @@ def shutdown(self): for r in self.requests: try: - r.shutdown(socket.SHUT_RDWR) + #r.shutdown(socket.SHUT_RDWR) r.close() except Exception: pass @@ -215,13 +215,13 @@ conn = self.get_connection() r= conn.getresponse() conn.close() - except socket.timeout: + except (socket.timeout, socket.gaierror, socket.error): r = None data = None header = None - if r: + if True: logger.log("PROXY", "\tresponse %s" % r.status) - if ut != self.URLTYPE_OFFLAW and r.status in (200,206,304): + if r and ut != self.URLTYPE_OFFLAW and r.status in (200,206,304): logger.log("PROXY", "\tgot response from server") data = r.read() if r.getheader("content-encoding") == "gzip": @@ -313,13 +313,14 @@ else: logger.popup("PROXY", "no cached dat. query for the dat.\n%s" % \ self.datkey()) - data = r.read() - self.wfile.write("HTTP/%d.%d %d %s\r\n" % - (r.version/10,r.version%10,r.status,r.reason)) - self.wfile.write(self.msg(r)) - self.wfile.write("\r\n") - self.wfile.write(data) - self.wfile.close() + if r: + data = r.read() + self.wfile.write("HTTP/%d.%d %d %s\r\n" % + (r.version/10,r.version%10,r.status,r.reason)) + self.wfile.write(self.msg(r)) + self.wfile.write("\r\n") + self.wfile.write(data) + self.wfile.close() try: conn = self.get_connection(['If-Modified-Since', 'Range', 'User-Agent']) From o2on-svn @ lists.sourceforge.jp Sun Aug 9 21:21:34 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 09 Aug 2009 21:21:34 +0900 Subject: [o2on-svn] [162] Ignore socket.gaierror in normal_proxy() Message-ID: <1249820494.074751.21303.nullmailer@users.sourceforge.jp> Revision: 162 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=162 Author: nawota Date: 2009-08-09 21:21:34 +0900 (Sun, 09 Aug 2009) Log Message: ----------- Ignore socket.gaierror in normal_proxy() Modified Paths: -------------- trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-09 12:15:14 UTC (rev 161) +++ trunk/opy2on/lib/o2on_server.py 2009-08-09 12:21:34 UTC (rev 162) @@ -167,7 +167,7 @@ conn = self.get_connection() r= conn.getresponse() conn.close() - except socket.timeout: + except (socket.timeout, socket.error, socket.gaierror): return self.wfile.write("HTTP/%d.%d %d %s\r\n" % (r.version/10,r.version%10,r.status,r.reason)) From o2on-svn @ lists.sourceforge.jp Thu Aug 13 09:57:06 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Thu, 13 Aug 2009 09:57:06 +0900 Subject: [o2on-svn] =?utf-8?b?WzE2M10gRklYOiAgIzE4MTY5IG9weTJvbuOCkuaWsA==?= =?utf-8?b?44OQ44O844K444On44Oz44Go6Kqk6KqN44GZ44KL?= Message-ID: <1250125026.711467.20320.nullmailer@users.sourceforge.jp> Revision: 163 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=163 Author: osa_p Date: 2009-08-13 09:57:06 +0900 (Thu, 13 Aug 2009) Log Message: ----------- FIX: #18169 opy2on???????吾??潟?茯よ???? Ticket Links: :----------- http://sourceforge.jp/projects/o2on/tracker/detail/18169 Modified Paths: -------------- trunk/o2on/src.o2on/O2Node.h trunk/o2on/src.o2on/O2NodeDB.cpp trunk/o2on/src.o2on/O2Protocol.h Modified: trunk/o2on/src.o2on/O2Node.h =================================================================== --- trunk/o2on/src.o2on/O2Node.h 2009-08-09 12:21:34 UTC (rev 162) +++ trunk/o2on/src.o2on/O2Node.h 2009-08-13 00:57:06 UTC (rev 163) @@ -41,6 +41,9 @@ pubT pubkey; //local wstring ua; + wstring proto_ver; + wstring app_name; + wstring app_ver; uint status; wstring flags; time_t lastlink; Modified: trunk/o2on/src.o2on/O2NodeDB.cpp =================================================================== --- trunk/o2on/src.o2on/O2NodeDB.cpp 2009-08-09 12:21:34 UTC (rev 162) +++ trunk/o2on/src.o2on/O2NodeDB.cpp 2009-08-13 00:57:06 UTC (rev 163) @@ -53,12 +53,15 @@ O2NodeDB:: touch_preprocessor(O2Node &node) { - // O2/0.2 (o2on/0.02.0027; Win32) - if (node.ua.size() > 13) { - wstring node_ver = node.ua.substr(13, 9); - if (wcscmp(node_ver.c_str(), ver) > 0) + if (wcscmp(node.app_name.c_str(), _T(APP_NAME)) == 0) { + // ????????????????? + wchar_t tmpW[64]; + swprintf_s(tmpW, 64, L"%1d.%02d.%04d", APP_VER_MAJOR, APP_VER_MINOR, APP_BUILDNO); + + if (wcscmp(node.app_ver.c_str(), tmpW) > 0) NewVerDetectionFlag = true; } + if (node.port == 0) { AddPort0Node(node); return false; Modified: trunk/o2on/src.o2on/O2Protocol.h =================================================================== --- trunk/o2on/src.o2on/O2Protocol.h 2009-08-09 12:21:34 UTC (rev 162) +++ trunk/o2on/src.o2on/O2Protocol.h 2009-08-13 00:57:06 UTC (rev 163) @@ -13,6 +13,7 @@ #include "httpheader.h" #include "O2Profile.h" #include "O2Node.h" +#include "O2Version.h" @@ -230,6 +231,51 @@ return false; ascii2unicode(it->second, node.ua); + // ?????????????????????+ // O2/0.2 (o2on/0.02.0027; Win32) + // O2/0.2 (opy2on/0.00.0001; Linux x86_64) + //???O2/?????????? (????/????????; ???)??ョ????? + size_t ProtoNamePos = node.ua.find(L"O2/", 0); + if (ProtoNamePos != 0) { + // ????????????? + return false; + } + size_t AppNamePeriodPos = node.ua.find(L" ", ProtoNamePos); + if ( AppNamePeriodPos == wstring::npos ) { + // ????????????????????? + return false; + } + node.proto_ver = node.ua.substr( 3, AppNamePeriodPos - 3 ); + size_t AppNameStartPos = node.ua.find(L"(", AppNamePeriodPos); + if (( AppNameStartPos == wstring::npos ) || ((AppNameStartPos + 1) == node.ua.size() )) { + // ???????????????? + return false; + } + AppNameStartPos++; + size_t AppNameEndPos = node.ua.find(L"/", AppNameStartPos); + if ( AppNameEndPos == wstring::npos ) { + // ????????u??????? + return false; + } + node.app_name = node.ua.substr( AppNameStartPos, AppNameEndPos - AppNameStartPos ); + if ((wcscmp(node.app_name.c_str(), _T(APP_NAME)) == 0) && ((AppNameEndPos + 1) < node.ua.size() )) { + // ????????????????? + AppNameEndPos++; + size_t VerEndPos = node.ua.find(L";", AppNameEndPos); + if ( VerEndPos == wstring::npos ) { + // ???????????u??????? + return false; + } + node.app_ver = node.ua.substr(AppNameEndPos, VerEndPos - AppNameEndPos); + } +#ifdef O2DEBUG + else { + wchar_t tmpW[128]; + swprintf_s(tmpW, 128, L"??????:%s\n", node.app_name.c_str()); + TRACEW(tmpW); + } +#endif + return true; } From o2on-svn @ lists.sourceforge.jp Sun Aug 16 16:11:49 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 16 Aug 2009 16:11:49 +0900 Subject: [o2on-svn] [164] logger.popup() fallback to logger.log() when dbus is not supported. Message-ID: <1250406709.550238.4759.nullmailer@users.sourceforge.jp> Revision: 164 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=164 Author: nawota Date: 2009-08-16 16:11:49 +0900 (Sun, 16 Aug 2009) Log Message: ----------- logger.popup() fallback to logger.log() when dbus is not supported. Modified Paths: -------------- trunk/opy2on/lib/o2on_util.py Modified: trunk/opy2on/lib/o2on_util.py =================================================================== --- trunk/opy2on/lib/o2on_util.py 2009-08-13 00:57:06 UTC (rev 163) +++ trunk/opy2on/lib/o2on_util.py 2009-08-16 07:11:49 UTC (rev 164) @@ -44,11 +44,14 @@ print l def popup(self, categ, s): if o2on_config.UseDBus: - if not self.bus: self.bus = dbus.SessionBus() - obj = self.bus.get_object("org.freedesktop.Notifications", - "/org/freedesktop/Notifications") - obj.Notify(AppName, 0, '', AppName+" "+categ,s, [], {}, -1, - dbus_interface="org.freedesktop.Notifications") + try: + if not self.bus: self.bus = dbus.SessionBus() + obj = self.bus.get_object("org.freedesktop.Notifications", + "/org/freedesktop/Notifications") + obj.Notify(AppName, 0, '', AppName+" "+categ,s, [], {}, -1, + dbus_interface="org.freedesktop.Notifications") + except dbus.exceptions.DBusException: + self.log(categ,s) else: self.log(categ, s) From o2on-svn @ lists.sourceforge.jp Sun Aug 16 16:12:03 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 16 Aug 2009 16:12:03 +0900 Subject: [o2on-svn] [165] Change default Timeout to 300 Message-ID: <1250406723.040617.6003.nullmailer@users.sourceforge.jp> Revision: 165 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=165 Author: nawota Date: 2009-08-16 16:12:02 +0900 (Sun, 16 Aug 2009) Log Message: ----------- Change default Timeout to 300 Modified Paths: -------------- trunk/opy2on/o2on_config.py.sample Modified: trunk/opy2on/o2on_config.py.sample =================================================================== --- trunk/opy2on/o2on_config.py.sample 2009-08-16 07:11:49 UTC (rev 164) +++ trunk/opy2on/o2on_config.py.sample 2009-08-16 07:12:02 UTC (rev 165) @@ -27,7 +27,7 @@ # DatSaveAsGzip = True # 接続タイムアウト (単位: 秒) -SocketTimeout = 20 +SocketTimeout = 300 # このノードの名前 今のところ日本語(ascii以外)はうまく動かない。 NodeName = "" From o2on-svn @ lists.sourceforge.jp Sun Aug 16 22:28:38 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 16 Aug 2009 22:28:38 +0900 Subject: [o2on-svn] [166] Decoding is now compatible with o2on. Message-ID: <1250429318.848130.12478.nullmailer@users.sourceforge.jp> Revision: 166 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=166 Author: nawota Date: 2009-08-16 22:28:38 +0900 (Sun, 16 Aug 2009) Log Message: ----------- Decoding is now compatible with o2on. Modified Paths: -------------- trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-08-16 07:12:02 UTC (rev 165) +++ trunk/opy2on/lib/o2on_key.py 2009-08-16 13:28:38 UTC (rev 166) @@ -16,6 +16,7 @@ import hashlib import time import errno +import codecs from o2on_const import KeyDBFile, regHosts import o2on_config @@ -23,6 +24,14 @@ import o2on_dat import o2on_node +def my_replace_handler(inst): + return ((u"\u30fb", inst.start+2)) + +try: + codecs.lookup_error('opy2on_replace') +except LookupError: + codecs.register_error('opy2on_replace', my_replace_handler) + class Key: def __init__(self): self.hash = None @@ -56,12 +65,12 @@ self.size = len(data) first = data.split("\n",1)[0] try: - first = first.replace("\x86\xa6", "\x81E").decode('cp932').encode('utf-8') + first = first.decode('cp932').encode('utf-8') except UnicodeDecodeError, inst: try: first = first.decode('euc_jp').encode('utf-8') except UnicodeDecodeError, inst: - raise Exception("Couldn't decode first line %s" % dat.datpath()) + first = first.decode('cp932','opy2on_replace').encode('utf-8') m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) if m: self.title = m.group(1) self.url = "http://xxx.%s/test/read.cgi/%s/%s/" % (dat.domain, Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-16 07:12:02 UTC (rev 165) +++ trunk/opy2on/lib/o2on_server.py 2009-08-16 13:28:38 UTC (rev 166) @@ -22,6 +22,7 @@ import threading import select from errno import ECONNRESET, EPIPE, ETIMEDOUT +import codecs import o2on_config from o2on_const import regHosts, ProtocolVer, AppName @@ -32,6 +33,14 @@ import o2on_im import o2on_util +def my_replace_handler(inst): + return ((u"\u30fb", inst.start+2)) + +try: + codecs.lookup_error('opy2on_replace') +except LookupError: + codecs.register_error('opy2on_replace', my_replace_handler) + class O2ONServer(BaseHTTPServer.HTTPServer): def __init__(self, handler, port, g): BaseHTTPServer.HTTPServer.__init__(self, @@ -68,7 +77,7 @@ def shutdown(self): for r in self.requests: try: - #r.shutdown(socket.SHUT_RDWR) + r.shutdown(socket.SHUT_RDWR) r.close() except Exception: pass @@ -196,11 +205,12 @@ m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) if not m: return "" try: - first = first.replace("\x86\xa6", "\x81E").decode('cp932').encode('utf-8') + first = first.decode('cp932').encode('utf-8') except UnicodeDecodeError, inst: try: first = first.decode('euc_jp').encode('utf-8') - except UnicodeDecodeError, inst: raise inst + except UnicodeDecodeError, inst: + first = first.decode('cp932','opy2on_replace').encode('utf-8') m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) if not m: return "" return m.group(1) From o2on-svn @ lists.sourceforge.jp Sun Aug 16 22:39:44 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 16 Aug 2009 22:39:44 +0900 Subject: [o2on-svn] [167] Don't respond when proxied server doesn't respond. Message-ID: <1250429984.023714.23840.nullmailer@users.sourceforge.jp> Revision: 167 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=167 Author: nawota Date: 2009-08-16 22:39:43 +0900 (Sun, 16 Aug 2009) Log Message: ----------- Don't respond when proxied server doesn't respond. Modified Paths: -------------- trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-08-16 13:28:38 UTC (rev 166) +++ trunk/opy2on/lib/o2on_server.py 2009-08-16 13:39:43 UTC (rev 167) @@ -178,6 +178,7 @@ conn.close() except (socket.timeout, socket.error, socket.gaierror): return + if not r: return self.wfile.write("HTTP/%d.%d %d %s\r\n" % (r.version/10,r.version%10,r.status,r.reason)) self.wfile.write(self.msg(r)) From o2on-svn @ lists.sourceforge.jp Sat Sep 5 08:17:48 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 05 Sep 2009 08:17:48 +0900 Subject: [o2on-svn] =?utf-8?b?WzE2OF0gIGJvYXJkbWFwIOOBq+WtmOWcqOOBl+OBqg==?= =?utf-8?b?44GESUTjgYznmbvpjLLjgZXjgozjgabjgYTjgovloLTlkIjjgavlr77lh6Y=?= Message-ID: <1252106268.219986.7811.nullmailer@users.sourceforge.jp> Revision: 168 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=168 Author: nawota Date: 2009-09-05 08:17:48 +0900 (Sat, 05 Sep 2009) Log Message: ----------- boardmap に存在しないIDが登録されている場合に対処 Modified Paths: -------------- trunk/opy2on/lib/o2on_node.py Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2009-08-16 13:39:43 UTC (rev 167) +++ trunk/opy2on/lib/o2on_node.py 2009-09-04 23:17:48 UTC (rev 168) @@ -340,7 +340,7 @@ self.glob = glob self.KBuckets = [] self.port0nodes = [] - self.lock = threading.Lock() + self.lock = threading.RLock() for x in range(0,160): self.KBuckets.append([]) self.nodes = dict() self.boardmap = dict() @@ -402,7 +402,11 @@ if len(self.boardmap[board])==0: del self.boardmap[board] return [] - return map(lambda x: self.nodes[x], self.boardmap[board]) + res = [] + for x in self.boardmap[board]: + n = self.nodes.get(x) + if n: res.append(n) + return res def get_random_board(self): with self.lock: if len(self.boardmap) == 0: return None @@ -421,13 +425,12 @@ r = None with self.lock: if not board in self.boardmap: - self.boardmap[board] = [n.id] + self.boardmap[board] = [n.id, ] elif len(self.boardmap[board])<10: self.boardmap[board].append(n.id) else: nt = self.nodes.get(self.boardmap[board][0]) if not nt: - raise Exception del self.boardmap[board][0] self.boardmap[board].append(n.id) else: From o2on-svn @ lists.sourceforge.jp Sat Sep 5 21:43:29 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 05 Sep 2009 21:43:29 +0900 Subject: [o2on-svn] =?utf-8?b?WzE2OV0gIC0gYm9hcmRtYXAg44Gr5a2Y5Zyo44GX44Gq?= =?utf-8?b?44GESUTjgYznmbvpjLLjgZXjgozjgabjgYTjgozjgbDliYrpmaTjgpLooYw=?= =?utf-8?b?44Gq44GG44KI44GG44Gr?= Message-ID: <1252154609.110380.2375.nullmailer@users.sourceforge.jp> Revision: 169 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=169 Author: nawota Date: 2009-09-05 21:43:29 +0900 (Sat, 05 Sep 2009) Log Message: ----------- - boardmap に存在しないIDが登録されていれば削除を行なうように - publishmap に key が登録されていない場合に対処 Modified Paths: -------------- trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_node.py Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-09-04 23:17:48 UTC (rev 168) +++ trunk/opy2on/lib/o2on_key.py 2009-09-05 12:43:29 UTC (rev 169) @@ -201,8 +201,10 @@ del self.keys[k.idkeyhash()] l = hash_xor_bitlength(self.glob.prof.mynode.id, k.hash) self.lenmap[l].remove(k.idkeyhash()) - self.publishmap[k.published].remove(k.idkeyhash()) - self.datkeymap[k.hash].remove(k.idkeyhash()) + if self.publishmap[k.published].get(k.idkeyhash()): + self.publishmap[k.published].remove(k.idkeyhash()) + if self.datkeymap[k.hash].get(k.idkeyhash()): + self.datkeymap[k.hash].remove(k.idkeyhash()) if len(self.lenmap[l]) == 0: del self.lenmap[l] if len(self.publishmap[k.published])==0: del self.publishmap[k.published] if len(self.datkeymap[k.hash])==0: del self.datkeymap[k.hash] Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2009-09-04 23:17:48 UTC (rev 168) +++ trunk/opy2on/lib/o2on_node.py 2009-09-05 12:43:29 UTC (rev 169) @@ -403,9 +403,14 @@ del self.boardmap[board] return [] res = [] + rem = [] for x in self.boardmap[board]: n = self.nodes.get(x) if n: res.append(n) + else: rem.append(x) + for x in rem: + self.boardmap[board].remove(x) + if len(self.boardmap[board]) == 0: del self.boardmap[board] return res def get_random_board(self): with self.lock: From o2on-svn @ lists.sourceforge.jp Sat Sep 5 21:47:03 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sat, 05 Sep 2009 21:47:03 +0900 Subject: [o2on-svn] [170] Fix last commit. Message-ID: <1252154823.093112.6653.nullmailer@users.sourceforge.jp> Revision: 170 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=170 Author: nawota Date: 2009-09-05 21:47:03 +0900 (Sat, 05 Sep 2009) Log Message: ----------- Fix last commit. Modified Paths: -------------- trunk/opy2on/lib/o2on_key.py Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-09-05 12:43:29 UTC (rev 169) +++ trunk/opy2on/lib/o2on_key.py 2009-09-05 12:47:03 UTC (rev 170) @@ -201,9 +201,9 @@ del self.keys[k.idkeyhash()] l = hash_xor_bitlength(self.glob.prof.mynode.id, k.hash) self.lenmap[l].remove(k.idkeyhash()) - if self.publishmap[k.published].get(k.idkeyhash()): + if k.idkeyhash() in self.publishmap[k.published]: self.publishmap[k.published].remove(k.idkeyhash()) - if self.datkeymap[k.hash].get(k.idkeyhash()): + if k.idkeyhash() in self.datkeymap[k.hash]: self.datkeymap[k.hash].remove(k.idkeyhash()) if len(self.lenmap[l]) == 0: del self.lenmap[l] if len(self.publishmap[k.published])==0: del self.publishmap[k.published] From o2on-svn @ lists.sourceforge.jp Sun Sep 13 13:39:10 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Sun, 13 Sep 2009 13:39:10 +0900 Subject: [o2on-svn] [171] parse first line before encoding. Message-ID: <1252816750.720429.22525.nullmailer@users.sourceforge.jp> Revision: 171 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=171 Author: nawota Date: 2009-09-13 13:39:10 +0900 (Sun, 13 Sep 2009) Log Message: ----------- parse first line before encoding. return 404 when client_address could not get. Modified Paths: -------------- trunk/opy2on/lib/o2on_key.py trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_key.py =================================================================== --- trunk/opy2on/lib/o2on_key.py 2009-09-05 12:47:03 UTC (rev 170) +++ trunk/opy2on/lib/o2on_key.py 2009-09-13 04:39:10 UTC (rev 171) @@ -64,6 +64,9 @@ data = dat.data() self.size = len(data) first = data.split("\n",1)[0] + m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) + if m: first = m.group(1) + else: self.title = None try: first = first.decode('cp932').encode('utf-8') except UnicodeDecodeError, inst: @@ -71,8 +74,7 @@ first = first.decode('euc_jp').encode('utf-8') except UnicodeDecodeError, inst: first = first.decode('cp932','opy2on_replace').encode('utf-8') - m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) - if m: self.title = m.group(1) + self.title = first self.url = "http://xxx.%s/test/read.cgi/%s/%s/" % (dat.domain, dat.board, dat.datnum) @@ -174,7 +176,8 @@ res = [] for x in sorted(self.publishmap.keys()): for y in self.publishmap[x]: - res.append(self.keys[y]) + tmp = self.keys.get(y) + if tmp: res.append(tmp) return res def published(self, idkeyhash, publish_time): if len(idkeyhash) != 20: raise Exception Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-09-05 12:47:03 UTC (rev 170) +++ trunk/opy2on/lib/o2on_server.py 2009-09-13 04:39:10 UTC (rev 171) @@ -386,8 +386,6 @@ if ".." in datpath: return self.response_400("datpath include ..") if not dat.setpath(datpath): return self.response_400("invalid datpath") if not dat.save(data): - logger.log("P2PSERVER", - "I don't like this omiyage dat %s" % self.client_address[0]) return self.response_400("invalid omiyage") else: self.server.glob.datdb.add_dat(dat) @@ -573,7 +571,7 @@ def response_400(self, reason=""): logger = self.server.glob.logger logger.log("P2PSERVER", - "response 400 %s (%s)" % (self.client_address[0], reason)) + "response 400 %s (%s)" % (hexlify(self.client_address[0]), reason)) logger.log("P2PSERVER", "\tpath was %s" % self.path) logger.log("P2PSERVER", "\theader was %s" % self.headers) header = common_header.copy() @@ -582,7 +580,6 @@ self.wfile.write("\r\n") self.wfile.close() def response_404(self): - #print "p2p server response 404 %s" % self.client_address[0] header = common_header.copy() self.wfile.write("HTTP/1.0 404 Not Found\r\n") for h in header: self.wfile.write("%s: %s\r\n" % (h,header[h])) @@ -598,6 +595,8 @@ def do_POST(self): self.do_GET() def do_GET(self): + if not self.client_address: return self.response_404() + self.server.glob.logger.log("P2PSERVER", "connection came %s" % (self.path)) nid = self.headers.getheader('X-O2-Node-ID') From o2on-svn @ lists.sourceforge.jp Tue Sep 22 20:39:48 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Tue, 22 Sep 2009 20:39:48 +0900 Subject: [o2on-svn] =?utf-8?q?=5B172=5D_ReCheckIP=2C__ForceShutdown_?= =?utf-8?b?44KS6L+95Yqg?= Message-ID: <1253619588.158711.13138.nullmailer@users.sourceforge.jp> Revision: 172 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=172 Author: nawota Date: 2009-09-22 20:39:48 +0900 (Tue, 22 Sep 2009) Log Message: ----------- ReCheckIP, ForceShutdown を追加 Modified Paths: -------------- trunk/opy2on/lib/o2on_job.py trunk/opy2on/lib/o2on_node.py trunk/opy2on/o2on_config.py.sample trunk/opy2on/opy2on.py Modified: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py 2009-09-13 04:39:10 UTC (rev 171) +++ trunk/opy2on/lib/o2on_job.py 2009-09-22 11:39:48 UTC (rev 172) @@ -34,6 +34,10 @@ self.name = name self.glob = g self.sec = s + self.node = None + def shutdown(self): + if self.node: self.node.shutdown() + self.node = None def stop(self): self.finish = True def wakeup(self): @@ -61,8 +65,10 @@ self.glob.logger.log("JOBMANAGER", "job %s started" % self.name) while not self.finish: #t = time.time() + self.node = None self.dojob(self.glob.nodedb, self.glob.logger, self.glob.prof, self.glob.datdb, self.glob.datquery) + self.node = None if self.finish: break diff = int(self.sec) #int(self.sec - (time.time()-t)) if 0 Revision: 173 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=173 Author: nawota Date: 2009-09-30 17:47:33 +0900 (Wed, 30 Sep 2009) Log Message: ----------- Proxy Server: POST に対応 Admin Server: /dats を実装 lib/o2on_dat: class dat に title を取得する関数 title を追加 強制終了時に強制終了するスレッドの名前を表示 Modified Paths: -------------- trunk/opy2on/lib/o2on_dat.py trunk/opy2on/lib/o2on_server.py trunk/opy2on/opy2on.py Modified: trunk/opy2on/lib/o2on_dat.py =================================================================== --- trunk/opy2on/lib/o2on_dat.py 2009-09-22 11:39:48 UTC (rev 172) +++ trunk/opy2on/lib/o2on_dat.py 2009-09-30 08:47:33 UTC (rev 173) @@ -52,6 +52,27 @@ return os.path.join(o2on_config.DatDir, self.domain, self.board, self.datnum[:4],self.datnum+".dat") raise Exception + def title(self): + dp = self.datpath() + if os.path.exists(dp): f=open(dp) + elif os.path.exists(dp+".gz"): f=gzip.GzipFile(dp+".gz") + else: f=None + if f: + first=f.readline() + f.close() + m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) + if not m: return None + try: + first = first.decode('cp932').encode('utf-8') + except UnicodeDecodeError, inst: + try: + first = first.decode('euc_jp').encode('utf-8') + except UnicodeDecodeError, inst: + first = first.decode('cp932','opy2on_replace').encode('utf-8') + m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) + if not m: return None + return m.group(1) + return None def data(self): dp = self.datpath() if os.path.exists(dp): f=open(dp) Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-09-22 11:39:48 UTC (rev 172) +++ trunk/opy2on/lib/o2on_server.py 2009-09-30 08:47:33 UTC (rev 173) @@ -162,7 +162,7 @@ for r in remove: if r in hr: del hr[r] conn = httplib.HTTPConnection(loc) - conn.request("GET",p.path, None, hr) + conn.request(self.command, p.path, self.rfile.read(), hr) return conn def msg(self,r): res = '' @@ -215,13 +215,17 @@ m = re.compile(r'^.*<>.*<>.*<>.*<>(.*)$').match(first) if not m: return "" return m.group(1) + def do_POST(self): + self.do_GET() def do_GET(self): logger = self.server.glob.logger logger.log("PROXY", "proxy requested %s" % self.path) ut = self.urltype() if ut in (self.URLTYPE_UNKNOWN, self.URLTYPE_NORMAL, self.URLTYPE_MACHI): + # 普通のプロキシとして動作 self.normal_proxy() return + # datがリクエストされた try: conn = self.get_connection() r= conn.getresponse() @@ -235,6 +239,7 @@ if r and ut != self.URLTYPE_OFFLAW and r.status in (200,206,304): logger.log("PROXY", "\tgot response from server") data = r.read() + # gzipをdecode if r.getheader("content-encoding") == "gzip": sf = StringIO.StringIO(data) dec = gzip.GzipFile(fileobj=sf) @@ -891,6 +896,50 @@
""") self.wfile.write(self.html_footer) + def dats(self, args): + self.send_common("dats", "Dat") + if len(args)==0: + regBoard = re.compile(r'^\t[^\t.]+\.([^\t]+)\t([^\t]+)\t([^\t]+)$') + inul = False + f=open(o2on_config.Path2channel_brd) + for line in f: + m = regBoard.match(line) + if m: + boardname = m.group(3).decode('cp932').encode('utf-8') + if os.path.exists(os.path.join(o2on_config.DatDir, + m.group(1), m.group(2))): + self.wfile.write("
  • %s
  • \n" % \ + (m.group(1), m.group(2), boardname)) + else: + self.wfile.write("
  • %s
  • \n" % boardname) + else: + pos = line.find("\t") + if pos == -1: continue + if inul: self.wfile.write("\n") + else: self.wfile.write("
      \n") + inul = True + self.wfile.write("
    • %s
        \n" % \ + (line[:pos].decode("cp932").encode('utf-8'))) + f.close() + self.wfile.write("
      \n") + self.wfile.write(self.html_footer) + elif len(args)==2: + regdat = re.compile('^(\d+)\.dat(?:\.gz)?$') + self.wfile.write("") + for root,dirs,files in os.walk(os.path.join(o2on_config.DatDir, + args[0], + args[1])): + for f in files: + m=regdat.match(f) + path = args[0]+"/"+args[1]+"/"+m.group(1) + dkhash = o2on_util.datkeyhash(path) + dat = self.server.glob.datdb.get(dkhash) + if dat: + self.wfile.write("\n" % \ + (dat.title() or "Unknown Title", + args[0], args[1], m.group(1))) + self.wfile.write("
      スレタイURL
      %shttp://xxx.%s/test/read.cgi/%s/%s/
      \n") + self.wfile.write(self.html_footer) def status(self,args): self.send_common("status", "Status Summary") glob = self.server.glob Modified: trunk/opy2on/opy2on.py =================================================================== --- trunk/opy2on/opy2on.py 2009-09-22 11:39:48 UTC (rev 172) +++ trunk/opy2on/opy2on.py 2009-09-30 08:47:33 UTC (rev 173) @@ -196,7 +196,7 @@ shutcount += 1 if o2on_config.ForceShutdown != None and \ o2on_config.ForceShutdown / 8 < shutcount: - glob.logger.popup("GLOBAL", "Force Shutdown") + glob.logger.popup("GLOBAL", "Force Shutdown %s" % j.name) j.shutdown() c += 1 glob.logger.log("GLOBAL", "Finished %d/%d" % (c, n)) From o2on-svn @ lists.sourceforge.jp Wed Sep 30 17:55:54 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Wed, 30 Sep 2009 17:55:54 +0900 Subject: [o2on-svn] =?utf-8?b?WzE3NF0gUHJveHkgU2VydmVyOiAgUE9TVCDjgavlr74=?= =?utf-8?b?5b+c5pmC44Gu44OQ44Kw44KS5L+u5q2j?= Message-ID: <1254300954.591732.17459.nullmailer@users.sourceforge.jp> Revision: 174 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=174 Author: nawota Date: 2009-09-30 17:55:54 +0900 (Wed, 30 Sep 2009) Log Message: ----------- Proxy Server: POST に対応時のバグを修正 Modified Paths: -------------- trunk/opy2on/lib/o2on_server.py Modified: trunk/opy2on/lib/o2on_server.py =================================================================== --- trunk/opy2on/lib/o2on_server.py 2009-09-30 08:47:33 UTC (rev 173) +++ trunk/opy2on/lib/o2on_server.py 2009-09-30 08:55:54 UTC (rev 174) @@ -162,7 +162,9 @@ for r in remove: if r in hr: del hr[r] conn = httplib.HTTPConnection(loc) - conn.request(self.command, p.path, self.rfile.read(), hr) + if self.command == "GET": body = None + else: body=self.rfile.read() + conn.request(self.command, p.path, body, hr) return conn def msg(self,r): res = '' From o2on-svn @ lists.sourceforge.jp Wed Sep 30 18:07:22 2009 From: o2on-svn @ lists.sourceforge.jp (o2on svn commit) Date: Wed, 30 Sep 2009 18:07:22 +0900 Subject: [o2on-svn] =?utf-8?b?WzE3NV0gIFJlY2hlY2tJUOOBjOOBhuOBvuOBj+WLlQ==?= =?utf-8?b?44GE44Gm44GE44Gq44GL44Gj44Gf44OQ44Kw44KS5L+u5q2j?= Message-ID: <1254301642.568265.7156.nullmailer@users.sourceforge.jp> Revision: 175 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=175 Author: nawota Date: 2009-09-30 18:07:22 +0900 (Wed, 30 Sep 2009) Log Message: ----------- RecheckIPがうまく動いていなかったバグを修正 Modified Paths: -------------- trunk/opy2on/lib/o2on_job.py Modified: trunk/opy2on/lib/o2on_job.py =================================================================== --- trunk/opy2on/lib/o2on_job.py 2009-09-30 08:55:54 UTC (rev 174) +++ trunk/opy2on/lib/o2on_job.py 2009-09-30 09:07:22 UTC (rev 175) @@ -172,18 +172,17 @@ self.glob.logger.log("GETIP", inst) else: if r: - if not prof.mynode.ip: - ip = o2on_node.e2ip(r[:8]) - if not regLocalIP.match(ip): - prof.mynode.ip = ip - if o2on_config.ReCheckIP == None: - self.finish = True - else: - self.sec = o2on_config.ReCheckIP * 60 + ip = o2on_node.e2ip(r[:8]) + if not regLocalIP.match(ip): + if o2on_config.ReCheckIP == None: + self.finish = True + else: + self.sec = o2on_config.ReCheckIP * 60 + if prof.mynode.ip != ip: logger.popup("GETIP","Got Global IP %s" % ip) - nodes.add_node(n) - break - else: break + prof.mynode.ip = ip + nodes.add_node(n) + break class AskNodeCollectionThread(JobThread): def __init__(self, g):