Tatsuki SUGIURA
sugi****@users*****
2006年 7月 12日 (水) 20:41:45 JST
Index: slashjp/plugins/Blob/Blob.pm diff -u slashjp/plugins/Blob/Blob.pm:1.3 slashjp/plugins/Blob/Blob.pm:1.4 --- slashjp/plugins/Blob/Blob.pm:1.3 Fri Dec 31 21:35:51 2004 +++ slashjp/plugins/Blob/Blob.pm Wed Jul 12 20:41:45 2006 @@ -1,7 +1,7 @@ # This code is a part of Slash, and is released under the GPL. -# Copyright 1997-2004 by Open Source Development Network. See README +# Copyright 1997-2005 by Open Source Technology Group. See README # and COPYING for more information, or see http://slashcode.com/. -# $Id: Blob.pm,v 1.3 2004/12/31 12:35:51 oliver Exp $ +# $Id: Blob.pm,v 1.4 2006/07/12 11:41:45 sugi Exp $ package Slash::Blob; @@ -9,40 +9,29 @@ use DBIx::Password; use Slash; use Slash::Utility; +use MIME::Types; use Digest::MD5 'md5_hex'; use vars qw($VERSION); use base 'Exporter'; use base 'Slash::DB::Utility'; -($VERSION) = ' $Revision: 1.3 $ ' =~ /\$Revision:\s+([^\s]+)/; +($VERSION) = ' $Revision: 1.4 $ ' =~ /\$Revision:\s+([^\s]+)/; -# Mime/Type hash (couldn't find a module that I liked that would do this -Brian -# there are plenty of other methods out there, this needs to be replaced -- pudge -my %mimetypes = ( - jpeg => 'image/jpeg', - jpg => 'image/jpeg', - gif => 'image/gif', - png => 'image/png', - tiff => 'image/tiff', - tif => 'image/tiff', - ps => 'application/postscript', - eps => 'application/postscript', - zip => 'application/zip', - doc => 'application/msword', - xls => 'application/ms-excel', - pdf => 'application/pdf', - gz => 'application/x-gzip', - bz2 => 'application/x-bzip2', - rpm => 'application/x-rpm', +# When this plugin was first written, it used a hardcoded hash to +# store MIME types. Now we use the MIME::Types module. But for +# backwards compatibility, here are the overrides for just the +# four types where our hardcoded hash differed from the (current) +# values returned by MIME::Types. Honestly my guess is that +# MIME::Types is right and we were wrong, and we should remove +# this (except for 'text'), but for now, let's make it completely +# backwards compatible. + +my %mimetype_overrides = ( mp3 => 'audio/mp3', - ra => 'audio/x-realaudio', - html => 'text/html', - htm => 'text/html', - txt => 'text/plain', + rpm => 'application/x-rpm', text => 'text/plain', - xml => 'text/xml', - rtf => 'text/rtf', + xls => 'application/ms-excel', ); sub new { @@ -67,10 +56,14 @@ my $prime = $self->{'_prime'}; $values->{seclev} ||= 0; - # Couldn't find a module that did this if (!$values->{content_type} && $values->{filename}) { (my $ext = lc $values->{filename}) =~ s/^.*\.([^.]+)$/$1/s; - $values->{content_type} = $mimetypes{$ext}; + if ($mimetype_overrides{$ext}) { + $values->{content_type} = $mimetype_overrides{$ext}; + } else { + my $m = MIME::Types->new(); + $values->{content_type} = $m->mimeTypeOf($values->{filename}) if $m; + } } $values->{content_type} ||= 'application/octet-stream'; @@ -83,7 +76,87 @@ $self->sqlDo("UPDATE $self->{'_table'} SET reference_count=(reference_count +1) WHERE id = '$found'"); } else { $values->{$prime} = $id; - $self->sqlInsert($table, $values); + + # if the size of the data is greater than the size of the max + # packet MySQL can accept, let's set it higher before saving the + # data -- pudge + + my $len = length $values->{data}; + my $var = 'max_allowed_packet'; + my $value; + + my $base = 1024**2; # 1MB + if ($len > $base) { + $value = $self->sqlGetVar($var); + my $needed = $len + $base; + + if ($value < $needed) { + return unless $self->sqlSetVar($var, $needed*2); + my $check = $self->sqlGetVar($var); + if ($check < $needed) { + errorLog("Value of $var is $check, should be $needed\n"); + return undef; + } + + # easily turn off for testing + my $do_chunk = 1; + my($size, $data); + + # chunking will be used here because 1. on old + # 3.x client lib, you cannot set the max packet size + # larger, and 2. sometimes the MySQL server "goes + # away" with a lot of data anyway. although in such + # cases, we have trouble *getting* the data back, + # but this is a problem for another day. -- pudge + if ($do_chunk) { + # smarter? + $size = $base >= $value ? $base/2 : $base; + $data = $values->{data}; + $values->{data} = substr($data, 0, $size, ''); + } + + $self->sqlInsert($table, $values) or return undef; + + if ($do_chunk) { + while (length $data) { + my $chunk = $self->sqlQuote(substr($data, 0, $size, '')); + my $ok = $self->sqlUpdate($table, { + -data => "CONCAT(data, $chunk)" + }, $where + ); + + if (!$ok) { # abort + $self->sqlDelete($table, $where); + return undef; + } + } + } + + # the new value is only session-specific anyway, + # but set it back for good measure + $self->sqlSetVar($var, $value); + + } else { + undef $value; + } + } + + # true $value means we already saved the data + unless ($value) { + $self->sqlInsert($table, $values) or return undef; + } + + # verify we saved what we think we did + # (note: even when we cannot retrieve all the data we saved, + # the MD5() check still works, so the data is all there; maybe + # some other MySQL setting about getting large amounts of data + # is giving us problems in the tests -- pudge + my $md5 = $self->sqlSelect('MD5(data)', $table, $where); + unless ($md5 eq $id) { + errorLog("md5:$md5 != id:$id\n"); + $self->sqlDelete($table, $where); + return undef; + } } return $found || $id ; @@ -148,7 +221,8 @@ data => $values->{data}, }; - my $id = $self->create($content); + my $id = $self->create($content) or return undef; + my $content_type = $self->get($id, 'content_type'); my $file_content = { Index: slashjp/plugins/Blob/PLUGIN diff -u slashjp/plugins/Blob/PLUGIN:1.2 slashjp/plugins/Blob/PLUGIN:1.3 --- slashjp/plugins/Blob/PLUGIN:1.2 Fri Dec 24 05:13:34 2004 +++ slashjp/plugins/Blob/PLUGIN Wed Jul 12 20:41:45 2006 @@ -1,4 +1,4 @@ -# $Id: PLUGIN,v 1.2 2004/12/23 20:13:34 oliver Exp $ +# $Id: PLUGIN,v 1.3 2006/07/12 11:41:45 sugi Exp $ name=Blob description="Generic binary storage" mysql_schema=mysql_schema Index: slashjp/plugins/Blob/README diff -u /dev/null slashjp/plugins/Blob/README:1.1.2.1 --- /dev/null Wed Jul 12 20:41:45 2006 +++ slashjp/plugins/Blob/README Wed Jul 12 20:41:45 2006 @@ -0,0 +1 @@ +This plugin requires the MIME::Types perl module. Index: slashjp/plugins/Blob/blob.pl diff -u slashjp/plugins/Blob/blob.pl:1.3 slashjp/plugins/Blob/blob.pl:1.4 --- slashjp/plugins/Blob/blob.pl:1.3 Fri Dec 31 21:35:51 2004 +++ slashjp/plugins/Blob/blob.pl Wed Jul 12 20:41:45 2006 @@ -1,8 +1,8 @@ #!/usr/bin/perl -w # This code is a part of Slash, and is released under the GPL. -# Copyright 1997-2004 by Open Source Development Network. See README +# Copyright 1997-2005 by Open Source Technology Group. See README # and COPYING for more information, or see http://slashcode.com/. -# $Id: blob.pl,v 1.3 2004/12/31 12:35:51 oliver Exp $ +# $Id: blob.pl,v 1.4 2006/07/12 11:41:45 sugi Exp $ use strict; use Slash; @@ -32,6 +32,11 @@ content_type => $data->{content_type}, filename => $data->{filename}, do_etag => 1, + dis_type => 'inline', # best to default to inline, + # users can choose to download + # if they wish, and most file + # types will auto-download anyway, + # even if set to inline content => $data->{data} }); } Index: slashjp/plugins/Blob/blobLoader diff -u slashjp/plugins/Blob/blobLoader:1.2 slashjp/plugins/Blob/blobLoader:1.3 --- slashjp/plugins/Blob/blobLoader:1.2 Fri Dec 24 05:13:34 2004 +++ slashjp/plugins/Blob/blobLoader Wed Jul 12 20:41:45 2006 @@ -1,8 +1,8 @@ #!/usr/bin/perl -w # This code is a part of Slash, and is released under the GPL. -# Copyright 1997-2004 by Open Source Development Network. See README +# Copyright 1997-2005 by Open Source Technology Group. See README # and COPYING for more information, or see http://slashcode.com/. -# $Id: blobLoader,v 1.2 2004/12/23 20:13:34 oliver Exp $ +# $Id: blobLoader,v 1.3 2006/07/12 11:41:45 sugi Exp $ use strict; use File::Basename; @@ -13,7 +13,7 @@ use vars qw( $slashdb $werder $constants $junk ); -(my $VERSION) = ' $Revision: 1.2 $ ' =~ /\$Revision:\s+([^\s]+)/; +(my $VERSION) = ' $Revision: 1.3 $ ' =~ /\$Revision:\s+([^\s]+)/; my $PROGNAME = basename($0); my %opts; @@ -32,24 +32,22 @@ exit(0); } -my ($file, $data); -open($file, $opts{f}); -while (<$file>) { +my ($fh, $data); +open($fh, $opts{f}); +while (<$fh>) { $data .= $_; } -close($file); +close($fh); my $content = { - seclev => $opts{s}, - content_type => $opts{c}, - data => $data, + seclev => $opts{'s'}, + content_type => $opts{c} || undef, + data => $data, + filename => $opts{f}, }; -$content->{filename} = $opts{f} unless $opts{c}; - my $id = $blobs->create($content); -print "Stored in $id\n"; - +print "$id $opts{f}\n"; sub usage { print "*** $_[0]\n" if $_[0]; @@ -79,7 +77,7 @@ $PROGNAME $VERSION This code is a part of Slash, and is released under the GPL. -Copyright 1997-2004 by Open Source Development Network. See README +Copyright 1997-2005 by Open Source Technology Group. See README and COPYING for more information, or see http://slashcode.com/. EOT Index: slashjp/plugins/Blob/clean_blobs.pl diff -u slashjp/plugins/Blob/clean_blobs.pl:1.2 slashjp/plugins/Blob/clean_blobs.pl:1.3 --- slashjp/plugins/Blob/clean_blobs.pl:1.2 Fri Dec 24 05:13:34 2004 +++ slashjp/plugins/Blob/clean_blobs.pl Wed Jul 12 20:41:45 2006 @@ -1,6 +1,6 @@ #!/usr/bin/perl -w -# $Id: clean_blobs.pl,v 1.2 2004/12/23 20:13:34 oliver Exp $ +# $Id: clean_blobs.pl,v 1.3 2006/07/12 11:41:45 sugi Exp $ use strict; use Slash::Constants ':slashd'; Index: slashjp/plugins/Blob/dump diff -u slashjp/plugins/Blob/dump:1.2 slashjp/plugins/Blob/dump:1.3 --- slashjp/plugins/Blob/dump:1.2 Fri Dec 24 05:13:34 2004 +++ slashjp/plugins/Blob/dump Wed Jul 12 20:41:45 2006 @@ -1,5 +1,5 @@ # -# $Id: dump,v 1.2 2004/12/23 20:13:34 oliver Exp $ +# $Id: dump,v 1.3 2006/07/12 11:41:45 sugi Exp $ # INSERT INTO vars (name, value, description) VALUES ('story_files', '1', 'Allow stories to use files from Slash::Blob'); Index: slashjp/plugins/Blob/fileadmin.pl diff -u slashjp/plugins/Blob/fileadmin.pl:1.3 slashjp/plugins/Blob/fileadmin.pl:1.4 --- slashjp/plugins/Blob/fileadmin.pl:1.3 Fri Dec 31 21:35:51 2004 +++ slashjp/plugins/Blob/fileadmin.pl Wed Jul 12 20:41:45 2006 @@ -1,8 +1,8 @@ #!/usr/bin/perl -w # This code is a part of Slash, and is released under the GPL. -# Copyright 1997-2004 by Open Source Development Network. See README +# Copyright 1997-2005 by Open Source Technology Group. See README # and COPYING for more information, or see http://slashcode.com/. -# $Id: fileadmin.pl,v 1.3 2004/12/31 12:35:51 oliver Exp $ +# $Id: fileadmin.pl,v 1.4 2006/07/12 11:41:45 sugi Exp $ use strict; use Slash 2.003; @@ -12,7 +12,7 @@ use vars qw($VERSION); -($VERSION) = ' $Revision: 1.3 $ ' =~ /\$Revision:\s+([^\s]+)/; +($VERSION) = ' $Revision: 1.4 $ ' =~ /\$Revision:\s+([^\s]+)/; sub main { my $slashdb = getCurrentDB(); @@ -106,7 +106,7 @@ local $/; $data = <$fh>; } - $form->{file_content} =~ s|^.+?([^/:\\]+)$|$1|; + $form->{file_content} =~ s|^.*?([^/:\\]+)$|$1|; } my $content = { @@ -118,7 +118,13 @@ description => $form->{description}, }; - $blobdb->createFileForStory($content); + unless ($blobdb->createFileForStory($content)) { + print "<p><b>File not saved, check logs for errors</b><br>"; + printf "Filename: %s, description: %s, data size: %dK</p>", + strip_literal($form->{file_content}), + strip_literal($form->{description}), + length($data) / 1024; + } } if ($form->{delete}) { Index: slashjp/plugins/Blob/mysql_schema diff -u slashjp/plugins/Blob/mysql_schema:1.3 slashjp/plugins/Blob/mysql_schema:1.4 --- slashjp/plugins/Blob/mysql_schema:1.3 Fri Dec 31 21:35:51 2004 +++ slashjp/plugins/Blob/mysql_schema Wed Jul 12 20:41:45 2006 @@ -1,5 +1,5 @@ # -# $Id: mysql_schema,v 1.3 2004/12/31 12:35:51 oliver Exp $ +# $Id: mysql_schema,v 1.4 2006/07/12 11:41:45 sugi Exp $ # # @@ -24,7 +24,7 @@ DROP TABLE IF EXISTS story_files; CREATE TABLE story_files ( id INT(5) NOT NULL AUTO_INCREMENT, - stoid MEDIUMINT UNSIGNED NOT NULL DEFAULT '', + stoid MEDIUMINT UNSIGNED NOT NULL DEFAULT '0', description VARCHAR(80) NOT NULL DEFAULT '', file_id VARCHAR(32) NOT NULL DEFAULT '', isimage ENUM('no', 'yes') DEFAULT 'no' NOT NULL,
SRAD