[24823] users/jberry/mpwa
source_changes at macosforge.org
source_changes at macosforge.org
Sat May 5 15:02:51 PDT 2007
Revision: 24823
http://trac.macosforge.org/projects/macports/changeset/24823
Author: jberry at macports.org
Date: 2007-05-05 15:02:50 -0700 (Sat, 05 May 2007)
Log Message:
-----------
mpwa: use multiple file 64K file blobs per file to avoid in memory issues and mysql buffer sizing issues.
Modified Paths:
--------------
users/jberry/mpwa/app/controllers/port_pkg_file_controller.rb
users/jberry/mpwa/app/models/port_pkg.rb
users/jberry/mpwa/app/models/port_pkg_file.rb
users/jberry/mpwa/doc/schema-thoughts.sql
users/jberry/mpwa/lib/mpwa-conf.rb
Modified: users/jberry/mpwa/app/controllers/port_pkg_file_controller.rb
===================================================================
--- users/jberry/mpwa/app/controllers/port_pkg_file_controller.rb 2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/app/controllers/port_pkg_file_controller.rb 2007-05-05 22:02:50 UTC (rev 24823)
@@ -51,6 +51,7 @@
def emit
file = PortPkgFile.find(params[:id])
- send_data file.file_blob.data, :filename => file.file_path, :type => file.mime_type, :disposition => 'inline'
+ send_data file.data, :filename => file.file_path, :type => file.mime_type, :disposition => 'inline'
end
+
end
Modified: users/jberry/mpwa/app/models/port_pkg.rb
===================================================================
--- users/jberry/mpwa/app/models/port_pkg.rb 2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/app/models/port_pkg.rb 2007-05-05 22:02:50 UTC (rev 24823)
@@ -109,11 +109,11 @@
meta.categories.each { |c| self.add_tag(c) }
# Save unpacked data into a file
- self.files << PortPkgFile.from_path(pkgPath, tempDirPath, :mimetype => 'application/x-macports-portpkg')
+ self.files << PortPkgFile.from_path(self, pkgPath, tempDirPath, :mimetype => 'application/vnd.macports.portpkg')
# Save files from the expanded package
expandedPkgPath.find do |p|
- self.files << PortPkgFile.from_path(p, tempDirPath) if p.file?
+ self.files << PortPkgFile.from_path(self, p, tempDirPath) if p.file?
end
# Save the pkg (maybe we shouldn't?)
Modified: users/jberry/mpwa/app/models/port_pkg_file.rb
===================================================================
--- users/jberry/mpwa/app/models/port_pkg_file.rb 2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/app/models/port_pkg_file.rb 2007-05-05 22:02:50 UTC (rev 24823)
@@ -1,3 +1,5 @@
+require 'stringio'
+
require 'file_blob'
require 'mpwa-conf'
@@ -3,8 +5,8 @@
class PortPkgFile < ActiveRecord::Base
belongs_to :port_pkg
- has_one :file_blob
+ #has_many :file_blob -- we don't use this association to avoid keeping many blobs in memory
- def PortPkgFile.from_path(path, path_root = nil, options = {})
- port_pkg_file = PortPkgFile.new
+ def PortPkgFile.from_path(port_pkg, path, path_root = nil, options = {})
+ port_pkg_file = PortPkgFile.new(:port_pkg => port_pkg)
return port_pkg_file.read_from_path(path, path_root, options)
end
@@ -18,24 +20,63 @@
mimetype = options[:mimetype] || PortPkgFile.mimetype_from_path(path)
reported_path = options[:filename] || path_root.nil? ?
path.to_s : Pathname.new(path).relative_path_from(path_root).to_s
+
File.open(path, "r") { |f| read_from_file(f, :path => reported_path, :mimetype => mimetype) }
return self
end
def read_from_file(file, options = {})
- # Create a new blob
- blob = FileBlob.new
- blob.read(file)
- self.file_blob = blob
-
- # Save other file information
+ # Save file meta information
self.file_path = options[:path]
- self.length = blob.data.length
+ self.length = 0
self.mime_type = options[:mimetype] || 'application/octet-stream'
- self.md5 = Digest::MD5.hexdigest(blob.data)
- self.sha256 = Digest::SHA256.hexdigest(blob.data)
+ # Save so that we get a primary id for the blob associations
+ self.save
+ # Create digesters for our digests
+ md5 = Digest::MD5.new
+ sha256 = Digest::SHA256.new
+
+ # Read the file, creating blobs of data as we go
+ buf = ''
+ length = 0
+ seq = 0
+ while (file.read(MPWA::MAX_BLOB_SIZE, buf))
+ # Update the digests
+ md5.update buf
+ sha256.update buf
+
+ # Create a new bob
+ blob = FileBlob.create(:port_pkg_file => self, :data => buf, :sequence => seq)
+ length = length + buf.length
+
+ seq = seq + 1
+ end
+
+ # Finish up
+ self.md5 = md5.hexdigest
+ self.sha256 = sha256.hexdigest
+ self.length = length
+
+ self.save
return self
end
+
+ def write_to_file(file)
+ seq = 0
+ while (blob = FileBlob.find(:first, :conditions => "port_pkg_file_id=#{self.id} and sequence=#{seq}"))
+ file.write(blob.data)
+ seq = seq + 1
+ end
+ end
+
+ def data()
+ StringIO.open("rw") do |f|
+ write_to_file(f)
+ f.rewind
+ f.read
+ end
+ end
+
end
Modified: users/jberry/mpwa/doc/schema-thoughts.sql
===================================================================
--- users/jberry/mpwa/doc/schema-thoughts.sql 2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/doc/schema-thoughts.sql 2007-05-05 22:02:50 UTC (rev 24823)
@@ -72,6 +72,7 @@
create table File_Blobs (
id bigint not null primary key auto_increment,
port_pkg_file_id bigint not null,
+ sequence int not null,
data blob
);
Modified: users/jberry/mpwa/lib/mpwa-conf.rb
===================================================================
--- users/jberry/mpwa/lib/mpwa-conf.rb 2007-05-05 19:33:16 UTC (rev 24822)
+++ users/jberry/mpwa/lib/mpwa-conf.rb 2007-05-05 22:02:50 UTC (rev 24823)
@@ -2,4 +2,6 @@
FILETOOL = "/usr/bin/file"
XARTOOL = "/opt/local/bin/xar"
PORTTOOL = "/opt/local/bin/port"
+
+ MAX_BLOB_SIZE = 1024*64
end
\ No newline at end of file
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/macports-changes/attachments/20070505/a4793927/attachment.html
More information about the macports-changes
mailing list