myexperiment-hackers
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[myexperiment-hackers] [2844] branches/datasets: Changed data set zippin


From: noreply
Subject: [myexperiment-hackers] [2844] branches/datasets: Changed data set zipping to occur in memory
Date: Fri, 25 Nov 2011 06:39:55 -0500 (EST)

Revision
2844
Author
fbacall
Date
2011-11-25 06:39:54 -0500 (Fri, 25 Nov 2011)

Log Message

Changed data set zipping to occur in memory

Modified Paths

Diff

Modified: branches/datasets/app/controllers/application.rb (2843 => 2844)


--- branches/datasets/app/controllers/application.rb	2011-11-24 14:18:59 UTC (rev 2843)
+++ branches/datasets/app/controllers/application.rb	2011-11-25 11:39:54 UTC (rev 2844)
@@ -1191,10 +1191,4 @@
     end
 
   end
-
-  #For sending zip files stored in Tempfile objects, ensures they are closed and can be garbage collected
-  def send_and_close(file, filename)
-    send_file file.path, :disposition => 'attachment', :filename => filename
-    file.close
-  end
 end

Modified: branches/datasets/app/controllers/data_sets_controller.rb (2843 => 2844)


--- branches/datasets/app/controllers/data_sets_controller.rb	2011-11-24 14:18:59 UTC (rev 2843)
+++ branches/datasets/app/controllers/data_sets_controller.rb	2011-11-25 11:39:54 UTC (rev 2844)
@@ -50,7 +50,8 @@
   end
 
   def download
-    send_and_close @data_set.create_zip(current_user), @data_set.archive_file_name
+    send_data @data_set.create_zip(current_user).string, :disposition => "attachment",
+              :filename => @data_set.archive_file_name
   end
 
   def new

Modified: branches/datasets/app/models/data_set.rb (2843 => 2844)


--- branches/datasets/app/models/data_set.rb	2011-11-24 14:18:59 UTC (rev 2843)
+++ branches/datasets/app/models/data_set.rb	2011-11-25 11:39:54 UTC (rev 2844)
@@ -10,7 +10,7 @@
 
   acts_as_site_entity
 
-  validates_presence_of :title
+  validates_presence_of :title, :workflow, :contributor
 
   format_attribute :description
 
@@ -24,26 +24,34 @@
   #this association is just to ensure text data is deleted when the data set is. not actually used.
   has_many :text_datas, :dependent => :destroy
 
-  # Zips the pack up and returns the file
+  validates_inclusion_of :category, :in => ["example_data"] #Need some more categories!
+
+  # Zips the pack up and returns a StringIO object containing its contents
   def create_zip(user)
-    stats = {"input" => {:text => 0, :files => 0, :hidden => 0},
+    # Some stats to be included in the metadata file
+    stats = {"input" =>  {:text => 0, :files => 0, :hidden => 0},
              "output" => {:text => 0, :files => 0, :hidden => 0}}
 
     #Create the zip file
-    file = new_zipfile(archive_file_name) do |zipfile|
-      #Add the data
+    new_zip(StringIO.new) do |zipfile|
+
+      #Add each data item to the zip. Inputs/outputs are seperated into folders. Each input/output datum is named as
+      # the port it relates to, followed by a dash, followed by either the name of the file if it is a blob,
+      # or "text.txt" if it is just text data.
+
       relationships.each do |data_item|
         data = ""
         port = data_item.objekt
+
         if data.kind_of?(Blob)
           if Authorization.is_authorized?("download", nil, data, user)
-            zipfile.add_file("#{port.port_type}s/#{port.name} - #{data.local_name}", data.content_blob.data)
+            zipfile.add_data("#{port.port_type}s/#{port.name} - #{data.local_name}", data.content_blob.data)
             stats[port.port_type][:files] += 1
           else
             stats[port.port_type][:hidden] += 1
           end
         elsif data.kind_of?(TextData)
-          zipfile.add_file("#{port.port_type}s/#{port.name} - text.txt", data.data)
+          zipfile.add_data("#{port.port_type}s/#{port.name} - text.txt", data.data)
           stats[port.port_type][:text] += 1
         else
           raise "Unsupported data type"
@@ -51,10 +59,8 @@
       end
 
       #Add metadata
-      zipfile.add_file("_metadata.txt", self.metadata(stats))
+      zipfile.add_data("_metadata.txt", self.metadata(stats))
     end
-
-    file
   end
 
   def archive_file_name(no_timestamp=false)

Modified: branches/datasets/app/views/data_sets/show.rhtml (2843 => 2844)


--- branches/datasets/app/views/data_sets/show.rhtml	2011-11-24 14:18:59 UTC (rev 2843)
+++ branches/datasets/app/views/data_sets/show.rhtml	2011-11-25 11:39:54 UTC (rev 2844)
@@ -9,7 +9,7 @@
     <ul class="sectionIcons">
       <% if Authorization.is_authorized?("download", nil, @workflow, current_user) %>
         <li>
-          <%= icon('download', download_workflow_data_set_path(@workflow, @data_set), nil, nil, 'Download data set') %>
+          <%= icon('download', download_workflow_data_set_path(@workflow, @data_set), nil, nil, 'Download as a zip file') %>
         </li>
       <% end %>
       <% if mine?(@workflow) %>

Modified: branches/datasets/lib/zip_in_memory.rb (2843 => 2844)


--- branches/datasets/lib/zip_in_memory.rb	2011-11-24 14:18:59 UTC (rev 2843)
+++ branches/datasets/lib/zip_in_memory.rb	2011-11-25 11:39:54 UTC (rev 2844)
@@ -4,30 +4,45 @@
 # See license.txt for details.
 
 
-# An easier way of making zip files without excessive use of the file system.
+# An easier way of making zip files without necessarily using the file system.
+# This avoids issues to do with deleting files after sending to the client, without deleting them before they are
+#  fully sent.
 # Adapted from from:
 # http://blog.devinterface.com/2010/02/create-zip-files-on-the-fly/
+# and
+# http://stackoverflow.com/questions/4797315/rails-on-the-fly-streaming-of-output-in-zip-format
 
 require 'zip/zip'
 
 module ZipInMemory
 
-  # "name" not important, as Tempfile is always unique
-  def new_zipfile(name)
-    t = Tempfile.new(name)
-    Zip::ZipOutputStream.open(t.path) do |z|
+  # Takes an IO object, eg. a stream.
+  # Writes the zip file to the object and returns it back.
+  def new_zip(io)
+    Zip::IOOutputStream.open(io) do |z|
       yield z
     end
-    t
   end
 
 end
 
 module Zip
-  class ZipOutputStream
-    def add_file(title, data)
-      put_next_entry title
-      print data
+  # Extension to ZipOutputStream to make it take a generic IO object rather than a file path.
+  class IOOutputStream < ZipOutputStream
+    def initialize io
+      super '-'
+      @outputStream = io
     end
+
+    def stream
+      @outputStream
+    end
+
+    # A quick way to add data to the zip file. Takes the title of the file to be written to the archive
+    # (can include directories, which will be created where necessary), and the data for the file.
+    def add_data(title, data)
+      self.put_next_entry title
+      self << data
+    end
   end
 end

reply via email to

[Prev in Thread] Current Thread [Next in Thread]