Simplify Export/Import APIs and Jobs

app/api/chemotion/export_api.rb

 module Chemotion
   class ExportAPI < Grape::API
+
     resource :exports do
 
-      before do
-        # TODO: validate collection_id, check permissions
-        # handle nested collections
-        @collection_ids = params[:collections]
-        @format = params[:format]
+      desc "Create export job"
+      params do
+        requires :collections, type: Array[Integer]
+        requires :format, type: String
+        requires :nested, type: Boolean
+      end
+      post do
+        collection_ids = params[:collections].uniq
+        format = params[:format]
+        nested = params[:nested] == true
+
+
+        # check if the user is allowed to export these collections
+        collection_ids.each do |collection_id|
+          begin
+            collection = Collection.belongs_to_or_shared_by(current_user.id, current_user.group_ids).find(collection_id)
+          rescue ActiveRecord::RecordNotFound
+            error!('401 Unauthorized', 401)
+          end
+        end
+
+        # create an id for the export
+        export_id = SecureRandom.uuid
+
+        # create the lock file
+        lock_file_path = Export::ExportCollections.lock_file_path(export_id, format)
+        File.open(lock_file_path, 'w') {}
+
+        # run the asyncronous export job and return its id to the client
+        ExportCollectionsJob.perform_later(export_id, collection_ids, format, nested)
+
+        # return the export_id to the client
+        return {:export_id => export_id}
       end
 
       desc "Poll export job"
@@ -14,19 +43,19 @@ module Chemotion
         requires :id, type: String
       end
       get '/:id' do
-        job_id = params[:id]
+        export_id = params[:id]
 
-        # look for the export file
-        ['json', 'zip'].each do |fmt|
-          file_name = File.join('public', fmt, "#{job_id}.#{fmt}")
-          lock_file_name = file_name + '.lock'
+        # look for the lock file file
+        ['json', 'zip'].each do |format|
+          file_path = Export::ExportCollections.file_path(export_id, format)
+          lock_file_path = Export::ExportCollections.lock_file_path(export_id, format)
 
-          if File.exist?(file_name) and !File.exist?(lock_file_name)
+          if File.exist?(file_path) and !File.exist?(lock_file_path)
             return {
               :status => 'COMPLETED',
-              :url => "/#{fmt}/#{job_id}.#{fmt}"
+              :url => Export::ExportCollections.file_url(export_id, format)
             }
-          elsif File.exist?(lock_file_name)
+          elsif File.exist?(lock_file_path)
             return {:status => 'EXECUTING'}
           end
         end
@@ -34,15 +63,6 @@ module Chemotion
         error! :not_found, 404
       end
 
-      desc "Create export job"
-      params do
-        requires :collections, type: Array[Integer]
-        requires :format, type: String
-      end
-      post do
-        ExportCollectionsJob.perform_later(@format, @collection_ids)
     end
   end
-
-  end
 end

app/api/chemotion/import_api.rb

@@ -3,24 +3,20 @@ module Chemotion
     resource :imports do
 
       post do
-        # create the import job in order to have the job id
-        job = ImportCollectionsJob.new
+        # create an id for the import
+        import_id = SecureRandom.uuid
 
-        # create a directory `tmp/imports/<job_id>` for this import
-        directory = File.join('tmp', 'import', job.job_id)
-        FileUtils.mkdir_p(directory) unless File.directory?(directory)
-
-        # store the file in the `tmp/imports/<job_id>` dir
-        file_path = File.join(directory, params[:file][:filename])
+        # store the file as `tmp/imports/<import_id>.zip`
+        file_path =  Import::ImportCollections.zip_file_path(import_id)
         File.open(file_path, 'wb') do |file|
             file.write(params[:file][:tempfile].read)
         end
 
         # run the asyncronous import job
-        ImportCollectionsJob.perform_later(directory, params[:file][:filename], current_user.id)
+        ImportCollectionsJob.perform_later(import_id, current_user.id)
 
-        # return the job id to the client
-        return job.job_id
+        # return the import_id to the client
+        return {:import_id => import_id}
       end
     end
   end

app/assets/javascripts/components/contextActions/ModalCollectionExport.js

@@ -33,8 +33,9 @@ export default class ModalCollectionExport extends React.Component {
     const { onHide, action } = this.props;
 
     let params = {
-      format: 'zip',
       collections: [uiState.currentCollection.id],
+      format: 'zip',
+      nested: true
     }
     action(params);
 

app/assets/javascripts/components/fetchers/ExportCollectionsFetcher.js

@@ -15,9 +15,10 @@ export default class ExportCollectionsFetcher {
     }).then((response) => {
       return response.json()
     }).then((json) => {
+      console.log(json.export_id);
       // after a short delay, start polling
       setTimeout(() => {
-        ExportCollectionsFetcher.pollJob(json.job_id)
+        ExportCollectionsFetcher.pollJob(json.export_id)
       }, 1000);
     }).catch((errorMessage) => {
       console.log(errorMessage);
@@ -26,9 +27,9 @@ export default class ExportCollectionsFetcher {
     return promise;
   }
 
-  static pollJob(jobId) {
+  static pollJob(exportId) {
 
-    let promise = fetch(`/api/v1/exports/${jobId}`, {
+    let promise = fetch(`/api/v1/exports/${exportId}`, {
       credentials: 'same-origin',
       method: 'GET',
       headers: {
@@ -41,7 +42,7 @@ export default class ExportCollectionsFetcher {
       if (json.status == 'EXECUTING') {
         // continue polling
         setTimeout(() => {
-          ExportCollectionsFetcher.pollJob(jobId);
+          ExportCollectionsFetcher.pollJob(exportId);
         }, 4000);
       } else if (json.status == 'COMPLETED') {
         // download the file, headers will prevent the browser from reloading the page

app/jobs/export_collections_job.rb

 class ExportCollectionsJob < ActiveJob::Base
   queue_as :export_collections
 
-  # rescue_from(ActiveRecord::RecordNotFound) do; end
-
-  before_enqueue do |job|
-    # create lock file
-    File.open(lock_file_name(job.arguments.first, job.job_id), 'w') {}
-  end
-
-  after_perform do |job|
-    # remove lock file
-    lock_file_name = lock_file_name(job.arguments.first, job.job_id)
-    File.delete(lock_file_name) if File.exist?(lock_file_name)
-  end
-
-  def perform(fmt, collection_ids)
-    # prepare the collections for export
-    export = Export::ExportCollections.new
-    export.prepare_data collection_ids
-
-    case fmt
-    when 'json'
-      # write the json file public/json/
-      File.write(file_name(fmt, self.job_id), export.to_json())
-    when 'zip'
-      # create a zip buffer
-      zip = Zip::OutputStream.write_buffer do |zip|
-        # write the json file into the zip file
-        zip.put_next_entry File.join('data.json')
-        zip.write export.to_json()
-
-        # write all attachemnts into an attachments directory
-        export.attachments.each do |attachment|
-          zip.put_next_entry File.join('attachments', attachment.filename)
-          zip.write attachment.read_file
-        end
-      end
-      zip.rewind
-
-      # write the zip file to public/zip/
-      File.write(file_name(fmt, self.job_id), zip.read)
-    end
-  end
-
-  private
-
-  def file_name(fmt, job_id)
-    return File.join('public', fmt, "#{job_id}.#{fmt}")
-  end
-
-  def lock_file_name(fmt, job_id)
-    return file_name(fmt, job_id) + '.lock'
+  def perform(export_id, collection_ids, format, nested)
+    export = Export::ExportCollections.new(export_id, collection_ids, format, nested)
+    export.prepare_data
+    export.to_file
+    export.cleanup
   end
 end

app/jobs/import_collections_job.rb

 class ImportCollectionsJob < ActiveJob::Base
   queue_as :import_collections
 
-  def perform(directory, file_name, current_user_id)
-    import = Import::ImportCollections.new(directory, file_name, current_user_id)
-    import.process
-
-    # cleanup
-    FileUtils.remove_dir(directory) if File.exist?(directory)
+  def perform(import_id, current_user_id)
+    import = Import::ImportCollections.new(import_id, current_user_id)
+    import.extract
+    import.read
+    import.import
+    import.cleanup
   end
 end

lib/export/export_collections.rb

 module Export
   class ExportCollections
 
-    attr_reader :data
-    attr_reader :uuids
-    attr_reader :attachments
+    # static methods
+    class << self
+      def file_path(export_id, format)
+        File.join('public', format, "#{export_id}.#{format}")
+      end
+      def file_url(export_id, format)
+        "/#{format}/#{export_id}.#{format}"
+      end
+      def lock_file_path(export_id, format)
+        File.join('public', format, "#{export_id}.lock")
+      end
+    end
+
+    def initialize(export_id, collection_ids, format, nested)
+      @export_id = export_id
+      @collection_ids = collection_ids
+      @format = format
+      @nested = nested
+
+      @file_path = ExportCollections.file_path(export_id, format)
+      @lock_file_path = ExportCollections.lock_file_path(export_id, format)
 
-    def initialize
       @data = {}
       @uuids = {}
       @attachments = []
     end
 
-    def to_json()
+    def to_json
       @data.to_json()
     end
 
-    def prepare_data(collection_ids)
+    def to_file
+      case @format
+      when 'json'
+        # write the json file public/json/
+        File.write(@file_path, @data.to_json())
+
+      when 'zip'
+        # create a zip buffer
+        zip = Zip::OutputStream.write_buffer do |zip|
+          # write the json file into the zip file
+          zip.put_next_entry File.join('data.json')
+          zip.write @data.to_json()
+
+          # write all attachemnts into an attachments directory
+          @attachments.each do |attachment|
+            zip.put_next_entry File.join('attachments', attachment.filename)
+            zip.write attachment.read_file
+          end
+        end
+        zip.rewind
+
+        # write the zip file to public/zip/
+        File.write(@file_path, zip.read)
+      end
+    end
+
+    def prepare_data
+      # get the collections from the database
+      collections = Collection.find(@collection_ids)
+
+      # fetch collections for export
+      fetch_many(collections, {
+        :user_id => 'User'
+      })
+
       # loop over all collections
-      fetch_collections(collection_ids).each do |collection|
+      collections.each do |collection|
 
         # fetch samples
         fetch_many(collection.samples, {
@@ -131,15 +182,14 @@ module Export
           fetch_literals(research_plan)
         end
       end
-      self
     end
 
-    def fetch_collections(collection_ids)
-      collections = Collection.find(collection_ids)
-      fetch_many(collections)
-      collections
+    def cleanup
+      File.delete(@lock_file_path) if File.exist?(@lock_file_path)
     end
 
+    private
+
     def fetch_containers(containable)
       containable_type = containable.class.name
 

lib/import/import_collections.rb

@@ -3,26 +3,31 @@ require 'json'
 module Import
   class ImportCollections
 
-    def initialize(directory, file_name, current_user_id)
-      @directory = directory
-      @file_name = file_name
+    # static methods
+    class << self
+      def zip_file_path(import_id)
+        File.join('tmp', 'import', "#{import_id}.zip")
+      end
+
+      def directory_path(import_id)
+        File.join('tmp', 'import', import_id)
+      end
+    end
+
+    def initialize(import_id, current_user_id)
+      @import_id = import_id
       @current_user_id = current_user_id
 
+      @zip_file_path = ImportCollections.zip_file_path(import_id)
+      @directory = ImportCollections.directory_path(import_id)
+
       @data = nil
       @instances = {}
       @attachments = []
     end
 
-    def process
-      extract
-      read
-      import
-      self
-    end
-
     def extract()
-      file_path = File.join(@directory, @file_name)
-      Zip::File.open(file_path) do |zip_file|
+      Zip::File.open(@zip_file_path) do |zip_file|
         zip_file.each do |f|
           fpath = File.join(@directory, f.name)
           fdir = File.dirname(fpath)
@@ -57,6 +62,13 @@ module Import
       end
     end
 
+    def cleanup
+      File.delete(@zip_file_path) if File.exist?(@zip_file_path)
+      FileUtils.remove_dir(@directory) if File.exist?(@directory)
+    end
+
+    private
+
     def import_collections
       @data.fetch('Collection', []).each do |uuid, fields|
         # create the collection