Commit cbc33c56 authored by hh1966's avatar hh1966
Browse files

Simplify Export/Import APIs and Jobs

parent f3206947
module Chemotion
class ExportAPI < Grape::API
resource :exports do
before do
# TODO: validate collection_id, check permissions
# handle nested collections
@collection_ids = params[:collections]
@format = params[:format]
desc "Create export job"
params do
requires :collections, type: Array[Integer]
requires :format, type: String
requires :nested, type: Boolean
end
post do
collection_ids = params[:collections].uniq
format = params[:format]
nested = params[:nested] == true
# check if the user is allowed to export these collections
collection_ids.each do |collection_id|
begin
collection = Collection.belongs_to_or_shared_by(current_user.id, current_user.group_ids).find(collection_id)
rescue ActiveRecord::RecordNotFound
error!('401 Unauthorized', 401)
end
end
# create an id for the export
export_id = SecureRandom.uuid
# create the lock file
lock_file_path = Export::ExportCollections.lock_file_path(export_id, format)
File.open(lock_file_path, 'w') {}
# run the asyncronous export job and return its id to the client
ExportCollectionsJob.perform_later(export_id, collection_ids, format, nested)
# return the export_id to the client
return {:export_id => export_id}
end
desc "Poll export job"
......@@ -14,19 +43,19 @@ module Chemotion
requires :id, type: String
end
get '/:id' do
job_id = params[:id]
export_id = params[:id]
# look for the export file
['json', 'zip'].each do |fmt|
file_name = File.join('public', fmt, "#{job_id}.#{fmt}")
lock_file_name = file_name + '.lock'
# look for the lock file file
['json', 'zip'].each do |format|
file_path = Export::ExportCollections.file_path(export_id, format)
lock_file_path = Export::ExportCollections.lock_file_path(export_id, format)
if File.exist?(file_name) and !File.exist?(lock_file_name)
if File.exist?(file_path) and !File.exist?(lock_file_path)
return {
:status => 'COMPLETED',
:url => "/#{fmt}/#{job_id}.#{fmt}"
:url => Export::ExportCollections.file_url(export_id, format)
}
elsif File.exist?(lock_file_name)
elsif File.exist?(lock_file_path)
return {:status => 'EXECUTING'}
end
end
......@@ -34,15 +63,6 @@ module Chemotion
error! :not_found, 404
end
desc "Create export job"
params do
requires :collections, type: Array[Integer]
requires :format, type: String
end
post do
ExportCollectionsJob.perform_later(@format, @collection_ids)
end
end
end
end
......@@ -3,24 +3,20 @@ module Chemotion
resource :imports do
post do
# create the import job in order to have the job id
job = ImportCollectionsJob.new
# create an id for the import
import_id = SecureRandom.uuid
# create a directory `tmp/imports/<job_id>` for this import
directory = File.join('tmp', 'import', job.job_id)
FileUtils.mkdir_p(directory) unless File.directory?(directory)
# store the file in the `tmp/imports/<job_id>` dir
file_path = File.join(directory, params[:file][:filename])
# store the file as `tmp/imports/<import_id>.zip`
file_path = Import::ImportCollections.zip_file_path(import_id)
File.open(file_path, 'wb') do |file|
file.write(params[:file][:tempfile].read)
end
# run the asyncronous import job
ImportCollectionsJob.perform_later(directory, params[:file][:filename], current_user.id)
ImportCollectionsJob.perform_later(import_id, current_user.id)
# return the job id to the client
return job.job_id
# return the import_id to the client
return {:import_id => import_id}
end
end
end
......
......@@ -33,8 +33,9 @@ export default class ModalCollectionExport extends React.Component {
const { onHide, action } = this.props;
let params = {
format: 'zip',
collections: [uiState.currentCollection.id],
format: 'zip',
nested: true
}
action(params);
......
......@@ -15,9 +15,10 @@ export default class ExportCollectionsFetcher {
}).then((response) => {
return response.json()
}).then((json) => {
console.log(json.export_id);
// after a short delay, start polling
setTimeout(() => {
ExportCollectionsFetcher.pollJob(json.job_id)
ExportCollectionsFetcher.pollJob(json.export_id)
}, 1000);
}).catch((errorMessage) => {
console.log(errorMessage);
......@@ -26,9 +27,9 @@ export default class ExportCollectionsFetcher {
return promise;
}
static pollJob(jobId) {
static pollJob(exportId) {
let promise = fetch(`/api/v1/exports/${jobId}`, {
let promise = fetch(`/api/v1/exports/${exportId}`, {
credentials: 'same-origin',
method: 'GET',
headers: {
......@@ -41,7 +42,7 @@ export default class ExportCollectionsFetcher {
if (json.status == 'EXECUTING') {
// continue polling
setTimeout(() => {
ExportCollectionsFetcher.pollJob(jobId);
ExportCollectionsFetcher.pollJob(exportId);
}, 4000);
} else if (json.status == 'COMPLETED') {
// download the file, headers will prevent the browser from reloading the page
......
class ExportCollectionsJob < ActiveJob::Base
queue_as :export_collections
# rescue_from(ActiveRecord::RecordNotFound) do; end
before_enqueue do |job|
# create lock file
File.open(lock_file_name(job.arguments.first, job.job_id), 'w') {}
end
after_perform do |job|
# remove lock file
lock_file_name = lock_file_name(job.arguments.first, job.job_id)
File.delete(lock_file_name) if File.exist?(lock_file_name)
end
def perform(fmt, collection_ids)
# prepare the collections for export
export = Export::ExportCollections.new
export.prepare_data collection_ids
case fmt
when 'json'
# write the json file public/json/
File.write(file_name(fmt, self.job_id), export.to_json())
when 'zip'
# create a zip buffer
zip = Zip::OutputStream.write_buffer do |zip|
# write the json file into the zip file
zip.put_next_entry File.join('data.json')
zip.write export.to_json()
# write all attachemnts into an attachments directory
export.attachments.each do |attachment|
zip.put_next_entry File.join('attachments', attachment.filename)
zip.write attachment.read_file
end
end
zip.rewind
# write the zip file to public/zip/
File.write(file_name(fmt, self.job_id), zip.read)
end
end
private
def file_name(fmt, job_id)
return File.join('public', fmt, "#{job_id}.#{fmt}")
end
def lock_file_name(fmt, job_id)
return file_name(fmt, job_id) + '.lock'
def perform(export_id, collection_ids, format, nested)
export = Export::ExportCollections.new(export_id, collection_ids, format, nested)
export.prepare_data
export.to_file
export.cleanup
end
end
class ImportCollectionsJob < ActiveJob::Base
queue_as :import_collections
def perform(directory, file_name, current_user_id)
import = Import::ImportCollections.new(directory, file_name, current_user_id)
import.process
# cleanup
FileUtils.remove_dir(directory) if File.exist?(directory)
def perform(import_id, current_user_id)
import = Import::ImportCollections.new(import_id, current_user_id)
import.extract
import.read
import.import
import.cleanup
end
end
module Export
class ExportCollections
attr_reader :data
attr_reader :uuids
attr_reader :attachments
# static methods
class << self
def file_path(export_id, format)
File.join('public', format, "#{export_id}.#{format}")
end
def file_url(export_id, format)
"/#{format}/#{export_id}.#{format}"
end
def lock_file_path(export_id, format)
File.join('public', format, "#{export_id}.lock")
end
end
def initialize(export_id, collection_ids, format, nested)
@export_id = export_id
@collection_ids = collection_ids
@format = format
@nested = nested
@file_path = ExportCollections.file_path(export_id, format)
@lock_file_path = ExportCollections.lock_file_path(export_id, format)
def initialize
@data = {}
@uuids = {}
@attachments = []
end
def to_json()
def to_json
@data.to_json()
end
def prepare_data(collection_ids)
def to_file
case @format
when 'json'
# write the json file public/json/
File.write(@file_path, @data.to_json())
when 'zip'
# create a zip buffer
zip = Zip::OutputStream.write_buffer do |zip|
# write the json file into the zip file
zip.put_next_entry File.join('data.json')
zip.write @data.to_json()
# write all attachemnts into an attachments directory
@attachments.each do |attachment|
zip.put_next_entry File.join('attachments', attachment.filename)
zip.write attachment.read_file
end
end
zip.rewind
# write the zip file to public/zip/
File.write(@file_path, zip.read)
end
end
def prepare_data
# get the collections from the database
collections = Collection.find(@collection_ids)
# fetch collections for export
fetch_many(collections, {
:user_id => 'User'
})
# loop over all collections
fetch_collections(collection_ids).each do |collection|
collections.each do |collection|
# fetch samples
fetch_many(collection.samples, {
......@@ -131,15 +182,14 @@ module Export
fetch_literals(research_plan)
end
end
self
end
def fetch_collections(collection_ids)
collections = Collection.find(collection_ids)
fetch_many(collections)
collections
def cleanup
File.delete(@lock_file_path) if File.exist?(@lock_file_path)
end
private
def fetch_containers(containable)
containable_type = containable.class.name
......
......@@ -3,26 +3,31 @@ require 'json'
module Import
class ImportCollections
def initialize(directory, file_name, current_user_id)
@directory = directory
@file_name = file_name
# static methods
class << self
def zip_file_path(import_id)
File.join('tmp', 'import', "#{import_id}.zip")
end
def directory_path(import_id)
File.join('tmp', 'import', import_id)
end
end
def initialize(import_id, current_user_id)
@import_id = import_id
@current_user_id = current_user_id
@zip_file_path = ImportCollections.zip_file_path(import_id)
@directory = ImportCollections.directory_path(import_id)
@data = nil
@instances = {}
@attachments = []
end
def process
extract
read
import
self
end
def extract()
file_path = File.join(@directory, @file_name)
Zip::File.open(file_path) do |zip_file|
Zip::File.open(@zip_file_path) do |zip_file|
zip_file.each do |f|
fpath = File.join(@directory, f.name)
fdir = File.dirname(fpath)
......@@ -57,6 +62,13 @@ module Import
end
end
def cleanup
File.delete(@zip_file_path) if File.exist?(@zip_file_path)
FileUtils.remove_dir(@directory) if File.exist?(@directory)
end
private
def import_collections
@data.fetch('Collection', []).each do |uuid, fields|
# create the collection
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment