From 1320e3a61a7e66ddcf1c12109b9e3954c673be0a Mon Sep 17 00:00:00 2001 From: mjy Date: Mon, 4 Apr 2022 19:53:36 -0500 Subject: [PATCH 1/8] w2830 wip --- app/views/downloads/api/v1/status.json.jbuilder | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 app/views/downloads/api/v1/status.json.jbuilder diff --git a/app/views/downloads/api/v1/status.json.jbuilder b/app/views/downloads/api/v1/status.json.jbuilder new file mode 100644 index 0000000000..accfe2c869 --- /dev/null +++ b/app/views/downloads/api/v1/status.json.jbuilder @@ -0,0 +1,7 @@ +json.download do + json.partial! '/downloads/api/v1/attributes', download: @download +end + +json.status do + json.status download_status(@download) +end From 99f300d5b524a653e005fd86a92231701873641f Mon Sep 17 00:00:00 2001 From: mjy Date: Tue, 5 Apr 2022 14:49:03 -0500 Subject: [PATCH 2/8] WIP --- app/controllers/downloads_controller.rb | 19 ++++++++++++++++++- app/helpers/downloads_helper.rb | 9 +++++++++ app/models/download.rb | 14 ++++++++++++++ .../downloads/api/v1/status.json.jbuilder | 18 +++++++++++++----- config/routes/api_v1.rb | 4 ++++ lib/export/download.rb | 7 +++++++ lib/export/dwca.rb | 1 + 7 files changed, 66 insertions(+), 6 deletions(-) diff --git a/app/controllers/downloads_controller.rb b/app/controllers/downloads_controller.rb index a772a0c9c0..db61b42d4f 100644 --- a/app/controllers/downloads_controller.rb +++ b/app/controllers/downloads_controller.rb @@ -2,7 +2,7 @@ class DownloadsController < ApplicationController include DataControllerConfiguration::ProjectDataControllerConfiguration before_action :set_download, only: [:show, :download_file, :destroy, :update, :file, :api_show, :edit] - before_action :set_download_api, only: [:api_file, :api_show] + before_action :set_download_api, only: [:api_file, :api_show, :api_status, :api_terminate] after_action -> { set_pagination_headers(:downloads) }, only: [:api_index], if: :json_request? @@ -96,6 +96,19 @@ def api_show render '/downloads/api/v1/show' end + def api_status + render '/downloads/api/v1/status' + end + + def api_build + @download = Download.new(api_build_params) + render '/downloads/api/v1/status' + end + + def api_terminate + render '/downloads/api/v1/status' + end + private def filter_params @@ -113,4 +126,8 @@ def set_download_api def download_params params.require(:download).permit(:is_public, :name, :expires ) end + + def api_build_params + params.permit(:type, predicate_extensions: {}) + end end diff --git a/app/helpers/downloads_helper.rb b/app/helpers/downloads_helper.rb index 9d3d3f8a27..5cba2b818f 100644 --- a/app/helpers/downloads_helper.rb +++ b/app/helpers/downloads_helper.rb @@ -17,4 +17,13 @@ def download_file_api_url(download) nil end end + + def download_status(download) + return nil if download.nil? || !download.persisted? + + return { + ready: download.ready?, + expired: download.expired?, + } + end end diff --git a/app/models/download.rb b/app/models/download.rb index e2fb808b2a..14085c4141 100644 --- a/app/models/download.rb +++ b/app/models/download.rb @@ -55,6 +55,15 @@ class Download < ApplicationRecord validates_presence_of :expires validates_presence_of :type + def build_async(record_scope, predicate_extension_params: {}, download_params: {} ) + # MEH! + download_params.each do |k, v| + self.send("#{k}=".to_sym, v) + end + + ::DwcaCreateDownloadJob.perform_later(self, core_scope: record_scope.to_sql, predicate_extension_params: predicate_extension_params) + end + # Gets the downloads storage path def self.storage_path STORAGE_PATH @@ -95,6 +104,10 @@ def delete_file FileUtils.rm_rf(path) end + def api_buildable? + false + end + private STORAGE_PATH = Rails.root.join(Rails.env.test? ? 'tmp' : '', "downloads#{ENV['TEST_ENV_NUMBER']}").freeze @@ -114,5 +127,6 @@ def save_file require_dependency 'download/bibtex' require_dependency 'download/coldp' require_dependency 'download/dwc_archive' +require_dependency 'download/dwc_archive/complete' require_dependency 'download/sql_project_dump' require_dependency 'download/text' diff --git a/app/views/downloads/api/v1/status.json.jbuilder b/app/views/downloads/api/v1/status.json.jbuilder index accfe2c869..0d9dc5849d 100644 --- a/app/views/downloads/api/v1/status.json.jbuilder +++ b/app/views/downloads/api/v1/status.json.jbuilder @@ -1,7 +1,15 @@ -json.download do - json.partial! '/downloads/api/v1/attributes', download: @download -end +if @download.valid? + + json.download do + json.partial! '/downloads/api/v1/attributes', download: @download + end + + json.status do + json.status download_status(@download) + end -json.status do - json.status download_status(@download) +else + json.status do + json.error_messages @download.errors.messages + end end diff --git a/config/routes/api_v1.rb b/config/routes/api_v1.rb index 8f763c7980..e2583973a9 100644 --- a/config/routes/api_v1.rb +++ b/config/routes/api_v1.rb @@ -45,10 +45,14 @@ get '/otus/:otu_id/inventory/images', to: '/images#api_image_inventory', as: :api_images get '/otus/:id', to: '/otus#api_show' + get '/downloads/build', to: '/downloads#api_build', as: :api_download_build get '/downloads/:id', to: '/downloads#api_show' get '/downloads', to: '/downloads#api_index' get '/downloads/:id/file', to: '/downloads#api_file', as: :api_download_file + get '/downloads/:id/terminate', to: '/downloads#api_terminate', as: :api_download_terminate + get '/downloads/:id/status', to: '/downloads#api_status', as: :api_download_status + get '/dwc_occurrences', to: '/dwc_occurrences#api_index' get '/taxon_names', to: '/taxon_names#api_index' diff --git a/lib/export/download.rb b/lib/export/download.rb index 25d294c5ed..7a1d8d4ecb 100644 --- a/lib/export/download.rb +++ b/lib/export/download.rb @@ -109,6 +109,13 @@ def self.sort_column_headers(column_names = [], column_order = []) unsorted + sorted end +# # @return [Download, false] +# # +# # Given a set of API param, create a download of a cnkkk +# def self.delegate_build(params = {}) + +# end + end diff --git a/lib/export/dwca.rb b/lib/export/dwca.rb index c79a080fd1..272e77fc87 100644 --- a/lib/export/dwca.rb +++ b/lib/export/dwca.rb @@ -28,6 +28,7 @@ module Dwca def self.download_async(record_scope, request = nil, predicate_extension_params: {}) name = "dwc-a_#{DateTime.now}.zip" + # TODO: move fixed attributes to model download = ::Download::DwcArchive.create!( name: "DwC Archive generated at #{Time.now}.", description: 'A Darwin Core archive.', From b30d5fc3d83a0c8847f2e29e37cbbcbec44d7b23 Mon Sep 17 00:00:00 2001 From: mjy Date: Thu, 21 Apr 2022 12:58:45 -0600 Subject: [PATCH 3/8] Alpha of #2830 for complete DwC archives via the API. Minor features for Project creation and show. --- .../collection_objects/filter_params.rb | 4 +- .../concerns/forgery_protection.rb | 12 +- app/controllers/downloads_controller.rb | 27 ++-- app/controllers/projects_controller.rb | 9 +- app/controllers/taxon_names_controller.rb | 2 +- app/helpers/downloads_helper.rb | 7 +- app/helpers/projects_helper.rb | 7 +- app/helpers/workbench/sessions_helper.rb | 6 +- app/jobs/dwca_create_download_job.rb | 2 +- app/models/download.rb | 30 +++- app/models/download/dwc_archive/complete.rb | 24 +++ app/models/project.rb | 2 + app/views/downloads/_attributes.html.erb | 5 + app/views/downloads/_attributes.json.jbuilder | 2 +- .../api/v1/_attributes.json.jbuilder | 7 +- app/views/downloads/api/v1/show.json.jbuilder | 10 +- .../downloads/api/v1/status.json.jbuilder | 15 -- app/views/projects/_form.html.erb | 6 + app/views/projects/show.html.erb | 23 ++- config/routes/api_v1.rb | 16 +- .../20220421182414_add_sha2_to_download.rb | 5 + db/schema.rb | 3 +- .../download/dwc_archive/complete_spec.rb | 148 ++++++++++++++++++ 23 files changed, 301 insertions(+), 71 deletions(-) create mode 100644 app/models/download/dwc_archive/complete.rb delete mode 100644 app/views/downloads/api/v1/status.json.jbuilder create mode 100644 db/migrate/20220421182414_add_sha2_to_download.rb create mode 100644 spec/models/download/dwc_archive/complete_spec.rb diff --git a/app/controllers/collection_objects/filter_params.rb b/app/controllers/collection_objects/filter_params.rb index 4d9474375f..b28fb96300 100644 --- a/app/controllers/collection_objects/filter_params.rb +++ b/app/controllers/collection_objects/filter_params.rb @@ -88,7 +88,7 @@ def collection_object_filter_params # TODO: check user_id: [] - a[:user_id] = params[:user_id] if params[:user_id] && is_project_member_by_id(params[:user_id], sessions_current_project_id) # double check vs. setting project_id from API + a[:user_id] = params[:user_id] if params[:user_id] && is_project_member_by_id?(params[:user_id], sessions_current_project_id) # double check vs. setting project_id from API a end @@ -165,7 +165,7 @@ def collection_object_api_params # } ) - a[:user_id] = params[:user_id] if params[:user_id] && is_project_member_by_id(params[:user_id], sessions_current_project_id) # double check vs. setting project_id from API + a[:user_id] = params[:user_id] if params[:user_id] && is_project_member_by_id?(params[:user_id], sessions_current_project_id) # double check vs. setting project_id from API a end end diff --git a/app/controllers/concerns/forgery_protection.rb b/app/controllers/concerns/forgery_protection.rb index 4ab42c4d96..5d78805c10 100644 --- a/app/controllers/concerns/forgery_protection.rb +++ b/app/controllers/concerns/forgery_protection.rb @@ -7,12 +7,14 @@ module ForgeryProtection protect_from_forgery def handle_unverified_request - respond_to do |format| - format.html do - flash[:notice] = "Your last request could not be fulfilled. Please retry." - redirect_to '/' + unless @api_request # @locodelassembly please check this out, this disables CSRF for API requests that are delete/delete + respond_to do |format| + format.html do + flash[:notice] = "Your last request could not be fulfilled. Please retry." + redirect_to '/' + end + format.json { render body: '{ "success": false }', status: :unprocessable_entity } end - format.json { render body: '{ "success": false }', status: :unprocessable_entity } end end diff --git a/app/controllers/downloads_controller.rb b/app/controllers/downloads_controller.rb index db61b42d4f..fc547aad91 100644 --- a/app/controllers/downloads_controller.rb +++ b/app/controllers/downloads_controller.rb @@ -2,7 +2,7 @@ class DownloadsController < ApplicationController include DataControllerConfiguration::ProjectDataControllerConfiguration before_action :set_download, only: [:show, :download_file, :destroy, :update, :file, :api_show, :edit] - before_action :set_download_api, only: [:api_file, :api_show, :api_status, :api_terminate] + before_action :set_download_api, only: [:api_file, :api_show, :api_destroy] after_action -> { set_pagination_headers(:downloads) }, only: [:api_index], if: :json_request? @@ -78,11 +78,12 @@ def file end def api_index - @downloads = Download.where(is_public: true, project_id: sessions_current_project_id) + @downloads = Download.where(project_id: sessions_current_project_id) .order('downloads.id').page(params[:page]).per(params[:per]) render '/downloads/api/v1/index' end + # GET /api/v1/downloads/123/file.json def api_file if @download.ready? @download.increment!(:times_downloaded) @@ -96,17 +97,24 @@ def api_show render '/downloads/api/v1/show' end - def api_status - render '/downloads/api/v1/status' + # DELETE /api/v1/downloads/1.json + def api_destroy + if @download.destroy + render json: {id: @download.id_was, status: :destroyed} + else + render json: {}, status: :unprocessable_entity + end end + # /api/v1/downloads/build?type=Download::DwcArchive::Complete def api_build - @download = Download.new(api_build_params) - render '/downloads/api/v1/status' + @download = Download.create(api_build_params) + render '/downloads/api/v1/show' end def api_terminate - render '/downloads/api/v1/status' + @download.terminate # TODO, add method, or change to :destroy + render '/downloads/api/v1/show' end private @@ -116,11 +124,12 @@ def filter_params end def set_download - @download = Download.unscoped.where(project_id: sessions_current_project_id).find(params[:id]) + # Why .unscoped ? + @download = Download.where(project_id: sessions_current_project_id).find(params[:id]) end def set_download_api - @download = Download.unscoped.where(is_public: true, project_id: sessions_current_project_id).find(params[:id]) + @download = Download.where(is_public: true, project_id: sessions_current_project_id).find(params[:id]) end def download_params diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 488a91c3bf..6fcaa2cb83 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -124,7 +124,14 @@ def set_project end def project_params - params.require(:project).permit(:name, :set_new_api_access_token, :clear_api_access_token, Project.key_value_preferences, Project.array_preferences, Project.hash_preferences) + params.require(:project).permit( + :name, + :set_new_api_access_token, + :clear_api_access_token, + Project.key_value_preferences, + Project.array_preferences, + Project.hash_preferences, + project_members_attributes: [:user_id, :destroy]) end def go_to diff --git a/app/controllers/taxon_names_controller.rb b/app/controllers/taxon_names_controller.rb index b73163647b..6833ebc5ba 100644 --- a/app/controllers/taxon_names_controller.rb +++ b/app/controllers/taxon_names_controller.rb @@ -373,7 +373,7 @@ def api_params ).to_h.symbolize_keys.merge(project_id: sessions_current_project_id) # TODO: see config in collection objects controller - # a[:user_id] = params[:user_id] if params[:user_id] && is_project_member_by_id(params[:user_id], sessions_current_project_id) # double check vs. setting project_id from API + # a[:user_id] = params[:user_id] if params[:user_id] && is_project_member_by_id?(params[:user_id], sessions_current_project_id) # double check vs. setting project_id from API # a end diff --git a/app/helpers/downloads_helper.rb b/app/helpers/downloads_helper.rb index 5cba2b818f..9ad0bedd99 100644 --- a/app/helpers/downloads_helper.rb +++ b/app/helpers/downloads_helper.rb @@ -18,12 +18,13 @@ def download_file_api_url(download) end end + # @return [Hash] + # calculated attributes used in /download responses def download_status(download) return nil if download.nil? || !download.persisted? - return { ready: download.ready?, - expired: download.expired?, - } + expired: download.expired? + } end end diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 93dcdfd617..6c9214edb3 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -24,7 +24,10 @@ def projects_list(projects) projects.collect { |p| content_tag(:li, project_link(p)) }.join.html_safe end - # Came from application_controller + def project_login_link(project) + return nil unless (!is_project_member_by_id?(sessions_current_user_id, sessions_current_project_id) && (sessions_current_project_id != project.id)) + link_to('Login to ' + project.name, select_project_path(project), class: ['button-default']) + end def invalid_object(object) !(!object.try(:project_id) || project_matches(object)) @@ -34,6 +37,4 @@ def project_matches(object) object.try(:project_id) == sessions_current_project_id end - - end diff --git a/app/helpers/workbench/sessions_helper.rb b/app/helpers/workbench/sessions_helper.rb index 42db2addbd..12fec77b3a 100644 --- a/app/helpers/workbench/sessions_helper.rb +++ b/app/helpers/workbench/sessions_helper.rb @@ -127,11 +127,7 @@ def is_superuser? sessions_signed_in? && ( is_administrator? || is_project_administrator? ) end - def is_project_member?(user, project) - project.project_members.include?(user) # TODO - change to ID - end - - def is_project_member_by_id(user_id, project_id) + def is_project_member_by_id?(user_id, project_id) ProjectMember.where(user_id: user_id, project_id: project_id).any? end diff --git a/app/jobs/dwca_create_download_job.rb b/app/jobs/dwca_create_download_job.rb index 34f2088fe8..b43f2befb0 100644 --- a/app/jobs/dwca_create_download_job.rb +++ b/app/jobs/dwca_create_download_job.rb @@ -1,11 +1,11 @@ class DwcaCreateDownloadJob < ApplicationJob queue_as :dwca_export + # @param download [a Download instance] # @param core_scope [String, ActiveRecord::Relation] # String of SQL generated from the scope # SQL must return a list of DwcOccurrence records # take a download, and a list of scopes, and save the result to the download, that's all - # @return # # TODO: handle extension scopes def perform(download, core_scope: nil, extension_scopes: {biological_associations: nil}, predicate_extension_params: {}) begin diff --git a/app/models/download.rb b/app/models/download.rb index 14085c4141..d6a3e2ef2a 100644 --- a/app/models/download.rb +++ b/app/models/download.rb @@ -55,12 +55,7 @@ class Download < ApplicationRecord validates_presence_of :expires validates_presence_of :type - def build_async(record_scope, predicate_extension_params: {}, download_params: {} ) - # MEH! - download_params.each do |k, v| - self.send("#{k}=".to_sym, v) - end - + def build_async(record_scope, predicate_extension_params: {}) ::DwcaCreateDownloadJob.perform_later(self, core_scope: record_scope.to_sql, predicate_extension_params: predicate_extension_params) end @@ -112,14 +107,35 @@ def api_buildable? STORAGE_PATH = Rails.root.join(Rails.env.test? ? 'tmp' : '', "downloads#{ENV['TEST_ENV_NUMBER']}").freeze + # TODO: check performance on 50-100mb files + def set_sha2 + if @source_file_path + s = ::Digest::SHA2.new + File.open(@source_file_path) do |f| + while chunk = f.read(256) # only load 256 bytes at a time + s << chunk + end + end + self.update_column(:sha2, s.hexdigest) + end + end + def dir_path str = id.to_s.rjust(9, '0') STORAGE_PATH.join(str[-str.length..-7], str[-6..-4], str[-3..-1]) end + # This is the only method to move a temporary file + # to its location on the file server. + # + # ActiveJob generating files trigger this method + # by .updating the filename attribute. def save_file FileUtils.mkdir_p(dir_path) - FileUtils.cp(@source_file_path, file_path) if @source_file_path + if @source_file_path + FileUtils.cp(@source_file_path, file_path) + set_sha2 + end end end diff --git a/app/models/download/dwc_archive/complete.rb b/app/models/download/dwc_archive/complete.rb new file mode 100644 index 0000000000..15fe60a7ce --- /dev/null +++ b/app/models/download/dwc_archive/complete.rb @@ -0,0 +1,24 @@ +# Only one per project. Includes the complete current contents of DwCOccurrences. +class Download::DwcArchive::Complete < Download::DwcArchive + + # Can be built with/out data attributes + attr_accessor :predicate_extensions + + # Default values + attribute :name, default: -> { "dwc-a_complete_#{DateTime.now}.zip" } + attribute :description, default: 'A Darwin Core archive of the complete TaxonWorks DwcOccurrence table' + attribute :filename, default: -> { "dwc-a_complete_#{DateTime.now}.zip" } + attribute :expires, default: -> { 1.month.from_now } + attribute :request, default: -> { '/api/v1/downloads/build?type=Download::DwcArchive::Complete' } + attribute :is_public, default: -> { 1 } + + after_save :build, unless: :ready? # prevent infinite loop callbacks + + validates_uniqueness_of :type, scope: [:project_id], message: 'Only one Download::DwcArchive::Complete is allowed. Destroy the old version first.' + + def build + record_scope = ::DwcOccurrence.where(project_id: project_id) + build_async(record_scope) + end + +end diff --git a/app/models/project.rb b/app/models/project.rb index 4a456a41cb..2daa6b4296 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -117,6 +117,8 @@ class Project < ApplicationRecord validates_presence_of :name validates_uniqueness_of :name + accepts_nested_attributes_for :project_members, allow_destroy: true + def project_administrators users.joins(:project_members).where(project_members: {is_project_administrator: true}) end diff --git a/app/views/downloads/_attributes.html.erb b/app/views/downloads/_attributes.html.erb index 7e15c3f933..f739ef5f85 100644 --- a/app/views/downloads/_attributes.html.erb +++ b/app/views/downloads/_attributes.html.erb @@ -45,6 +45,11 @@ Total records (estimate): <%= @download.total_records -%>

+ +

+ SHA2: + <%= @download.sha2 %> +

<% else %>

Status: diff --git a/app/views/downloads/_attributes.json.jbuilder b/app/views/downloads/_attributes.json.jbuilder index fb2087c655..e9f3bbb5c9 100644 --- a/app/views/downloads/_attributes.json.jbuilder +++ b/app/views/downloads/_attributes.json.jbuilder @@ -1,4 +1,4 @@ -json.extract! download, :id, :name, :description, :request, :expires, :times_downloaded, :total_records, :is_public, :type, :created_by_id, :updated_by_id, :project_id, :created_at, :updated_at +json.extract! download, :id, :name, :description, :request, :expires, :times_downloaded, :total_records, :is_public, :type, :sha2, :created_by_id, :updated_by_id, :project_id, :created_at, :updated_at json.ready download.ready? json.file_url file_download_url(download) json.partial! '/shared/data/all/metadata', object: download diff --git a/app/views/downloads/api/v1/_attributes.json.jbuilder b/app/views/downloads/api/v1/_attributes.json.jbuilder index 327b3c7204..9874d96b8c 100644 --- a/app/views/downloads/api/v1/_attributes.json.jbuilder +++ b/app/views/downloads/api/v1/_attributes.json.jbuilder @@ -1,2 +1,5 @@ -json.extract! download, :id, :name, :type, :description, :filename, :request, :expires, :created_at, :updated_at, :project_id -json.download api_v1_api_download_file_url(download) +json.extract! download, :id, :name, :type, :description, :filename, :times_downloaded, :request, :expires, :sha2, :created_at, :updated_at, :project_id + +if download.persisted? && !download.expired? + json.file api_v1_download_file_url(download) +end diff --git a/app/views/downloads/api/v1/show.json.jbuilder b/app/views/downloads/api/v1/show.json.jbuilder index 82813fa362..549d185b99 100644 --- a/app/views/downloads/api/v1/show.json.jbuilder +++ b/app/views/downloads/api/v1/show.json.jbuilder @@ -1 +1,9 @@ -json.partial! '/downloads/api/v1/attributes', download: @download +if @download.persisted? + json.status download_status(@download) + + json.download do + json.partial! '/downloads/api/v1/attributes', download: @download + end +else # has to be invalid at this point + json.error_messages @download.errors.messages +end diff --git a/app/views/downloads/api/v1/status.json.jbuilder b/app/views/downloads/api/v1/status.json.jbuilder deleted file mode 100644 index 0d9dc5849d..0000000000 --- a/app/views/downloads/api/v1/status.json.jbuilder +++ /dev/null @@ -1,15 +0,0 @@ -if @download.valid? - - json.download do - json.partial! '/downloads/api/v1/attributes', download: @download - end - - json.status do - json.status download_status(@download) - end - -else - json.status do - json.error_messages @download.errors.messages - end -end diff --git a/app/views/projects/_form.html.erb b/app/views/projects/_form.html.erb index f01f5a6cba..00d34f9f84 100644 --- a/app/views/projects/_form.html.erb +++ b/app/views/projects/_form.html.erb @@ -13,6 +13,12 @@ <%= radio_button_tag 'project[clear_api_access_token]', true %> <%= f.label :clear_api_access_token %> Disables public API access to all project data. + <% if !is_project_member_by_id?(sessions_current_user_id, sessions_current_project_id) -%> +

+ <%= check_box_tag 'project[project_members_attributes][0][user_id]', sessions_current_user_id, true %> <%= f.label :add_yourself_to_project %>
+
+ <% end %> +
<%= f.submit %>
diff --git a/app/views/projects/show.html.erb b/app/views/projects/show.html.erb index f15bda82f2..78d64efc4f 100644 --- a/app/views/projects/show.html.erb +++ b/app/views/projects/show.html.erb @@ -1,10 +1,11 @@

<%= @project.name -%>

+<%= project_login_link(@project) -%>

Edit

    - <%= content_tag(:li, link_to('Edit', edit_project_path(@project)) ) %> - <%= content_tag(:li, link_to('Preferences', project_preferences_task_path, data: { turbolinks: false } ) ) %> + <%= tag.li( link_to('Edit', edit_project_path(@project)) ) %> + <%= tag.li( link_to('Preferences', project_preferences_task_path, data: { turbolinks: false } ) ) %>

Project attributes

@@ -16,9 +17,9 @@

Members

    - <%= content_tag(:li, add_project_member_link(@project)) -%> - <%= content_tag(:li, link_to('Add many project members', many_new_project_members_path(project_member: {project_id: @project}) )) -%> - <%= content_tag(:li, link_to('Add new user', signup_path)) -%> + <%= tag.li( add_project_member_link(@project)) -%> + <%= tag.li( link_to('Add many project members', many_new_project_members_path(project_member: {project_id: @project}) )) -%> + <%= tag.li( link_to('Add new user', signup_path)) -%>
<% if @project.users.any? -%> @@ -63,7 +64,17 @@

Metadata

    - <%= content_tag(:li, link_to('Stats', project_activity_task_path, data: { turbolinks: false })) -%> + <%= tag.li(link_to('Stats', project_activity_task_path, data: { turbolinks: false })) -%>
+ +

API Helpers

+
    + <% if is_project_member_by_id?(sessions_current_user_id, sessions_current_project_id) %> + <%= tag.li("?project_token=#{@project.api_access_token}&token=#{sessions_current_user.api_access_token}") -%> + <% else %> + <%= tag.li("?project_token=#{@project.api_access_token}") -%> + <% end %> +
+
diff --git a/config/routes/api_v1.rb b/config/routes/api_v1.rb index e2583973a9..49239db4ef 100644 --- a/config/routes/api_v1.rb +++ b/config/routes/api_v1.rb @@ -36,23 +36,23 @@ defaults authenticate_user: true, authenticate_project: true do # authenticated by user and project get '/both_authenticated', to: 'base#index' + + post '/downloads/build', to: '/downloads#api_build', as: :download_build + delete '/downloads/:id', to: '/downloads#api_destroy', as: :download_destroy # validate type, etc. end + # There should be no post or delete in this section defaults authenticate_user_or_project: true do + get '/downloads/:id', to: '/downloads#api_show', as: :download_show + get '/downloads/:id/file', to: '/downloads#api_file', as: :download_file + get '/downloads', to: '/downloads#api_index' + get '/otus', to: '/otus#api_index' get '/otus/autocomplete', to: '/otus#api_autocomplete' get '/otus/:id/inventory/descendants', to: '/otus#api_descendants', as: :api_descendants get '/otus/:otu_id/inventory/images', to: '/images#api_image_inventory', as: :api_images get '/otus/:id', to: '/otus#api_show' - get '/downloads/build', to: '/downloads#api_build', as: :api_download_build - get '/downloads/:id', to: '/downloads#api_show' - get '/downloads', to: '/downloads#api_index' - get '/downloads/:id/file', to: '/downloads#api_file', as: :api_download_file - - get '/downloads/:id/terminate', to: '/downloads#api_terminate', as: :api_download_terminate - get '/downloads/:id/status', to: '/downloads#api_status', as: :api_download_status - get '/dwc_occurrences', to: '/dwc_occurrences#api_index' get '/taxon_names', to: '/taxon_names#api_index' diff --git a/db/migrate/20220421182414_add_sha2_to_download.rb b/db/migrate/20220421182414_add_sha2_to_download.rb new file mode 100644 index 0000000000..e07119ae70 --- /dev/null +++ b/db/migrate/20220421182414_add_sha2_to_download.rb @@ -0,0 +1,5 @@ +class AddSha2ToDownload < ActiveRecord::Migration[6.1] + def change + add_column :downloads, :sha2, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 21f27cdbff..b47a23bba1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2022_03_31_023656) do +ActiveRecord::Schema.define(version: 2022_04_21_182414) do # These are extensions that must be enabled in order to support this database enable_extension "fuzzystrmatch" @@ -659,6 +659,7 @@ t.boolean "is_public" t.string "type" t.integer "total_records" + t.string "sha2" t.index ["created_by_id"], name: "index_downloads_on_created_by_id" t.index ["filename"], name: "index_downloads_on_filename" t.index ["project_id"], name: "index_downloads_on_project_id" diff --git a/spec/models/download/dwc_archive/complete_spec.rb b/spec/models/download/dwc_archive/complete_spec.rb new file mode 100644 index 0000000000..a0aba9bbe6 --- /dev/null +++ b/spec/models/download/dwc_archive/complete_spec.rb @@ -0,0 +1,148 @@ +require 'rails_helper' + +RSpec.describe Download::DwcArchive::Complete, type: :model do + # TODO: Avoid having to merge file_path when factory already provides this + let(:valid_attributes) { + strip_housekeeping_attributes(FactoryBot.build(:valid_download).attributes.merge({ source_file_path: Rails.root.join('spec/files/downloads/Sample.zip') })) + } + + let(:valid_attributes_no_file) { + strip_housekeeping_attributes(FactoryBot.build(:valid_download_no_file).attributes) + } + + let(:download) { DownloadDwC.create! valid_attributes } + let(:download_no_file) { Download.create! valid_attributes_no_file } + let(:file_path) { Download.storage_path.join(*download.id.to_s.rjust(9, '0').scan(/.../), download.filename) } + + specify '#is_public defaults true' do + a = Download::DwcArchive::Complete.new + a.save! + expect(a.is_public).to eq(true) + end + + specify 'init' do + expect(Download::DwcArchive::Complete.new.save!).to be_truthy + end +=begin +describe "default scope" do +let(:expiry_dates) { [1.day.ago, 2.day.ago, 1.minute.from_now, 1.day.from_now, 1.week.from_now] } + +before do + expiry_dates.each do | time | + Download.create! valid_attributes.merge({ expires: time }) + end +end + +it "returns non-expired downloads only" do + expect(Download.all.count).to eq(3) +end + +it "can be avoided with 'unscoped'" do + expect(Download.unscoped.all.count).to eq(5) +end +end + +describe "#create" do + context "when source_file_path provided" do + it "stores file in download directory" do + expect(file_path.exist?).to be_truthy + end + end +end + +describe "#file_path" do + context "on create" do + it "points to storage location" do + expect(download.file_path).to eq(file_path) + end + end + + context "on load" do + it "points to storage location" do + loaded = Download.find(download.id) + expect(download.file_path).to eq(Download.storage_path.join(*loaded.id.to_s.rjust(9, '0').scan(/.../), loaded.filename)) + end + end + + describe "directory structure" do + it "is three-level deep composed of the zero-padded id split in groups of three (id < 10^9)" do + download = Download.new(valid_attributes) + download.id = 1234567 + download.save! + + expect(download.file_path).to eq(Download.storage_path.join('001', '234', '567', download.filename)) + end + + it "outermost directory is longer than the others when id >= 10^9" do + download = Download.new(valid_attributes) + download.id = 1234567890 + download.save! + + expect(download.file_path).to eq(Download.storage_path.join('1234', '567', '890', download.filename)) + end + end +end + +describe "#expired?" do + it "is true when expire date is lower than current time" do + download.expires = 1.second.ago + + expect(download.expired?).to be_truthy + end + + it "is false when expire is date is higher or equal to current time" do + download.expires = 1.day.from_now + + expect(download.expired?).to be_falsey + end +end + +describe "#ready?" do + context "when not expired" do + context "and created with file" do + it "is true when file is present" do + expect(download.ready?).to be_truthy + end + + it "is false when the file is deleted" do + download.delete_file + expect(download.ready?).to be_falsey + end + end + + context "and created without file" do + it "is false when file is absent" do + expect(download_no_file.ready?).to be_falsey + end + + it "is true when file is added afterwards" do + download_no_file.update!(source_file_path: Rails.root.join('spec/files/downloads/Sample.zip')) + expect(download_no_file.ready?).to be_truthy + end + end + end + context "when expired" do + it "it is false" do + download.expires = 1.second.ago + expect(download.ready?).to be_falsey + end + end +end + +describe "#delete file" do + it "removes the file from storage" do + download.delete_file + expect(file_path.exist?).to be_falsey + end +end + +describe "#destroy" do + it "Removes the file from storage" do + download.destroy + expect(file_path.exist?).to be_falsey + end +end + +=end + +end From 82ed718daea3c616d4aa09c166be487f5e86ed0b Mon Sep 17 00:00:00 2001 From: mjy Date: Thu, 21 Apr 2022 14:46:50 -0600 Subject: [PATCH 4/8] Return .unscoped. --- app/controllers/downloads_controller.rb | 10 ++++++---- app/models/download.rb | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/controllers/downloads_controller.rb b/app/controllers/downloads_controller.rb index fc547aad91..0624b26842 100644 --- a/app/controllers/downloads_controller.rb +++ b/app/controllers/downloads_controller.rb @@ -11,7 +11,7 @@ class DownloadsController < ApplicationController def index respond_to do |format| format.html do - @recent_objects = Download.recent_from_project_id(sessions_current_project_id).order(updated_at: :desc).limit(10) + @recent_objects = Download.unscoped.recent_from_project_id(sessions_current_project_id).order(updated_at: :desc).limit(10) render '/shared/data/all/index' end format.json { @@ -64,7 +64,8 @@ def update # GET /downloads/list # GET /downloads/list.json def list - @downloads = Download.where(project_id: sessions_current_project_id).order(:id).page(params[:page]).per(params[:per]) + # has a default scope + @downloads = Download.unscoped.where(project_id: sessions_current_project_id).order(:id).page(params[:page]).per(params[:per]) end # GET /downloads/1/file @@ -78,6 +79,7 @@ def file end def api_index + # If default scope is removed return here @downloads = Download.where(project_id: sessions_current_project_id) .order('downloads.id').page(params[:page]).per(params[:per]) render '/downloads/api/v1/index' @@ -125,11 +127,11 @@ def filter_params def set_download # Why .unscoped ? - @download = Download.where(project_id: sessions_current_project_id).find(params[:id]) + @download = Download.unscoped.where(project_id: sessions_current_project_id).find(params[:id]) end def set_download_api - @download = Download.where(is_public: true, project_id: sessions_current_project_id).find(params[:id]) + @download = Download.unscoped.where(is_public: true, project_id: sessions_current_project_id).find(params[:id]) end def download_params diff --git a/app/models/download.rb b/app/models/download.rb index d6a3e2ef2a..3ed5edd0d1 100644 --- a/app/models/download.rb +++ b/app/models/download.rb @@ -45,6 +45,7 @@ class Download < ApplicationRecord include Housekeeping include Shared::IsData + # TODO: consider removing default_scope { where('expires >= ?', Time.now) } after_save :save_file From c7f7e2b1e535f2c3c602660701d81bfcae06d543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A1n=20Lucas=20Pereira?= Date: Thu, 9 Jun 2022 00:15:14 -0300 Subject: [PATCH 5/8] Moved skipping forgery protection into controller --- app/controllers/concerns/forgery_protection.rb | 12 +++++------- app/controllers/downloads_controller.rb | 2 ++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/controllers/concerns/forgery_protection.rb b/app/controllers/concerns/forgery_protection.rb index 5d78805c10..4ab42c4d96 100644 --- a/app/controllers/concerns/forgery_protection.rb +++ b/app/controllers/concerns/forgery_protection.rb @@ -7,14 +7,12 @@ module ForgeryProtection protect_from_forgery def handle_unverified_request - unless @api_request # @locodelassembly please check this out, this disables CSRF for API requests that are delete/delete - respond_to do |format| - format.html do - flash[:notice] = "Your last request could not be fulfilled. Please retry." - redirect_to '/' - end - format.json { render body: '{ "success": false }', status: :unprocessable_entity } + respond_to do |format| + format.html do + flash[:notice] = "Your last request could not be fulfilled. Please retry." + redirect_to '/' end + format.json { render body: '{ "success": false }', status: :unprocessable_entity } end end diff --git a/app/controllers/downloads_controller.rb b/app/controllers/downloads_controller.rb index 0624b26842..b5eb39f7bf 100644 --- a/app/controllers/downloads_controller.rb +++ b/app/controllers/downloads_controller.rb @@ -6,6 +6,8 @@ class DownloadsController < ApplicationController after_action -> { set_pagination_headers(:downloads) }, only: [:api_index], if: :json_request? + skip_forgery_protection only: [:api_build, :api_destroy] + # GET /downloads # GET /downloads.json def index From b33c2350918a13376766f99b18930887fe9134bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A1n=20Lucas=20Pereira?= Date: Fri, 19 Aug 2022 20:03:50 -0300 Subject: [PATCH 6/8] Simplified SHA256 calculation and made 256 explicit given SHA2 is a family --- app/models/download.rb | 9 +-------- app/views/downloads/_attributes.html.erb | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/app/models/download.rb b/app/models/download.rb index 3ed5edd0d1..cb651465a5 100644 --- a/app/models/download.rb +++ b/app/models/download.rb @@ -108,16 +108,9 @@ def api_buildable? STORAGE_PATH = Rails.root.join(Rails.env.test? ? 'tmp' : '', "downloads#{ENV['TEST_ENV_NUMBER']}").freeze - # TODO: check performance on 50-100mb files def set_sha2 if @source_file_path - s = ::Digest::SHA2.new - File.open(@source_file_path) do |f| - while chunk = f.read(256) # only load 256 bytes at a time - s << chunk - end - end - self.update_column(:sha2, s.hexdigest) + self.update_column(:sha2, Digest::SHA256.file(@source_file_path).to_s) end end diff --git a/app/views/downloads/_attributes.html.erb b/app/views/downloads/_attributes.html.erb index f739ef5f85..32626251e5 100644 --- a/app/views/downloads/_attributes.html.erb +++ b/app/views/downloads/_attributes.html.erb @@ -47,7 +47,7 @@

- SHA2: + SHA256: <%= @download.sha2 %>

<% else %> From 3395b65b0c7a1803a4fcc387ea091b519df33863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A1n=20Lucas=20Pereira?= Date: Fri, 19 Aug 2022 20:14:46 -0300 Subject: [PATCH 7/8] Added test to confirm SHA256 is calculated and matches value from external tool --- spec/models/download/dwc_archive/complete_spec.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/models/download/dwc_archive/complete_spec.rb b/spec/models/download/dwc_archive/complete_spec.rb index a0aba9bbe6..171fdb454b 100644 --- a/spec/models/download/dwc_archive/complete_spec.rb +++ b/spec/models/download/dwc_archive/complete_spec.rb @@ -10,7 +10,7 @@ strip_housekeeping_attributes(FactoryBot.build(:valid_download_no_file).attributes) } - let(:download) { DownloadDwC.create! valid_attributes } + let(:download) { Download.create! valid_attributes } let(:download_no_file) { Download.create! valid_attributes_no_file } let(:file_path) { Download.storage_path.join(*download.id.to_s.rjust(9, '0').scan(/.../), download.filename) } @@ -23,6 +23,12 @@ specify 'init' do expect(Download::DwcArchive::Complete.new.save!).to be_truthy end + + specify 'sha2' do + # $ sha256sum spec/files/downloads/Sample.zip + # 26a2c50757bd7068e82b6698d7dfd5974ebe68e950b5454034593741c925023d spec/files/downloads/Sample.zip + expect(download.sha2).to eq('26a2c50757bd7068e82b6698d7dfd5974ebe68e950b5454034593741c925023d') + end =begin describe "default scope" do let(:expiry_dates) { [1.day.ago, 2.day.ago, 1.minute.from_now, 1.day.from_now, 1.week.from_now] } From 352a286f44b4777639bd78227dc3d3581d4a21ef Mon Sep 17 00:00:00 2001 From: mjy Date: Wed, 30 Nov 2022 11:48:24 -0600 Subject: [PATCH 8/8] Fix bad merge --- app/helpers/projects_helper.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 370d8c9bd8..9eb5b9a198 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -101,6 +101,4 @@ def project_classification(project) } end - ->>>>>>> development end