HYC-1774 - Bulkrax importer fixes (hyrax-4)

Gemfile

@@ -12,7 +12,7 @@ gem 'blacklight_advanced_search', '~> 8.0.0.alpha2'
 gem 'blacklight_oai_provider', '7.0.2'
 gem 'blacklight_range_limit', '~> 8.3.0'
 gem 'bootstrap', '~> 4.0'
-gem 'bulkrax', '~> 5.0'
+gem 'bulkrax', git: 'https://github.com/UNC-Libraries/bulkrax.git', branch: 'unc-bulkrax-5-development'
 gem 'clamav-client', require: 'clamav/client'
 # Use CoffeeScript for .coffee assets and views
 gem 'coffee-rails', '~> 5.0.0'

Gemfile.lock

@@ -1,3 +1,24 @@
+GIT
+  remote: https://github.com/UNC-Libraries/bulkrax.git
+  revision: ccbc380e1a51543017b9ddab5cd5d03f46fd293d
+  branch: unc-bulkrax-5-development
+  specs:
+    bulkrax (5.4.1)
+      bagit (~> 0.4)
+      coderay
+      dry-monads (~> 1.5.0)
+      iso8601 (~> 0.9.0)
+      kaminari
+      language_list (~> 1.2, >= 1.2.1)
+      libxml-ruby (~> 3.2.4)
+      loofah (>= 2.2.3)
+      oai (>= 0.4, < 2.x)
+      rack (>= 2.0.6)
+      rails (>= 5.1.6)
+      rdf (>= 2.0.2, < 4.0)
+      rubyzip
+      simple_form
+
 GIT
   remote: https://github.com/UNC-Libraries/hyrax.git
   revision: 916bb20935606fe84073cf303dca43c6bf0ecc7f
@@ -173,7 +194,7 @@ GEM
     babel-transpiler (0.7.0)
       babel-source (>= 4.0, < 6)
       execjs (~> 2.0)
-    bagit (0.4.5)
+    bagit (0.4.6)
       docopt (~> 0.5.0)
       validatable (~> 1.6)
     base64 (0.1.1)
@@ -227,20 +248,6 @@ GEM
       signet (~> 0.8)
       typhoeus
     builder (3.2.4)
-    bulkrax (5.2.1)
-      bagit (~> 0.4)
-      coderay
-      iso8601 (~> 0.9.0)
-      kaminari
-      language_list (~> 1.2, >= 1.2.1)
-      libxml-ruby (~> 3.2.4)
-      loofah (>= 2.2.3)
-      oai (>= 0.4, < 2.x)
-      rack (>= 2.0.6)
-      rails (>= 5.1.6)
-      rdf (>= 2.0.2, < 4.0)
-      rubyzip
-      simple_form
     byebug (11.1.3)
     cancancan (3.5.0)
     capybara (3.39.2)
@@ -383,7 +390,8 @@ GEM
     factory_bot_rails (6.2.0)
       factory_bot (~> 6.2.0)
       railties (>= 5.0.0)
-    faraday (2.7.10)
+    faraday (2.7.11)
+      base64
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-encoding (0.0.5)
@@ -408,8 +416,8 @@ GEM
     font-awesome-rails (4.7.0.8)
       railties (>= 3.2, < 8.0)
     geocoder (1.8.2)
-    globalid (1.1.0)
-      activesupport (>= 5.0)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
     google-apis-core (0.11.1)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.16.2, < 2.a)
@@ -626,7 +634,7 @@ GEM
     mini_racer (0.2.15)
       libv8 (> 7.3)
     minitar (0.9)
-    minitest (5.19.0)
+    minitest (5.20.0)
     multi_json (1.15.0)
     multi_xml (0.6.0)
     multipart-post (2.3.0)
@@ -641,7 +649,7 @@ GEM
       net-protocol
     net-protocol (0.2.1)
       timeout
-    net-smtp (0.3.3)
+    net-smtp (0.4.0)
       net-protocol
     nio4r (2.5.9)
     noid (0.9.0)
@@ -1061,7 +1069,7 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.11)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   x86_64-linux
@@ -1072,7 +1080,7 @@ DEPENDENCIES
   blacklight_oai_provider (= 7.0.2)
   blacklight_range_limit (~> 8.3.0)
   bootstrap (~> 4.0)
-  bulkrax (~> 5.0)
+  bulkrax!
   byebug (~> 11.1.3)
   capybara (~> 3.36)
   clamav-client

app/assets/stylesheets/unc_custom.css.scss

@@ -1118,6 +1118,10 @@ Hide it entirely until the issue is resolved upstreeam
     .nav-link[href="/leases?locale=en"], .nav-link[href^="/dashboard/profiles"] {
       display: none;
     }
+    /* Temporary fix so that bulkrax links format correctly, until the gem is updated */
+    .nav-item a {
+      display: block;
+    }
   }
 }
 
@@ -1143,6 +1147,12 @@ Hide it entirely until the issue is resolved upstreeam
   }
 }
 
+.bulkrax-align-text {
+  .panel-body {
+    width: 100%;
+  }
+}
+
 .search-form {
   label {
     visibility: hidden;

app/overrides/models/bulkrax/csv_entry_override.rb

@@ -16,9 +16,10 @@ def invalid_model_type(record)
   end
 
   def build_object(_key, value)
-    # [hyc-override] skip mapped fields that don't exist for the current record
-    data = hyrax_record.send(value['object']) if hyrax_record.respond_to?(value['object'])
-    return if data.nil? || data.empty?
+    return unless hyrax_record.respond_to?(value['object'])
+
+    data = hyrax_record.send(value['object'])
+    return if data.empty?
 
     data = data.to_a if data.is_a?(ActiveTriples::Relation)
     # [hyc-override] convert people objects to the hash serialization expected by bulkrax
@@ -37,6 +38,14 @@ def serialize_people(data)
 
   # [hyc-override] allow object_name to be passed down
   def object_metadata(data, object_name = nil)
+    # NOTE: What is `d` in this case:
+    #
+    #  "[{\"single_object_first_name\"=>\"Fake\", \"single_object_last_name\"=>\"Fakerson\", \"single_object_position\"=>\"Leader, Jester, Queen\", \"single_object_language\"=>\"english\"}]"
+    #
+    # The above is a stringified version of a Ruby string.  Using eval is a very bad idea as it
+    # will execute the value of `d` within the full Ruby interpreter context.
+    #
+    # TODO: Would it be possible to store this as a non-string?  Maybe the actual Ruby Array and Hash?
     data = data.map { |d| eval(d) }.flatten # rubocop:disable Security/Eval
 
     data.each_with_index do |obj, index|

app/overrides/models/concerns/bulkrax/has_matchers_override.rb

@@ -1,18 +1,20 @@
 # frozen_string_literal: true
+# [hyc-override] https://github.com/samvera-labs/bulkrax/tree/v5.3.0/app/models/concerns/bulkrax/has_matchers.rb
 
-# [hyc-override] updating the `multiple?` method to skip attributes without properties
 Bulkrax::HasMatchers.module_eval do
   def multiple?(field)
+    # [hyc-override] removed rights_statement from the list since it is not multiple for us
     @multiple_bulkrax_fields ||=
-      [
-        'file',
-        'remote_files',
-        "#{related_parents_parsed_mapping}",
-        "#{related_children_parsed_mapping}"
-      ]
+        %W[
+          file
+          remote_files
+          #{related_parents_parsed_mapping}
+          #{related_children_parsed_mapping}
+        ]
 
     return true if @multiple_bulkrax_fields.include?(field)
     return false if field == 'model'
+    # [hyc-override] updating the `multiple?` method to skip attributes without properties
     return false if factory_class.properties[field].blank?
 
     field_supported?(field) && factory_class&.properties&.[](field)&.[]('multiple')

app/overrides/parsers/bulkrax/csv_parser_override.rb

@@ -15,9 +15,9 @@ def write_import_file(file)
   end
 
   def valid_import?
-    import_strings = keys_without_numbers(import_fields.map(&:to_s))
-    error_alert = "Missing at least one required element, missing element(s) are: #{missing_elements(import_strings).join(', ')}"
-    raise StandardError, error_alert unless required_elements?(import_strings)
+    compressed_record = records.flat_map(&:to_a).partition { |_, v| !v }.flatten(1).to_h
+    error_alert = "Missing at least one required element, missing element(s) are: #{missing_elements(compressed_record).join(', ')}"
+    raise StandardError, error_alert unless required_elements?(compressed_record)
     # [hyc-override] explicitly raise error when file paths are not present
     raise StandardError.new 'file paths are invalid' unless file_paths.is_a?(Array)
     true

app/views/bulkrax/importers/new.html.erb

@@ -0,0 +1,30 @@
+<%# [hyc-override] https://github.com/samvera-labs/bulkrax/tree/v5.3.0/app/views/bulkrax/importers/new.html.erb %>
+<% provide :page_header do %>
+  <h1><span class="fa fa-edit" aria-hidden="true"></span> New Importer</h1>
+<% end %>
+
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default tabs importer-form">
+      <%= simple_form_for @importer, html: { multipart: true } do |form| %>
+        <%= render 'form', importer: @importer, form: form %>
+        <div class="panel-footer">
+          <div class='pull-right'>
+            <% if ENV['SHOW_CREATE_AND_VALIDATE'] == 'true' %>
+              <%# [hyc-override] Removing validate option since it is very buggy right now %>
+              <%#= form.button :submit, value: 'Create and Validate', class: 'btn btn-primary' %>
+              <%# | %>
+            <% end  %>
+            <%= form.button :submit, value: 'Create and Import', class: 'btn btn-primary' %>
+            |
+            <%# [hyc-override] removing "create" button since there is no separate functionality defined for it at the moment %>
+            <%#= form.button :submit, value: 'Create', class: 'btn btn-primary' %>
+            <%# | %>
+            <% cancel_path = form.object.persisted? ? importer_path(form.object) : importers_path %>
+            <%= link_to t('.cancel'), cancel_path, class: 'btn btn-default ' %>
+          </div>
+        </div>
+      <% end %>
+    </div>
+  </div>
+</div>

config/database.yml

@@ -28,6 +28,8 @@ development:
   username: hyrax
   password: <%= ENV['HYRAX_DATABASE_PASSWORD'] %>
   url: <%= ENV['DATABASE_URL'] %>
+  # Increase pool size on in dev, otherwise sidekiq (running in its own vm) runs out of connections
+  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } * 2 %>
 
   # The specified database role being used to connect to postgres.
   # To create additional roles in postgres see `$ createuser --help`.

config/initializers/bulkrax.rb

@@ -212,4 +212,6 @@
 end
 
 # Sidebar for hyrax 3+ support
-Hyrax::DashboardController.sidebar_partials[:repository_content] << 'hyrax/dashboard/sidebar/bulkrax_sidebar_additions' if Object.const_defined?(:Hyrax) && ::Hyrax::DashboardController&.respond_to?(:sidebar_partials)
+if Object.const_defined?(:Hyrax) && ::Hyrax::DashboardController&.respond_to?(:sidebar_partials)
+  Hyrax::DashboardController.sidebar_partials[:repository_content] << 'hyrax/dashboard/sidebar/bulkrax_sidebar_additions'
+end