Merge branch 'release/0.6.2'

.github/workflows/ci.yml

@@ -29,12 +29,12 @@ jobs:
         image: postgres:latest
         env:
           POSTGRES_HOST: 127.0.0.1
-          POSTGRES_PORT: 5555
+          POSTGRES_PORT: 5432
           POSTGRES_DB: geoblacklight_admin_development
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
         ports:
-          - 5555:5432
+          - 5432:5432
         # Set health checks to wait until postgres has started
         options:
           --health-cmd pg_isready

app/controllers/admin/document_distributions_controller.rb

@@ -125,27 +125,28 @@ def import
         raise ArgumentError, "File does not exist or is invalid."
       end
 
-      if DocumentDistribution.import(params.dig(:document_distribution, :distributions, :file))
+      success, errors = DocumentDistribution.import(params.dig(:document_distribution, :distributions, :file))
+      if success == true
         logger.debug("Distributions were created successfully.")
         if params[:document_id]
           redirect_to admin_document_document_distributions_path(@document), notice: "Distributions were created successfully."
         else
-          redirect_to admin_document_document_distributions_path, notice: "Distributions were created successfully."
+          redirect_to admin_document_distributions_path, notice: "Distributions were created successfully."
         end
       else
-        logger.debug("Distributions could not be created.")
+        logger.debug("Some distributions could not be created. #{errors.join(", ")}")
         if params[:document_id]
-          redirect_to admin_document_document_distributions_path(@document), warning: "Distributions could not be created."
+          redirect_to admin_document_document_distributions_path(@document), notice: "Some distributions could not be created. #{errors.join(", ")}"
         else
-          redirect_to admin_document_document_distributions_path, warning: "Distributions could not be created."
+          redirect_to admin_document_distributions_path, notice: "Some distributions could not be created. #{errors.join(", ")}"
         end
       end
     rescue => e
       logger.debug("Distributions could not be created. #{e}")
       if params[:document_id]
         redirect_to admin_document_document_distributions_path(@document), notice: "Distributions could not be created. #{e}"
       else
-        redirect_to admin_document_document_distributions_path, notice: "Distributions could not be created. #{e}"
+        redirect_to admin_document_distributions_path, notice: "Distributions could not be created. #{e}"
       end
     end
 

app/models/document.rb

@@ -147,10 +147,8 @@ def distributions
 
       # BEFORE - Apply Multiple Downloads
       # @TODO: Remove this once we've migrated to DocumentDistributions
-      if ENV["GBL_ADMIN_REFERENCES_MIGRATED"] == "false"
-        distributions = apply_downloads(distributions)
-        logger.debug("Document#distributions > downloads: #{distributions}")
-      end
+      distributions = apply_downloads(distributions)
+      logger.debug("Document#distributions > downloads: #{distributions}")
     end
 
     # BEFORE & AFTER - Apply Distributable Assets
@@ -192,12 +190,24 @@ def distributions_csv
     csv = []
 
     distributions.each do |key, value|
-      if key == "http://schema.org/downloadUrl"
+      if key == "http://schema.org/downloadUrl" || key == :"http://schema.org/downloadUrl"
         value.each do |download|
-          csv << [friendlier_id, ReferenceType.find_by(reference_uri: key).name, download["url"], download["label"]]
+          logger.debug("Document#distributions_csv > download: #{download.inspect}")
+
+          csv << [
+            friendlier_id,
+            ReferenceType.find_by(reference_uri: key).name,
+            download["url"],
+            download["label"]
+          ]
         end
       else
-        csv << [friendlier_id, ReferenceType.find_by(reference_uri: key)&.name, value, nil]
+        csv << [
+          friendlier_id,
+          ReferenceType.find_by(reference_uri: key)&.name,
+          value,
+          nil
+        ]
       end
     end
     csv
@@ -264,7 +274,7 @@ def apply_downloads(distributions)
 
     multiple_downloads = multiple_downloads.uniq { |d| [d[:label], d[:url]] } unless multiple_downloads.empty?
 
-    distributions[:"http://schema.org/downloadUrl"] = multiple_downloads.flatten unless multiple_downloads.empty?
+    distributions["http://schema.org/downloadUrl"] = multiple_downloads.flatten unless multiple_downloads.empty?
     distributions
   end
 

app/models/document_distribution.rb

@@ -65,14 +65,22 @@ def self.csv_column_names
   # @param file [File] the CSV file to import
   # @return [Boolean] true if import is successful
   def self.import(file)
+    @errors = []
+
     logger.debug("CSV Import")
     ::CSV.foreach(file.path, headers: true) do |row|
       logger.debug("CSV Row: #{row.to_hash}")
+
+      unless Document.exists?(friendlier_id: row.to_hash["friendlier_id"])
+        logger.debug("Document not found: #{row.to_hash["friendlier_id"]}")
+        @errors << "Document not found: #{row.to_hash["friendlier_id"]}"
+        next
+      end
+
       document_distribution = DocumentDistribution.find_or_initialize_by(
         friendlier_id: row.to_hash["friendlier_id"],
         reference_type_id: ReferenceType.find_by(name: row.to_hash["reference_type"]).id,
-        url: row.to_hash["distribution_url"],
-        label: row.to_hash["label"]
+        url: row.to_hash["distribution_url"]
       )
 
       logger.debug("Document Distribution: #{document_distribution.inspect}")
@@ -84,7 +92,7 @@ def self.import(file)
         label: row.to_hash["label"]
       )
     end
-    true
+    [@errors.empty?, @errors]
   end
 
   # Destroy All
@@ -132,7 +140,7 @@ def to_csv
   def to_aardvark_distribution
     hash = {}
     hash[reference_type.reference_uri.to_s] = url
-    hash[:label] = label if reference_type.reference_uri.to_s == "http://schema.org/downloadUrl"
+    hash["label"] = label if reference_type.reference_uri.to_s == "http://schema.org/downloadUrl"
     hash
   end
 

app/views/admin/document_distributions/import.html.erb

@@ -20,7 +20,10 @@
     </p>
 
     <h3 class="h4">Upload a CSV File</h3>
-    <%= simple_form_for DocumentDistribution.new, url: import_admin_document_distributions_path, method: :post, multipart: true do |f| %>
+    <%= simple_form_for DocumentDistribution.new, 
+        url: @document || params[:document_id] ? import_admin_document_document_distributions_path(@document) : import_admin_document_distributions_path, 
+        method: :post, 
+        multipart: true do |f| %>
       <div class="form-inputs">
         <%= f.simple_fields_for :distributions do |distribution_fields| %>
           <%= distribution_fields.input :file, as: :file, input_html: {} %>

app/views/admin/document_distributions/index.html.erb

@@ -59,32 +59,34 @@
           <h3>External – Document Distributions</h3>
         <% end %>
 
-        <table class="table table-striped table-bordered sortable">
-          <thead class="thead-dark">
-            <tr>
-              <th class="header">Layer Slug</th>
-              <th class="header">Type</th>
-              <th class="header">URI</th>
-              <th class="header">Value</th>
-              <th class="header">Label</th>
-              <th class="header" colspan="2">Actions</th>
-            </tr>
-          </thead>
-
-          <tbody>
-            <% @document_distributions.each do |document_distribution| %>
+        <div class="table-responsive">
+          <table class="table table-striped table-bordered sortable">
+            <thead class="thead-dark">
               <tr>
-                <td><%= link_to document_distribution.friendlier_id, admin_document_path(document_distribution.document) %></td>
-                <td><%= document_distribution.reference_type.reference_type %></td>
-                <td><%= document_distribution.reference_type.reference_uri %></td>
-                <td><%= link_to document_distribution.url, document_distribution.url, target: '_blank' %></td>
-                <td><%= document_distribution.label %></td>
-                <td><%= link_to 'Edit', edit_admin_document_document_distribution_path(document_distribution.document, document_distribution) %></td>
-                <td><%= link_to 'Destroy', admin_document_document_distribution_path(document_distribution.document, document_distribution), method: :delete, data: { confirm: 'Are you sure?' } %></td>
+                <th class="header">Layer Slug</th>
+                <th class="header">Type</th>
+                <th class="header">URI</th>
+                <th class="header">Value</th>
+                <th class="header">Label</th>
+                <th class="header" colspan="2">Actions</th>
               </tr>
-            <% end %>
-          </tbody>
-        </table>
+            </thead>
+
+            <tbody>
+              <% @document_distributions.each do |document_distribution| %>
+                <tr>
+                  <td><%= link_to document_distribution.friendlier_id, admin_document_path(document_distribution.document) %></td>
+                  <td><%= document_distribution.reference_type.reference_type %></td>
+                  <td><%= document_distribution.reference_type.reference_uri %></td>
+                  <td><%= link_to document_distribution.url, document_distribution.url, target: '_blank' %></td>
+                  <td><%= document_distribution.label %></td>
+                  <td><%= link_to 'Edit', edit_admin_document_document_distribution_path(document_distribution.document, document_distribution) %></td>
+                  <td><%= link_to 'Destroy', admin_document_document_distribution_path(document_distribution.document, document_distribution), method: :delete, data: { confirm: 'Are you sure?' } %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </div>
 
         <% if @pagy %>
           <h6>

app/views/admin/documents/_result_selected_options.html.erb

@@ -28,8 +28,8 @@
       <a class="dropdown-item" data-action="click->results#exportCsvDocumentDownloads" href="javascript:void(0);">
         CSV - Document Downloads
       </a>
-      <a class="dropdown-item" data-action="click->results#exportCsvDocumentReferences" href="javascript:void(0);">
-        CSV - Document References
+      <a class="dropdown-item" data-action="click->results#exportCsvDocumentDistributions" href="javascript:void(0);">
+        CSV - Document Distributions
       </a>
     </div>
   </span>

app/views/admin/imports/show.html.erb

@@ -131,8 +131,8 @@
 
     <nav>
       <div class="nav nav-tabs" id="import-state-tabs" role="tablist">
-        <a class="nav-item nav-link active" id="import-success-tab" data-toggle="tab" href="#import-state-success" role="tab" aria-controls="import-state-success" aria-selected="true">Success</a>
-        <a class="nav-item nav-link" id="import-failed-tab" data-toggle="tab" href="#import-state-failed" role="tab" aria-controls="import-state-failed" aria-selected="false">Failed</a>
+        <a class="nav-item nav-link active" id="import-success-tab" data-toggle="tab" href="#import-state-success" role="tab" aria-controls="import-state-success" aria-selected="true">Success <span class="badge badge-success"><%= @import_success_documents.count %></span></a>
+        <a class="nav-item nav-link" id="import-failed-tab" data-toggle="tab" href="#import-state-failed" role="tab" aria-controls="import-state-failed" aria-selected="false">Failed <span class="badge badge-danger"><%= @import_failed_documents.count %></span> </a>
       </div>
     </nav>
 

docs/upgrading.md

@@ -32,9 +32,9 @@ This is a list of manual changes that need to be made to the parent Rails applic
 
 #### DATA MIGRATION
 
-* Migrate existing AttrJson `dct_reference_s` values to the new `document_reference` table
-  * `rake geoblacklight_admin:references:migrate`
-* Audit the references migration
-  * `rake geoblacklight_admin:references:audit`
-* Set environment variable for `GBL_ADMIN_REFERENCES_MIGRATED` to `true`
-* In the Elements table, update the "References" element's import and export functions to distributions_json (from `references_json`)
+* In the Elements table, update the "References" element's import and export functions to distributions_json (from `references_json`)
+* Migrate existing AttrJson `dct_reference_s` values to the new `document_distributions` table
+  * `rake geoblacklight_admin:distributions:migrate`
+* Audit the distributions migration
+  * `rake geoblacklight_admin:distributions:audit`
+* Set environment variable for `GBL_ADMIN_DISTRIBUTIONS_MIGRATED` to `true`

lib/generators/geoblacklight_admin/config_generator.rb

@@ -216,7 +216,7 @@ def set_routes
               end
             end
 
-            # Document References
+            # Document Distributions
             resources :document_distributions, path: "distributions" do
               collection do
                 get "display_attach_form"

lib/generators/geoblacklight_admin/templates/.env.development.example

@@ -11,7 +11,7 @@ SOLR_URL=http://127.0.0.1:8983/solr/blacklight-core
 
 # PostgreSQL
 POSTGRES_HOST=127.0.0.1
-POSTGRES_PORT=5555
+POSTGRES_PORT=5432
 POSTGRES_DB=geoblacklight_admin_development
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres

lib/geoblacklight_admin/tasks/distributions.rake

@@ -18,9 +18,46 @@ namespace :geoblacklight_admin do
               url: distribution[2],
               label: distribution[3]
             )
+          rescue ActiveRecord::RecordInvalid => e
+            if !distribution[2].nil? && distribution[2].is_a?(Hash)
+              puts "Distribution rescued and skipped: #{distribution.inspect}"
+            else
+              puts "RecordInvalid processing distribution: #{distribution[0]} - #{e.inspect}"
+            end
+          rescue TypeError => e
+            puts "TypeError processing distribution: #{distribution[0]} - #{e.inspect}"
+            puts "Distribution: #{distribution.inspect}"
+
+            # Fix for #<TypeError: can't cast Hash>
+            # These are download links that are not already in an array
+            # ex. "{\"http://schema.org/url\":\"https://datacore.iu.edu/concern/data_sets/hx11xf65s\",\"http://schema.org/downloadUrl\":{\"label\":\"PDF\",\"url\":\"https://datacore.iu.edu/downloads/ms35t9074\"}}"
+            if !distribution[2].nil? && distribution[2].is_a?(Hash)
+              DocumentDistribution.find_or_create_by!(
+                friendlier_id: distribution[0],
+                reference_type_id: ReferenceType.find_by(name: distribution[1]).id,
+                url: distribution[2][:url],
+                label: distribution[2][:label]
+              )
+              puts "Distribution rescued and migrated: #{distribution.inspect}"
+            elsif distribution[2].nil? && distribution[2].is_a?(Array)
+              distribution[2].each do |download|
+                if download.is_a?(Hash) && download[:url].present? && download[:label].present?
+                  DocumentDistribution.find_or_create_by!(
+                    friendlier_id: distribution[0],
+                    reference_type_id: ReferenceType.find_by(name: distribution[1]).id,
+                    url: download[:url],
+                    label: download[:label]
+                  )
+                end
+              end
+              puts "Distribution array rescued and migrated: #{distribution.inspect}"
+            else
+              puts "Distribution not migrated: #{distribution.inspect}"
+            end
+          rescue => e
+            puts "Error processing distribution: #{distribution[0]} - #{e.inspect}"
+            puts "Distribution: #{distribution.inspect}"
           end
-        rescue => e
-          puts "\nError processing distributions for document: #{document.friendlier_id} - #{e.inspect}\n"
         end
         total_documents_processed += documents.size
         puts "Processed #{documents.size} documents in this batch, total processed: #{total_documents_processed}"

lib/geoblacklight_admin/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GeoblacklightAdmin
-  VERSION = "0.6.1"
+  VERSION = "0.6.2"
 end