diff --git a/.gitignore b/.gitignore index edadf6704..dca434ebf 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,4 @@ node_modules # Vite uses dotenv and suggests to ignore local-only env files. See # https://vitejs.dev/guide/env-and-mode.html#env-files *.local +db/gazetteer/* diff --git a/Gemfile b/Gemfile index c6b3f53fe..971dd5b31 100644 --- a/Gemfile +++ b/Gemfile @@ -82,7 +82,7 @@ gem 'twitter-typeahead-rails', '0.11.1.pre.corejavascript' gem 'geoblacklight', '4.4' # GBL Admin -gem 'geoblacklight_admin', git: "https://github.com/geobtaa/geoblacklight_admin.git", branch: "develop" +gem 'geoblacklight_admin', "~> 0.6.0" gem 'git', ">= 1.13" gem "rubyzip", ">= 1.3.0" gem "awesome_print" @@ -100,11 +100,11 @@ gem 'noticed' gem 'paper_trail' # Image migration -gem 'geoblacklight_sidecar_images', git: "https://github.com/geoblacklight/geoblacklight_sidecar_images.git", branch: "feature/statesman-update" +gem 'geoblacklight_sidecar_images', git: "https://github.com/geoblacklight/geoblacklight_sidecar_images.git", branch: "develop" gem 'mini_magick', '~> 4.9.4' gem "image_processing", ">= 1.2" -gem 'statesman', '~> 10.0' +gem 'statesman', '>= 12.0' gem 'sidekiq', '~> 6.4' gem 'sidekiq-failures', '~> 1.0.0' gem 'down', '~> 5.0' diff --git a/Gemfile.lock b/Gemfile.lock index 93f4ac296..8aa5987ed 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,11 +1,13 @@ GIT remote: https://github.com/cbeer/solr_wrapper.git - revision: 8decbdd1840e14a8b039ceead1060f864891408c + revision: 4d9bac111bb9bc84e0deb810b9454e8a3bc59057 branch: master specs: - solr_wrapper (4.0.2) - http + solr_wrapper (4.1.0) + faraday (~> 2.5) + faraday-follow_redirects minitar + ostruct retriable ruby-progressbar @@ -27,130 +29,93 @@ GIT GIT remote: https://github.com/geoblacklight/geoblacklight_sidecar_images.git - revision: 089e2a4a677a5d522b9d88fef0ef46cfa69961a0 - branch: feature/statesman-update + revision: ca23e5ef8eb4fa90d5c9cb955943deeb0dcba6f5 + branch: develop specs: - geoblacklight_sidecar_images (1.0.0) + geoblacklight_sidecar_images (1.1.0) faraday (>= 2.0) geoblacklight (~> 4.0) image_processing (~> 1.6) mimemagic (~> 0.3) mini_magick (~> 4.9.4) - rails (>= 5.2, < 7.1) - statesman (~> 10.0.0) - -GIT - remote: https://github.com/geobtaa/geoblacklight_admin.git - revision: 2ac63e33ea33ce6fe7eccc3b64600bf90292d346 - branch: develop - specs: - geoblacklight_admin (0.4.1) - active_storage_validations (~> 1.0) - amazing_print - blacklight (~> 7.33) - blacklight_advanced_search - blacklight_range_limit - bootstrap (~> 4.0) - chosen-rails (~> 1.10) - cocoon (~> 1.2) - config (~> 4.0) - devise (~> 4.7) - devise-bootstrap-views (~> 1.0) - devise_invitable (~> 2.0) - dotenv-rails (~> 2.8) - geoblacklight (~> 4.4) - haml (~> 5.2) - httparty (~> 0.21) - inline_svg (~> 1.9) - jquery-rails (~> 4.4) - kithe (~> 2.0) - noticed (~> 1.6) - pagy (~> 6.0) - paper_trail (~> 15.0) - pg (~> 1.4) - qa (~> 5.0) - rails (~> 7.0, < 8.0) - ruby-progressbar - simple_form (~> 5.0) - sprockets (< 4) - statesman (~> 10.0) - vite_rails (~> 3.0) - vite_ruby (>= 3.5) + rails (>= 5.2, < 8.0) + statesman (>= 3.4) GEM remote: https://rubygems.org/ specs: - Ascii85 (1.1.1) - actioncable (7.0.8.4) - actionpack (= 7.0.8.4) - activesupport (= 7.0.8.4) + Ascii85 (2.0.1) + actioncable (7.0.8.7) + actionpack (= 7.0.8.7) + activesupport (= 7.0.8.7) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (7.0.8.4) - actionpack (= 7.0.8.4) - activejob (= 7.0.8.4) - activerecord (= 7.0.8.4) - activestorage (= 7.0.8.4) - activesupport (= 7.0.8.4) + actionmailbox (7.0.8.7) + actionpack (= 7.0.8.7) + activejob (= 7.0.8.7) + activerecord (= 7.0.8.7) + activestorage (= 7.0.8.7) + activesupport (= 7.0.8.7) mail (>= 2.7.1) net-imap net-pop net-smtp - actionmailer (7.0.8.4) - actionpack (= 7.0.8.4) - actionview (= 7.0.8.4) - activejob (= 7.0.8.4) - activesupport (= 7.0.8.4) + actionmailer (7.0.8.7) + actionpack (= 7.0.8.7) + actionview (= 7.0.8.7) + activejob (= 7.0.8.7) + activesupport (= 7.0.8.7) mail (~> 2.5, >= 2.5.4) net-imap net-pop net-smtp rails-dom-testing (~> 2.0) - actionpack (7.0.8.4) - actionview (= 7.0.8.4) - activesupport (= 7.0.8.4) + actionpack (7.0.8.7) + actionview (= 7.0.8.7) + activesupport (= 7.0.8.7) rack (~> 2.0, >= 2.2.4) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (7.0.8.4) - actionpack (= 7.0.8.4) - activerecord (= 7.0.8.4) - activestorage (= 7.0.8.4) - activesupport (= 7.0.8.4) + actiontext (7.0.8.7) + actionpack (= 7.0.8.7) + activerecord (= 7.0.8.7) + activestorage (= 7.0.8.7) + activesupport (= 7.0.8.7) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.0.8.4) - activesupport (= 7.0.8.4) + actionview (7.0.8.7) + activesupport (= 7.0.8.7) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) active_record_query_trace (1.8.2) activerecord (>= 6.0.0) - active_storage_validations (1.1.4) - activejob (>= 5.2.0) - activemodel (>= 5.2.0) - activestorage (>= 5.2.0) - activesupport (>= 5.2.0) - activejob (7.0.8.4) - activesupport (= 7.0.8.4) + active_storage_validations (1.3.5) + activejob (>= 6.1.4) + activemodel (>= 6.1.4) + activestorage (>= 6.1.4) + activesupport (>= 6.1.4) + marcel (>= 1.0.3) + activejob (7.0.8.7) + activesupport (= 7.0.8.7) globalid (>= 0.3.6) - activemodel (7.0.8.4) - activesupport (= 7.0.8.4) - activerecord (7.0.8.4) - activemodel (= 7.0.8.4) - activesupport (= 7.0.8.4) - activerecord-import (1.8.1) + activemodel (7.0.8.7) + activesupport (= 7.0.8.7) + activerecord (7.0.8.7) + activemodel (= 7.0.8.7) + activesupport (= 7.0.8.7) + activerecord-import (2.0.0) activerecord (>= 4.2) - activestorage (7.0.8.4) - actionpack (= 7.0.8.4) - activejob (= 7.0.8.4) - activerecord (= 7.0.8.4) - activesupport (= 7.0.8.4) + activestorage (7.0.8.7) + actionpack (= 7.0.8.7) + activejob (= 7.0.8.7) + activerecord (= 7.0.8.7) + activesupport (= 7.0.8.7) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (7.0.8.4) + activesupport (7.0.8.7) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -164,31 +129,33 @@ GEM net-http-persistent amazing_print (1.6.0) ansi (1.5.0) - appsignal (4.0.6) + appsignal (4.2.2) + logger rack - attr_json (2.4.0) - activerecord (>= 6.0.0, < 7.3) + attr_json (2.5.0) + activerecord (>= 6.0.0, < 8.1) autoprefixer-rails (10.4.19.0) execjs (~> 2) awesome_print (1.9.2) aws-eventstream (1.3.0) - aws-partitions (1.971.0) - aws-sdk-core (3.203.0) + aws-partitions (1.1023.0) + aws-sdk-core (3.214.0) aws-eventstream (~> 1, >= 1.3.0) - aws-partitions (~> 1, >= 1.651.0) + aws-partitions (~> 1, >= 1.992.0) aws-sigv4 (~> 1.9) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.89.0) - aws-sdk-core (~> 3, >= 3.203.0) + aws-sdk-kms (1.96.0) + aws-sdk-core (~> 3, >= 3.210.0) aws-sigv4 (~> 1.5) - aws-sdk-s3 (1.160.0) - aws-sdk-core (~> 3, >= 3.203.0) + aws-sdk-s3 (1.176.1) + aws-sdk-core (~> 3, >= 3.210.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.5) - aws-sigv4 (1.9.1) + aws-sigv4 (1.10.1) aws-eventstream (~> 1, >= 1.0.2) - axe-core-api (4.10.0) + axe-core-api (4.10.2) dumb_delegator + ostruct virtus axiom-types (0.1.1) descendants_tracker (~> 0.0.4) @@ -217,11 +184,11 @@ GEM jquery-rails rails (>= 3.0) tether-rails - blazer (3.0.4) + blazer (3.1.0) activerecord (>= 6.1) chartkick (>= 5) csv - railties (>= 6.1) + railties (>= 7) safely_block (>= 0.4) bootsnap (1.9.4) msgpack (~> 1.0) @@ -245,7 +212,7 @@ GEM capybara-selenium (0.0.6) capybara selenium-webdriver - chartkick (5.1.0) + chartkick (5.1.2) childprocess (5.1.0) logger (~> 1.5) chosen-rails (1.10.0) @@ -271,14 +238,14 @@ GEM connection_pool (2.4.1) content_disposition (1.0.0) crass (1.0.6) - csv (3.3.0) - database_cleaner (2.0.2) + csv (3.3.1) + database_cleaner (2.1.0) database_cleaner-active_record (>= 2, < 3) database_cleaner-active_record (2.2.0) activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) - date (3.3.4) + date (3.4.1) deep_merge (1.2.2) deprecation (1.1.0) activesupport @@ -307,12 +274,13 @@ GEM railties (>= 3.2) down (5.4.2) addressable (~> 2.8) - dry-cli (1.1.0) + dry-cli (1.2.0) dry-configurable (1.2.0) dry-core (~> 1.0, < 2) zeitwerk (~> 2.6) - dry-core (1.0.1) + dry-core (1.0.2) concurrent-ruby (~> 1.0) + logger zeitwerk (~> 2.6) dry-inflector (1.1.0) dry-initializer (3.1.1) @@ -345,20 +313,23 @@ GEM erubi (1.13.0) ethon (0.16.0) ffi (>= 1.15.0) - execjs (2.9.1) + execjs (2.10.0) factory_bot (6.5.0) activesupport (>= 5.0.0) - factory_bot_rails (6.4.3) - factory_bot (~> 6.4) + factory_bot_rails (6.4.4) + factory_bot (~> 6.5) railties (>= 5.0.0) - faraday (2.11.0) - faraday-net_http (>= 2.0, < 3.4) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json logger - faraday-net_http (3.3.0) - net-http - faraday-net_http_persistent (2.1.0) + faraday-follow_redirects (0.3.0) + faraday (>= 1, < 3) + faraday-net_http (3.4.0) + net-http (>= 0.5.0) + faraday-net_http_persistent (2.3.0) faraday (~> 2.5) - net-http-persistent (~> 4.0) + net-http-persistent (>= 4.0.4, < 5) faraday-retry (2.2.1) faraday (~> 2.0) fastimage (2.3.1) @@ -368,9 +339,9 @@ GEM rake flamegraph (0.9.5) foreman (0.88.1) - fx (0.8.0) - activerecord (>= 6.0.0) - railties (>= 6.0.0) + fx (0.9.0) + activerecord (>= 7.0) + railties (>= 7.0) geo_combine (0.9.2) activesupport faraday-net_http_persistent (~> 2.0) @@ -394,10 +365,43 @@ GEM rgeo-geojson sprockets-rails (~> 3.0) vite_rails (~> 3.0) - geocoder (1.8.3) + geoblacklight_admin (0.6.1) + active_storage_validations (~> 1.0) + amazing_print + blacklight (~> 7.33) + blacklight_advanced_search + blacklight_range_limit + bootstrap (~> 4.0) + chosen-rails (~> 1.10) + cocoon (~> 1.2) + config (~> 4.0) + devise (~> 4.7) + devise-bootstrap-views (~> 1.0) + dotenv-rails (~> 2.8) + geoblacklight (~> 4.4) + haml (~> 5.2) + httparty (~> 0.21) + inline_svg (~> 1.9) + jquery-rails (~> 4.4) + kithe (~> 2.0) + mutex_m (~> 0.2.0) + noticed (~> 1.6) + pagy (~> 6.0) + paper_trail (~> 15.0) + pg (~> 1.4) + qa (~> 5.0) + rails (~> 7.0, < 7.3) + ruby-progressbar + simple_form (~> 5.0) + sprockets (~> 3.0) + statesman (~> 12.0) + vite_rails (~> 3.0) + vite_ruby (>= 3.5) + zeitwerk (~> 2.6) + geocoder (1.8.4) base64 (>= 0.1.0) csv (>= 3.0.0) - git (2.3.0) + git (2.3.3) activesupport (>= 5.0) addressable (~> 2.8) process_executer (~> 1.1) @@ -411,7 +415,7 @@ GEM execjs (~> 2.0) sprockets (>= 2.0.0) tilt (>= 1.2) - hashdiff (1.1.1) + hashdiff (1.1.2) hashery (2.1.2) hashie (5.0.0) http (5.2.0) @@ -420,7 +424,7 @@ GEM http-cookie (~> 1.0) http-form_data (~> 2.2) llhttp-ffi (~> 0.5.0) - http-cookie (1.0.7) + http-cookie (1.0.8) domain_name (~> 0.5) http-form_data (2.3.0) httparty (0.22.0) @@ -428,7 +432,7 @@ GEM mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) httpclient (2.8.3) - i18n (1.14.5) + i18n (1.14.6) concurrent-ruby (~> 1.0) ice_nine (0.11.2) image_processing (1.13.0) @@ -437,7 +441,7 @@ GEM inline_svg (1.10.0) activesupport (>= 3.0) nokogiri (>= 1.6) - jbuilder (2.12.0) + jbuilder (2.13.0) actionview (>= 5.0.0) activesupport (>= 5.0.0) jmespath (1.6.2) @@ -445,8 +449,10 @@ GEM rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) - json-schema (5.0.0) + json (2.9.0) + json-schema (5.1.1) addressable (~> 2.8) + bigdecimal (~> 3.1) kaminari (1.2.2) activesupport (>= 4.1.0) kaminari-actionview (= 1.2.2) @@ -459,14 +465,14 @@ GEM activerecord kaminari-core (= 1.2.2) kaminari-core (1.2.2) - kithe (2.15.1) + kithe (2.16.0) attr_json (~> 2.0) fastimage (~> 2.0) fx (>= 0.6.0, < 1) marcel mini_mime pdf-reader (~> 2.0) - rails (>= 6.0, < 8.0) + rails (>= 6.0, < 8.1) rsolr (~> 2.2) ruby-progressbar (~> 1.0) shrine (~> 3.3) @@ -491,8 +497,8 @@ GEM llhttp-ffi (0.5.0) ffi-compiler (~> 1.0) rake (~> 13.0) - logger (1.6.1) - loofah (2.22.0) + logger (1.6.3) + loofah (2.23.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) m (1.5.1) @@ -513,17 +519,18 @@ GEM matrix (0.4.2) memory_profiler (0.9.14) method_source (1.1.0) - mime-types (3.5.2) + mime-types (3.6.0) + logger mime-types-data (~> 3.2015) - mime-types-data (3.2024.0903) + mime-types-data (3.2024.1203) mimemagic (0.4.3) nokogiri (~> 1) rake mini_magick (4.9.5) mini_mime (1.1.5) - mini_portile2 (2.8.7) + mini_portile2 (2.8.8) minitar (1.0.2) - minitest (5.25.1) + minitest (5.25.4) minitest-ci (3.4.0) minitest (>= 5.0.6) minitest-reporters (1.7.1) @@ -531,18 +538,19 @@ GEM builder minitest (>= 5.0) ruby-progressbar - msgpack (1.7.2) + msgpack (1.7.5) multi_json (1.15.0) multi_xml (0.7.1) bigdecimal (~> 3.1) - net-ftp (0.3.7) + mutex_m (0.2.0) + net-ftp (0.3.8) net-protocol time - net-http (0.4.1) + net-http (0.6.0) uri - net-http-persistent (4.0.2) + net-http-persistent (4.0.5) connection_pool (~> 2.2) - net-imap (0.4.16) + net-imap (0.5.2) date net-protocol net-pop (0.1.2) @@ -551,15 +559,15 @@ GEM timeout net-smtp (0.5.0) net-protocol - nio4r (2.7.3) - nokogiri (1.16.7) + nio4r (2.7.4) + nokogiri (1.17.2) mini_portile2 (~> 2.8.2) racc (~> 1.4) noticed (1.6.3) http (>= 4.0.0) rails (>= 5.2.0) orm_adapter (0.5.0) - ostruct (0.6.0) + ostruct (0.6.1) pagy (6.5.0) paper_trail (15.2.0) activerecord (>= 6.1) @@ -567,34 +575,34 @@ GEM parslet (2.0.0) pastel (0.8.0) tty-color (~> 0.5) - pdf-reader (2.12.0) - Ascii85 (~> 1.0) + pdf-reader (2.13.0) + Ascii85 (>= 1.0, < 3.0, != 2.0.0) afm (~> 0.2.1) hashery (~> 2.0) ruby-rc4 ttfunk - pg (1.5.8) + pg (1.5.9) pointless_feedback (4.1.5) airrecord (~> 1.0) jquery-rails (>= 4.0) rails (>= 4.0) typhoeus (~> 0.7, >= 0.7.3) popper_js (1.16.1) - process_executer (1.1.0) + process_executer (1.2.0) public_suffix (6.0.1) - puma (5.6.8) + puma (5.6.9) nio4r (~> 2.0) - qa (5.13.0) + qa (5.14.0) activerecord-import deprecation faraday (< 3.0, != 2.0.0) geocoder ldpath nokogiri (~> 1.6) - rails (>= 5.0, < 8.0) + rails (>= 5.0, < 8.1) rdf racc (1.8.1) - rack (2.2.9) + rack (2.2.10) rack-cors (2.0.2) rack (>= 2.0.0) rack-mini-profiler (2.3.4) @@ -603,30 +611,30 @@ GEM rack rack-test (2.1.0) rack (>= 1.3) - rails (7.0.8.4) - actioncable (= 7.0.8.4) - actionmailbox (= 7.0.8.4) - actionmailer (= 7.0.8.4) - actionpack (= 7.0.8.4) - actiontext (= 7.0.8.4) - actionview (= 7.0.8.4) - activejob (= 7.0.8.4) - activemodel (= 7.0.8.4) - activerecord (= 7.0.8.4) - activestorage (= 7.0.8.4) - activesupport (= 7.0.8.4) + rails (7.0.8.7) + actioncable (= 7.0.8.7) + actionmailbox (= 7.0.8.7) + actionmailer (= 7.0.8.7) + actionpack (= 7.0.8.7) + actiontext (= 7.0.8.7) + actionview (= 7.0.8.7) + activejob (= 7.0.8.7) + activemodel (= 7.0.8.7) + activerecord (= 7.0.8.7) + activestorage (= 7.0.8.7) + activesupport (= 7.0.8.7) bundler (>= 1.15.0) - railties (= 7.0.8.4) + railties (= 7.0.8.7) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.0) + rails-html-sanitizer (1.6.2) loofah (~> 2.21) - nokogiri (~> 1.14) - railties (7.0.8.4) - actionpack (= 7.0.8.4) - activesupport (= 7.0.8.4) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (7.0.8.7) + actionpack (= 7.0.8.7) + activesupport (= 7.0.8.7) method_source rake (>= 12.2) thor (~> 1.0) @@ -640,17 +648,17 @@ GEM bcp47_spec (~> 0.2) bigdecimal (~> 3.1, >= 3.1.5) link_header (~> 0.0, >= 0.0.8) - rdf-vocab (3.3.1) + rdf-vocab (3.3.2) rdf (~> 3.3) redis (4.8.1) - regexp_parser (2.9.2) + regexp_parser (2.9.3) request_store (1.7.0) rack (>= 1.4) responders (3.1.1) actionpack (>= 5.2) railties (>= 5.2) retriable (3.1.2) - rexml (3.3.7) + rexml (3.4.0) rgeo (3.0.1) rgeo-geojson (2.2.0) multi_json (~> 1.15) @@ -706,13 +714,13 @@ GEM tilt scrub_rb (1.0.1) sd_notify (0.1.1) - selenium-webdriver (4.24.0) + selenium-webdriver (4.27.0) base64 (~> 0.2) logger (~> 1.4) rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - semantic_range (3.0.0) + semantic_range (3.1.0) shrine (3.6.0) content_disposition (~> 1.0) down (~> 5.1) @@ -733,13 +741,13 @@ GEM docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) - simplecov-html (0.12.3) + simplecov-html (0.13.1) simplecov_json_formatter (0.1.4) sitemap_generator (6.0.2) builder (~> 3.0) slop (4.10.1) spring (4.2.1) - sprockets (3.7.3) + sprockets (3.7.5) base64 concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -750,20 +758,20 @@ GEM sqlite3 (1.7.3) mini_portile2 (~> 2.8.0) stackprof (0.2.26) - statesman (10.0.0) + statesman (12.1.0) stimulus-rails (1.3.4) railties (>= 6.0.0) temple (0.10.3) - terser (1.2.3) + terser (1.2.4) execjs (>= 0.3.0, < 3) tether-rails (1.4.0) rails (>= 3.1) thor (1.3.2) thread_safe (0.3.6) tilt (2.4.0) - time (0.4.0) + time (0.4.1) date - timeout (0.4.1) + timeout (0.4.3) traject (3.8.2) concurrent-ruby (>= 0.8.0) dot-properties (>= 0.1.1) @@ -792,7 +800,7 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) unf (0.2.0) - uri (0.13.1) + uri (1.0.2) view_component (2.83.0) activesupport (>= 5.2.0, < 8.0) concurrent-ruby (~> 1.0) @@ -801,11 +809,13 @@ GEM axiom-types (~> 0.1) coercible (~> 1.0) descendants_tracker (~> 0.0, >= 0.0.3) - vite_rails (3.0.17) - railties (>= 5.1, < 8) + vite_rails (3.0.19) + railties (>= 5.1, < 9) vite_ruby (~> 3.0, >= 3.2.2) - vite_ruby (3.8.2) + vite_ruby (3.9.1) dry-cli (>= 0.7, < 2) + logger (~> 1.6) + mutex_m rack-proxy (~> 0.6, >= 0.6.1) zeitwerk (~> 2.2) warden (1.2.9) @@ -829,7 +839,7 @@ GEM xpath (3.2.0) nokogiri (~> 1.8) yell (2.2.2) - zeitwerk (2.6.18) + zeitwerk (2.7.1) PLATFORMS ruby @@ -868,7 +878,7 @@ DEPENDENCIES foreman geoblacklight (= 4.4) geoblacklight-icons! - geoblacklight_admin! + geoblacklight_admin (~> 0.6.0) geoblacklight_sidecar_images! git (>= 1.13) haml @@ -912,7 +922,7 @@ DEPENDENCIES spring sqlite3 (~> 1.4) stackprof (~> 0.2.12) - statesman (~> 10.0) + statesman (>= 12.0) stimulus-rails terser turbolinks diff --git a/README.md b/README.md index 6757ffd40..c4ae4db32 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This geoportal connects users to digital geospatial resources, including GIS datasets, web services, and digitized historical maps from multiple data clearinghouses and library catalogs. The interface offers both text and place based search options, and item pages display descriptive metadata records along with external links to download, view, or read more about the resources. This site is solely an aggregator and does not host any data. Read more about the Geospatial Data Discovery Project here. -This project is collectively managed by librarians and geospatial specialists at 14 research institutions from across the Big Ten Academic Alliance including: +This project is collectively managed by librarians and geospatial specialists at 15 research institutions from across the Big Ten Academic Alliance including: * Indiana University Bloomington * Michigan State University @@ -24,11 +24,11 @@ This project is collectively managed by librarians and geospatial specialists at ### Technology and Hosting -The geoportal is built with Geoblacklight, a multi-institutional open-source collaboration for finding and sharing geospatial data. The technical infrastructure for the project is provided by the University of Minnesota Libraries Web Development department. +The geoportal is built with GeoBlacklight, a multi-institutional open-source collaboration for finding and sharing geospatial data. The technical infrastructure for the project is provided by the University of Minnesota Libraries Web Development department. ### Contributing Visit the [project wiki](https://github.com/BTAA-Geospatial-Data-Project/geoportal/wiki) for developer getting started documentation. #### Release Version -B1G Geoportal Version v5.11.0 / GeoBlacklight v4.4.0 / GeoBlacklight Admin v0.4.1 (develop branch) +B1G Geoportal Version v5.12.0 / GeoBlacklight v4.4.0 / GeoBlacklight Admin v0.6.1(develop branch) diff --git a/app/assets/javascripts/application.js b/app/assets/javascripts/application.js index 5cf3ccc07..b9d596114 100644 --- a/app/assets/javascripts/application.js +++ b/app/assets/javascripts/application.js @@ -40,6 +40,7 @@ //= require geoblacklight/viewers/viewer //= require ./geoportal/viewers/map //= require ./geoportal/viewers/b1g_image +//= require ./geoportal/viewers/cog //= require ./geoportal/viewers/download //= require ./geoportal/viewers/esri //= require ./geoportal/viewers/esri/dynamic_map_layer @@ -50,10 +51,10 @@ //= require ./geoportal/viewers/iiif_manifest //= require ./geoportal/viewers/index_map //= require geoblacklight/viewers/oembed - -//= require ./geoportal/viewers/wms +//= require ./geoportal/viewers/pmtiles //= require ./geoportal/viewers/tilejson //= require ./geoportal/viewers/tms +//= require ./geoportal/viewers/wms //= require ./geoportal/viewers/wmts //= require ./geoportal/viewers/xyz diff --git a/app/assets/javascripts/geoportal/viewers/cog.js b/app/assets/javascripts/geoportal/viewers/cog.js new file mode 100644 index 000000000..639d33e88 --- /dev/null +++ b/app/assets/javascripts/geoportal/viewers/cog.js @@ -0,0 +1,5 @@ +//= require geoblacklight/viewers/viewer + +GeoBlacklight.Viewer.Cog = GeoBlacklight.Viewer.extend({ + load: function() {} +}); \ No newline at end of file diff --git a/app/assets/javascripts/geoportal/viewers/iiif.js b/app/assets/javascripts/geoportal/viewers/iiif.js new file mode 100644 index 000000000..e50ca262b --- /dev/null +++ b/app/assets/javascripts/geoportal/viewers/iiif.js @@ -0,0 +1,25 @@ +//= require geoblacklight/viewers/viewer + +GeoBlacklight.Viewer.Iiif = GeoBlacklight.Viewer.extend({ + load: function() { + this.adjustLayout(); + + this.map = L.map(this.element, { + center: [0, 0], + crs: L.CRS.Simple, + zoom: 0 + }); + this.loadControls(); + this.iiifLayer = L.tileLayer.iiif(this.data.url) + .addTo(this.map); + }, + + adjustLayout: function() { + + // hide attribute table + $('#table-container').hide(); + + // expand viewer element + $(this.element).parent().attr('class', 'col-md-12'); + } +}); diff --git a/app/assets/javascripts/geoportal/viewers/oembed.js b/app/assets/javascripts/geoportal/viewers/oembed.js new file mode 100644 index 000000000..004d6d371 --- /dev/null +++ b/app/assets/javascripts/geoportal/viewers/oembed.js @@ -0,0 +1,13 @@ +//= require geoblacklight/viewers/viewer + +GeoBlacklight.Viewer.Oembed = GeoBlacklight.Viewer.extend({ + load: function() { + var $el = $(this.element); + $.getJSON(this.data.url, function(data){ + if (data === null){ + return; + } + $el.html(data.html); + }); + }, +}); diff --git a/app/assets/javascripts/geoportal/viewers/pmtiles.js b/app/assets/javascripts/geoportal/viewers/pmtiles.js new file mode 100644 index 000000000..359a0bbc4 --- /dev/null +++ b/app/assets/javascripts/geoportal/viewers/pmtiles.js @@ -0,0 +1,5 @@ +//= require geoblacklight/viewers/viewer + +GeoBlacklight.Viewer.Pmtiles = GeoBlacklight.Viewer.extend({ + load: function() {} +}); \ No newline at end of file diff --git a/app/assets/stylesheets/geoportal/_todo.scss b/app/assets/stylesheets/geoportal/_todo.scss index 1d9bf08aa..d57c7d50a 100644 --- a/app/assets/stylesheets/geoportal/_todo.scss +++ b/app/assets/stylesheets/geoportal/_todo.scss @@ -24,7 +24,13 @@ a { border-color: $visited-color; } } -a.btn { text-decoration: none; } +a.btn { + text-decoration: none; + &:visited { + color: $white; + border-color: $white; + } +} a.btn.btn-primary { color: $white; background-color: $btaa-blue; @@ -244,7 +250,6 @@ span.list-view.document-counter { // * Form Controlls .form-control { color: $black !important; - background-color: $white !important; } // * Input Placeholder diff --git a/app/assets/stylesheets/geoportal/geoportal.scss b/app/assets/stylesheets/geoportal/geoportal.scss index 1ca830585..2ed7a2c0e 100644 --- a/app/assets/stylesheets/geoportal/geoportal.scss +++ b/app/assets/stylesheets/geoportal/geoportal.scss @@ -22,3 +22,8 @@ .pull-right { float: right; } + +// Fix input#q in search form +input#q { + background-color: $white !important; +} diff --git a/app/channels/application_cable/connection.rb b/app/channels/application_cable/connection.rb index 2c84e59bf..e59083764 100644 --- a/app/channels/application_cable/connection.rb +++ b/app/channels/application_cable/connection.rb @@ -20,4 +20,4 @@ def find_verified_user end end end -end +end \ No newline at end of file diff --git a/app/channels/export_channel.rb b/app/channels/export_channel.rb index 742084409..ee9bf4b5f 100644 --- a/app/channels/export_channel.rb +++ b/app/channels/export_channel.rb @@ -9,4 +9,4 @@ def subscribed def unsubscribed # Any cleanup needed when channel is unsubscribed end -end +end \ No newline at end of file diff --git a/app/javascript/channels/export_channel.js b/app/javascript/channels/export_channel.js index b58579644..ea90f3ac6 100644 --- a/app/javascript/channels/export_channel.js +++ b/app/javascript/channels/export_channel.js @@ -1,18 +1,18 @@ import consumer from "./consumer" -consumer.subscriptions.create({ channel: "ExportChannel" }, { +export default consumer.subscriptions.create({ channel: "ExportChannel" }, { connected() { // Called when the subscription is ready for use on the server - console.log('Export Channel Connected'); + console.log("GBL Admin - ExportChannel connected"); }, disconnected() { // Called when the subscription has been terminated by the server - console.log('Export Channel Disconnected'); + console.log("GBL Admin - ExportChannel disconnected"); }, received(data) { - console.log('Export Channel Received'); + console.log('GBL Admin - ExportChannel received!'); console.log(data); if (data['progress']) { @@ -27,4 +27,4 @@ consumer.subscriptions.create({ channel: "ExportChannel" }, { } } } -}); +}); \ No newline at end of file diff --git a/app/javascript/channels/index.js b/app/javascript/channels/index.js index 0cfcf7491..30454868f 100644 --- a/app/javascript/channels/index.js +++ b/app/javascript/channels/index.js @@ -1,5 +1,2 @@ -// Load all the channels within this directory and all subdirectories. -// Channel files must be named *_channel.js. - -const channels = require.context('.', true, /_channel\.js$/) -channels.keys().forEach(channels) +export { default as consumer } from './consumer'; +export { default as exportChannel } from './export_channel'; diff --git a/app/javascript/entrypoints/application.js b/app/javascript/entrypoints/application.js index 619e35528..9dec89a2a 100644 --- a/app/javascript/entrypoints/application.js +++ b/app/javascript/entrypoints/application.js @@ -22,7 +22,7 @@ console.log('Vite ⚡️ Rails') // ActiveStorage.start() // // // Import all channels. -// const channels = import.meta.globEager('./**/*_channel.js') +import '../channels'; // Example: Import a stylesheet in app/frontend/index.css // import '~/index.css' diff --git a/app/views/robots/robots.text.erb b/app/views/robots/robots.text.erb index 18bafc477..ee22a3d71 100644 --- a/app/views/robots/robots.text.erb +++ b/app/views/robots/robots.text.erb @@ -9,6 +9,7 @@ User-agent: * Disallow: / <%- else -%> User-agent: * +Crawl-delay: 5 Disallow: /?q= Disallow: /?f Disallow: /?_ @@ -37,4 +38,6 @@ User-agent: DotBot Disallow: / User-agent: DataForSeoBot Disallow: / +User-agent: ImagesiftBot +Disallow: / <%- end -%> diff --git a/config/cable.yml b/config/cable.yml index 232fc5be8..b03f164f4 100644 --- a/config/cable.yml +++ b/config/cable.yml @@ -1,10 +1,9 @@ +# config/cable.yml +production: + adapter: postgresql + development: - adapter: async + adapter: postgresql test: - adapter: async - -production: - adapter: redis - url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> - channel_prefix: geoblacklight_production + adapter: postgresql \ No newline at end of file diff --git a/config/environments/development.rb b/config/environments/development.rb index 3d3f7f715..a80ac11cc 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -64,12 +64,12 @@ # Raises error for missing translations. # config.i18n.raise_on_missing_translations = true - # config.active_job.queue_adapter = :inline config.active_job.queue_adapter = :sidekiq + Redis.exists_returns_integer = true # Annotate rendered view with file names. # config.action_view.annotate_rendered_view_with_filenames = true # Uncomment if you wish to allow Action Cable access from any origin. - # config.action_cable.disable_request_forgery_protection = true + config.action_cable.disable_request_forgery_protection = true end diff --git a/config/initializers/mime_types.rb b/config/initializers/mime_types.rb index 5b678f929..3426ef927 100644 --- a/config/initializers/mime_types.rb +++ b/config/initializers/mime_types.rb @@ -13,3 +13,4 @@ Mime::Type.register "text/csv", :csv_document_downloads Mime::Type.register "text/csv", :csv_document_access_links +Mime::Type.register "text/csv", :csv_document_distributions diff --git a/config/routes.rb b/config/routes.rb index e3f928b50..6a5eae6f9 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,4 +1,6 @@ Rails.application.routes.draw do + mount ActionCable.server => "/cable" + get 'about', :to => redirect('https://gin.btaa.org/') get 'help', :to => redirect('https://gin.btaa.org/guides/') get 'robots.:format' => 'robots#robots' @@ -78,147 +80,118 @@ end end - #################### - # GBL‡ADMIN +#################### +# GBL‡ADMIN - # Bulk Actions - resources :bulk_actions do - patch :run, on: :member - patch :revert, on: :member - end +# Bulk Actions +resources :bulk_actions do + patch :run, on: :member + patch :revert, on: :member +end - # Users - devise_for :users, controllers: {invitations: "devise/invitations"}, skip: [:registrations] - as :user do - get "/sign_in" => "devise/sessions#new" # custom path to login/sign_in - get "/sign_up" => "devise/registrations#new", :as => "new_user_registration" # custom path to sign_up/registration - get "users/edit" => "devise/registrations#edit", :as => "edit_user_registration" - put "users" => "devise/registrations#update", :as => "user_registration" - end +# Users +devise_for :users, skip: [:registrations] +as :user do + get "/sign_in" => "devise/sessions#new" # custom path to login/sign_in + get "/sign_up" => "devise/registrations#new", :as => "new_user_registration" # custom path to sign_up/registration + get "users/edit" => "devise/registrations#edit", :as => "edit_user_registration" + put "users" => "devise/registrations#update", :as => "user_registration" +end - namespace :admin do +namespace :admin do - authenticate :user, ->(user) { user.admin? } do - mount Sidekiq::Web => "/sidekiq" - end - - devise_for :users, controllers: {invitations: "devise/invitations"}, skip: [:registrations] - # Root - root to: "documents#index" - - # Assets - # Note "assets" is Rails reserved word for routing, oops. So we use - # asset_files. - resources :assets, path: "asset_files" do - collection do - get "display_attach_form" - post "attach_files" + authenticate :user, ->(user) { user.admin? } do + mount Sidekiq::Web => "/sidekiq" + end - get "destroy_all" - post "destroy_all" - end + devise_for :users - post :sort, on: :collection - end + # Root + root to: "documents#index" - # Bulk Actions - resources :bulk_actions do - patch :run, on: :member - patch :revert, on: :member - end + # Assets + # Note "assets" is Rails reserved word for routing, oops. So we use + # asset_files. + resources :assets, path: "asset_files" do + collection do + get "display_attach_form" + post "attach_files" - # Imports - resources :imports do - resources :mappings - resources :import_documents, only: [:show] - patch :run, on: :member + get "destroy_all" + post "destroy_all" end - # Elements - resources :elements do - post :sort, on: :collection - end + post :sort, on: :collection + end - # Form Elements - resources :form_elements do - post :sort, on: :collection - end - resources :form_header, path: :form_elements, controller: :form_elements - resources :form_group, path: :form_elements, controller: :form_elements - resources :form_control, path: :form_elements, controller: :form_elements - resources :form_feature, path: :form_elements, controller: :form_elements - - # Notifications - resources :notifications do - put "batch", on: :collection - end + # Bulk Actions + resources :bulk_actions do + patch :run, on: :member + patch :revert, on: :member + end - # Users - get "users/index" - - # Bookmarks - resources :bookmarks - delete "/bookmarks", to: "bookmarks#destroy", as: :bookmarks_destroy_by_fkeys - - # Search controller - get "/search" => "search#index" - - # AdvancedSearch controller - get '/advanced_search' => 'advanced_search#index', constraints: lambda { |req| req.format == :json } - get '/advanced_search/facets' => 'advanced_search#facets', constraints: lambda { |req| req.format == :json } - get '/advanced_search/facet/:id' => 'advanced_search#facet', constraints: lambda { |req| req.format == :json }, as: 'advanced_search_facet' - - # Ids controller - get '/api/ids' => 'ids#index', constraints: lambda { |req| req.format == :json } - get '/api' => 'api#index', constraints: lambda { |req| req.format == :json } - get '/api/fetch' => 'api#fetch', constraints: lambda { |req| req.format == :json } - get '/api/facet/:id' => 'api#facet', constraints: lambda { |req| req.format == :json } - get '/api/tableau_export' => 'api#tableau_export' - - # Documents - resources :documents do - get "admin" - get "versions" - - # DocumentAccesses - resources :document_accesses, path: "access" do - collection do - get "import" - post "import" - - get "destroy_all" - post "destroy_all" - end - end + # Imports + resources :imports do + resources :mappings + resources :import_documents, only: [:show] + patch :run, on: :member + end - # DocumentDownloads - resources :document_downloads, path: "downloads" do - collection do - get "import" - post "import" + # Elements + resources :elements do + post :sort, on: :collection + end - get "destroy_all" - post "destroy_all" - end - end + # Form Elements + resources :form_elements do + post :sort, on: :collection + end + resources :form_header, path: :form_elements, controller: :form_elements + resources :form_group, path: :form_elements, controller: :form_elements + resources :form_control, path: :form_elements, controller: :form_elements + resources :form_feature, path: :form_elements, controller: :form_elements + + # Reference Types + resources :reference_types do + post :sort, on: :collection + end - # Document Assets - resources :document_assets, path: "assets" do - collection do - get "display_attach_form" - post "attach_files" + # Notifications + resources :notifications do + put "batch", on: :collection + end - get "destroy_all" - post "destroy_all" - end - end + # Users + get "users/index" + + # Bookmarks + resources :bookmarks + delete "/bookmarks", to: "bookmarks#destroy", as: :bookmarks_destroy_by_fkeys + + # Search controller + get "/search" => "search#index" + + # AdvancedSearch controller + get '/advanced_search' => 'advanced_search#index', constraints: lambda { |req| req.format == :json } + get '/advanced_search/facets' => 'advanced_search#facets', constraints: lambda { |req| req.format == :json } + get '/advanced_search/facet/:id' => 'advanced_search#facet', constraints: lambda { |req| req.format == :json }, as: 'advanced_search_facet' + + # Ids controller + get '/api/ids' => 'ids#index', constraints: lambda { |req| req.format == :json } + get '/api' => 'api#index', constraints: lambda { |req| req.format == :json } + get '/api/fetch' => 'api#fetch', constraints: lambda { |req| req.format == :json } + get '/api/facet/:id' => 'api#facet', constraints: lambda { |req| req.format == :json } + + # Documents + resources :documents do + get "admin" + get "versions" - collection do - get "fetch" - end + collection do + get "fetch" end - # Document Accesses + # DocumentAccesses resources :document_accesses, path: "access" do collection do get "import" @@ -229,7 +202,18 @@ end end - # Document Downloads + # Document Assets + resources :document_assets, path: "assets" do + collection do + get "display_attach_form" + post "attach_files" + + get "destroy_all" + post "destroy_all" + end + end + + # DocumentDownloads resources :document_downloads, path: "downloads" do collection do get "import" @@ -240,40 +224,88 @@ end end - # Document Assets - resources :document_assets, path: "assets" do + # Document Distributions + resources :document_distributions, path: "distributions" do collection do get "display_attach_form" post "attach_files" + get "import" + post "import" + get "destroy_all" post "destroy_all" end end + end - # Assets - get "/asset_files/ingest", to: "assets#display_attach_form", as: "assets_ingest" - post "/asset_files/ingest", to: "assets#attach_files" - - # DocumentAssets - get "/documents/:id/ingest", to: "document_assets#display_attach_form", as: "asset_ingest" - post "/documents/:id/ingest", to: "document_assets#attach_files" - mount Kithe::AssetUploader.upload_endpoint(:cache) => "/direct_upload", :as => :direct_app_upload - - resources :collections, except: [:show] - - # Note "assets" is Rails reserved word for routing, oops. So we use - # asset_files. - resources :assets, path: "asset_files", except: [:new, :create] do - member do - put "convert_to_child_work" - end + # Document Accesses + resources :document_accesses, path: "access" do + collection do + get "import" + post "import" + + get "destroy_all" + post "destroy_all" end + end - # @TODO - # mount Qa::Engine => "/authorities" - mount ActionCable.server => "/cable" + # Document Downloads + resources :document_downloads, path: "downloads" do + collection do + get "import" + post "import" + + get "destroy_all" + post "destroy_all" + end + end + + # Document Distributions + resources :document_distributions, path: "distributions" do + collection do + get "import" + post "import" + + get "destroy_all" + post "destroy_all" + end + end + + # Document Assets + resources :document_assets, path: "assets" do + collection do + get "display_attach_form" + post "attach_files" + + get "destroy_all" + post "destroy_all" + end + end + + # Assets + get "/asset_files/ingest", to: "assets#display_attach_form", as: "assets_ingest" + post "/asset_files/ingest", to: "assets#attach_files" + + # DocumentAssets + get "/documents/:id/ingest", to: "document_assets#display_attach_form", as: "asset_ingest" + post "/documents/:id/ingest", to: "document_assets#attach_files" + + # Asset Direct Upload + mount Kithe::AssetUploader.upload_endpoint(:cache) => "/direct_upload", :as => :direct_app_upload + + resources :collections, except: [:show] + + # Note "assets" is Rails reserved word for routing, oops. So we use + # asset_files. + resources :assets, path: "asset_files", except: [:new, :create] do + member do + put "convert_to_child_work" + end + end + # @TODO + # mount Qa::Engine => "/authorities" # @TODO authenticate :user, ->(user) { user } do mount Blazer::Engine, at: "blazer" diff --git a/db/migrate/20241009200524_create_admin_reference_types.rb b/db/migrate/20241009200524_create_admin_reference_types.rb new file mode 100644 index 000000000..4a68e1476 --- /dev/null +++ b/db/migrate/20241009200524_create_admin_reference_types.rb @@ -0,0 +1,13 @@ +class CreateAdminReferenceTypes < ActiveRecord::Migration[7.0] + def change + create_table :reference_types do |t| + t.string :name # short name, ex: "cog" + t.string :reference_type # human-readable name, ex: "Cloud Optimized GeoTIFF (COG)" + t.string :reference_uri # key name, ex: "https://github.com/cogeotiff/cog-spec" + t.boolean :label, default: false # optional download label + t.text :note # optional note + t.integer :position # position for sorting + t.timestamps + end + end +end \ No newline at end of file diff --git a/db/migrate/20241010161420_create_document_references.rb b/db/migrate/20241010161420_create_document_references.rb new file mode 100644 index 000000000..458a18399 --- /dev/null +++ b/db/migrate/20241010161420_create_document_references.rb @@ -0,0 +1,14 @@ +class CreateDocumentReferences < ActiveRecord::Migration[7.0] + def change + create_table :document_references do |t| + t.string :friendlier_id, null: false + t.references :reference_type, null: false, foreign_key: true + t.string :url + t.string :label + t.integer :position + t.timestamps + + t.index [:friendlier_id, :reference_type_id, :url], unique: true, name: 'document_references_unique_index' + end + end +end \ No newline at end of file diff --git a/db/migrate/20241120238823_rename_references_to_distributions.rb b/db/migrate/20241120238823_rename_references_to_distributions.rb new file mode 100644 index 000000000..7bf4153ac --- /dev/null +++ b/db/migrate/20241120238823_rename_references_to_distributions.rb @@ -0,0 +1,5 @@ +class RenameReferencesToDistributions < ActiveRecord::Migration[7.0] + def change + rename_table :document_references, :document_distributions + end +end \ No newline at end of file diff --git a/db/schema.rb b/db/schema.rb index 651a13d95..c59d94473 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_06_19_183528) do +ActiveRecord::Schema[7.0].define(version: 2024_11_20_238823) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -214,13 +214,11 @@ t.string "name" t.string "request", null: false t.string "scope", null: false - t.string "field_name" - t.string "field_value" + t.string "field_name", null: false + t.string "field_value", null: false t.text "notes" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.string "action_type", default: "ChangePublicationState", null: false - t.string "action" end create_table "document_accesses", force: :cascade do |t| @@ -231,6 +229,18 @@ t.datetime "updated_at", null: false end + create_table "document_distributions", force: :cascade do |t| + t.string "friendlier_id", null: false + t.bigint "reference_type_id", null: false + t.string "url" + t.string "label" + t.integer "position" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["friendlier_id", "reference_type_id", "url"], name: "document_references_unique_index", unique: true + t.index ["reference_type_id"], name: "index_document_distributions_on_reference_type_id" + end + create_table "document_downloads", force: :cascade do |t| t.string "friendlier_id" t.string "label" @@ -301,83 +311,6 @@ t.datetime "updated_at", null: false end - create_table "good_job_batches", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.text "description" - t.jsonb "serialized_properties" - t.text "on_finish" - t.text "on_success" - t.text "on_discard" - t.text "callback_queue_name" - t.integer "callback_priority" - t.datetime "enqueued_at", precision: nil - t.datetime "discarded_at", precision: nil - t.datetime "finished_at", precision: nil - end - - create_table "good_job_executions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.uuid "active_job_id", null: false - t.text "job_class" - t.text "queue_name" - t.jsonb "serialized_params" - t.datetime "scheduled_at", precision: nil - t.datetime "finished_at", precision: nil - t.text "error" - t.integer "error_event", limit: 2 - t.index ["active_job_id", "created_at"], name: "index_good_job_executions_on_active_job_id_and_created_at" - end - - create_table "good_job_processes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.jsonb "state" - end - - create_table "good_job_settings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.text "key" - t.jsonb "value" - t.index ["key"], name: "index_good_job_settings_on_key", unique: true - end - - create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.text "queue_name" - t.integer "priority" - t.jsonb "serialized_params" - t.datetime "scheduled_at", precision: nil - t.datetime "performed_at", precision: nil - t.datetime "finished_at", precision: nil - t.text "error" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.uuid "active_job_id" - t.text "concurrency_key" - t.text "cron_key" - t.uuid "retried_good_job_id" - t.datetime "cron_at", precision: nil - t.uuid "batch_id" - t.uuid "batch_callback_id" - t.boolean "is_discrete" - t.integer "executions_count" - t.text "job_class" - t.integer "error_event", limit: 2 - t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at" - t.index ["active_job_id"], name: "index_good_jobs_on_active_job_id" - t.index ["batch_callback_id"], name: "index_good_jobs_on_batch_callback_id", where: "(batch_callback_id IS NOT NULL)" - t.index ["batch_id"], name: "index_good_jobs_on_batch_id", where: "(batch_id IS NOT NULL)" - t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)" - t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at" - t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at", unique: true - t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))" - t.index ["priority", "created_at"], name: "index_good_jobs_jobs_on_priority_created_at_when_unfinished", order: { priority: "DESC NULLS LAST" }, where: "(finished_at IS NULL)" - t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" - t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" - end - create_table "image_upload_transitions", force: :cascade do |t| t.string "to_state", null: false t.text "metadata" @@ -513,6 +446,17 @@ t.datetime "updated_at", precision: nil, null: false end + create_table "reference_types", force: :cascade do |t| + t.string "name" + t.string "reference_type" + t.string "reference_uri" + t.boolean "label", default: false + t.text "note" + t.integer "position" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "searches", force: :cascade do |t| t.text "query_params" t.integer "user_id" @@ -613,6 +557,7 @@ add_foreign_key "bulk_action_document_transitions", "bulk_action_documents" add_foreign_key "bulk_action_documents", "bulk_actions" add_foreign_key "bulk_action_transitions", "bulk_actions" + add_foreign_key "document_distributions", "reference_types" add_foreign_key "image_upload_transitions", "solr_document_sidecars" add_foreign_key "import_document_transitions", "import_documents" add_foreign_key "import_documents", "imports" diff --git a/db/seeds.rb b/db/seeds.rb index 53bbc16ca..5dc147044 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -5,4 +5,43 @@ # # cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }]) # Mayor.create(name: 'Emanuel', city: cities.first) -GeoblacklightAdmin::Engine.load_seed \ No newline at end of file +GeoblacklightAdmin::Engine.load_seed + +# db/seeds.rb + +reference_types = [ + { name: 'arcgis_dynamic_map_layer', reference_type: 'ArcGIS DynamicMapLayer', reference_uri: 'urn:x-esri:serviceType:ArcGIS#DynamicMapLayer', label: false, note: '-', position: 1 }, + { name: 'arcgis_feature_layer', reference_type: 'ArcGIS FeatureLayer', reference_uri: 'urn:x-esri:serviceType:ArcGIS#FeatureLayer', label: false, note: '-', position: 2 }, + { name: 'arcgis_image_map_layer', reference_type: 'ArcGIS ImageMapLayer', reference_uri: 'urn:x-esri:serviceType:ArcGIS#ImageMapLayer', label: false, note: '-', position: 3 }, + { name: 'arcgis_tiled_map_layer', reference_type: 'ArcGIS TiledMapLayer', reference_uri: 'urn:x-esri:serviceType:ArcGIS#TiledMapLayer', label: false, note: '-', position: 4 }, + { name: 'cog', reference_type: 'Cloud Optimized GeoTIFF (COG)', reference_uri: 'https://github.com/cogeotiff/cog-spec', label: false, note: '-', position: 5 }, + { name: 'documentation_download', reference_type: 'Data dictionary / supplemental documentation', reference_uri: 'http://lccn.loc.gov/sh85035852', label: false, note: 'Functions as a link to download documentation (not a viewer)', position: 6 }, + { name: 'documentation_external', reference_type: 'Documentation (External)', reference_uri: 'http://schema.org/url', label: false, note: '-', position: 7 }, + { name: 'download', reference_type: 'Download file', reference_uri: 'http://schema.org/downloadUrl', label: true, note: 'Link to download file (for multiple files see the multiple downloads guidelines)', position: 8 }, + { name: 'geo_json', reference_type: 'GeoJSON', reference_uri: 'http://geojson.org/geojson-spec.html', label: false, note: '-', position: 9 }, + { name: 'full_layer_description', reference_type: 'Full layer description', reference_uri: 'http://schema.org/url', label: false, note: 'To view further descriptive information about the layer or a link to its landing page', position: 10 }, + { name: 'iiif_image', reference_type: 'International Image Interoperability Framework (IIIF) Image API', reference_uri: 'http://iiif.io/api/image', label: false, note: 'Load the image viewer using Leaflet-IIIF', position: 11 }, + { name: 'iiif_manifest', reference_type: 'International Image Interoperability Framework (IIIF) Presentation API Manifest', reference_uri: 'http://iiif.io/api/presentation#manifest', label: false, note: 'View the IIIF manifest - uses the Clover viewer by default https://samvera-labs.github.io/clover-iiif/docs', position: 12 }, + { name: 'image', reference_type: 'Image file', reference_uri: 'http://schema.org/image', label: true, note: '-', position: 13 }, + { name: 'metadata_fgdc', reference_type: 'Metadata in FGDC', reference_uri: 'http://www.opengis.net/cat/csw/csdgm', label: false, note: 'Provides an HTML view of an XML file in the FGDC standard', position: 14 }, + { name: 'metadata_html', reference_type: 'Metadata in HTML', reference_uri: 'http://www.w3.org/1999/xhtml', label: false, note: 'View structured metadata in any standard expressed as HTML', position: 15 }, + { name: 'metadata_iso', reference_type: 'Metadata in ISO 19139', reference_uri: 'http://www.isotc211.org/schemas/2005/gmd/', label: false, note: 'Provides an HTML view of an XML file in the ISO 19139 standard', position: 16 }, + { name: 'metadata_mods', reference_type: 'Metadata in MODS', reference_uri: 'http://www.loc.gov/mods/v3', label: false, note: 'Provides a raw XML view of metadata in the MODS format', position: 17 }, + { name: 'oembed', reference_type: 'oEmbed', reference_uri: 'https://oembed.com', label: false, note: '-', position: 18 }, + { name: 'open_index_map', reference_type: 'OpenIndexMap', reference_uri: 'https://openindexmaps.org', label: false, note: 'Provides an interactive preview of a GeoJSON file formatted as an OpenIndexMap', position: 19 }, + { name: 'pmtiles', reference_type: 'PMTiles', reference_uri: 'https://github.com/protomaps/PMTiles', label: false, note: '-', position: 20 }, + { name: 'thumbnail', reference_type: 'Thumbnail file', reference_uri: 'http://schema.org/thumbnailUrl', label: true, note: '-', position: 21 }, + { name: 'tile_map_service', reference_type: 'Tile Mapping Service (TMS)', reference_uri: 'https://wiki.osgeo.org/wiki/Tile_Map_Service_Specification', label: false, note: '-', position: 22 }, + { name: 'tile_json', reference_type: 'TileJSON', reference_uri: 'https://github.com/mapbox/tilejson-spec', label: false, note: '-', position: 23 }, + { name: 'wcs', reference_type: 'Web Coverage Service (WCS)', reference_uri: 'http://www.opengis.net/def/serviceType/ogc/wcs', label: false, note: '-', position: 24 }, + { name: 'wfs', reference_type: 'Web Feature Service (WFS)', reference_uri: 'http://www.opengis.net/def/serviceType/ogc/wfs', label: false, note: 'Provides a to download generated vector datasets (GeoJSON, shapefile)', position: 25 }, + { name: 'wmts', reference_type: 'Web Mapping Service (WMS)', reference_uri: 'http://www.opengis.net/def/serviceType/ogc/wms', label: false, note: 'Provides a service to visually preview a layer and inspect its features', position: 26 }, + { name: 'wms', reference_type: 'WMTS', reference_uri: 'http://www.opengis.net/def/serviceType/ogc/wmts', label: false, note: '-', position: 27 }, + { name: 'xyz_tiles', reference_type: 'XYZ tiles', reference_uri: 'https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames', label: false, note: 'Link to an XYZ tile server', position: 28 } +] + +reference_types.each do |attributes| + ReferenceType.create!(attributes) +end + +puts "Reference types seeded successfully." \ No newline at end of file diff --git a/docs/geodev_references_audit_11072024.txt b/docs/geodev_references_audit_11072024.txt new file mode 100644 index 000000000..b29863268 --- /dev/null +++ b/docs/geodev_references_audit_11072024.txt @@ -0,0 +1,145 @@ +--- Audit Start --- + +NO MATCH — Duplicate holc-scan.jpg URL +Document: ed5705ef-7c1a-4497-b7af-fd971cee8e40 +CSV References Sorted: [["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=hudson-co.-nj", nil], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "GeoTIFF"], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "JPEG"]] +Document References Sorted: [["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=hudson-co.-nj", nil], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "JPEG"]] +Processed 1000 documents in this batch, total processed: 1000 +Processed 1000 documents in this batch, total processed: 2000 +Processed 1000 documents in this batch, total processed: 3000 + +NO MATCH - Duplicate rectified.zip URL +Document: 32027b5a-ea8a-42dd-875c-518cd3ab5811 +CSV References Sorted: [["32027b5a-ea8a-42dd-875c-518cd3ab5811", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=charleston-wv", nil], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/holc-scan.jpg", "JPG"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF (inset)"]] +Document References Sorted: [["32027b5a-ea8a-42dd-875c-518cd3ab5811", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=charleston-wv", nil], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/holc-scan.jpg", "JPG"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF"]] +Processed 1000 documents in this batch, total processed: 4000 +Processed 1000 documents in this batch, total processed: 5000 +Processed 1000 documents in this batch, total processed: 6000 +Processed 1000 documents in this batch, total processed: 7000 +Processed 1000 documents in this batch, total processed: 8000 +Processed 1000 documents in this batch, total processed: 9000 +Processed 1000 documents in this batch, total processed: 10000 +Processed 1000 documents in this batch, total processed: 11000 +Processed 1000 documents in this batch, total processed: 12000 +Processed 1000 documents in this batch, total processed: 13000 +Processed 1000 documents in this batch, total processed: 14000 +Processed 1000 documents in this batch, total processed: 15000 +Processed 1000 documents in this batch, total processed: 16000 +Processed 1000 documents in this batch, total processed: 17000 +Processed 1000 documents in this batch, total processed: 18000 +Processed 1000 documents in this batch, total processed: 19000 +Processed 1000 documents in this batch, total processed: 20000 +Processed 1000 documents in this batch, total processed: 21000 +Processed 1000 documents in this batch, total processed: 22000 +Processed 1000 documents in this batch, total processed: 23000 +Processed 1000 documents in this batch, total processed: 24000 +Processed 1000 documents in this batch, total processed: 25000 +Processed 1000 documents in this batch, total processed: 26000 +Processed 1000 documents in this batch, total processed: 27000 +Processed 1000 documents in this batch, total processed: 28000 +Processed 1000 documents in this batch, total processed: 29000 +Processed 1000 documents in this batch, total processed: 30000 +Processed 1000 documents in this batch, total processed: 31000 +Processed 1000 documents in this batch, total processed: 32000 +Processed 1000 documents in this batch, total processed: 33000 +Processed 1000 documents in this batch, total processed: 34000 +Processed 1000 documents in this batch, total processed: 35000 +Processed 1000 documents in this batch, total processed: 36000 +Processed 1000 documents in this batch, total processed: 37000 +Processed 1000 documents in this batch, total processed: 38000 +Processed 1000 documents in this batch, total processed: 39000 +Processed 1000 documents in this batch, total processed: 40000 +Processed 1000 documents in this batch, total processed: 41000 +Processed 1000 documents in this batch, total processed: 42000 +Processed 1000 documents in this batch, total processed: 43000 +Processed 1000 documents in this batch, total processed: 44000 +Processed 1000 documents in this batch, total processed: 45000 +Processed 1000 documents in this batch, total processed: 46000 +Processed 1000 documents in this batch, total processed: 47000 +Processed 1000 documents in this batch, total processed: 48000 +Processed 1000 documents in this batch, total processed: 49000 +Processed 1000 documents in this batch, total processed: 50000 +Processed 1000 documents in this batch, total processed: 51000 +Processed 1000 documents in this batch, total processed: 52000 +Processed 1000 documents in this batch, total processed: 53000 +Processed 1000 documents in this batch, total processed: 54000 +Processed 1000 documents in this batch, total processed: 55000 +Processed 1000 documents in this batch, total processed: 56000 +Processed 1000 documents in this batch, total processed: 57000 +Processed 1000 documents in this batch, total processed: 58000 +Processed 1000 documents in this batch, total processed: 59000 +Processed 1000 documents in this batch, total processed: 60000 +Processed 1000 documents in this batch, total processed: 61000 +Processed 1000 documents in this batch, total processed: 62000 +Processed 1000 documents in this batch, total processed: 63000 +Processed 1000 documents in this batch, total processed: 64000 +Processed 1000 documents in this batch, total processed: 65000 +Processed 1000 documents in this batch, total processed: 66000 +Processed 1000 documents in this batch, total processed: 67000 +Processed 1000 documents in this batch, total processed: 68000 +Processed 1000 documents in this batch, total processed: 69000 +Processed 1000 documents in this batch, total processed: 70000 +Processed 1000 documents in this batch, total processed: 71000 +Processed 1000 documents in this batch, total processed: 72000 +Processed 1000 documents in this batch, total processed: 73000 +Processed 1000 documents in this batch, total processed: 74000 +Processed 1000 documents in this batch, total processed: 75000 +Processed 1000 documents in this batch, total processed: 76000 +Processed 1000 documents in this batch, total processed: 77000 +Processed 1000 documents in this batch, total processed: 78000 +Processed 1000 documents in this batch, total processed: 79000 +Processed 1000 documents in this batch, total processed: 80000 +Processed 1000 documents in this batch, total processed: 81000 +Processed 1000 documents in this batch, total processed: 82000 +Processed 1000 documents in this batch, total processed: 83000 +Processed 1000 documents in this batch, total processed: 84000 +Processed 1000 documents in this batch, total processed: 85000 +Processed 1000 documents in this batch, total processed: 86000 +Processed 1000 documents in this batch, total processed: 87000 +Processed 1000 documents in this batch, total processed: 88000 +Processed 1000 documents in this batch, total processed: 89000 +Processed 1000 documents in this batch, total processed: 90000 +Processed 1000 documents in this batch, total processed: 91000 +Processed 1000 documents in this batch, total processed: 92000 + +# ERROR - harvard_download in dct_references_s (we don't support those) +Error auditing references for document: harvard-g7064-s2-1834-k3 - # +Processed 1000 documents in this batch, total processed: 93000 +Processed 1000 documents in this batch, total processed: 94000 +Processed 1000 documents in this batch, total processed: 95000 +Processed 1000 documents in this batch, total processed: 96000 + +NO MATCH - Duplicate holc-scan.jpg URL +Document: 82a3b7a2-f1b8-4a64-9832-a227ddd1a66d +CSV References Sorted: [["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=staten-island-ny", nil], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "GeoTIFF"], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "JPEG"]] +Document References Sorted: [["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=staten-island-ny", nil], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "JPEG"]] +Processed 1000 documents in this batch, total processed: 97000 +Processed 1000 documents in this batch, total processed: 98000 +Processed 1000 documents in this batch, total processed: 99000 +Processed 1000 documents in this batch, total processed: 100000 +Processed 1000 documents in this batch, total processed: 101000 +Processed 1000 documents in this batch, total processed: 102000 +Processed 1000 documents in this batch, total processed: 103000 + +NO MATCH - Duplicate shp_bdry_votingdistricts.zip URL +Document: test +CSV References Sorted: [["test", "download", "https://example.com", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "KMZ"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "Shapefile"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "anotherLabel"], ["test", "download", "https:/anotherExample.com", "anotherLabel"]] +Document References Sorted: [["test", "download", "https://example.com", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "KMZ"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "Shapefile"], ["test", "download", "https:/anotherExample.com", "anotherLabel"]] + +NO MATCH - Bad COG (array of URLs) +Document: btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166 +CSV References Sorted: [["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "cog", ["https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/51011e42-c559-47dd-8e18-f6388ecdc083/f2d92d1dfb83d2361547c62c5321be52.tif", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6b7500e0-acea-45c3-a8b2-47a8b2a5fb20/acc4aa6b5e9ff7f346a6a3f1b13cc2df.tif"], nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "documentation_external", "http://maps.indiana.edu/layerGallery.html", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "download", "http://maps.indiana.edu/download/Environment/Land_Cover_Impervious_Surfaces_2006.zip", "GeoTIFF"]] +Document References Sorted: [["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "cog", "[\"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/51011e42-c559-47dd-8e18-f6388ecdc083/f2d92d1dfb83d2361547c62c5321be52.tif\", \"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6b7500e0-acea-45c3-a8b2-47a8b2a5fb20/acc4aa6b5e9ff7f346a6a3f1b13cc2df.tif\"]", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "documentation_external", "http://maps.indiana.edu/layerGallery.html", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "download", "http://maps.indiana.edu/download/Environment/Land_Cover_Impervious_Surfaces_2006.zip", "GeoTIFF"]] +Processed 1000 documents in this batch, total processed: 104000 +Processed 1000 documents in this batch, total processed: 105000 +Processed 1000 documents in this batch, total processed: 106000 +Processed 1000 documents in this batch, total processed: 107000 +Processed 1000 documents in this batch, total processed: 108000 + +NO MATCH - Bad image (array of URLs) +Document: 59bd8ed8-088c-4932-97c9-544135dd9d64 +CSV References Sorted: [["59bd8ed8-088c-4932-97c9-544135dd9d64", "documentation_external", "https://michiganology.org/uncategorized/IO_59bd8ed8-088c-4932-97c9-544135dd9d64", nil], ["59bd8ed8-088c-4932-97c9-544135dd9d64", "image", ["https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/88b43100-06a1-47e0-8fe0-797b1768f4e0/952fffe1b6f9f9367fdec0cfca0b3c2b.png", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/29421b81-8152-4fff-972e-90f98ed57ee2/aa172e2dd422e038b09b0f081e82084e.png"], nil]] +Document References Sorted: [["59bd8ed8-088c-4932-97c9-544135dd9d64", "documentation_external", "https://michiganology.org/uncategorized/IO_59bd8ed8-088c-4932-97c9-544135dd9d64", nil], ["59bd8ed8-088c-4932-97c9-544135dd9d64", "image", "[\"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/88b43100-06a1-47e0-8fe0-797b1768f4e0/952fffe1b6f9f9367fdec0cfca0b3c2b.png\", \"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/29421b81-8152-4fff-972e-90f98ed57ee2/aa172e2dd422e038b09b0f081e82084e.png\"]", nil]] +Processed 1000 documents in this batch, total processed: 109000 +Processed 178 documents in this batch, total processed: 109178 +--- Audit End --- diff --git a/docs/localhost_references_audit_11082024.txt b/docs/localhost_references_audit_11082024.txt new file mode 100644 index 000000000..a399aaf20 --- /dev/null +++ b/docs/localhost_references_audit_11082024.txt @@ -0,0 +1,229 @@ +--- Audit Start --- + +NO MATCH - Duplicate holc-scan.jpg URL +Document: ed5705ef-7c1a-4497-b7af-fd971cee8e40 +CSV References Sorted: [["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=hudson-co.-nj", nil], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "GeoTIFF"], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "JPEG"]] +Document References Sorted: [["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=hudson-co.-nj", nil], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "JPEG"]] +Processed 1000 documents in this batch, total processed: 1000 +Processed 1000 documents in this batch, total processed: 2000 +Processed 1000 documents in this batch, total processed: 3000 + +NO MATCH - Duplicate rectified.zip URL +Document: 32027b5a-ea8a-42dd-875c-518cd3ab5811 +CSV References Sorted: [["32027b5a-ea8a-42dd-875c-518cd3ab5811", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=charleston-wv", nil], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/holc-scan.jpg", "JPG"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF (inset)"]] +Document References Sorted: [["32027b5a-ea8a-42dd-875c-518cd3ab5811", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=charleston-wv", nil], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/holc-scan.jpg", "JPG"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF"]] +Processed 1000 documents in this batch, total processed: 4000 + +NO MATCH +Document: 8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7 +CSV References Sorted: [["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/bcddb6b0-5a7c-4337-b5b2-385c03696b14/dba7e29239b5d738ebc61556ced3bd3f.zip", "wicolu02.zip"], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/bcddb6b0-5a7c-4337-b5b2-385c03696b14/dba7e29239b5d738ebc61556ced3bd3f.zip", "Shapefile"], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/83f2dd82-d3be-4064-85e1-fe0b3171e1fc/ba285518fe6a561b627dbdda66ba4455.pmtiles", nil]] +Document References Sorted: [["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/bcddb6b0-5a7c-4337-b5b2-385c03696b14/dba7e29239b5d738ebc61556ced3bd3f.zip", "wicolu02.zip"], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/83f2dd82-d3be-4064-85e1-fe0b3171e1fc/ba285518fe6a561b627dbdda66ba4455.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 5000 +Processed 1000 documents in this batch, total processed: 6000 +Processed 1000 documents in this batch, total processed: 7000 +Processed 1000 documents in this batch, total processed: 8000 +Processed 1000 documents in this batch, total processed: 9000 + +NO MATCH +Document: 9e5aa5c8-071f-43ab-9121-19433ca85d6e +CSV References Sorted: [["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/7c39c703-92d2-43f2-bd70-31ee228ff6ae/ac773faa312d4889104cf69d5f2a01e5.zip", "queelu02.zip"], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/7c39c703-92d2-43f2-bd70-31ee228ff6ae/ac773faa312d4889104cf69d5f2a01e5.zip", "Shapefile"], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/66eacb4b-9288-43dc-92aa-edee0c79cac0/2b2007b7b67d73828c76e80f17bb5a08.pmtiles", nil]] +Document References Sorted: [["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/7c39c703-92d2-43f2-bd70-31ee228ff6ae/ac773faa312d4889104cf69d5f2a01e5.zip", "queelu02.zip"], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/66eacb4b-9288-43dc-92aa-edee0c79cac0/2b2007b7b67d73828c76e80f17bb5a08.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 10000 + +NO MATCH +Document: btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927 +CSV References Sorted: [["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "cog", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ad2ac386-d3bf-4830-8044-9a1118805f09/7f3333d450a445df7fa068d3aee9537a.tif", nil], ["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/35843439-cf4c-485d-8176-581d9cf12305/c62a6e98889841135d3b3b611833f0ed.zip", "Aquifer_Sensitivity_Near_Surface.zip"], ["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/35843439-cf4c-485d-8176-581d9cf12305/c62a6e98889841135d3b3b611833f0ed.zip", "GeoTIFF"]] +Document References Sorted: [["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "cog", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ad2ac386-d3bf-4830-8044-9a1118805f09/7f3333d450a445df7fa068d3aee9537a.tif", nil], ["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/35843439-cf4c-485d-8176-581d9cf12305/c62a6e98889841135d3b3b611833f0ed.zip", "Aquifer_Sensitivity_Near_Surface.zip"]] +Processed 1000 documents in this batch, total processed: 11000 + +NO MATCH +Document: 0f9688d8-de7d-4d7c-8339-9c2ac314ee98 +CSV References Sorted: [["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/155cac25-9e6e-4042-905b-8ea59aa3fa64/2c58fb8d4806ef2b64312c64a9817032.zip", "prinlu02.zip"], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/155cac25-9e6e-4042-905b-8ea59aa3fa64/2c58fb8d4806ef2b64312c64a9817032.zip", "Shapefile"], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ea6a508b-36fa-4f5a-9794-203f78379871/4b5e2ffea0cdebd9240693757ad6cf7f.pmtiles", nil]] +Document References Sorted: [["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/155cac25-9e6e-4042-905b-8ea59aa3fa64/2c58fb8d4806ef2b64312c64a9817032.zip", "prinlu02.zip"], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ea6a508b-36fa-4f5a-9794-203f78379871/4b5e2ffea0cdebd9240693757ad6cf7f.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 12000 + +NO MATCH +Document: 5fc8dc18-62e6-4316-ba99-1a61534ea088 +CSV References Sorted: [["5fc8dc18-62e6-4316-ba99-1a61534ea088", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/2e1ba025-123b-4eaf-8a0c-ef3d5581ac16/a7b1ffe10d7fffd5616c5ba90a0b97cc.zip", "washlu02.zip"], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/2e1ba025-123b-4eaf-8a0c-ef3d5581ac16/a7b1ffe10d7fffd5616c5ba90a0b97cc.zip", "Shapefile"], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ec0fdd6d-1646-44b3-88f9-223e341adfc4/5670f2caa5330fd6a1579d5b55350dda.pmtiles", nil]] +Document References Sorted: [["5fc8dc18-62e6-4316-ba99-1a61534ea088", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/2e1ba025-123b-4eaf-8a0c-ef3d5581ac16/a7b1ffe10d7fffd5616c5ba90a0b97cc.zip", "washlu02.zip"], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ec0fdd6d-1646-44b3-88f9-223e341adfc4/5670f2caa5330fd6a1579d5b55350dda.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 13000 +Processed 1000 documents in this batch, total processed: 14000 +Processed 1000 documents in this batch, total processed: 15000 +Processed 1000 documents in this batch, total processed: 16000 + +NO MATCH +Document: e65d1fb2-5e3f-47b5-9341-d638931b508c +CSV References Sorted: [["e65d1fb2-5e3f-47b5-9341-d638931b508c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/742a7bce-eeb2-43ec-9f34-aff92ff38137/6287bb36020c79d998db621e8a374d6a.zip", "talblu02.zip"], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/742a7bce-eeb2-43ec-9f34-aff92ff38137/6287bb36020c79d998db621e8a374d6a.zip", "Shapefile"], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f9580be1-dd0c-4d2a-a81b-2d33fcf3308f/c37de751689c4351d44699d4158ea205.pmtiles", nil]] +Document References Sorted: [["e65d1fb2-5e3f-47b5-9341-d638931b508c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/742a7bce-eeb2-43ec-9f34-aff92ff38137/6287bb36020c79d998db621e8a374d6a.zip", "talblu02.zip"], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f9580be1-dd0c-4d2a-a81b-2d33fcf3308f/c37de751689c4351d44699d4158ea205.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 17000 + +NO MATCH +Document: 099bda09-d518-4a8e-8981-c0eb6e48495c +CSV References Sorted: [["099bda09-d518-4a8e-8981-c0eb6e48495c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/cea57932-72aa-4721-b94a-cec77d264b8f/8c32420230a07bbfab0bff0303c0bc99.zip", "howalu02.zip"], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/cea57932-72aa-4721-b94a-cec77d264b8f/8c32420230a07bbfab0bff0303c0bc99.zip", "Shapefile"], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/fec5b374-bed8-4397-9e7b-2533ea916031/2900232d9f00f80bf38fa7db455fdc7c.pmtiles", nil]] +Document References Sorted: [["099bda09-d518-4a8e-8981-c0eb6e48495c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/cea57932-72aa-4721-b94a-cec77d264b8f/8c32420230a07bbfab0bff0303c0bc99.zip", "howalu02.zip"], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/fec5b374-bed8-4397-9e7b-2533ea916031/2900232d9f00f80bf38fa7db455fdc7c.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 18000 + +NO MATCH +Document: demo-01 +CSV References Sorted: [["demo-01", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/1386180e-8d9b-4adc-b00f-d4fe1ce1d36c/448c534106e7d70f12f72e357351290c.zip", "geodatabase.gdb.zip"], ["demo-01", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/8f6f9903-fa07-40b8-b6bb-85316081c020/32332dc98388b849f8cbacbe9d706818.zip", "shapefile.zip"], ["demo-01", "metadata_html", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/9110e3ff-e5ca-4fd0-91a4-97417c2cdce8/57cf879488900529fea70d7654ccd0b9.html", nil], ["demo-01", "metadata_iso", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/3dab566a-9d04-40bd-b1fb-083a6e8bede1/fe994aa784d30437a134eb80ecf6212d.xml", nil], ["demo-01", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f78f027e-5ab2-4db4-a2a1-c34f9fe36626/495373519c7039d18be25db5bfa5e9a0.pmtiles", nil]] +Document References Sorted: [["demo-01", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/1386180e-8d9b-4adc-b00f-d4fe1ce1d36c/448c534106e7d70f12f72e357351290c.zip", "geodatabase.gdb.zip"], ["demo-01", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/8f6f9903-fa07-40b8-b6bb-85316081c020/32332dc98388b849f8cbacbe9d706818.zip", "shapefile.zip"], ["demo-01", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/1386180e-8d9b-4adc-b00f-d4fe1ce1d36c/448c534106e7d70f12f72e357351290c.zip", "geodatabase.gdb.zip"], ["demo-01", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/8f6f9903-fa07-40b8-b6bb-85316081c020/32332dc98388b849f8cbacbe9d706818.zip", "shapefile.zip"], ["demo-01", "metadata_html", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/9110e3ff-e5ca-4fd0-91a4-97417c2cdce8/57cf879488900529fea70d7654ccd0b9.html", nil], ["demo-01", "metadata_iso", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/3dab566a-9d04-40bd-b1fb-083a6e8bede1/fe994aa784d30437a134eb80ecf6212d.xml", nil], ["demo-01", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f78f027e-5ab2-4db4-a2a1-c34f9fe36626/495373519c7039d18be25db5bfa5e9a0.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 19000 +Processed 1000 documents in this batch, total processed: 20000 + +NO MATCH +Document: e0d15cad-86d4-4509-9b4e-758c1100d38d +CSV References Sorted: [["e0d15cad-86d4-4509-9b4e-758c1100d38d", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/ece861d7-0128-4181-9cd8-9e675d5bff28/b1a24a50f4c4a7a54dce428a5e7de09f.zip", "somelu02.zip"], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ece861d7-0128-4181-9cd8-9e675d5bff28/b1a24a50f4c4a7a54dce428a5e7de09f.zip", "Shapefile"], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/5e0b4359-2466-4c9a-8a67-e850373c92da/9016f58c38a31109f989a47094a75c35.pmtiles", nil]] +Document References Sorted: [["e0d15cad-86d4-4509-9b4e-758c1100d38d", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ece861d7-0128-4181-9cd8-9e675d5bff28/b1a24a50f4c4a7a54dce428a5e7de09f.zip", "somelu02.zip"], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/5e0b4359-2466-4c9a-8a67-e850373c92da/9016f58c38a31109f989a47094a75c35.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 21000 +Processed 1000 documents in this batch, total processed: 22000 +Processed 1000 documents in this batch, total processed: 23000 +Processed 1000 documents in this batch, total processed: 24000 +Processed 1000 documents in this batch, total processed: 25000 +Processed 1000 documents in this batch, total processed: 26000 + +NO MATCH +Document: f245136b-0d60-4315-abb0-52d0fbee4c5c +CSV References Sorted: [["f245136b-0d60-4315-abb0-52d0fbee4c5c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/fb544a75-16c4-4d84-88f3-7a3a26a330a4/14c781373ed8c16fa3ee69a6eed2c10b.zip", "charlu02.zip"], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/fb544a75-16c4-4d84-88f3-7a3a26a330a4/14c781373ed8c16fa3ee69a6eed2c10b.zip", "Shapefile"], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/d3c052e8-2021-4010-9f9b-28cd39ced401/7b9a5f03ec43669e51b1c38c9a40df4a.pmtiles", nil]] +Document References Sorted: [["f245136b-0d60-4315-abb0-52d0fbee4c5c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/fb544a75-16c4-4d84-88f3-7a3a26a330a4/14c781373ed8c16fa3ee69a6eed2c10b.zip", "charlu02.zip"], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/d3c052e8-2021-4010-9f9b-28cd39ced401/7b9a5f03ec43669e51b1c38c9a40df4a.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 27000 +Processed 1000 documents in this batch, total processed: 28000 +Processed 1000 documents in this batch, total processed: 29000 +Processed 1000 documents in this batch, total processed: 30000 +Processed 1000 documents in this batch, total processed: 31000 + +NO MATCH +Document: aug-5-2024 +CSV References Sorted: [["aug-5-2024", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/cc88fc5e-9dc4-4f09-89cf-22335910716c/91b4c31ee446c420390a16571a3971ab.csv", nil], ["aug-5-2024", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/02c06377-071d-4019-b89b-ee5b937776cd/efd61e3342b3b14b389ba7ad08d5065e.zip", "Geodatabase"], ["aug-5-2024", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/e0984e99-e174-45ac-be0e-a17101197f8e/7dc9b32d545b515323d5ae068aee9026.zip", "Shapefile"], ["aug-5-2024", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/e0984e99-e174-45ac-be0e-a17101197f8e/7dc9b32d545b515323d5ae068aee9026.zip", "Mixed"], ["aug-5-2024", "metadata_fgdc", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/4cb38259-b1ac-4ab7-b075-dc1cd548ba7b/3cec5aeb7d6f68a0939204f2d3c3993c.xml", nil], ["aug-5-2024", "metadata_html", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ba4716ea-c86d-4ea1-98b5-a7433daacdfb/dc523dc6e0a1d6533b8ce49fcee297e4.html", nil], ["aug-5-2024", "metadata_iso", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/b5c33f20-16a5-4bad-83e1-8cbc65aad67a/2fbc5ed3ad78410cbf7c1d39ac7470bb.xml", nil], ["aug-5-2024", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/af264c4c-862a-46dd-a3b4-88973a8fa754/c8858797b551c94a92ecbe8387d37b85.pmtiles", nil]] +Document References Sorted: [["aug-5-2024", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/cc88fc5e-9dc4-4f09-89cf-22335910716c/91b4c31ee446c420390a16571a3971ab.csv", nil], ["aug-5-2024", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/02c06377-071d-4019-b89b-ee5b937776cd/efd61e3342b3b14b389ba7ad08d5065e.zip", "Geodatabase"], ["aug-5-2024", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/e0984e99-e174-45ac-be0e-a17101197f8e/7dc9b32d545b515323d5ae068aee9026.zip", "Shapefile"], ["aug-5-2024", "metadata_fgdc", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/4cb38259-b1ac-4ab7-b075-dc1cd548ba7b/3cec5aeb7d6f68a0939204f2d3c3993c.xml", nil], ["aug-5-2024", "metadata_html", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ba4716ea-c86d-4ea1-98b5-a7433daacdfb/dc523dc6e0a1d6533b8ce49fcee297e4.html", nil], ["aug-5-2024", "metadata_iso", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/b5c33f20-16a5-4bad-83e1-8cbc65aad67a/2fbc5ed3ad78410cbf7c1d39ac7470bb.xml", nil], ["aug-5-2024", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/af264c4c-862a-46dd-a3b4-88973a8fa754/c8858797b551c94a92ecbe8387d37b85.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 32000 + +NO MATCH +Document: btaa-3020c203-5325-4bec-92e3-4a16e267c14e +CSV References Sorted: [["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "documentation_external", "http://maps.indiana.edu/previewMaps/Infrastructure/Railroads_Active_Abandoned_INDOT.html", nil], ["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/0027dc65-5d68-4d1e-a996-efd7886a1f5c/360dbb031fbaa6f01db6e6046beb228b.zip", "Shapefile"]] +Document References Sorted: [["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "documentation_external", "http://maps.indiana.edu/previewMaps/Infrastructure/Railroads_Active_Abandoned_INDOT.html", nil], ["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/0027dc65-5d68-4d1e-a996-efd7886a1f5c/360dbb031fbaa6f01db6e6046beb228b.zip", "Shapefile"], ["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/0027dc65-5d68-4d1e-a996-efd7886a1f5c/360dbb031fbaa6f01db6e6046beb228b.zip", "Shapefile"]] +Processed 1000 documents in this batch, total processed: 33000 +Processed 1000 documents in this batch, total processed: 34000 + +NO MATCH +Document: 87d2a9e8-7c16-40a9-8fab-155a1c6b209d +CSV References Sorted: [["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/bcec399e-2730-49de-99ce-863ea2153c66/5923ebeafb0f6cd35886c0055b8a9f09.zip", "calvlu02.zip"], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/bcec399e-2730-49de-99ce-863ea2153c66/5923ebeafb0f6cd35886c0055b8a9f09.zip", "Shapefile"], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6bf2d6a1-3027-4a44-ab14-ce9aa6e24565/d0460a41554de8fe92e78393338423c7.pmtiles", nil]] +Document References Sorted: [["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/bcec399e-2730-49de-99ce-863ea2153c66/5923ebeafb0f6cd35886c0055b8a9f09.zip", "calvlu02.zip"], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6bf2d6a1-3027-4a44-ab14-ce9aa6e24565/d0460a41554de8fe92e78393338423c7.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 35000 +Processed 1000 documents in this batch, total processed: 36000 +Processed 1000 documents in this batch, total processed: 37000 + +NO MATCH +Document: df107945-d91f-40f0-8802-504544918d48 +CSV References Sorted: [["df107945-d91f-40f0-8802-504544918d48", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["df107945-d91f-40f0-8802-504544918d48", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/520d12e8-a4f3-426a-aa51-4c9f61f7448e/4360868ca1fa192b55fe04ce71d0882d.zip", "fredlu02.zip"], ["df107945-d91f-40f0-8802-504544918d48", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/520d12e8-a4f3-426a-aa51-4c9f61f7448e/4360868ca1fa192b55fe04ce71d0882d.zip", "Shapefile"], ["df107945-d91f-40f0-8802-504544918d48", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/a0543f18-79a1-49d3-ba9a-dda899ca9574/61d5276991895701d076ace7d6881e8d.pmtiles", nil]] +Document References Sorted: [["df107945-d91f-40f0-8802-504544918d48", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["df107945-d91f-40f0-8802-504544918d48", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/520d12e8-a4f3-426a-aa51-4c9f61f7448e/4360868ca1fa192b55fe04ce71d0882d.zip", "fredlu02.zip"], ["df107945-d91f-40f0-8802-504544918d48", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/a0543f18-79a1-49d3-ba9a-dda899ca9574/61d5276991895701d076ace7d6881e8d.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 38000 + +NO MATCH +Document: ca1590a9-6077-4e2c-a029-beefc614c9a2 +CSV References Sorted: [["ca1590a9-6077-4e2c-a029-beefc614c9a2", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/db90a215-2255-41d2-9f75-ad12c3515257/600fda0a7b49192369ac0262fde3610d.zip", "harflu02.zip"], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/db90a215-2255-41d2-9f75-ad12c3515257/600fda0a7b49192369ac0262fde3610d.zip", "Shapefile"], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/12a76d2b-eb87-4f81-a135-f5cecf087304/b5d74dcbaa8216fb199a9e55f2e4d555.pmtiles", nil]] +Document References Sorted: [["ca1590a9-6077-4e2c-a029-beefc614c9a2", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/db90a215-2255-41d2-9f75-ad12c3515257/600fda0a7b49192369ac0262fde3610d.zip", "harflu02.zip"], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/12a76d2b-eb87-4f81-a135-f5cecf087304/b5d74dcbaa8216fb199a9e55f2e4d555.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 39000 +Processed 1000 documents in this batch, total processed: 40000 +Processed 1000 documents in this batch, total processed: 41000 + +NO MATCH +Document: a3b32ab0-3149-4f1d-b6e9-9c51dafe5613 +CSV References Sorted: [["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/eaa41daf-752a-4f4f-9c43-1742f3e8f2e5/ce0ece3a00dd056344c18f11010d086d.zip", "garrlu02.zip"], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/eaa41daf-752a-4f4f-9c43-1742f3e8f2e5/ce0ece3a00dd056344c18f11010d086d.zip", "Shapefile"], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/46ed8c47-6729-4d5c-99a2-c31e91207bbf/daa05ab70dbbadd384996454a1fa92a1.pmtiles", nil]] +Document References Sorted: [["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/eaa41daf-752a-4f4f-9c43-1742f3e8f2e5/ce0ece3a00dd056344c18f11010d086d.zip", "garrlu02.zip"], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/46ed8c47-6729-4d5c-99a2-c31e91207bbf/daa05ab70dbbadd384996454a1fa92a1.pmtiles", nil]] + +NO MATCH +Document: 1da88d05-7b51-48ae-b5d5-4711c052b3f2 +CSV References Sorted: [["1da88d05-7b51-48ae-b5d5-4711c052b3f2", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/f55e0cc3-3684-491c-99bc-702014c2b61b/293cd53d936793143105427e9a2b6781.zip", "Surficial_Unconsolidated_Thickness_DEM.zip"]] +Document References Sorted: [["1da88d05-7b51-48ae-b5d5-4711c052b3f2", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/f55e0cc3-3684-491c-99bc-702014c2b61b/293cd53d936793143105427e9a2b6781.zip", "Surficial_Unconsolidated_Thickness_DEM.zip"], ["1da88d05-7b51-48ae-b5d5-4711c052b3f2", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f55e0cc3-3684-491c-99bc-702014c2b61b/293cd53d936793143105427e9a2b6781.zip", "Surficial_Unconsolidated_Thickness_DEM.zip"]] +Processed 1000 documents in this batch, total processed: 42000 +Processed 1000 documents in this batch, total processed: 43000 +Processed 1000 documents in this batch, total processed: 44000 +Processed 1000 documents in this batch, total processed: 45000 +Processed 1000 documents in this batch, total processed: 46000 +Processed 1000 documents in this batch, total processed: 47000 +Processed 1000 documents in this batch, total processed: 48000 +Processed 1000 documents in this batch, total processed: 49000 +Processed 1000 documents in this batch, total processed: 50000 +Processed 1000 documents in this batch, total processed: 51000 +Processed 1000 documents in this batch, total processed: 52000 +Processed 1000 documents in this batch, total processed: 53000 +Processed 1000 documents in this batch, total processed: 54000 +Processed 1000 documents in this batch, total processed: 55000 +Processed 1000 documents in this batch, total processed: 56000 +Processed 1000 documents in this batch, total processed: 57000 +Processed 1000 documents in this batch, total processed: 58000 +Processed 1000 documents in this batch, total processed: 59000 +Processed 1000 documents in this batch, total processed: 60000 +Processed 1000 documents in this batch, total processed: 61000 +Processed 1000 documents in this batch, total processed: 62000 +Processed 1000 documents in this batch, total processed: 63000 +Processed 1000 documents in this batch, total processed: 64000 +Processed 1000 documents in this batch, total processed: 65000 +Processed 1000 documents in this batch, total processed: 66000 +Processed 1000 documents in this batch, total processed: 67000 +Processed 1000 documents in this batch, total processed: 68000 +Processed 1000 documents in this batch, total processed: 69000 +Processed 1000 documents in this batch, total processed: 70000 +Processed 1000 documents in this batch, total processed: 71000 +Processed 1000 documents in this batch, total processed: 72000 +Processed 1000 documents in this batch, total processed: 73000 +Processed 1000 documents in this batch, total processed: 74000 +Processed 1000 documents in this batch, total processed: 75000 +Processed 1000 documents in this batch, total processed: 76000 +Processed 1000 documents in this batch, total processed: 77000 +Processed 1000 documents in this batch, total processed: 78000 +Processed 1000 documents in this batch, total processed: 79000 +Processed 1000 documents in this batch, total processed: 80000 +Processed 1000 documents in this batch, total processed: 81000 +Processed 1000 documents in this batch, total processed: 82000 +Processed 1000 documents in this batch, total processed: 83000 +Processed 1000 documents in this batch, total processed: 84000 +Processed 1000 documents in this batch, total processed: 85000 +Processed 1000 documents in this batch, total processed: 86000 +Processed 1000 documents in this batch, total processed: 87000 +Processed 1000 documents in this batch, total processed: 88000 +Processed 1000 documents in this batch, total processed: 89000 +Processed 1000 documents in this batch, total processed: 90000 +Processed 1000 documents in this batch, total processed: 91000 +Processed 1000 documents in this batch, total processed: 92000 + +Error auditing references for document: harvard-g7064-s2-1834-k3 - # +Processed 1000 documents in this batch, total processed: 93000 +Processed 1000 documents in this batch, total processed: 94000 +Processed 1000 documents in this batch, total processed: 95000 +Processed 1000 documents in this batch, total processed: 96000 + +NO MATCH - Duplicate holc-scan.jpg URL +Document: 82a3b7a2-f1b8-4a64-9832-a227ddd1a66d +CSV References Sorted: [["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=staten-island-ny", nil], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "GeoTIFF"], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "JPEG"]] +Document References Sorted: [["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=staten-island-ny", nil], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "JPEG"]] +Processed 1000 documents in this batch, total processed: 97000 +Processed 1000 documents in this batch, total processed: 98000 +Processed 1000 documents in this batch, total processed: 99000 +Processed 1000 documents in this batch, total processed: 100000 +Processed 1000 documents in this batch, total processed: 101000 +Processed 1000 documents in this batch, total processed: 102000 +Processed 1000 documents in this batch, total processed: 103000 + +NO MATCH +Document: test +CSV References Sorted: [["test", "download", "https://example.com", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "KMZ"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "Shapefile"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "anotherLabel"], ["test", "download", "https:/anotherExample.com", "anotherLabel"]] +Document References Sorted: [["test", "download", "https://example.com", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "KMZ"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "Shapefile"], ["test", "download", "https:/anotherExample.com", "anotherLabel"]] + +NO MATCH - Array of cog URLs? +Document: btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166 +CSV References Sorted: [["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "cog", ["https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/51011e42-c559-47dd-8e18-f6388ecdc083/f2d92d1dfb83d2361547c62c5321be52.tif", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6b7500e0-acea-45c3-a8b2-47a8b2a5fb20/acc4aa6b5e9ff7f346a6a3f1b13cc2df.tif"], nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "documentation_external", "http://maps.indiana.edu/layerGallery.html", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "download", "http://maps.indiana.edu/download/Environment/Land_Cover_Impervious_Surfaces_2006.zip", "GeoTIFF"]] +Document References Sorted: [["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "cog", "[\"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/51011e42-c559-47dd-8e18-f6388ecdc083/f2d92d1dfb83d2361547c62c5321be52.tif\", \"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6b7500e0-acea-45c3-a8b2-47a8b2a5fb20/acc4aa6b5e9ff7f346a6a3f1b13cc2df.tif\"]", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "documentation_external", "http://maps.indiana.edu/layerGallery.html", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "download", "http://maps.indiana.edu/download/Environment/Land_Cover_Impervious_Surfaces_2006.zip", "GeoTIFF"]] +Processed 1000 documents in this batch, total processed: 104000 +Processed 1000 documents in this batch, total processed: 105000 +Processed 1000 documents in this batch, total processed: 106000 +Processed 1000 documents in this batch, total processed: 107000 +Processed 1000 documents in this batch, total processed: 108000 + +NO MATCH - Array of image URLs? +Document: 59bd8ed8-088c-4932-97c9-544135dd9d64 +CSV References Sorted: [["59bd8ed8-088c-4932-97c9-544135dd9d64", "documentation_external", "https://michiganology.org/uncategorized/IO_59bd8ed8-088c-4932-97c9-544135dd9d64", nil], ["59bd8ed8-088c-4932-97c9-544135dd9d64", "image", ["https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/88b43100-06a1-47e0-8fe0-797b1768f4e0/952fffe1b6f9f9367fdec0cfca0b3c2b.png", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/29421b81-8152-4fff-972e-90f98ed57ee2/aa172e2dd422e038b09b0f081e82084e.png"], nil]] +Document References Sorted: [["59bd8ed8-088c-4932-97c9-544135dd9d64", "documentation_external", "https://michiganology.org/uncategorized/IO_59bd8ed8-088c-4932-97c9-544135dd9d64", nil], ["59bd8ed8-088c-4932-97c9-544135dd9d64", "image", "[\"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/88b43100-06a1-47e0-8fe0-797b1768f4e0/952fffe1b6f9f9367fdec0cfca0b3c2b.png\", \"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/29421b81-8152-4fff-972e-90f98ed57ee2/aa172e2dd422e038b09b0f081e82084e.png\"]", nil]] +Processed 1000 documents in this batch, total processed: 109000 +Processed 178 documents in this batch, total processed: 109178 +--- Audit End --- diff --git a/docs/references_migration.txt b/docs/references_migration.txt new file mode 100644 index 000000000..27a194a5c --- /dev/null +++ b/docs/references_migration.txt @@ -0,0 +1,21 @@ +# Migration steps + +1. Set `GBL_ADMIN_REFERENCES_MIGRATED=false` in .env.production +2. Deploy code to box +3. Migrate references +4. Audit references +5. Set `GBL_ADMIN_REFERENCES_MIGRATED=true` in .env.production +6. Restart puma and sidekiq via: + +``` +sudo systemctl restart puma +sudo systemctl restart sidekiq +``` + +7. Reindex documents + +``` +RAILS_ENV=production bundle exec rake geoblacklight_admin:solr:reindex +``` + +8. PROFIT! diff --git a/solr/conf/schema.xml b/solr/conf/schema.xml index c8f475539..02343d0ea 100644 --- a/solr/conf/schema.xml +++ b/solr/conf/schema.xml @@ -226,4 +226,7 @@ + + + diff --git a/yarn.lock b/yarn.lock index 8bd8854d5..6fc1e4872 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1647,9 +1647,9 @@ integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== "@geoblacklight/admin@^0.4.4": - version "0.4.4" - resolved "https://registry.yarnpkg.com/@geoblacklight/admin/-/admin-0.4.4.tgz#359033da67b5a10f4e713f8bbc9a9802aa011064" - integrity sha512-De3OtOzzNs0ZpHm+D3FLeiJDvF4nFpsJ7wJXHUB9nU1KKjUDi1DObxvAgboACbRt/iG0XYEHQbsB5vw21EsbfQ== + version "0.4.12" + resolved "https://registry.yarnpkg.com/@geoblacklight/admin/-/admin-0.4.12.tgz#a6270411af86bebc5d833ac5c1743ef4f78c42bd" + integrity sha512-BlP9CiwuUig3N5TeayHX64chuScej70jKI+pcpu6J5nEypEgai3k/LUgyVabjnzmg831FYf8QJ97yk++d1xQ6A== dependencies: "@hotwired/stimulus" "^3.2.2" "@nathanvda/cocoon" "^1.2.14" @@ -2664,9 +2664,9 @@ "@babel/runtime" "^7.13.10" "@rails/actioncable@^6.0.0": - version "6.1.7" - resolved "https://registry.yarnpkg.com/@rails/actioncable/-/actioncable-6.1.7.tgz#8b4506925d3f7146a70941e4db7ce9dda99f99ae" - integrity sha512-F6S74NGpxhbbDRFsQFGYqefRfZPgYvePNtz9hHKYOqLturrsqrDoG+UcrxEGHsvqDUorMYfx4Wl3K8smmk/u2g== + version "6.1.710" + resolved "https://registry.yarnpkg.com/@rails/actioncable/-/actioncable-6.1.710.tgz#5c77515cbc9b8fb5118afa0aa0048192b33c33ea" + integrity sha512-YDzMuh8mDRNLFXiexlLVnF1NUwk1bXwwO1gtEhOCsCqIsL6D21JfxpMwAD5x8qzrMJ1J0RZVa2hLl7ofeg+Mqw== "@rails/actioncable@^7.0.7-2": version "7.1.3" @@ -2674,9 +2674,9 @@ integrity sha512-ojNvnoZtPN0pYvVFtlO7dyEN9Oml1B6IDM+whGKVak69MMYW99lC2NOWXWeE3bmwEydbP/nn6ERcpfjHVjYQjA== "@rails/activestorage@^6.0.0": - version "6.1.7" - resolved "https://registry.yarnpkg.com/@rails/activestorage/-/activestorage-6.1.7.tgz#5aaae9f4d10800fdb4fd6fe26fd8b4218579c6e3" - integrity sha512-h++k8LBLns4O8AqzdaFp1TsCLP9VSc2hgWI37bjzJ+4D995X7Rd8kdkRmXRaNAUlHDJgy6RpnbhBJ5oiIgWTDw== + version "6.1.710" + resolved "https://registry.yarnpkg.com/@rails/activestorage/-/activestorage-6.1.710.tgz#bc914716f642ba233f033b7765a44dc3bfb626f5" + integrity sha512-hLXxAtn7hSWXkTzMGOmQmjuzJ0FzVw8j/Zi8DGS+DG9x4uPQjl+ZEVdty7pFcsBCjkpejtk0TChcBQlLW8sgOg== dependencies: spark-md5 "^3.0.0" @@ -2688,9 +2688,9 @@ spark-md5 "^3.0.1" "@rails/ujs@^6.0.0": - version "6.1.7" - resolved "https://registry.yarnpkg.com/@rails/ujs/-/ujs-6.1.7.tgz#b09dc5b2105dd267e8374c47e4490240451dc7f6" - integrity sha512-0e7WQ4LE/+LEfW2zfAw9ppsB6A8RmxbdAUPAF++UT80epY+7emuQDkKXmaK0a9lp6An50RvzezI0cIQjp1A58w== + version "6.1.710" + resolved "https://registry.yarnpkg.com/@rails/ujs/-/ujs-6.1.710.tgz#14cbf938c49985af51cca52d117f9c358ea4a579" + integrity sha512-NUS8nUkYHGSn/EDg3oJCkpMweSKeFxA4ef6pcnW2fJEZ7RcoWdJa1rr9vamUoqzNR+Ctd8ezsgBMAjxUahVB1w== "@rails/ujs@^7.0.7-2": version "7.1.3"