DP-827: Reorganize into a proper module (#3)

bin/gingr

@@ -1,115 +1,4 @@
 #!/usr/bin/env ruby
-require 'find'
-require 'json'
-require 'thor'
-require_relative '../lib/data_handler'
-require_relative '../lib/geoserver_publisher'
-require_relative '../lib/import_util'
-require_relative '../lib/solr_indexer'
+require_relative '../lib/gingr'
 
-# Gingr module
-module Gingr
-  # ingestion tasks
-  class Import < Thor
-    include Config
-    include ImportUtil
-
-    Thor.check_unknown_options!
-
-    desc 'solr',
-         'Giving a directory path, it will index all json files from the directory/sub-directory to solr'
-    long_desc <<-TEXT, wrapping: false
-          examples:\n
-          1) ruby bin/import solr tmp/test_public \n
-          2) ruby bin/import solr tmp/test_public  --no-update_reference_field \n
-            (it will update reference urls from 'dct_references_s' field in each geoblacklight json file \n
-            with current download_url, geoserver_url, geoserver_secure_url)
-    TEXT
-    option :download_url
-    option :geoserver_url
-    option :geoserver_secure_url
-    option :update_reference_field, type: :boolean, default: false
-    option :solr_url
-    def solr(dir_path)
-      reference_urls = ImportUtil.get_reference_urls(options)
-      solr_url = options[:solr_url] || ENV.fetch('SOLR_URL', nil)
-      ImportUtil.index_solr_from_dir(dir_path, solr_url, reference_urls)
-      txt = "all json files under '#{dir_path}' and subdirectories have been indexed to solr #{solr_url} successfully"
-      Config.logger.info(txt)
-    end
-
-    desc 'geoserver', 'publish a giving shapefile or GeoTIFF file to a geoserver'
-    long_desc <<-TEXT, wrapping: false
-         examples: \n
-         1) ruby bin/import geoserver fk4cr7f93g.shp \n
-         2) ruby bin/import geoserver fk4h14n50v.shp  --no-is-public
-    TEXT
-    option :geoserver_url
-    option :is_public, type: :boolean, default: true
-    def geoserver(filename)
-      url = options[:geoserver_url]
-      url ||= options[:is_public] ? ENV.fetch('GEOSERVER_URL', nil) : ENV.fetch('GEOSERVER_SECURE_URL', nil)
-      publisher = GeoserverPublisher.new(url)
-      publisher.update(filename)
-      Config.logger.info("'#{filename}' - published to geoserver #{url} successfully")
-    end
-
-    desc 'unpack',
-         'unpack a giving zip file, move shapefiles and GeoTIFF files to geoserver_root, other files to spatial_root'
-    long_desc <<-TEXT, wrapping: false
-         * When giving a zip file without path, it will look for a zip file under /app/import/
-    TEXT
-    option :spatial_root
-    option :geoserver_root
-    def unpack(zipfile)
-      zipfile_path = zipfile == File.basename(zipfile) ? File.join(ImportUtil.root_path, 'import', zipfile) : zipfile
-      DataHandler.spatial_root = options[:spatial_root] || ENV.fetch('SPATIAL_ROOT', nil)
-      DataHandler.geoserver_root = options[:geoserver_root] || ENV.fetch('GEOSERVER_ROOT', nil)
-
-      temp_path = File.join(Dir.pwd, 'tmp')
-      DataHandler.extract_and_move(zipfile_path, temp_path)
-    end
-
-    desc 'all',
-         'unpack a giving zip file, move files, index json files to solr and publish geofiles to geoservers'
-    long_desc <<-TEXT, wrapping: false
-          1) move all geofiles to geoserver_root \n
-          2) move all data.zip, ISO19139.xml and document files to spatial_root \n
-          2) index all geoblacklight json files to solr \n
-          3) publish all shapefiles and GeoTIFF files to geoserver \n
-    TEXT
-    option :solr_url
-    option :update_reference_field, type: :boolean, default: false
-    option :spatial_root
-    option :geoserver_root
-    option :geoserver_url
-    option :geoserver_secure_url
-    def all(zipfile)
-      unpacked = unpack(zipfile)
-      solr(unpacked[:extract_to_path])
-
-      geofile_names = unpacked[:geofile_name_hash]
-      ImportUtil.publish_geoservers(geofile_names, options)
-      Config.logger.info("#{zipfile} - all imported")
-    end
-
-    desc 'geoserver_workspace', 'create a workspace in a geoserver'
-    long_desc <<-LONGDESC
-         This is for spec test. Geodata website only needs one workspace "UCB"
-    LONGDESC
-    option :geoserver_url
-    option :is_public, type: :boolean, default: true
-    def geoserver_workspace(name)
-      url = options[:geoserver_url]
-      url ||= options[:is_public] ? ENV.fetch('GEOSERVER_URL', nil) : ENV.fetch('GEOSERVER_SECURE_URL', nil)
-      publisher = GeoserverPublisher.new(url)
-      publisher.create_workspace(name)
-      Config.logger.info("geoserver workspace '#{name}' - created successfully")
-    end
-
-    def self.exit_on_failure?
-      true
-    end
-  end
-end
-Gingr::Import.start(ARGV)
+Gingr::Cli.start(ARGV)

bin/gingr-examples.sh

@@ -2,28 +2,28 @@
 
 # unpack zipfile
 echo "1 - unpack zipfile starting..."
-ruby bin/import unpack spec/fixture/zipfile/test_public.zip
+ruby bin/gingr unpack spec/fixture/zipfile/test_public.zip
 echo "unpack zipfile ends"
 
 # index solr
 echo "2 - index solr starting..."
-ruby bin/import solr tmp/test_public/
+ruby bin/gingr solr tmp/test_public/
 echo "index solr ends"
 
 # publish geoserver (need testing both public and restricted geoserver?)
 echo "3.1 - create geoserver workspace ..."
-ruby bin/import geoserver_workspace UCB
+ruby bin/gingr geoserver_workspace UCB
 echo "create create geoserver workspace ends"
 
 echo "3.2 - publish vector to geoserver starting..."
-ruby bin/import geoserver fk4cr7f93g.shp
+ruby bin/gingr geoserver fk4cr7f93g.shp
 echo "publish vector to geoserver ends"
 
 # echo "3.3 - publish raster to geoserver starting..."
-# ruby bin/import geoserver bbbb.TIFF
+# ruby bin/gingr geoserver bbbb.TIFF
 # echo "publish raster to geoserver ends"
 
 # import all with a zipfile
 echo "4 - import all starting..."
-ruby bin/import all spec/fixture/zipfile/test_public.zip
+ruby bin/gingr all spec/fixture/zipfile/test_public.zip
 echo "import all ends"

lib/gingr.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# monkey-patch first
+require_relative 'monkeypatch/geoserver/publish'
+
+require_relative 'gingr/cli'
+require_relative 'gingr/config'
+require_relative 'gingr/data_handler'
+require_relative 'gingr/geoserver_publisher'
+require_relative 'gingr/import_util'
+require_relative 'gingr/solr_indexer'
+
+module Gingr
+  #
+end

lib/gingr/cli.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+require 'thor'
+require_relative 'config'
+require_relative 'import_util'
+
+module Gingr
+  class Cli < Thor
+    include Config
+    include ImportUtil
+
+    Thor.check_unknown_options!
+
+    desc 'solr',
+         'Giving a directory path, it will index all json files from the directory/sub-directory to solr'
+    long_desc <<-TEXT, wrapping: false
+          examples:\n
+          1) ruby bin/import solr tmp/test_public \n
+          2) ruby bin/import solr tmp/test_public  --no-update_reference_field \n
+            (it will update reference urls from 'dct_references_s' field in each geoblacklight json file \n
+            with current download_url, geoserver_url, geoserver_secure_url)
+    TEXT
+    option :download_url
+    option :geoserver_url
+    option :geoserver_secure_url
+    option :update_reference_field, type: :boolean, default: false
+    option :solr_url
+    def solr(dir_path)
+      reference_urls = ImportUtil.get_reference_urls(options)
+      solr_url = options[:solr_url] || ENV.fetch('SOLR_URL', nil)
+      ImportUtil.index_solr_from_dir(dir_path, solr_url, reference_urls)
+      txt = "all json files under '#{dir_path}' and subdirectories have been indexed to solr #{solr_url} successfully"
+      Config.logger.info(txt)
+    end
+
+    desc 'geoserver', 'publish a giving shapefile or GeoTIFF file to a geoserver'
+    long_desc <<-TEXT, wrapping: false
+         examples: \n
+         1) ruby bin/import geoserver fk4cr7f93g.shp \n
+         2) ruby bin/import geoserver fk4h14n50v.shp  --no-is-public
+    TEXT
+    option :geoserver_url
+    option :is_public, type: :boolean, default: true
+    def geoserver(filename)
+      url = options[:geoserver_url]
+      url ||= options[:is_public] ? ENV.fetch('GEOSERVER_URL', nil) : ENV.fetch('GEOSERVER_SECURE_URL', nil)
+      publisher = GeoserverPublisher.new(url)
+      publisher.update(filename)
+      Config.logger.info("'#{filename}' - published to geoserver #{url} successfully")
+    end
+
+    desc 'unpack',
+         'unpack a giving zip file, move shapefiles and GeoTIFF files to geoserver_root, other files to spatial_root'
+    long_desc <<-TEXT, wrapping: false
+         * When giving a zip file without path, it will look for a zip file under /app/import/
+    TEXT
+    option :spatial_root
+    option :geoserver_root
+    def unpack(zipfile)
+      zipfile_path = zipfile == File.basename(zipfile) ? File.join(ImportUtil.root_path, 'import', zipfile) : zipfile
+      DataHandler.spatial_root = options[:spatial_root] || ENV.fetch('SPATIAL_ROOT', nil)
+      DataHandler.geoserver_root = options[:geoserver_root] || ENV.fetch('GEOSERVER_ROOT', nil)
+
+      temp_path = File.join(Dir.pwd, 'tmp')
+      DataHandler.extract_and_move(zipfile_path, temp_path)
+    end
+
+    desc 'all',
+         'unpack a giving zip file, move files, index json files to solr and publish geofiles to geoservers'
+    long_desc <<-TEXT, wrapping: false
+          1) move all geofiles to geoserver_root \n
+          2) move all data.zip, ISO19139.xml and document files to spatial_root \n
+          2) index all geoblacklight json files to solr \n
+          3) publish all shapefiles and GeoTIFF files to geoserver \n
+    TEXT
+    option :solr_url
+    option :update_reference_field, type: :boolean, default: false
+    option :spatial_root
+    option :geoserver_root
+    option :geoserver_url
+    option :geoserver_secure_url
+    def all(zipfile)
+      unpacked = unpack(zipfile)
+      solr(unpacked[:extract_to_path])
+
+      geofile_names = unpacked[:geofile_name_hash]
+      ImportUtil.publish_geoservers(geofile_names, options)
+      Config.logger.info("#{zipfile} - all imported")
+    end
+
+    desc 'geoserver_workspace', 'create a workspace in a geoserver'
+    long_desc <<-LONGDESC
+         This is for spec test. Geodata website only needs one workspace "UCB"
+    LONGDESC
+    option :geoserver_url
+    option :is_public, type: :boolean, default: true
+    def geoserver_workspace(name)
+      url = options[:geoserver_url]
+      url ||= options[:is_public] ? ENV.fetch('GEOSERVER_URL', nil) : ENV.fetch('GEOSERVER_SECURE_URL', nil)
+      publisher = GeoserverPublisher.new(url)
+      publisher.create_workspace(name)
+      Config.logger.info("geoserver workspace '#{name}' - created successfully")
+    end
+
+    def self.exit_on_failure?
+      true
+    end
+  end
+end

lib/config.rb → lib/gingr/config.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-
 require 'berkeley_library/logging'
 
 # Gingr

lib/data_handler.rb → lib/gingr/data_handler.rb

@@ -1,13 +1,10 @@
 # frozen_string_literal: true
-
-require 'zip'
+require 'fileutils'
 require 'pathname'
+require 'zip'
 require_relative 'config'
-require 'fileutils'
 
-# Ginger module
 module Gingr
-  # handle ingestion data
   module DataHandler
     include Gingr::Config
 

lib/geoserver_publisher.rb → lib/gingr/geoserver_publisher.rb

@@ -1,14 +1,12 @@
 # frozen_string_literal: true
-
-require_relative 'publish'
+require_relative '../monkeypatch/geoserver/publish'
 require 'uri'
 require_relative 'config'
 
-# Ginger module
 module Gingr
-  include Gingr::Config
-  # publish services to geoserver
   class GeoserverPublisher
+    include Gingr::Config
+
     def initialize(url)
       uri = URI(url)
       @conn = Geoserver::Publish::Connection.new({ 'url' => rest_url(uri), 'user' => uri.user,

lib/import_util.rb → lib/gingr/import_util.rb

@@ -1,15 +1,13 @@
 # frozen_string_literal: true
-
+require 'uri'
 require_relative 'config'
-require_relative 'solr_indexer'
 require_relative 'geoserver_publisher'
-
-require 'uri'
+require_relative 'solr_indexer'
 
 module Gingr
-  # util methods for bin/import
   module ImportUtil
     include Gingr::Config
+
     class << self
       def publish_geoservers(geofile_names, options)
         publish_geoserver_files(geofile_names[:public], options[:geoserver_url], true)

lib/solr_indexer.rb → lib/gingr/solr_indexer.rb

@@ -1,14 +1,12 @@
 # frozen_string_literal: true
-
-require 'rsolr'
 require 'faraday/net_http_persistent'
+require 'rsolr'
 require_relative 'config'
 
-# Ginger module
 module Gingr
-  include Gingr::Config
-  # index solr for Gingr
   class SolrIndexer
+    include Gingr::Config
+
     attr_reader :solr, :need_update_reference_urls
 
     def initialize(url, reference_urls = {})

lib/publish.rb → lib/monkeypatch/geoserver/publish.rb

@@ -1,26 +1,10 @@
 # frozen_string_literal: true
+require 'geoserver/publish'
 
-require 'erb'
-require 'faraday'
-require 'json'
-require 'yaml'
-
+# Monkey-patch geoserver-publish gem to prefix everything with berkeley_
+# @note Is this really necessary?
 module Geoserver
-  # from geoserver-publish gem: get a specific store name
   module Publish
-    require 'geoserver/publish/config'
-    require 'geoserver/publish/connection'
-    require 'geoserver/publish/coverage'
-    require 'geoserver/publish/coverage_store'
-    require 'geoserver/publish/create'
-    require 'geoserver/publish/data_store'
-    require 'geoserver/publish/feature_type'
-    require 'geoserver/publish/geowebcache'
-    require 'geoserver/publish/layer'
-    require 'geoserver/publish/style'
-    require 'geoserver/publish/version'
-    require 'geoserver/publish/workspace'
-
     def self.delete_geotiff(workspace_name:, id:, connection: nil)
       coverage_store_name = "berkeley_#{id}"
       CoverageStore.new(connection).delete(workspace_name:, coverage_store_name:)
@@ -47,12 +31,5 @@ def self.shapefile(workspace_name:, file_path:, id:, title: nil, connection: nil
       create_feature_type(workspace_name:, data_store_name:, feature_type_name: id, title:,
                           connection:)
     end
-
-    def self.root
-      Pathname.new(File.expand_path('../..', __dir__))
-    end
-
-    class Error < StandardError
-    end
   end
 end