Skip to content

Commit

Permalink
Ingestion reporting
Browse files Browse the repository at this point in the history
  • Loading branch information
yzhoubk committed Oct 31, 2024
1 parent 2ef64cf commit 1484468
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 3 deletions.
20 changes: 17 additions & 3 deletions lib/gingr/cli.rb
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,10 @@ def all(zipfile)
geoserver_urls = options.slice(:geoserver_url, :geoserver_secure_url).transform_keys(&:to_sym)
failed_files = Gingr::GeoserverPublisher.publish_inventory(geofile_names, **geoserver_urls)

logger.info("Total ingested records: #{total_indexed}")
logger.error("#{failed_files.join(';')} failed published to geoservers.") unless failed_files.empty?
logger.info("#{zipfile} - all imported")
report(total_indexed, failed_files, zipfile)
# logger.info("Total ingested records: #{total_indexed}")
# logger.error("#{failed_files.join(';')} failed published to geoservers.") unless failed_files.empty?
# logger.info("#{zipfile} - all imported")
end

desc 'geoserver_workspace', 'create a workspace in a geoserver'
Expand All @@ -134,5 +135,18 @@ def set_data_handler(spatial_root, goserver_root)
gingr_watch_root_dir ||= ENV['GINGR_WATCH_DIRECTORY'] || '/opt/app/data/gingr'
DataHandler.processing_root = File.join(gingr_watch_root_dir, 'processing')
end

def report(total_indexed, failed_files, zipfile)
if total_indexed.nil?
logger.error('Solr indexing failed')
logger.info("#{zipfile} - not imported")
return
end
logger.info("#{zipfile} - all imported, total records: #{total_indexed}")
return if failed_files.empty?

logger.warn("#{zipfile} - some shapefile or GeoTIFF files not published to Geoservers")
logger.error("Failed to published geo files: #{failed_files.join('; ')}")
end
end
end
3 changes: 3 additions & 0 deletions lib/gingr/solr_indexer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ def index_directory(directory)
.size
@solr.commit
total_indexed
rescue StandardError => e
logger.error "Indexing directory '#{directory}' failed: #{e.message}"
nil
end

def update_reference_urls!(doc)
Expand Down

0 comments on commit 1484468

Please sign in to comment.