From 6c51346b6f150a69391794b7909b30592acbbe0e Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Sat, 2 Mar 2024 22:03:25 +0100 Subject: [PATCH] Feature: Enhance SOLR integration and add a Schema API (#54) * add an abstraction to SOLR integeration and add Schema API * add SOLR Schema API tests * update SOLR backend configuration and init * use the new Solr connector in the model search interface * update search test to cover the new automatic indexing and unindexing * handle the solr container initialization when running docker for tests * add omit_norms options for SolrSchemaGenerator * fix solr schema initial dynamic fields declaration and replace the usage of mapping-ISOLatin1Accent * delay the schema generation to after model declarations or in demand * add solr edismax fitlers tests * fix indexBatch and unindexBatch tests * add security checks to the index and unindex functions * change dynamic fields names to have less code migration * update clear_all_schema to remove all copy and normal fields * add an option to force solr initialization if wanted * handle indexing embed objects of a model * add index update option * fix clear all schema to just remove all the fields and recreate them * add index_enabled? helper for models * perform a status test when initializing the solr connector * extract init_search_connection function from init_search_connections * fix typo in indexOptimize call * add solr search using HTTP post instead of GET for large queries --- .ruby-version | 1 + docker-compose.yml | 9 +- lib/goo.rb | 49 +++- lib/goo/base/settings/attribute.rb | 16 +- lib/goo/config/config.rb | 2 +- lib/goo/search/search.rb | 194 +++++++++---- lib/goo/search/solr/solr_admin.rb | 79 ++++++ lib/goo/search/solr/solr_connector.rb | 41 +++ lib/goo/search/solr/solr_query.rb | 94 +++++++ lib/goo/search/solr/solr_schema.rb | 184 ++++++++++++ lib/goo/search/solr/solr_schema_generator.rb | 279 +++++++++++++++++++ rakelib/docker_based_test.rake | 14 + test/solr/test_solr.rb | 122 ++++++++ test/test_search.rb | 249 ++++++++++++++++- 14 files changed, 1252 insertions(+), 81 deletions(-) create mode 100644 .ruby-version create mode 100644 lib/goo/search/solr/solr_admin.rb create mode 100644 lib/goo/search/solr/solr_connector.rb create mode 100644 lib/goo/search/solr/solr_query.rb create mode 100644 lib/goo/search/solr/solr_schema.rb create mode 100644 lib/goo/search/solr/solr_schema_generator.rb create mode 100644 test/solr/test_solr.rb diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 00000000..6a81b4c8 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +2.7.8 diff --git a/docker-compose.yml b/docker-compose.yml index 6bd6cd56..463a1b92 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,15 +10,10 @@ services: retries: 30 solr-ut: - image: ontoportal/solr-ut:0.0.2 + image: solr:8.11.2 ports: - 8983:8983 - healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] - start_period: 10s - interval: 10s - timeout: 5s - retries: 5 + command: bin/solr start -cloud -f agraph-ut: image: franzinc/agraph:v8.1.0 diff --git a/lib/goo.rb b/lib/goo.rb index 283ab5b4..adf73d3a 100644 --- a/lib/goo.rb +++ b/lib/goo.rb @@ -42,6 +42,7 @@ module Goo @@model_by_name = {} @@search_backends = {} @@search_connection = {} + @@search_collections = {} @@default_namespace = nil @@id_prefix = nil @@redis_client = nil @@ -101,7 +102,7 @@ def self.language_includes(lang) end def self.add_namespace(shortcut, namespace,default=false) - if !(namespace.instance_of? RDF::Vocabulary) + unless namespace.instance_of? RDF::Vocabulary raise ArgumentError, "Namespace must be a RDF::Vocabulary object" end @@namespaces[shortcut.to_sym] = namespace @@ -252,11 +253,9 @@ def self.configure raise ArgumentError, "Configuration needs to receive a code block" end yield self - configure_sanity_check() + configure_sanity_check - if @@search_backends.length > 0 - @@search_backends.each { |name, val| @@search_connection[name] = RSolr.connect(url: search_conf(name), timeout: 1800, open_timeout: 1800) } - end + init_search_connections @@namespaces.freeze @@sparql_backends.freeze @@ -280,8 +279,44 @@ def self.search_conf(name=:main) return @@search_backends[name][:service] end - def self.search_connection(name=:main) - return @@search_connection[name] + def self.search_connection(collection_name) + return search_client(collection_name).solr + end + + def self.search_client(collection_name) + @@search_connection[collection_name] + end + + def self.add_search_connection(collection_name, search_backend = :main, &block) + @@search_collections[collection_name] = { + search_backend: search_backend, + block: block_given? ? block : nil + } + end + + def self.search_connections + @@search_connection + end + + def self.init_search_connection(collection_name, search_backend = :main, block = nil, force: false) + return @@search_connection[collection_name] if @@search_connection[collection_name] && !force + + @@search_connection[collection_name] = SOLR::SolrConnector.new(search_conf(search_backend), collection_name) + if block + block.call(@@search_connection[collection_name].schema_generator) + @@search_connection[collection_name].enable_custom_schema + end + @@search_connection[collection_name].init(force) + @@search_connection[collection_name] + end + + + def self.init_search_connections(force=false) + @@search_collections.each do |collection_name, backend| + search_backend = backend[:search_backend] + block = backend[:block] + init_search_connection(collection_name, search_backend, block, force: force) + end end def self.sparql_query_client(name=:main) diff --git a/lib/goo/base/settings/attribute.rb b/lib/goo/base/settings/attribute.rb index dda5fdd5..dbf52b78 100644 --- a/lib/goo/base/settings/attribute.rb +++ b/lib/goo/base/settings/attribute.rb @@ -158,11 +158,6 @@ def list?(attr) attribute_settings(attr)[:enforce].include?(:list) end - def index_attribute?(attr) - return false if attribute_settings(attr).nil? - attribute_settings(attr)[:index] - end - def transitive?(attr) return false unless @model_settings[:attributes].include?(attr) attribute_settings(attr)[:transitive] == true @@ -212,6 +207,17 @@ def attribute_uri(attr, *args) Goo.vocabulary(nil)[attr] end + + def indexable?(attr) + setting = attribute_settings(attr.to_sym) + setting && (setting[:index].nil? || setting[:index] == true) + end + + def fuzzy_searchable?(attr) + attribute_settings(attr)[:fuzzy_search] == true + end + + private def set_no_list_by_default(options) diff --git a/lib/goo/config/config.rb b/lib/goo/config/config.rb index ff51e8b7..4c51a223 100644 --- a/lib/goo/config/config.rb +++ b/lib/goo/config/config.rb @@ -20,7 +20,7 @@ def config(&block) @settings.goo_path_query ||= ENV['GOO_PATH_QUERY'] || '/sparql/' @settings.goo_path_data ||= ENV['GOO_PATH_DATA'] || '/data/' @settings.goo_path_update ||= ENV['GOO_PATH_UPDATE'] || '/update/' - @settings.search_server_url ||= ENV['SEARCH_SERVER_URL'] || 'http://localhost:8983/solr/term_search_core1' + @settings.search_server_url ||= ENV['SEARCH_SERVER_URL'] || 'http://localhost:8983/solr' @settings.redis_host ||= ENV['REDIS_HOST'] || 'localhost' @settings.redis_port ||= ENV['REDIS_PORT'] || 6379 @settings.bioportal_namespace ||= ENV['BIOPORTAL_NAMESPACE'] || 'http://data.bioontology.org/' diff --git a/lib/goo/search/search.rb b/lib/goo/search/search.rb index 1dc72ea9..dceb856f 100644 --- a/lib/goo/search/search.rb +++ b/lib/goo/search/search.rb @@ -1,4 +1,5 @@ require 'rsolr' +require_relative 'solr/solr_connector' module Goo @@ -8,102 +9,183 @@ def self.included(base) base.extend(ClassMethods) end - def index(connection_name=:main) + def index(connection_name = nil, to_set = nil) raise ArgumentError, "ID must be set to be able to index" if @id.nil? - doc = indexable_object - Goo.search_connection(connection_name).add(doc) + document = indexable_object(to_set) + + return if document.blank? || document[:id].blank? + + connection_name ||= self.class.search_collection_name + unindex(connection_name) + self.class.search_client(connection_name).index_document(document) end - def index_update(to_set, connection_name=:main) + def index_update(attributes_to_update, connection_name = nil, to_set = nil) raise ArgumentError, "ID must be set to be able to index" if @id.nil? - raise ArgumentError, "Field names to be updated in index must be provided" if to_set.nil? + raise ArgumentError, "Field names to be updated in index must be provided" if attributes_to_update.blank? + + old_doc = self.class.search("id:\"#{index_id}\"").dig("response", "docs")&.first + + raise ArgumentError, "ID must be set to be able to index" if old_doc.blank? + doc = indexable_object(to_set) - doc.each { |key, val| - next if key === :id - doc[key] = {set: val} - } + doc.each do |key, val| + next unless attributes_to_update.any? { |attr| key.to_s.eql?(attr.to_s) || key.to_s.include?("#{attr}_") } + old_doc[key] = val + end + + connection_name ||= self.class.search_collection_name + unindex(connection_name) - Goo.search_connection(connection_name).update( - data: "[#{doc.to_json}]", - headers: { 'Content-Type' => 'application/json' } - ) + old_doc.reject! { |k, v| k.to_s.end_with?('_sort') || k.to_s.end_with?('_sorts') } + old_doc.delete("_version_") + self.class.search_client(connection_name).index_document(old_doc) end - def unindex(connection_name=:main) - id = index_id - Goo.search_connection(connection_name).delete_by_id(id) + def unindex(connection_name = nil) + connection_name ||= self.class.search_collection_name + self.class.search_client(connection_name).delete_by_id(index_id) end # default implementation, should be overridden by child class - def index_id() + def index_id raise ArgumentError, "ID must be set to be able to index" if @id.nil? @id.to_s end # default implementation, should be overridden by child class - def index_doc(to_set=nil) + def index_doc(to_set = nil) raise NoMethodError, "You must define method index_doc in your class for it to be indexable" end - def indexable_object(to_set=nil) - doc = index_doc(to_set) - # use resource_id for the actual term id because :id is a Solr reserved field - doc[:resource_id] = doc[:id].to_s - doc[:id] = index_id.to_s - doc + def embedded_doc + raise NoMethodError, "You must define method embedded_doc in your class for it to be indexable" end + def indexable_object(to_set = nil) + begin + document = index_doc(to_set) + rescue NoMethodError + document = self.to_hash.reject { |k, _| !self.class.indexable?(k) } + document.transform_values! do |v| + is_array = v.is_a?(Array) + v = Array(v).map do |x| + if x.is_a?(Goo::Base::Resource) + x.embedded_doc rescue x.id.to_s + else + if x.is_a?(RDF::URI) + x.to_s + else + x.respond_to?(:object) ? x.object : x + end + end + end + is_array ? v : v.first + end + end + + document = document.reduce({}) do |h, (k, v)| + if v.is_a?(Hash) + v.each { |k2, v2| h["#{k}_#{k2}".to_sym] = v2 } + else + h[k] = v + end + h + end + + model_name = self.class.model_name.to_s.downcase + document.delete(:id) + document.delete("id") + + document.transform_keys! do |k| + self.class.index_document_attr(k) + end + + document[:resource_id] = self.id.to_s + document[:resource_model] = model_name + document[:id] = index_id.to_s + document + end module ClassMethods - def search(q, params={}, connection_name=:main) - params["q"] = q - Goo.search_connection(connection_name).post('select', :data => params) + def index_enabled? + !@model_settings[:search_collection].nil? end - def indexBatch(collection, connection_name=:main) - docs = Array.new - collection.each do |c| - docs << c.indexable_object + def enable_indexing(collection_name, search_backend = :main, &block) + @model_settings[:search_collection] = collection_name + + if block_given? + # optional block to generate custom schema + Goo.add_search_connection(collection_name, search_backend, &block) + else + Goo.add_search_connection(collection_name, search_backend) end - Goo.search_connection(connection_name).add(docs) + + after_save :index + after_destroy :unindex end - def unindexBatch(collection, connection_name=:main) - docs = Array.new - collection.each do |c| - docs << c.index_id - end - Goo.search_connection(connection_name).delete_by_id(docs) + def search_collection_name + @model_settings[:search_collection] + end + + def search_client(connection_name = search_collection_name) + Goo.search_client(connection_name) + end + + def custom_schema?(connection_name = search_collection_name) + search_client(connection_name).custom_schema? + end + + def schema_generator + Goo.search_client(search_collection_name).schema_generator + end + + def index_document_attr(key) + return key.to_s if custom_schema? || self.attribute_settings(key).nil? + + type = self.datatype(key) + is_list = self.list?(key) + fuzzy = self.fuzzy_searchable?(key) + search_client.index_document_attr(key, type, is_list, fuzzy) + end + + def search(q, params = {}, connection_name = search_collection_name) + search_client(connection_name).search(q, params) + end + + def submit_search_query(query, params = {}, connection_name = search_collection_name) + search_client(connection_name).submit_search_query(query, params) + end + + def indexBatch(collection, connection_name = search_collection_name) + docs = collection.map(&:indexable_object) + search_client(connection_name).index_document(docs) end - def unindexByQuery(query, connection_name=:main) - Goo.search_connection(connection_name).delete_by_query(query) + def unindexBatch(collection, connection_name = search_collection_name) + docs = collection.map(&:index_id) + search_client(connection_name).delete_by_id(docs) end - # Get the doc that will be indexed in solr - def get_indexable_object() - # To make the code less readable the guys that wrote it managed to hide the real function called by this line - # It is "get_index_doc" in ontologies_linked_data Class.rb - doc = self.class.model_settings[:search_options][:document].call(self) - doc[:resource_id] = doc[:id].to_s - doc[:id] = get_index_id.to_s - # id: clsUri_ONTO-ACRO_submissionNumber. i.e.: http://lod.nal.usda.gov/nalt/5260_NALT_4 - doc + def unindexByQuery(query, connection_name = search_collection_name) + search_client(connection_name).delete_by_query(query) end - def indexCommit(attrs=nil, connection_name=:main) - Goo.search_connection(connection_name).commit(:commit_attributes => attrs || {}) + def indexCommit(attrs = nil, connection_name = search_collection_name) + search_client(connection_name).index_commit(attrs) end - def indexOptimize(attrs=nil, connection_name=:main) - Goo.search_connection(connection_name).optimize(:optimize_attributes => attrs || {}) + def indexOptimize(attrs = nil, connection_name = search_collection_name) + search_client(connection_name).index_optimize(attrs) end - def indexClear(connection_name=:main) - # WARNING: this deletes ALL data from the index - unindexByQuery("*:*", connection_name) + # WARNING: this deletes ALL data from the index + def indexClear(connection_name = search_collection_name) + search_client(connection_name).clear_all_data end end end diff --git a/lib/goo/search/solr/solr_admin.rb b/lib/goo/search/solr/solr_admin.rb new file mode 100644 index 00000000..4d20271b --- /dev/null +++ b/lib/goo/search/solr/solr_admin.rb @@ -0,0 +1,79 @@ +module SOLR + module Administration + + def admin_url + "#{@solr_url}/admin" + end + + def solr_alive? + collections_url = URI.parse("#{admin_url}/collections?action=CLUSTERSTATUS") + http = Net::HTTP.new(collections_url.host, collections_url.port) + request = Net::HTTP::Get.new(collections_url.request_uri) + + begin + response = http.request(request) + return response.code.eql?("200") && JSON.parse(response.body).dig("responseHeader", "status").eql?(0) + rescue StandardError => e + return false + end + end + + def fetch_all_collections + collections_url = URI.parse("#{admin_url}/collections?action=LIST") + + http = Net::HTTP.new(collections_url.host, collections_url.port) + request = Net::HTTP::Get.new(collections_url.request_uri) + + begin + response = http.request(request) + raise StandardError, "Failed to fetch collections. HTTP #{response.code}: #{response.message}" unless response.code.to_i == 200 + rescue StandardError => e + raise StandardError, "Failed to fetch collections. #{e.message}" + end + + collections = [] + if response.is_a?(Net::HTTPSuccess) + collections = JSON.parse(response.body)['collections'] + end + + collections + end + + def create_collection(name = @collection_name, num_shards = 1, replication_factor = 1) + return if collection_exists?(name) + create_collection_url = URI.parse("#{admin_url}/collections?action=CREATE&name=#{name}&numShards=#{num_shards}&replicationFactor=#{replication_factor}") + + http = Net::HTTP.new(create_collection_url.host, create_collection_url.port) + request = Net::HTTP::Post.new(create_collection_url.request_uri) + + begin + response = http.request(request) + raise StandardError, "Failed to create collection. HTTP #{response.code}: #{response.message}" unless response.code.to_i == 200 + rescue StandardError => e + raise StandardError, "Failed to create collection. #{e.message}" + end + end + + def delete_collection(collection_name = @collection_name) + return unless collection_exists?(collection_name) + + delete_collection_url = URI.parse("#{admin_url}/collections?action=DELETE&name=#{collection_name}") + + http = Net::HTTP.new(delete_collection_url.host, delete_collection_url.port) + request = Net::HTTP::Post.new(delete_collection_url.request_uri) + + begin + response = http.request(request) + raise StandardError, "Failed to delete collection. HTTP #{response.code}: #{response.message}" unless response.code.to_i == 200 + rescue StandardError => e + raise StandardError, "Failed to delete collection. #{e.message}" + end + + end + + def collection_exists?(collection_name) + fetch_all_collections.include?(collection_name.to_s) + end + end +end + diff --git a/lib/goo/search/solr/solr_connector.rb b/lib/goo/search/solr/solr_connector.rb new file mode 100644 index 00000000..e367f5cd --- /dev/null +++ b/lib/goo/search/solr/solr_connector.rb @@ -0,0 +1,41 @@ +require 'rsolr' +require_relative 'solr_schema_generator' +require_relative 'solr_schema' +require_relative 'solr_admin' +require_relative 'solr_query' + +module SOLR + + class SolrConnector + include Schema, Administration, Query + attr_reader :solr + + def initialize(solr_url, collection_name) + @solr_url = solr_url + @collection_name = collection_name + @solr = RSolr.connect(url: collection_url) + + # Perform a status test and wait up to 30 seconds before raising an error + wait_time = 0 + max_wait_time = 30 + until solr_alive? || wait_time >= max_wait_time + sleep 1 + wait_time += 1 + end + raise "Solr instance not reachable within #{max_wait_time} seconds" unless solr_alive? + + + @custom_schema = false + end + + def init(force = false) + return if collection_exists?(@collection_name) && !force + + create_collection + + init_schema + end + + end +end + diff --git a/lib/goo/search/solr/solr_query.rb b/lib/goo/search/solr/solr_query.rb new file mode 100644 index 00000000..401feb37 --- /dev/null +++ b/lib/goo/search/solr/solr_query.rb @@ -0,0 +1,94 @@ +module SOLR + module Query + + def clear_all_data + delete_by_query('*:*') + end + + def collection_url + "#{@solr_url}/#{@collection_name}" + end + + def index_commit(attrs = nil) + @solr.commit(:commit_attributes => attrs || {}) + end + + def index_optimize(attrs = nil) + @solr.optimize(:optimize_attributes => attrs || {}) + end + + def index_document(document, commit: true) + @solr.add(document) + @solr.commit if commit + end + + def index_document_attr(key, type, is_list, fuzzy_search) + dynamic_field(type: type, is_list: is_list, is_fuzzy_search: fuzzy_search).gsub('*', key.to_s) + end + + def delete_by_id(document_id, commit: true) + return if document_id.nil? + + @solr.delete_by_id(document_id) + @solr.commit if commit + end + + def delete_by_query(query) + @solr.delete_by_query(query) + @solr.commit + end + + def search(query, params = {}) + params[:q] = query + @solr.get('select', params: params) + end + + def submit_search_query(query, params = {}) + uri = ::URI.parse("#{collection_url}/select") + + http = Net::HTTP.new(uri.host, uri.port) + request = Net::HTTP::Post.new(uri.request_uri) + + params[:q] = query + request.set_form_data(params) + + response = http.request(request) + + if response.is_a?(Net::HTTPSuccess) + JSON.parse(response.body) + else + puts "Error: #{response.code} - #{response.message}" + nil + end + end + + private + + def dynamic_field(type:, is_list:, is_fuzzy_search: false) + return is_list ? '*_texts' : '*_text' if is_fuzzy_search + + dynamic_type = case type + when :uri, :string, nil + '*_t' + when :integer + '*_i' + when :boolean + '*_b' + when :date_time + '*_dt' + when :float + '*_f' + else + # Handle unknown data types or raise an error based on your specific requirements + raise ArgumentError, "Unsupported ORM data type: #{type}" + end + + if is_list + dynamic_type = dynamic_type.eql?('*_t') ? "*_txt" : "#{dynamic_type}s" + end + + dynamic_type + end + end +end + diff --git a/lib/goo/search/solr/solr_schema.rb b/lib/goo/search/solr/solr_schema.rb new file mode 100644 index 00000000..8c38fd2f --- /dev/null +++ b/lib/goo/search/solr/solr_schema.rb @@ -0,0 +1,184 @@ +module SOLR + module Schema + + def fetch_schema + uri = URI.parse("#{@solr_url}/#{@collection_name}/schema") + http = Net::HTTP.new(uri.host, uri.port) + + request = Net::HTTP::Get.new(uri.path, 'Content-Type' => 'application/json') + response = http.request(request) + + if response.code.to_i == 200 + @schema = JSON.parse(response.body)["schema"] + else + raise StandardError, "Failed to upload schema. HTTP #{response.code}: #{response.body}" + end + end + + def schema + @schema ||= fetch_schema + end + + def all_fields + schema["fields"] + end + + def all_copy_fields + schema["copyFields"] + end + + def all_dynamic_fields + schema["dynamicFields"] + end + + def all_fields_types + schema["fieldTypes"] + end + + def fetch_all_fields + fetch_schema["fields"] + end + + def fetch_all_copy_fields + fetch_schema["copyFields"] + end + + def fetch_all_dynamic_fields + fetch_schema["dynamicFields"] + end + + def fetch_all_fields_types + fetch_schema["fieldTypes"] + end + + def schema_generator + @schema_generator ||= SolrSchemaGenerator.new + end + + def init_collection(num_shards = 1, replication_factor = 1) + create_collection_url = URI.parse("#{@solr_url}/admin/collections?action=CREATE&name=#{@collection_name}&numShards=#{num_shards}&replicationFactor=#{replication_factor}") + + http = Net::HTTP.new(create_collection_url.host, create_collection_url.port) + request = Net::HTTP::Post.new(create_collection_url.request_uri) + + begin + response = http.request(request) + raise StandardError, "Failed to create collection. HTTP #{response.code}: #{response.message}" unless response.code.to_i == 200 + rescue StandardError => e + raise StandardError, "Failed to create collection. #{e.message}" + end + end + + def init_schema(generator = schema_generator) + clear_all_schema(generator) + fetch_schema + default_fields = all_fields.map { |f| f['name'] } + + solr_schema = { + "add-field-type": generator.field_types_to_add, + 'add-field' => generator.fields_to_add.reject { |f| default_fields.include?(f[:name]) }, + 'add-dynamic-field' => generator.dynamic_fields_to_add, + 'add-copy-field' => generator.copy_fields_to_add + } + + update_schema(solr_schema) + end + + def custom_schema? + @custom_schema + end + + def enable_custom_schema + @custom_schema = true + end + + def clear_all_schema(generator = schema_generator) + init_ft = generator.field_types_to_add.map { |f| f[:name] } + dynamic_fields = all_dynamic_fields.map { |f| { name: f['name'] } } + copy_fields = all_copy_fields.map { |f| { source: f['source'], dest: f['dest'] } } + fields_types = all_fields_types.select { |f| init_ft.include?(f['name']) }.map { |f| { name: f['name']} } + fields = all_fields.reject { |f| %w[id _version_ ].include?(f['name']) }.map { |f| { name: f['name'] } } + + upload_schema('delete-copy-field' => copy_fields) unless copy_fields.empty? + upload_schema('delete-dynamic-field' => dynamic_fields) unless dynamic_fields.empty? + upload_schema('delete-field' => fields) unless copy_fields.empty? + upload_schema('delete-field-type' => fields_types) unless fields_types.empty? + end + + def map_to_indexer_type(orm_data_type) + case orm_data_type + when :uri + 'string' # Assuming a string field for URIs + when :string, nil # Default to 'string' if no type is given + 'text_general' # Assuming a generic text field for strings + when :integer + 'pint' + when :boolean + 'boolean' + when :date_time + 'pdate' + when :float + 'pfloat' + else + # Handle unknown data types or raise an error based on your specific requirements + raise ArgumentError, "Unsupported ORM data type: #{orm_data_type}" + end + end + + def delete_field(name) + update_schema('delete-field' => [ + { name: name } + ]) + end + + def add_field(name, type, indexed: true, stored: true, multi_valued: false) + update_schema('add-field' => [ + { name: name, type: type, indexed: indexed, stored: stored, multiValued: multi_valued } + ]) + end + + def add_dynamic_field(name, type, indexed: true, stored: true, multi_valued: false) + update_schema('add-dynamic-field' => [ + { name: name, type: type, indexed: indexed, stored: stored, multiValued: multi_valued } + ]) + end + + def add_copy_field(source, dest) + update_schema('add-copy-field' => [ + { source: source, dest: dest } + ]) + end + + def fetch_field(name) + fetch_all_fields.select { |f| f['name'] == name }.first + end + + def update_schema(schema_json) + permitted_actions = %w[add-field add-copy-field add-dynamic-field add-field-type delete-copy-field delete-dynamic-field delete-field delete-field-type] + + unless permitted_actions.any? { |action| schema_json.key?(action) } + raise StandardError, "The schema need to implement at least one of this actions: #{permitted_actions.join(', ')}" + end + upload_schema(schema_json) + fetch_schema + end + + private + + def upload_schema(schema_json) + uri = URI.parse("#{@solr_url}/#{@collection_name}/schema") + http = Net::HTTP.new(uri.host, uri.port) + + request = Net::HTTP::Post.new(uri.path, 'Content-Type' => 'application/json') + request.body = schema_json.to_json + response = http.request(request) + if response.code.to_i == 200 + response + else + raise StandardError, "Failed to upload schema. HTTP #{response.code}: #{response.body}" + end + end + + end +end + diff --git a/lib/goo/search/solr/solr_schema_generator.rb b/lib/goo/search/solr/solr_schema_generator.rb new file mode 100644 index 00000000..bc1e4693 --- /dev/null +++ b/lib/goo/search/solr/solr_schema_generator.rb @@ -0,0 +1,279 @@ +module SOLR + + class SolrSchemaGenerator + + attr_reader :schema + + def initialize + @schema = {} + end + + def add_field(name, type, indexed: true, stored: true, multi_valued: false, omit_norms: nil) + @schema['add-field'] ||= [] + af = { name: name.to_s, type: type, indexed: indexed, stored: stored, multiValued: multi_valued} + af[:omitNorms] = omit_norms unless omit_norms.nil? + @schema['add-field'] << af + end + + def add_dynamic_field(name, type, indexed: true, stored: true, multi_valued: false, omit_norms: nil) + @schema['add-dynamic-field'] ||= [] + df = { name: name.to_s, type: type, indexed: indexed, stored: stored, multiValued: multi_valued } + df[:omitNorms] = omit_norms unless omit_norms.nil? + @schema['add-dynamic-field'] << df + end + + def add_copy_field(source, dest) + @schema['add-copy-field'] ||= [] + @schema['add-copy-field'] << { source: source, dest: dest } + end + + def add_field_type(type_definition) + @schema['add-field-type'] ||= [] + @schema['add-field-type'] << type_definition + end + + def fields_to_add + custom_fields = @schema['add-field'] || [] + custom_fields + init_fields + end + + def dynamic_fields_to_add + custom_fields = @schema['add-dynamic-field'] || [] + custom_fields + init_dynamic_fields + end + + def copy_fields_to_add + custom_fields = @schema['add-copy-field'] || [] + custom_fields + init_copy_fields + end + + def field_types_to_add + custom_fields = @schema['add-field-type'] || [] + custom_fields + init_fields_types + end + + def init_fields_types + [ + { + "name": "string_ci", + "class": "solr.TextField", + "sortMissingLast": true, + "omitNorms": true, + "queryAnalyzer": + { + "tokenizer": { + "class": "solr.KeywordTokenizerFactory" + }, + "filters": [ + { + "class": "solr.LowerCaseFilterFactory" + } + ] + } + }, + { + "name": "text_suggest_ngram", + "class": "solr.TextField", + "positionIncrementGap": "100", + "analyzer": { + "tokenizer": { + "class": "solr.StandardTokenizerFactory" + }, + "filters": [ + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.EdgeNGramTokenizerFactory", + "minGramSize": 1, + "maxGramSize": 25 + } + ] + } + }, + { + "name": "text_suggest_edge", + "class": "solr.TextField", + "positionIncrementGap": "100", + "indexAnalyzer": { + "tokenizer": { + "class": "solr.KeywordTokenizerFactory" + }, + "char_filters": [ + { + "class": "solr.MappingCharFilterFactory", + "mapping": "solr/resources/org/apache/lucene/analysis/miscellaneous/MappingCharFilter.greekaccent" + } + ], + "filters": [ + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([\\.,;:-_])", + "replacement": " ", + "replace": "all" + }, + { + "class": "solr.EdgeNGramFilterFactory", + "minGramSize": 1, + "maxGramSize": 30, + "preserveOriginal": true + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\\w\\d\\*æøåÆØÅ ])", + "replacement": "", + "replace": "all" + } + ] + }, + "queryAnalyzer": { + "tokenizer": { + "class": "solr.KeywordTokenizerFactory" + }, + "char_filters": [ + { + "class": "solr.MappingCharFilterFactory", + "mapping": "solr/resources/org/apache/lucene/analysis/miscellaneous/MappingCharFilter.greekaccent" + } + ], + "filters": [ + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([\\.,;:-_])", + "replacement": " ", + "replace": "all" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\\w\\d\\*æøåÆØÅ ])", + "replacement": "", + "replace": "all" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "^(.{30})(.*)?", + "replacement": "$1", + "replace": "all" + } + ] + } + }, + { + "name": "text_suggest", + "class": "solr.TextField", + "positionIncrementGap": 100, + indexAnalyzer: { + "char_filters": [ + { + "class": "solr.MappingCharFilterFactory", + "mapping": "solr/resources/org/apache/lucene/analysis/miscellaneous/MappingCharFilter.greekaccent" + } + ], + "tokenizer": { + "class": "solr.StandardTokenizerFactory" + }, + "filters": [ + { + "class": "solr.WordDelimiterGraphFilterFactory", + "generateWordParts": "1", + "generateNumberParts": "1", + "catenateWords": "1", + "catenateNumbers": "1", + "catenateAll": "1", + "splitOnCaseChange": "1", + "splitOnNumerics": "1", + "preserveOriginal": "1" + }, + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\w\d*æøåÆØÅ ])", + "replacement": " ", + "replace": "all" + } + ] + }, + queryAnalyzer: { + "char_filters": [ + { + "class": "solr.MappingCharFilterFactory", + "mapping": "solr/resources/org/apache/lucene/analysis/miscellaneous/MappingCharFilter.greekaccent" + } + ], + "tokenizer": { + "class": "solr.StandardTokenizerFactory" + }, + "filters": [ + { + "class": "solr.WordDelimiterGraphFilterFactory", + "generateWordParts": "0", + "generateNumberParts": "0", + "catenateWords": "0", + "catenateNumbers": "0", + "catenateAll": "0", + "splitOnCaseChange": "0", + "splitOnNumerics": "0" + }, + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\w\d*æøåÆØÅ ])", + "replacement": " ", + "replace": "all" + } + ] + } + } + ] + end + + def init_fields + [ + #{ name: "_version_", type: "plong", indexed: true, stored: true, multiValued: false }, + { name: "resource_id", type: "string", indexed: true, multiValued: false, required: true, stored: true }, + { name: "resource_model", type: "string", indexed: true, multiValued: false, required: true, stored: false }, + { name: "_text_", type: "text_general", indexed: true, multiValued: true, stored: false }, + ] + end + + def init_dynamic_fields + [ + {"name": "*_t", "type": "text_general", stored: true, "multiValued": false }, + {"name": "*_txt", "type": "text_general", stored: true, "multiValued": true}, + {"name": "*_i", "type": "pint", stored: true }, + {"name": "*_is", "type": "pints", stored: true }, + {"name": "*_f", "type": "pfloat", stored: true }, + {"name": "*_fs", "type": "pfloats", stored: true }, + {"name": "*_b", "type": "boolean", stored: true }, + {"name": "*_bs", "type": "booleans", stored: true }, + {"name": "*_dt", "type": "pdate", stored: true }, + {"name": "*_dts", "type": "pdate", stored: true , multiValued: true}, + { "name": "*Exact", "type": "string_ci", "multiValued": true, stored: false }, + { "name": "*Suggest", "type": "text_suggest", "omitNorms": true, stored: false, "multiValued": true }, + { "name": "*SuggestEdge", "type": "text_suggest_edge", stored: false, "multiValued": true }, + { "name": "*SuggestNgram", "type": "text_suggest_ngram", stored: false, "omitNorms": true, "multiValued": true }, + { "name": "*_text", "type": "text_general", stored: true, "multiValued": false }, + { "name": "*_texts", "type": "text_general", stored: true, "multiValued": true }, + {"name": "*_sort", "type": "string", stored: false }, + {"name": "*_sorts", "type": "strings", stored: false , "multiValued": true}, + ] + end + + def init_copy_fields + [ + { source: "*_text", dest: %w[_text_ *Exact *Suggest *SuggestEdge *SuggestNgram *_sort] }, + { source: "*_texts", dest: %w[_text_ *Exact *Suggest *SuggestEdge *SuggestNgram *_sorts] }, + ] + end + end +end diff --git a/rakelib/docker_based_test.rake b/rakelib/docker_based_test.rake index d9b334f4..c84879a9 100644 --- a/rakelib/docker_based_test.rake +++ b/rakelib/docker_based_test.rake @@ -5,6 +5,20 @@ namespace :test do namespace :docker do task :up do system("docker compose up -d") || abort("Unable to start docker containers") + unless system("curl -sf http://localhost:8983/solr || exit 1") + printf("waiting for Solr container to initialize") + sec = 0 + until system("curl -sf http://localhost:8983/solr || exit 1") do + sleep(1) + printf(".") + sec += 1 + if sec > 30 + abort(" Solr container hasn't initialized properly") + end + end + printf("\n") + end + end task :down do #system("docker compose --profile fs --profile ag stop") diff --git a/test/solr/test_solr.rb b/test/solr/test_solr.rb new file mode 100644 index 00000000..6428bc8a --- /dev/null +++ b/test/solr/test_solr.rb @@ -0,0 +1,122 @@ +require_relative '../test_case' +require 'benchmark' + + +class TestSolr < MiniTest::Unit::TestCase + def self.before_suite + @@connector = SOLR::SolrConnector.new(Goo.search_conf, 'test') + @@connector.delete_collection('test') + @@connector.init + end + + def self.after_suite + @@connector.delete_collection('test') + end + + def test_add_collection + connector = @@connector + connector.create_collection('test2') + all_collections = connector.fetch_all_collections + assert_includes all_collections, 'test2' + end + + def test_delete_collection + connector = @@connector + test_add_collection + connector.delete_collection('test2') + + all_collections = connector.fetch_all_collections + refute_includes all_collections, 'test2' + end + + def test_schema_generator + connector = @@connector + + all_fields = connector.all_fields + + connector.schema_generator.fields_to_add.each do |f| + field = all_fields.select { |x| x["name"].eql?(f[:name]) }.first + refute_nil field + assert_equal field["type"], f[:type] + assert_equal field["indexed"], f[:indexed] + assert_equal field["stored"], f[:stored] + assert_equal field["multiValued"], f[:multiValued] + end + + copy_fields = connector.all_copy_fields + connector.schema_generator.copy_fields_to_add.each do |f| + field = copy_fields.select { |x| x["source"].eql?(f[:source]) }.first + refute_nil field + assert_equal field["source"], f[:source] + assert_includes f[:dest], field["dest"] + end + + dynamic_fields = connector.all_dynamic_fields + + connector.schema_generator.dynamic_fields_to_add.each do |f| + field = dynamic_fields.select { |x| x["name"].eql?(f[:name]) }.first + refute_nil field + assert_equal field["name"], f[:name] + assert_equal field["type"], f[:type] + assert_equal field["multiValued"], f[:multiValued] + assert_equal field["stored"], f[:stored] + end + + connector.clear_all_schema + connector.fetch_schema + all_fields = connector.all_fields + connector.schema_generator.fields_to_add.each do |f| + field = all_fields.select { |x| x["name"].eql?(f[:name]) }.first + assert_nil field + end + + copy_fields = connector.all_copy_fields + connector.schema_generator.copy_fields_to_add.each do |f| + field = copy_fields.select { |x| x["source"].eql?(f[:source]) }.first + assert_nil field + end + + dynamic_fields = connector.all_dynamic_fields + connector.schema_generator.dynamic_fields_to_add.each do |f| + field = dynamic_fields.select { |x| x["name"].eql?(f[:name]) }.first + assert_nil field + end + end + + def test_add_field + connector = @@connector + add_field('test', connector) + + + field = connector.fetch_all_fields.select { |f| f['name'] == 'test' }.first + + refute_nil field + assert_equal field['type'], 'string' + assert_equal field['indexed'], true + assert_equal field['stored'], true + assert_equal field['multiValued'], true + + connector.delete_field('test') + end + + def test_delete_field + connector = @@connector + + add_field('test', connector) + + connector.delete_field('test') + + field = connector.all_fields.select { |f| f['name'] == 'test' }.first + + assert_nil field + end + + private + + def add_field(name, connector) + if connector.fetch_field(name) + connector.delete_field(name) + end + connector.add_field(name, 'string', indexed: true, stored: true, multi_valued: true) + end +end diff --git a/test/test_search.rb b/test/test_search.rb index 180062d1..0bba79d9 100644 --- a/test/test_search.rb +++ b/test/test_search.rb @@ -3,9 +3,9 @@ module TestSearch class TermSearch < Goo::Base::Resource - model :term_search, name_with: :id + model :term_search, name_with: lambda { |resource| uuid_uri_generator(resource) } attribute :prefLabel, enforce: [:existence] - attribute :synonym # array of strings + attribute :synonym, enforce: [:list] # array of strings attribute :definition # array of strings attribute :submissionAcronym, enforce: [:existence] attribute :submissionId, enforce: [:existence, :integer] @@ -14,6 +14,39 @@ class TermSearch < Goo::Base::Resource attribute :semanticType attribute :cui + enable_indexing(:term_search) do | schema_generator | + schema_generator.add_field(:prefLabel, 'text_general', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:synonym, 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:definition, 'string', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:submissionAcronym, 'string', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:submissionId, 'pint', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:cui, 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:semanticType, 'text_general', indexed: true, stored: true, multi_valued: true) + + # Copy fields for term search + schema_generator.add_copy_field('prefLabel', '_text_') + # for exact search + schema_generator.add_copy_field('prefLabel', 'prefLabelExact') + + # Matches whole terms in the suggest text + schema_generator.add_copy_field('prefLabel', 'prefLabelSuggest') + + # Will match from the left of the field, e.g. if the document field + # is "A brown fox" and the query is "A bro", it will match, but not "brown" + schema_generator.add_copy_field('prefLabel', 'prefLabelSuggestEdge') + + # Matches any word in the input field, with implicit right truncation. + # This means that the field "A brown fox" will be matched by query "bro". + # We use this to get partial matches, but these would be boosted lower than exact and left-anchored + schema_generator.add_copy_field('prefLabel', 'prefLabelSuggestNgram') + + schema_generator.add_copy_field('synonym', '_text_') + schema_generator.add_copy_field('synonym', 'synonymExact') + schema_generator.add_copy_field('synonym', 'synonymSuggest') + schema_generator.add_copy_field('synonym', 'synonymSuggestEdge') + schema_generator.add_copy_field('synonym', 'synonymSuggestNgram') + end + def index_id() "#{self.id.to_s}_#{self.submissionAcronym}_#{self.submissionId}" end @@ -23,8 +56,45 @@ def index_doc(to_set = nil) end end + class TermSearch2 < Goo::Base::Resource + model :term_search2, name_with: :prefLabel + attribute :prefLabel, enforce: [:existence], fuzzy_search: true + attribute :synonym, enforce: [:list] + attribute :definition + attribute :submissionAcronym, enforce: [:existence] + attribute :submissionId, enforce: [:existence, :integer] + attribute :private, enforce: [:boolean], default: false, index: false + # Dummy attributes to validate non-searchable files + attribute :semanticType + attribute :cui + + enable_indexing(:test_solr) + end + + class TermSearch3 < Goo::Base::Resource + model :term_search3, name_with: :prefLabel + attribute :prefLabel, enforce: [:existence] + attribute :synonym, enforce: [:list] + attribute :definition + attribute :submissionAcronym, enforce: [:existence] + attribute :submissionId, enforce: [:existence, :integer] + attribute :private, enforce: [:boolean], default: false, index: false + # Dummy attributes to validate non-searchable files + attribute :semanticType + attribute :cui + + attribute :object, enforce: [:term_search] + attribute :object_list, enforce: [:term_search, :list] + + + enable_indexing(:test_solr) + end + class TestModelSearch < MiniTest::Unit::TestCase + def self.before_suite + Goo.init_search_connections(true) + end def setup @terms = [ TermSearch.new( @@ -61,7 +131,21 @@ def setup submissionId: 2, semanticType: "Neoplastic Process", cui: "C0375111" - ) + ), + TermSearch.new( + id: RDF::URI.new("http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#Melanoma2"), + prefLabel: "Melanoma with cutaneous melanoma syndrome", + synonym: [ + "Cutaneous Melanoma", + "Skin Cancer", + "Malignant Melanoma" + ], + definition: "Melanoma refers to a malignant skin cancer", + submissionAcronym: "NCIT", + submissionId: 2, + semanticType: "Neoplastic Process", + cui: "C0025202" + ), ] end @@ -78,6 +162,98 @@ def test_search assert_equal @terms[1].prefLabel, resp["response"]["docs"][0]["prefLabel"] end + def test_search_filters + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelExact^100 prefLabelSuggestEdge^50 synonymSuggestEdge^10 prefLabelSuggestNgram synonymSuggestNgram resource_id cui semanticType", + "pf"=>"prefLabelSuggest^50", + } + resp = TermSearch.search("Melanoma wi", params) + assert_equal(3, resp["response"]["numFound"]) + assert_equal @terms[2].prefLabel, resp["response"]["docs"][0]["prefLabel"] + end + + def test_search_exact_filter + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelExact", + } + resp = TermSearch.search("Melanoma", params) + assert_equal(1, resp["response"]["numFound"]) + assert_equal @terms[0].prefLabel, resp["response"]["docs"][0]["prefLabel"] + end + + def test_search_suggest_edge_filter + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelSuggestEdge", + } + resp = TermSearch.search("Melanoma with", params) + assert_equal(1, resp["response"]["numFound"]) + assert_equal @terms[2].prefLabel, resp["response"]["docs"][0]["prefLabel"] + + resp = TermSearch.search("Melanoma", params) + assert_equal(2, resp["response"]["numFound"]) + assert_equal @terms[0].prefLabel, resp["response"]["docs"][0]["prefLabel"] + end + + def test_search_suggest_ngram_filter + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelSuggestNgram", + } + resp = TermSearch.search("cutaneous", params) + assert_equal(1, resp["response"]["numFound"]) + assert_equal @terms[2].prefLabel, resp["response"]["docs"][0]["prefLabel"] + + resp = TermSearch.search("eous", params) + assert_equal(0, resp["response"]["numFound"]) + end + + def test_search_suggest_filter + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelSuggest", + } + resp = TermSearch.search("cutaneous test with Neoplasm Melanoma", params) + assert_equal(3, resp["response"]["numFound"]) + + + resp = TermSearch.search("mel", params) + assert_equal(0, resp["response"]["numFound"]) + end + def test_unindex TermSearch.indexClear() @terms[1].index() @@ -120,7 +296,7 @@ def test_indexBatch TermSearch.indexBatch(@terms) TermSearch.indexCommit() resp = TermSearch.search("*:*") - assert_equal 2, resp["response"]["docs"].length + assert_equal @terms.size, resp["response"]["docs"].length end def test_unindexBatch @@ -128,7 +304,7 @@ def test_unindexBatch TermSearch.indexBatch(@terms) TermSearch.indexCommit() resp = TermSearch.search("*:*") - assert_equal 2, resp["response"]["docs"].length + assert_equal @terms.size, resp["response"]["docs"].length TermSearch.unindexBatch(@terms) TermSearch.indexCommit() @@ -142,6 +318,69 @@ def test_indexClear resp = TermSearch.search("*:*") assert_equal 0, resp["response"]["docs"].length end + + def test_index_on_save_delete + TermSearch2.find("test").first&.delete + TermSearch3.find("test2").first&.delete + + term = TermSearch2.new(prefLabel: "test", + submissionId: 1, + definition: "definition of test", + synonym: ["synonym1", "synonym2"], + submissionAcronym: "test", + private: true + ) + + term2 = TermSearch3.new(prefLabel: "test2", + submissionId: 1, + definition: "definition of test2", + synonym: ["synonym1", "synonym2"], + submissionAcronym: "test", + private: true, + object: TermSearch.new(prefLabel: "test", submissionAcronym: 'acronym', submissionId: 1 ).save, + object_list: [TermSearch.new(prefLabel: "test2",submissionAcronym: 'acronym2', submissionId: 2).save, + TermSearch.new(prefLabel: "test3", submissionAcronym: 'acronym3', submissionId: 3).save] + ) + + term.save + term2.save + + # set as not indexed in model definition + refute_includes TermSearch2.search_client.fetch_all_fields.map{|f| f["name"]}, "private_b" + refute_includes TermSearch2.search_client.fetch_all_fields.map{|f| f["name"]}, "private_b" + + + indexed_term = TermSearch2.search("id:#{term.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + indexed_term2 = TermSearch3.search("id:#{term2.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + + term.indexable_object.each do |k, v| + assert_equal v, indexed_term[k.to_s] + end + + term2.indexable_object.each do |k, v| + assert_equal v, indexed_term2[k.to_s] + end + + term2.definition = "new definition of test2" + term2.synonym = ["new synonym1", "new synonym2"] + term2.save + + indexed_term2 = TermSearch3.search("id:#{term2.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + + term2.indexable_object.each do |k, v| + assert_equal v, indexed_term2[k.to_s] + end + + term2.delete + term.delete + + indexed_term = TermSearch2.submit_search_query("id:#{term.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + indexed_term2 = TermSearch3.submit_search_query("id:#{term2.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + + assert_nil indexed_term + assert_nil indexed_term2 + + end end end