Skip to content

Commit

Permalink
update delete zombie mapping counts to check also submission deletion
Browse files Browse the repository at this point in the history
  • Loading branch information
@syphax-bouazzouni
syphax-bouazzouni committed Nov 3, 2024
1 parent 955167b commit 5ffc197
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 84 deletions.
32 changes: 11 additions & 21 deletions lib/ontologies_linked_data/concerns/mappings/mapping_counts.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,7 @@ def mapping_counts(enable_debug = false, logger = nil, reload_cache = false, arr
exter_total += v
end
counts[external_uri.to_s] = exter_total
if enable_debug
logger.info("Time for External Mappings took #{Time.now - t0} sec. records #{exter_total}")
end
logger.info("Time for External Mappings took #{Time.now - t0} sec. records #{exter_total}") if enable_debug
LinkedData.settings.interportal_hash ||= {}
# Counting for Interportal mappings
LinkedData.settings.interportal_hash.each_key do |acro|
Expand Down Expand Up @@ -94,7 +92,7 @@ def create_mapping_count_totals_for_ontologies(logger, arr_acronyms)
end

ontologies = LinkedData::Models::Ontology.where.include(:acronym).all.map(&:acronym)
delete_zombie_mapping_count(persistent_counts.values, ontologies)
delete_zombie_mapping_count(persistent_counts.values, new_counts)

num_counts = new_counts.keys.length
ctr = 0
Expand Down Expand Up @@ -160,8 +158,6 @@ def create_mapping_count_pairs_for_ontologies(logger, arr_acronyms)
# temp_dir = '/Users/mdorf/Downloads/test/'
# temp_file_path = File.join(File.dirname(file_path), "test.ttl")
# fsave = File.open(temp_file_path, "a")
ontologies = LinkedData::Models::Ontology.where.include(:acronym).all.map(&:acronym)

latest_submissions.each do |acr, sub|
self.handle_triple_store_downtime(logger) if Goo.backend_4s?
new_counts = nil
Expand All @@ -175,14 +171,10 @@ def create_mapping_count_pairs_for_ontologies(logger, arr_acronyms)
LinkedData::Models::MappingCount.where(pair_count: true).and(ontologies: acr)
.include(:ontologies, :count).all.each do |m|
other = m.ontologies.first

if other == acr
other = m.ontologies[1]
end
other = m.ontologies.last if other == acr
persistent_counts[other] = m
end

delete_zombie_mapping_count(persistent_counts.values, ontologies)
delete_zombie_mapping_count(persistent_counts.values, new_counts)

num_counts = new_counts.keys.length
logger.info("Ontology: #{acr}. #{num_counts} mapping pair counts to record...")
Expand All @@ -198,7 +190,7 @@ def create_mapping_count_pairs_for_ontologies(logger, arr_acronyms)
if new_count.zero?
inst.delete
elsif new_count != inst.count
inst.bring_remaining
inst.bring_remaining if inst.persistent?
inst.pair_count = true
inst.count = new_count

Expand All @@ -216,13 +208,12 @@ def create_mapping_count_pairs_for_ontologies(logger, arr_acronyms)
end
end
else
next unless ontologies.include?(other)
next unless new_counts.key?(other)

m = LinkedData::Models::MappingCount.new
m.count = new_count
m.ontologies = [acr, other]
m.pair_count = true
puts "creating mapping count #{m.ontologies}"
begin
if m.valid?
m.save()
Expand Down Expand Up @@ -254,17 +245,16 @@ def create_mapping_count_pairs_for_ontologies(logger, arr_acronyms)

private

def delete_zombie_mapping_count(mappings_count, ontologies)
def delete_zombie_mapping_count(existent_counts, new_counts)
special_mappings = ["http://data.bioontology.org/metadata/ExternalMappings",
"http://data.bioontology.org/metadata/InterportalMappings/agroportal",
"http://data.bioontology.org/metadata/InterportalMappings/ncbo",
"http://data.bioontology.org/metadata/InterportalMappings/sifr"]

mappings_count.each do |mapping|
next mapping if mapping.ontologies.all? { |x| ontologies.include?(x) }
next mapping if mapping.ontologies.size == 1 && !(mapping.ontologies & special_mappings).empty?

next mapping unless mapping.persistent?
existent_counts.each do |mapping|
next if mapping.ontologies.size == 1 && !(mapping.ontologies & special_mappings).empty?
next if new_counts.keys.include?(mapping.ontologies.first)
next unless mapping.persistent?

mapping.delete
end
Expand Down
134 changes: 78 additions & 56 deletions test/models/test_mappings.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,11 @@

class TestMapping < LinkedData::TestOntologyCommon


ONT_ACR1 = 'MAPPING_TEST1'
ONT_ACR2 = 'MAPPING_TEST2'
ONT_ACR3 = 'MAPPING_TEST3'
ONT_ACR4 = 'MAPPING_TEST4'


def self.before_suite
backend_4s_delete
ontologies_parse
Expand All @@ -18,26 +16,26 @@ def self.before_suite
def self.ontologies_parse
helper = LinkedData::TestOntologyCommon.new(self)
helper.submission_parse(ONT_ACR1,
"MappingOntTest1",
"./test/data/ontology_files/BRO_v3.3.owl", 11,
process_rdf: true, extract_metadata: false)
"MappingOntTest1",
"./test/data/ontology_files/BRO_v3.3.owl", 11,
process_rdf: true, extract_metadata: false)
helper.submission_parse(ONT_ACR2,
"MappingOntTest2",
"./test/data/ontology_files/CNO_05.owl", 22,
process_rdf: true, extract_metadata: false)
"MappingOntTest2",
"./test/data/ontology_files/CNO_05.owl", 22,
process_rdf: true, extract_metadata: false)
helper.submission_parse(ONT_ACR3,
"MappingOntTest3",
"./test/data/ontology_files/aero.owl", 33,
process_rdf: true, extract_metadata: false)
"MappingOntTest3",
"./test/data/ontology_files/aero.owl", 33,
process_rdf: true, extract_metadata: false)
helper.submission_parse(ONT_ACR4,
"MappingOntTest4",
"./test/data/ontology_files/fake_for_mappings.owl", 44,
process_rdf: true, extract_metadata: false)
"MappingOntTest4",
"./test/data/ontology_files/fake_for_mappings.owl", 44,
process_rdf: true, extract_metadata: false)
end

def test_mapping_count_delete
def test_mapping_count_ontology_delete
assert create_count_mapping > 2, 'Mapping count should exceed the value of 2'

counts = LinkedData::Models::MappingCount.where.include(:ontologies).all.map(&:ontologies)

assert(counts.any? { |x| x.include?(ONT_ACR4) }, 'Mapping count of ONT_ACR4 should exist')
Expand All @@ -48,6 +46,34 @@ def test_mapping_count_delete

counts = LinkedData::Models::MappingCount.where.include(:ontologies).all.map(&:ontologies)
refute(counts.any? { |x| x.include?(ONT_ACR4) }, 'Mapping count of deleted ontologies should not exist anymore')
submission_parse(ONT_ACR4,
"MappingOntTest4",
"./test/data/ontology_files/fake_for_mappings.owl", 45,
process_rdf: true, extract_metadata: false)
end

def test_mapping_count_submission_delete
assert create_count_mapping > 2, 'Mapping count should exceed the value of 2'

counts = LinkedData::Models::MappingCount.where.include(:ontologies).all.map(&:ontologies)

assert(counts.any? { |x| x.include?(ONT_ACR4) }, 'Mapping count of ONT_ACR4 should exist')

sub = LinkedData::Models::Ontology.find(ONT_ACR4).first.latest_submission
sub.bring_remaining
sub.archive(force: true)
sub.delete

LinkedData::Mappings.create_mapping_counts(Logger.new(TestLogFile.new))

counts = LinkedData::Models::MappingCount.where.include(:ontologies).all.map(&:ontologies)
refute(counts.any? { |x| x.include?(ONT_ACR4) }, "Mapping count of deleted ontologies should not exist anymore: #{counts}")

LinkedData::Models::Ontology.find(ONT_ACR4).first.delete
submission_parse(ONT_ACR4,
"MappingOntTest4",
"./test/data/ontology_files/fake_for_mappings.owl", 46,
process_rdf: true, extract_metadata: false)
end

def test_mapping_count_models
Expand All @@ -63,34 +89,32 @@ def test_mapping_count_models
assert LinkedData::Models::MappingCount.where(ontologies: "BRO").all.count == 1
m = LinkedData::Models::MappingCount.new
assert !m.valid?
m.ontologies = ["BRO","FMA"]
m.ontologies = ["BRO", "FMA"]
m.count = 321
m.pair_count = true
assert m.valid?
m.save
assert LinkedData::Models::MappingCount.where(ontologies: "BRO").all.count == 2
result = LinkedData::Models::MappingCount.where(ontologies: "BRO")
.and(ontologies: "FMA").include(:count).all
.and(ontologies: "FMA").include(:count).all
assert result.length == 1
assert result.first.count == 321
result = LinkedData::Models::MappingCount.where(ontologies: "BRO")
.and(pair_count: true)
.include(:count)
.all
.and(pair_count: true)
.include(:count)
.all
assert result.length == 1
assert result.first.count == 321
LinkedData::Models::MappingCount.where.all(&:delete)
end



def test_mappings_ontology
LinkedData::Models::RestBackupMapping.all.each do |m|
LinkedData::Mappings.delete_rest_mapping(m.id)
end

assert create_count_mapping > 2
#bro
# bro
ont1 = LinkedData::Models::Ontology.where({ :acronym => ONT_ACR1 }).to_a[0]

latest_sub = ont1.latest_submission
Expand All @@ -100,7 +124,7 @@ def test_mappings_ontology
size = 10
page_no = 1
while keep_going
page = LinkedData::Mappings.mappings_ontology(latest_sub,page_no, size)
page = LinkedData::Mappings.mappings_ontology(latest_sub, page_no, size)
assert_instance_of(Goo::Base::Page, page)
keep_going = (page.length == size)
mappings += page
Expand All @@ -126,9 +150,9 @@ def test_mappings_ontology
end
assert create_count_mapping > 2

by_ont_counts = LinkedData::Mappings.mapping_ontologies_count(latest_sub,nil)
by_ont_counts = LinkedData::Mappings.mapping_ontologies_count(latest_sub, nil)
total = 0
by_ont_counts.each do |k,v|
by_ont_counts.each do |k, v|
total += v
end
assert(by_ont_counts.length == 2)
Expand All @@ -139,12 +163,12 @@ def test_mappings_ontology
assert_equal(by_ont_counts["MAPPING_TEST4"], 8)
assert_equal(total, 18)
assert_equal(mappings.length, 18)
assert_equal(same_uri,10)
assert_equal(same_uri, 10)
assert_equal(cui, 3)
assert_equal(loom,5)
assert_equal(loom, 5)
mappings.each do |map|
class_mappings = LinkedData::Mappings.mappings_ontology(
latest_sub,1,100,map.classes[0].id)
latest_sub, 1, 100, map.classes[0].id)
assert class_mappings.length > 0
class_mappings.each do |cmap|
assert validate_mapping(map)
Expand All @@ -154,9 +178,9 @@ def test_mappings_ontology

def test_mappings_two_ontologies
assert create_count_mapping > 2, "Mapping count should exceed the value of 2"
#bro
# bro
ont1 = LinkedData::Models::Ontology.where({ :acronym => ONT_ACR1 }).to_a[0]
#fake ont
# fake ont
ont2 = LinkedData::Models::Ontology.where({ :acronym => ONT_ACR4 }).to_a[0]

latest_sub1 = ont1.latest_submission
Expand All @@ -168,8 +192,7 @@ def test_mappings_two_ontologies
size = 5
page_no = 1
while keep_going
page = LinkedData::Mappings.mappings_ontologies(latest_sub1,latest_sub2,
page_no, size)
page = LinkedData::Mappings.mappings_ontologies(latest_sub1, latest_sub2, page_no, size, nil, true)
assert_instance_of(Goo::Base::Page, page)
keep_going = (page.length == size)
mappings += page
Expand All @@ -182,7 +205,7 @@ def test_mappings_two_ontologies
assert_equal(map.classes[0].submission.ontology.acronym,
latest_sub1.ontology.acronym)
assert_equal(map.classes[1].submission.ontology.acronym,
latest_sub2.ontology.acronym)
latest_sub2.ontology.acronym)
if map.source == "CUI"
cui += 1
elsif map.source == "SAME_URI"
Expand All @@ -194,8 +217,7 @@ def test_mappings_two_ontologies
end
assert validate_mapping(map), "mapping is not valid"
end
count = LinkedData::Mappings.mapping_ontologies_count(latest_sub1,
latest_sub2)
count = LinkedData::Mappings.mapping_ontologies_count(latest_sub1, latest_sub2, true)

assert_equal(count, mappings.length)
assert_equal(5, same_uri)
Expand All @@ -212,7 +234,7 @@ def test_mappings_rest
mappings_created = []

3.times do |i|
classes = get_mapping_classes(term_a:mapping_term_a[i], term_b: mapping_term_b[i],
classes = get_mapping_classes(term_a: mapping_term_a[i], term_b: mapping_term_b[i],
submissions_a: submissions_a[i], submissions_b: submissions_b[i])

mappings_created << create_rest_mapping(relation: RDF::URI.new(relations[i]),
Expand All @@ -231,10 +253,10 @@ def test_mappings_rest
if m.source == "REST"
rest_mapping_count += 1
assert_equal m.classes.length, 2
c1 = m.classes.select {
|c| c.submission.id.to_s["TEST1"] }.first
c1 = m.classes.select {
|c| c.submission.id.to_s["TEST1"] }.first
c2 = m.classes.select {
|c| c.submission.id.to_s["TEST2"] }.first
|c| c.submission.id.to_s["TEST2"] }.first
assert c1 != nil
assert c2 != nil
ia = mapping_term_a.index c1.id.to_s
Expand All @@ -244,31 +266,31 @@ def test_mappings_rest
assert ia == ib
end
end
#depending on the order could be 2 or three
#some other test is leaving mappings persisted
# depending on the order could be 2 or three
# some other test is leaving mappings persisted
assert rest_mapping_count > 1 || rest_mapping_count < 4
#in a new submission we should have moved the rest mappings
# in a new submission we should have moved the rest mappings
helper = LinkedData::TestOntologyCommon.new(self)
helper.submission_parse(ONT_ACR1,
"MappingOntTest1",
"./test/data/ontology_files/BRO_v3.3.owl", 12,
process_rdf: true, extract_metadata: false)
"MappingOntTest1",
"./test/data/ontology_files/BRO_v3.3.owl", 12,
process_rdf: true, extract_metadata: false)

assert create_count_mapping > 2

latest_sub1 = LinkedData::Models::Ontology.find(RDF::URI.new(ont_id)).first.latest_submission
mappings = LinkedData::Mappings.mappings_ontology(latest_sub1, 1, 1000)
rest_mapping_count = 0
mappings.each do |m|
rest_mapping_count += 1 if m.source == "REST"
rest_mapping_count += 1 if m.source == "REST"
end
assert_equal 3, rest_mapping_count
end

def test_get_rest_mapping
mapping_term_a, mapping_term_b, submissions_a, submissions_b, relations, user = rest_mapping_data

classes = get_mapping_classes(term_a:mapping_term_a[0], term_b: mapping_term_b[0],
classes = get_mapping_classes(term_a: mapping_term_a[0], term_b: mapping_term_b[0],
submissions_a: submissions_a[0], submissions_b: submissions_b[0])

mappings_created = []
Expand Down Expand Up @@ -308,21 +330,21 @@ def get_mapping_classes(term_a:, term_b:, submissions_a:, submissions_b:)
def rest_mapping_data
mapping_term_a = ["http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Image_Algorithm",
"http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Image",
"http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Integration_and_Interoperability_Tools" ]
"http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Integration_and_Interoperability_Tools"]
submissions_a = [
"http://data.bioontology.org/ontologies/MAPPING_TEST1/submissions/latest",
"http://data.bioontology.org/ontologies/MAPPING_TEST1/submissions/latest",
"http://data.bioontology.org/ontologies/MAPPING_TEST1/submissions/latest" ]
"http://data.bioontology.org/ontologies/MAPPING_TEST1/submissions/latest"]
mapping_term_b = ["http://purl.org/incf/ontology/Computational_Neurosciences/cno_alpha.owl#cno_0000202",
"http://purl.org/incf/ontology/Computational_Neurosciences/cno_alpha.owl#cno_0000203",
"http://purl.org/incf/ontology/Computational_Neurosciences/cno_alpha.owl#cno_0000205" ]
"http://purl.org/incf/ontology/Computational_Neurosciences/cno_alpha.owl#cno_0000205"]
submissions_b = [
"http://data.bioontology.org/ontologies/MAPPING_TEST2/submissions/latest",
"http://data.bioontology.org/ontologies/MAPPING_TEST2/submissions/latest",
"http://data.bioontology.org/ontologies/MAPPING_TEST2/submissions/latest" ]
relations = [ "http://www.w3.org/2004/02/skos/core#exactMatch",
"http://www.w3.org/2004/02/skos/core#closeMatch",
"http://www.w3.org/2004/02/skos/core#relatedMatch" ]
"http://data.bioontology.org/ontologies/MAPPING_TEST2/submissions/latest"]
relations = ["http://www.w3.org/2004/02/skos/core#exactMatch",
"http://www.w3.org/2004/02/skos/core#closeMatch",
"http://www.w3.org/2004/02/skos/core#relatedMatch"]

user = LinkedData::Models::User.where.include(:username).all[0]
assert user != nil
Expand Down
Loading

0 comments on commit 5ffc197

Please sign in to comment.