diff --git a/examples/neural_reasoner_retrieval.py b/examples/neural_reasoner_retrieval.py index a2ecb6a..4a4e065 100644 --- a/examples/neural_reasoner_retrieval.py +++ b/examples/neural_reasoner_retrieval.py @@ -117,7 +117,6 @@ def execute(args): nominals = inds_in_sig # (10) All combinations of 3 for Nominals, e.g. {martin, heinz, markus} nominal_combinations = set( OWLObjectOneOf(combination)for combination in itertools.combinations(nominals, 3)) - # (11) NC UNION NC. unions = concept_reducer(nc, opt=OWLObjectUnionOf) # (12) NC INTERSECTION NC. diff --git a/tests/test_neural_reasoner_accuracy.py b/tests/test_neural_reasoner_accuracy.py new file mode 100644 index 0000000..ea3b2c7 --- /dev/null +++ b/tests/test_neural_reasoner_accuracy.py @@ -0,0 +1,198 @@ +from owlapy.owl_neural_reasoners.owl_neural_reasoner import OWLNeuralReasoner +from owlapy.owl_reasoner import StructuralReasoner +from owlapy.owl_ontology_manager import OntologyManager +from owlapy.utils import concept_reducer, concept_reducer_properties, jaccard_similarity, f1_set_similarity +from owlapy.class_expression import ( + OWLObjectUnionOf, + OWLObjectIntersectionOf, + OWLObjectSomeValuesFrom, + OWLObjectAllValuesFrom, + OWLObjectMinCardinality, + OWLObjectMaxCardinality, + OWLObjectOneOf, +) +import time +from typing import Tuple, Set +import pandas as pd +import random +import itertools + +def test_retrieval_performance(): + # Set up variables + path_kg = "KGs/Family/family-benchmark_rich_background.owl" + path_kge_model = None + gamma = 0.8 + seed = 42 + ratio_sample_object_prop = 1.0 + ratio_sample_nc = 1.0 + num_nominals = 10 + min_jaccard_similarity = 1.0 + min_f1_score = 1.0 + + # (1) Initialize knowledge base. + symbolic_kb = StructuralReasoner(ontology=OntologyManager().load_ontology(path=path_kg)) + + # (2) Initialize Neural OWL Reasoner. + if path_kge_model: + neural_owl_reasoner = OWLNeuralReasoner(path_neural_embedding=path_kge_model, gamma=gamma) + else: + neural_owl_reasoner = OWLNeuralReasoner(path_of_kb=path_kg, gamma=gamma) + + # Fix the random seed. + random.seed(seed) + + ################################################################### + # GENERATE DL CONCEPTS TO EVALUATE RETRIEVAL PERFORMANCES + + # (3) R: Extract object properties. + object_properties = {i for i in symbolic_kb.get_root_ontology().object_properties_in_signature()} + + # (3.1) Subsample if required. + if ratio_sample_object_prop and len(object_properties) > 0: + object_properties = {i for i in random.sample(population=list(object_properties), + k=max(1, int(len(object_properties) * ratio_sample_object_prop)))} + + # (4) R⁻: Inverse of object properties. + object_properties_inverse = {i.get_inverse_property() for i in object_properties} + + # (5) R*: R UNION R⁻. + object_properties_and_inverse = object_properties.union(object_properties_inverse) + + # (6) NC: Named owl concepts. + nc = {i for i in symbolic_kb.get_root_ontology().classes_in_signature()} + + if ratio_sample_nc and len(nc) > 0: + # (6.1) Subsample if required. + nc = {i for i in random.sample(population=list(nc), k=max(1, int(len(nc) * ratio_sample_nc)))} + + # (7) NC⁻: Complement of NC. + nnc = {i.get_object_complement_of() for i in nc} + + # (8) NC*: NC UNION NC⁻. + nc_star = nc.union(nnc) + + # (9) Retrieve random Nominals. + inds_in_sig = list(symbolic_kb.get_root_ontology().individuals_in_signature()) + if len(inds_in_sig) > num_nominals: + nominals = set(random.sample(inds_in_sig, num_nominals)) + else: + nominals = set(inds_in_sig) + + # (10) All combinations of 3 for Nominals. + nominal_combinations = set(OWLObjectOneOf(combination) for combination in itertools.combinations(nominals, 3)) + + # (13) NC* UNION NC*. + unions_nc_star = concept_reducer(nc_star, opt=OWLObjectUnionOf) + + # (14) NC* INTERSECTION NC*. + intersections_nc_star = concept_reducer(nc_star, opt=OWLObjectIntersectionOf) + + # (15) ∃ r. C s.t. C ∈ NC* and r ∈ R* . + exist_nc_star = concept_reducer_properties( + concepts=nc_star, + properties=object_properties_and_inverse, + cls=OWLObjectSomeValuesFrom, + ) + + # (16) ∀ r. C s.t. C ∈ NC* and r ∈ R* . + for_all_nc_star = concept_reducer_properties( + concepts=nc_star, + properties=object_properties_and_inverse, + cls=OWLObjectAllValuesFrom, + ) + + # (17) ≥ n r. C and ≤ n r. C, s.t. C ∈ NC* and r ∈ R* . + min_cardinality_nc_star_1 = concept_reducer_properties( + concepts=nc_star, + properties=object_properties_and_inverse, + cls=OWLObjectMinCardinality, + cardinality=1, + ) + min_cardinality_nc_star_2 = concept_reducer_properties( + concepts=nc_star, + properties=object_properties_and_inverse, + cls=OWLObjectMinCardinality, + cardinality=2, + ) + min_cardinality_nc_star_3 = concept_reducer_properties( + concepts=nc_star, + properties=object_properties_and_inverse, + cls=OWLObjectMinCardinality, + cardinality=3, + ) + + max_cardinality_nc_star_1 = concept_reducer_properties( + concepts=nc_star, + properties=object_properties_and_inverse, + cls=OWLObjectMaxCardinality, + cardinality=1, + ) + max_cardinality_nc_star_2 = concept_reducer_properties( + concepts=nc_star, + properties=object_properties_and_inverse, + cls=OWLObjectMaxCardinality, + cardinality=2, + ) + max_cardinality_nc_star_3 = concept_reducer_properties( + concepts=nc_star, + properties=object_properties_and_inverse, + cls=OWLObjectMaxCardinality, + cardinality=3, + ) + + # (18) ∃ r. Nominal s.t. Nominal ∈ Nominals and r ∈ R* . + exist_nominals = concept_reducer_properties( + concepts=nominal_combinations, + properties=object_properties_and_inverse, + cls=OWLObjectSomeValuesFrom, + ) + + ################################################################### + # Retrieval Results + + def concept_retrieval(retriever_func, c) -> Tuple[Set[str], float]: + start_time = time.time() + return {i.str for i in retriever_func.instances(c)}, time.time() - start_time + + # Combine all concepts + concepts = list( + itertools.chain( + nc, + nnc, + unions_nc_star, + intersections_nc_star, + exist_nc_star, + for_all_nc_star, + min_cardinality_nc_star_1, + min_cardinality_nc_star_2, + min_cardinality_nc_star_3, + max_cardinality_nc_star_1, + max_cardinality_nc_star_2, + max_cardinality_nc_star_3, + exist_nominals, + ) + ) + random.shuffle(concepts) + + data = [] + + for expression in concepts: + retrieval_y, _ = concept_retrieval(symbolic_kb, expression) + retrieval_neural_y, _ = concept_retrieval(neural_owl_reasoner, expression) + jaccard_sim = jaccard_similarity(retrieval_y, retrieval_neural_y) + f1_sim = f1_set_similarity(retrieval_y, retrieval_neural_y) + data.append({ + "Jaccard Similarity": jaccard_sim, + "F1": f1_sim, + }) + + df = pd.DataFrame(data) + + mean_jaccard_similarity = df["Jaccard Similarity"].mean() + assert mean_jaccard_similarity >= min_jaccard_similarity, \ + f"Mean Jaccard Similarity {mean_jaccard_similarity} is less than the threshold {min_jaccard_similarity}" + + mean_f1_score = df["F1"].mean() + assert mean_f1_score >= min_f1_score, \ + f"Mean F1 Score {mean_f1_score} is less than the threshold {min_f1_score}" +