Feature 1419 competency question coverage report

.github/workflows/automated-testing.yml

@@ -12,6 +12,50 @@ jobs:
       - name: make
         run: |
           make
+      - name: validate-profile
+        run: | 
+          echo "java -jar build/robot.jar validate-profile --input build/oeo/$(cat VERSION)/oeo-full.owl --profile Full -vvv --output merged-validation.txt"
+          java -jar build/robot.jar validate-profile --input build/oeo/$(cat VERSION)/oeo-full.owl --profile Full -vvv --output merged-validation.txt
+      - name: verify
+        run: |
+          java -jar build/robot.jar verify --input build/oeo/$(cat VERSION)/oeo-full.owl --queries tests/verify/*
+      - uses: actions/upload-artifact@master
+        with:
+          name: build-artifacts
+          path: build/oeo
+  test:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.7'
+          architecture: x64
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
+          java-version: '11'
+      - uses: actions/download-artifact@master
+        with:
+          name: build-artifacts
+          path: build/oeo
+      - name: install python dependencies
+        run: pip install -r src/scripts/requirements.txt
+      - name: setup robot 1.9.0
+        run: |
+          wget https://github.com/ontodev/robot/releases/download/v1.9.0/robot.jar -O build/robot19.jar
+      - name: Build ETD xlsx
+        run: |
+          java -jar build/robot19.jar merge --input build/oeo/$(cat VERSION)/oeo-full.omn \
+            --include-annotations true \
+          export  --header "ID|LABEL|definition" \
+                  --prefix "OEO: http://openenergy-platform.org/ontology/oeo/OEO_" \
+                  --sort "LABEL" \
+                  --export $(pwd)/build/oeo/$(cat VERSION)/etd.xlsx
+      - name: Build ETD csv
+        run: |
+          python $(pwd)/src/scripts/etd/etd.py $(pwd)/build/oeo/$(cat VERSION)
       - name: consistency
         run: |
           wget https://github.com/owlcs/releases/raw/master/HermiT/org.semanticweb.hermit-packaged-1.4.6.519-SNAPSHOT.jar -O build/hermit.jar
@@ -22,21 +66,48 @@ jobs:
           fi
           echo "Ontology is inconsistent: $OUT"
           exit 1
-      - name: validate-profile
-        run: | 
-          echo "java -jar build/robot.jar validate-profile --input build/oeo/$(cat VERSION)/oeo-full.owl --profile Full -vvv --output merged-validation.txt"
-          java -jar build/robot.jar validate-profile --input build/oeo/$(cat VERSION)/oeo-full.owl --profile Full -vvv --output merged-validation.txt
-      - name: verify
-        run: |
-          java -jar build/robot.jar verify --input build/oeo/$(cat VERSION)/oeo-full.owl --queries tests/verify/*
-      - name: competency
-        run: |
-          bash tests/competency_questions/run_questions.sh "java -jar build/hermit.jar" $(pwd)/build/oeo/$(cat VERSION)/oeo-full.owl true
-          bash tests/competency_questions/run_questions.sh "java -jar build/hermit.jar" $(pwd)/build/oeo/$(cat VERSION)/oeo-full.owl false
-      - name: Upload Artifacts
+      - name: Upload Ontology
+        if: always()
         uses: actions/upload-artifact@v3
         with:
           name: build-files
           path: |
             build/**/*
-            !build/**/*.jar
+            !build/**/*.jar
+      - name: competency
+        continue-on-error: true
+        run: |
+          pytest -s -v
+          # bash tests/competency_questions/run_questions.sh "java -jar build/hermit.jar" $(pwd)/build/oeo/$(cat VERSION)/oeo-full.owl true
+          # bash tests/competency_questions/run_questions.sh "java -jar build/hermit.jar" $(pwd)/build/oeo/$(cat VERSION)/oeo-full.owl false
+      - name: Upload Artifacts
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-report
+          path: build/report.json
+      - name: Get Coverage for badge 
+        run : |
+          echo "COVERAGE=$(head build/report.json | grep -o '"coverage": "[^"]*"' | grep -o '[^"]*\%')" >> $GITHUB_ENV  
+          REF=${{ github.ref }}     
+          IFS='/' read -ra PATHS <<< "$REF"
+          BRANCH_NAME="${PATHS[1]}_${PATHS[2]}"
+          echo "BRANCH=$(echo ${BRANCH_NAME})" >> $GITHUB_ENV
+      - name: Coverage Badge
+        uses: schneegans/[email protected]
+        with:
+          auth: ${{ secrets.GIST_SECRET }}
+          gistID: 6d00affa9fbc89c79684d62091d96551
+          filename: open_energy_ontology__${{ env.BRANCH }}.json
+          label: CQ Coverage
+          message: ${{ env.COVERAGE }}
+          color: green
+#           bash tests/competency_questions/run_questions.sh "java -jar build/hermit.jar" $(pwd)/build/oeo/$(cat VERSION)/oeo-full.owl true
+#           bash tests/competency_questions/run_questions.sh "java -jar build/hermit.jar" $(pwd)/build/oeo/$(cat VERSION)/oeo-full.owl false
+#       - name: Upload Artifacts
+#         uses: actions/upload-artifact@v3
+#         with:
+#           name: build-files
+#           path: |
+#             build/**/*
+#             !build/**/*.jar

README.md

@@ -2,6 +2,7 @@
 
 [![License: CC0-1.0](https://img.shields.io/badge/License-CC0%201.0-lightgrey.svg)](http://creativecommons.org/publicdomain/zero/1.0/)
 ![GitHub release (latest by date)](https://img.shields.io/github/v/release/OpenEnergyPlatform/ontology)
+![Coverage Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/areleu/6d00affa9fbc89c79684d62091d96551/raw/open_energy_ontology__heads_feature-1419-competency-question-coverage-report.json)
 
 # Open Energy Family - Open Energy Ontology (OEO)
 

src/scripts/etd/etd.py

@@ -25,9 +25,7 @@
     df = df.replace('\n', '<br>', regex=True)
     df = df.sort_values("LABEL", key=lambda col: col.str.strip().str.lower())
     # Create one table per letter:
-
-    pathlib.Path(cwd + "/src/scripts/etd/glossary/").mkdir(parents=True, exist_ok=True) 
-
+    pathlib.Path(target_path).joinpath("glossary").mkdir(parents=True, exist_ok=True) 
     # header = GLOSSARY_HEADER + " ".join([f"[{letter}]({BASE_LINK_WIKI}{letter})" for letter in string.ascii_uppercase]) + "\n"
 
     # with open(cwd + "/src/scripts/etd/glossary/glossary.md", "w") as fil:
@@ -55,4 +53,4 @@
     df_csv["ID"] = df_csv["ID"].str.replace("http://openenergy-platform.org/ontology/oeo/oeo-physical/", "")
     df_csv["ID"] = df_csv["ID"].str.replace("http://openenergy-platform.org/ontology/oeo/oeo-model/", "")
     df_csv["ID"] = df_csv["ID"].str.replace(":", "_")
-    df_csv.to_csv(pathlib.Path(target_path).joinpath("glossary/glossary.csv").as_posix(), index=False)
+    df_csv.to_csv(pathlib.Path(target_path).joinpath("glossary/glossary.csv").as_posix(), index=False)

src/scripts/requirements.txt

@@ -1,3 +1,5 @@
 pandas
 tabulate
-openpyxl
+openpyxl
+pytest
+pytest_harvest

tests/competency_questions/q4.omn → tests/competency_questions/physical/completeness/004_equivalences_to_flourinated_greenhouse_gases.omn

@@ -9,7 +9,7 @@ Ontology: <http://openenergy-platform.org/ontology/cc/> <http://openenergy-platf
 Class: OEO_00000026
 Class: OEO_00000038
 Class: OEO_00000219
-Class: OEO_00000321
+Class: OEO_00000322
 
 Class: OEO_00000013
     EquivalentTo: OEO_00000026 or OEO_00000038 or OEO_00000219 or OEO_00000322

tests/competency_questions/q5.omn → tests/competency_questions/physical/completeness/005_methane_has_normal_state_of_matter_gaseous.omn

@@ -8,7 +8,7 @@ Ontology: <http://openenergy-platform.org/ontology/cc/> <http://openenergy-platf
 
 Class: OEO_00000025
 ObjectProperty: OEO_00000529
-Class: OEO_00000182
+Individual: OEO_00000182
 
 Class: OEO_00000025
     SubClassOf: OEO_00000529 value OEO_00000182

tests/competency_questions/q12.omn → tests/competency_questions/physical/completeness/012_gas_fired_power_unit_is_fueled_power_unit.omn

@@ -4,7 +4,7 @@ Prefix: : <http://openenergy-platform.org/ontology/oeo/>
 Ontology: <http://openenergy-platform.org/ontology/cc/> <http://openenergy-platform.org/ontology/cc/>
 
 
-#Is a gas fired power unit a fuelled power unit?
+#Is a gas fired power unit a fueled power unit?
 
 Class: OEO_00000017
 Class: OEO_00000175

tests/competency_questions/q47.omn → tests/competency_questions/physical/completeness/047_portion_of_hydrogen_is_energy_carrier.omn

@@ -8,7 +8,7 @@ Ontology: <http://openenergy-platform.org/ontology/cc/> <http://openenergy-platf
 # Every portion of hydrogen is an energy carrier
 
 ObjectProperty: obo:RO_0000091 # has disposion
-Class: OEO_00000028 # energy carrier disposition
+Class: OEO_00000151 # energy carrier disposition
 
 Class: OEO_00000220 # hydrogen
-    SubClassOf: obo:RO_0000091 some OEO_00000028
+    SubClassOf: obo:RO_0000091 some OEO_00000151

tests/competency_questions/q1.omn → tests/competency_questions/physical/soundness/001_biofuel_is_renewable_fuel.omn

@@ -3,11 +3,12 @@ Prefix: : <http://openenergy-platform.org/ontology/oeo/>
 Ontology: <http://openenergy-platform.org/ontology/cc/> <http://openenergy-platform.org/ontology/cc/>
 
 # Competency Question: Is  biofuel renewable fuel?
- 
+
 Class: OEO_00000033
 Class: OEO_00000072
 
 Class: OEO_00000072
   SubClassOf: OEO_00000033
-
-# This competency question gets deprecated with this pull request: https://github.com/OpenEnergyPlatform/ontology/pull/1409
+
+# This competency question gets deprecated with this pull request: https://github.com/OpenEnergyPlatform/ontology/pull/1409
+# This means it should be false

tests/competency_questions/q29.omn → tests/competency_questions/physical/soundness/029_biofuel_is_classified_as_renewable_fuel.omn

@@ -12,4 +12,5 @@ Class: OEO_00000033
 Class: OEO_00000072
     SubClassOf: OEO_00000033
 
-# This competency question gets deprecated with this pull request: https://github.com/OpenEnergyPlatform/ontology/pull/1409
+# This competency question gets deprecated with this pull request: https://github.com/OpenEnergyPlatform/ontology/pull/1409
+# This means it should be false

tests/conftest.py

@@ -0,0 +1,5 @@
+
+
+def pytest_addoption(parser):
+    parser.addoption("--selected", action="store", default="")
+    parser.addoption("--report", action="store", default="report.json")

tests/test_competency_questions.py

@@ -0,0 +1,173 @@
+from glob import glob
+import pandas as pd
+from subprocess import Popen, PIPE
+from pathlib import Path
+import json
+import re
+from pytest_harvest import saved_fixture
+import pytest
+import os
+import shutil
+
+CWD = os.getcwd()
+with open(Path(CWD).joinpath("VERSION").as_posix()) as version_file:
+    __version__ = version_file.readlines()[0].strip()
+
+HERMIT_JAR = Path(CWD).joinpath("build/hermit.jar").as_posix()
+JAVA_PATH = shutil.which("java")
+ONTOLOGY_PATH = (
+    Path(CWD)
+    .joinpath("build/oeo")
+    .joinpath(f"{__version__}")
+    .joinpath("oeo-full.omn")
+    .as_posix()
+)
+EXISTING_TERMS_AND_DEFINITONS = (
+    Path(CWD).joinpath("build/oeo").joinpath(f"{__version__}").joinpath("glossary/glossary.csv")
+)
+COMPETENCY_QUESTION_DIRECTORY = Path(CWD).joinpath("tests/competency_questions")
+
+
+@pytest.fixture
+@saved_fixture
+def existing_terms_and_definitons():
+    """
+    Fixture containing the final report
+    """
+    terms_and_definitions = pd.read_csv(EXISTING_TERMS_AND_DEFINITONS)
+    return list(terms_and_definitions["ID"])
+
+
+def pytest_generate_tests(metafunc):
+    if "competency_question_path" in metafunc.fixturenames:
+        competency_question_list = [
+            p for p in glob(COMPETENCY_QUESTION_DIRECTORY.as_posix() + "/**/*.omn", recursive=True)
+        ]
+        selected_filer = metafunc.config.getoption("--selected")
+        competency_question_list = [
+            cq for cq in competency_question_list if selected_filer in cq
+        ]
+        metafunc.parametrize("competency_question_path", competency_question_list)
+
+
+def check_competency_question(conclusion, premise):
+    """Check competency question against the given ontology
+
+    Args:
+        conclusion (str): A filepath to the competency question.
+        premise (str): A filepath to the reference ontology.
+
+    Raises:
+        RuntimeError: If calling HermiT raises an error this will be raised as a RuntimeError.
+
+    Returns:
+        (boolean): Returns True if the competency question was sucessfully checked against the ontology.
+    """
+    conclusion = Path(conclusion).resolve().as_posix()
+    p = Popen(
+        [
+            f"{JAVA_PATH}",
+            "-jar",
+            f"{HERMIT_JAR}",
+            f"--premise=file:///{premise}",
+            f"--conclusion=file:///{conclusion}",
+            "-E",
+        ],
+        stdin=PIPE,
+        stdout=PIPE,
+        stderr=PIPE,
+    )
+    output, err = p.communicate(b"input data that is passed to subprocess' stdin")
+    if len(err) == 0:
+        return eval(output.strip().capitalize())
+    else:
+        raise RuntimeError(f"{err}")
+
+
+def search_term_in_question_file(question, pattern=r"\w+_\d{8}"):
+    """Search all the Ontology terms in a specific competency question file
+
+    Args:
+        question (sts): Filepath to the competency question file
+        pattern (regexp, optional): Regular expression of the terms to be searched. Defaults to r"\w+_\d{8}".
+
+    Returns:
+        list: List of all the terms covered by the CQ.
+    """
+    terms = []
+    with open(question, "r") as question_file:
+        for line in question_file.readlines():
+            terms.extend(re.findall(pattern, line))
+    return terms
+
+
+def test_competency_question(
+    competency_question_path, existing_terms_and_definitons, results_bag
+):
+    """Metatest to produce competency question tests."""
+    results_bag.questions = {}
+    results_bag.terms = {}
+    name = Path(competency_question_path).stem
+    is_soundness = True if "soundness" in competency_question_path else False
+    failure = ""
+    query_result = False
+    if not name in results_bag.questions:
+        results_bag.questions[name] = {"passed": False, "covers": []}
+    try:
+        query_result = check_competency_question(
+            competency_question_path, ONTOLOGY_PATH
+        )
+        if is_soundness:
+            query_result = not query_result
+        results_bag.questions[name]["passed"] = query_result
+    except RuntimeError as e:
+        results_bag.questions[name]["passed"] = False
+        results_bag.questions[name]["error"] = str(e)
+        failure = str(e)
+
+    terms = search_term_in_question_file(competency_question_path)
+    results_bag.questions[name]["covers"].extend(terms)
+    results_bag.questions[name]["covers"] = list(
+        set(results_bag.questions[name]["covers"])
+    )
+    for term in terms:
+        if term in existing_terms_and_definitons:
+            if not term in results_bag.terms:
+                results_bag.terms[term] = {"covered": False, "by": []}
+            results_bag.terms[term]["covered"] = True
+            results_bag.terms[term]["by"].append(name)
+    if len(failure) > 0:
+        raise RuntimeError(failure)
+
+    assert query_result, f"{name} failed"
+
+
+def test_synthesis(fixture_store, existing_terms_and_definitons):
+    """This test should run at the end. Produces the final report.
+
+    Args:
+        fixture_store (fixture): These are all the availible fixtures.
+        existing_terms_and_definitons (fixture): Fixture with existing terms and definitions.
+    """
+    initial_terms = {t : {"covered": False, "by": []} for t in existing_terms_and_definitons if "OEO_" in t}
+    report = {"coverage": "0%", "questions": {}, "terms": initial_terms}
+    for v in fixture_store["results_bag"].values():
+        report["questions"].update(v["questions"])
+        for term in v["terms"].keys():
+            if term in report["terms"]:
+                report["terms"][term] ["covered"] = True
+                report["terms"][term]["by"].extend(v["terms"][term]["by"])
+                report["terms"][term]["by"] = list(set(report["terms"][term]["by"]))
+                report["terms"][term]["by"] = list(set(report["terms"][term]["by"]))
+            else:
+                report["terms"][term] = {
+                    "covered": True,
+                    "by": list(set(report["terms"][term]["by"])),
+                }
+
+    coverage = sum([1 for c in report.get("terms", {}).values() if c["covered"]]) / len(
+        [term for term in existing_terms_and_definitons if "OEO_" in term]
+    )
+    report["coverage"] = "{:.0%}".format(coverage)
+    with open(Path(CWD).joinpath("build/report.json"), "w") as f:
+        json.dump(report, f, indent=4)