For comparison only: DEV branch

src/curation/upheno-config.yaml

@@ -379,6 +379,8 @@ upheno_profiles:
 pattern_repos:
   - obophenotype/upheno/contents/src/patterns/dosdp-dev
   - obophenotype/upheno/contents/src/patterns/dosdp-patterns
+exclude_patterns:
+  - abnormalMorphologyOfAnatomicalEntity.yaml
 release_artefacts: 
   - base
   - simple-non-classified

src/ontology/Makefile

@@ -1,7 +1,7 @@
 # ----------------------------------------
 # Makefile for upheno
 # Generated using ontology-development-kit
-# ODK Version: v1.4
+# ODK Version: v1.4.3
 # ----------------------------------------
 # IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use upheno.Makefile instead
 
@@ -43,14 +43,14 @@ REPORT_PROFILE_OPTS =
 OBO_FORMAT_OPTIONS =        
 SPARQL_VALIDATION_CHECKS =  owldef-self-reference iri-range label-with-iri multiple-replaced_by 
 SPARQL_EXPORTS =            basic-report class-count-by-prefix edges xrefs obsoletes synonyms 
-ODK_VERSION_MAKEFILE =      v1.4
+ODK_VERSION_MAKEFILE =      v1.4.3
 
 TODAY ?=                    $(shell date +%Y-%m-%d)
 OBODATE ?=                  $(shell date +'%d:%m:%Y %H:%M')
 VERSION=                    $(TODAY)
 ANNOTATE_ONTOLOGY_VERSION = annotate -V $(ONTBASE)/releases/$(VERSION)/$@ --annotation owl:versionInfo $(VERSION)
 ANNOTATE_CONVERT_FILE =     annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) convert -f ofn --output [email protected] && mv [email protected] $@
-OTHER_SRC =                 $(PATTERNDIR)/definitions.owl 
+OTHER_SRC =                 $(PATTERNDIR)/definitions.owl $(COMPONENTSDIR)/upheno-deprecated.owl 
 ONTOLOGYTERMS =             $(TMPDIR)/ontologyterms.txt
 EDIT_PREPROCESSED =         $(TMPDIR)/$(ONT)-preprocess.owl
 PATTERNDIR=                 ../patterns
@@ -345,6 +345,45 @@ refresh-%:
 no-mirror-refresh-%:
 	$(MAKE) IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl -B
 
+
+# ----------------------------------------
+# Components
+# ----------------------------------------
+# Some ontologies contain external and internal components. A component is included in the ontology in its entirety.
+
+COMP=true # Global parameter to bypass component generation
+
+.PHONY: all_components
+all_components: $(OTHER_SRC)
+
+.PHONY: recreate-components
+recreate-components:
+	$(MAKE) COMP=true IMP=false MIR=true PAT=true IMP_LARGE=false all_components -B
+
+.PHONY: no-mirror-recreate-components
+no-mirror-recreate-components:
+	$(MAKE) COMP=true IMP=false MIR=false PAT=true IMP_LARGE=false all_components -B
+
+.PHONY: recreate-%
+recreate-%:
+	$(MAKE) COMP=true IMP=false IMP_LARGE=false MIR=true PAT=true $(COMPONENTSDIR)/$*.owl -B
+
+.PHONY: no-mirror-recreate-%
+no-mirror-recreate-%:
+	$(MAKE) COMP=true IMP=false IMP_LARGE=false MIR=false PAT=true $(COMPONENTSDIR)/$*.owl -B
+
+$(COMPONENTSDIR)/%: | $(COMPONENTSDIR)
+	touch $@
+.PRECIOUS: $(COMPONENTSDIR)/%
+
+
+
+$(COMPONENTSDIR)/upheno-deprecated.owl: $(TEMPLATEDIR)/obsolete.tsv
+	if [ $(COMP) = true ] ; then $(ROBOT) template  \
+		$(patsubst %, --template %, $^) \
+		$(ANNOTATE_CONVERT_FILE); fi
+
+.PRECIOUS: $(COMPONENTSDIR)/upheno-deprecated.owl
 # ----------------------------------------
 # Mirroring upstream ontologies
 # ----------------------------------------

src/ontology/upheno-odk.yaml

@@ -278,3 +278,9 @@ import_group:
     - id: pato
     - id: cl
     - id: mpath
+components:
+  products:
+    - filename: upheno-deprecated.owl
+      use_template: TRUE
+      templates:
+        - obsolete.tsv

src/scripts/create_sssom.py

@@ -0,0 +1,106 @@
+
+import pandas as pd
+import os
+import yaml
+import glob
+import argparse
+from sssom.context import get_converter
+from sssom.parsers import from_sssom_dataframe
+from sssom.writers import write_table
+
+def main():
+    parser = argparse.ArgumentParser(description='Create SSSOM file from upheno id map and pattern matches')
+    parser.add_argument('--upheno_id_map', type=str, help='upheno id map file')
+    parser.add_argument('--patterns_dir', type=str, help='directory containing pattern files')
+    parser.add_argument('--matches_dir', type=str, help='directory containing pattern matches')
+    args = parser.parse_args()
+    create_upheno_sssom(args.upheno_id_map, args.patterns_dir, args.matches_dir)
+
+def get_id_columns(pattern_file):
+    try:
+        with open(pattern_file, "r") as stream:
+            pattern_json = yaml.safe_load(stream)
+            idcolumns = list(pattern_json["vars"].keys())
+            return idcolumns
+    except Exception as exc:
+        print("Could not get id columns: " + pattern_file)
+        return None
+
+def create_upheno_sssom(upheno_id_map, patterns_dir, matches_dir):
+
+    all_pattern_matches_map = dict()
+
+    for pattern_match_tsv in glob.glob(matches_dir + "/**/*.tsv"):
+        pattern_name = os.path.basename( pattern_match_tsv ).split(".")[0]
+        df = pd.read_csv(pattern_match_tsv, sep='\t')
+        if pattern_name in all_pattern_matches_map:
+            all_pattern_matches_map[pattern_name] = pd.concat([ all_pattern_matches_map[pattern_name], df ])
+        else:
+            all_pattern_matches_map[pattern_name] = df
+
+
+
+    cache_pattern_file_to_idcolumn = dict()
+
+    df = pd.read_csv(upheno_id_map, sep='\t')
+
+    sssom = []
+
+    converter = get_converter()
+
+    for index, row in df.iterrows():
+        tokens = row['id'].split('-')
+        fillers = tokens[:-1]
+        pattern_name = tokens[-1].split('.')[0]
+        pattern_file = pattern_name + ".yaml"
+        id_columns = cache_pattern_file_to_idcolumn.get(pattern_file)
+        if id_columns == None:
+            id_columns = get_id_columns(os.path.join(patterns_dir, pattern_file))
+            cache_pattern_file_to_idcolumn[pattern_file] = id_columns
+        if id_columns == None:
+            continue
+        # print(tokens)
+        # print(pattern_file)
+        # print(id_columns)
+        # print(fillers)
+        tsv_df = all_pattern_matches_map[pattern_name]
+        #filtered = tsv[lambda df: filter_row(df, id_columns, fillers) ]
+
+        mask = pd.Series(True, index=tsv_df.index)
+        for col, filler in zip(id_columns, fillers):
+            mask = mask & (tsv_df[col] == filler)
+        subset_df = tsv_df[mask]
+
+        # print(subset_df)
+
+        upheno_id = row['defined_class']
+
+        for index, row in subset_df.iterrows():
+            species_specific_id = row['defined_class']
+            sssom.append([
+                converter.compress(upheno_id),
+                "semapv:crossSpeciesExactMatch", 
+                converter.compress(species_specific_id),
+                "semapv:LogicalMatching"
+                ])
+
+    df_out = pd.DataFrame(sssom, columns=['subject_id', 'predicate_id', 'object_id', 'mapping_justification'])
+
+    meta = dict()
+    meta['mapping_set_id'] = 'https://data.monarchinitiative.org/mappings/upheno/upheno-species-independent.sssom.tsv'
+    msdf = from_sssom_dataframe(df_out, prefix_map=converter, meta=meta)
+    msdf.clean_prefix_map()
+    write_table(msdf, open("upheno-species-independent.sssom.tsv", "w"))
+
+def filter_row(df, id_columns, fillers):
+    n = 0
+    while n < len(id_columns):
+        column = id_columns[n]
+        filler = fillers[n]
+        if df[column] != filler:
+            return False
+    return True
+
+if __name__ == "__main__":
+    main()
+

src/scripts/lib.py

@@ -115,6 +115,9 @@ def get_min_upheno_id(self):
     def get_pattern_repos(self):
         return self.config.get("pattern_repos")
 
+    def get_exclude_patterns(self):
+        return self.config.get("exclude_patterns")
+
     def get_working_directory(self):
         return self.config.get("working_directory")
 

src/scripts/run.sh

@@ -9,4 +9,4 @@
 # we therefore map the whole repo (../..) to a docker volume.
 #
 # See README-editors.md for more details.
-docker run -v $PWD/../../:/work -w /work/src/scripts --rm -ti obolibrary/odkfull:dev "$@"
+docker run -v $PWD/../../:/work -w /work/src/scripts --rm -ti obolibrary/odkfull:v1.3.1 "$@"

src/scripts/upheno_pipeline.sh

@@ -49,7 +49,14 @@ sh run.sh python3 upheno_prepare.py ../curation/upheno-config.yaml
 #    from the previous step using dosdp. Add taxon restrictions 
 
 sh run.sh python3 upheno_create_profiles.py ../curation/upheno-config.yaml
+
 sh run.sh python3 upheno-stats.py ../curation/upheno-config.yaml
+
+sh run.sh python3 create_sssom.py \
+    --upheno_id_map ../curation/upheno_id_map.txt \
+    --patterns_dir ../curation/patterns-for-matching \
+    --matches_dir ../curation/pattern-matches
+
 cd ../ontology/
 sh run.sh make o sim reports
 echo "Release successfully completed, ready to deploy."

src/scripts/upheno_prepare.py

@@ -12,6 +12,7 @@
 import sys
 import urllib.request
 import warnings
+import json
 from subprocess import CalledProcessError, check_call
 
 import pandas as pd
@@ -207,15 +208,17 @@ def get_pattern_urls(upheno_pattern_repos):
     return upheno_patterns
 
 
-def download_patterns(upheno_pattern_repos, pattern_dir):
+def download_patterns(upheno_pattern_repos, pattern_dir, exclude_patterns):
     upheno_patterns = get_pattern_urls(upheno_pattern_repos)
     filenames = []
     for url in upheno_patterns:
-        print("Downloading " + url)
         filename = os.path.basename(url)
         file_path = os.path.join(pattern_dir, filename)
+        print("Downloading " + filename + " from " + url + " to " + pattern_dir)
+        if filename in exclude_patterns:
+            continue
         if not upheno_config.is_skip_pattern_download():
-            try:
+            # try:
                 x = urllib.request.urlopen(url).read()
                 y = ruamel.yaml.round_trip_load(x, preserve_quotes=True)
                 print(file_path)
@@ -227,8 +230,23 @@ def download_patterns(upheno_pattern_repos, pattern_dir):
                 with open(file_path, "w") as outfile:
                     ruamel.yaml.round_trip_dump(y, outfile, explicit_start=True, width=5000)
 
-            except Exception as exc:
-                print(exc)
+                # generate inheres_in matches for any inheres_in_part_of patterns
+                #
+                if "RO:0002314" in y["relations"].values(): # inheres in part of
+                    new_pattern = y.copy()
+                    new_pattern["relations"] = {}
+                    for k,v in y["relations"].items():
+                        if v == "RO:0002314":
+                            new_pattern["relations"][k] = "RO:0000052"
+                        else:
+                            new_pattern["relations"][k] = v
+                    new_file_path = os.path.splitext(file_path)[0] + "-modified.yaml"
+                    with open(new_file_path, "w") as outfile:
+                        ruamel.yaml.round_trip_dump(new_pattern, outfile, explicit_start=True, width=5000)
+                        filenames.append(new_file_path)
+
+            # except Exception as exc:
+                # print(exc)
 
         if os.path.isfile(file_path):
             filenames.append(filename)
@@ -386,6 +404,42 @@ def prepare_species_specific_phenotype_ontologies(config):
             )
             robot_upheno_component(o_base_taxon, remove_eqs_file)
 
+def postprocess_modified_patterns(upheno_config, pattern_files, matches_dir):
+    patterns = []
+    delete_files = []
+    delete_files.extend(pattern_files)
+
+    for pattern_path in pattern_files:
+        pid = os.path.basename(pattern_path).replace(".yaml", "")
+        patterns.append(pid)
+
+    for id in upheno_config.get_phenotype_ontologies():
+        oid_matches_path = os.path.join(matches_dir, id)
+        for pattern in patterns:
+            # Load both the modified and unm modified tsv files
+            # merge them and write them back to the unmodified file
+            unmodified_tsv_path = os.path.join(oid_matches_path, pattern + ".tsv")
+            modified_tsv_path = os.path.join(oid_matches_path, pattern + "-modification.tsv")
+            if not os.path.exists(modified_tsv_path):
+                continue
+            if not os.path.exists(unmodified_tsv_path):
+                continue
+            df_unmodified = pd.read_csv(unmodified_tsv_path, sep="\t")
+            df_modified = pd.read_csv(modified_tsv_path, sep="\t")
+            df_combined = pd.concat([df_unmodified, df_modified])
+            # Remove duplicate rows
+            df_final = df_combined.drop_duplicates()
+            df_final.to_csv(unmodified_tsv_path, sep="\t", index=False)
+            delete_files.append(modified_tsv_path)
+
+    # Delete the modified tsv files and their corresponding patterns:
+    for file_path in delete_files:
+        if os.path.exists(file_path):
+            os.remove(file_path)
+
+
+
+
 
 def match_patterns(upheno_config, pattern_files, matches_dir, pattern_dir, overwrite=True):
     patterns = []
@@ -413,6 +467,14 @@ def match_patterns(upheno_config, pattern_files, matches_dir, pattern_dir, overw
             dosdp_pattern_match(ontology_path, pattern_string2, pattern_dir, outdir, TIMEOUT)
         else:
             print("Matches for ({}) already made, bypassing.".format(outdir))
+
+    postprocess_modified_patterns = [
+        os.path.join(pattern_dir, f)
+        for f in os.listdir(pattern_dir)
+        if os.path.isfile(os.path.join(pattern_dir, f)) and f.endswith("-modification.yaml")
+        ]
+
+    postprocess_modified_patterns(upheno_config, postprocess_modified_patterns, matches_dir)
 
 
 def add_taxon_restrictions(
@@ -479,13 +541,16 @@ def download_sources(dir, overwrite=True):
 
 
 print("### Download patterns ###")
-pattern_files = download_patterns(upheno_config.get_pattern_repos(), pattern_dir)
+exclude_patterns = upheno_config.get_exclude_patterns()
+pattern_files = download_patterns(upheno_config.get_pattern_repos(), pattern_dir, exclude_patterns)
 pattern_files = [
     os.path.join(pattern_dir, f)
     for f in os.listdir(pattern_dir)
     if os.path.isfile(os.path.join(pattern_dir, f)) and f.endswith(".yaml")
 ]
 
+exit(0)
+
 print("### Download sources ###")
 print("ROBOT args: " + os.environ["ROBOT_JAVA_ARGS"])
 download_sources(module_dir, upheno_config.is_overwrite_ontologies())