Merge remote-tracking branch 'origin/master' into itrb-test

code/ARAX/ARAXQuery/ARAX_decorator.py

@@ -112,7 +112,7 @@ def decorate_edges(self, response: ARAXResponse, kind: Optional[str] = "RTX-KG2"
         """
         kg = response.envelope.message.knowledge_graph
         response.debug(f"Decorating edges with EPC info from KG2c")
-        supported_kinds = {"RTX-KG2", "NGD"}
+        supported_kinds = {"RTX-KG2", "NGD", "SEMMEDDB"}
         if kind not in supported_kinds:
             response.error(f"Supported values for ARAXDecorator.decorate_edges()'s 'kind' parameter are: "
                            f"{supported_kinds}")
@@ -124,6 +124,11 @@ def decorate_edges(self, response: ARAXResponse, kind: Optional[str] = "RTX-KG2"
                                      if edge.sources and any(retrieval_source.resource_id == self.kg2_infores_curie and
                                                              retrieval_source.resource_role == "aggregator_knowledge_source"
                                                              for retrieval_source in edge.sources)}
+        elif kind == "SEMMEDDB":
+            edge_keys_to_decorate = {edge_id for edge_id, edge in kg.edges.items()
+                                     if edge.sources and any(retrieval_source.resource_id == "infores:semmeddb" and
+                                                             retrieval_source.resource_role == "primary_knowledge_source"
+                                                             for retrieval_source in edge.sources)}
         else:
             edge_keys_to_decorate = {edge_id for edge_id, edge in kg.edges.items()
                                      if edge.predicate == "biolink:occurs_together_in_literature_with"}
@@ -206,7 +211,7 @@ def decorate_edges(self, response: ARAXResponse, kind: Optional[str] = "RTX-KG2"
                 existing_attribute_type_ids = {attribute.attribute_type_id for attribute in bare_edge.attributes} if bare_edge.attributes else set()
                 new_attributes = []
                 # Create KG2 edge-specific attributes
-                if kind == "RTX-KG2":
+                if kind == "RTX-KG2" or kind == "SEMMEDDB":
                     if attribute_type_id_map["kg2_ids"] not in existing_attribute_type_ids:
                         new_attributes.append(self.create_attribute("kg2_ids", list(joined_kg2_ids)))
                     if joined_publications and attribute_type_id_map["publications"] not in existing_attribute_type_ids:

code/ARAX/ARAXQuery/ARAX_expander.py

@@ -388,7 +388,10 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
                         infer_input_parameters = {"action": "drug_treatment_graph_expansion",'node_curie': object_curie, 'qedge_id': inferred_qedge_key}
                         inferer = ARAXInfer()
                         infer_response = inferer.apply(response, infer_input_parameters)
-                        return infer_response
+                        # return infer_response
+                        response = infer_response
+                        overarching_kg = eu.convert_standard_kg_to_qg_organized_kg(message.knowledge_graph)
+
                     elif set(['biolink:regulates']).intersection(set(qedge.predicates)): # Figure out if this is a "regulates" query, then use call XCRG models
                         # Call XCRG models and simply return whatever it returns
                         # Get the subject and object of this edge
@@ -425,7 +428,8 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
                             infer_input_parameters = {"action": "chemical_gene_regulation_graph_expansion", 'object_qnode_id' : qedge.object, 'object_curie': object_curie, 'qedge_id': inferred_qedge_key, 'regulation_type': regulation_type}
                         inferer = ARAXInfer()
                         infer_response = inferer.apply(response, infer_input_parameters)
-                        return infer_response
+                        response = infer_response
+                        overarching_kg = eu.convert_standard_kg_to_qg_organized_kg(message.knowledge_graph)
                     else:
                         log.info(f"Qedge {inferred_qedge_key} has knowledge_type == inferred, but the query is not "
                                  f"DTD-related (e.g., 'biolink:ameliorates', 'biolink:treats') or CRG-related ('biolink:regulates') according to the specified predicate. Will answer using the normal 'fill' strategy (not creative mode).")
@@ -434,10 +438,13 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
                                 f"the qedges has knowledge_type == inferred. Will answer using the normal 'fill' strategy "
                                 f"(not creative mode).")
 
-
         # Expand any specified edges
         if qedge_keys_to_expand:
             query_sub_graph = self._extract_query_subgraph(qedge_keys_to_expand, query_graph, log)
+            if mode != "RTXKG2":
+                if inferred_qedge_keys and len(query_graph.edges) == 1:
+                    for edge in query_sub_graph.edges.keys():
+                        query_sub_graph.edges[edge].knowledge_type = 'lookup'
             if log.status != 'OK':
                 return response
             log.debug(f"Query graph for this Expand() call is: {query_sub_graph.to_dict()}")
@@ -473,7 +480,10 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
 
                 # Create a query graph for this edge (that uses curies found in prior steps)
                 one_hop_qg = self._get_query_graph_for_edge(qedge_key, query_graph, overarching_kg, log)
-
+                if mode != "RTXKG2":
+                    if inferred_qedge_keys and len(query_graph.edges) == 1:
+                        for edge in one_hop_qg.edges.keys():
+                            one_hop_qg.edges[edge].knowledge_type = 'lookup'
                 # Figure out the prune threshold (use what user provided or otherwise do something intelligent)
                 if parameters.get("prune_threshold"):
                     pre_prune_threshold = parameters["prune_threshold"]
@@ -486,7 +496,10 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
                     for qnode_key in fulfilled_qnode_keys:
                         num_kg_nodes = len(overarching_kg.nodes_by_qg_id[qnode_key])
                         if num_kg_nodes > pre_prune_threshold:
-                            overarching_kg = self._prune_kg(qnode_key, pre_prune_threshold, overarching_kg, query_graph, log)
+                            if inferred_qedge_keys and len(inferred_qedge_keys) == 1:
+                                overarching_kg = self._prune_kg(qnode_key, pre_prune_threshold, overarching_kg, message.query_graph, log)
+                            else:
+                                overarching_kg = self._prune_kg(qnode_key, pre_prune_threshold, overarching_kg, query_graph, log)
                             # Re-formulate the QG for this edge now that the KG has been slimmed down
                             one_hop_qg = self._get_query_graph_for_edge(qedge_key, query_graph, overarching_kg, log)
                 if log.status != 'OK':
@@ -650,7 +663,10 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
                     self._apply_any_kryptonite_edges(overarching_kg, message.query_graph,
                                                      message.encountered_kryptonite_edges_info, response)
                     # Remove any paths that are now dead-ends
-                    overarching_kg = self._remove_dead_end_paths(query_graph, overarching_kg, response)
+                    if inferred_qedge_keys and len(inferred_qedge_keys) == 1:
+                        overarching_kg = self._remove_dead_end_paths(message.query_graph, overarching_kg, response)
+                    else:
+                        overarching_kg = self._remove_dead_end_paths(query_graph, overarching_kg, response)
                     if response.status != 'OK':
                         return response
 
@@ -694,10 +710,14 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
 
             # Override node types to only include descendants of what was asked for in the QG (where applicable) #1360
             self._override_node_categories(message.knowledge_graph, message.query_graph, log)
+        elif mode == "RTXKG2":
+            decorator = ARAXDecorator()
+            decorator.decorate_edges(response, kind="SEMMEDDB")
+
 
         # Map canonical curies back to the input curies in the QG (where applicable) #1622
         self._map_back_to_input_curies(message.knowledge_graph, query_graph, log)
-        if mode != "RTXKG2":    
+        if mode == "RTXKG2":
             eu.remove_semmeddb_edges_and_nodes_with_low_publications(message.knowledge_graph, response)
             overarching_kg = eu.convert_standard_kg_to_qg_organized_kg(message.knowledge_graph)
         # Return the response and done

code/ARAX/ARAXQuery/ARAX_ranker.py

@@ -625,8 +625,29 @@ def aggregate_scores_dmk(self, response):
         #print(float(len(ranks_list)))
         result_scores = sum(ranks_list)/float(len(ranks_list))
         #print(result_scores)
+
+        # Replace Inferred Results Score with Probability score calculated by xDTD model
+        inferred_qedge_keys = [qedge_key for qedge_key, qedge in message.query_graph.edges.items() 
+                               if qedge.knowledge_type == "inferred"]
         for result, score in zip(results, result_scores):
             result.analyses[0].score = score  # For now we only ever have one Analysis per Result
+            if inferred_qedge_keys:
+                inferred_qedge_key = inferred_qedge_keys[0]
+                edge_bindings =  result.analyses[0].edge_bindings
+                inferred_edge_bindings = []
+                if edge_bindings:
+                    inferred_edge_bindings = edge_bindings.get(inferred_qedge_key,[])
+                for edge_name in inferred_edge_bindings:
+                    edge_id = edge_name.id
+                    edge_attributes = message.knowledge_graph.edges[edge_id].attributes
+                    if edge_attributes is not None:
+                        for edge_attribute in edge_attributes:
+                            if edge_attribute.original_attribute_name == 'probability_treats' and edge_attribute.value is not None:
+                                result.analyses[0].score = float(edge_attribute.value)
+
+
+
+
 
         # for result in message.results:
         #     self.result_confidence_maker(result)

code/ARAX/ARAXQuery/Expand/expand_utilities.py

@@ -598,7 +598,7 @@ def remove_semmeddb_edges_and_nodes_with_low_publications(kg: KnowledgeGraph, lo
             log.error(tb, error_code=error_type.__name__)
             log.error(f"Something went wrong removing semmeddb edges from the knowledge graph")
         else:
-            log.info(f"{edges_removed_counter} Semmeddb Edges with low publications successfully removed")
+            log.info(f"{edges_removed_counter} Semmeddb Edges with low publication count successfully removed")
 
 
 def is_expand_created_subclass_qedge_key(qedge_key: str, qg: QueryGraph) -> bool:

code/ARAX/test/test_ARAX_expand.py

@@ -1114,12 +1114,18 @@ def test_xdtd_expand():
     nodes_by_qg_id, edges_by_qg_id, message = _run_query_and_do_standard_testing(json_query=query, return_message=True)
     assert message.auxiliary_graphs
     for edge in edges_by_qg_id["t_edge"].values():
-        assert edge.attributes
-        support_graph_attributes = [attribute for attribute in edge.attributes if attribute.attribute_type_id == "biolink:support_graphs"]
-        assert support_graph_attributes
-        assert len(support_graph_attributes) == 1
-        support_graph_attribute = support_graph_attributes[0]
-        assert support_graph_attribute.value[0] in message.auxiliary_graphs
+        inferred_edge = False
+        for source in edge.sources:
+            if source.resource_role == "primary_knowledge_source" and source.resource_id == "infores:arax":
+                inferred_edge = True
+        # Perform Tests only for inferred edges
+        if inferred_edge:
+            assert edge.attributes
+            support_graph_attributes = [attribute for attribute in edge.attributes if attribute.attribute_type_id == "biolink:support_graphs"]
+            assert support_graph_attributes
+            assert len(support_graph_attributes) == 1
+            support_graph_attribute = support_graph_attributes[0]
+            assert support_graph_attribute.value[0] in message.auxiliary_graphs
 
 
 @pytest.mark.slow

code/RTXConfiguration.py

@@ -11,7 +11,7 @@
 import yaml
 from pygit2 import Repository, discover_repository
 
-DEBUG = False
+DEBUG = True
 
 class RTXConfiguration:
 

code/UI/interactive/rtx.js

@@ -69,6 +69,7 @@ function main() {
     UIstate["version"] = checkUIversion(false);
     UIstate["scorestep"] = 0.1;
     UIstate["maxresults"] = 1000;
+    UIstate["prevtimestampobj"] = null;
     document.getElementById("menuapiurl").href = providers["ARAX"].url + "/ui/";
 
     load_meta_knowledge_graph();
@@ -2348,11 +2349,37 @@ function there_was_an_error() {
     document.getElementById("menunumresults").classList.remove("numold");
 }
 
+
+function calc_timespan(obj) {
+    if (!obj.dataset.timestamp)
+	return;
+
+    obj.style.background = "#ff0";
+
+    if (UIstate["prevtimestampobj"]) {
+	let units = " seconds";
+	let diff = Math.abs(obj.dataset.timestamp - UIstate["prevtimestampobj"].dataset.timestamp)/1000;
+	if (diff>66) {
+	    diff/=60;
+	    units = " minutes";
+	}
+	showJSONpopup("Elapsed Time","Elapsed time is: "+diff+units,false);
+
+	obj.style.background = null;
+	UIstate["prevtimestampobj"].style.background = null;
+	UIstate["prevtimestampobj"] = null;
+    }
+    else {
+	UIstate["prevtimestampobj"] = obj;
+    }
+}
+
 function process_log(logarr) {
     var status = {};
     for (var s of ["ERROR","WARNING","INFO","DEBUG"]) {
 	status[s] = 0;
     }
+    let starttime = null;
     for (var msg of logarr) {
 	if (msg.prefix) { // upconvert TRAPI 0.9.3 --> 1.0
 	    msg.level = msg.level_str;
@@ -2362,7 +2389,14 @@ function process_log(logarr) {
 	status[msg.level]++;
 
 	var span = document.createElement("span");
+	span.title = "Click to display elapsed time between two events";
 	span.className = "hoverable msg " + msg.level;
+	span.dataset.timestamp = Date.parse(msg.timestamp);
+	span.setAttribute('onclick', 'calc_timespan(this);');
+
+	if (!starttime)
+	    starttime = Date.parse(msg.timestamp);
+	span.dataset.timestamp = Date.parse(msg.timestamp);
 
         if (msg.level == "DEBUG") { span.style.display = 'none'; }
 
@@ -2372,23 +2406,36 @@ function process_log(logarr) {
 	span2.appendChild(document.createTextNode('\u00A0'));
         span.appendChild(span2);
 
-	span.appendChild(document.createTextNode('\u00A0'));
-
-	span.appendChild(document.createTextNode(msg.timestamp+" "+msg.level+": "));
+        span2 = document.createElement("span");
+	span2.appendChild(document.createTextNode('\u00A0'));
+	span2.appendChild(document.createTextNode(msg.timestamp+" "+msg.level+": "));
 	if (msg.code)
-	    span.appendChild(document.createTextNode("["+msg.code+"] "));
+	    span2.appendChild(document.createTextNode("["+msg.code+"] "));
+
+	span2.appendChild(document.createTextNode('\u00A0'));
+	span2.appendChild(document.createTextNode('\u00A0'));
+	span2.appendChild(document.createTextNode('\u00A0'));
+	span2.appendChild(document.createTextNode(msg.message));
+        span.appendChild(span2);
 
-	span.appendChild(document.createTextNode('\u00A0'));
-	span.appendChild(document.createTextNode('\u00A0'));
-	span.appendChild(document.createTextNode('\u00A0'));
-	span.appendChild(document.createTextNode(msg.message));
-	span.appendChild(document.createElement("br"));
+        let units = " s";
+	let diff = Math.abs(span.dataset.timestamp - starttime)/1000;
+	if (diff>66) {
+	    diff/=60;
+	    units = " m";
+	}
+	span2 = document.createElement("span");
+	span2.style.float = 'right';
+        span2.appendChild(document.createTextNode(diff+units));
+        span.appendChild(span2);
 
 	document.getElementById("logdiv").appendChild(span);
     }
     document.getElementById("menunummessages").innerHTML = logarr.length;
-    if (status.ERROR > 0) document.getElementById("menunummessages").classList.add('numnew','msgERROR');
-    else if (status.WARNING > 0) document.getElementById("menunummessages").classList.add('numnew','msgWARNING');
+    if (status.ERROR > 0)
+	document.getElementById("menunummessages").classList.add('numnew','msgERROR');
+    else if (status.WARNING > 0)
+	document.getElementById("menunummessages").classList.add('numnew','msgWARNING');
     for (var s of ["ERROR","WARNING","INFO","DEBUG"]) {
 	document.getElementById("count_"+s).innerHTML += ": "+status[s];
     }

code/UI/interactive/rtx.version

@@ -1 +1 @@
-Wobbly Manta ray
+Same Chipmunk

requirements.txt

@@ -31,9 +31,9 @@ treelib==1.6.1
 asyncio==3.4.3
 aiohttp==3.8.5
 boto3==1.24.59
-tornado==6.3.2
+tornado==6.3.3
 MarkupSafe==2.1.2
-reasoner-validator==3.8.1
+reasoner-validator==3.8.4
 pronto==2.5.3
 pygit2==1.10.0
 tabulate==0.9.0