diff --git a/src/vfb_connect/cross_server_tools.py b/src/vfb_connect/cross_server_tools.py index c76c7190..6a3b2f2b 100644 --- a/src/vfb_connect/cross_server_tools.py +++ b/src/vfb_connect/cross_server_tools.py @@ -823,21 +823,18 @@ def get_terms_by_xref(self, xrefs: iter, db='', summary=True, return_dataframe=T return self.neo_query_wrapper.get_terms_by_xref(xrefs, db=db, summary=summary, return_dataframe=False) - def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, return_just_ids=True, verbose=False): - """Map a list external DB IDs to VFB IDs + def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, return_just_ids=True, verbose=False, datasource_only=True): + """Map a list of external DB IDs to VFB short_form IDs - :param acc: An iterable (e.g. a list) of external IDs (e.g. neuprint bodyIDs). Can be in the form of 'db:acc' or just 'acc'. + :param acc: An iterable (e.g. a list) of external DB IDs. :param db: optional specify the VFB id (short_form) of an external DB to map to. (use get_dbs to find options) :param id_type: optionally specify an external id_type :param reverse_return: Boolean: Optional (see return) :param return_just_ids: Boolean: Optional (see return) - :param verbose: Optional. If `True`, prints the running query and found terms. Default `False`. :return: if `reverse_return` is False: - dict { acc : [{ db: : vfb_id : } + dict { acc : [{ db: : vfb_id : } Return if `reverse_return` is `True`: - dict { VFB_id : [{ db: : acc : } - if `return_just_ids` is `True`: - return just the VFB_ids in a list + dict { vfb_id : [{ db: : acc : } """ if isinstance(acc, str): if ':' in acc and db == '': @@ -852,19 +849,27 @@ def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, retur new_acc.append(temp_acc) else: new_acc.append(xref.split(':')[-1]) + else: + new_acc.append(xref) acc = new_acc if isinstance(acc, list) and all(isinstance(x, int) for x in acc): acc = [str(x) for x in acc] print(f"Converted to strings: {acc}") if verbose else None if db in VFB_DBS_2_SYMBOLS.keys(): db = VFB_DBS_2_SYMBOLS[db] - result = self.neo_query_wrapper.xref_2_vfb_id(acc=acc, db=db, id_type=id_type, reverse_return=reverse_return, verbose=verbose) + if db not in self.get_dbs(): + db = self.lookup_id(db) + result = self.neo_query_wrapper.xref_2_vfb_id(acc=acc, db=db, id_type=id_type, reverse_return=reverse_return, verbose=verbose, datasource_only=datasource_only) print(result) if verbose else None if return_just_ids & reverse_return: return [x.key for x in result] if return_just_ids and not reverse_return: id_list = [] for id in acc: + if id not in result.keys(): + print(f"No match found for {id} returning xref {':'.join([db,id])}") if verbose else None + id_list.append(":".join([db,id])) + continue id_list.append(result[id][0]['vfb_id']) # This takes the first match only if len(result[id]) > 1: print(f"Multiple matches found for {id}: {result[id]}") diff --git a/src/vfb_connect/neo/query_wrapper.py b/src/vfb_connect/neo/query_wrapper.py index f42cf7a3..42fc3862 100644 --- a/src/vfb_connect/neo/query_wrapper.py +++ b/src/vfb_connect/neo/query_wrapper.py @@ -445,18 +445,16 @@ def vfb_id_2_neuprint_bodyID(self, vfb_id, db=''): mapping = self.vfb_id_2_xrefs(vfb_id, db=db, reverse_return=True) return [int(k) for k, v in mapping.items()] - def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, verbose=False): - """Map a list external DB IDs to VFB IDs - - :param acc: An iterable (e.g. a list) of external IDs (e.g. neuprint bodyIDs). - :param db: optional specify the VFB id (short_form) of an external DB to map to. (use get_dbs to find options) - :param id_type: optionally specify an external id_type - :param reverse_return: Boolean: Optional (see return) - :return: if `reverse_return` is False: - dict { acc : [{ db: : vfb_id : } - Return if `reverse_return` is `True`: - dict { VFB_id : [{ db: : acc : } - """ + def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, verbose=False, datasource_only=False): + """Map a list of external DB IDs to VFB short_form IDs + + :param acc: An iterable (e.g., a list) of external DB IDs. + :param db: Optional. Specify the VFB ID (short_form) of an external database to map to. (Use `get_dbs()` to find options). + :param id_type: Optional. Specify an external ID type to filter the mapping results. + :param reverse_return: Optional. If `True`, returns the results in reverse order. Default is `False`. + :return: A dictionary of mappings from external DB IDs to VFB IDs. + :rtype: dict + """ if isinstance(acc, str): acc = [acc] match = "MATCH (s:Individual)<-[r:database_cross_reference]-(i:Entity) WHERE" @@ -468,6 +466,8 @@ def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, verbo conditions.append("s.short_form = '%s'" % db) if id_type: conditions.append("r.id_type = '%s'" % id_type) + if datasource_only: + conditions.append("s.is_data_source = [True]") condition_clauses = ' AND '.join(conditions) ret = "RETURN r.accession[0] as key, " \ "collect({ db: s.short_form, vfb_id: i.short_form }) as mapping" diff --git a/src/vfb_connect/schema/test/vfb_term_test.py b/src/vfb_connect/schema/test/vfb_term_test.py index 540f3937..cad1c763 100644 --- a/src/vfb_connect/schema/test/vfb_term_test.py +++ b/src/vfb_connect/schema/test/vfb_term_test.py @@ -502,5 +502,18 @@ def test_vfbterm_xref(self): print(self.vfb.xref_2_vfb_id(term.xref_id, return_just_ids=True, verbose=True)) self.assertEqual(self.vfb.xref_2_vfb_id(term.xref_id, return_just_ids=True)[0], term.id) + def test_load_synapes(self): + term = self.vfb.term('VFB_jrchk6dr') + print("got term ", term) + term.load_skeleton(template='JRC2018Unisex') + con = term.load_skeleton_synaptic_connections(verbose=True).to_dict('records') + print(con[0]) + self.assertGreater(len(con),10) + term = self.vfb.term('VFB_00102gjr') + print("got term ", term) + con = term.load_skeleton_synaptic_connections().to_dict('records') + print(con[0]) + self.assertGreater(len(con),10) + if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/src/vfb_connect/schema/vfb_term.py b/src/vfb_connect/schema/vfb_term.py index 8ac1959a..4d9bba65 100644 --- a/src/vfb_connect/schema/vfb_term.py +++ b/src/vfb_connect/schema/vfb_term.py @@ -12,6 +12,9 @@ import webbrowser +NEUPRINT_TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InZmYndvcmtzaG9wLm5ldXJvZmx5MjAyMEBnbWFpbC5jb20iLCJsZXZlbCI6Im5vYXV0aCIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg2Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tWXFDN21NRXd3TlEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQU1adXVjbU5zaXhXZDRhM0VyTTQ0ODBMa2IzNDdvUlpfUS9zOTYtYy9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3OTQwOTE4ODd9.ceg4mrj2o-aOhK0NHNGmBacg8R34PBPoLBwhCo4uOCQ' +CATMAID_TOKEN = None + neuron_containing_anatomy_tags = [ "Painted_domain", "Synaptic_neuropil_domain", @@ -2684,21 +2687,175 @@ def load_volume(self, template=None, verbose=False, query_by_label=True, force_r self._volume.label = self.name self._volume.id = self.id + def _extract_url_parameter(self, url, parameter='dataset', verbose=False): + """ + Extract the value of a query parameter from a URL. + """ + from urllib.parse import urlparse, parse_qs + # Parse the URL + parsed_url = urlparse(url) + print("Parsed URL: ", parsed_url) if verbose else None + # Extract the query parameters as a dictionary + query_params = parse_qs(parsed_url.query) + print("Query Params: ", query_params) if verbose else None + # Get the value of the 'dataset' parameter + param_value = query_params.get(parameter) + print("Param Value: ", param_value) if verbose else None + # Return the first value if the parameter exists + if param_value: + print(f"Found {parameter} parameter {param_value} of type {type(param_value)}") if verbose else None + if isinstance(param_value, list): + print("Returning first value: ", param_value[0]) if verbose else None + return param_value[0] + if isinstance(param_value, str): + print("Returning value: ", param_value) if verbose else None + return param_value + return None + def load_skeleton_synaptic_connections(self, template=None, verbose=False): """ Load the synaptic connections for the neuron's skeleton. """ + import flybrains + import navis + from navis import transforms + import pymaid + from navis.interfaces.neuprint import fetch_synapses, Client + + # Load the correct template template = self.get_default_template(template=template) + + # Check if skeleton is loaded, else load it if not self._skeleton or self._skeleton_template != template: - print(f"No skeleton loaded yet for {self.name} so loading...") if verbose else None - template = self.get_default_template(template=template) + if verbose: + print(f"No skeleton loaded yet for {self.name} so loading...") self.load_skeleton(template=template, verbose=verbose) + if self._skeleton: - print(f"Loading synaptic connections for {self.name}...") if verbose else None - xref = self.xref - # TODO: Load synaptic connections - # see https://github.com/navis-org/navis/blob/1eead062710af6adabc9e9c40196ad7be029cb52/navis/interfaces/neuprint.py#L491 - print("FEATURE NOT YET IMPLEMENTED") + if verbose: + print(f"Loading synaptic connections for {self.name}...") + + if 'catmaid' in self.xref_url: + if verbose: + print("Loading synaptic connections from CATMAID...") + + skid = int(self.xref_accession) + db = self.xref_id.split(':')[0] + server = "https://" + self.xref_url.split('://')[-1].split('/')[0] + pid = self._extract_url_parameter(self.xref_url, parameter='pid', verbose=verbose) + + # Connect to CATMAID + catmaid = pymaid.connect_catmaid(server=server, project_id=pid, api_token=CATMAID_TOKEN, max_threads=10) + connectors = pymaid.get_connectors([skid], remote_instance=catmaid) + + # Get connector details to link node_ids + links = pymaid.get_connector_details(connectors.connector_id) + connectors['node_id'] = connectors.connector_id.map(links.set_index('connector_id').node_id.to_dict()) + + # Perform alignment if needed + available_transforms = transforms.registry.summary() + source, target = self.determine_transform(db, available_transforms, template) + + if source and target: + if verbose: + print(f"Transforming from {source} to {target}...") + aligned_connectors = navis.xform_brain(connectors, source=source, target=target) + else: + aligned_connectors = connectors + + # Attach connectors to the skeleton + aligned_connectors['type'] = aligned_connectors['type'].str.lower() # Ensure 'type' is lowercase + self._skeleton._set_connectors(aligned_connectors) + if verbose: + print(f"Connectors set for {self.name}") + return aligned_connectors + + elif 'neuprint' in self.xref_url: + if verbose: + print("Loading synaptic connections from neuprint...") + + bodyId = int(self.xref_accession) + db = self.xref_id.split(':')[0] + server = self.xref_url.split('://')[-1].split('/')[0] + ds = self._extract_url_parameter(self.xref_url, parameter='dataset', verbose=verbose) + + # Connect to neuprint + client = Client(server=server, dataset=ds, token=NEUPRINT_TOKEN) + connectors = fetch_synapses(bodyId, client=client) + + # Get transforms if needed + available_transforms = transforms.registry.summary() + source, target = self.determine_transform(ds, available_transforms, template) + + if source and target: + if verbose: + print(f"Transforming from {source} to {target}...") + aligned_connectors = navis.xform_brain(connectors, source=source, target=target) + else: + aligned_connectors = connectors + + # Ensure 'connector_id' exists and bodyId gets mapped to 'id' + if 'connector_id' not in aligned_connectors.columns: + aligned_connectors['connector_id'] = aligned_connectors.index + if 'id' not in aligned_connectors.columns: + acc = aligned_connectors.bodyId.to_list() + vfb_ids = self.vfb.xref_2_vfb_id(acc=acc, db=db, return_just_ids=True, verbose=verbose) + if len(vfb_ids) != len(acc): + print("Some bodyIds could not be mapped to VFB IDs.") + aligned_connectors['id'] = vfb_ids + + # Attach connectors to the skeleton + aligned_connectors['type'] = aligned_connectors['type'].str.lower() # Ensure 'type' is lowercase + if verbose: + print(f"Setting connectors for {self.name}...") + if verbose: + print("Connectors: ", aligned_connectors) + self._skeleton._set_connectors(aligned_connectors) + if verbose: + print(f"Connectors set for {self.name}") + return aligned_connectors + + else: + if verbose: + print("Unknown data source for synaptic connections.") + return None + + else: + if verbose: + print("Skeleton not loaded; cannot load synaptic connections.") + return None + + def determine_transform(self, name, available_transforms, template): + """ + Determine the source and target brain space for the transformation. + """ + source = None + target = None + + if name.upper() in available_transforms.source.to_list(): + source = name.upper() + elif name.lower() in available_transforms.source.to_list(): + source = name.lower() + + if self.vfb.lookup_name(template) in available_transforms.target.to_list(): + target = template + + if not source or not target: + if 'fafb' in name.lower(): + source = 'FAFB' + target = 'JRC2018U' + elif 'fanc' in name.lower(): + source = 'FANC' + target = 'JRC2018U' + elif 'hemibrain' in name.lower(): + source = 'hemibrain' + target = 'JRC2018U' + elif 'optic-lobe' in name.lower(): + source = 'JRCFIB2022Mraw' + target = 'JRC2018U' + + return source, target + def get_default_template(self, template=None):