Skip to content

Commit

Permalink
synapsis basic
Browse files Browse the repository at this point in the history
  • Loading branch information
@Robbie1977
Robbie1977 committed Sep 5, 2024
1 parent bc83611 commit 2c767d3
Show file tree
Hide file tree
Showing 3 changed files with 135 additions and 125 deletions.
22 changes: 12 additions & 10 deletions src/vfb_connect/cross_server_tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -823,21 +823,18 @@ def get_terms_by_xref(self, xrefs: iter, db='', summary=True, return_dataframe=T

return self.neo_query_wrapper.get_terms_by_xref(xrefs, db=db, summary=summary, return_dataframe=False)

def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, return_just_ids=True, verbose=False):
"""Map a list external DB IDs to VFB IDs
def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, return_just_ids=True, verbose=False, datasource_only=True):
"""Map a list of external DB IDs to VFB short_form IDs
:param acc: An iterable (e.g. a list) of external IDs (e.g. neuprint bodyIDs). Can be in the form of 'db:acc' or just 'acc'.
:param acc: An iterable (e.g. a list) of external DB IDs.
:param db: optional specify the VFB id (short_form) of an external DB to map to. (use get_dbs to find options)
:param id_type: optionally specify an external id_type
:param reverse_return: Boolean: Optional (see return)
:param return_just_ids: Boolean: Optional (see return)
:param verbose: Optional. If `True`, prints the running query and found terms. Default `False`.
:return: if `reverse_return` is False:
dict { acc : [{ db: <db> : vfb_id : <VFB_id> }
dict { acc : [{ db: <db> : vfb_id : <VFB_id> }
Return if `reverse_return` is `True`:
dict { VFB_id : [{ db: <db> : acc : <acc> }
if `return_just_ids` is `True`:
return just the VFB_ids in a list
dict { vfb_id : [{ db: <db> : acc : <acc> }
"""
if isinstance(acc, str):
if ':' in acc and db == '':
Expand All @@ -852,18 +849,23 @@ def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, retur
new_acc.append(temp_acc)
else:
new_acc.append(xref.split(':')[-1])
else:
new_acc.append(xref)
acc = new_acc
if db in VFB_DBS_2_SYMBOLS.keys():
db = VFB_DBS_2_SYMBOLS[db]
result = self.neo_query_wrapper.xref_2_vfb_id(acc=acc, db=db, id_type=id_type, reverse_return=reverse_return, verbose=verbose)
if db not in self.get_dbs():
db = self.lookup_id(db)
result = self.neo_query_wrapper.xref_2_vfb_id(acc=acc, db=db, id_type=id_type, reverse_return=reverse_return, verbose=verbose, datasource_only=datasource_only)
print(result) if verbose else None
if return_just_ids & reverse_return:
return [x.key for x in result]
if return_just_ids and not reverse_return:
id_list = []
for id in acc:
if id not in result.keys():
print(f"No match found for {id}")
print(f"No match found for {id} returning xref {":".join([db,id])}") if verbose else None
id_list.append(":".join([db,id]))
continue
id_list.append(result[id][0]['vfb_id']) # This takes the first match only
if len(result[id]) > 1:
Expand Down
24 changes: 12 additions & 12 deletions src/vfb_connect/neo/query_wrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,18 +444,16 @@ def vfb_id_2_neuprint_bodyID(self, vfb_id, db=''):
mapping = self.vfb_id_2_xrefs(vfb_id, db=db, reverse_return=True)
return [int(k) for k, v in mapping.items()]

def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, verbose=False):
"""Map a list external DB IDs to VFB IDs
:param acc: An iterable (e.g. a list) of external IDs (e.g. neuprint bodyIDs).
:param db: optional specify the VFB id (short_form) of an external DB to map to. (use get_dbs to find options)
:param id_type: optionally specify an external id_type
:param reverse_return: Boolean: Optional (see return)
:return: if `reverse_return` is False:
dict { acc : [{ db: <db> : vfb_id : <VFB_id> }
Return if `reverse_return` is `True`:
dict { VFB_id : [{ db: <db> : acc : <acc> }
"""
def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, verbose=False, datasource_only=False):
"""Map a list of external DB IDs to VFB short_form IDs
:param acc: An iterable (e.g., a list) of external DB IDs.
:param db: Optional. Specify the VFB ID (short_form) of an external database to map to. (Use `get_dbs()` to find options).
:param id_type: Optional. Specify an external ID type to filter the mapping results.
:param reverse_return: Optional. If `True`, returns the results in reverse order. Default is `False`.
:return: A dictionary of mappings from external DB IDs to VFB IDs.
:rtype: dict
"""
if isinstance(acc, str):
acc = [acc]
match = "MATCH (s:Individual)<-[r:database_cross_reference]-(i:Entity) WHERE"
Expand All @@ -467,6 +465,8 @@ def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, verbo
conditions.append("s.short_form = '%s'" % db)
if id_type:
conditions.append("r.id_type = '%s'" % id_type)
if datasource_only:
conditions.append("s.is_data_source = [True]")
condition_clauses = ' AND '.join(conditions)
ret = "RETURN r.accession[0] as key, " \
"collect({ db: s.short_form, vfb_id: i.short_form }) as mapping"
Expand Down
214 changes: 111 additions & 103 deletions src/vfb_connect/schema/vfb_term.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1572,6 +1572,7 @@ def __init__(self, id=None, term: Optional[Term] = None, related_terms: Optional
from vfb_connect import vfb
self.vfb = vfb
self.debug = verbose
self._return_type = self.vfb._return_type # Default to global version but can be set to id, name (list) or full (VFBTerms)
if id is not None:
if isinstance(id, list):
id = id[0]
Expand Down Expand Up @@ -2719,132 +2720,139 @@ def load_skeleton_synaptic_connections(self, template=None, verbose=False):
import flybrains
import navis
from navis import transforms
import pymaid
from navis.interfaces.neuprint import fetch_synapses, Client

# Load the correct template
template = self.get_default_template(template=template)

# Check if skeleton is loaded, else load it
if not self._skeleton or self._skeleton_template != template:
print(f"No skeleton loaded yet for {self.name} so loading...") if verbose else None
template = self.get_default_template(template=template)
if verbose:
print(f"No skeleton loaded yet for {self.name} so loading...")
self.load_skeleton(template=template, verbose=verbose)

if self._skeleton:
print(f"Loading synaptic connections for {self.name}...") if verbose else None
if verbose:
print(f"Loading synaptic connections for {self.name}...")

if 'catmaid' in self.xref_url:
print("Loading synaptic connections from CATMAID...") if verbose else None
skid = int(self.xref_id.split(':')[-1])
if verbose:
print("Loading synaptic connections from CATMAID...")

skid = int(self.xref_accession)
db = self.xref_id.split(':')[0]
print("CATMAID skeleton ID: ", skid) if verbose else None
server = "https://" + self.xref_url.split('://')[-1].split('/')[0]
print("CATMAID Server: ", server) if verbose else None
pid = self._extract_url_parameter(self.xref_url, parameter='pid', verbose=verbose)
import pymaid

# Connect to CATMAID
catmaid = pymaid.connect_catmaid(server=server, project_id=pid, api_token=CATMAID_TOKEN, max_threads=10)
connectors = pymaid.get_connectors([skid],remote_instance=catmaid)
print("Connectors: ", connectors) if verbose else None
# Check available transforms
connectors = pymaid.get_connectors([skid], remote_instance=catmaid)

# Get connector details to link node_ids
links = pymaid.get_connector_details(connectors.connector_id)
connectors['node_id'] = connectors.connector_id.map(links.set_index('connector_id').node_id.to_dict())

# Perform alignment if needed
available_transforms = transforms.registry.summary()
print("Available transforms: ", available_transforms) if verbose else None
target = None
source = None
name = db.split('_')[-1].upper()
if not 'L1EM' in name:
if name in available_transforms.source.to_list():
source = name
if name.lower() in available_transforms.source.to_list():
source = name.lower()
if self.vfb.lookup_name(self._skeleton_template) in available_transforms.target.to_list():
target = self._skeleton_template
if not target or not source:
print("falling back to manual transforms...") if verbose else None
if 'fafb' in name.lower():
source = 'FAFB'
target = 'JRC2018U'
JRC2018U = flybrains.JRC2018U
FAFB14 = flybrains.FAFB
elif 'fanc' in name.lower():
source = 'FANC'
target = 'JRC2018U'
FANC = flybrains.FANC
JRC2018U = flybrains.JRC2018U
else:
print("No transform available for ", db)
print("Transforming from ", source, " to ", target) if verbose else None
source, target = self.determine_transform(db, available_transforms, template)

if source and target:
if verbose:
print(f"Transforming from {source} to {target}...")
aligned_connectors = navis.xform_brain(connectors, source=source, target=target)
else:
aligned_connectors = connectors
print("FULL FEATURE NOT YET IMPLEMENTED")

# Attach connectors to the skeleton
aligned_connectors['type'] = aligned_connectors['type'].str.lower() # Ensure 'type' is lowercase
self._skeleton._set_connectors(aligned_connectors)
if verbose:
print(f"Connectors set for {self.name}")
return aligned_connectors

if 'neuprint' in self.xref_url:
print("Loading synaptic connections from neuprint...") if verbose else None
from navis.interfaces.neuprint import fetch_synapses, Client


bodyId = int(self.xref_id.split(':')[-1])

elif 'neuprint' in self.xref_url:
if verbose:
print("Loading synaptic connections from neuprint...")

bodyId = int(self.xref_accession)
db = self.xref_id.split(':')[0]
print("Neuprint bodyId: ", bodyId) if verbose else None
server = self.xref_url.split('://')[-1].split('/')[0]
print("Neuprint Server: ", server) if verbose else None
ds = self._extract_url_parameter(self.xref_url, parameter='dataset', verbose=verbose)
print("Neuprint Dataset: ", ds) if verbose else None

# Connect to neuprint
client = Client(server=server, dataset=ds, token=NEUPRINT_TOKEN)
connectors = fetch_synapses(bodyId, client=client)
print("Connectors: ", connectors) if verbose else None
# Check available transforms

# Get transforms if needed
available_transforms = transforms.registry.summary()
print("Available transforms: ", available_transforms) if verbose else None
target = None
source = None
if '%3A' in ds:
ds = ds.replace('%3A', ':')
name = ds.split(':')[0].upper()
if name in available_transforms.source.to_list():
source = name
if name.lower() in available_transforms.source.to_list():
source = name.lower()
if self.vfb.lookup_name(self._skeleton_template) in available_transforms.target.to_list():
target = self._skeleton_template
if not target or not source:
print("falling back to manual transforms...") if verbose else None
if 'manc' in name.lower():
source = 'MANC'
target = 'JRCVNC2018U'
elif 'hemibrain' in name.lower():
source = 'hemibrain'
target = 'JRC2018U'
elif 'optic-lobe' in name.lower():
source = 'JRCFIB2022Mraw'
target = 'JRC2018U'
else:
print("No transform available for ", ds)
print("Transforming from ", source, " to ", target) if verbose else None
aligned_connectors = navis.xform_brain(connectors, source=source, target=target)

print("Aligned Connectors before: ", aligned_connectors) if verbose else None
if len(aligned_connectors) > 0:
# Adding 'connector_id' from index if not already present
if 'connector_id' not in aligned_connectors.columns:
aligned_connectors['connector_id'] = aligned_connectors.index

# Convert 'bodyId' to 'id' using vfb.xref_2_vfb_id method
source, target = self.determine_transform(ds, available_transforms, template)

if source and target:
if verbose:
print(f"Transforming from {source} to {target}...")
aligned_connectors = navis.xform_brain(connectors, source=source, target=target)
else:
aligned_connectors = connectors

# Ensure 'connector_id' exists and bodyId gets mapped to 'id'
if 'connector_id' not in aligned_connectors.columns:
aligned_connectors['connector_id'] = aligned_connectors.index
if 'id' not in aligned_connectors.columns:
acc = aligned_connectors.bodyId.to_list()
print("Body IDs: ", acc) if verbose else None
print(f"type(acc): {type(acc)}") if verbose else None
print(f"acc[0]: {type(acc[0])}") if verbose else None
print("DB: ", db) if verbose else None
vfb_ids = self.vfb.xref_2_vfb_id(acc=acc, db=db, return_just_ids=True, verbose=verbose)
print("VFB IDs: ", vfb_ids) if verbose else None
if 'id' not in aligned_connectors.columns and vfb_ids:
print("Adding 'id' column to aligned_connectors") if verbose else None
aligned_connectors['id'] = vfb_ids
print("Aligned Connectors after: ", aligned_connectors) if verbose else None
print("FULL FEATURE NOT YET IMPLEMENTED")
return aligned_connectors
self._skeleton._set_connectors(aligned_connectors)
print("Connectors set for ", self.name) if verbose else None
print(self._skeleton.connectors) if verbose else None
return
if len(vfb_ids) != len(acc):
print("Some bodyIds could not be mapped to VFB IDs.")
aligned_connectors['id'] = vfb_ids

# Attach connectors to the skeleton
aligned_connectors['type'] = aligned_connectors['type'].str.lower() # Ensure 'type' is lowercase
self._skeleton._set_connectors(aligned_connectors)
if verbose:
print(f"Connectors set for {self.name}")
return aligned_connectors

else:
if verbose:
print("Unknown data source for synaptic connections.")
return None

else:
if verbose:
print("Skeleton not loaded; cannot load synaptic connections.")
return None

def determine_transform(self, name, available_transforms, template):
"""
Determine the source and target brain space for the transformation.
"""
source = None
target = None

if name.upper() in available_transforms.source.to_list():
source = name.upper()
elif name.lower() in available_transforms.source.to_list():
source = name.lower()

if self.vfb.lookup_name(template) in available_transforms.target.to_list():
target = template

if not source or not target:
if 'fafb' in name.lower():
source = 'FAFB'
target = 'JRC2018U'
elif 'fanc' in name.lower():
source = 'FANC'
target = 'JRC2018U'
elif 'hemibrain' in name.lower():
source = 'hemibrain'
target = 'JRC2018U'
elif 'optic-lobe' in name.lower():
source = 'JRCFIB2022Mraw'
target = 'JRC2018U'

return source, target

# TODO: Load synaptic connections
# see https://github.com/navis-org/navis/blob/1eead062710af6adabc9e9c40196ad7be029cb52/navis/interfaces/neuprint.py#L491
print("FEATURE NOT YET IMPLEMENTED")


def get_default_template(self, template=None):
Expand Down

0 comments on commit 2c767d3

Please sign in to comment.