Merge pull request #108 from VirtualFlyBrain/master

Bringing upto latest so tests work

.github/workflows/publish-to-pypi.yml

@@ -7,13 +7,13 @@ on:
 jobs:
   build-n-publish:
     name: Build and publish Python 🐍 distributions 📦 to PyPI
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@master
-    - name: Set up Python 3.7
+    - name: Set up Python 3.9
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: 3.9
     - name: Install wheel and setuptools
       run: >-
         python -m

.github/workflows/test_notebooks.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Install dependencies
         run: |
@@ -20,7 +20,7 @@ jobs:
 
       - name: Execute the notebooks
         run: |
-          jupyter nbconvert --to notebook --execute docs/source/tutorials/*.ipynb
+          jupyter nbconvert --to notebook --execute --ExecutePreprocessor.timeout=600 docs/source/tutorials/*.ipynb
 
       - uses: actions/upload-artifact@v2
         with:

.readthedocs.yaml

@@ -0,0 +1,15 @@
+version: 2
+
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.8"
+
+sphinx:
+   configuration: docs/source/conf.py
+
+python:
+  install:
+    - requirements: docs/requirements.txt
+    - method: pip
+      path: .

doc/vfb_connect/cross_server_tools.md

@@ -53,7 +53,7 @@ Classes
         `classification = 'class expression' e.g. "'Kenyon cell'" or "'neuron' that overlaps 'lateral horn'".
 
     `get_neurons_upstream_of(self, neuron, weight, classification=None, query_by_label=True, return_dataframe=True)`
-    :   Get all neurons downstream of individual `neuron` (short_form if query_by_label=False, otherwise label)
+    :   Get all neurons upstream of individual `neuron` (short_form if query_by_label=False, otherwise label)
         with connection strength > threshold.  Optionally restrict target neurons to those specified by
         `classification = 'class expression' e.g. "'Kenyon cell'" or "'neuron' that overlaps 'lateral horn'".
 
@@ -74,4 +74,13 @@ Classes
     `get_vfb_link(self, short_forms: <built-in function iter>, template)`
     :   Takes a list of VFB IDs (short_forms) and the name (label) of a template.
         Returns a link to VFB loading all available images
-        of neurons on that template.
+        of neurons on that template.
+
+    `get_gene_function_filters(self)`
+    :   Returns a list of all unique gene function labels in the database in alphabetical order.
+
+    `get_transcriptomic_profile(self, cell_type, gene_type=False, return_dataframe=True)`
+    :   Takes a cell_type (name, ID or symbol) from the Drosophila anatomy ontology as a String.
+        Returns transcriptomics data for clusters annotated as the given cell_type (and subtypes).
+        Can optionally be restricted to genes of a particular function by specifying gene_type as a String.
+        Available gene functions can be retrieved by running get_gene_function_filters().

docs/requirements.txt

@@ -12,3 +12,4 @@ docutils
 myst-parser
 sphinx_rtd_theme
 
+numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability

docs/source/API_reference.rst

@@ -96,6 +96,13 @@ Connectivity queries
 .. autofunction:: vfb_connect.cross_server_tools.VfbConnect.get_neurons_downstream_of
 .. autofunction:: vfb_connect.cross_server_tools.VfbConnect.get_neurons_upstream_of
 
+Transcriptomics Queries
+~~~~~~~~~~~~~~~~~~~~~~~
+
+**Methods for retrieving transcriptomics data**
+
+.. autofunction:: vfb_connect.cross_server_tools.VfbConnect.get_transcriptomic_profile
+.. autofunction:: vfb_connect.cross_server_tools.VfbConnect.get_gene_function_filters
 
 VFB link generation
 ~~~~~~~~~~~~~~~~~~~~

src/vfb_connect/cross_server_tools.py

@@ -353,10 +353,73 @@ def get_images_by_type(self, class_expression, template, image_folder,
                                                  image_type=image_type,
                                                  stomp=stomp)
 
+    def get_gene_function_filters(self):
+        """Get list of all gene function labels.
 
+        :return: List of unique gene function labels in alphabetical order.
+        :rtype: list
+        """
+        query = ("MATCH (g:Gene) RETURN DISTINCT apoc.coll.subtract(labels(g), "
+                 "['Class', 'Entity', 'hasScRNAseq', 'Feature', 'Gene']) AS gene_labels")
+        result = self.neo_query_wrapper._query(query)
+        labels = []
+        for r in result:
+            labels.extend(r['gene_labels'])
+        labels = sorted(list(set(labels)))
+        return labels
+
+    def get_transcriptomic_profile(self, cell_type, gene_type=False, return_dataframe=True):
+        """Get gene expression data for a given cell_type.
+
+        Returns a DataFrame of gene expression data for clusters of cells annotated as cell_type (or subtypes).
+        Can optionally restrict to a gene_type - these can be retrieved by running get_gene_function_filters.
+        If no data is found, returns False.
+
+        :param cell_type: The ID, name or symbol of a class in the Drosophila Anatomy Ontology (FBbt).
+        :param gene_type: Optional. A gene function label - these can be retrieved by running get_gene_function_filters().
+        :return: DataFrame with gene expression data for clusters of cells annotated as cell_type (or subtypes).
+        :rtype: DataFrame
+        """
 
+        try:
+            cell_type_short_form = self.lookup[cell_type].replace(':', '_')
+        except KeyError:
+            if cell_type.replace('_', ':') in self.lookup.values():
+                cell_type_short_form = cell_type.replace(':', '_')
+            else:
+                raise KeyError("cell_type must be a valid ID, label or symbol from the Drosophila Anatomy Ontology")
 
+        if not cell_type_short_form.startswith('FBbt'):
+            raise KeyError("cell_type must be a valid ID, label or symbol from the Drosophila Anatomy Ontology")
 
-
-
+        if gene_type:
+            if gene_type not in self.get_gene_function_filters():
+                raise KeyError("gene_type must be a valid gene function label, try running get_gene_function_filters()")
+            else:
+                gene_label = ':' + gene_type
+        else:
+            gene_label = ''
+
+        query = ("MATCH (g:Gene:Class%s)<-[e:expresses]-(clus:Cluster:Individual)-"
+                 "[:composed_primarily_of]->(c2:Class)-[:SUBCLASSOF*0..]->(c1:Neuron:Class) "
+                 "WHERE c1.short_form = '%s' "
+                 "MATCH (clus)-[:part_of]->()-[:has_part]->(sa:Sample:Individual) "
+                 "OPTIONAL MATCH (sa)-[:part_of]->(sex:Class) "
+                 "WHERE sex.short_form IN ['FBbt_00007011', 'FBbt_00007004'] "
+                 "OPTIONAL MATCH (sa)-[:overlaps]->(tis:Class:Anatomy) "
+                 "OPTIONAL MATCH (clus)-[:has_source]->(ds:DataSet:Individual) "
+                 "OPTIONAL MATCH (ds)-[:has_reference]->(p:pub:Individual) "
+                 "OPTIONAL MATCH (ds)-[dbx:database_cross_reference]->(s:Site:Individual) "
+                 "RETURN DISTINCT c2.label AS cell_type, c2.short_form AS cell_type_id, "
+                 "sex.label AS sample_sex, COLLECT(tis.label) AS sample_tissue, "
+                 "p.miniref[0] as ref, g.label AS gene, g.short_form AS gene_id, "
+                 "apoc.coll.subtract(labels(g), ['Class', 'Entity', 'hasScRNAseq', 'Feature', 'Gene']) AS function, "
+                 "e.expression_extent[0] as extent, toFloat(e.expression_level[0]) as level order by cell_type, g.label"
+                 % (gene_label, cell_type_short_form))
+        r = self.nc.commit_list([query])
+        dc = dict_cursor(r)
+        if return_dataframe:
+            return pd.DataFrame.from_records(dc)
+        else:
+            return dc
 

src/vfb_connect/neo/neo4j_tools.py

@@ -89,12 +89,12 @@ def __init__(self, endpoint = get_default_servers()['neo_endpoint'],
         self.base_uri = endpoint
         self.usr = usr
         self.pwd = pwd
-        self.commit = "/db/data/transaction/commit"
-        self.headers = {}
+        self.commit = "/db/neo4j/tx/commit"
+        self.headers = {'Content-type': 'application/json'}
         if not self.test_connection():
-            print("Connecting to Neo4j v4+")
-            self.commit = "/db/neo4j/tx/commit"
-            self.headers = {'Content-type': 'application/json'}
+            print("Falling back to Neo4j v3 connection")
+            self.commit = "/db/data/transaction/commit"
+            self.headers = {}
             self.test_connection()
 
     def commit_list(self, statements, return_graphs=False):

src/vfb_connect/neo/query_wrapper.py

@@ -410,18 +410,18 @@ def _get_anatomical_individual_TermInfo_by_type(self, classification, summary=Fa
         typ = 'Get JSON for Individual:Anatomy_by_type'
         qs = Template(self.queries[typ]).substitute(ID=classification)
         if summary:
-            return self._termInfo_2_summary(self._query(qs), typ='Get JSON for Individual:Anatomy')
+            return self._termInfo_2_summary(self._query(qs), typ='Get JSON for Individual')
         else:
             return self._query(qs)
 
     def _termInfo_2_summary(self, TermInfo, typ):
         # type_2_summary = {
-        #     'Get JSON for Individual:Anatomy': '_populate_instance_summary_tab',
+        #     'Get JSON for Individual': '_populate_instance_summary_tab',
         #     'Get JSON for Class': '_populate_anatomical_entity_summary',
         # }
         dc = []
         for r in TermInfo:
-            if typ == 'Get JSON for Individual:Anatomy':
+            if typ == 'Get JSON for Individual':
                 dc.append(_populate_instance_summary_tab(r))
             elif typ == 'Get JSON for Class':
                 dc.append(_populate_anatomical_entity_summary(r))
@@ -437,7 +437,7 @@ def get_anatomical_individual_TermInfo(self, short_forms, summary=False):
         :param summary: Optional.  Returns summary reports if `True`. Default `False`
         :rtype: list of VFB_json or summary_report_json
         """
-        return self._get_TermInfo(short_forms, typ='Get JSON for Individual:Anatomy', summary=summary)
+        return self._get_TermInfo(short_forms, typ='Get JSON for Individual', summary=summary)
 
     def get_type_TermInfo(self, short_forms, summary=False):
         """
@@ -467,4 +467,4 @@ def get_template_TermInfo(self, short_forms):
         :param summary: Optional.  Returns summary reports if `True`. Default `False`
         :rtype: list of VFB_json or summary_report_json
         """
-        return self._get_TermInfo(short_forms, typ='Get JSON for Template')
+        return self._get_TermInfo(short_forms, typ='Get JSON for Template')