Merge pull request #1838 from AllenInstitute/rc/2.6.0

rc/2.6.0

CHANGELOG.md

@@ -1,6 +1,11 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.6.0] = 2021-02-05
+- Adds ability to write and read behavior only NWB files
+- Adds eye tracking ellipse fits and metadata as new NWB data stream
+- OPhys Behavior data retrieval methods no longer depend on ROIs being ordered identically in different files.
+
 ## [2.5.0] = 2021-01-29
 -  Adds unfiltered running speed as new data stream
 -  run_demixing gracefully ignores any ROIs that are not in the input trace file

allensdk/__init__.py

@@ -37,7 +37,8 @@
 
 
 
-__version__ = '2.5.0'
+
+__version__ = '2.6.0'
 
 
 try:

allensdk/brain_observatory/behavior/behavior_ophys_session.py

@@ -437,6 +437,16 @@ def eye_tracking(self) -> pd.DataFrame:
     def eye_tracking(self, value):
         self._eye_tracking = value
 
+    @property
+    def eye_tracking_rig_geometry(self) -> dict:
+        """Get the eye tracking rig geometry associated with an ophys experiment"""
+        return self.api.get_eye_tracking_rig_geometry()
+
+    @property
+    def eye_gaze_mapping_file_path(self) -> str:
+        """Get h5 filepath containing eye gaze behavior of the experiment's subject"""
+        return self.api.get_eye_gaze_mapping_file_path()
+
     def cache_clear(self) -> None:
         """Convenience method to clear the api cache, if applicable."""
         try:

allensdk/brain_observatory/behavior/behavior_session.py

@@ -5,7 +5,7 @@
 import inspect
 
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
-    BehaviorLimsApi)
+    BehaviorLimsApi, BehaviorNwbApi)
 from allensdk.brain_observatory.behavior.session_apis.abcs import BehaviorBase
 from allensdk.brain_observatory.running_speed import RunningSpeed
 
@@ -36,30 +36,30 @@ def from_lims(cls, behavior_session_id: int) -> "BehaviorSession":
     @classmethod
     def from_nwb_path(
             cls, nwb_path: str, **api_kwargs: Any) -> "BehaviorSession":
-        return NotImplementedError
+        return cls(api=BehaviorNwbApi.from_path(path=nwb_path, **api_kwargs))
 
     @property
     def behavior_session_id(self) -> int:
         """Unique identifier for this experimental session.
         :rtype: int
         """
-        return self.api.behavior_session_id
+        return self.api.get_behavior_session_id()
 
     @property
     def ophys_session_id(self) -> Optional[int]:
         """The unique identifier for the ophys session associated
         with this behavior session (if one exists)
         :rtype: int
         """
-        return self.api.ophys_session_id
+        return self.api.get_ophys_session_id()
 
     @property
     def ophys_experiment_ids(self) -> Optional[List[int]]:
         """The unique identifiers for the ophys experiment(s) associated
         with this behavior session (if one exists)
         :rtype: int
         """
-        return self.api.ophys_experiment_ids
+        return self.api.get_ophys_experiment_ids()
 
     @property
     def licks(self) -> pd.DataFrame:

allensdk/brain_observatory/behavior/metadata_processing.py

@@ -26,6 +26,11 @@
     "previously trained, but with the lick-response "
     "sensor withdrawn (passive/open loop mode)."
 )
+TRAINING_DESCRIPTION = (
+    "A training session where a mouse performs a visual change detection task "
+    "with a set of natural scenes. Successfully completing the task delivers "
+    "a water reward via a lick spout."
+)
 
 
 def get_expt_description(session_type: str) -> str:
@@ -55,8 +60,11 @@ def get_expt_description(session_type: str) -> str:
     ophys_4_6 = dict.fromkeys(["OPHYS_4", "OPHYS_6"], OPHYS_4_6_DESCRIPTION)
     ophys_2_5 = {"OPHYS_2": OPHYS_2_DESCRIPTION,
                  "OPHYS_5": OPHYS_5_DESCRIPTION}
+    training = dict.fromkeys(
+        ["TRAINING_1", "TRAINING_2", "TRAINING_3", "TRAINING_4", "TRAINING_5"],
+        TRAINING_DESCRIPTION)
 
-    expt_description_dict = {**ophys_1_3, **ophys_2_5, **ophys_4_6}
+    expt_description_dict = {**ophys_1_3, **ophys_2_5, **ophys_4_6, **training}
 
     # Session type string will look something like: OPHYS_4_images_A
     truncated_session_type = "_".join(session_type.split("_")[:2])

allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py

@@ -184,7 +184,6 @@ def _get_behavior_summary_table(self,
             SELECT
                 bs.id AS behavior_session_id,
                 bs.ophys_session_id,
-                bs.behavior_training_id,
                 equipment.name as equipment_name,
                 bs.date_of_acquisition,
                 d.id as donor_id,

allensdk/brain_observatory/behavior/schemas.py

@@ -61,6 +61,10 @@ class SubjectMetadataSchema(RaisingSchema):
 class BehaviorMetadataSchema(RaisingSchema):
     """This schema contains metadata pertaining to behavior.
     """
+    neurodata_type = 'BehaviorMetadata'
+    neurodata_type_inc = 'LabMetaData'
+    neurodata_doc = "Metadata for behavior and behavior + ophys experiments"
+    neurodata_skip = {"experiment_datetime"}
 
     behavior_session_uuid = fields.UUID(
         doc='MTrain record for session, also called foraging_id',
@@ -71,6 +75,21 @@ class BehaviorMetadataSchema(RaisingSchema):
              'visual_stimulus from the monitor'),
         required=True,
     )
+    session_type = fields.String(
+        doc='Experimental session description',
+        allow_none=True,
+        required=True,
+    )
+    # 'experiment_datetime' will be stored in
+    # pynwb NWBFile 'session_start_time' attr
+    experiment_datetime = fields.DateTime(
+        doc='Date of the experiment (UTC, as string)',
+        required=True,
+    )
+    rig_name = fields.String(
+        doc='Name of behavior or optical physiology experiment rig',
+        required=True,
+    )
 
 
 class NwbOphysMetadataSchema(RaisingSchema):
@@ -123,10 +142,6 @@ class OphysMetadataSchema(NwbOphysMetadataSchema):
         doc='Id for this ophys session',
         required=True,
     )
-    rig_name = fields.String(
-        doc='Name of optical physiology experiment rig',
-        required=True,
-    )
     field_of_view_width = fields.Int(
         doc='Width of optical physiology imaging plane in pixels',
         required=True,
@@ -143,7 +158,7 @@ class OphysBehaviorMetadataSchema(BehaviorMetadataSchema, OphysMetadataSchema):
     """
 
     neurodata_type = 'OphysBehaviorMetadata'
-    neurodata_type_inc = 'LabMetaData'
+    neurodata_type_inc = 'BehaviorMetadata'
     neurodata_doc = "Metadata for behavior + ophys experiments"
     # Fields to skip converting to extension
     # They already exist as attributes for the following pyNWB classes:
@@ -152,18 +167,6 @@ class OphysBehaviorMetadataSchema(BehaviorMetadataSchema, OphysMetadataSchema):
                       "targeted_structure", "experiment_datetime",
                       "ophys_frame_rate"}
 
-    session_type = fields.String(
-        doc='Experimental session description',
-        allow_none=True,
-        required=True,
-    )
-    # 'experiment_datetime' will be stored in
-    # pynwb NWBFile 'session_start_time' attr
-    experiment_datetime = fields.DateTime(
-        doc='Date of the experiment (UTC, as string)',
-        required=True,
-    )
-
 
 class CompleteOphysBehaviorMetadataSchema(OphysBehaviorMetadataSchema,
                                           SubjectMetadataSchema):
@@ -230,3 +233,54 @@ class BehaviorTaskParametersSchema(RaisingSchema):
         doc='Total number of stimuli frames',
         required=True,
     )
+
+
+class EyeTrackingRigGeometry(RaisingSchema):
+    """Eye tracking rig geometry"""
+    values = fields.Float(
+        doc='position/rotation with respect to (x, y, z)',
+        required=True,
+        shape=(3,)
+    )
+    unit_of_measurement = fields.Str(
+        doc='Unit of measurement for the data',
+        required=True
+    )
+
+
+class OphysEyeTrackingRigMetadataSchema(RaisingSchema):
+    """This schema encompasses metadata for ophys experiment rig
+    """
+    neurodata_type = 'OphysEyeTrackingRigMetadata'
+    neurodata_type_inc = 'NWBDataInterface'
+    neurodata_doc = "Metadata for ophys experiment rig"
+
+    equipment = fields.Str(
+        doc='Description of rig',
+        required=True
+    )
+    monitor_position = fields.Nested(
+        EyeTrackingRigGeometry,
+        doc='position of monitor (x, y, z)',
+        required=True
+    )
+    camera_position = fields.Nested(
+        EyeTrackingRigGeometry,
+        doc='position of camera (x, y, z)',
+        required=True
+    )
+    led_position = fields.Nested(
+        EyeTrackingRigGeometry,
+        doc='position of LED (x, y, z)',
+        required=True
+    )
+    monitor_rotation = fields.Nested(
+        EyeTrackingRigGeometry,
+        doc='rotation of monitor (x, y, z)',
+        required=True
+    )
+    camera_rotation = fields.Nested(
+        EyeTrackingRigGeometry,
+        doc='rotation of camera (x, y, z)',
+        required=True
+    )

allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py

@@ -27,7 +27,7 @@ def get_licks(self) -> pd.DataFrame:
 
         Returns
         -------
-        np.ndarray
+        pd.Dataframe
             A dataframe containing lick timestamps.
         """
         raise NotImplementedError()

allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py

@@ -152,7 +152,7 @@ def get_stimulus_presentations(self) -> pd.DataFrame:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def get_eye_tracking(self) -> pd.DataFrame:
+    def get_eye_tracking(self) -> Optional[pd.DataFrame]:
         """Get eye tracking data from behavior + ophys session.
 
         Returns

allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py

@@ -1,5 +1,7 @@
 # data_io classes for behavior only
+from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_nwb_api import BehaviorNwbApi  # noqa: F401, E501
 from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_lims_api import BehaviorLimsApi  # noqa: F401, E501
+from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_json_api import BehaviorJsonApi  # noqa: F401, E501
 
 # data_io classes for behavior + ophys
 from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_nwb_api import BehaviorOphysNwbApi  # noqa: F401, E501

allensdk/brain_observatory/behavior/session_apis/data_io/behavior_json_api.py

@@ -0,0 +1,75 @@
+import logging
+from datetime import datetime
+import pytz
+
+from allensdk.brain_observatory.behavior.session_apis.data_transforms import (
+    BehaviorDataXforms)
+
+
+class BehaviorJsonApi(BehaviorDataXforms):
+    """A data fetching class that serves as an API for fetching 'raw'
+    data from a json file necessary (but not sufficient) for filling
+    a 'BehaviorSession'.
+
+    Most 'raw' data provided by this API needs to be processed by
+    BehaviorDataXforms methods in order to usable by 'BehaviorSession's.
+
+    This class is used by the write_nwb module for behavior sessions.
+    """
+
+    def __init__(self, data):
+        self.data = data
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def get_behavior_session_id(self) -> int:
+        return self.data['behavior_session_id']
+
+    def get_foraging_id(self) -> int:
+        return self.data['foraging_id']
+
+    def get_rig_name(self) -> str:
+        """Get the name of the experiment rig (ex: CAM2P.3)"""
+        return self.data['rig_name']
+
+    def get_sex(self) -> str:
+        """Get the sex of the subject (ex: 'M', 'F', or 'unknown')"""
+        return self.data['sex']
+
+    def get_age(self) -> str:
+        """Get the age of the subject (ex: 'P15', 'Adult', etc...)"""
+        return self.data['age']
+
+    def get_stimulus_name(self) -> str:
+        """Get the name of the stimulus presented for a behavior or
+        behavior + ophys experiment"""
+        return self.data['stimulus_name']
+
+    def get_experiment_date(self) -> datetime:
+        """Get the acquisition date of an ophys experiment"""
+        return pytz.utc.localize(
+            datetime.strptime(self.data['date_of_acquisition'],
+                              "%Y-%m-%d %H:%M:%S"))
+
+    def get_reporter_line(self) -> str:
+        """Get the (gene) reporter line for the subject associated with a
+        behavior or behavior + ophys experiment"""
+        return self.data['reporter_line']
+
+    def get_driver_line(self) -> str:
+        """Get the (gene) driver line for the subject associated with a
+        behavior or behavior + ophys experiment"""
+        return self.data['driver_line']
+
+    def get_full_genotype(self) -> str:
+        """Get the full genotype of the subject associated with a
+        behavior or behavior + ophys experiment"""
+        return self.data['full_genotype']
+
+    def get_behavior_stimulus_file(self) -> str:
+        """Get the filepath to the StimulusPickle file for the session"""
+        return self.data['behavior_stimulus_file']
+
+    def get_external_specimen_name(self) -> int:
+        """Get the external specimen id (LabTracks ID) for the subject
+        associated with a behavior experiment"""
+        return int(self.data['external_specimen_name'])

allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py

@@ -43,7 +43,6 @@ def __init__(self, behavior_session_id: int,
         ids = self._get_ids()
         self.ophys_experiment_ids = ids.get("ophys_experiment_ids")
         self.ophys_session_id = ids.get("ophys_session_id")
-        self.behavior_training_id = ids.get("behavior_training_id")
         self.foraging_id = ids.get("foraging_id")
         self.ophys_container_id = ids.get("ophys_container_id")
 
@@ -82,16 +81,15 @@ def _get_ids(self) -> Dict[str, Optional[Union[int, List[int]]]]:
         """Fetch ids associated with this behavior_session_id. If there is no
         id, return None.
         :returns: Dictionary of ids with the following keys:
-            behavior_training_id: int -- Only if was a training session
-            ophys_session_id: int -- None if have behavior_training_id
+            ophys_session_id: int
             ophys_experiment_ids: List[int] -- only if have ophys_session_id
             foraging_id: int
         :rtype: dict
         """
         # Get all ids from the behavior_sessions table
         query = f"""
             SELECT
-                ophys_session_id, behavior_training_id, foraging_id
+                ophys_session_id, foraging_id
             FROM
                 behavior_sessions
             WHERE
@@ -141,6 +139,18 @@ def get_behavior_session_id(self) -> int:
         """Getter to be consistent with BehaviorOphysLimsApi."""
         return self.behavior_session_id
 
+    def get_ophys_experiment_ids(self) -> Optional[List[int]]:
+        return self.ophys_experiment_ids
+
+    def get_ophys_session_id(self) -> Optional[int]:
+        return self.ophys_session_id
+
+    def get_foraging_id(self) -> int:
+        return self.foraging_id
+
+    def get_ophys_container_id(self) -> Optional[int]:
+        return self.ophys_container_id
+
     def get_behavior_stimulus_file(self) -> str:
         """Return the path to the StimulusPickle file for a session.
         :rtype: str