Merge pull request #2248 from AllenInstitute/rc/2.13.2

start rc/2.13.2

CHANGELOG.md

@@ -1,6 +1,12 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.13.2] = TBD
+- Fixes bug that caused file paths on windows machines to be incorrect in Visual behavior user-facing classes
+- Updates to support MESO.2
+- Loosens/updates required versions for several dependencies
+- Updates in order to generate valid NWB files for Neuropixels Visual Coding data collected between 2019 and 2021
+
 ## [2.13.1] = 2021-10-04
 - Fixes bug that was preventing the BehaviorSession from properly instantiating passive sessions.
 

allensdk/__init__.py

@@ -35,7 +35,7 @@
 #
 import logging
 
-__version__ = '2.13.1'
+__version__ = '2.13.2'
 
 
 try:

allensdk/brain_observatory/behavior/behavior_ophys_experiment.py

@@ -100,7 +100,8 @@ def to_nwb(self) -> NWBFile:
                                     ophys_timestamps=self._ophys_timestamps)
         self._motion_correction.to_nwb(nwbfile=nwbfile)
         self._eye_tracking.to_nwb(nwbfile=nwbfile)
-        self._eye_tracking_rig_geometry.to_nwb(nwbfile=nwbfile)
+        if self._eye_tracking_rig_geometry is not None:
+            self._eye_tracking_rig_geometry.to_nwb(nwbfile=nwbfile)
 
         return nwbfile
     # ==================== class and utility methods ======================
@@ -110,7 +111,7 @@ def from_lims(cls,
                   ophys_experiment_id: int,
                   eye_tracking_z_threshold: float = 3.0,
                   eye_tracking_dilation_frames: int = 2,
-                  events_filter_scale: float = 2.0,
+                  events_filter_scale_seconds: float = 2.0/31.0,
                   events_filter_n_time_steps: int = 20,
                   exclude_invalid_rois=True,
                   skip_eye_tracking=False) -> \
@@ -123,7 +124,7 @@ def from_lims(cls,
             See `BehaviorOphysExperiment.from_nwb`
         eye_tracking_dilation_frames
             See `BehaviorOphysExperiment.from_nwb`
-        events_filter_scale
+        events_filter_scale_seconds
             See `BehaviorOphysExperiment.from_nwb`
         events_filter_n_time_steps
             See `BehaviorOphysExperiment.from_nwb`
@@ -159,19 +160,26 @@ def _get_eye_tracking_table(sync_file: SyncFile):
         lims_db = db_connection_creator(
             fallback_credentials=LIMS_DB_CREDENTIAL_MAP
         )
-        sync_file = SyncFile.from_lims(db=lims_db,
-                                       ophys_experiment_id=ophys_experiment_id)
-        stimulus_timestamps = StimulusTimestamps.from_sync_file(
-            sync_file=sync_file)
-        behavior_session_id = BehaviorSessionId.from_lims(
-            db=lims_db, ophys_experiment_id=ophys_experiment_id)
+
         is_multiplane_session = _is_multi_plane_session()
+
         meta = BehaviorOphysMetadata.from_lims(
             ophys_experiment_id=ophys_experiment_id, lims_db=lims_db,
             is_multiplane=is_multiplane_session
         )
+
+        sync_file = SyncFile.from_lims(db=lims_db,
+                                       ophys_experiment_id=ophys_experiment_id)
+
         monitor_delay = calculate_monitor_delay(
             sync_file=sync_file, equipment=meta.behavior_metadata.equipment)
+
+        stimulus_timestamps = StimulusTimestamps.from_sync_file(
+            sync_file=sync_file,
+            monitor_delay=monitor_delay)
+
+        behavior_session_id = BehaviorSessionId.from_lims(
+            db=lims_db, ophys_experiment_id=ophys_experiment_id)
         date_of_acquisition = DateOfAcquisitionOphys.from_lims(
             ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
         behavior_session = BehaviorSession.from_lims(
@@ -198,7 +206,7 @@ def _get_eye_tracking_table(sync_file: SyncFile):
             ophys_timestamps=ophys_timestamps,
             segmentation_mask_image_spacing=projections.max_projection.spacing,
             events_params=EventsParams(
-                filter_scale=events_filter_scale,
+                filter_scale_seconds=events_filter_scale_seconds,
                 filter_n_time_steps=events_filter_n_time_steps),
             exclude_invalid_rois=exclude_invalid_rois
         )
@@ -227,7 +235,7 @@ def _get_eye_tracking_table(sync_file: SyncFile):
     def from_nwb(cls, nwbfile: NWBFile,
                  eye_tracking_z_threshold: float = 3.0,
                  eye_tracking_dilation_frames: int = 2,
-                 events_filter_scale: float = 2.0,
+                 events_filter_scale_seconds: float = 2.0/31.0,
                  events_filter_n_time_steps: int = 20,
                  exclude_invalid_rois=True
                  ) -> "BehaviorOphysExperiment":
@@ -244,10 +252,11 @@ def from_nwb(cls, nwbfile: NWBFile,
             Determines the number of adjacent frames that will be marked
             as 'likely_blink' when performing blink detection for
             `eye_tracking` data, by default 2
-        events_filter_scale : float, optional
+        events_filter_scale_seconds : float, optional
             Stdev of halfnorm distribution used to convolve ophys events with
             a 1d causal half-gaussian filter to smooth it for visualization,
-            by default 2.0
+            in seconds (by default 2.0/31.0; this value has been found to
+            perform well on Allen Institute data across multiple platforms).
         events_filter_n_time_steps : int, optional
             Number of time steps to use for convolution of ophys events
         exclude_invalid_rois
@@ -265,7 +274,7 @@ def _is_multi_plane_session():
             nwbfile=nwbfile,
             segmentation_mask_image_spacing=projections.max_projection.spacing,
             events_params=EventsParams(
-                filter_scale=events_filter_scale,
+                filter_scale_seconds=events_filter_scale_seconds,
                 filter_n_time_steps=events_filter_n_time_steps
             ),
             exclude_invalid_rois=exclude_invalid_rois
@@ -303,7 +312,7 @@ def from_json(cls,
                   session_data: dict,
                   eye_tracking_z_threshold: float = 3.0,
                   eye_tracking_dilation_frames: int = 2,
-                  events_filter_scale: float = 2.0,
+                  events_filter_scale_seconds: float = 2.0/31.0,
                   events_filter_n_time_steps: int = 20,
                   exclude_invalid_rois=True,
                   skip_eye_tracking=False) -> \
@@ -317,7 +326,7 @@ def from_json(cls,
             See `BehaviorOphysExperiment.from_nwb`
         eye_tracking_dilation_frames
             See `BehaviorOphysExperiment.from_nwb`
-        events_filter_scale
+        events_filter_scale_seconds
             See `BehaviorOphysExperiment.from_nwb`
         events_filter_n_time_steps
             See `BehaviorOphysExperiment.from_nwb`
@@ -377,7 +386,7 @@ def _get_eye_tracking_table(sync_file: SyncFile):
             ophys_timestamps=ophys_timestamps,
             segmentation_mask_image_spacing=projections.max_projection.spacing,
             events_params=EventsParams(
-                filter_scale=events_filter_scale,
+                filter_scale_seconds=events_filter_scale_seconds,
                 filter_n_time_steps=events_filter_n_time_steps),
             exclude_invalid_rois=exclude_invalid_rois
         )
@@ -729,6 +738,8 @@ def eye_tracking_rig_geometry(self) -> dict:
                 monitor_position_mm (array of float)
                 monitor_rotation_deg (array of float)
         """
+        if self._eye_tracking_rig_geometry is None:
+            return dict()
         return self._eye_tracking_rig_geometry.to_dict()['rig_geometry']
 
     @property

allensdk/brain_observatory/behavior/behavior_project_cache/tables/util/experiments_table_utils.py

@@ -34,7 +34,7 @@ def add_experience_level_to_experiment_table(
     session_123 = experiments_table.session_number.isin([1, 2, 3])
     familiar_indices = experiments_table[session_123].index.values
 
-    experiments_table.at[familiar_indices, 'experience_level'] = 'Familiar'
+    experiments_table.loc[familiar_indices, 'experience_level'] = 'Familiar'
 
     session_4 = (experiments_table.session_number == 4)
     zero_prior_exp = (experiments_table.prior_exposures_to_image_set == 0)
@@ -43,17 +43,15 @@ def add_experience_level_to_experiment_table(
                           session_4
                           & zero_prior_exp].index.values
 
-    experiments_table.at[novel_indices,
-                         'experience_level'] = 'Novel 1'
+    experiments_table.loc[novel_indices, 'experience_level'] = 'Novel 1'
 
     session_456 = experiments_table.session_number.isin([4, 5, 6])
     nonzero_prior_exp = (experiments_table.prior_exposures_to_image_set != 0)
     novel_gt_1_indices = experiments_table[
                              session_456
                              & nonzero_prior_exp].index.values
 
-    experiments_table.at[novel_gt_1_indices,
-                         'experience_level'] = 'Novel >1'
+    experiments_table.loc[novel_gt_1_indices, 'experience_level'] = 'Novel >1'
 
     return experiments_table
 
@@ -88,7 +86,7 @@ def add_passive_flag_to_ophys_experiment_table(
 
     session_25 = experiments_table.session_number.isin([2, 5])
     passive_indices = experiments_table[session_25].index.values
-    experiments_table.at[passive_indices, 'passive'] = True
+    experiments_table.loc[passive_indices, 'passive'] = True
 
     return experiments_table
 

allensdk/brain_observatory/behavior/behavior_session.py

@@ -101,22 +101,24 @@ def from_json(cls,
         `BehaviorSession` instance
 
         """
+        if monitor_delay is None:
+            monitor_delay = cls._get_monitor_delay()
+
         behavior_session_id = BehaviorSessionId.from_json(
             dict_repr=session_data)
         stimulus_file = StimulusFile.from_json(dict_repr=session_data)
         stimulus_timestamps = StimulusTimestamps.from_json(
-            dict_repr=session_data)
+            dict_repr=session_data,
+            monitor_delay=monitor_delay)
         running_acquisition = RunningAcquisition.from_json(
-            dict_repr=session_data)
+            dict_repr=session_data,
+            monitor_delay=monitor_delay)
         raw_running_speed = RunningSpeed.from_json(
             dict_repr=session_data, filtered=False
         )
         running_speed = RunningSpeed.from_json(dict_repr=session_data)
         metadata = BehaviorMetadata.from_json(dict_repr=session_data)
 
-        if monitor_delay is None:
-            monitor_delay = cls._get_monitor_delay()
-
         licks, rewards, stimuli, task_parameters, trials = \
             cls._read_data_from_stimulus_file(
                 stimulus_file=stimulus_file,
@@ -179,14 +181,20 @@ def from_lims(cls, behavior_session_id: int,
                 fallback_credentials=LIMS_DB_CREDENTIAL_MAP
             )
 
+        if monitor_delay is None:
+            monitor_delay = cls._get_monitor_delay()
+
         behavior_session_id = BehaviorSessionId(behavior_session_id)
         stimulus_file = StimulusFile.from_lims(
             db=lims_db, behavior_session_id=behavior_session_id.value)
         if stimulus_timestamps is None:
             stimulus_timestamps = StimulusTimestamps.from_stimulus_file(
-                stimulus_file=stimulus_file)
+                stimulus_file=stimulus_file,
+                monitor_delay=monitor_delay)
         running_acquisition = RunningAcquisition.from_lims(
-            lims_db, behavior_session_id.value
+            lims_db,
+            behavior_session_id.value,
+            monitor_delay=monitor_delay
         )
         raw_running_speed = RunningSpeed.from_lims(
             lims_db, behavior_session_id.value, filtered=False,
@@ -200,9 +208,6 @@ def from_lims(cls, behavior_session_id: int,
             behavior_session_id=behavior_session_id, lims_db=lims_db
         )
 
-        if monitor_delay is None:
-            monitor_delay = cls._get_monitor_delay()
-
         licks, rewards, stimuli, task_parameters, trials = \
             cls._read_data_from_stimulus_file(
                 stimulus_file=stimulus_file,
@@ -904,8 +909,7 @@ def _read_data_from_stimulus_file(
             stimulus_file=stimulus_file,
             stimulus_timestamps=stimulus_timestamps,
             licks=licks,
-            rewards=rewards,
-            monitor_delay=trial_monitor_delay
+            rewards=rewards
         )
         return licks, rewards, stimuli, task_parameters, trials
 

allensdk/brain_observatory/behavior/data_files/_data_file_abc.py

@@ -2,6 +2,8 @@
 from typing import Any, Union
 from pathlib import Path
 
+from allensdk.internal.core.lims_utilities import safe_system_path
+
 
 class DataFile(abc.ABC):
     """An abstract class that prototypes methods for accessing internal
@@ -15,7 +17,7 @@ class DataFile(abc.ABC):
     """
 
     def __init__(self, filepath: Union[str, Path]):  # pragma: no cover
-        self._filepath: str = str(filepath)
+        self._filepath: str = safe_system_path(str(filepath))
         self._data = self.load_data(filepath=self._filepath)
 
     @property

allensdk/brain_observatory/behavior/data_files/demix_file.py

@@ -61,6 +61,6 @@ def load_data(filepath: Union[str, Path]) -> pd.DataFrame:
         with h5py.File(filepath, 'r') as in_file:
             traces = in_file['data'][()]
             roi_id = in_file['roi_names'][()]
-            idx = pd.Index(roi_id, name='cell_roi_id', dtype=int)
+            idx = pd.Index(roi_id, name='cell_roi_id').astype('int64')
             return pd.DataFrame({'corrected_fluorescence': list(traces)},
                                 index=idx)

allensdk/brain_observatory/behavior/data_files/dff_file.py

@@ -61,5 +61,5 @@ def load_data(filepath: Union[str, Path]) -> pd.DataFrame:
         with h5py.File(filepath, 'r') as raw_file:
             traces = np.asarray(raw_file['data'], dtype=np.float64)
             roi_names = np.asarray(raw_file['roi_names'])
-            idx = pd.Index(roi_names, name='cell_roi_id', dtype=int)
+            idx = pd.Index(roi_names, name='cell_roi_id').astype('int64')
             return pd.DataFrame({'dff': [x for x in traces]}, index=idx)