Merge pull request #2246 from jsiegle/ecephys-updates

Updates required for ecephys NWB packaging

CHANGELOG.md

@@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file.
 - Fixes bug that caused file paths on windows machines to be incorrect in Visual behavior user-facing classes
 - Updates to support MESO.2
 - Loosens/updates required versions for several dependencies
+- Updates in order to generate valid NWB files for Neuropixels Visual Coding data collected between 2019 and 2021
 
 ## [2.13.1] = 2021-10-04
 - Fixes bug that was preventing the BehaviorSession from properly instantiating passive sessions.

allensdk/brain_observatory/comparison_utils.py

@@ -6,7 +6,7 @@
 import numpy as np
 import pandas as pd
 import xarray as xr
-from pandas.util.testing import assert_frame_equal
+from pandas.testing import assert_frame_equal
 
 
 def compare_fields(x1: Any, x2: Any, err_msg="",

allensdk/brain_observatory/ecephys/__init__.py

@@ -1,7 +1,6 @@
 import numpy as np
 
 
-
 UNIT_FILTER_DEFAULTS = {
     "amplitude_cutoff_maximum": {
         "value": 0.1,
@@ -23,7 +22,7 @@ def get_unit_filter_value(key, pop=True, replace_none=True, **source):
         value = source.pop(key, UNIT_FILTER_DEFAULTS[key]["value"])
     else:
         value = source.get(key, UNIT_FILTER_DEFAULTS[key]["value"])
-    
+
     if value is None and replace_none:
         value = UNIT_FILTER_DEFAULTS[key]["missing"]
 

allensdk/brain_observatory/ecephys/align_timestamps/__main__.py

@@ -6,7 +6,8 @@
 from ._schemas import InputParameters, OutputParameters
 from .barcode_sync_dataset import BarcodeSyncDataset
 from .channel_states import extract_barcodes_from_states, \
-    extract_splits_from_states
+    extract_splits_from_states, \
+    extract_splits_from_barcode_times
 from .probe_synchronizer import ProbeSynchronizer
 
 
@@ -29,6 +30,12 @@ def align_timestamps(args):
             channel_states, timestamps, probe["sampling_rate"]
         )
 
+        barcode_split_times = extract_splits_from_barcode_times(
+            probe_barcode_times
+        )
+
+        probe_split_times = np.union1d(probe_split_times, barcode_split_times)
+
         print("Split times:")
         print(probe_split_times)
 
@@ -92,6 +99,7 @@ def align_timestamps(args):
             "global_probe_lfp_sampling_rate"] = lfp_sampling_rate
         this_probe_output_info["output_paths"] = mapped_files
         this_probe_output_info["name"] = probe["name"]
+        this_probe_output_info["split_times"] = probe_split_times
 
         probe_output_info.append(this_probe_output_info)
 

allensdk/brain_observatory/ecephys/align_timestamps/_schemas.py

@@ -1,6 +1,6 @@
-from argschema import ArgSchema, ArgSchemaParser
+from argschema import ArgSchema
 from argschema.schemas import DefaultSchema
-from argschema.fields import Nested, InputDir, String, Float, Dict, Int, List
+from argschema.fields import Nested, String, Float, Dict, Int, List
 
 
 class ProbeMappable(DefaultSchema):
@@ -10,40 +10,50 @@ class ProbeMappable(DefaultSchema):
     )
     input_path = String(
         required=True,
-        help="Input path for this file. Should point to a file containing a 1D timestamps array with values in probe samples.",
+        help="""Input path for this file. Should point to a file containing a 1D
+                timestamps array with values in probe samples.""",
     )
     output_path = String(
         required=True,
-        help="Output path for the mapped version of this file. Will write a 1D timestamps array with values in seconds on the master clock.",
+        help="""Output path for the mapped version of this file. Will write a 1D
+                timestamps array with values in seconds on the master clock."""
     )
 
 
 class ProbeInputParameters(DefaultSchema):
     name = String(required=True, help="Identifier for this probe")
     sampling_rate = Float(
         required=True,
-        help="The sampling rate of the probe, in Hz, assessed on the probe clock.",
+        help="""The sampling rate of the probe, in Hz, assessed on
+                the probe clock.""",
     )
     lfp_sampling_rate = Float(
-        required=True, help="The sampling rate of the LFP collected on this probe."
+        required=True, help="""The sampling rate of the LFP collected on this
+                            probe."""
     )
     start_index = Int(
-        default=0, help="Sample index of probe recording start time. Defaults to 0."
+        default=0, help="""Sample index of probe recording start time.
+        Defaults to 0."""
     )
     barcode_channel_states_path = String(
         required=True,
-        help="Path to the channel states file. This file contains a 1-dimensional array whose axis is events and whose "
-        "values indicate the state of the channel line (rising or falling) at that event.",
+        help="""Path to the channel states file. This file contains a
+                1-dimensional array whose axis is events and whose
+                values indicate the state of the channel line (rising or
+                falling) at that event.""",
     )
     barcode_timestamps_path = String(
         required=True,
-        help="Path to the timestamps file. This file contains a 1-dimensional array whose axis is events and whose "
-        "values indicate the sample on which each event was detected.",
+        help="""Path to the timestamps file. This file contains a 1-dimensional
+         array whose axis is events and whose values indicate the sample
+         on which each event was detected.""",
     )
     mappable_timestamp_files = Nested(
         ProbeMappable,
         many=True,
-        help="Timestamps files for this probe. Describe the times (in probe samples) when e.g. lfp samples were taken or spike events occured",
+        help="""Timestamps files for this probe. Describe the times (in probe
+                samples) when e.g. lfp samples were taken or spike events
+                occured""",
     )
 
 
@@ -54,7 +64,9 @@ class InputParameters(ArgSchema):
         help="Probes whose data will be aligned to the master clock.",
     )
     sync_h5_path = String(
-        required=True, help="path to h5 file containing syncronization information"
+        required=True,
+        help="""path to h5 file containing syncronization
+                information"""
     )
 
 
@@ -66,15 +78,24 @@ class ProbeOutputParameters(DefaultSchema):
     )
     total_time_shift = Float(
         required=True,
-        help="Translation (in seconds) from master->probe times computed for this probe.",
+        help="""Translation (in seconds) from master->probe times computed
+                 for this probe.""",
     )
     global_probe_sampling_rate = Float(
         required=True,
-        help="The sampling rate of this probe in Hz, assessed on the master clock.",
+        help="""The sampling rate of this probe in Hz, assessed on the master
+                clock.""",
     )
     global_probe_lfp_sampling_rate = Float(
         required=True,
-        help="The sampling rate of LFP collected on this probe in Hz, assessed on the master clock.",
+        help="""The sampling rate of LFP collected on this probe in Hz,
+                 assessed on the master clock.""",
+    )
+    split_times = List(
+        Float(),
+        required=True,
+        help="""Start/stop times of likely dropped data, due to gaps in
+                recording or irregular barcode intervals"""
     )