From a01ab404da4359bbc9225c3e3d3cf55856ae1013 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sat, 7 Dec 2024 15:10:57 +0100 Subject: [PATCH 01/10] resolve miniscope negative timestamps --- .../interfaces/miniscope_imaging_interface.py | 5 ---- .../zaki_2024/zaki_2024_nwbconverter.py | 29 +++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py index 9c514ff..688b04f 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py @@ -370,11 +370,6 @@ def get_original_timestamps(self) -> np.ndarray: timestamps_file_path = self.miniscope_folder / "timeStamps.csv" assert timestamps_file_path.exists(), f"Miniscope timestamps file not found in {self.miniscope_folder}" timestamps_seconds = get_miniscope_timestamps(file_path=timestamps_file_path) - # Shift when the first timestamp is negative - # TODO: Figure why, I copied from Miniscope. Need to shift also session_start_time - if timestamps_seconds[0] < 0.0: - timestamps_seconds += abs(timestamps_seconds[0]) - return timestamps_seconds def add_to_nwbfile( diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index 90c5712..06d4f60 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -1,7 +1,10 @@ """Primary NWBConverter class for this dataset.""" +from datetime import timedelta + from neuroconv import NWBConverter from neuroconv.datainterfaces import VideoInterface +from neuroconv.utils.dict import DeepDict from interfaces import ( MinianSegmentationInterface, @@ -31,3 +34,29 @@ class Zaki2024NWBConverter(NWBConverter): ShockStimuli=Zaki2024ShockStimuliInterface, CellRegistration=Zaki2024CellRegistrationInterface, ) + + def get_metadata(self) -> DeepDict: + metadata = super().get_metadata() + + if "MiniscopeImaging" in self.data_interface_objects: + imaging_interface = self.data_interface_objects["MiniscopeImaging"] + imaging_timestamps = imaging_interface.get_original_timestamps() + # If the first timestamp in the imaging data is negative, adjust the session start time + # to ensure all timestamps are positive. This is done by shifting the session start time + # backward by the absolute value of the negative timestamp. + if imaging_timestamps[0] < 0.0: + time_shift = timedelta(seconds=abs(imaging_timestamps[0])) + session_start_time = imaging_interface.get_metadata()["NWBFile"]["session_start_time"] + metadata["NWBFile"].update(session_start_time=session_start_time - time_shift) + return metadata + + def temporally_align_data_interfaces(self): + if "MiniscopeImaging" in self.data_interface_objects: + imaging_interface = self.data_interface_objects["MiniscopeImaging"] + imaging_timestamps = imaging_interface.get_original_timestamps() + # Align the starting times of all data interfaces when the imaging data's first timestamp is negative. + # This is done by calculating a time shift based on the absolute value of the first negative timestamp. + # The time shift is applied to all relevant data interfaces to ensure temporal alignment. + if imaging_timestamps[0] < 0.0: + time_shift = abs(imaging_timestamps[0]) + imaging_interface.set_aligned_timestamps(imaging_timestamps + time_shift) \ No newline at end of file From 9bdca3cb8009741ae90e12e053a62f15274df7c3 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sat, 7 Dec 2024 15:12:28 +0100 Subject: [PATCH 02/10] align minian interface --- .../zaki_2024/interfaces/minian_interface.py | 26 +++++++++++++++++-- .../zaki_2024/zaki_2024_nwbconverter.py | 12 ++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py index 8f64290..ad0dd52 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py @@ -154,7 +154,7 @@ def get_accepted_list(self) -> list: accepted_list: list List of accepted ROI ids. """ - return list(range(self.get_num_rois())) + return self.get_roi_ids() def get_rejected_list(self) -> list: """Get a list of rejected ROI ids. @@ -312,6 +312,28 @@ def get_dtype(self) -> DtypeType: def get_channel_names(self) -> list[str]: return ["OpticalChannel"] + def get_original_timestamps(self, stub_test: bool = False) -> list[np.ndarray]: + """ + Retrieve the original unaltered timestamps for the data in this interface. + + This function should retrieve the data on-demand by re-initializing the IO. + + Returns + ------- + timestamps : numpy.ndarray + The timestamps for the data stream. + stub_test : bool, default: False + This method scans through each video; a process which can take some time to complete. + + To limit that scan to a small number of frames, set `stub_test=True`. + """ + max_frames = 100 if stub_test else None + with self._video_capture(file_path=str(self.file_path)) as video: + # fps = video.get_video_fps() # There is some debate about whether the OpenCV timestamp + # method is simply returning range(length) / fps 100% of the time for any given format + timestamps = video.get_video_timestamps(max_frames=max_frames) + return timestamps + def get_video( self, start_frame: Optional[int] = None, end_frame: Optional[int] = None, channel: int = 0 ) -> np.ndarray: @@ -440,7 +462,7 @@ def add_to_nwbfile( xy_translation=xy_translation, ) - ophys = get_module(nwbfile, "ophys") + ophys = get_module(nwbfile, name="ophys", description="Data processed with MiniAn software") if "MotionCorrection" not in ophys.data_interfaces: motion_correction = MotionCorrection(name="MotionCorrection") ophys.add(motion_correction) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index 06d4f60..e1afc7b 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -59,4 +59,14 @@ def temporally_align_data_interfaces(self): # The time shift is applied to all relevant data interfaces to ensure temporal alignment. if imaging_timestamps[0] < 0.0: time_shift = abs(imaging_timestamps[0]) - imaging_interface.set_aligned_timestamps(imaging_timestamps + time_shift) \ No newline at end of file + imaging_interface.set_aligned_timestamps(imaging_timestamps + time_shift) + + if "MinianSegmentation" in self.data_interface_objects: + segmentation_interface = self.data_interface_objects["MinianSegmentation"] + segmentation_timestamps = segmentation_interface.get_original_timestamps() + segmentation_interface.set_aligned_timestamps(segmentation_timestamps + time_shift) + + # if "MinianMotionCorrection" in self.data_interface_objects: + # motion_correction_interface = self.data_interface_objects["MinianMotionCorrection"] + # motion_correction_timestamps = motion_correction_interface.get_original_timestamps() + # motion_correction_interface.set_aligned_timestamps(segmentation_timestamps + time_shift) From e9ef0ccf561fb46d1efc5c2897299659bb8ceb4b Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sat, 7 Dec 2024 15:12:58 +0100 Subject: [PATCH 03/10] align sleep intervals --- ...aki_2024_sleep_classification_interface.py | 53 +++++++++++-------- .../zaki_2024/zaki_2024_nwbconverter.py | 9 ++++ 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_sleep_classification_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_sleep_classification_interface.py index 942a9b3..3cd5773 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_sleep_classification_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_sleep_classification_interface.py @@ -1,12 +1,13 @@ """Primary class for converting experiment-specific behavior.""" -from pynwb.file import NWBFile +import pandas as pd +from pydantic import FilePath +from typing import Optional, List +from pynwb.file import NWBFile +from pynwb.epoch import TimeIntervals from neuroconv.basedatainterface import BaseDataInterface from neuroconv.utils import DeepDict -from pydantic import FilePath -from typing import Optional -from pynwb.epoch import TimeIntervals class Zaki2024SleepClassificationInterface(BaseDataInterface): @@ -14,12 +15,14 @@ class Zaki2024SleepClassificationInterface(BaseDataInterface): keywords = ["behavior", "sleep stages"] - def __init__(self, file_path: FilePath, video_sampling_frequency: float, verbose: bool = False): + def __init__(self, file_path: FilePath, sampling_frequency: float, verbose: bool = False): # This should load the data lazily and prepare variables you need self.file_path = file_path self.verbose = verbose - self.video_sampling_frequency = video_sampling_frequency + self.sampling_frequency = sampling_frequency + self._start_times = None + self._stop_times = None def get_metadata(self) -> DeepDict: # Automatically retrieve as much metadata as possible from the source files available @@ -27,33 +30,40 @@ def get_metadata(self) -> DeepDict: return metadata - def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict] = None): - - import pandas as pd - + def get_sleep_states_times(self): sleep_behavior_df = pd.read_csv(self.file_path) import numpy as np # Note this will have the first row as None - shifted_sleep_state = sleep_behavior_df['SleepState'].shift() - start_indices = np.where(sleep_behavior_df['SleepState'] != shifted_sleep_state)[0] + shifted_sleep_state = sleep_behavior_df["SleepState"].shift() + start_indices = np.where(sleep_behavior_df["SleepState"] != shifted_sleep_state)[0] stop_indices = [i - 1 for i in start_indices[1:]] stop_indices.append(len(sleep_behavior_df) - 1) stop_indices = np.array(stop_indices) - start_frames = sleep_behavior_df['Frame'][start_indices].values - start_times = start_frames / self.video_sampling_frequency - stop_frames = sleep_behavior_df['Frame'][stop_indices].values - stop_times = stop_frames / self.video_sampling_frequency - sleep_state = sleep_behavior_df['SleepState'][start_indices].values + start_frames = sleep_behavior_df["Frame"][start_indices].values + start_times = self._start_times if self._start_times is not None else start_frames / self.sampling_frequency + stop_frames = sleep_behavior_df["Frame"][stop_indices].values + stop_times = self._stop_times if self._stop_times is not None else stop_frames / self.sampling_frequency + sleep_state = sleep_behavior_df["SleepState"][start_indices].values + + return start_times, stop_times, sleep_state + + def set_aligned_interval_times(self, start_times: List[float], stop_times: List[float]) -> None: + self._start_times = start_times + self._stop_times = stop_times + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict] = None): + + start_times, stop_times, sleep_state = self.get_sleep_states_times() description = ( "Sleep states classified with custom algorithm using the data " "from the HD-X02 sensor (EEG, EMG, temperature, etc.)." - ) - + ) + sleep_intervals = TimeIntervals(name="SleepIntervals", description=description) column_description = """ Sleep State Classification, it can be one of the following: @@ -63,7 +73,7 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict] = None): - 'wake': State of full consciousness when the animal is alert, responsive to the environment, and capable of voluntary movement """ sleep_intervals.add_column(name="sleep_state", description=column_description) - + for start_time, stop_time, state in zip(start_times, stop_times, sleep_state): sleep_intervals.add_interval(start_time=start_time, stop_time=stop_time, sleep_state=state) @@ -71,6 +81,5 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict] = None): sleep_module = nwbfile.create_processing_module(name="sleep", description="Sleep data") else: sleep_module = nwbfile.processing["sleep"] - - sleep_module.add(sleep_intervals) + sleep_module.add(sleep_intervals) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index e1afc7b..cfa980d 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -70,3 +70,12 @@ def temporally_align_data_interfaces(self): # motion_correction_interface = self.data_interface_objects["MinianMotionCorrection"] # motion_correction_timestamps = motion_correction_interface.get_original_timestamps() # motion_correction_interface.set_aligned_timestamps(segmentation_timestamps + time_shift) + + if "SleepClassification" in self.data_interface_objects: + sleep_classification_interface = self.data_interface_objects["SleepClassification"] + start_times, stop_times, sleep_states = sleep_classification_interface.get_sleep_states_times() + start_times += time_shift + stop_times += time_shift + sleep_classification_interface.set_aligned_interval_times( + start_times=start_times, stop_times=stop_times + ) \ No newline at end of file From 4ccd3be7dab7c7132b4b8470544b465b1d56d5f4 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sat, 7 Dec 2024 15:13:17 +0100 Subject: [PATCH 04/10] align edf singal --- .../interfaces/zaki_2024_edf_interface.py | 17 +++++++++++------ .../zaki_2024/zaki_2024_nwbconverter.py | 5 ++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py index c123cfe..d5ccbd5 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py @@ -5,7 +5,6 @@ from pynwb.device import Device from neuroconv.basedatainterface import BaseDataInterface -from neuroconv.utils import DeepDict from mne.io import read_raw_edf from datetime import datetime, timedelta @@ -19,9 +18,13 @@ def __init__( verbose: bool = False, ): self.file_path = Path(file_path) + self._starting_time = None self.verbose = verbose super().__init__(file_path=file_path) + def set_aligned_starting_time(self, aligned_starting_time: float): + self._starting_time = aligned_starting_time + def add_to_nwbfile( self, nwbfile: NWBFile, @@ -49,8 +52,6 @@ def add_to_nwbfile( stop_datetime_timestamp : datetime, optional The ending timestamp for slicing the data. If specified, data will be included only up to this time. Default is None, which includes data until the end. - **conversion_options - Additional options for data conversion (not currently used directly in this function). Returns ------- @@ -73,8 +74,6 @@ def add_to_nwbfile( "description": "EMG signal recorder with HD-X02 wireless telemetry probe", "unit": "volts", }, - # TODO: Figure out if the units of activity are correct, the raw format marks Volts - # TODO: My own reading of the header indicates that the physical units is counts "Activity": { "name": "ActivitySignal", "description": "Activity signal recorder with HD-X02 wireless telemetry probe. It refers to the motion of the probe relative to the receiver and it can be used as a proxy for locomotion.", @@ -107,6 +106,8 @@ def add_to_nwbfile( else: data = data[:, start_idx:end_idx] + starting_time = self._starting_time if self._starting_time is not None else starting_time + for channel_index, channel_name in enumerate(channels_dict.keys()): time_series_kwargs = channels_dict[channel_name].copy() time_series_kwargs.update( @@ -132,8 +133,12 @@ def __init__( ): self.file_paths = file_paths self.verbose = verbose + self._starting_time = 0.0 super().__init__(file_paths=file_paths) + def set_aligned_starting_time(self, aligned_starting_time: float): + self._starting_time = aligned_starting_time + def add_to_nwbfile( self, nwbfile: NWBFile, @@ -205,7 +210,7 @@ def add_to_nwbfile( for channel_index, channel_name in enumerate(channels_dict.keys()): time_series_kwargs = channels_dict[channel_name].copy() time_series_kwargs.update( - data=concatenated_data[channel_index], starting_time=0.0, rate=edf_reader.info["sfreq"] + data=concatenated_data[channel_index], starting_time=self._starting_time, rate=edf_reader.info["sfreq"] ) time_series = TimeSeries(**time_series_kwargs) nwbfile.add_acquisition(time_series) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index cfa980d..741ab86 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -78,4 +78,7 @@ def temporally_align_data_interfaces(self): stop_times += time_shift sleep_classification_interface.set_aligned_interval_times( start_times=start_times, stop_times=stop_times - ) \ No newline at end of file + ) + if "EDFSignals" in self.data_interface_objects: + edf_signals_interface = self.data_interface_objects["EDFSignals"] + edf_signals_interface.set_aligned_starting_time(time_shift) From b5a6116cc6071f4077dd95838eb8157a563b6cfa Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sat, 7 Dec 2024 15:13:39 +0100 Subject: [PATCH 05/10] align eztrack and video --- .../zaki_2024/interfaces/eztrack_interface.py | 58 +++++++++++++------ .../zaki_2024/zaki_2024_nwbconverter.py | 17 ++++++ 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/eztrack_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/eztrack_interface.py index 3787eb5..5001fbd 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/eztrack_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/eztrack_interface.py @@ -1,6 +1,8 @@ """Primary class for converting experiment-specific behavior.""" import numpy as np +import pandas as pd + from pynwb import TimeSeries from pynwb.epoch import TimeIntervals from pynwb.file import NWBFile @@ -8,7 +10,7 @@ from neuroconv.basedatainterface import BaseDataInterface from neuroconv.utils import DeepDict from pydantic import FilePath -from typing import Optional +from typing import Optional, List class EzTrackFreezingBehaviorInterface(BaseDataInterface): @@ -22,6 +24,9 @@ def __init__(self, file_path: FilePath, video_sampling_frequency: float, verbose self.file_path = file_path self.verbose = verbose self.video_sampling_frequency = video_sampling_frequency + self._start_times = None + self._stop_times = None + self._starting_time = None def get_metadata(self) -> DeepDict: # Automatically retrieve as much metadata as possible from the source files available @@ -29,22 +34,52 @@ def get_metadata(self) -> DeepDict: return metadata - def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict] = None): + def get_interval_times(self): + # Extract start and stop times of the freezing events + # From the discussion wih the author, the freezing events are the frames where the freezing behavior is 100 + freezing_behavior_df = pd.read_csv(self.file_path) + freezing_values = freezing_behavior_df["Freezing"].values + changes_in_freezing = np.diff(freezing_values) + freezing_start = np.where(changes_in_freezing == 100)[0] + 1 + freezing_stop = np.where(changes_in_freezing == -100)[0] + 1 - import pandas as pd + start_frames = freezing_behavior_df["Frame"].values[freezing_start] + stop_frames = freezing_behavior_df["Frame"].values[freezing_stop] + + start_times = ( + self._start_times if self._start_times is not None else start_frames / self.video_sampling_frequency + ) + stop_times = self._stop_times if self._stop_times is not None else stop_frames / self.video_sampling_frequency + return start_times, stop_times + def get_starting_time(self) -> float: freezing_behavior_df = pd.read_csv(self.file_path) + return freezing_behavior_df["Frame"].values[0] / self.video_sampling_frequency + + def set_aligned_interval_times(self, start_times: List[float], stop_times: List[float]) -> None: + self._start_times = start_times + self._stop_times = stop_times + + def set_aligned_starting_time(self, aligned_start_time) -> None: + self._starting_time = aligned_start_time + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict] = None, stub_test: bool = False): + + freezing_behavior_df = pd.read_csv(self.file_path) + + start_times, stop_times = self.get_interval_times() # Extract motion data motion_data = freezing_behavior_df["Motion"].values + starting_time = self._starting_time if self._starting_time is not None else self.get_starting_time() motion_series = TimeSeries( name="MotionSeries", description="ezTrack measures the motion of the animal by assessing the number of pixels of the behavioral " "video whose grayscale change exceeds a particular threshold from one frame to the next.", - data=motion_data, + data=motion_data[:100] if stub_test else motion_data, unit="n.a", - starting_time=freezing_behavior_df["Frame"][0] / self.video_sampling_frequency, + starting_time=starting_time, rate=self.video_sampling_frequency, ) @@ -54,19 +89,6 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict] = None): freeze_threshold = freezing_behavior_df["FreezeThresh"].unique()[0] min_freeze_duration = freezing_behavior_df["MinFreezeDuration"].unique()[0] - # Extract start and stop times of the freezing events - # From the discussion wih the author, the freezing events are the frames where the freezing behavior is 100 - freezing_values = freezing_behavior_df["Freezing"].values - changes_in_freezing = np.diff(freezing_values) - freezing_start = np.where(changes_in_freezing == 100)[0] + 1 - freezing_stop = np.where(changes_in_freezing == -100)[0] + 1 - - start_frames = freezing_behavior_df["Frame"].values[freezing_start] - stop_frames = freezing_behavior_df["Frame"].values[freezing_stop] - - start_times = start_frames / self.video_sampling_frequency - stop_times = stop_frames / self.video_sampling_frequency - description = f""" Freezing behavior intervals generated using EzTrack software for file {file}. Parameters used include a motion cutoff of {motion_cutoff}, freeze threshold of {freeze_threshold}, diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index 741ab86..a7412ce 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -82,3 +82,20 @@ def temporally_align_data_interfaces(self): if "EDFSignals" in self.data_interface_objects: edf_signals_interface = self.data_interface_objects["EDFSignals"] edf_signals_interface.set_aligned_starting_time(time_shift) + + if "FreezingBehavior" in self.data_interface_objects: + freezing_behavior_interface = self.data_interface_objects["FreezingBehavior"] + start_times, stop_times = freezing_behavior_interface.get_interval_times() + start_times += time_shift + stop_times += time_shift + freezing_behavior_interface.set_aligned_interval_times( + start_times=start_times, stop_times=stop_times + ) + starting_time = freezing_behavior_interface.get_starting_time() + freezing_behavior_interface.set_aligned_starting_time(starting_time + time_shift) + + if "Video" in self.data_interface_objects: + video_interface = self.data_interface_objects["Video"] + video_timestamps = video_interface.get_original_timestamps() + aligned_video_timestamps = video_timestamps + time_shift + video_interface.set_aligned_timestamps(aligned_video_timestamps) From ecca8a6cb7b1797610352c292e7ae228e0c4b8d7 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sat, 7 Dec 2024 15:15:53 +0100 Subject: [PATCH 06/10] add session start time timezone --- .../zaki_2024/zaki_2024_convert_session.py | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py index 35b1965..4201505 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py @@ -1,7 +1,7 @@ """Primary script to run to convert an entire session for of data using the NWBConverter.""" import time - +import pytz from pathlib import Path from typing import Union from datetime import datetime, timedelta @@ -62,16 +62,16 @@ def session_to_nwb( source_data.update(dict(MinianSegmentation=dict(folder_path=minian_folder_path))) conversion_options.update(dict(MinianSegmentation=dict(stub_test=stub_test))) - motion_corrected_video = minian_folder_path / "minian_mc.mp4" - if motion_corrected_video.is_file(): - source_data.update( - dict( - MinianMotionCorrection=dict(folder_path=minian_folder_path, video_file_path=motion_corrected_video) - ) - ) - conversion_options.update(dict(MinianMotionCorrection=dict(stub_test=stub_test))) - elif verbose and not motion_corrected_video.is_file(): - print(f"No motion corrected data found at {motion_corrected_video}") + # motion_corrected_video = minian_folder_path / "minian_mc.mp4" + # if motion_corrected_video.is_file(): + # source_data.update( + # dict( + # MinianMotionCorrection=dict(folder_path=minian_folder_path, video_file_path=motion_corrected_video) + # ) + # ) + # conversion_options.update(dict(MinianMotionCorrection=dict(stub_test=stub_test))) + # elif verbose and not motion_corrected_video.is_file(): + # print(f"No motion corrected data found at {motion_corrected_video}") # Add Behavioral Video if video_file_path: @@ -133,7 +133,7 @@ def session_to_nwb( sleep_classification_file_path = Path(sleep_classification_file_path) assert sleep_classification_file_path.is_file(), f"{sleep_classification_file_path} does not exist" source_data.update( - dict(SleepClassification=dict(file_path=sleep_classification_file_path, video_sampling_frequency=30.0)) + dict(SleepClassification=dict(file_path=sleep_classification_file_path, sampling_frequency=15.0)) ) # Add Shock Stimuli times for FC sessions @@ -154,6 +154,9 @@ def session_to_nwb( session_start_time = datetime.strptime(datetime_str, "%Y_%m_%d %H_%M_%S") metadata["NWBFile"]["session_start_time"] = session_start_time + eastern = pytz.timezone("US/Eastern") + metadata["NWBFile"]["session_start_time"] = eastern.localize(metadata["NWBFile"]["session_start_time"]) + # Update default metadata with the editable in the corresponding yaml file editable_metadata_path = Path(__file__).parent / "zaki_2024_metadata.yaml" editable_metadata = load_dict_from_file(editable_metadata_path) @@ -180,10 +183,10 @@ def session_to_nwb( if __name__ == "__main__": - subject_id = "Ca_EEG3-4" - session_type = "OfflineDay2Session1" + subject_id = "Ca_EEG2-1" + session_type = "NeutralExposure" # session_id = subject_id + "_" + session_type - stub_test = False + stub_test = True verbose = True yaml_file_path = Path(__file__).parent / "utils/conversion_parameters.yaml" conversion_parameter_dict = load_dict_from_file(yaml_file_path) From f981d9cf6ed10eea466bebaf61c059b42461eaec Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sat, 7 Dec 2024 16:09:55 +0100 Subject: [PATCH 07/10] minor fixes --- src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py | 4 ---- .../zaki_2024/zaki_2024_convert_all_sessions.py | 4 ++-- src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py | 2 +- src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py | 1 - 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py index ad0dd52..4dbf01b 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py @@ -166,10 +166,6 @@ def get_rejected_list(self) -> list: """ return list() - def get_roi_ids(self) -> list: - dataset = self._read_zarr_group("/A.zarr") - return list(dataset["unit_id"]) - def get_traces_dict(self) -> dict: """Get traces as a dictionary with key as the name of the ROiResponseSeries. diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py index 79ce922..c66d7fc 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py @@ -122,8 +122,8 @@ def get_session_to_nwb_kwargs_per_session( data_dir_path = Path("D:/") output_dir_path = Path("D:/cai_lab_conversion_nwb/") max_workers = 1 - verbose = True - stub_test = True + verbose = False + stub_test = False dataset_to_nwb( data_dir_path=data_dir_path, output_dir_path=output_dir_path, diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py index 4201505..f274e63 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py @@ -186,7 +186,7 @@ def session_to_nwb( subject_id = "Ca_EEG2-1" session_type = "NeutralExposure" # session_id = subject_id + "_" + session_type - stub_test = True + stub_test = False verbose = True yaml_file_path = Path(__file__).parent / "utils/conversion_parameters.yaml" conversion_parameter_dict = load_dict_from_file(yaml_file_path) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index a7412ce..abbcd14 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -60,7 +60,6 @@ def temporally_align_data_interfaces(self): if imaging_timestamps[0] < 0.0: time_shift = abs(imaging_timestamps[0]) imaging_interface.set_aligned_timestamps(imaging_timestamps + time_shift) - if "MinianSegmentation" in self.data_interface_objects: segmentation_interface = self.data_interface_objects["MinianSegmentation"] segmentation_timestamps = segmentation_interface.get_original_timestamps() From 07eb22d832c820f2de0b2588e0d4797e7de9532b Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sat, 7 Dec 2024 17:03:04 +0100 Subject: [PATCH 08/10] minor fixes --- .../zaki_2024/zaki_2024_convert_session.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py index f274e63..88dea30 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py @@ -31,8 +31,9 @@ def session_to_nwb( sleep_classification_file_path: Union[str, Path] = None, shock_stimulus: dict = None, ): - print(f"Converting session {session_id}") + if verbose: + print(f"Converting session {session_id}") start = time.time() output_dir_path = Path(output_dir_path) @@ -87,9 +88,9 @@ def session_to_nwb( source_data.update( dict(FreezingBehavior=dict(file_path=freezing_output_file_path, video_sampling_frequency=30.0)) ) + conversion_options.update(dict(FreezingBehavior=dict(stub_test=stub_test))) # Add EEG, EMG, Temperature and Activity signals - if edf_file_path: edf_file_path = Path(edf_file_path) assert edf_file_path.is_file(), f"{edf_file_path} does not exist" @@ -144,7 +145,7 @@ def session_to_nwb( ShockStimuli=shock_stimulus, ) - converter = Zaki2024NWBConverter(source_data=source_data) + converter = Zaki2024NWBConverter(source_data=source_data, verbose=verbose) # Add datetime to conversion metadata = converter.get_metadata() @@ -184,9 +185,9 @@ def session_to_nwb( if __name__ == "__main__": subject_id = "Ca_EEG2-1" - session_type = "NeutralExposure" # + session_type = "FC" # session_id = subject_id + "_" + session_type - stub_test = False + stub_test = True verbose = True yaml_file_path = Path(__file__).parent / "utils/conversion_parameters.yaml" conversion_parameter_dict = load_dict_from_file(yaml_file_path) From f1d52f075968f7518ca568b7c64b22ca4d6b360a Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sat, 7 Dec 2024 17:03:25 +0100 Subject: [PATCH 09/10] bug fix --- .../zaki_2024/interfaces/miniscope_imaging_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py index 688b04f..0a1dfac 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py @@ -386,7 +386,7 @@ def add_to_nwbfile( from neuroconv.tools.roiextractors import add_photon_series_to_nwbfile - miniscope_timestamps = self.get_original_timestamps() + miniscope_timestamps = self.get_timestamps() imaging_extractor = self.imaging_extractor if stub_test: From a32d893d4ef1bd63d15c38b8b6fcafb398c21c4f Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Sun, 8 Dec 2024 10:54:52 +0100 Subject: [PATCH 10/10] minor fix --- src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index abbcd14..c0ba848 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -68,7 +68,7 @@ def temporally_align_data_interfaces(self): # if "MinianMotionCorrection" in self.data_interface_objects: # motion_correction_interface = self.data_interface_objects["MinianMotionCorrection"] # motion_correction_timestamps = motion_correction_interface.get_original_timestamps() - # motion_correction_interface.set_aligned_timestamps(segmentation_timestamps + time_shift) + # motion_correction_interface.set_aligned_timestamps(motion_correction_timestamps + time_shift) if "SleepClassification" in self.data_interface_objects: sleep_classification_interface = self.data_interface_objects["SleepClassification"]