From b7ab33734e63e7cd38f0e511c2536e8c1857cb99 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Wed, 23 Oct 2024 17:45:11 +0200 Subject: [PATCH 01/18] add conversion script --- .../zaki_2024/zaki_2024_convert_session.py | 82 +++++++++++++++---- 1 file changed, 64 insertions(+), 18 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py index 42d88f0..88b7e47 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py @@ -1,14 +1,24 @@ """Primary script to run to convert an entire session for of data using the NWBConverter.""" + from pathlib import Path from typing import Union -import datetime -from zoneinfo import ZoneInfo +from datetime import datetime +import pandas as pd from neuroconv.utils import load_dict_from_file, dict_deep_update from zaki_2024_nwbconverter import Zaki2024NWBConverter -def session_to_nwb(data_dir_path: Union[str, Path], output_dir_path: Union[str, Path], subject_id: str, session_id: str, stub_test: bool = False): + +def session_to_nwb( + data_dir_path: Union[str, Path], + output_dir_path: Union[str, Path], + subject_id: str, + session_id: str, + date_str: str, + time_str: str, + stub_test: bool = False, +): data_dir_path = Path(data_dir_path) output_dir_path = Path(output_dir_path) @@ -21,20 +31,46 @@ def session_to_nwb(data_dir_path: Union[str, Path], output_dir_path: Union[str, source_data = dict() conversion_options = dict() + raw_data_dir_path = data_dir_path / "Ca_EEG_Experiment" / subject_id + + # Add Imaging + miniscope_folder_path = ( + raw_data_dir_path / (subject_id + "_Sessions") / session_id / date_str / time_str / "miniscope" + ) + source_data.update(dict(MiniscopeImaging=dict(folder_path=miniscope_folder_path))) + conversion_options.update(dict(MiniscopeImaging=dict(stub_test=stub_test))) + # Add Segmentation minian_folder_path = data_dir_path / "Ca_EEG_Calcium" / subject_id / session_id / "minian" source_data.update(dict(MinianSegmentation=dict(folder_path=minian_folder_path))) - conversion_options.update(dict(MinianSegmentation=dict(stub_test=stub_test))) + # conversion_options.update(dict(MinianSegmentation=dict(stub_test=stub_test))) + + # Add Behavioral Video + video_file_paths = [raw_data_dir_path / (subject_id + "_Sessions") / session_id / (session_id + ".wmv")] + source_data.update(dict(Video=dict(file_paths=video_file_paths))) + conversion_options.update(dict(Video=dict(stub_test=stub_test))) + + # Add Freezing Analysis output + csv_file_path = raw_data_dir_path / (subject_id + "_Sessions") / session_id / (session_id + "_FreezingOutput.csv") + source_data.update(dict(FreezingBehavior=dict(file_path=csv_file_path, video_sampling_frequency=30.0))) + # conversion_options.update(dict(FreezingBehavior=dict(stub_test=stub_test))) + + # Add EEG, EMG, Temperature and Activity signals + # TODO discuss how to slice this data + datetime_obj = datetime.strptime(date_str, "%Y_%m_%d") + reformatted_date_str = datetime_obj.strftime("_%m%d%y") + edf_file_path = data_dir_path / "Ca_EEG_EDF" / (subject_id + "_EDF") / (subject_id + reformatted_date_str + ".edf") + source_data.update(dict(EDFSignals=dict(file_path=edf_file_path))) converter = Zaki2024NWBConverter(source_data=source_data) # Add datetime to conversion metadata = converter.get_metadata() - datetime.datetime( - year=2020, month=1, day=1, tzinfo=ZoneInfo("US/Eastern") - ) - date = datetime.datetime.today() # TO-DO: Get this from author - metadata["NWBFile"]["session_start_time"] = date + # if session_start_time has been already set from other interfaces do not override + if not metadata["NWBFile"]["session_start_time"]: + datetime_str = date_str + " " + time_str + session_start_time = datetime.strptime(datetime_str, "%Y_%m_%d %H_%M_%S") + metadata["NWBFile"]["session_start_time"] = session_start_time # Update default metadata with the editable in the corresponding yaml file editable_metadata_path = Path(__file__).parent / "zaki_2024_metadata.yaml" @@ -44,7 +80,9 @@ def session_to_nwb(data_dir_path: Union[str, Path], output_dir_path: Union[str, metadata["Subject"]["subject_id"] = subject_id # Run conversion - converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options, overwrite=True) + converter.run_conversion( + metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options, overwrite=True + ) if __name__ == "__main__": @@ -55,11 +93,19 @@ def session_to_nwb(data_dir_path: Union[str, Path], output_dir_path: Union[str, task = "NeutralExposure" session_id = subject_id + "_" + task output_dir_path = Path("D:/cai_lab_conversion_nwb/") - stub_test = False - - session_to_nwb(data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - stub_test=stub_test, - subject_id=subject_id, - session_id=session_id - ) + stub_test = True + session_times_file_path = data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_SessionTimes.csv") + df = pd.read_csv(session_times_file_path) + session_row = df[df["Session"] == task].iloc[0] + date_str = session_row["Date"] + time_str = session_row["Time"] + + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + stub_test=stub_test, + subject_id=subject_id, + session_id=session_id, + date_str=date_str, + time_str=time_str, + ) From cec68e3f2397b5d9555329e4703a3751afae09ea Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Thu, 24 Oct 2024 12:53:26 +0200 Subject: [PATCH 02/18] add motion correction data --- .../zaki_2024/interfaces/minian_interface.py | 19 ------- .../zaki_2024/utils/__init__.py | 2 +- .../zaki_2024/utils/motion_correction.py | 28 ++++++++++ .../zaki_2024/zaki_2024_nwbconverter.py | 51 +++++++++++++++++++ 4 files changed, 80 insertions(+), 20 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py index e7f5c3d..1b3fcc9 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py @@ -136,25 +136,6 @@ def _read_timestamps_from_csv(self): return filtered_df["Time Stamp (ms)"].to_numpy() - def get_motion_correction_data(self) -> np.ndarray: - """Read the xy shifts in the 'motion' field from the zarr object. - - Parameters - ---------- - field: str - The field to read from the zarr object. - - Returns - ------- - motion_correction: numpy.ndarray - The first column is the x shifts. The second column is the y shifts. - """ - dataset = self._read_zarr_group(f"/motion.zarr") - # from zarr field motion.zarr/shift_dim we can verify that the two column refer respectively to - # ['height','width'] --> ['y','x']. Following best practice we swap the two columns - motion_correction = dataset["motion"][:, [1, 0]] - return motion_correction - def get_image_size(self): dataset = self._read_zarr_group("/A.zarr") height = dataset["height"].shape[0] diff --git a/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py b/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py index 5181b87..a9b7d56 100644 --- a/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py +++ b/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py @@ -1,2 +1,2 @@ -from .motion_correction import add_motion_correction +from .motion_correction import add_motion_correction, load_motion_correction_data from .cell_registration import add_cell_registration, get_global_ids_from_csv diff --git a/src/cai_lab_to_nwb/zaki_2024/utils/motion_correction.py b/src/cai_lab_to_nwb/zaki_2024/utils/motion_correction.py index 150c128..a395a41 100644 --- a/src/cai_lab_to_nwb/zaki_2024/utils/motion_correction.py +++ b/src/cai_lab_to_nwb/zaki_2024/utils/motion_correction.py @@ -2,10 +2,38 @@ import numpy as np from neuroconv.tools import get_module +from pydantic.types import PathType from pynwb import NWBFile, TimeSeries from roiextractors.extraction_tools import DtypeType +def load_motion_correction_data(folder_path: PathType) -> np.ndarray: + """Read the xy shifts in the 'motion' field from the zarr object. + + Parameters + ---------- + folder_path: PathType + Path to minian output. + + Returns + ------- + motion_correction: numpy.ndarray + The first column is the x shifts. The second column is the y shifts. + """ + import zarr + import warnings + + if "/motion.zarr" not in zarr.open(folder_path, mode="r"): + warnings.warn(f"Group '/motion.zarr' not found in the Zarr store.", UserWarning) + return None + else: + dataset = zarr.open(str(folder_path) + "/motion.zarr", "r") + # from zarr field motion.zarr/shift_dim we can verify that the two column refer respectively to + # ['height','width'] --> ['y','x']. Following best practice we swap the two columns + motion_correction = dataset["motion"][:, [1, 0]] + return motion_correction + + def add_motion_correction( nwbfile: NWBFile, motion_correction_series: np.ndarray, diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index b7d93d5..158690b 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -1,9 +1,15 @@ """Primary NWBConverter class for this dataset.""" +from fontTools.misc.cython import returns +from pynwb import NWBFile +from typing import Optional + from neuroconv import NWBConverter from neuroconv.datainterfaces import VideoInterface +from utils import add_motion_correction, load_motion_correction_data + from interfaces import ( MinianSegmentationInterface, Zaki2024EDFInterface, @@ -24,3 +30,48 @@ class Zaki2024NWBConverter(NWBConverter): FreezingBehavior=EzTrackFreezingBehaviorInterface, Video=VideoInterface, ) + + # TODO decide which datastream set the session start time + # def get_metadata(self) -> DeepDict: + # if "" not in self.data_interface_objects: + # return super().get_metadata() + # + # # Explicitly set session_start_time to ... start time + # metadata = super().get_metadata() + # session_start_time = self.data_interface_objects[""] + # metadata["NWBFile"]["session_start_time"] = session_start_time + # + # return metadata + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata, conversion_options: Optional[dict] = None) -> None: + super().add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, conversion_options=conversion_options) + + # Add motion correction + if "MinianSegmentation" in self.data_interface_objects: + minian_interface = self.data_interface_objects["MinianSegmentation"] + minian_folder_path = minian_interface.source_data["folder_path"] + interface_name = "MiniscopeImaging" + one_photon_series_name = metadata["Ophys"]["OnePhotonSeries"][0]["name"] + + motion_correction = load_motion_correction_data(minian_folder_path) + if interface_name in conversion_options: + if "stub_test" in conversion_options[interface_name]: + if conversion_options[interface_name]["stub_test"]: + num_frames = 100 + motion_correction = motion_correction[:num_frames, :] + + add_motion_correction( + nwbfile=nwbfile, + motion_correction_series=motion_correction, + one_photon_series_name=one_photon_series_name, + ) + + # TODO discuss time alignment with author + # def temporally_align_data_interfaces(self): + # aligned_starting_time = 0 + # if "MiniscopeImaging" in self.data_interface_classes: + # miniscope_interface = self.data_interface_classes["MiniscopeImaging"] + # miniscope_interface.set_aligned_starting_time(aligned_starting_time=aligned_starting_time) + # if "MinianSegmentation" in self.data_interface_classes: + # minian_interface = self.data_interface_classes["MinianSegmentation"] + # minian_interface.set_aligned_starting_time(aligned_starting_time=aligned_starting_time) From ce25fd0fff876c859e8a4028adcca54be056f218 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Thu, 24 Oct 2024 17:46:25 +0200 Subject: [PATCH 03/18] add reminder to include cell registration --- .../zaki_2024/zaki_2024_nwbconverter.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index 158690b..832f6e5 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -20,7 +20,7 @@ class Zaki2024NWBConverter(NWBConverter): - """Primary conversion class for my extracellular electrophysiology dataset.""" + """Primary conversion class Cai Lab dataset.""" data_interface_classes = dict( MiniscopeImaging=MiniscopeImagingInterface, @@ -46,8 +46,8 @@ class Zaki2024NWBConverter(NWBConverter): def add_to_nwbfile(self, nwbfile: NWBFile, metadata, conversion_options: Optional[dict] = None) -> None: super().add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, conversion_options=conversion_options) - # Add motion correction if "MinianSegmentation" in self.data_interface_objects: + # Add motion correction minian_interface = self.data_interface_objects["MinianSegmentation"] minian_folder_path = minian_interface.source_data["folder_path"] interface_name = "MiniscopeImaging" @@ -66,6 +66,14 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata, conversion_options: Optiona one_photon_series_name=one_photon_series_name, ) + # TODO Add cell global_ids + # global_roi_ids = get_global_ids_from_csv() + # add_cell_registration( + # nwbfile=nwbfile, + # global_roi_ids=global_roi_ids, + # plane_segmentation_name="PlaneSegmentation", + # ) + # TODO discuss time alignment with author # def temporally_align_data_interfaces(self): # aligned_starting_time = 0 From 3f93b59669b70f1ec6102a99a96c154c44583dcb Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Thu, 24 Oct 2024 17:46:57 +0200 Subject: [PATCH 04/18] add checks for data stream existence --- .../zaki_2024/zaki_2024_convert_session.py | 76 +++++++++++++------ 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py index 88b7e47..d929729 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py @@ -4,19 +4,31 @@ from typing import Union from datetime import datetime import pandas as pd - +import json from neuroconv.utils import load_dict_from_file, dict_deep_update from zaki_2024_nwbconverter import Zaki2024NWBConverter +def get_miniscope_folder_path(folder_path: Union[str, Path]): + folder_path = Path(folder_path) + if folder_path.is_dir(): + general_metadata_json = folder_path / "metaData.json" + assert general_metadata_json.exists(), f"General metadata json not found in {folder_path}" + with open(general_metadata_json) as f: + general_metadata = json.load(f) + miniscope_name = general_metadata["miniscopes"][0] + return folder_path / miniscope_name.replace(" ", "_") + else: + print("No Miniscope data found at {}".format(folder_path)) + return None + + def session_to_nwb( data_dir_path: Union[str, Path], output_dir_path: Union[str, Path], subject_id: str, session_id: str, - date_str: str, - time_str: str, stub_test: bool = False, ): @@ -31,36 +43,57 @@ def session_to_nwb( source_data = dict() conversion_options = dict() - raw_data_dir_path = data_dir_path / "Ca_EEG_Experiment" / subject_id + session_times_file_path = data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_SessionTimes.csv") + df = pd.read_csv(session_times_file_path) + session_row = df[df["Session"] == task].iloc[0] + date_str = session_row["Date"] + time_str = session_row["Time"] + + experiment_dir_path = data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_Sessions") # Add Imaging - miniscope_folder_path = ( - raw_data_dir_path / (subject_id + "_Sessions") / session_id / date_str / time_str / "miniscope" - ) - source_data.update(dict(MiniscopeImaging=dict(folder_path=miniscope_folder_path))) - conversion_options.update(dict(MiniscopeImaging=dict(stub_test=stub_test))) + folder_path = experiment_dir_path / session_id / date_str / time_str + miniscope_folder_path = get_miniscope_folder_path(folder_path) + if miniscope_folder_path is not None: + source_data.update(dict(MiniscopeImaging=dict(folder_path=miniscope_folder_path))) + conversion_options.update(dict(MiniscopeImaging=dict(stub_test=stub_test))) # Add Segmentation minian_folder_path = data_dir_path / "Ca_EEG_Calcium" / subject_id / session_id / "minian" - source_data.update(dict(MinianSegmentation=dict(folder_path=minian_folder_path))) - # conversion_options.update(dict(MinianSegmentation=dict(stub_test=stub_test))) + if minian_folder_path.is_dir(): + source_data.update(dict(MinianSegmentation=dict(folder_path=minian_folder_path))) + # conversion_options.update(dict(MinianSegmentation=dict(stub_test=stub_test))) + else: + print("No Minian data found at {}".format(minian_folder_path)) # Add Behavioral Video - video_file_paths = [raw_data_dir_path / (subject_id + "_Sessions") / session_id / (session_id + ".wmv")] - source_data.update(dict(Video=dict(file_paths=video_file_paths))) - conversion_options.update(dict(Video=dict(stub_test=stub_test))) + video_file_path = experiment_dir_path / session_id / (session_id + ".wmv") + if video_file_path.is_file(): + source_data.update(dict(Video=dict(file_paths=[video_file_path]))) + conversion_options.update(dict(Video=dict(stub_test=stub_test))) + else: + print("No behavioral video found at {}".format(video_file_path)) # Add Freezing Analysis output - csv_file_path = raw_data_dir_path / (subject_id + "_Sessions") / session_id / (session_id + "_FreezingOutput.csv") - source_data.update(dict(FreezingBehavior=dict(file_path=csv_file_path, video_sampling_frequency=30.0))) - # conversion_options.update(dict(FreezingBehavior=dict(stub_test=stub_test))) + freezing_output_file_path = experiment_dir_path / session_id / (session_id + "_FreezingOutput.csv") + if freezing_output_file_path.is_file(): + source_data.update( + dict(FreezingBehavior=dict(file_path=freezing_output_file_path, video_sampling_frequency=30.0)) + ) + # conversion_options.update(dict(FreezingBehavior=dict(stub_test=stub_test))) + else: + print("No freezing output csv file found at {}".format(freezing_output_file_path)) # Add EEG, EMG, Temperature and Activity signals # TODO discuss how to slice this data datetime_obj = datetime.strptime(date_str, "%Y_%m_%d") reformatted_date_str = datetime_obj.strftime("_%m%d%y") edf_file_path = data_dir_path / "Ca_EEG_EDF" / (subject_id + "_EDF") / (subject_id + reformatted_date_str + ".edf") - source_data.update(dict(EDFSignals=dict(file_path=edf_file_path))) + if edf_file_path.is_file(): + source_data.update(dict(EDFSignals=dict(file_path=edf_file_path))) + # conversion_options.update(dict(EDFSignals=dict(stub_test=stub_test))) + else: + print("No .edf file found at {}".format(edf_file_path)) converter = Zaki2024NWBConverter(source_data=source_data) @@ -94,11 +127,6 @@ def session_to_nwb( session_id = subject_id + "_" + task output_dir_path = Path("D:/cai_lab_conversion_nwb/") stub_test = True - session_times_file_path = data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_SessionTimes.csv") - df = pd.read_csv(session_times_file_path) - session_row = df[df["Session"] == task].iloc[0] - date_str = session_row["Date"] - time_str = session_row["Time"] session_to_nwb( data_dir_path=data_dir_path, @@ -106,6 +134,4 @@ def session_to_nwb( stub_test=stub_test, subject_id=subject_id, session_id=session_id, - date_str=date_str, - time_str=time_str, ) From efa934f646c712d108eb55627c1197633172b2a4 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Thu, 24 Oct 2024 18:24:27 +0200 Subject: [PATCH 05/18] add convert all sessions --- .../zaki_2024_convert_all_sessions.py | 52 +++++++++++++++---- .../zaki_2024/zaki_2024_convert_session.py | 32 +++++++++--- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py index 51565b8..6aa9398 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py @@ -1,4 +1,5 @@ """Primary script to run to convert all sessions in a dataset using session_to_nwb.""" + from pathlib import Path from typing import Union from concurrent.futures import ProcessPoolExecutor, as_completed @@ -6,7 +7,7 @@ import traceback from tqdm import tqdm -from .zaki_2024_convert_session import session_to_nwb +from zaki_2024_convert_session import session_to_nwb def dataset_to_nwb( @@ -30,6 +31,7 @@ def dataset_to_nwb( Whether to print verbose output, by default True """ data_dir_path = Path(data_dir_path) + output_dir_path = Path(output_dir_path) session_to_nwb_kwargs_per_session = get_session_to_nwb_kwargs_per_session( data_dir_path=data_dir_path, ) @@ -39,7 +41,7 @@ def dataset_to_nwb( for session_to_nwb_kwargs in session_to_nwb_kwargs_per_session: session_to_nwb_kwargs["output_dir_path"] = output_dir_path session_to_nwb_kwargs["verbose"] = verbose - exception_file_path = data_dir_path / f"ERROR_.txt" # Add error file path here + exception_file_path = data_dir_path / f"ERROR_.txt" # Add error file path here futures.append( executor.submit( safe_session_to_nwb, @@ -86,24 +88,52 @@ def get_session_to_nwb_kwargs_per_session( list[dict[str, Any]] A list of dictionaries containing the kwargs for session_to_nwb for each session. """ - ##### - # # Implement this function to return the kwargs for session_to_nwb for each session - # This can be a specific list with hard-coded sessions, a path expansion or any conversion specific logic that you might need - ##### - raise NotImplementedError + ##### + # # Implement this function to return the kwargs for session_to_nwb for each session + # This can be a specific list with hard-coded sessions, a path expansion or any conversion specific logic that you might need + ##### + import pandas as pd + + subjects_df = pd.read_excel(data_dir_path / "Ca_EEG_Design.xlsx") + subjects = subjects_df["Mouse"] + session_to_nwb_kwargs_per_session = [] + for subject_id in subjects: + session_times_file_path = data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_SessionTimes.csv") + if session_times_file_path.is_file(): + session_times_df = pd.read_csv(session_times_file_path) + tasks = [task for task in session_times_df["Session"] if "Offline" not in task] + for task in tasks: + session_id = subject_id + "_" + task + session_row = session_times_df[session_times_df["Session"] == task].iloc[0] + date_str = session_row["Date"] + time_str = session_row["Time"] + session_to_nwb_kwargs_per_session.append( + dict( + data_dir_path=data_dir_path, + subject_id=subject_id, + session_id=session_id, + stub_test=True, + date_str=date_str, + time_str=time_str, + ) + ) + else: + print("Subject {} not found".format(subject_id)) + + return session_to_nwb_kwargs_per_session if __name__ == "__main__": # Parameters for conversion - data_dir_path = Path("/Directory/With/Raw/Formats/") - output_dir_path = Path("~/conversion_nwb/") + data_dir_path = Path("D:/") + output_dir_path = Path("D:/cai_lab_conversion_nwb/") max_workers = 1 - verbose = False + verbose = True dataset_to_nwb( data_dir_path=data_dir_path, output_dir_path=output_dir_path, max_workers=max_workers, - verbose=False, + verbose=verbose, ) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py index d929729..ee6c9a0 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py @@ -1,5 +1,7 @@ """Primary script to run to convert an entire session for of data using the NWBConverter.""" +import time + from pathlib import Path from typing import Union from datetime import datetime @@ -29,8 +31,14 @@ def session_to_nwb( output_dir_path: Union[str, Path], subject_id: str, session_id: str, + date_str: str, + time_str: str, stub_test: bool = False, + verbose: bool = True, ): + print("Converting session {}".format(session_id)) + if verbose: + start = time.time() data_dir_path = Path(data_dir_path) output_dir_path = Path(output_dir_path) @@ -43,12 +51,6 @@ def session_to_nwb( source_data = dict() conversion_options = dict() - session_times_file_path = data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_SessionTimes.csv") - df = pd.read_csv(session_times_file_path) - session_row = df[df["Session"] == task].iloc[0] - date_str = session_row["Date"] - time_str = session_row["Time"] - experiment_dir_path = data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_Sessions") # Add Imaging @@ -117,6 +119,16 @@ def session_to_nwb( metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options, overwrite=True ) + if verbose: + stop_time = time.time() + conversion_time_seconds = stop_time - start + if conversion_time_seconds <= 60 * 3: + print(f"Conversion took {conversion_time_seconds:.2f} seconds") + elif conversion_time_seconds <= 60 * 60: + print(f"Conversion took {conversion_time_seconds / 60:.2f} minutes") + else: + print(f"Conversion took {conversion_time_seconds / 60 / 60:.2f} hours") + if __name__ == "__main__": @@ -127,11 +139,17 @@ def session_to_nwb( session_id = subject_id + "_" + task output_dir_path = Path("D:/cai_lab_conversion_nwb/") stub_test = True - + session_times_file_path = data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_SessionTimes.csv") + df = pd.read_csv(session_times_file_path) + session_row = df[df["Session"] == task].iloc[0] + date_str = session_row["Date"] + time_str = session_row["Time"] session_to_nwb( data_dir_path=data_dir_path, output_dir_path=output_dir_path, stub_test=stub_test, subject_id=subject_id, session_id=session_id, + date_str=date_str, + time_str=time_str, ) From 79456f11d404d4618bfd11692f4f89f9db2490ba Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Fri, 25 Oct 2024 13:15:19 +0200 Subject: [PATCH 06/18] fix small bug on opening zarr store --- src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py index 1b3fcc9..0b70cc5 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py @@ -65,7 +65,7 @@ def _read_zarr_group(self, zarr_group=""): zarr.open The zarr object specified by self.folder_path. """ - if zarr_group not in zarr.open(self.folder_path, mode="r"): + if zarr_group not in zarr.open(str(self.folder_path)): warnings.warn(f"Group '{zarr_group}' not found in the Zarr store.", UserWarning) return None else: From 921e95c7de77787061ce69d6fc8234b7a30503a2 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Fri, 25 Oct 2024 13:15:40 +0200 Subject: [PATCH 07/18] add stub option for minian and edf signals --- .../zaki_2024/interfaces/zaki_2024_edf_interface.py | 7 ++++++- src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py | 5 ++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py index 40c4c24..257829e 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py @@ -29,7 +29,9 @@ def get_timestamps_reference_time(self): edf_reader = read_raw_edf(input_fname=self.file_path, verbose=self.verbose) return edf_reader.info["meas_date"] - def add_to_nwbfile(self, nwbfile: NWBFile, **conversion_options) -> NWBFile: + def add_to_nwbfile( + self, nwbfile: NWBFile, stub_test: bool = False, stub_frames: int = 100, **conversion_options + ) -> NWBFile: channels_dict = { "Temp": { @@ -59,6 +61,9 @@ def add_to_nwbfile(self, nwbfile: NWBFile, **conversion_options) -> NWBFile: edf_reader = read_raw_edf(input_fname=self.file_path, verbose=self.verbose) data, times = edf_reader.get_data(picks=list(channels_dict.keys()), return_times=True) data = data.astype("float32") + if stub_test: + data = data[:, :stub_frames] + times = times[:stub_frames] for channel_index, channel_name in enumerate(channels_dict.keys()): time_series_kwargs = channels_dict[channel_name].copy() time_series_kwargs.update(data=data[channel_index], timestamps=times) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py index ee6c9a0..bcbf8f2 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py @@ -64,7 +64,7 @@ def session_to_nwb( minian_folder_path = data_dir_path / "Ca_EEG_Calcium" / subject_id / session_id / "minian" if minian_folder_path.is_dir(): source_data.update(dict(MinianSegmentation=dict(folder_path=minian_folder_path))) - # conversion_options.update(dict(MinianSegmentation=dict(stub_test=stub_test))) + conversion_options.update(dict(MinianSegmentation=dict(stub_test=stub_test))) else: print("No Minian data found at {}".format(minian_folder_path)) @@ -82,7 +82,6 @@ def session_to_nwb( source_data.update( dict(FreezingBehavior=dict(file_path=freezing_output_file_path, video_sampling_frequency=30.0)) ) - # conversion_options.update(dict(FreezingBehavior=dict(stub_test=stub_test))) else: print("No freezing output csv file found at {}".format(freezing_output_file_path)) @@ -93,7 +92,7 @@ def session_to_nwb( edf_file_path = data_dir_path / "Ca_EEG_EDF" / (subject_id + "_EDF") / (subject_id + reformatted_date_str + ".edf") if edf_file_path.is_file(): source_data.update(dict(EDFSignals=dict(file_path=edf_file_path))) - # conversion_options.update(dict(EDFSignals=dict(stub_test=stub_test))) + conversion_options.update(dict(EDFSignals=dict(stub_test=stub_test))) else: print("No .edf file found at {}".format(edf_file_path)) From f39fdc753a390f86fb876faf0b8bdfbc87deecf3 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Fri, 25 Oct 2024 13:16:55 +0200 Subject: [PATCH 08/18] add TODO on edf interface --- .../zaki_2024/interfaces/zaki_2024_edf_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py index 257829e..4b12f9b 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/zaki_2024_edf_interface.py @@ -61,6 +61,7 @@ def add_to_nwbfile( edf_reader = read_raw_edf(input_fname=self.file_path, verbose=self.verbose) data, times = edf_reader.get_data(picks=list(channels_dict.keys()), return_times=True) data = data.astype("float32") + # TODO select the correct time range if stub_test: data = data[:, :stub_frames] times = times[:stub_frames] From b51bdc9e95bf1eda86ef2333491fcceaa1e1118c Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Fri, 25 Oct 2024 18:40:08 +0200 Subject: [PATCH 09/18] add ophys metadata file --- .../zaki_2024/zaki_2024_ophys_metadata.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml new file mode 100644 index 0000000..6fb2a43 --- /dev/null +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml @@ -0,0 +1,21 @@ +Ophys: + Device: + - name: Miniscope + description: The V4 Miniscope is the head-mounted miniature microscope part of the UCLA Miniscope imaging platform. + manufacturer: UCLA Miniscope. + OnePhotonSeries: + - name: OnePhotonSeries + description: Imaging data from Miniscope. + imaging_plane: ImagingPlane + unit: n.a. + ImagingPlane: + - name: ImagingPlane + description: Imaging plane for Miniscope imaging data. + excitation_lambda: 496.0 + location: CA1 + device: Miniscope + optical_channel: + - name: Green + description: Green channel of the microscope. + emission_lambda: 513.0 + indicator: GCaMP6f \ No newline at end of file From 799559280189fd8d48865f4263be9d38fb6300c8 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Fri, 25 Oct 2024 18:55:45 +0200 Subject: [PATCH 10/18] add minian motion correction interface --- .../zaki_2024/interfaces/minian_interface.py | 199 +++++++++++++++++- 1 file changed, 197 insertions(+), 2 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py index 0b70cc5..09e8143 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py @@ -2,13 +2,19 @@ import warnings import numpy as np import pandas as pd +from neuroconv import BaseDataInterface +from neuroconv.tools import get_module -from roiextractors.extraction_tools import PathType +from roiextractors.extraction_tools import PathType, DtypeType, get_package from neuroconv.datainterfaces.ophys.basesegmentationextractorinterface import BaseSegmentationExtractorInterface from roiextractors.segmentationextractor import SegmentationExtractor +from roiextractors.imagingextractor import ImagingExtractor from typing import Optional -from pynwb import NWBFile +from pynwb import NWBFile, TimeSeries +from pynwb.ophys import MotionCorrection, CorrectedImageStack, OnePhotonSeries + +from explore_segmentation_data import extractor class MinianSegmentationExtractor(SegmentationExtractor): @@ -250,3 +256,192 @@ def add_to_nwbfile( plane_segmentation_name=plane_segmentation_name, iterator_options=iterator_options, ) + + +class _MinianMotionCorrectedVideoExtractor(ImagingExtractor): + """An auxiliar extractor to get data from a single Minian motion corrected video (.mp4) file. + + This format consists of a single video (.mp4) + """ + + extractor_name = "_MinianMotionCorrectedVideo" + + def __init__(self, file_path: PathType): + """Create a _MinianMotionCorrectedVideoExtractor instance from a file path. + + Parameters + ---------- + file_path: PathType + The file path to the Minian motion corrected video (.mp4) file. + """ + from neuroconv.datainterfaces.behavior.video.video_utils import VideoCaptureContext + + self._video_capture = VideoCaptureContext + self._cv2 = get_package(package_name="cv2", installation_instructions="pip install opencv-python-headless") + self.file_path = file_path + super().__init__() + + cap = self._cv2.VideoCapture(str(self.file_path)) + + self._num_frames = int(cap.get(self._cv2.CAP_PROP_FRAME_COUNT)) + + # Get the frames per second (fps) + self._sampling_frequency = cap.get(self._cv2.CAP_PROP_FPS) + self.frame_width = int(cap.get(self._cv2.CAP_PROP_FRAME_WIDTH) / 2) + self.frame_height = int(cap.get(self._cv2.CAP_PROP_FRAME_HEIGHT)) + + _, frame = cap.read() + self._dtype = frame.dtype + + # Release the video capture object + cap.release() + + def get_num_frames(self) -> int: + return self._num_frames + + def get_num_channels(self) -> int: + return 1 + + def get_image_size(self) -> tuple[int, int]: + return (self.frame_height, self.frame_width) + + def get_sampling_frequency(self): + return self._sampling_frequency + + def get_dtype(self) -> DtypeType: + return self._dtype + + def get_channel_names(self) -> list[str]: + return ["OpticalChannel"] + + def get_video( + self, start_frame: Optional[int] = None, end_frame: Optional[int] = None, channel: int = 0 + ) -> np.ndarray: + """Get the video frames. + + Parameters + ---------- + start_frame: int, optional + Start frame index (inclusive). + end_frame: int, optional + End frame index (exclusive). + channel: int, optional + Channel index. + + Returns + ------- + video: numpy.ndarray + The video frames. + + Notes + ----- + The grayscale conversion is based on minian + https://github.com/denisecailab/minian/blob/f64c456ca027200e19cf40a80f0596106918fd09/minian/utilities.py#LL272C12-L272C12 + """ + if channel != 0: + raise NotImplementedError( + f"The {self.extractor_name}Extractor does not currently support multiple color channels." + ) + + end_frame = end_frame or self.get_num_frames() + start_frame = start_frame or 0 + + video = np.empty(shape=(end_frame - start_frame, *self.get_image_size()), dtype=self.get_dtype()) + with self._video_capture(file_path=str(self.file_path)) as video_obj: + # Set the starting frame position + video_obj.current_frame = start_frame + for frame_number in range(end_frame - start_frame): + frame = next(video_obj) + + # motion corrected video in minian are saved side by side (right side) with non-corrected video + # thus we need to extract only the right side of the frames + video[frame_number] = self._cv2.cvtColor(frame[:, self.frame_width :], self._cv2.COLOR_RGB2GRAY) + + return video + + +class MinianMotionCorrectionInterface(BaseDataInterface): + """Data interface for adding the motion corrected image data produced by Minian software.""" + + display_name = "MinianMotionCorrection" + associated_suffixes = (".mp4", ".zarr") + info = "Interface for motion corrected imaging data produced by Minian software." + + def __init__(self, folder_path: PathType, verbose: bool = True): + """ + Interface for motion corrected imaging data produced by Minian software. + + Parameters + ---------- + folder_path : PathType + Path to .zarr path (Minian output). + verbose : bool, default True + Whether to print progress + """ + super().__init__(folder_path=folder_path, verbose=verbose) + + def add_to_nwbfile( + self, + nwbfile: NWBFile, + metadata: dict, + corrected_image_stack_name: str = "CorrectedImageStack", + stub_test: bool = False, + ) -> None: + + # extract xy_shift + assert "/motion.zarr" in zarr.open(self.folder_path), f"Group '/motion.zarr' not found in the Zarr store." + dataset = zarr.open(str(self.folder_path) + "/motion.zarr") + # from zarr field motion.zarr/shift_dim we can verify that the two column refer respectively to + # ['height','width'] --> ['y','x']. Following best practice we swap the two columns + xy_shifts = dataset["motion"][:, [1, 0]] + + # extract corrected image stack + motion_corrected_video_file_path = self.folder_path / "minian_mc.mp4" + extractor = _MinianMotionCorrectedVideoExtractor(file_path=str(motion_corrected_video_file_path)) + end_frame = 100 if stub_test else None + motion_corrected_data = extractor.get_video(end_frame=end_frame) + + # add motion correction + name_suffix = corrected_image_stack_name.replace("CorrectedImageStack", "") + original_one_photon_series_name = f"OnePhotonSeries{name_suffix}" + assert ( + original_one_photon_series_name in nwbfile.acquisition + ), f"The one photon series '{original_one_photon_series_name}' does not exist in the NWBFile." + + original_one_photon_series = nwbfile.acquisition[original_one_photon_series_name] + + timestamps = original_one_photon_series.timestamps + assert timestamps is not None, "The timestamps for the original one photon series must be set." + assert len(xy_shifts) == len(timestamps), "The length of the xy shifts must match the length of the timestamps." + + xy_translation = TimeSeries( + name="xy_translation", # name must be 'xy_translation' + data=xy_shifts[:end_frame, :], + description=f"The x, y shifts for the {original_one_photon_series_name} imaging data.", + unit="px", + timestamps=timestamps[:end_frame], + ) + + corrected_one_photon_series = OnePhotonSeries( + name="corrected", # name must be 'corrected' + data=motion_corrected_data, + imaging_plane=original_one_photon_series.imaging_plane, + timestamps=timestamps[:end_frame], + unit="n.a.", + ) + + corrected_image_stack = CorrectedImageStack( + name=corrected_image_stack_name, + corrected=corrected_one_photon_series, + original=original_one_photon_series, + xy_translation=xy_translation, + ) + + ophys = get_module(nwbfile, "ophys") + if "MotionCorrection" not in ophys.data_interfaces: + motion_correction = MotionCorrection(name="MotionCorrection") + ophys.add(motion_correction) + else: + motion_correction = ophys.data_interfaces["MotionCorrection"] + + motion_correction.add_corrected_image_stack(corrected_image_stack) From 071ce8d4a27abb051818b9fd5f34a9b4af6140d5 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Mon, 28 Oct 2024 14:22:26 +0100 Subject: [PATCH 11/18] update subject age --- src/cai_lab_to_nwb/zaki_2024/zaki_2024_metadata.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_metadata.yaml b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_metadata.yaml index dc300ec..e89724c 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_metadata.yaml +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_metadata.yaml @@ -37,5 +37,5 @@ NWBFile: - Zaki, Yosif Subject: species: Mus musculus - age: TBD # in ISO 8601, such as "P1W2D" - sex: TBD # One of M, F, U, or O + age: P12W/P15W # in ISO 8601, such as "P1W2D" + sex: U # One of M, F, U, or O From 35a41cf6c6279c241209164300ee563fb16703c5 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Mon, 28 Oct 2024 14:24:46 +0100 Subject: [PATCH 12/18] add motion correction interface --- .../assets/experimental_setup.excalidraw | 82 +++++++++---------- .../zaki_2024/interfaces/__init__.py | 2 +- .../zaki_2024/interfaces/minian_interface.py | 27 +++--- .../zaki_2024_convert_all_sessions.py | 2 +- ...zaki_2024_convert_conditioning_session.py} | 8 ++ .../zaki_2024/zaki_2024_nwbconverter.py | 70 +++++++--------- 6 files changed, 94 insertions(+), 97 deletions(-) rename src/cai_lab_to_nwb/zaki_2024/{zaki_2024_convert_session.py => zaki_2024_convert_conditioning_session.py} (94%) diff --git a/src/cai_lab_to_nwb/zaki_2024/assets/experimental_setup.excalidraw b/src/cai_lab_to_nwb/zaki_2024/assets/experimental_setup.excalidraw index 2bce307..539d341 100644 --- a/src/cai_lab_to_nwb/zaki_2024/assets/experimental_setup.excalidraw +++ b/src/cai_lab_to_nwb/zaki_2024/assets/experimental_setup.excalidraw @@ -1,12 +1,12 @@ { "type": "excalidraw", "version": 2, - "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", + "source": "https://excalidraw-jetbrains-plugin", "elements": [ { "type": "text", - "version": 116, - "versionNonce": 1625403132, + "version": 117, + "versionNonce": 908106764, "isDeleted": false, "id": "L-aY7li9S6byMi9dRpI0g", "fillStyle": "solid", @@ -19,14 +19,14 @@ "y": 385.6060606060606, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", - "width": 49.51994323730469, + "width": 49.51997375488281, "height": 25, "seed": 1759143978, "groupIds": [], "frameId": null, "roundness": null, "boundElements": [], - "updated": 1728393699195, + "updated": 1730110773641, "link": null, "locked": false, "fontSize": 20, @@ -111,8 +111,8 @@ }, { "type": "text", - "version": 182, - "versionNonce": 1519505092, + "version": 183, + "versionNonce": 785382068, "isDeleted": false, "id": "5skludqUfD0GwbR4KnOgE", "fillStyle": "solid", @@ -125,14 +125,14 @@ "y": 378.0858585858586, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", - "width": 57.71992492675781, + "width": 57.719970703125, "height": 25, "seed": 887374454, "groupIds": [], "frameId": null, "roundness": null, "boundElements": [], - "updated": 1728393699196, + "updated": 1730110773641, "link": null, "locked": false, "fontSize": 20, @@ -217,8 +217,8 @@ }, { "type": "text", - "version": 215, - "versionNonce": 726298492, + "version": 216, + "versionNonce": 1648966284, "isDeleted": false, "id": "xwJEPjW4IczAvod45gqWw", "fillStyle": "solid", @@ -231,14 +231,14 @@ "y": 372.57828282828285, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", - "width": 56.899932861328125, + "width": 56.89996337890625, "height": 25, "seed": 1865596333, "groupIds": [], "frameId": null, "roundness": null, "boundElements": [], - "updated": 1728393699196, + "updated": 1730110773642, "link": null, "locked": false, "fontSize": 20, @@ -253,8 +253,8 @@ }, { "type": "text", - "version": 237, - "versionNonce": 1483953732, + "version": 238, + "versionNonce": 770081844, "isDeleted": false, "id": "EMIlW1zYnf5zJmKH25xk7", "fillStyle": "solid", @@ -267,14 +267,14 @@ "y": 375.0934343434344, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", - "width": 56.459930419921875, + "width": 56.4599609375, "height": 25, "seed": 1682743459, "groupIds": [], "frameId": null, "roundness": null, "boundElements": [], - "updated": 1728393699197, + "updated": 1730110773642, "link": null, "locked": false, "fontSize": 20, @@ -289,8 +289,8 @@ }, { "type": "text", - "version": 246, - "versionNonce": 466034684, + "version": 247, + "versionNonce": 1826228492, "isDeleted": false, "id": "kwlpwOZPvqUt2M6P7dWmp", "fillStyle": "solid", @@ -303,14 +303,14 @@ "y": 372.9823232323234, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", - "width": 56.899932861328125, + "width": 56.89996337890625, "height": 25, "seed": 1778501603, "groupIds": [], "frameId": null, "roundness": null, "boundElements": [], - "updated": 1728393699197, + "updated": 1730110773643, "link": null, "locked": false, "fontSize": 20, @@ -1517,8 +1517,8 @@ }, { "type": "text", - "version": 101, - "versionNonce": 293442884, + "version": 102, + "versionNonce": 1819270580, "isDeleted": false, "id": "h2z4qkZL4niP7z-OkJqNJ", "fillStyle": "solid", @@ -1531,14 +1531,14 @@ "y": 431.7045454545455, "strokeColor": "#1971c2", "backgroundColor": "#e8590c", - "width": 73.67991638183594, + "width": 73.679931640625, "height": 25, "seed": 847432859, "groupIds": [], "frameId": null, "roundness": null, "boundElements": [], - "updated": 1728393699198, + "updated": 1730110773644, "link": null, "locked": false, "fontSize": 20, @@ -1625,8 +1625,8 @@ }, { "type": "text", - "version": 185, - "versionNonce": 2022232444, + "version": 186, + "versionNonce": 961349516, "isDeleted": false, "id": "AgsUL343nkP_YeK0xO6lR", "fillStyle": "solid", @@ -1639,14 +1639,14 @@ "y": 414.7186883807135, "strokeColor": "#1971c2", "backgroundColor": "#e8590c", - "width": 73.67991638183594, + "width": 73.679931640625, "height": 25, "seed": 1837666779, "groupIds": [], "frameId": null, "roundness": null, "boundElements": [], - "updated": 1728393699200, + "updated": 1730110773645, "link": null, "locked": false, "fontSize": 20, @@ -1697,8 +1697,8 @@ }, { "type": "text", - "version": 201, - "versionNonce": 921256444, + "version": 202, + "versionNonce": 328337204, "isDeleted": false, "id": "yVDlE4w1dC7oAQXrBm1Ji", "fillStyle": "solid", @@ -1711,14 +1711,14 @@ "y": 414.32952844424653, "strokeColor": "#1971c2", "backgroundColor": "#e8590c", - "width": 73.67991638183594, + "width": 73.679931640625, "height": 25, "seed": 1149625717, "groupIds": [], "frameId": null, "roundness": null, "boundElements": [], - "updated": 1728393699200, + "updated": 1730110773645, "link": null, "locked": false, "fontSize": 20, @@ -1913,8 +1913,8 @@ }, { "type": "text", - "version": 264, - "versionNonce": 1348900432, + "version": 266, + "versionNonce": 1886964276, "isDeleted": false, "id": "hTKFkUsoeSb91JPHBBE3d", "fillStyle": "solid", @@ -1934,7 +1934,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1726851109249, + "updated": 1730110773724, "link": null, "locked": false, "fontSize": 20, @@ -1945,12 +1945,12 @@ "containerId": null, "originalText": "When the animal is taken out of the vivarium then the EEG is sort of meaningless, signal strength data in the edf reader should indicate this.", "lineHeight": 1.2, - "baseline": 18 + "baseline": 19 }, { "type": "text", - "version": 308, - "versionNonce": 1593859664, + "version": 310, + "versionNonce": 133636876, "isDeleted": false, "id": "cEPATEz6uaUihnMDzjGnh", "fillStyle": "solid", @@ -1975,7 +1975,7 @@ "type": "arrow" } ], - "updated": 1726851152071, + "updated": 1730110773725, "link": null, "locked": false, "fontSize": 20, @@ -1986,7 +1986,7 @@ "containerId": null, "originalText": "We should build the pipeline assuming those are present\nThey are not present in the two sesions that we got.", "lineHeight": 1.2, - "baseline": 42 + "baseline": 43 }, { "type": "arrow", diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/__init__.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/__init__.py index 9d11419..88ab8d2 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/__init__.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/__init__.py @@ -1,5 +1,5 @@ from .eztrack_interface import EzTrackFreezingBehaviorInterface from .zaki_2024_edf_interface import Zaki2024EDFInterface -from .minian_interface import MinianSegmentationInterface +from .minian_interface import MinianSegmentationInterface, MinianMotionCorrectionInterface from .zaki_2024_sleep_classification_interface import Zaki2024SleepClassificationInterface from .miniscope_imaging_interface import MiniscopeImagingInterface diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py index 09e8143..8f64290 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/minian_interface.py @@ -14,8 +14,6 @@ from pynwb import NWBFile, TimeSeries from pynwb.ophys import MotionCorrection, CorrectedImageStack, OnePhotonSeries -from explore_segmentation_data import extractor - class MinianSegmentationExtractor(SegmentationExtractor): """A SegmentationExtractor for Minian. @@ -306,7 +304,7 @@ def get_image_size(self) -> tuple[int, int]: return (self.frame_height, self.frame_width) def get_sampling_frequency(self): - return self._sampling_frequency + return None def get_dtype(self) -> DtypeType: return self._dtype @@ -367,18 +365,22 @@ class MinianMotionCorrectionInterface(BaseDataInterface): associated_suffixes = (".mp4", ".zarr") info = "Interface for motion corrected imaging data produced by Minian software." - def __init__(self, folder_path: PathType, verbose: bool = True): + def __init__(self, folder_path: PathType, video_file_path: PathType, verbose: bool = True): """ Interface for motion corrected imaging data produced by Minian software. Parameters ---------- folder_path : PathType - Path to .zarr path (Minian output). + Path to .zarr store (Minian output). + video_file_path : PathType + Path to .mp4 video of motion corrected images. verbose : bool, default True Whether to print progress """ - super().__init__(folder_path=folder_path, verbose=verbose) + super().__init__(folder_path=folder_path, video_file_path=video_file_path, verbose=verbose) + self.folder_path = folder_path + self.video_file_path = video_file_path def add_to_nwbfile( self, @@ -395,9 +397,14 @@ def add_to_nwbfile( # ['height','width'] --> ['y','x']. Following best practice we swap the two columns xy_shifts = dataset["motion"][:, [1, 0]] + csv_file = self.folder_path / "timeStamps.csv" + df = pd.read_csv(csv_file) + frame_numbers = dataset["frame"] + filtered_df = df[df["Frame Number"].isin(frame_numbers)] * 1e-3 + timestamps = filtered_df["Time Stamp (ms)"].to_numpy() + # extract corrected image stack - motion_corrected_video_file_path = self.folder_path / "minian_mc.mp4" - extractor = _MinianMotionCorrectedVideoExtractor(file_path=str(motion_corrected_video_file_path)) + extractor = _MinianMotionCorrectedVideoExtractor(file_path=str(self.video_file_path)) end_frame = 100 if stub_test else None motion_corrected_data = extractor.get_video(end_frame=end_frame) @@ -410,10 +417,6 @@ def add_to_nwbfile( original_one_photon_series = nwbfile.acquisition[original_one_photon_series_name] - timestamps = original_one_photon_series.timestamps - assert timestamps is not None, "The timestamps for the original one photon series must be set." - assert len(xy_shifts) == len(timestamps), "The length of the xy shifts must match the length of the timestamps." - xy_translation = TimeSeries( name="xy_translation", # name must be 'xy_translation' data=xy_shifts[:end_frame, :], diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py index 6aa9398..67306be 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_all_sessions.py @@ -7,7 +7,7 @@ import traceback from tqdm import tqdm -from zaki_2024_convert_session import session_to_nwb +from zaki_2024_convert_conditioning_session import session_to_nwb def dataset_to_nwb( diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py similarity index 94% rename from src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py rename to src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py index bcbf8f2..6271290 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py @@ -68,6 +68,14 @@ def session_to_nwb( else: print("No Minian data found at {}".format(minian_folder_path)) + # Add Motion Correction + motion_corrected_video = minian_folder_path / "minian_mc.mp4" + if motion_corrected_video.is_file(): + source_data.update( + dict(MinianMotionCorrection=dict(folder_path=minian_folder_path, video_file_path=motion_corrected_video)) + ) + conversion_options.update(dict(MinianMotionCorrection=dict(stub_test=stub_test))) + # Add Behavioral Video video_file_path = experiment_dir_path / session_id / (session_id + ".wmv") if video_file_path.is_file(): diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index 832f6e5..9f9f1a6 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -16,6 +16,7 @@ EzTrackFreezingBehaviorInterface, Zaki2024SleepClassificationInterface, MiniscopeImagingInterface, + MinianMotionCorrectionInterface, ) @@ -25,61 +26,46 @@ class Zaki2024NWBConverter(NWBConverter): data_interface_classes = dict( MiniscopeImaging=MiniscopeImagingInterface, MinianSegmentation=MinianSegmentationInterface, + MinianMotionCorrection=MinianMotionCorrectionInterface, SleepClassification=Zaki2024SleepClassificationInterface, EDFSignals=Zaki2024EDFInterface, FreezingBehavior=EzTrackFreezingBehaviorInterface, Video=VideoInterface, ) + +""" # TODO decide which datastream set the session start time - # def get_metadata(self) -> DeepDict: - # if "" not in self.data_interface_objects: - # return super().get_metadata() - # - # # Explicitly set session_start_time to ... start time - # metadata = super().get_metadata() - # session_start_time = self.data_interface_objects[""] - # metadata["NWBFile"]["session_start_time"] = session_start_time - # - # return metadata + def get_metadata(self) -> DeepDict: + if "" not in self.data_interface_objects: + return super().get_metadata() + + # Explicitly set session_start_time to ... start time + metadata = super().get_metadata() + session_start_time = self.data_interface_objects[""] + metadata["NWBFile"]["session_start_time"] = session_start_time + return metadata + + # TODO Add cell global_ids def add_to_nwbfile(self, nwbfile: NWBFile, metadata, conversion_options: Optional[dict] = None) -> None: super().add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, conversion_options=conversion_options) if "MinianSegmentation" in self.data_interface_objects: - # Add motion correction - minian_interface = self.data_interface_objects["MinianSegmentation"] - minian_folder_path = minian_interface.source_data["folder_path"] - interface_name = "MiniscopeImaging" - one_photon_series_name = metadata["Ophys"]["OnePhotonSeries"][0]["name"] - - motion_correction = load_motion_correction_data(minian_folder_path) - if interface_name in conversion_options: - if "stub_test" in conversion_options[interface_name]: - if conversion_options[interface_name]["stub_test"]: - num_frames = 100 - motion_correction = motion_correction[:num_frames, :] - - add_motion_correction( + global_roi_ids = get_global_ids_from_csv() + add_cell_registration( nwbfile=nwbfile, - motion_correction_series=motion_correction, - one_photon_series_name=one_photon_series_name, + global_roi_ids=global_roi_ids, + plane_segmentation_name="PlaneSegmentation", ) - # TODO Add cell global_ids - # global_roi_ids = get_global_ids_from_csv() - # add_cell_registration( - # nwbfile=nwbfile, - # global_roi_ids=global_roi_ids, - # plane_segmentation_name="PlaneSegmentation", - # ) - # TODO discuss time alignment with author - # def temporally_align_data_interfaces(self): - # aligned_starting_time = 0 - # if "MiniscopeImaging" in self.data_interface_classes: - # miniscope_interface = self.data_interface_classes["MiniscopeImaging"] - # miniscope_interface.set_aligned_starting_time(aligned_starting_time=aligned_starting_time) - # if "MinianSegmentation" in self.data_interface_classes: - # minian_interface = self.data_interface_classes["MinianSegmentation"] - # minian_interface.set_aligned_starting_time(aligned_starting_time=aligned_starting_time) + def temporally_align_data_interfaces(self): + aligned_starting_time = 0 + if "MiniscopeImaging" in self.data_interface_classes: + miniscope_interface = self.data_interface_classes["MiniscopeImaging"] + miniscope_interface.set_aligned_starting_time(aligned_starting_time=aligned_starting_time) + if "MinianSegmentation" in self.data_interface_classes: + minian_interface = self.data_interface_classes["MinianSegmentation"] + minian_interface.set_aligned_starting_time(aligned_starting_time=aligned_starting_time) +""" From 316fa778d1d3233ae86e85b37581ec5fcdd7ee3b Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Mon, 28 Oct 2024 15:29:45 +0100 Subject: [PATCH 13/18] add description for miniscope device --- .../zaki_2024/interfaces/miniscope_imaging_interface.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py index 7023cfc..379c8f5 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py @@ -249,7 +249,10 @@ def get_metadata(self) -> DeepDict: device_metadata = metadata["Ophys"]["Device"][0] miniscope_config = deepcopy(self._miniscope_config) device_name = miniscope_config.pop("name") - device_metadata.update(name=device_name, **miniscope_config) + description = ( + "The Miniscope is the head-mounted miniature microscope part of the " "UCLA Miniscope imaging platform." + ) + device_metadata.update(name=device_name, description=description, **miniscope_config) # Add link to Device for ImagingPlane imaging_plane_metadata = metadata["Ophys"]["ImagingPlane"][0] imaging_plane_metadata.update( From 2e57187b7d9b2f5c2ffa517b2ba82a1a193ea806 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Mon, 28 Oct 2024 16:32:09 +0100 Subject: [PATCH 14/18] update ophys metadata --- .../interfaces/miniscope_imaging_interface.py | 5 ++-- .../zaki_2024_convert_conditioning_session.py | 5 +++- .../zaki_2024/zaki_2024_nwbconverter.py | 30 +++++++++++++++---- .../zaki_2024/zaki_2024_ophys_metadata.yaml | 8 ++--- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py index 379c8f5..f42ce4f 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py @@ -250,13 +250,12 @@ def get_metadata(self) -> DeepDict: miniscope_config = deepcopy(self._miniscope_config) device_name = miniscope_config.pop("name") description = ( - "The Miniscope is the head-mounted miniature microscope part of the " "UCLA Miniscope imaging platform." + "The Miniscope is the head-mounted miniature microscope part of the UCLA Miniscope imaging platform." ) - device_metadata.update(name=device_name, description=description, **miniscope_config) + device_metadata.update(description=description, **miniscope_config) # Add link to Device for ImagingPlane imaging_plane_metadata = metadata["Ophys"]["ImagingPlane"][0] imaging_plane_metadata.update( - device=device_name, imaging_rate=self._metadata_frame_rate, ) one_photon_series_metadata = metadata["Ophys"]["OnePhotonSeries"][0] diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py index 6271290..76324b5 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py @@ -104,7 +104,10 @@ def session_to_nwb( else: print("No .edf file found at {}".format(edf_file_path)) - converter = Zaki2024NWBConverter(source_data=source_data) + ophys_metadata_path = Path(__file__).parent / "zaki_2024_ophys_metadata.yaml" + ophys_metadata = load_dict_from_file(ophys_metadata_path) + + converter = Zaki2024NWBConverter(source_data=source_data, ophys_metadata=ophys_metadata) # Add datetime to conversion metadata = converter.get_metadata() diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index 9f9f1a6..b07e150 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -1,14 +1,11 @@ """Primary NWBConverter class for this dataset.""" -from fontTools.misc.cython import returns -from pynwb import NWBFile -from typing import Optional +from copy import deepcopy from neuroconv import NWBConverter - +from neuroconv.utils import DeepDict from neuroconv.datainterfaces import VideoInterface - -from utils import add_motion_correction, load_motion_correction_data +from typing import Dict from interfaces import ( MinianSegmentationInterface, @@ -33,6 +30,27 @@ class Zaki2024NWBConverter(NWBConverter): Video=VideoInterface, ) + def __init__( + self, + source_data: Dict[str, dict], + verbose: bool = True, + ophys_metadata=Dict[str, dict], + ): + self.verbose = verbose + self._validate_source_data(source_data=source_data, verbose=self.verbose) + self.data_interface_objects = { + name: data_interface(**source_data[name]) + for name, data_interface in self.data_interface_classes.items() + if name in source_data + } + self.ophys_metadata = ophys_metadata + + def get_metadata(self) -> DeepDict: + metadata = super().get_metadata() + metadata["Ophys"]["OnePhotonSeries"] = self.ophys_metadata["Ophys"]["OnePhotonSeries"] + metadata["Ophys"]["ImagingPlane"] = self.ophys_metadata["Ophys"]["ImagingPlane"] + return metadata + """ # TODO decide which datastream set the session start time diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml index 6fb2a43..2b4e289 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml @@ -1,8 +1,4 @@ Ophys: - Device: - - name: Miniscope - description: The V4 Miniscope is the head-mounted miniature microscope part of the UCLA Miniscope imaging platform. - manufacturer: UCLA Miniscope. OnePhotonSeries: - name: OnePhotonSeries description: Imaging data from Miniscope. @@ -13,9 +9,9 @@ Ophys: description: Imaging plane for Miniscope imaging data. excitation_lambda: 496.0 location: CA1 - device: Miniscope + device: Microscope optical_channel: - - name: Green + - name: OpticalChannel description: Green channel of the microscope. emission_lambda: 513.0 indicator: GCaMP6f \ No newline at end of file From c236951b67800cca8351e8f2290f0456143e5c4d Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Mon, 28 Oct 2024 16:39:05 +0100 Subject: [PATCH 15/18] minor fixes --- .../interfaces/miniscope_imaging_interface.py | 6 ++---- .../zaki_2024_convert_conditioning_session.py | 5 +---- .../zaki_2024/zaki_2024_metadata.yaml | 17 +++++++++++++++ .../zaki_2024/zaki_2024_nwbconverter.py | 21 ------------------- .../zaki_2024/zaki_2024_ophys_metadata.yaml | 17 --------------- 5 files changed, 20 insertions(+), 46 deletions(-) delete mode 100644 src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py index f42ce4f..db2e362 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py @@ -248,16 +248,14 @@ def get_metadata(self) -> DeepDict: device_metadata = metadata["Ophys"]["Device"][0] miniscope_config = deepcopy(self._miniscope_config) - device_name = miniscope_config.pop("name") + miniscope_config.pop("name") description = ( "The Miniscope is the head-mounted miniature microscope part of the UCLA Miniscope imaging platform." ) device_metadata.update(description=description, **miniscope_config) # Add link to Device for ImagingPlane imaging_plane_metadata = metadata["Ophys"]["ImagingPlane"][0] - imaging_plane_metadata.update( - imaging_rate=self._metadata_frame_rate, - ) + imaging_plane_metadata.update(imaging_rate=self._metadata_frame_rate) one_photon_series_metadata = metadata["Ophys"]["OnePhotonSeries"][0] one_photon_series_metadata.update(unit="px") diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py index 76324b5..6271290 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py @@ -104,10 +104,7 @@ def session_to_nwb( else: print("No .edf file found at {}".format(edf_file_path)) - ophys_metadata_path = Path(__file__).parent / "zaki_2024_ophys_metadata.yaml" - ophys_metadata = load_dict_from_file(ophys_metadata_path) - - converter = Zaki2024NWBConverter(source_data=source_data, ophys_metadata=ophys_metadata) + converter = Zaki2024NWBConverter(source_data=source_data) # Add datetime to conversion metadata = converter.get_metadata() diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_metadata.yaml b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_metadata.yaml index e89724c..bf535e7 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_metadata.yaml +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_metadata.yaml @@ -39,3 +39,20 @@ Subject: species: Mus musculus age: P12W/P15W # in ISO 8601, such as "P1W2D" sex: U # One of M, F, U, or O +Ophys: + OnePhotonSeries: + - name: OnePhotonSeries + description: Imaging data from Miniscope. + imaging_plane: ImagingPlane + unit: n.a. + ImagingPlane: + - name: ImagingPlane + description: Imaging plane for Miniscope imaging data. + excitation_lambda: 496.0 + location: CA1 + device: Microscope + optical_channel: + - name: OpticalChannel + description: Green channel of the microscope. + emission_lambda: 513.0 + indicator: GCaMP6f \ No newline at end of file diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py index b07e150..e26bbe3 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_nwbconverter.py @@ -30,27 +30,6 @@ class Zaki2024NWBConverter(NWBConverter): Video=VideoInterface, ) - def __init__( - self, - source_data: Dict[str, dict], - verbose: bool = True, - ophys_metadata=Dict[str, dict], - ): - self.verbose = verbose - self._validate_source_data(source_data=source_data, verbose=self.verbose) - self.data_interface_objects = { - name: data_interface(**source_data[name]) - for name, data_interface in self.data_interface_classes.items() - if name in source_data - } - self.ophys_metadata = ophys_metadata - - def get_metadata(self) -> DeepDict: - metadata = super().get_metadata() - metadata["Ophys"]["OnePhotonSeries"] = self.ophys_metadata["Ophys"]["OnePhotonSeries"] - metadata["Ophys"]["ImagingPlane"] = self.ophys_metadata["Ophys"]["ImagingPlane"] - return metadata - """ # TODO decide which datastream set the session start time diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml deleted file mode 100644 index 2b4e289..0000000 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_ophys_metadata.yaml +++ /dev/null @@ -1,17 +0,0 @@ -Ophys: - OnePhotonSeries: - - name: OnePhotonSeries - description: Imaging data from Miniscope. - imaging_plane: ImagingPlane - unit: n.a. - ImagingPlane: - - name: ImagingPlane - description: Imaging plane for Miniscope imaging data. - excitation_lambda: 496.0 - location: CA1 - device: Microscope - optical_channel: - - name: OpticalChannel - description: Green channel of the microscope. - emission_lambda: 513.0 - indicator: GCaMP6f \ No newline at end of file From f4a95326e58e23203e93f29292fdcaad3a885cc4 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Mon, 28 Oct 2024 16:41:23 +0100 Subject: [PATCH 16/18] minor --- .../zaki_2024/zaki_2024_convert_conditioning_session.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py index 6271290..1c913bb 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py @@ -75,6 +75,8 @@ def session_to_nwb( dict(MinianMotionCorrection=dict(folder_path=minian_folder_path, video_file_path=motion_corrected_video)) ) conversion_options.update(dict(MinianMotionCorrection=dict(stub_test=stub_test))) + else: + print("No motion corrected data found at {}".format(motion_corrected_video)) # Add Behavioral Video video_file_path = experiment_dir_path / session_id / (session_id + ".wmv") From aa752682541a70ed8077912219febc6ef7f4e853 Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Mon, 28 Oct 2024 16:43:35 +0100 Subject: [PATCH 17/18] remove add_motion_correction util fun --- .../zaki_2024/utils/__init__.py | 1 - .../zaki_2024/utils/motion_correction.py | 80 ------------------- 2 files changed, 81 deletions(-) delete mode 100644 src/cai_lab_to_nwb/zaki_2024/utils/motion_correction.py diff --git a/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py b/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py index a9b7d56..38262bf 100644 --- a/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py +++ b/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py @@ -1,2 +1 @@ -from .motion_correction import add_motion_correction, load_motion_correction_data from .cell_registration import add_cell_registration, get_global_ids_from_csv diff --git a/src/cai_lab_to_nwb/zaki_2024/utils/motion_correction.py b/src/cai_lab_to_nwb/zaki_2024/utils/motion_correction.py deleted file mode 100644 index b09af0f..0000000 --- a/src/cai_lab_to_nwb/zaki_2024/utils/motion_correction.py +++ /dev/null @@ -1,80 +0,0 @@ -from typing import List - -import numpy as np -from neuroconv.tools import get_module -from pydantic.types import PathType -from pynwb import NWBFile, TimeSeries -from roiextractors.extraction_tools import DtypeType - - -def load_motion_correction_data(folder_path: PathType) -> np.ndarray: - """Read the xy shifts in the 'motion' field from the zarr object. - - Parameters - ---------- - folder_path: PathType - Path to minian output. - - Returns - ------- - motion_correction: numpy.ndarray - The first column is the x shifts. The second column is the y shifts. - """ - import zarr - import warnings - - if "/motion.zarr" not in zarr.open(folder_path, mode="r"): - warnings.warn(f"Group '/motion.zarr' not found in the Zarr store.", UserWarning) - return None - else: - dataset = zarr.open(str(folder_path) + "/motion.zarr", "r") - # from zarr field motion.zarr/shift_dim we can verify that the two column refer respectively to - # ['height','width'] --> ['y','x']. Following best practice we swap the two columns - motion_correction = dataset["motion"][:, [1, 0]] - return motion_correction - - -def add_motion_correction( - nwbfile: NWBFile, - motion_correction_series: np.ndarray, - one_photon_series_name: str, -) -> None: - """Add motion correction data to the NWBFile. - - The x, y shifts for the imaging data (identified by 'one_photon_series_name' are added to the NWBFile as a TimeSeries. - The series is added to the 'ophys' processing module. - - Parameters - ---------- - nwbfile: NWBFile - The NWBFile where the motion correction time series will be added to. - motion_correction_series: numpy.ndarray - The x, y shifts for the imaging data. - one_photon_series_name: str - The name of the one photon series in the NWBFile. - """ - - assert ( - one_photon_series_name in nwbfile.acquisition - ), f"The one photon series '{one_photon_series_name}' does not exist in the NWBFile." - name_suffix = one_photon_series_name.replace("OnePhotonSeries", "") - motion_correction_time_series_name = "MotionCorrectionSeries" + name_suffix - ophys = get_module(nwbfile, "ophys") - if motion_correction_time_series_name in ophys.data_interfaces: - raise ValueError( - f"The motion correction time series '{motion_correction_time_series_name}' already exists in the NWBFile." - ) - - one_photon_series = nwbfile.acquisition[one_photon_series_name] - num_frames = one_photon_series.data.maxshape[0] - assert ( - num_frames == motion_correction_series.shape[0] - ), f"The number of frames for motion correction ({motion_correction_series.shape[0]}) does not match the number of frames ({num_frames}) from the {one_photon_series_name} imaging data." - xy_translation = TimeSeries( - name="MotionCorrectionSeries" + name_suffix, - description=f"The x, y shifts for the {one_photon_series_name} imaging data.", - data=motion_correction_series, - unit="px", - timestamps=one_photon_series.timestamps, - ) - ophys.add(xy_translation) From 29e5abbf8c0ad9553309e8a882a98259bfa141ec Mon Sep 17 00:00:00 2001 From: alessandratrapani Date: Tue, 29 Oct 2024 09:55:57 +0100 Subject: [PATCH 18/18] nest print statements within elif verbose --- .../zaki_2024_convert_conditioning_session.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py index 1c913bb..f8bbd0f 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_conditioning_session.py @@ -65,7 +65,7 @@ def session_to_nwb( if minian_folder_path.is_dir(): source_data.update(dict(MinianSegmentation=dict(folder_path=minian_folder_path))) conversion_options.update(dict(MinianSegmentation=dict(stub_test=stub_test))) - else: + elif verbose: print("No Minian data found at {}".format(minian_folder_path)) # Add Motion Correction @@ -75,7 +75,7 @@ def session_to_nwb( dict(MinianMotionCorrection=dict(folder_path=minian_folder_path, video_file_path=motion_corrected_video)) ) conversion_options.update(dict(MinianMotionCorrection=dict(stub_test=stub_test))) - else: + elif verbose: print("No motion corrected data found at {}".format(motion_corrected_video)) # Add Behavioral Video @@ -83,7 +83,7 @@ def session_to_nwb( if video_file_path.is_file(): source_data.update(dict(Video=dict(file_paths=[video_file_path]))) conversion_options.update(dict(Video=dict(stub_test=stub_test))) - else: + elif verbose: print("No behavioral video found at {}".format(video_file_path)) # Add Freezing Analysis output @@ -92,7 +92,7 @@ def session_to_nwb( source_data.update( dict(FreezingBehavior=dict(file_path=freezing_output_file_path, video_sampling_frequency=30.0)) ) - else: + elif verbose: print("No freezing output csv file found at {}".format(freezing_output_file_path)) # Add EEG, EMG, Temperature and Activity signals @@ -103,7 +103,7 @@ def session_to_nwb( if edf_file_path.is_file(): source_data.update(dict(EDFSignals=dict(file_path=edf_file_path))) conversion_options.update(dict(EDFSignals=dict(stub_test=stub_test))) - else: + elif verbose: print("No .edf file found at {}".format(edf_file_path)) converter = Zaki2024NWBConverter(source_data=source_data)