From 854b5dc6c601e741c015fc258142b9f50bf279d2 Mon Sep 17 00:00:00 2001 From: danielsf Date: Mon, 4 Oct 2021 21:00:51 -0700 Subject: [PATCH 1/5] updte nightly build script --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2602d372a..be41b0412 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: image: ["allensdk_local_py37:latest"] - branch: ["master", "rc/2.13.1"] + branch: ["master", "rc/2.13.2"] steps: - uses: actions/checkout@v2 with: From 67e69c3e5222738301f8c608af5c18f2b20a7f2e Mon Sep 17 00:00:00 2001 From: Adam Amster Date: Tue, 7 Dec 2021 15:34:44 -0500 Subject: [PATCH 2/5] Updates to support MESO.2 --- .../behavior/data_objects/stimuli/util.py | 4 ++- .../behavior/sync/__init__.py | 31 +++++++++++++------ .../behavior/test_sync_processing.py | 7 +++++ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/util.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/util.py index b3f38b7eb..34c461063 100644 --- a/allensdk/brain_observatory/behavior/data_objects/stimuli/util.py +++ b/allensdk/brain_observatory/behavior/data_objects/stimuli/util.py @@ -50,7 +50,9 @@ def calculate_monitor_delay(sync_file: SyncFile, 'CAM2P.3': 0.021390, 'CAM2P.4': 0.021102, 'CAM2P.5': 0.021192, - 'MESO.1': 0.03613} + 'MESO.1': 0.03613, + # TODO update with actual monitor delay once calculated + 'MESO.2': 0.03613} if equipment_name not in delay_lookup: msg = warning_msg diff --git a/allensdk/brain_observatory/behavior/sync/__init__.py b/allensdk/brain_observatory/behavior/sync/__init__.py index 50610532a..7740e37d0 100644 --- a/allensdk/brain_observatory/behavior/sync/__init__.py +++ b/allensdk/brain_observatory/behavior/sync/__init__.py @@ -32,7 +32,9 @@ def get_raw_stimulus_frames( """ try: - return dataset.get_edges("falling",'stim_vsync', "seconds") + return dataset.get_edges(kind="falling", + keys=["vsync_stim", "stim_vsync"], + units="seconds") except KeyError: if not permissive: raise @@ -63,7 +65,9 @@ def get_ophys_frames( """ try: - return dataset.get_edges("rising", '2p_vsync', "seconds") + return dataset.get_edges(kind="rising", + keys=["vsync_2p", "2p_vsync"], + units="seconds") except KeyError: if not permissive: raise @@ -142,8 +146,12 @@ def get_trigger( correspond to acquired ophys frames. """ - return dataset.get_edges( - "rising", ["2p_trigger", "acq_trigger"], "seconds", permissive) + keys = ["2p_trigger", "acq_trigger", "2p_acq_trigger", "2p_acquiring", + "stim_running"] + return dataset.get_edges(kind="rising", + keys=keys, + units="seconds", + permissive=permissive) def get_eye_tracking( @@ -165,8 +173,11 @@ def get_eye_tracking( dataset. """ - return dataset.get_edges( - "rising", ["cam2_exposure", "eye_tracking"], "seconds", permissive) + keys = ["cam2_exposure", "eye_tracking", "eye_frame_received"] + return dataset.get_edges(kind="rising", + keys=keys, + units="seconds", + permissive=permissive) def get_behavior_monitoring( @@ -189,9 +200,11 @@ def get_behavior_monitoring( sync dataset. """ - return dataset.get_edges( - "rising", ["cam1_exposure", "behavior_monitoring"], "seconds", - permissive) + keys = ["cam1_exposure", "behavior_monitoring", "beh_frame_received"] + return dataset.get_edges(kind="rising", + keys=keys, + units="seconds", + permissive=permissive) def get_sync_data( diff --git a/allensdk/test/brain_observatory/behavior/test_sync_processing.py b/allensdk/test/brain_observatory/behavior/test_sync_processing.py index 02d847999..0c46e40d2 100644 --- a/allensdk/test/brain_observatory/behavior/test_sync_processing.py +++ b/allensdk/test/brain_observatory/behavior/test_sync_processing.py @@ -41,15 +41,22 @@ def test_get_time_sync_integration(sync_path, sync_key, count_exp, last_exp): [sync.get_trigger, "foo", None, None, None], [sync.get_trigger, "2p_trigger", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_trigger, "acq_trigger", [1, 2, 3], [4, 5, 6], [1, 2, 3]], + [sync.get_trigger, "2p_acq_trigger", [1, 2, 3], [4, 5, 6], [1, 2, 3]], + [sync.get_trigger, "2p_acquiring", [1, 2, 3], [4, 5, 6], [1, 2, 3]], + [sync.get_trigger, "stim_running", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_eye_tracking, "cam2_exposure", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_eye_tracking, "eye_tracking", [1, 2, 3], [4, 5, 6], [1, 2, 3]], + [sync.get_eye_tracking, "eye_frame_received", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_behavior_monitoring, "cam1_exposure", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_behavior_monitoring, "behavior_monitoring", [1, 2, 3], [4, 5, 6], [1, 2, 3]], + [sync.get_behavior_monitoring, "beh_frame_received", [1, 2, 3], [4, 5, 6], + [1, 2, 3]], [sync.get_stim_photodiode, "stim_photodiode", [1, 2, 3], [4, 5, 6], [1, 2, 3, 4, 5, 6]], [sync.get_stim_photodiode, "photodiode", [1, 2, 3], [4, 5, 6], [1, 2, 3, 4, 5, 6]], [sync.get_lick_times, "lick_times", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_lick_times, "lick_sensor", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_ophys_frames, "2p_vsync", [1, 2, 3], [4, 5, 6], [1, 2, 3]], + [sync.get_ophys_frames, "vsync_2p", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_raw_stimulus_frames, "stim_vsync", [1, 2, 3], [4, 5, 6], [4, 5, 6]], ]) def test_timestamp_extractors(fn, key, rise, fall, expect): From 54c659aecf71648512b9d91c8f1131aa06d08c2b Mon Sep 17 00:00:00 2001 From: Adam Amster Date: Tue, 7 Dec 2021 15:46:28 -0500 Subject: [PATCH 3/5] lint --- .../behavior/sync/__init__.py | 129 +++++++++--------- .../behavior/test_sync_processing.py | 29 ++-- 2 files changed, 83 insertions(+), 75 deletions(-) diff --git a/allensdk/brain_observatory/behavior/sync/__init__.py b/allensdk/brain_observatory/behavior/sync/__init__.py index 7740e37d0..c0677b80e 100644 --- a/allensdk/brain_observatory/behavior/sync/__init__.py +++ b/allensdk/brain_observatory/behavior/sync/__init__.py @@ -3,32 +3,30 @@ @author: marinag """ -from itertools import chain -from typing import Dict, Any, Optional, List, Union -from allensdk.brain_observatory.behavior.sync.process_sync import ( - filter_digital, calculate_delay) # NOQA: E402 -from allensdk.brain_observatory.sync_dataset import Dataset as SyncDataset # NOQA: E402 +from typing import Dict, Optional, List, Union +from allensdk.brain_observatory.sync_dataset import \ + Dataset as SyncDataset import numpy as np -import scipy.stats as sps def get_raw_stimulus_frames( - dataset: SyncDataset, - permissive: bool = False + dataset: SyncDataset, + permissive: bool = False ) -> np.ndarray: - """ Report the raw timestamps of each stimulus frame. This corresponds to - the time at which the psychopy window's flip method returned, but not + """ Report the raw timestamps of each stimulus frame. This corresponds to + the time at which the psychopy window's flip method returned, but not necessarily to the time at which the stimulus frame was displayed. Parameters ---------- dataset : describes experiment timing - permissive : If True, None will be returned if timestamps are not found. If - False, a KeyError will be raised + permissive : If True, None will be returned if timestamps are not found. + If False, a KeyError will be raised Returns ------- - array of timestamps (floating point; seconds; relative to experiment start). + array of timestamps (floating point; seconds; relative to experiment + start). """ try: @@ -42,7 +40,7 @@ def get_raw_stimulus_frames( def get_ophys_frames( - dataset: SyncDataset, + dataset: SyncDataset, permissive: bool = False ) -> np.ndarray: """ Report the timestamps of each optical physiology video frame @@ -50,16 +48,17 @@ def get_ophys_frames( Parameters ---------- dataset : describes experiment timing + permissive : If True, None will be returned if timestamps are not found. + If False, a KeyError will be raised Returns ------- - array of timestamps (floating point; seconds; relative to experiment start). - permissive : If True, None will be returned if timestamps are not found. If - False, a KeyError will be raised + array of timestamps (floating point; seconds; relative to experiment + start). Notes ----- - use rising edge for Scientifica, falling edge for Nikon + use rising edge for Scientifica, falling edge for Nikon http://confluence.corp.alleninstitute.org/display/IT/Ophys+Time+Sync This function uses rising edges @@ -75,7 +74,7 @@ def get_ophys_frames( def get_lick_times( - dataset: SyncDataset, + dataset: SyncDataset, permissive: bool = False ) -> Optional[np.ndarray]: """ Report the timestamps of each detected lick @@ -83,38 +82,39 @@ def get_lick_times( Parameters ---------- dataset : describes experiment timing - permissive : If True, None will be returned if timestamps are not found. If - False, a KeyError will be raised + permissive : If True, None will be returned if timestamps are not found. + If False, a KeyError will be raised Returns ------- - array of timestamps (floating point; seconds; relative to experiment start) - or None. If None, no lick timestamps were found in this sync + array of timestamps (floating point; seconds; relative to experiment + start) + or None. If None, no lick timestamps were found in this sync dataset. """ return dataset.get_edges( "rising", ["lick_times", "lick_sensor"], "seconds", permissive) - + def get_stim_photodiode( - dataset: SyncDataset, + dataset: SyncDataset, permissive: bool = False ) -> Optional[List[float]]: - """ Report the timestamps of each detected sync square transition (both + """ Report the timestamps of each detected sync square transition (both black -> white and white -> black) in this experiment. Parameters ---------- dataset : describes experiment timing - permissive : If True, None will be returned if timestamps are not found. If - False, a KeyError will be raised + permissive : If True, None will be returned if timestamps are not found. + If False, a KeyError will be raised Returns ------- - array of timestamps (floating point; seconds; relative to experiment start) - or None. If None, no photodiode timestamps were found in this sync - dataset. + array of timestamps (floating point; seconds; relative to experiment + start) or None. If None, no photodiode timestamps were found in this sync + dataset. """ return dataset.get_edges( @@ -122,28 +122,28 @@ def get_stim_photodiode( def get_trigger( - dataset: SyncDataset, + dataset: SyncDataset, permissive: bool = False ) -> Optional[np.ndarray]: - """ Returns (as a 1-element array) the time at which optical physiology + """ Returns (as a 1-element array) the time at which optical physiology acquisition was started. Parameters ---------- dataset : describes experiment timing - permissive : If True, None will be returned if timestamps are not found. If - False, a KeyError will be raised + permissive : If True, None will be returned if timestamps are not found. + If False, a KeyError will be raised Returns ------- - timestamps (floating point; seconds; relative to experiment start) - or None. If None, no timestamps were found in this sync dataset. + timestamps (floating point; seconds; relative to experiment start) + or None. If None, no timestamps were found in this sync dataset. Notes ----- - Ophys frame timestamps can be recorded before acquisition start when - experimenters are setting up the recording session. These do not - correspond to acquired ophys frames. + Ophys frame timestamps can be recorded before acquisition start when + experimenters are setting up the recording session. These do not + correspond to acquired ophys frames. """ keys = ["2p_trigger", "acq_trigger", "2p_acq_trigger", "2p_acquiring", @@ -155,7 +155,7 @@ def get_trigger( def get_eye_tracking( - dataset: SyncDataset, + dataset: SyncDataset, permissive: bool = False ) -> Optional[np.ndarray]: """ Report the timestamps of each frame of the eye tracking video @@ -163,14 +163,14 @@ def get_eye_tracking( Parameters ---------- dataset : describes experiment timing - permissive : If True, None will be returned if timestamps are not found. If - False, a KeyError will be raised + permissive : If True, None will be returned if timestamps are not found. + If False, a KeyError will be raised Returns ------- - array of timestamps (floating point; seconds; relative to experiment start) - or None. If None, no eye tracking timestamps were found in this sync - dataset. + array of timestamps (floating point; seconds; relative to experiment + start) or None. If None, no eye tracking timestamps were found in this + sync dataset. """ keys = ["cam2_exposure", "eye_tracking", "eye_frame_received"] @@ -181,23 +181,23 @@ def get_eye_tracking( def get_behavior_monitoring( - dataset: SyncDataset, + dataset: SyncDataset, permissive: bool = False ) -> Optional[np.ndarray]: - """ Report the timestamps of each frame of the behavior + """ Report the timestamps of each frame of the behavior monitoring video Parameters ---------- dataset : describes experiment timing - permissive : If True, None will be returned if timestamps are not found. If - False, a KeyError will be raised + permissive : If True, None will be returned if timestamps are not found. + If False, a KeyError will be raised Returns ------- - array of timestamps (floating point; seconds; relative to experiment start) - or None. If None, no behavior monitoring timestamps were found in this - sync dataset. + array of timestamps (floating point; seconds; relative to experiment + start) or None. If None, no behavior monitoring timestamps were found in + this sync dataset. """ keys = ["cam1_exposure", "behavior_monitoring", "beh_frame_received"] @@ -211,17 +211,18 @@ def get_sync_data( sync_path: str, permissive: bool = False ) -> Dict[str, Union[List, np.ndarray, None]]: - """ Convenience function for extracting several timestamp arrays from a + """ Convenience function for extracting several timestamp arrays from a sync file. Parameters ---------- - sync_path : The hdf5 file here ought to be a Visual Behavior sync output - file. See allensdk.brain_observatory.sync_dataset for more details of - this format. - permissive : If True, None will be returned if timestamps are not found. If - False, a KeyError will be raised - + sync_path : The hdf5 file here ought to be a Visual Behavior sync output + file. See allensdk.brain_observatory.sync_dataset for more details of + this format. + + permissive : If True, None will be returned if timestamps are not found. + If False, a KeyError will be raised + Returns ------- A dictionary with the following keys. All timestamps in seconds: @@ -232,7 +233,7 @@ def get_sync_data( behavior_monitoring : timestamps of behavior monitoring video frame stim_photodiode : timestamps of each photodiode transition stimulus_times_no_delay : raw stimulus frame timestamps - Some values may be None. This indicates that the corresponding timestamps + Some values may be None. This indicates that the corresponding timestamps were not located in this sync file. """ @@ -243,7 +244,9 @@ def get_sync_data( 'lick_times': get_lick_times(sync_dataset, permissive), 'ophys_trigger': get_trigger(sync_dataset, permissive), 'eye_tracking': get_eye_tracking(sync_dataset, permissive), - 'behavior_monitoring': get_behavior_monitoring(sync_dataset, permissive), + 'behavior_monitoring': get_behavior_monitoring(sync_dataset, + permissive), 'stim_photodiode': get_stim_photodiode(sync_dataset, permissive), - 'stimulus_times_no_delay': get_raw_stimulus_frames(sync_dataset, permissive) + 'stimulus_times_no_delay': get_raw_stimulus_frames(sync_dataset, + permissive) } diff --git a/allensdk/test/brain_observatory/behavior/test_sync_processing.py b/allensdk/test/brain_observatory/behavior/test_sync_processing.py index 0c46e40d2..49aacc60f 100644 --- a/allensdk/test/brain_observatory/behavior/test_sync_processing.py +++ b/allensdk/test/brain_observatory/behavior/test_sync_processing.py @@ -15,8 +15,8 @@ "specimen_789992909", "ophys_session_819949602", ) -sync_path=os.path.join( - base_dir, +sync_path = os.path.join( + base_dir, "819949602_sync.h5" ) @@ -46,18 +46,24 @@ def test_get_time_sync_integration(sync_path, sync_key, count_exp, last_exp): [sync.get_trigger, "stim_running", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_eye_tracking, "cam2_exposure", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_eye_tracking, "eye_tracking", [1, 2, 3], [4, 5, 6], [1, 2, 3]], - [sync.get_eye_tracking, "eye_frame_received", [1, 2, 3], [4, 5, 6], [1, 2, 3]], - [sync.get_behavior_monitoring, "cam1_exposure", [1, 2, 3], [4, 5, 6], [1, 2, 3]], - [sync.get_behavior_monitoring, "behavior_monitoring", [1, 2, 3], [4, 5, 6], [1, 2, 3]], + [sync.get_eye_tracking, "eye_frame_received", [1, 2, 3], [4, 5, 6], + [1, 2, 3]], + [sync.get_behavior_monitoring, "cam1_exposure", [1, 2, 3], [4, 5, 6], + [1, 2, 3]], + [sync.get_behavior_monitoring, "behavior_monitoring", [1, 2, 3], [4, 5, 6], + [1, 2, 3]], [sync.get_behavior_monitoring, "beh_frame_received", [1, 2, 3], [4, 5, 6], [1, 2, 3]], - [sync.get_stim_photodiode, "stim_photodiode", [1, 2, 3], [4, 5, 6], [1, 2, 3, 4, 5, 6]], - [sync.get_stim_photodiode, "photodiode", [1, 2, 3], [4, 5, 6], [1, 2, 3, 4, 5, 6]], + [sync.get_stim_photodiode, "stim_photodiode", [1, 2, 3], [4, 5, 6], + [1, 2, 3, 4, 5, 6]], + [sync.get_stim_photodiode, "photodiode", [1, 2, 3], [4, 5, 6], + [1, 2, 3, 4, 5, 6]], [sync.get_lick_times, "lick_times", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_lick_times, "lick_sensor", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_ophys_frames, "2p_vsync", [1, 2, 3], [4, 5, 6], [1, 2, 3]], [sync.get_ophys_frames, "vsync_2p", [1, 2, 3], [4, 5, 6], [1, 2, 3]], - [sync.get_raw_stimulus_frames, "stim_vsync", [1, 2, 3], [4, 5, 6], [4, 5, 6]], + [sync.get_raw_stimulus_frames, "stim_vsync", [1, 2, 3], [4, 5, 6], + [4, 5, 6]], ]) def test_timestamp_extractors(fn, key, rise, fall, expect): @@ -66,18 +72,17 @@ def __init__(self): self.line_labels = [key, "1", "2"] def get_rising_edges(self, line, units): - if not line in self.line_labels: + if line not in self.line_labels: raise ValueError return rise def get_falling_edges(self, line, units): - if not line in self.line_labels: + if line not in self.line_labels: raise ValueError return fall if expect is None: - with pytest.raises(KeyError) as _err: + with pytest.raises(KeyError): fn(Ds()) else: assert np.allclose(expect, fn(Ds())) - From faaa3fdeadeff352131776a3e09a7e956f0fdfde Mon Sep 17 00:00:00 2001 From: Adam Amster Date: Tue, 7 Dec 2021 15:51:46 -0500 Subject: [PATCH 4/5] update docs --- CHANGELOG.md | 1 + doc_template/index.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0703434b..5f03c91ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file. ## [2.13.2] = TBD - Fixes bug that caused file paths on windows machines to be incorrect in Visual behavior user-facing classes +- Updates to support MESO.2 ## [2.13.1] = 2021-10-04 - Fixes bug that was preventing the BehaviorSession from properly instantiating passive sessions. diff --git a/doc_template/index.rst b/doc_template/index.rst index 54e5d52d9..a28af2afc 100644 --- a/doc_template/index.rst +++ b/doc_template/index.rst @@ -121,6 +121,7 @@ See the `mouse connectivity section `_ for more details. What's New - 2.13.2 ----------------------------------------------------------------------- - Fixes bug that caused file paths on windows machines to be incorrect in Visual behavior user-facing classes +- Updates to support MESO.2 What's New - 2.13.1 ----------------------------------------------------------------------- From 0ac514b84b62f2e794137f36a7c06d9f280f41d2 Mon Sep 17 00:00:00 2001 From: Adam Amster Date: Tue, 7 Dec 2021 15:54:34 -0500 Subject: [PATCH 5/5] add test for MESO.2 --- .../behavior/data_objects/test_trial_table.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_trial_table.py b/allensdk/test/brain_observatory/behavior/data_objects/test_trial_table.py index d1aea946f..a5fa4b179 100644 --- a/allensdk/test/brain_observatory/behavior/data_objects/test_trial_table.py +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_trial_table.py @@ -105,8 +105,9 @@ def setup_method(self, method): self.sync_file = SyncFile(filepath=str(test_data_dir / 'sync.h5')) self.trials = TrialTable(trials=trials) - def test_monitor_delay(self, monkeypatch): - equipment = Equipment(equipment_name='CAM2P.1') + @pytest.mark.parametrize('equipment_name', ('CAMP2.1', 'MESO.2')) + def test_monitor_delay(self, monkeypatch, equipment_name): + equipment = Equipment(equipment_name=equipment_name) def dummy_delay(self): return 1.12