diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cd0dfeea..01878c928 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Change Log All notable changes to this project will be documented in this file. +## [2.5.0] = 2021-01-29 +- Adds unfiltered running speed as new data stream +- run_demixing gracefully ignores any ROIs that are not in the input trace file + ## [2.4.1] = 2021-01-04 - update deprecated call to scipy.spatial.transform.Rotation.as_dcm() to .as_matrix() diff --git a/allensdk/__init__.py b/allensdk/__init__.py index 850c3e248..9f51d95c3 100644 --- a/allensdk/__init__.py +++ b/allensdk/__init__.py @@ -37,7 +37,7 @@ -__version__ = '2.4.1' +__version__ = '2.5.0' try: diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_session.py b/allensdk/brain_observatory/behavior/behavior_ophys_session.py index 03d1dd995..68a881d42 100644 --- a/allensdk/brain_observatory/behavior/behavior_ophys_session.py +++ b/allensdk/brain_observatory/behavior/behavior_ophys_session.py @@ -516,11 +516,11 @@ def _get_roi_masks_by_cell_roi_id(self, cell_roi_ids=None): table.set_index("cell_roi_id", inplace=True) table = table.loc[cell_roi_ids, :] - full_image_shape = table.iloc[0]["image_mask"].shape + full_image_shape = table.iloc[0]["roi_mask"].shape output = np.zeros((len(cell_roi_ids), full_image_shape[0], full_image_shape[1]), dtype=np.uint8) for ii, (_, row) in enumerate(table.iterrows()): - output[ii, :, :] = _translate_roi_mask(row["image_mask"], int(row["y"]), int(row["x"])) + output[ii, :, :] = _translate_roi_mask(row["roi_mask"], int(row["y"]), int(row["x"])) # Pixel spacing and units of mask image will match either the # max or avg projection image of 2P movie. diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py index a76648fa2..398933997 100644 --- a/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py +++ b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py @@ -44,9 +44,14 @@ def get_rewards(self) -> pd.DataFrame: raise NotImplementedError() @abc.abstractmethod - def get_running_data_df(self) -> pd.DataFrame: + def get_running_data_df(self, lowpass=True) -> pd.DataFrame: """Get running speed data. + Parameters + ---------- + lowpass: bool + Whether to return running speed with low pass filter applied or without + Returns ------- pd.DataFrame diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py index ec98f8105..3f56e2263 100644 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py +++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py @@ -61,9 +61,10 @@ def save(self, session_object): # Add running data to NWB in-memory object: unit_dict = {'v_sig': 'V', 'v_in': 'V', 'speed': 'cm/s', 'timestamps': 's', 'dx': 'cm'} - nwb.add_running_data_df_to_nwbfile(nwbfile, - session_object.running_data_df, - unit_dict) + nwb.add_running_data_dfs_to_nwbfile(nwbfile, + session_object.running_data_df, + session_object.raw_running_data_df, + unit_dict) # Add stimulus template data to NWB in-memory object: for name, image_data in session_object.stimulus_templates.items(): @@ -148,9 +149,22 @@ def get_ophys_session_id(self) -> int: def get_eye_tracking(self) -> int: raise NotImplementedError() - def get_running_data_df(self, **kwargs) -> pd.DataFrame: + def get_running_data_df(self, lowpass=True) -> pd.DataFrame: + """ + Gets the running data df + Parameters + ---------- + lowpass: bool + Whether to return running speed with or without low pass filter applied - running_speed = self.get_running_speed() + Returns + ------- + pd.DataFrame: + Dataframe containing various signals used to compute running + speed, and the filtered or unfiltered speed. + """ + + running_speed = self.get_running_speed(lowpass=lowpass) running_data_df = pd.DataFrame({'speed': running_speed.values}, index=pd.Index(running_speed.timestamps, @@ -263,6 +277,10 @@ def get_task_parameters(self) -> dict: def get_cell_specimen_table(self) -> pd.DataFrame: # NOTE: ROI masks are stored in full frame width and height arrays df = self.nwbfile.processing['ophys'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'].to_dataframe() + + # Because pynwb stores this field as "image_mask", it is renamed here + df = df.rename(columns={'image_mask': 'roi_mask'}) + df.index.rename('cell_roi_id', inplace=True) df['cell_specimen_id'] = [None if csid == -1 else csid for csid in df['cell_specimen_id'].values] diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py index 2c434a115..db73b2c2c 100644 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py +++ b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py @@ -458,7 +458,7 @@ def get_raw_cell_specimen_table_dict(self) -> dict: """.format(ophys_cell_seg_run_id) initial_cs_table = pd.read_sql(query, self.lims_db.get_connection()) cell_specimen_table = initial_cs_table.rename( - columns={'id': 'cell_roi_id', 'mask_matrix': 'image_mask'}) + columns={'id': 'cell_roi_id', 'mask_matrix': 'roi_mask'}) cell_specimen_table.drop(['ophys_experiment_id', 'ophys_cell_segmentation_run_id'], inplace=True, axis=1) diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py index 3d4ebde09..59ed52ebe 100644 --- a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py +++ b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py @@ -46,7 +46,7 @@ def get_cell_specimen_table(self): fov_height = self.get_field_of_view_shape()['height'] # Convert cropped ROI masks to uncropped versions - image_mask_list = [] + roi_mask_list = [] for cell_roi_id, table_row in cell_specimen_table.iterrows(): # Deserialize roi data into AllenSDK RoiMask object curr_roi = roi.RoiMask(image_w=fov_width, image_h=fov_height, @@ -55,10 +55,10 @@ def get_cell_specimen_table(self): curr_roi.y = table_row['y'] curr_roi.width = table_row['width'] curr_roi.height = table_row['height'] - curr_roi.mask = np.array(table_row['image_mask']) - image_mask_list.append(curr_roi.get_mask_plane().astype(np.bool)) + curr_roi.mask = np.array(table_row['roi_mask']) + roi_mask_list.append(curr_roi.get_mask_plane().astype(np.bool)) - cell_specimen_table['image_mask'] = image_mask_list + cell_specimen_table['roi_mask'] = roi_mask_list cell_specimen_table = cell_specimen_table[sorted(cell_specimen_table.columns)] cell_specimen_table.index.rename('cell_roi_id', inplace=True) @@ -340,7 +340,7 @@ def get_corrected_fluorescence_traces(self): ophys_timestamps = self.get_ophys_timestamps() num_trace_timepoints = corrected_fluorescence_traces.shape[1] - assert num_trace_timepoints, ophys_timestamps.shape[0] + assert num_trace_timepoints == ophys_timestamps.shape[0] df = pd.DataFrame( {'corrected_fluorescence': list(corrected_fluorescence_traces)}, index=pd.Index(cell_roi_id_list, name='cell_roi_id')) diff --git a/allensdk/brain_observatory/behavior/write_nwb/_schemas.py b/allensdk/brain_observatory/behavior/write_nwb/_schemas.py index 38f6df620..4249410ca 100644 --- a/allensdk/brain_observatory/behavior/write_nwb/_schemas.py +++ b/allensdk/brain_observatory/behavior/write_nwb/_schemas.py @@ -18,7 +18,7 @@ class CellSpecimenTable(RaisingSchema): height = Dict(String, Int, required=True) width = Dict(String, Int, required=True) mask_image_plane = Dict(String, Int, required=True) - image_mask = Dict(String, List(List(Boolean)), required=True) + roi_mask = Dict(String, List(List(Boolean)), required=True) diff --git a/allensdk/brain_observatory/nwb/__init__.py b/allensdk/brain_observatory/nwb/__init__.py index f0e5118f3..502009197 100644 --- a/allensdk/brain_observatory/nwb/__init__.py +++ b/allensdk/brain_observatory/nwb/__init__.py @@ -333,37 +333,48 @@ def add_running_speed_to_nwbfile(nwbfile, running_speed, name='speed', unit='cm/ unit=unit ) - running_mod = ProcessingModule('running', 'Running speed processing module') - nwbfile.add_processing_module(running_mod) + if 'running' in nwbfile.processing: + running_mod = nwbfile.processing['running'] + else: + running_mod = ProcessingModule('running', 'Running speed processing module') + nwbfile.add_processing_module(running_mod) running_mod.add_data_interface(running_speed_series) return nwbfile -def add_running_data_df_to_nwbfile(nwbfile, running_data_df, unit_dict, index_key='timestamps'): - ''' Adds running speed data to an NWBFile as timeseries in acquisition and processing +def add_running_data_dfs_to_nwbfile(nwbfile, running_data_df, running_data_df_unfiltered, unit_dict): + """Adds both unfiltered (raw) and filtered running speed data to an NWBFile as timeseries in acquisition and processing Parameters ---------- nwbfile : pynwb.NWBFile - File to which runnign speeds will be written - running_speed : pandas.DataFrame - Contains 'speed' and 'times', 'v_in', 'vsig', 'dx' - unit : str, optional + File to which running speeds will be written + running_data_df : pandas.DataFrame + Filtered running data + Contains 'speed', 'v_in', 'vsig', 'dx' + Note that 'v_in', 'vsig', 'dx' are expected to be the same as in running_data_df_unfiltered + running_data_df_unfiltered : pandas.DataFrame + Unfiltered (raw) Running data + Contains 'speed', 'v_in', 'vsig', 'dx' + Note that 'v_in', 'vsig', 'dx' are expected to be the same as in running_data_df + unit_dict : dict, optional SI units of running speed values Returns ------- nwbfile : pynwb.NWBFile - ''' - assert running_data_df.index.name == index_key - + """ running_speed = RunningSpeed(timestamps=running_data_df.index.values, values=running_data_df['speed'].values) + running_speed_unfiltered = RunningSpeed(timestamps=running_data_df_unfiltered.index.values, + values=running_data_df_unfiltered['speed'].values) + add_running_speed_to_nwbfile(nwbfile, running_speed, name='speed', unit=unit_dict['speed']) + add_running_speed_to_nwbfile(nwbfile, running_speed_unfiltered, name='speed_unfiltered', unit=unit_dict['speed']) running_mod = nwbfile.processing['running'] timestamps_ts = running_mod.get_data_interface('speed').timestamps @@ -883,9 +894,9 @@ def add_cell_specimen_table(nwbfile: NWBFile, imaging_plane=imaging_plane) for col_name in cell_roi_table.columns: - # the columns 'image_mask', 'pixel_mask', and 'voxel_mask' are already defined + # the columns 'roi_mask', 'pixel_mask', and 'voxel_mask' are already defined # in the nwb.ophys::PlaneSegmentation Object - if col_name not in ['id', 'mask_matrix', 'image_mask', 'pixel_mask', 'voxel_mask']: + if col_name not in ['id', 'mask_matrix', 'roi_mask', 'pixel_mask', 'voxel_mask']: # This builds the columns with name of column and description of column # both equal to the column name in the cell_roi_table plane_segmentation.add_column(col_name, @@ -895,13 +906,13 @@ def add_cell_specimen_table(nwbfile: NWBFile, # go through each roi and add it to the plan segmentation object for cell_roi_id, table_row in cell_roi_table.iterrows(): - # NOTE: The 'image_mask' in this cell_roi_table has already been + # NOTE: The 'roi_mask' in this cell_roi_table has already been # processing by the function from # allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api # get_cell_specimen_table() method. As a result, the ROI is stored in # an array that is the same shape as the FULL field of view of the # experiment (e.g. 512 x 512). - mask = table_row.pop('image_mask') + mask = table_row.pop('roi_mask') csid = table_row.pop('cell_specimen_id') table_row['cell_specimen_id'] = -1 if csid is None else csid diff --git a/allensdk/brain_observatory/nwb/nwb_api.py b/allensdk/brain_observatory/nwb/nwb_api.py index b951b6fca..8c74139c0 100644 --- a/allensdk/brain_observatory/nwb/nwb_api.py +++ b/allensdk/brain_observatory/nwb/nwb_api.py @@ -45,10 +45,23 @@ def from_path(cls, path, **kwargs): return cls(path=path, **kwargs) - def get_running_speed(self) -> RunningSpeed: - - values = self.nwbfile.modules['running'].get_data_interface('speed').data[:] - timestamps = self.nwbfile.modules['running'].get_data_interface('speed').timestamps[:] + def get_running_speed(self, lowpass=True) -> RunningSpeed: + """ + Gets the running speed + Parameters + ---------- + lowpass: bool + Whether to return the running speed with lowpass filter applied or without + + Returns + ------- + RunningSpeed: + The running speed + """ + + interface_name = 'speed' if lowpass else 'speed_unfiltered' + values = self.nwbfile.modules['running'].get_data_interface(interface_name).data[:] + timestamps = self.nwbfile.modules['running'].get_data_interface(interface_name).timestamps[:] return RunningSpeed( timestamps=timestamps, diff --git a/allensdk/internal/pipeline_modules/run_demixing.py b/allensdk/internal/pipeline_modules/run_demixing.py index bd9565024..f1c2b3fa1 100644 --- a/allensdk/internal/pipeline_modules/run_demixing.py +++ b/allensdk/internal/pipeline_modules/run_demixing.py @@ -16,7 +16,12 @@ import allensdk.core.json_utilities as ju import logging -EXCLUDE_LABELS = ["union", "duplicate", "motion_border" ] +EXCLUDE_LABELS = ["union", "duplicate", "motion_border", + "decrosstalk_ghost", + "decrosstalk_invalid_raw", + "decrosstalk_invalid_raw_active", + "decrosstalk_invalid_unmixed", + "decrosstalk_invalid_unmixed_active" ] def debug(experiment_id, local=False): OUTPUT_DIRECTORY = "/data/informatics/CAM/demix" @@ -34,13 +39,13 @@ def debug(experiment_id, local=False): """ % experiment_id) nrois = { roi['id']: dict(width=roi['width'], - height=roi['height'], + height=roi['height'], x=roi['x'], y=roi['y'], id=roi['id'], valid=roi['valid_roi'], mask=roi['mask_matrix'], - exclusion_labels=[]) + exclusion_labels=[]) for roi in rois } for exc_label in exc_labels: @@ -48,12 +53,12 @@ def debug(experiment_id, local=False): movie_path_response = lu.query(''' select wkf.filename, wkf.storage_directory from well_known_files wkf - join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id + join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id where wkf.attachable_id = {} and wkf.attachable_type = 'OphysExperiment' and wkft.name = 'MotionCorrectedImageStack' '''.format(experiment_id)) movie_h5_path = os.path.join(movie_path_response[0]['storage_directory'], movie_path_response[0]['filename']) - + exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id)) input_data = { @@ -62,9 +67,9 @@ def debug(experiment_id, local=False): "roi_masks": nrois.values(), "output_file": os.path.join(exp_dir, "demixed_traces.h5") } - - run_module(SCRIPT, - input_data, + + run_module(SCRIPT, + input_data, exp_dir, sdk_path=SDK_PATH, pbs=dict(vmem=160, @@ -99,8 +104,8 @@ def parse_input(data, exclude_labels): movie_shape = f["data"].shape[1:] with h5py.File(traces_h5, "r") as f: - traces = f["data"].value - trace_ids = [ int(rid) for rid in f["roi_names"].value ] + traces = f["data"][()] + trace_ids = [ int(rid) for rid in f["roi_names"][()] ] rois = get_path(data, "roi_masks", False) masks = None @@ -135,7 +140,7 @@ def main(): logging.debug("reading input") traces, masks, valid, trace_ids, movie_h5, output_h5 = parse_input(data, mod.args.exclude_labels) - + logging.debug("excluded masks: %s", str(zip(np.where(~valid)[0], trace_ids[~valid]))) output_dir = os.path.dirname(output_h5) plot_dir = os.path.join(output_dir, "demix_plots") @@ -145,7 +150,7 @@ def main(): logging.debug("reading movie") with h5py.File(movie_h5, 'r') as f: - movie = f['data'].value + movie = f['data'][()] # only demix non-union, non-duplicate ROIs valid_idxs = np.where(valid) @@ -154,9 +159,9 @@ def main(): logging.debug("demixing") demixed_traces, drop_frames = demixer.demix_time_dep_masks(demix_traces, movie, demix_masks) - - nt_inds = demixer.plot_negative_transients(demix_traces, - demixed_traces, + + nt_inds = demixer.plot_negative_transients(demix_traces, + demixed_traces, valid[valid_idxs], demix_masks, trace_ids[valid_idxs], @@ -164,17 +169,17 @@ def main(): logging.debug("rois with negative transients: %s", str(trace_ids[valid_idxs][nt_inds])) - nb_inds = demixer.plot_negative_baselines(demix_traces, - demixed_traces, + nb_inds = demixer.plot_negative_baselines(demix_traces, + demixed_traces, demix_masks, trace_ids[valid_idxs], plot_dir) - # negative baseline rois (and those that overlap with them) become nans + # negative baseline rois (and those that overlap with them) become nans logging.debug("rois with negative baselines (or overlap with them): %s", str(trace_ids[valid_idxs][nb_inds])) demixed_traces[nb_inds, :] = np.nan - logging.info("Saving output") + logging.info("Saving output") out_traces = np.zeros(traces.shape, dtype=demix_traces.dtype) out_traces[:] = np.nan out_traces[valid_idxs] = demixed_traces diff --git a/allensdk/test/brain_observatory/behavior/conftest.py b/allensdk/test/brain_observatory/behavior/conftest.py index d91e79f75..dd81ceab2 100644 --- a/allensdk/test/brain_observatory/behavior/conftest.py +++ b/allensdk/test/brain_observatory/behavior/conftest.py @@ -196,7 +196,7 @@ def cell_specimen_table(): 'max_correction_right': [1., 1.], 'mask_image_plane': [1, 1], 'ophys_cell_segmentation_run_id': [1, 1], - 'image_mask': [np.array([[True, True], + 'roi_mask': [np.array([[True, True], [True, False]]), np.array([[True, True], [False, True]])]}, diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py index 9301781e6..4d3045d7d 100644 --- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py +++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py @@ -115,6 +115,7 @@ def test_visbeh_ophys_data_set(): 'stage': u'OPHYS_6_images_B'} +@pytest.mark.skip(reason="on-prem data source missing") @pytest.mark.requires_bamboo def test_legacy_dff_api(): @@ -197,7 +198,7 @@ def get_cell_specimen_table(self): "cell_roi_id": [1, 2], "y": [1, 1], "x": [2, 1], - "image_mask": [roi_1, roi_2] + "roi_mask": [roi_1, roi_2] }, index=pd.Index(data=[10, 11], name="cell_specimen_id") ) diff --git a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py index 794003012..28ebfdc28 100644 --- a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py +++ b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py @@ -9,37 +9,25 @@ import allensdk.brain_observatory.nwb as nwb from allensdk.brain_observatory.behavior.session_apis.data_io import ( BehaviorOphysNwbApi) -from allensdk.brain_observatory.behavior.schemas import ( - BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_running_speed_to_nwbfile(nwbfile, running_speed, roundtrip, roundtripper): - - nwbfile = nwb.add_running_speed_to_nwbfile(nwbfile, running_speed) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - running_speed_obt = obt.get_running_speed() - assert np.allclose(running_speed.timestamps, running_speed_obt.timestamps) - assert np.allclose(running_speed.values, running_speed_obt.values) @pytest.mark.parametrize('roundtrip', [True, False]) def test_add_running_data_df_to_nwbfile(nwbfile, running_data_df, roundtrip, roundtripper): + # Just make it different from running_data_df + running_data_df_unfiltered = running_data_df.copy() + running_data_df_unfiltered['speed'] = running_data_df['speed'] * 2 unit_dict = {'v_sig': 'V', 'v_in': 'V', 'speed': 'cm/s', 'timestamps': 's', 'dx': 'cm'} - nwbfile = nwb.add_running_data_df_to_nwbfile(nwbfile, running_data_df, unit_dict) + nwbfile = nwb.add_running_data_dfs_to_nwbfile(nwbfile, running_data_df=running_data_df, + running_data_df_unfiltered=running_data_df_unfiltered, unit_dict=unit_dict) if roundtrip: obt = roundtripper(nwbfile, BehaviorOphysNwbApi) else: obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - pd.testing.assert_frame_equal(running_data_df, obt.get_running_data_df()) + pd.testing.assert_frame_equal(running_data_df, obt.get_running_data_df(lowpass=True)) + pd.testing.assert_frame_equal(running_data_df_unfiltered, obt.get_running_data_df(lowpass=False)) @pytest.mark.parametrize('roundtrip', [True, False]) diff --git a/doc_template/index.rst b/doc_template/index.rst index c3f6e27ab..e28be8c2f 100644 --- a/doc_template/index.rst +++ b/doc_template/index.rst @@ -91,7 +91,13 @@ The Allen SDK provides Python code for accessing experimental metadata along wit See the `mouse connectivity section `_ for more details. -What's New - 2.3.2 (December 21, 2020) +What's New - 2.5.0 (January 29, 2021) +----------------------------------------------------------------------- +- Adds unfiltered running speed as new data stream +- run_demixing gracefully ignores any ROIs that are not in the input trace file + + +What's New - 2.4.0 (December 21, 2020) ----------------------------------------------------------------------- As of the 2.4.0 release: - When running raster_plot on a spike_times dataframe, the spike times from each unit are plotted twice. (thank you @dgmurx)