From c5ace8a3e64669e6306cb8b622c44f08f013533b Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Thu, 26 Jun 2025 18:18:54 -0700 Subject: [PATCH 01/14] Update maxwell.py Extract stimulation and Maxwell spike events --- .../extractors/neoextractors/maxwell.py | 139 +++++++++++++++--- 1 file changed, 117 insertions(+), 22 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/maxwell.py b/src/spikeinterface/extractors/neoextractors/maxwell.py index 1ec210541e..c6df3b6cf7 100644 --- a/src/spikeinterface/extractors/neoextractors/maxwell.py +++ b/src/spikeinterface/extractors/neoextractors/maxwell.py @@ -36,7 +36,7 @@ class MaxwellRecordingExtractor(NeoBaseRecordingExtractor): rec_name : str, default: None When the file contains several recordings you need to specify the one you want to extract. (rec_name='rec0000'). - install_maxwell_plugin : bool, default: True + install_maxwell_plugin : bool, default: False If True, install the maxwell plugin for neo. block_index : int, default: None If there are several blocks (experiments), specify the block index you want to load @@ -52,7 +52,7 @@ def __init__( block_index=None, all_annotations=False, rec_name=None, - install_maxwell_plugin=True, + install_maxwell_plugin=False, use_names_as_ids: bool = False, ): if install_maxwell_plugin: @@ -91,8 +91,7 @@ def install_maxwell_plugin(self, force_download=False): auto_install_maxwell_hdf5_compression_plugin(force_download=False) -_maxwell_event_dtype = np.dtype([("frame", "int64"), ("state", "int8"), ("time", "float64")]) - +_maxwell_event_dtype = np.dtype([("id", "int8"), ("frame", "uint32"), ("time", "float64"), ("state", "uint32"), ("message", "object")]) class MaxwellEventExtractor(BaseEvent): """ @@ -106,10 +105,48 @@ def __init__(self, file_path): h5_file = h5py.File(self.file_path, mode="r") version = int(h5_file["version"][0].decode()) fs = 20000 - + + if version < 20190530: + raise NotImplementedError(f"Version {self.version} not supported") + + # get ttl events bits = h5_file["bits"] - bit_states = bits["bits"] - channel_ids = np.unique(bit_states[bit_states != 0]) + + channel_ids = np.zeros((0),dtype=np.int8) + if len(bits)>0: + bit_state = bits["bits"] + channel_ids = np.int8(np.unique(bit_state[bit_state != 0])) + if -1 in channel_ids or 1 in channel_ids: + raise ValueError("TTL bits cannot be -1 or 1.") + + # access data_store from h5_file + data_store_keys = [x for x in h5_file["data_store"].keys()] + data_store_keys_id = [("events" in h5_file["data_store"][x].keys()) and ("groups" in h5_file["data_store"][x].keys()) for x in data_store_keys] + data_store = data_store_keys[data_store_keys_id.index(True)] + + # get stim events + event_raw = h5_file["data_store"][data_store]["events"] + channel_ids_stim = np.int8(np.unique([x[1] for x in event_raw])) + if -1 in channel_ids_stim or 0 in channel_ids_stim: + raise ValueError("Stimulation bits cannot be -1 or 0.") + if len(channel_ids)>0: + if set(channel_ids) & set(channel_ids_stim): + raise ValueError("TTL and stimulation bits overlap.") + channel_ids = np.concatenate((channel_ids, channel_ids_stim), dtype=np.int8) + + # set spike events channel == -1 + spike_raw = h5_file["data_store"][data_store]["spikes"] + if len(spike_raw)>0: + channel_ids = np.concatenate((channel_ids, [-1]), dtype=np.int8) + + # event = np.zeros(len(event_raw), dtype=_maxwell_stim_dtype) + # event["frame"] = [x[0] for x in event_raw] - first_frame + # event["time"] = event["frame"] / self.fs + # event["type"] = [x[1] for x in event_raw] + # event["subtype"] = [x[2] for x in event_raw] + # event["amplitude"] = [int(x[3].split(b'{"amplitude":"')[1].split(b'","phase":"')[0].decode("utf-8")) for x in event_raw] + # event["phase"] = [int(x[3].split(b',"phase":"')[1].split(b'"}\n')[0].decode("utf-8")) for x in event_raw] + # self.stim = event BaseEvent.__init__(self, channel_ids, structured_dtype=_maxwell_event_dtype) event_segment = MaxwellEventSegment(h5_file, version, fs) @@ -123,24 +160,82 @@ def __init__(self, h5_file, version, fs): self.version = version self.bits = self.h5_file["bits"] self.fs = fs - + def get_events(self, channel_id, start_time, end_time): - if self.version != 20160704: - raise NotImplementedError(f"Version {self.version} not supported") + bits = self.bits + + # get ttl events + channel_ids = np.zeros((0),dtype=np.int8) + bit_channel = np.zeros((0),dtype=np.int8) + bit_frameno = np.zeros((0),dtype=np.uint32) + bit_state = np.zeros((0),dtype=np.uint32) + bit_message = np.zeros((0),dtype=object) + if len(bits)>0: + good_idx = np.where(bits["bits"] != 0)[0] + channel_ids = np.concatenate((channel_ids,np.int8(np.unique(bits["bits"][good_idx])))) + if 1 in channel_ids: + raise ValueError("TTL bits cannot be 1.") + bit_channel = np.concatenate((bit_channel,np.uint8(bits["bits"][good_idx]))) + bit_frameno = np.concatenate((bit_frameno,np.uint32(bits["frameno"][good_idx]))) + bit_state = np.concatenate((bit_state,np.uint32(bits["bits"][good_idx]))) + bit_message = np.concatenate((bit_message,[b'{}\n']*len(bit_state)),dtype=object) + + # access data_store from h5_file + h5_file = self.h5_file + data_store_keys = [x for x in h5_file["data_store"].keys()] + data_store_keys_id = [("events" in h5_file["data_store"][x].keys()) and ("groups" in h5_file["data_store"][x].keys()) for x in data_store_keys] + data_store = data_store_keys[data_store_keys_id.index(True)] + + # get stim events + event_raw = h5_file["data_store"][data_store]["events"] + channel_ids_stim = np.int8(np.unique([x[1] for x in event_raw])) + # if -1 in channel_ids_stim or 0 in channel_ids_stim: + # raise ValueError("Stimulation bits cannot be -1 or 0.") + stim_arr = np.array(event_raw) + bit_channel_stim = stim_arr["eventtype"] + bit_frameno_stim = stim_arr["frameno"] + bit_state_stim = stim_arr["eventid"] + bit_message_stim = stim_arr["eventmessage"] + # bit_channel_stim = np.array([x[1] for x in event_raw],dtype=np.uint8) + # bit_frameno_stim = np.array([x[0] for x in event_raw],dtype=np.uint32) + # bit_state_stim = np.array([x[2] for x in event_raw],dtype=np.uint32) + # bit_message_stim = np.array([x[3] for x in event_raw],dtype=object) + + # get spike events + spike_raw = h5_file["data_store"][data_store]["spikes"] + if len(spike_raw)>0: + channel_ids_spike = np.int8([-1]) + spike_arr = np.array(spike_raw) + bit_channel_spike = -np.ones(len(spike_arr),dtype=np.int8) + bit_frameno_spike = spike_arr["frameno"] + bit_state_spike= spike_arr["channel"] + bit_message_spike = spike_arr["amplitude"] + + # if len(channel_ids)>0: + # if set(channel_ids) & set(channel_ids_stim): + # raise ValueError("TTL and stimulation bits overlap.") + + # final array in order: spikes, stims, ttl + bit_channel = np.concatenate((bit_channel_spike, bit_channel_stim, bit_channel)) + bit_frameno = np.concatenate((bit_frameno_spike, bit_frameno_stim, bit_frameno)) + bit_state = np.concatenate((bit_state_spike, bit_state_stim, bit_state)) + bit_message = np.concatenate((bit_message_spike, bit_message_stim, bit_message)) + + first_frame = h5_file["data_store"][data_store]["groups/routed/frame_nos"][0] + bit_frameno = bit_frameno - first_frame - framevals = self.h5_file["sig"][-2:, 0] - first_frame = framevals[1] << 16 | framevals[0] - ttl_frames = self.bits["frameno"] - first_frame - ttl_states = self.bits["bits"] if channel_id is not None: - bits_channel_idx = np.where((ttl_states == channel_id) | (ttl_states == 0))[0] - ttl_frames = ttl_frames[bits_channel_idx] - ttl_states = ttl_states[bits_channel_idx] - ttl_states[ttl_states == 0] = -1 - event = np.zeros(len(ttl_frames), dtype=_maxwell_event_dtype) - event["frame"] = ttl_frames - event["time"] = ttl_frames / self.fs - event["state"] = ttl_states + good_idx = np.where(bit_channel == channel_id)[0] + bit_channel = bit_channel[good_idx] + bit_frameno = bit_frameno[good_idx] + bit_state = bit_state[good_idx] + bit_message = bit_message[good_idx] + event = np.zeros(len(bit_channel), dtype=_maxwell_event_dtype) + event["id"] = bit_channel + event["frame"] = bit_frameno + event["time"] = np.float64(bit_frameno) / self.fs + event["state"] = bit_state + event["message"] = bit_message if start_time is not None: event = event[event["time"] >= start_time] From 16cd0ae22521b2e0e23be4c012dd7edee0f03205 Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Thu, 26 Jun 2025 18:25:42 -0700 Subject: [PATCH 02/14] Update neobaseextractor.py Remove duplicate channel_id assignments (preferentially keep earlier ids) --- .../extractors/neoextractors/neobaseextractor.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index d66ce79aa3..201e92a569 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -228,6 +228,11 @@ def __init__( # need neo 0.10.0 signal_channels = self.neo_reader.header["signal_channels"] mask = signal_channels["stream_id"] == stream_id + + # if stream has duplicate channel_id assignments, correct this + seen = set() + dupid = [i for i,x in enumerate(signal_channels[mask]["id"]) if x in seen or seen.add(x)] + mask[dupid] = False signal_channels = signal_channels[mask] if use_names_as_ids: From 25b35890011688e96199fc7a831f3bd75f8bd282 Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Fri, 27 Jun 2025 08:30:15 -0700 Subject: [PATCH 03/14] Update neobaseextractor.py Removed duplicate removal temporarily --- .../extractors/neoextractors/neobaseextractor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index 201e92a569..03f8ef0806 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -230,10 +230,10 @@ def __init__( mask = signal_channels["stream_id"] == stream_id # if stream has duplicate channel_id assignments, correct this - seen = set() - dupid = [i for i,x in enumerate(signal_channels[mask]["id"]) if x in seen or seen.add(x)] - mask[dupid] = False - signal_channels = signal_channels[mask] + # seen = set() + # dupid = [i for i,x in enumerate(signal_channels[mask]["id"]) if x in seen or seen.add(x)] + # mask[dupid] = False + # signal_channels = signal_channels[mask] if use_names_as_ids: chan_names = signal_channels["name"] From 79a6a679e9825fab6517fbd8fd740998fbd182c1 Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Sun, 29 Jun 2025 21:27:00 -0700 Subject: [PATCH 04/14] Update neobaseextractor.py Remove 1) instances where multiple MaxWell electrodes are connected to the same channel (mix of electrode signals) and 2) duplicates where multiple MaxWell channels and connected to the same electrode (duplicates). --- .../neoextractors/neobaseextractor.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index 03f8ef0806..70beda2e07 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -5,6 +5,7 @@ from math import isclose import numpy as np +import pandas as pd import importlib from spikeinterface.core import ( @@ -228,12 +229,25 @@ def __init__( # need neo 0.10.0 signal_channels = self.neo_reader.header["signal_channels"] mask = signal_channels["stream_id"] == stream_id + mask_id = np.argwhere(mask).flatten().tolist() + + # remove all duplicate channel-to-electrode assignments + dupid = np.where(pd.DataFrame(signal_channels[mask]['name']).duplicated(keep='first')) + for i in dupid[0]: + mask[mask_id.pop(i)] = False + + # remove all duplicate channel assigments corresponding to different electrodes (channel is a mix of mulitple electrode signals) + signal_channels_chan, signal_channels_elec = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) + dupid = np.where(pd.DataFrame(signal_channels_chan).duplicated(keep=False)) + for i in dupid[0]: + mask[mask_id.pop(i)] = False + + # remove subsequent duplicated electrodes (single electrode saved to multiple channels) + dupid = np.where(pd.DataFrame(signal_channels_elec[mask]['name']).duplicated(keep='first')) + for i in dupid[0]: + mask[mask_id.pop(i)] = False - # if stream has duplicate channel_id assignments, correct this - # seen = set() - # dupid = [i for i,x in enumerate(signal_channels[mask]["id"]) if x in seen or seen.add(x)] - # mask[dupid] = False - # signal_channels = signal_channels[mask] + signal_channels = signal_channels[mask] if use_names_as_ids: chan_names = signal_channels["name"] From 5a71f968baf488472ff4c90258cf1f70248e4d24 Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Sun, 29 Jun 2025 21:30:36 -0700 Subject: [PATCH 05/14] Update neobaseextractor.py --- src/spikeinterface/extractors/neoextractors/neobaseextractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index 70beda2e07..d217c4c1f7 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -243,7 +243,7 @@ def __init__( mask[mask_id.pop(i)] = False # remove subsequent duplicated electrodes (single electrode saved to multiple channels) - dupid = np.where(pd.DataFrame(signal_channels_elec[mask]['name']).duplicated(keep='first')) + dupid = np.where(pd.DataFrame(signal_channels_elec).duplicated(keep='first')) for i in dupid[0]: mask[mask_id.pop(i)] = False From ffef869b62cd1eb9cc62bc832f4dd6aa59af2ad3 Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Sun, 29 Jun 2025 23:33:43 -0700 Subject: [PATCH 06/14] Update maxwell.py Significant update for MaxWell systems - now utilize events to extract MAXWELL SPIKE (via built in template matching) and STIMULATION events. --- .../extractors/neoextractors/maxwell.py | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/maxwell.py b/src/spikeinterface/extractors/neoextractors/maxwell.py index c6df3b6cf7..1f7547b89b 100644 --- a/src/spikeinterface/extractors/neoextractors/maxwell.py +++ b/src/spikeinterface/extractors/neoextractors/maxwell.py @@ -139,15 +139,6 @@ def __init__(self, file_path): if len(spike_raw)>0: channel_ids = np.concatenate((channel_ids, [-1]), dtype=np.int8) - # event = np.zeros(len(event_raw), dtype=_maxwell_stim_dtype) - # event["frame"] = [x[0] for x in event_raw] - first_frame - # event["time"] = event["frame"] / self.fs - # event["type"] = [x[1] for x in event_raw] - # event["subtype"] = [x[2] for x in event_raw] - # event["amplitude"] = [int(x[3].split(b'{"amplitude":"')[1].split(b'","phase":"')[0].decode("utf-8")) for x in event_raw] - # event["phase"] = [int(x[3].split(b',"phase":"')[1].split(b'"}\n')[0].decode("utf-8")) for x in event_raw] - # self.stim = event - BaseEvent.__init__(self, channel_ids, structured_dtype=_maxwell_event_dtype) event_segment = MaxwellEventSegment(h5_file, version, fs) self.add_event_segment(event_segment) @@ -189,18 +180,12 @@ def get_events(self, channel_id, start_time, end_time): # get stim events event_raw = h5_file["data_store"][data_store]["events"] channel_ids_stim = np.int8(np.unique([x[1] for x in event_raw])) - # if -1 in channel_ids_stim or 0 in channel_ids_stim: - # raise ValueError("Stimulation bits cannot be -1 or 0.") stim_arr = np.array(event_raw) bit_channel_stim = stim_arr["eventtype"] bit_frameno_stim = stim_arr["frameno"] bit_state_stim = stim_arr["eventid"] bit_message_stim = stim_arr["eventmessage"] - # bit_channel_stim = np.array([x[1] for x in event_raw],dtype=np.uint8) - # bit_frameno_stim = np.array([x[0] for x in event_raw],dtype=np.uint32) - # bit_state_stim = np.array([x[2] for x in event_raw],dtype=np.uint32) - # bit_message_stim = np.array([x[3] for x in event_raw],dtype=object) - + # get spike events spike_raw = h5_file["data_store"][data_store]["spikes"] if len(spike_raw)>0: @@ -211,10 +196,6 @@ def get_events(self, channel_id, start_time, end_time): bit_state_spike= spike_arr["channel"] bit_message_spike = spike_arr["amplitude"] - # if len(channel_ids)>0: - # if set(channel_ids) & set(channel_ids_stim): - # raise ValueError("TTL and stimulation bits overlap.") - # final array in order: spikes, stims, ttl bit_channel = np.concatenate((bit_channel_spike, bit_channel_stim, bit_channel)) bit_frameno = np.concatenate((bit_frameno_spike, bit_frameno_stim, bit_frameno)) From 0d35e2f8f4c1fa5512e4fd6552bc649628cf379c Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Sun, 29 Jun 2025 23:39:10 -0700 Subject: [PATCH 07/14] Update neobaseextractor.py --- .../extractors/neoextractors/neobaseextractor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index d217c4c1f7..603d421ca9 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -237,12 +237,13 @@ def __init__( mask[mask_id.pop(i)] = False # remove all duplicate channel assigments corresponding to different electrodes (channel is a mix of mulitple electrode signals) - signal_channels_chan, signal_channels_elec = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) + signal_channels_chan,_ = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) dupid = np.where(pd.DataFrame(signal_channels_chan).duplicated(keep=False)) for i in dupid[0]: mask[mask_id.pop(i)] = False # remove subsequent duplicated electrodes (single electrode saved to multiple channels) + _,signal_channels_elec = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) dupid = np.where(pd.DataFrame(signal_channels_elec).duplicated(keep='first')) for i in dupid[0]: mask[mask_id.pop(i)] = False From 5a485e14db5a07c48451dfe027936c132fbe2bb0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 06:46:14 +0000 Subject: [PATCH 08/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../extractors/neoextractors/maxwell.py | 69 +++++++++++-------- .../neoextractors/neobaseextractor.py | 14 ++-- 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/maxwell.py b/src/spikeinterface/extractors/neoextractors/maxwell.py index 1f7547b89b..4cee044ea1 100644 --- a/src/spikeinterface/extractors/neoextractors/maxwell.py +++ b/src/spikeinterface/extractors/neoextractors/maxwell.py @@ -91,7 +91,10 @@ def install_maxwell_plugin(self, force_download=False): auto_install_maxwell_hdf5_compression_plugin(force_download=False) -_maxwell_event_dtype = np.dtype([("id", "int8"), ("frame", "uint32"), ("time", "float64"), ("state", "uint32"), ("message", "object")]) +_maxwell_event_dtype = np.dtype( + [("id", "int8"), ("frame", "uint32"), ("time", "float64"), ("state", "uint32"), ("message", "object")] +) + class MaxwellEventExtractor(BaseEvent): """ @@ -105,23 +108,26 @@ def __init__(self, file_path): h5_file = h5py.File(self.file_path, mode="r") version = int(h5_file["version"][0].decode()) fs = 20000 - + if version < 20190530: raise NotImplementedError(f"Version {self.version} not supported") - + # get ttl events bits = h5_file["bits"] - - channel_ids = np.zeros((0),dtype=np.int8) - if len(bits)>0: + + channel_ids = np.zeros((0), dtype=np.int8) + if len(bits) > 0: bit_state = bits["bits"] channel_ids = np.int8(np.unique(bit_state[bit_state != 0])) if -1 in channel_ids or 1 in channel_ids: raise ValueError("TTL bits cannot be -1 or 1.") - + # access data_store from h5_file data_store_keys = [x for x in h5_file["data_store"].keys()] - data_store_keys_id = [("events" in h5_file["data_store"][x].keys()) and ("groups" in h5_file["data_store"][x].keys()) for x in data_store_keys] + data_store_keys_id = [ + ("events" in h5_file["data_store"][x].keys()) and ("groups" in h5_file["data_store"][x].keys()) + for x in data_store_keys + ] data_store = data_store_keys[data_store_keys_id.index(True)] # get stim events @@ -129,16 +135,16 @@ def __init__(self, file_path): channel_ids_stim = np.int8(np.unique([x[1] for x in event_raw])) if -1 in channel_ids_stim or 0 in channel_ids_stim: raise ValueError("Stimulation bits cannot be -1 or 0.") - if len(channel_ids)>0: + if len(channel_ids) > 0: if set(channel_ids) & set(channel_ids_stim): raise ValueError("TTL and stimulation bits overlap.") channel_ids = np.concatenate((channel_ids, channel_ids_stim), dtype=np.int8) # set spike events channel == -1 spike_raw = h5_file["data_store"][data_store]["spikes"] - if len(spike_raw)>0: + if len(spike_raw) > 0: channel_ids = np.concatenate((channel_ids, [-1]), dtype=np.int8) - + BaseEvent.__init__(self, channel_ids, structured_dtype=_maxwell_event_dtype) event_segment = MaxwellEventSegment(h5_file, version, fs) self.add_event_segment(event_segment) @@ -151,30 +157,33 @@ def __init__(self, h5_file, version, fs): self.version = version self.bits = self.h5_file["bits"] self.fs = fs - + def get_events(self, channel_id, start_time, end_time): bits = self.bits - + # get ttl events - channel_ids = np.zeros((0),dtype=np.int8) - bit_channel = np.zeros((0),dtype=np.int8) - bit_frameno = np.zeros((0),dtype=np.uint32) - bit_state = np.zeros((0),dtype=np.uint32) - bit_message = np.zeros((0),dtype=object) - if len(bits)>0: + channel_ids = np.zeros((0), dtype=np.int8) + bit_channel = np.zeros((0), dtype=np.int8) + bit_frameno = np.zeros((0), dtype=np.uint32) + bit_state = np.zeros((0), dtype=np.uint32) + bit_message = np.zeros((0), dtype=object) + if len(bits) > 0: good_idx = np.where(bits["bits"] != 0)[0] - channel_ids = np.concatenate((channel_ids,np.int8(np.unique(bits["bits"][good_idx])))) + channel_ids = np.concatenate((channel_ids, np.int8(np.unique(bits["bits"][good_idx])))) if 1 in channel_ids: raise ValueError("TTL bits cannot be 1.") - bit_channel = np.concatenate((bit_channel,np.uint8(bits["bits"][good_idx]))) - bit_frameno = np.concatenate((bit_frameno,np.uint32(bits["frameno"][good_idx]))) - bit_state = np.concatenate((bit_state,np.uint32(bits["bits"][good_idx]))) - bit_message = np.concatenate((bit_message,[b'{}\n']*len(bit_state)),dtype=object) - + bit_channel = np.concatenate((bit_channel, np.uint8(bits["bits"][good_idx]))) + bit_frameno = np.concatenate((bit_frameno, np.uint32(bits["frameno"][good_idx]))) + bit_state = np.concatenate((bit_state, np.uint32(bits["bits"][good_idx]))) + bit_message = np.concatenate((bit_message, [b"{}\n"] * len(bit_state)), dtype=object) + # access data_store from h5_file h5_file = self.h5_file data_store_keys = [x for x in h5_file["data_store"].keys()] - data_store_keys_id = [("events" in h5_file["data_store"][x].keys()) and ("groups" in h5_file["data_store"][x].keys()) for x in data_store_keys] + data_store_keys_id = [ + ("events" in h5_file["data_store"][x].keys()) and ("groups" in h5_file["data_store"][x].keys()) + for x in data_store_keys + ] data_store = data_store_keys[data_store_keys_id.index(True)] # get stim events @@ -185,15 +194,15 @@ def get_events(self, channel_id, start_time, end_time): bit_frameno_stim = stim_arr["frameno"] bit_state_stim = stim_arr["eventid"] bit_message_stim = stim_arr["eventmessage"] - + # get spike events spike_raw = h5_file["data_store"][data_store]["spikes"] - if len(spike_raw)>0: + if len(spike_raw) > 0: channel_ids_spike = np.int8([-1]) spike_arr = np.array(spike_raw) - bit_channel_spike = -np.ones(len(spike_arr),dtype=np.int8) + bit_channel_spike = -np.ones(len(spike_arr), dtype=np.int8) bit_frameno_spike = spike_arr["frameno"] - bit_state_spike= spike_arr["channel"] + bit_state_spike = spike_arr["channel"] bit_message_spike = spike_arr["amplitude"] # final array in order: spikes, stims, ttl diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index 603d421ca9..bf3cc32bf5 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -232,22 +232,22 @@ def __init__( mask_id = np.argwhere(mask).flatten().tolist() # remove all duplicate channel-to-electrode assignments - dupid = np.where(pd.DataFrame(signal_channels[mask]['name']).duplicated(keep='first')) + dupid = np.where(pd.DataFrame(signal_channels[mask]["name"]).duplicated(keep="first")) for i in dupid[0]: mask[mask_id.pop(i)] = False - + # remove all duplicate channel assigments corresponding to different electrodes (channel is a mix of mulitple electrode signals) - signal_channels_chan,_ = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) + signal_channels_chan, _ = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) dupid = np.where(pd.DataFrame(signal_channels_chan).duplicated(keep=False)) for i in dupid[0]: mask[mask_id.pop(i)] = False - + # remove subsequent duplicated electrodes (single electrode saved to multiple channels) - _,signal_channels_elec = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) - dupid = np.where(pd.DataFrame(signal_channels_elec).duplicated(keep='first')) + _, signal_channels_elec = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) + dupid = np.where(pd.DataFrame(signal_channels_elec).duplicated(keep="first")) for i in dupid[0]: mask[mask_id.pop(i)] = False - + signal_channels = signal_channels[mask] if use_names_as_ids: From de6177ccefb7f7f4cd038899f1aaf0d6739a342d Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Mon, 30 Jun 2025 09:59:12 -0700 Subject: [PATCH 09/14] Update neobaseextractor.py Avoids using pandas and fixed error with pop use --- .../neoextractors/neobaseextractor.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index bf3cc32bf5..fb2e3e027c 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -5,7 +5,6 @@ from math import isclose import numpy as np -import pandas as pd import importlib from spikeinterface.core import ( @@ -229,24 +228,26 @@ def __init__( # need neo 0.10.0 signal_channels = self.neo_reader.header["signal_channels"] mask = signal_channels["stream_id"] == stream_id - mask_id = np.argwhere(mask).flatten().tolist() # remove all duplicate channel-to-electrode assignments - dupid = np.where(pd.DataFrame(signal_channels[mask]["name"]).duplicated(keep="first")) - for i in dupid[0]: - mask[mask_id.pop(i)] = False - + mask_id = np.argwhere(mask).flatten() + [u,u_i,u_v,u_c] = np.unique(signal_channels[mask]['name'], return_index=True, return_inverse=True, return_counts=True) + for i in u_v[u_i[u_c>1]]: + mask[mask_id[np.argwhere(signal_channels[mask]['name'] == u[i])[1:].flatten()]] = False + # remove all duplicate channel assigments corresponding to different electrodes (channel is a mix of mulitple electrode signals) - signal_channels_chan, _ = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) - dupid = np.where(pd.DataFrame(signal_channels_chan).duplicated(keep=False)) - for i in dupid[0]: - mask[mask_id.pop(i)] = False - + mask_id = np.argwhere(mask).flatten() + signal_channels_chan,_ = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) + [u,u_i,u_v,u_c] = np.unique(signal_channels_chan, return_index=True, return_inverse=True, return_counts=True) + for i in u_v[u_i[u_c>1]]: + mask[mask_id[np.argwhere(signal_channels_chan == u[i])[:].flatten()]] = False + # remove subsequent duplicated electrodes (single electrode saved to multiple channels) - _, signal_channels_elec = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) - dupid = np.where(pd.DataFrame(signal_channels_elec).duplicated(keep="first")) - for i in dupid[0]: - mask[mask_id.pop(i)] = False + mask_id = np.argwhere(mask).flatten() + _,signal_channels_elec = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) + [u,u_i,u_v,u_c] = np.unique(signal_channels_elec, return_index=True, return_inverse=True, return_counts=True) + for i in u_v[u_i[u_c>1]]: + mask[mask_id[np.argwhere(signal_channels_elec == u[i])[1:].flatten()]] = False signal_channels = signal_channels[mask] From ae5dedc24d3bcde3f60f512fffdd0a9f308a4643 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 16:59:38 +0000 Subject: [PATCH 10/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../neoextractors/neobaseextractor.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index fb2e3e027c..f0156942de 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -231,22 +231,24 @@ def __init__( # remove all duplicate channel-to-electrode assignments mask_id = np.argwhere(mask).flatten() - [u,u_i,u_v,u_c] = np.unique(signal_channels[mask]['name'], return_index=True, return_inverse=True, return_counts=True) - for i in u_v[u_i[u_c>1]]: - mask[mask_id[np.argwhere(signal_channels[mask]['name'] == u[i])[1:].flatten()]] = False - + [u, u_i, u_v, u_c] = np.unique( + signal_channels[mask]["name"], return_index=True, return_inverse=True, return_counts=True + ) + for i in u_v[u_i[u_c > 1]]: + mask[mask_id[np.argwhere(signal_channels[mask]["name"] == u[i])[1:].flatten()]] = False + # remove all duplicate channel assigments corresponding to different electrodes (channel is a mix of mulitple electrode signals) mask_id = np.argwhere(mask).flatten() - signal_channels_chan,_ = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) - [u,u_i,u_v,u_c] = np.unique(signal_channels_chan, return_index=True, return_inverse=True, return_counts=True) - for i in u_v[u_i[u_c>1]]: + signal_channels_chan, _ = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) + [u, u_i, u_v, u_c] = np.unique(signal_channels_chan, return_index=True, return_inverse=True, return_counts=True) + for i in u_v[u_i[u_c > 1]]: mask[mask_id[np.argwhere(signal_channels_chan == u[i])[:].flatten()]] = False - + # remove subsequent duplicated electrodes (single electrode saved to multiple channels) mask_id = np.argwhere(mask).flatten() - _,signal_channels_elec = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) - [u,u_i,u_v,u_c] = np.unique(signal_channels_elec, return_index=True, return_inverse=True, return_counts=True) - for i in u_v[u_i[u_c>1]]: + _, signal_channels_elec = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) + [u, u_i, u_v, u_c] = np.unique(signal_channels_elec, return_index=True, return_inverse=True, return_counts=True) + for i in u_v[u_i[u_c > 1]]: mask[mask_id[np.argwhere(signal_channels_elec == u[i])[1:].flatten()]] = False signal_channels = signal_channels[mask] From df735f050fdd3c3b7e2261b2a959dfca91d498c2 Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Mon, 30 Jun 2025 10:11:55 -0700 Subject: [PATCH 11/14] Update maxwell.py --- src/spikeinterface/extractors/neoextractors/maxwell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/neoextractors/maxwell.py b/src/spikeinterface/extractors/neoextractors/maxwell.py index 4cee044ea1..66b74103ef 100644 --- a/src/spikeinterface/extractors/neoextractors/maxwell.py +++ b/src/spikeinterface/extractors/neoextractors/maxwell.py @@ -52,7 +52,7 @@ def __init__( block_index=None, all_annotations=False, rec_name=None, - install_maxwell_plugin=False, + install_maxwell_plugin=True, use_names_as_ids: bool = False, ): if install_maxwell_plugin: From 3eca5ba9f430445240707084de8111d1b52421d8 Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Mon, 30 Jun 2025 10:15:51 -0700 Subject: [PATCH 12/14] Update maxwell.py --- src/spikeinterface/extractors/neoextractors/maxwell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/extractors/neoextractors/maxwell.py b/src/spikeinterface/extractors/neoextractors/maxwell.py index 66b74103ef..9000069805 100644 --- a/src/spikeinterface/extractors/neoextractors/maxwell.py +++ b/src/spikeinterface/extractors/neoextractors/maxwell.py @@ -36,7 +36,7 @@ class MaxwellRecordingExtractor(NeoBaseRecordingExtractor): rec_name : str, default: None When the file contains several recordings you need to specify the one you want to extract. (rec_name='rec0000'). - install_maxwell_plugin : bool, default: False + install_maxwell_plugin : bool, default: True If True, install the maxwell plugin for neo. block_index : int, default: None If there are several blocks (experiments), specify the block index you want to load From be427f997a5dbe6fc923db4182e2118eabe03aa6 Mon Sep 17 00:00:00 2001 From: cmccrimmon <103812790+cmccrimmon@users.noreply.github.com> Date: Tue, 1 Jul 2025 09:58:56 -0700 Subject: [PATCH 13/14] Update neobaseextractor.py --- .../neoextractors/neobaseextractor.py | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index f0156942de..c42cff39eb 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -228,29 +228,21 @@ def __init__( # need neo 0.10.0 signal_channels = self.neo_reader.header["signal_channels"] mask = signal_channels["stream_id"] == stream_id - - # remove all duplicate channel-to-electrode assignments - mask_id = np.argwhere(mask).flatten() - [u, u_i, u_v, u_c] = np.unique( - signal_channels[mask]["name"], return_index=True, return_inverse=True, return_counts=True - ) - for i in u_v[u_i[u_c > 1]]: - mask[mask_id[np.argwhere(signal_channels[mask]["name"] == u[i])[1:].flatten()]] = False - + # remove all duplicate channel assigments corresponding to different electrodes (channel is a mix of mulitple electrode signals) mask_id = np.argwhere(mask).flatten() - signal_channels_chan, _ = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) - [u, u_i, u_v, u_c] = np.unique(signal_channels_chan, return_index=True, return_inverse=True, return_counts=True) - for i in u_v[u_i[u_c > 1]]: + signal_channels_chan,_ = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) + [u, u_c] = np.unique(signal_channels_chan, return_counts=True) + for i in np.argwhere(u_c>1).flatten(): mask[mask_id[np.argwhere(signal_channels_chan == u[i])[:].flatten()]] = False - + # remove subsequent duplicated electrodes (single electrode saved to multiple channels) mask_id = np.argwhere(mask).flatten() - _, signal_channels_elec = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) - [u, u_i, u_v, u_c] = np.unique(signal_channels_elec, return_index=True, return_inverse=True, return_counts=True) - for i in u_v[u_i[u_c > 1]]: + _,signal_channels_elec = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) + [u, u_c] = np.unique(signal_channels_elec, return_counts=True) + for i in np.argwhere(u_c>1).flatten(): mask[mask_id[np.argwhere(signal_channels_elec == u[i])[1:].flatten()]] = False - + signal_channels = signal_channels[mask] if use_names_as_ids: From cbd1a714f4474785c39e31cb2a4e9f1d7ed08639 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 16:59:23 +0000 Subject: [PATCH 14/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../extractors/neoextractors/neobaseextractor.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py index c42cff39eb..5d37a666b6 100644 --- a/src/spikeinterface/extractors/neoextractors/neobaseextractor.py +++ b/src/spikeinterface/extractors/neoextractors/neobaseextractor.py @@ -228,21 +228,21 @@ def __init__( # need neo 0.10.0 signal_channels = self.neo_reader.header["signal_channels"] mask = signal_channels["stream_id"] == stream_id - + # remove all duplicate channel assigments corresponding to different electrodes (channel is a mix of mulitple electrode signals) mask_id = np.argwhere(mask).flatten() - signal_channels_chan,_ = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) + signal_channels_chan, _ = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) [u, u_c] = np.unique(signal_channels_chan, return_counts=True) - for i in np.argwhere(u_c>1).flatten(): + for i in np.argwhere(u_c > 1).flatten(): mask[mask_id[np.argwhere(signal_channels_chan == u[i])[:].flatten()]] = False - + # remove subsequent duplicated electrodes (single electrode saved to multiple channels) mask_id = np.argwhere(mask).flatten() - _,signal_channels_elec = map(list, zip(*(x.split(' ') for x in signal_channels[mask]['name']))) + _, signal_channels_elec = map(list, zip(*(x.split(" ") for x in signal_channels[mask]["name"]))) [u, u_c] = np.unique(signal_channels_elec, return_counts=True) - for i in np.argwhere(u_c>1).flatten(): + for i in np.argwhere(u_c > 1).flatten(): mask[mask_id[np.argwhere(signal_channels_elec == u[i])[1:].flatten()]] = False - + signal_channels = signal_channels[mask] if use_names_as_ids: