diff --git a/pdm.lock b/pdm.lock index be57333..8287002 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:1c3d81ec6290a8d72305c77a33dbec714092ba8a5cc02913a57209fc3f95075e" +content_hash = "sha256:87086b7720a1f78a76da5e1b521ea20e04a922fd05963ccd05ecc31bd75fd821" [[metadata.targets]] requires_python = ">=3.9" @@ -1670,25 +1670,9 @@ files = [ {file = "ndx_pose-0.1.1-py2.py3-none-any.whl", hash = "sha256:229718b494bf34f2e7f73d6e185b074a46169420b57e8573944a14b280b0a472"}, ] -[[package]] -name = "neo" -version = "0.13.2" -requires_python = ">=3.8" -summary = "Neo is a package for representing electrophysiology data in Python, together with support for reading a wide range of neurophysiology file formats" -groups = ["default"] -dependencies = [ - "numpy>=1.19.5", - "packaging", - "quantities>=0.14.1", -] -files = [ - {file = "neo-0.13.2-py3-none-any.whl", hash = "sha256:51d908c1c92f58705cdc58bdc2aeb0c6b91dd734d2c83b04d0d6f9461eb255a9"}, - {file = "neo-0.13.2.tar.gz", hash = "sha256:6980d9542e795d5351fd5b5d9d394104a47f30d0ef7f0511faca6a7b5e2772bc"}, -] - [[package]] name = "npc-ephys" -version = "0.1.22" +version = "0.1.23" requires_python = ">=3.9" summary = "Tools for accessing and processing raw ephys data, compatible with data in the cloud." groups = ["default"] @@ -1698,14 +1682,16 @@ dependencies = [ "npc-lims>=0.1.73", "npc-sync>=0.1.12", "pandas>=2.2.0", + "polars>=0.20.26", + "pyarrow>=14.0", "pynwb>=2.8.0", "tqdm>=4.66.1", - "wavpack-numcodecs>=0.1.5", + "wavpack-numcodecs!=0.2.0,>=0.1.5", "zarr<2.18.1", ] files = [ - {file = "npc_ephys-0.1.22-py3-none-any.whl", hash = "sha256:1cbd320bced0d7581b4e6e9b662122ef475a2b67c543f0449ebe2d8b9f221c92"}, - {file = "npc_ephys-0.1.22.tar.gz", hash = "sha256:c5dabdb0acda60b95c810494cd4cc759a299c32f2cc2724f906ebeea2dd1c0fc"}, + {file = "npc_ephys-0.1.23-py3-none-any.whl", hash = "sha256:3133bee243452ef16eb47d00a49019a6a2de441ef5024e0dad526162190a298c"}, + {file = "npc_ephys-0.1.23.tar.gz", hash = "sha256:eb48fe84364905e24e53c2b441d6041da9cc3d188551dcb9e5e73502e9e74d04"}, ] [[package]] @@ -2198,31 +2184,17 @@ files = [ [[package]] name = "polars" -version = "0.20.26" +version = "1.7.1" requires_python = ">=3.8" summary = "Blazingly fast DataFrame library" groups = ["default"] files = [ - {file = "polars-0.20.26-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:97d0e4b6ab6b47fa07798b447189ee9505d2085ec1a64a6aa8a65fdd429cd49f"}, - {file = "polars-0.20.26-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c270e366b4d8b672b204e7d48e39d255641d3d2b7bdc3a0ccd968cf53934657f"}, - {file = "polars-0.20.26-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db35d6eed508256a797c7f1b8e9dec4aae9c11b891797b2d38fac5627d072d34"}, - {file = "polars-0.20.26-cp38-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:25b00bd5cf44929722aa6389706559c5e8cedd6db2cfc38b27b706ed37e1b2af"}, - {file = "polars-0.20.26-cp38-abi3-win_amd64.whl", hash = "sha256:b22063acc815bc5c6d2e24292ff771ca0df306ecf97e8f6899924a1ec6d3f136"}, - {file = "polars-0.20.26.tar.gz", hash = "sha256:fa83d130562a5180a47f8763a7bb9f408dbbf51eafc1380e8a2951be8ce05a2c"}, -] - -[[package]] -name = "probeinterface" -version = "0.2.23" -requires_python = ">=3.8" -summary = "Python package to handle probe layout, geometry and wiring to device." -groups = ["default"] -dependencies = [ - "numpy", -] -files = [ - {file = "probeinterface-0.2.23-py3-none-any.whl", hash = "sha256:f5226dc550798a39000a25b112966ffa5ad4955f7ec803e3d41f06308af66e1b"}, - {file = "probeinterface-0.2.23.tar.gz", hash = "sha256:75aa922c52678a0796ac317e6390e54074bbfb5587e3ee735d1def9a5df375e3"}, + {file = "polars-1.7.1-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:589c1b5a9b5167f3c49713212cbeccc39e3a0e12577e21331c50dbf7178e32ed"}, + {file = "polars-1.7.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c955cca9d109ed5d79f4498915ec80590aa2e4619bc40bafbbeb5a160fcb166e"}, + {file = "polars-1.7.1-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cd675e4a306b2da57a1b688e65382aaa9e992dd7156b485fbd7f39892a3d784"}, + {file = "polars-1.7.1-cp38-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:45c255749b49bee244d10baeb69057580a0a397125b014bc8854b73ba5bdf45e"}, + {file = "polars-1.7.1-cp38-abi3-win_amd64.whl", hash = "sha256:a9004a907fc8e923dda27879f7e6eea8e06a753e160d08e606c8b9b5f914f911"}, + {file = "polars-1.7.1.tar.gz", hash = "sha256:3323bf6b3f1cf55212ddd35f044af8a1aa02033bca17d06f3852325e0da93a80"}, ] [[package]] @@ -2603,20 +2575,6 @@ files = [ {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] -[[package]] -name = "quantities" -version = "0.15.0" -requires_python = ">=3.8" -summary = "Support for physical quantities with units, based on numpy" -groups = ["default"] -dependencies = [ - "numpy>=1.20", -] -files = [ - {file = "quantities-0.15.0-py3-none-any.whl", hash = "sha256:589bdadcbbdc1c10950120c6d197f7e71ac145512a4b4ac5fd40d4946709d6ec"}, - {file = "quantities-0.15.0.tar.gz", hash = "sha256:9ea31e2a0d7517cf24d546b14146def9292639993a616cca61b875ef796b4b2b"}, -] - [[package]] name = "redis" version = "5.0.8" @@ -2967,26 +2925,6 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -[[package]] -name = "spikeinterface" -version = "0.101.0" -requires_python = "<4.0,>=3.8" -summary = "Python toolkit for analysis, visualization, and comparison of spike sorting output" -groups = ["default"] -dependencies = [ - "neo>=0.13.0", - "numpy<2.0,>=1.20", - "packaging", - "probeinterface>=0.2.23", - "threadpoolctl>=3.0.0", - "tqdm", - "zarr<2.18,>=2.16", -] -files = [ - {file = "spikeinterface-0.101.0-py3-none-any.whl", hash = "sha256:cc475411cdda8cc695b54607bf088c44783c1adccbd82a8e701b6c2a09a1c66d"}, - {file = "spikeinterface-0.101.0.tar.gz", hash = "sha256:a5a6c3a6c5a62627c5cb341dd364fbed232f4a7a19ed67a274065fd5f3de3ace"}, -] - [[package]] name = "tables" version = "3.9.2" diff --git a/pyproject.toml b/pyproject.toml index 65f73bd..e759936 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ authors = [ { name = "Sam Gale", email = "samg@alleninstitute.org" }, ] dependencies = [ - "h5py>=3.9.0", "pandas>=2.0", "pynwb>=2.7.0", "rich>=13.5.2", @@ -19,11 +18,8 @@ dependencies = [ "scipy>=1.9.3", "DynamicRoutingTask<0.1.106", "numba>=0.57.1", - "opencv-python-headless>=4.8.0.76", "python-dotenv>=1.0.0", "ndx-events>=0.2.0", - "pyarrow>=14.0", - "spikeinterface>=0.98.2", "hdmf>=3.14.0", "ndx-pose>=0.1.1", "tables>=3.9.2", @@ -31,13 +27,12 @@ dependencies = [ "pydantic>=2.6.4", "npc-lims>=0.1.174", "npc-sync>=0.1.18", - "npc-ephys>=0.1.22", + "npc-ephys>=0.1.23", "npc-stim>=0.1.8", "npc-samstim>=0.1.4", "npc-session>=0.1.34", "npc-mvr>=0.1.6", "npc-io>=0.1.27", - "polars>=0.20.26", ] requires-python = ">=3.9" readme = "README.md" diff --git a/src/npc_sessions/sessions.py b/src/npc_sessions/sessions.py index f26f7d8..19f6056 100644 --- a/src/npc_sessions/sessions.py +++ b/src/npc_sessions/sessions.py @@ -2085,7 +2085,7 @@ def _manipulator_positions(self) -> pynwb.core.DynamicTable: f"{self.id} has no log.csv file to get manipulator coordinates" ) from exc - df = utils.get_newscale_coordinates( + df = npc_ephys.get_newscale_coordinates( self.newscale_log_path, f"{self.id.date}_{self.ephys_settings_xml_data.start_time.isoformat()}", ) diff --git a/src/npc_sessions/utils/__init__.py b/src/npc_sessions/utils/__init__.py index 9f9d482..f0b865a 100644 --- a/src/npc_sessions/utils/__init__.py +++ b/src/npc_sessions/utils/__init__.py @@ -1,5 +1,4 @@ from npc_sessions.utils.electrodes import * from npc_sessions.utils.intervals import * from npc_sessions.utils.misc import * -from npc_sessions.utils.newscale import * -from npc_sessions.utils.videos import * +from npc_sessions.utils.videos import * \ No newline at end of file diff --git a/src/npc_sessions/utils/newscale.py b/src/npc_sessions/utils/newscale.py deleted file mode 100644 index 3036c63..0000000 --- a/src/npc_sessions/utils/newscale.py +++ /dev/null @@ -1,220 +0,0 @@ -from __future__ import annotations - -import collections -import datetime -import logging -from collections.abc import Iterable - -import npc_io -import npc_session -import numpy as np -import pandas as pd -import polars as pl - -logger = logging.getLogger(__name__) - -SERIAL_NUM_TO_PROBE_LETTER = ( - { - "SN32148": "A", - "SN32142": "B", - "SN32144": "C", - "SN32149": "D", - "SN32135": "E", - "SN24273": "F", - } # NP.0 - | { - "SN40911": "A", - "SN40900": "B", - "SN40912": "C", - "SN40913": "D", - "SN40914": "E", - "SN40910": "F", - } # NP.1 - | { - "SN45356": "A", - "SN45484": "B", - "SN45485": "C", - "SN45359": "D", - "SN45482": "E", - "SN45361": "F", - } # NP.2 - | { - "SN40906": "A", - "SN40908": "B", - "SN40907": "C", - "SN41084": "D", - "SN40903": "E", - "SN40902": "F", - } # NP.3 -) -SHORT_TRAVEL_SERIAL_NUMBERS = { - "SN32148", - "SN32142", - "SN32144", - "SN32149", - "SN32135", - "SN24273", -} -SHORT_TRAVEL_RANGE = 6_000 -LONG_TRAVEL_RANGE = 15_000 -NEWSCALE_LOG_COLUMNS = ( - "last_movement_dt", - "device_name", - "x", - "y", - "z", - "x_virtual", - "y_virtual", - "z_virtual", -) - - -def get_newscale_data(path: npc_io.PathLike) -> pl.DataFrame: - """ - >>> df = get_newscale_data('s3://aind-ephys-data/ecephys_686740_2023-10-23_14-11-05/behavior/log.csv') - """ - return pl.read_csv( - source=npc_io.from_pathlike(path).as_posix(), - new_columns=NEWSCALE_LOG_COLUMNS, - try_parse_dates=True, - ) - - -def get_newscale_coordinates( - newscale_log_path: npc_io.PathLike, - recording_start_time: ( - str | datetime.datetime | npc_session.DatetimeRecord | None - ) = None, -) -> pd.DataFrame: - """Returns the coordinates of each probe at the given time, by scanning for the most-recent prior movement on each motor. - - - looks up the timestamp of movement preceding `recording_start_time` - - if not provided, attempt to parse experiment (sync) start time from `newscale_log_path`: - assumes manipulators were not moved after the start time - - >>> df = get_newscale_coordinates('s3://aind-ephys-data/ecephys_686740_2023-10-23_14-11-05/behavior/log.csv', '2023-10-23 14-11-05') - >>> list(df['x']) - [6278.0, 6943.5, 7451.0, 4709.0, 4657.0, 5570.0] - >>> list(df['z']) - [11080.0, 8573.0, 6500.0, 8107.0, 8038.0, 9125.0] - """ - newscale_log_path = npc_io.from_pathlike(newscale_log_path) - if recording_start_time is None: - try: - start = npc_session.DatetimeRecord(newscale_log_path.as_posix()) - except ValueError as exc: - raise ValueError( - f"`recording_start_time` must be provided to indicate start of ephys recording: no time could be parsed from {newscale_log_path.as_posix()}" - ) from exc - else: - start = npc_session.DatetimeRecord(recording_start_time) - - movement = pl.col(NEWSCALE_LOG_COLUMNS[0]) - serial_number = pl.col(NEWSCALE_LOG_COLUMNS[1]) - df = get_newscale_data(newscale_log_path) - - # if experiment date isn't in df, the log file didn't cover this experiment - - # we can't continue - if start.dt.date() not in df["last_movement_dt"].dt.date(): - raise IndexError( - f"no movement data found for experiment date {start.dt.date()} in {newscale_log_path.as_posix()}" - ) - - recent_df = df.filter( - pl.col("last_movement_dt").dt.date() - > (start.dt.date() - datetime.timedelta(hours=24)) - ) - recent_z_values = recent_df["z"].str.strip_chars().cast(pl.Float32).to_numpy() - z_inverted: bool = is_z_inverted(recent_z_values) - - df = ( - df.filter(movement < start.dt) - .group_by(serial_number) - .agg( - pl.col(NEWSCALE_LOG_COLUMNS[:-3]).sort_by(movement).last() - ) # get last-moved for each manipulator - .top_k(6, by=movement) - ) - - # serial numbers have an extra leading space - manipulators = df.get_column(NEWSCALE_LOG_COLUMNS[1]).str.strip_chars() - df = df.with_columns(manipulators) - # convert str floats to floats - for column in NEWSCALE_LOG_COLUMNS[2:8]: - if column not in df.columns: - continue - df = df.with_columns(df.get_column(column).str.strip_chars().cast(pl.Float64)) - probes = manipulators.replace( - {k: f"probe{v}" for k, v in SERIAL_NUM_TO_PROBE_LETTER.items()} - ).alias("electrode_group_name") - - # correct z values - z = df["z"] - for idx, device in enumerate(df["device_name"]): - if z_inverted: - z[idx] = get_z_travel(device) - z[idx] - df = df.with_columns(z) - - # add time of last movement relative to start of recording - df = df.with_columns( - (pl.col("last_movement_dt") - start.dt) - .dt.total_seconds() - .alias("last_movement_time") - ) - - df = ( - df.insert_column(index=0, column=probes) - .sort(pl.col("electrode_group_name")) - .to_pandas() - ) - # nwb doesn't support `Timestamp` - df.last_movement_dt = df.last_movement_dt.astype("str") # type: ignore[attr-defined] - return df - - -def get_z_travel(serial_number: str) -> int: - """ - >>> get_z_travel('SN32144') - 6000 - >>> get_z_travel('SN40911') - 15000 - """ - if serial_number not in SERIAL_NUM_TO_PROBE_LETTER: - raise ValueError( - f"{serial_number=} is not a known serial number: need to update {__file__}" - ) - if serial_number in SHORT_TRAVEL_SERIAL_NUMBERS: - return SHORT_TRAVEL_RANGE - return LONG_TRAVEL_RANGE - - -def is_z_inverted(z_values: Iterable[float]) -> bool: - """ - The limits of the z-axis are [0-6000] for NP.0 and [0-15000] for NP.1-3. The - NewScale software sometimes (but not consistently) inverts the z-axis, so - retracted probes have a z-coordinate of 6000 or 15000 not 0. This function checks - the values in the z-column and tries to determine if the z-axis is inverted. - - Assumptions: - - the manipulators spend more time completely retracted than completely extended - - >>> is_z_inverted([0, 3000, 3000, 0]) - False - >>> is_z_inverted([15000, 3000, 3000, 15000]) - True - """ - c = collections.Counter(np.round(list(z_values), -2)) - is_long_travel = bool(c[LONG_TRAVEL_RANGE]) - travel_range = LONG_TRAVEL_RANGE if is_long_travel else SHORT_TRAVEL_RANGE - return c[0] < c[travel_range] - - -if __name__ == "__main__": - import doctest - - import dotenv - - dotenv.load_dotenv(dotenv.find_dotenv(usecwd=True)) - doctest.testmod( - optionflags=(doctest.IGNORE_EXCEPTION_DETAIL | doctest.NORMALIZE_WHITESPACE) - )