diff --git a/optinist/api/dataclass/bar.py b/optinist/api/dataclass/bar.py new file mode 100644 index 000000000..42d46d1d3 --- /dev/null +++ b/optinist/api/dataclass/bar.py @@ -0,0 +1,35 @@ +import numpy as np +import pandas as pd + +from optinist.api.dataclass.base import BaseData +from optinist.api.utils.filepath_creater import join_filepath +from optinist.api.utils.json_writer import JsonWriter + + +class BarData(BaseData): + def __init__(self, data, index=None, file_name='bar'): + super().__init__(file_name) + data = np.array(data) + + assert data.ndim <= 2, 'Bar Dimension Error' + + if data.ndim == 1: + data = data[np.newaxis] + + assert data.ndim == 2, 'Bar Dimesion is not 2' + + self.data = data + + # indexを指定 + if index is not None: + self.index = index + else: + self.index = np.arange(len(self.data)) + + def save_json(self, json_dir): + self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) + df = pd.DataFrame( + self.data, + index=self.index, + ) + JsonWriter.write_as_split(self.json_path, df) diff --git a/optinist/api/dataclass/base.py b/optinist/api/dataclass/base.py new file mode 100644 index 000000000..e256645df --- /dev/null +++ b/optinist/api/dataclass/base.py @@ -0,0 +1,13 @@ +import gc + + +class BaseData: + def __init__(self, file_name): + self.file_name = file_name + + def save_json(self, json_dir): + pass + + def __del__(self): + del self + gc.collect() diff --git a/optinist/api/dataclass/csv.py b/optinist/api/dataclass/csv.py new file mode 100644 index 000000000..a2509b2a7 --- /dev/null +++ b/optinist/api/dataclass/csv.py @@ -0,0 +1,38 @@ +import numpy as np +import pandas as pd + +from optinist.api.dataclass.base import BaseData + +from optinist.api.utils.filepath_creater import create_directory, join_filepath +from optinist.api.utils.json_writer import JsonWriter + + +class CsvData(BaseData): + def __init__(self, data, params, file_name='csv'): + super().__init__(file_name) + + if isinstance(data, str): + self.data = pd.read_csv(data, header=None).values + + if params["transpose"]: + self.data = self.data.T + + if params["setHeader"] is not None: + header = params["setHeader"] + self.data = self.data[header:] + else: + self.data = np.array(data) + + if self.data.ndim == 1: + self.data = self.data[np.newaxis, :] + + def save_json(self, json_dir): + # timeseriesだけはdirを返す + self.json_path = join_filepath([json_dir, self.file_name]) + create_directory(self.json_path) + + for i, data in enumerate(self.data): + JsonWriter.write_as_split( + join_filepath([self.json_path, f'{str(i)}.json']), + data + ) diff --git a/optinist/api/dataclass/dataclass.py b/optinist/api/dataclass/dataclass.py index 8a832c8e0..7f168fd04 100644 --- a/optinist/api/dataclass/dataclass.py +++ b/optinist/api/dataclass/dataclass.py @@ -1,325 +1,11 @@ -import numpy as np -import pandas as pd -import imageio -import tifffile -import gc -from pynwb import NWBFile - - -from optinist.api.utils.filepath_creater import create_directory, join_filepath -from optinist.api.dir_path import DIRPATH -from optinist.api.dataclass.utils import create_images_list -from optinist.api.utils.json_writer import JsonWriter - - -class NWBFile(NWBFile): - pass - - -class BaseData: - def __init__(self, file_name): - self.file_name = file_name - - def save_json(self, json_dir): - pass - - -class ImageData(BaseData): - def __init__(self, data, file_name='image'): - super().__init__(file_name) - - self.json_path = None - - if data is None: - self.path = None - elif isinstance(data, str): - self.path = data - elif isinstance(data, list) and isinstance(data[0], str): - self.path = data - else: - _dir = join_filepath([DIRPATH.OUTPUT_DIR, "tiff", file_name]) - create_directory(_dir) - - _path = join_filepath([_dir, f'{file_name}.tif']) - tifffile.imsave(_path, data) - self.path = [_path] - - del data - gc.collect() - - @property - def data(self): - if isinstance(self.path, list): - return np.concatenate([imageio.volread(p) for p in self.path]) - else: - return np.array(imageio.volread(self.path)) - - def save_json(self, json_dir): - self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) - JsonWriter.write_as_split( - self.json_path, - create_images_list(self.data) - ) - - def __del__(self): - del self - gc.collect() - - -class TimeSeriesData(BaseData): - def __init__(self, data, std=None, index=None, cell_numbers=None, file_name='timeseries'): - super().__init__(file_name) - - assert data.ndim <= 2, 'TimeSeries Dimension Error' - - if isinstance(data, str): - self.data = pd.read_csv(data, header=None).values - else: - self.data = data - - if len(self.data.shape) == 1: - self.data = self.data[np.newaxis, :] - - self.std = std - - # indexを指定 - if index is not None: - self.index = index - else: - self.index = np.arange(len(self.data[0])) - - # cell番号を表示 - if cell_numbers is not None: - self.cell_numbers = cell_numbers + 1 - else: - self.cell_numbers = range(1, len(self.data) + 1) - - def save_json(self, json_dir): - - # timeseriesだけはdirを返す - self.json_path = join_filepath([json_dir, self.file_name]) - create_directory(self.json_path, delete_dir=True) - - for i, cell_i in enumerate(self.cell_numbers): - data = self.data[i] - if self.std is not None: - std = self.std[i] - df = pd.DataFrame( - np.concatenate([data[:, np.newaxis], std[:, np.newaxis]], axis=1), - index=self.index, - columns=["data", "std"] - ) - else: - df = pd.DataFrame( - data, - index=self.index, - columns=["data"] - ) - - JsonWriter.write( - join_filepath([self.json_path, f'{str(cell_i)}.json']), - df - ) - - def __del__(self): - del self - gc.collect() - - -class FluoData(TimeSeriesData): - def __init__(self, data, std=None, index=None, file_name='fluo'): - super().__init__( - data=data, - std=std, - index=index, - file_name=file_name, - ) - - -class BehaviorData(TimeSeriesData): - def __init__(self, data, std=None, index=None, file_name='behavior'): - super().__init__( - data=data, - std=std, - index=index, - file_name=file_name, - ) - - -class CsvData(BaseData): - def __init__(self, data, params, file_name='csv'): - super().__init__(file_name) - - if isinstance(data, str): - self.data = pd.read_csv(data, header=None).values - - if params["transpose"]: - self.data = self.data.T - - if params["setHeader"] is not None: - header = params["setHeader"] - self.data = self.data[header:] - else: - self.data = np.array(data) - - if self.data.ndim == 1: - self.data = self.data[np.newaxis, :] - - def save_json(self, json_dir): - # timeseriesだけはdirを返す - self.json_path = join_filepath([json_dir, self.file_name]) - create_directory(self.json_path) - - for i, data in enumerate(self.data): - JsonWriter.write_as_split( - join_filepath([self.json_path, f'{str(i)}.json']), - data - ) - - def __del__(self): - del self - gc.collect() - - -class HeatMapData(BaseData): - def __init__(self, data, columns=None, file_name='heatmap'): - super().__init__(file_name) - self.data = data - - # indexを指定 - if columns is not None: - self.columns = columns - else: - self.columns = np.arange(len(self.data[0])) - - def save_json(self, json_dir): - self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) - df = pd.DataFrame( - self.data, - columns=self.columns, - ) - JsonWriter.write_as_split(self.json_path, df) - - def __del__(self): - del self - gc.collect() - - -class RoiData(BaseData): - def __init__(self, data, file_name='roi'): - super().__init__(file_name) - - images = create_images_list(data) - - _dir = join_filepath([DIRPATH.OUTPUT_DIR, "tiff", file_name]) - create_directory(_dir) - self.path = join_filepath([_dir, f'{file_name}.tif']) - tifffile.imsave(self.path, images) - - del images, data - gc.collect() - - @property - def data(self): - data = np.array(imageio.volread(self.path)) - if data.ndim == 3: - return data[0] - elif data.ndim == 2: - return data - - def save_json(self, json_dir): - self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) - JsonWriter.write_as_split( - self.json_path, - create_images_list(self.data) - ) - - def __del__(self): - del self - gc.collect() - - -class Suite2pData(BaseData): - def __init__(self, data, file_name='suite2p'): - super().__init__(file_name) - self.data = data - - def __del__(self): - del self - gc.collect() - - -class IscellData(BaseData): - def __init__(self, data, file_name='iscell'): - super().__init__(file_name) - self.data = data - - def __del__(self): - del self - gc.collect() - - -class ScatterData(BaseData): - def __init__(self, data, file_name='scatter'): - super().__init__(file_name) - - assert data.ndim <= 2, 'Scatter Dimension Error' - - self.data = data.T - - def save_json(self, json_dir): - self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) - JsonWriter.write_as_split(self.json_path, self.data) - - def __del__(self): - del self - gc.collect() - - -class BarData(BaseData): - def __init__(self, data, index=None, file_name='bar'): - super().__init__(file_name) - data = np.array(data) - - assert data.ndim <= 2, 'Bar Dimension Error' - - if data.ndim == 1: - data = data[np.newaxis] - - assert data.ndim == 2, 'Bar Dimesion is not 2' - - self.data = data - - # indexを指定 - if index is not None: - self.index = index - else: - self.index = np.arange(len(self.data)) - - def save_json(self, json_dir): - self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) - df = pd.DataFrame( - self.data, - index=self.index, - ) - JsonWriter.write_as_split(self.json_path, df) - - def __del__(self): - del self - gc.collect() - - -class HTMLData(BaseData): - def __init__(self, data, file_name='html'): - super().__init__(file_name) - self.data = data - - def save_json(self, json_dir): - self.json_path = join_filepath([json_dir, f"{self.file_name}.html"]) - - with open(self.json_path, "w") as f: - f.write(self.data) - - def __del__(self): - del self - gc.collect() +from optinist.api.dataclass.base import * +from optinist.api.dataclass.bar import * +from optinist.api.dataclass.csv import * +from optinist.api.dataclass.heatmap import * +from optinist.api.dataclass.html import * +from optinist.api.dataclass.image import * +from optinist.api.dataclass.iscell import * +from optinist.api.dataclass.nwb import * +from optinist.api.dataclass.scatter import * +from optinist.api.dataclass.suite2p import * +from optinist.api.dataclass.timeseries import * \ No newline at end of file diff --git a/optinist/api/dataclass/heatmap.py b/optinist/api/dataclass/heatmap.py new file mode 100644 index 000000000..43778ef48 --- /dev/null +++ b/optinist/api/dataclass/heatmap.py @@ -0,0 +1,27 @@ +import numpy as np +import pandas as pd + +from optinist.api.utils.filepath_creater import join_filepath +from optinist.api.utils.json_writer import JsonWriter + +from optinist.api.dataclass.base import BaseData + + +class HeatMapData(BaseData): + def __init__(self, data, columns=None, file_name='heatmap'): + super().__init__(file_name) + self.data = data + + # indexを指定 + if columns is not None: + self.columns = columns + else: + self.columns = np.arange(len(self.data[0])) + + def save_json(self, json_dir): + self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) + df = pd.DataFrame( + self.data, + columns=self.columns, + ) + JsonWriter.write_as_split(self.json_path, df) diff --git a/optinist/api/dataclass/html.py b/optinist/api/dataclass/html.py new file mode 100644 index 000000000..3e1035af7 --- /dev/null +++ b/optinist/api/dataclass/html.py @@ -0,0 +1,14 @@ +from optinist.api.dataclass.base import BaseData +from optinist.api.utils.filepath_creater import join_filepath + + +class HTMLData(BaseData): + def __init__(self, data, file_name='html'): + super().__init__(file_name) + self.data = data + + def save_json(self, json_dir): + self.json_path = join_filepath([json_dir, f"{self.file_name}.html"]) + + with open(self.json_path, "w") as f: + f.write(self.data) diff --git a/optinist/api/dataclass/image.py b/optinist/api/dataclass/image.py new file mode 100644 index 000000000..5fd4cb93f --- /dev/null +++ b/optinist/api/dataclass/image.py @@ -0,0 +1,78 @@ +import numpy as np +import imageio +import tifffile +import gc + +from optinist.api.utils.filepath_creater import create_directory, join_filepath +from optinist.api.dir_path import DIRPATH +from optinist.api.dataclass.utils import create_images_list +from optinist.api.utils.json_writer import JsonWriter +from optinist.api.dataclass.base import BaseData + + +class ImageData(BaseData): + def __init__(self, data, file_name='image'): + super().__init__(file_name) + + self.json_path = None + + if data is None: + self.path = None + elif isinstance(data, str): + self.path = data + elif isinstance(data, list) and isinstance(data[0], str): + self.path = data + else: + _dir = join_filepath([DIRPATH.OUTPUT_DIR, "tiff", file_name]) + create_directory(_dir) + + _path = join_filepath([_dir, f'{file_name}.tif']) + tifffile.imsave(_path, data) + self.path = [_path] + + del data + gc.collect() + + @property + def data(self): + if isinstance(self.path, list): + return np.concatenate([imageio.volread(p) for p in self.path]) + else: + return np.array(imageio.volread(self.path)) + + def save_json(self, json_dir): + self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) + JsonWriter.write_as_split( + self.json_path, + create_images_list(self.data) + ) + + +class RoiData(BaseData): + def __init__(self, data, file_name='roi'): + super().__init__(file_name) + + images = create_images_list(data) + + _dir = join_filepath([DIRPATH.OUTPUT_DIR, "tiff", file_name]) + create_directory(_dir) + self.path = join_filepath([_dir, f'{file_name}.tif']) + tifffile.imsave(self.path, images) + + del images, data + gc.collect() + + @property + def data(self): + data = np.array(imageio.volread(self.path)) + if data.ndim == 3: + return data[0] + elif data.ndim == 2: + return data + + def save_json(self, json_dir): + self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) + JsonWriter.write_as_split( + self.json_path, + create_images_list(self.data) + ) diff --git a/optinist/api/dataclass/iscell.py b/optinist/api/dataclass/iscell.py new file mode 100644 index 000000000..f5ba7edc7 --- /dev/null +++ b/optinist/api/dataclass/iscell.py @@ -0,0 +1,7 @@ +from optinist.api.dataclass.base import BaseData + + +class IscellData(BaseData): + def __init__(self, data, file_name='iscell'): + super().__init__(file_name) + self.data = data diff --git a/optinist/api/dataclass/nwb.py b/optinist/api/dataclass/nwb.py new file mode 100644 index 000000000..7c6c57c7c --- /dev/null +++ b/optinist/api/dataclass/nwb.py @@ -0,0 +1,5 @@ +from pynwb import NWBFile + + +class NWBFile(NWBFile): + pass diff --git a/optinist/api/dataclass/scatter.py b/optinist/api/dataclass/scatter.py new file mode 100644 index 000000000..4c434d02f --- /dev/null +++ b/optinist/api/dataclass/scatter.py @@ -0,0 +1,16 @@ +from optinist.api.dataclass.base import BaseData +from optinist.api.utils.filepath_creater import join_filepath +from optinist.api.utils.json_writer import JsonWriter + + +class ScatterData(BaseData): + def __init__(self, data, file_name='scatter'): + super().__init__(file_name) + + assert data.ndim <= 2, 'Scatter Dimension Error' + + self.data = data.T + + def save_json(self, json_dir): + self.json_path = join_filepath([json_dir, f"{self.file_name}.json"]) + JsonWriter.write_as_split(self.json_path, self.data) diff --git a/optinist/api/dataclass/suite2p.py b/optinist/api/dataclass/suite2p.py new file mode 100644 index 000000000..ade219d1f --- /dev/null +++ b/optinist/api/dataclass/suite2p.py @@ -0,0 +1,7 @@ +from optinist.api.dataclass.base import BaseData + + +class Suite2pData(BaseData): + def __init__(self, data, file_name='suite2p'): + super().__init__(file_name) + self.data = data diff --git a/optinist/api/dataclass/timeseries.py b/optinist/api/dataclass/timeseries.py new file mode 100644 index 000000000..df6aa804c --- /dev/null +++ b/optinist/api/dataclass/timeseries.py @@ -0,0 +1,82 @@ +import numpy as np +import pandas as pd + +from optinist.api.dataclass.base import BaseData +from optinist.api.utils.filepath_creater import create_directory, join_filepath +from optinist.api.utils.json_writer import JsonWriter + + +class TimeSeriesData(BaseData): + def __init__(self, data, std=None, index=None, cell_numbers=None, file_name='timeseries'): + super().__init__(file_name) + + assert data.ndim <= 2, 'TimeSeries Dimension Error' + + if isinstance(data, str): + self.data = pd.read_csv(data, header=None).values + else: + self.data = data + + if len(self.data.shape) == 1: + self.data = self.data[np.newaxis, :] + + self.std = std + + # indexを指定 + if index is not None: + self.index = index + else: + self.index = np.arange(len(self.data[0])) + + # cell番号を表示 + if cell_numbers is not None: + self.cell_numbers = cell_numbers + 1 + else: + self.cell_numbers = range(1, len(self.data) + 1) + + def save_json(self, json_dir): + + # timeseriesだけはdirを返す + self.json_path = join_filepath([json_dir, self.file_name]) + create_directory(self.json_path, delete_dir=True) + + for i, cell_i in enumerate(self.cell_numbers): + data = self.data[i] + if self.std is not None: + std = self.std[i] + df = pd.DataFrame( + np.concatenate([data[:, np.newaxis], std[:, np.newaxis]], axis=1), + index=self.index, + columns=["data", "std"] + ) + else: + df = pd.DataFrame( + data, + index=self.index, + columns=["data"] + ) + + JsonWriter.write( + join_filepath([self.json_path, f'{str(cell_i)}.json']), + df + ) + + +class FluoData(TimeSeriesData): + def __init__(self, data, std=None, index=None, file_name='fluo'): + super().__init__( + data=data, + std=std, + index=index, + file_name=file_name, + ) + + +class BehaviorData(TimeSeriesData): + def __init__(self, data, std=None, index=None, file_name='behavior'): + super().__init__( + data=data, + std=std, + index=index, + file_name=file_name, + ) diff --git a/optinist/test_data/output/snakemake/input_0/data_endoscope.pkl b/optinist/test_data/output/snakemake/input_0/data_endoscope.pkl index df5127ef3..b5166dcf5 100644 Binary files a/optinist/test_data/output/snakemake/input_0/data_endoscope.pkl and b/optinist/test_data/output/snakemake/input_0/data_endoscope.pkl differ diff --git a/optinist/version.py b/optinist/version.py index ae8d6a90b..0150b3562 100644 --- a/optinist/version.py +++ b/optinist/version.py @@ -3,7 +3,7 @@ _MINOR = "1" # On main and in a nightly release the patch should be one ahead of the last # released build. -_PATCH = "1" +_PATCH = "2" # This is mainly for nightly builds which have the suffix ".dev$DATE". See # https://semver.org/#is-v123-a-semantic-version for the semantics. _SUFFIX = "" diff --git a/requirements.txt b/requirements.txt index ac88f00db..78ff261cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,6 @@ future pynwb psutil nbstripout -# snakemake -git+https://github.com/ShogoAkiyama/snakemake@main#egg=snakemake +snakemake +# git+https://github.com/ShogoAkiyama/snakemake@main#egg=snakemake jinja2==3.0.3 diff --git a/setup.py b/setup.py index 7493a0ddb..c0a62a2ef 100644 --- a/setup.py +++ b/setup.py @@ -7,10 +7,10 @@ def _requires_from_file(filepath): def take_package_name(name): - if "snakemake" in name: - return f"snakemake @ https://github.com/ShogoAkiyama/snakemake/archive/refs/tags/v7.7.2-post1.zip" - else: - return name.strip() + # if "snakemake" in name: + # return f"snakemake @ https://github.com/ShogoAkiyama/snakemake/archive/refs/tags/v7.7.2-post1.zip" + # else: + return name.strip() with open(filepath) as fp: return [take_package_name(pkg_name) for pkg_name in fp.readlines()]