From ccdbd2015d66acd374bf388a5b8abd414b710bea Mon Sep 17 00:00:00 2001 From: Ruan Comelli Date: Fri, 11 Jun 2021 14:43:28 -0300 Subject: [PATCH] Minor code improvement, including adding more imports to __init__ --- boiling_learning/datasets/creators.py | 14 +- boiling_learning/datasets/datasets.py | 29 ++- boiling_learning/preprocessing/__init__.py | 1 + boiling_learning/preprocessing/image.py | 6 +- boiling_learning/utils/__init__.py | 1 + image_generators/steady_state.py | 3 +- run_experiment.py | 23 +-- sandbox/sandbox.py | 228 --------------------- 8 files changed, 34 insertions(+), 271 deletions(-) delete mode 100644 sandbox/sandbox.py diff --git a/boiling_learning/datasets/creators.py b/boiling_learning/datasets/creators.py index 8fd9aa71..aa8ac078 100644 --- a/boiling_learning/datasets/creators.py +++ b/boiling_learning/datasets/creators.py @@ -11,8 +11,8 @@ import boiling_learning.utils as bl_utils from boiling_learning.datasets.datasets import ( DatasetSplits, - tf_concatenate, - tf_train_val_test_split, + concatenate, + train_val_test_split, ) from boiling_learning.io.io import DatasetTriplet from boiling_learning.management.Manager import Manager @@ -41,7 +41,7 @@ def experiment_video_dataset_creator( t.as_tf_py_function(pack_tuple=True), num_parallel_calls=AUTOTUNE ) - ds_train, ds_val, ds_test = tf_train_val_test_split(ds, splits) + ds_train, ds_val, ds_test = train_val_test_split(ds, splits) if dataset_size is not None: ds_train = ds_train.take(dataset_size) @@ -53,7 +53,7 @@ def experiment_video_dataset_creator( snapshot_path = bl_utils.ensure_dir(snapshot_path) if dataset_size is not None: - num_shards = min([dataset_size, num_shards]) + num_shards = min(dataset_size, num_shards) ds_train = ds_train.apply( bl_preprocessing.snapshotter( @@ -175,12 +175,12 @@ def dataset_creator( tuple, mit.unzip(ds_dict.values()) ) - ds_train = tf_concatenate(datasets_train) + ds_train = concatenate(datasets_train) if None in datasets_val: ds_val = None else: - ds_val = tf_concatenate(datasets_val) - ds_test = tf_concatenate(datasets_test) + ds_val = concatenate(datasets_val) + ds_test = concatenate(datasets_test) if dataset_size is not None: ds_train = ds_train.take(dataset_size) diff --git a/boiling_learning/datasets/datasets.py b/boiling_learning/datasets/datasets.py index e60dfe2b..15291421 100644 --- a/boiling_learning/datasets/datasets.py +++ b/boiling_learning/datasets/datasets.py @@ -115,7 +115,7 @@ def to_str(self): ) -def tf_concatenate(datasets: Iterable[tf.data.Dataset]) -> tf.data.Dataset: +def concatenate(datasets: Iterable[tf.data.Dataset]) -> tf.data.Dataset: datasets = deque(datasets) if not datasets: @@ -128,7 +128,14 @@ def tf_concatenate(datasets: Iterable[tf.data.Dataset]) -> tf.data.Dataset: return ds -def tf_train_val_test_split( +def flatten_zip(*ds: tf.data.Dataset) -> tf.data.Dataset: + if len(ds) == 1: + return ds[0] + else: + return tf.data.Dataset.zip(ds) + + +def train_val_test_split( ds: tf.data.Dataset, splits: DatasetSplits, shuffle: bool = False ) -> DatasetTriplet: """#TODO(docstring): describe here @@ -139,12 +146,6 @@ def tf_train_val_test_split( Consequently it is safe to be used early in the pipeline to avoid consuming test data. """ - def flatten_zip(*ds: tf.data.Dataset) -> tf.data.Dataset: - if len(ds) == 1: - return ds[0] - else: - return tf.data.Dataset.zip(ds) - if splits.val == 0: split_train, split_test = mathutils.proportional_ints( splits.train, splits.test @@ -181,7 +182,7 @@ def flatten_zip(*ds: tf.data.Dataset) -> tf.data.Dataset: return ds_train, ds_val, ds_test -def tf_train_val_test_split_concat( +def train_val_test_split_concat( datasets: Iterable[tf.data.Dataset], splits: DatasetSplits ) -> DatasetTriplet: """#TODO(docstring): describe here @@ -199,16 +200,16 @@ def tf_train_val_test_split_concat( raise ValueError('argument *datasets* must be a non-empty iterable.') ds = datasets.popleft() - ds_train, ds_val, ds_test = tf_train_val_test_split(ds, splits) + ds_train, ds_val, ds_test = train_val_test_split(ds, splits) if ds_val is None: for ds in datasets: - ds_train_, _, ds_test_ = tf_train_val_test_split(ds, splits) + ds_train_, _, ds_test_ = train_val_test_split(ds, splits) ds_train = ds_train.concatenate(ds_train_) ds_test = ds_test.concatenate(ds_test_) else: for ds in datasets: - ds_train_, ds_val_, ds_test_ = tf_train_val_test_split(ds, splits) + ds_train_, ds_val_, ds_test_ = train_val_test_split(ds, splits) ds_train = ds_train.concatenate(ds_train_) ds_val = ds_val.concatenate(ds_val_) ds_test = ds_test.concatenate(ds_test_) @@ -266,9 +267,7 @@ def take( if isinstance(count, int): return ds.take(count) elif isinstance(count, Fraction): - return tf_train_val_test_split(ds, splits=DatasetSplits(train=count))[ - 0 - ] + return train_val_test_split(ds, splits=DatasetSplits(train=count))[0] else: raise TypeError( f'*count* must be either *int* or *Fraction*, got {type(count)}.' diff --git a/boiling_learning/preprocessing/__init__.py b/boiling_learning/preprocessing/__init__.py index 48a1dcb7..0fff1908 100644 --- a/boiling_learning/preprocessing/__init__.py +++ b/boiling_learning/preprocessing/__init__.py @@ -1,4 +1,5 @@ from boiling_learning.preprocessing.Case import * from boiling_learning.preprocessing.ExperimentalData import * from boiling_learning.preprocessing.ExperimentVideo import * +from boiling_learning.preprocessing.ImageDataset import * from boiling_learning.preprocessing.preprocessing import * diff --git a/boiling_learning/preprocessing/image.py b/boiling_learning/preprocessing/image.py index da3eeb7b..8854ac4e 100644 --- a/boiling_learning/preprocessing/image.py +++ b/boiling_learning/preprocessing/image.py @@ -1,3 +1,4 @@ +import math from typing import Any, Iterable, Optional, Tuple, TypeVar, Union import numpy as np @@ -159,7 +160,10 @@ def grayscale(image: ImageType) -> tf.Tensor: def downscale( image: ImageType, factors: Tuple[int, int], antialias: bool = False ) -> tf.Tensor: - sizes = (image.shape[0] // factors[0], image.shape[1] // factors[1]) + sizes = ( + math.ceil(image.shape[0] / factors[0]), + math.ceil(image.shape[1] / factors[1]), + ) return tf.image.resize( image, sizes, method='bilinear', antialias=antialias ) diff --git a/boiling_learning/utils/__init__.py b/boiling_learning/utils/__init__.py index 734feacd..ce2b8738 100644 --- a/boiling_learning/utils/__init__.py +++ b/boiling_learning/utils/__init__.py @@ -1,2 +1,3 @@ from boiling_learning.utils.FrozenDict import * +from boiling_learning.utils.Parameters import * from boiling_learning.utils.utils import * diff --git a/image_generators/steady_state.py b/image_generators/steady_state.py index db57398b..22e47365 100644 --- a/image_generators/steady_state.py +++ b/image_generators/steady_state.py @@ -1,8 +1,9 @@ +from pathlib import Path + import matplotlib as mpl import matplotlib.pyplot as plt import modin.pandas as pd import seaborn as sns -from pathlib2 import Path STEADY_STATE_DIR = Path(__file__).parent.parent / 'Selected Experiments' STEADY_STATE_PATH = STEADY_STATE_DIR / 'SteadyState.csv' diff --git a/run_experiment.py b/run_experiment.py index f0a6dce4..c71e973a 100644 --- a/run_experiment.py +++ b/run_experiment.py @@ -5,6 +5,7 @@ from datetime import datetime from pathlib import Path +import matplotlib as mpl import modin.pandas as pd import nidaqmx import numpy as np @@ -66,7 +67,6 @@ def surface_area(self): ) -#%% # ------------------------------------------------------- # Settings # ------------------------------------------------------- @@ -224,7 +224,6 @@ def correct_wire_temperature(reference_file): # For timestamps: -#%% """ Support definitions ------------------------------------------------------- """ @@ -283,7 +282,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): bl.utils.print_verbose(cond, *args, **kwargs) -#%% # ------------------------------------------------------- # Channel definitions # ------------------------------------------------------- @@ -339,7 +337,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): ChannelType.INPUT, ) -#%% """ Print system information ------------------------------------------------------- """ @@ -361,7 +358,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): ('anything', 'info'), f'> Types in ChannelType: {[x for x in ChannelType]}' ) -#%% """ Load calibration polynomial ------------------------------------------------------- """ @@ -374,7 +370,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): f'> Calibrated polynomial:\n{calibrated_polynomial}', ) -#%% # ------------------------------------------------------- # Initialize # ------------------------------------------------------- @@ -397,7 +392,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): output_writer = csv.writer(output_file) - #%% # ------------------------------------------------------- # Setup channels # ------------------------------------------------------- @@ -458,7 +452,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): f'experiment samp_clk_rate: {experiment.timing.samp_clk_rate}', ) - #%% # ------------------------------------------------------- # Run experiment # ------------------------------------------------------- @@ -472,12 +465,10 @@ def print_if_must(keys, *args, conds=None, **kwargs): # Header # ------------------------------------------------------- print_if_must(('anything', 'info'), f'> Iteration {iter_count}') - #%% # ------------------------------------------------------- # Time measurement # ------------------------------------------------------- elapsed_time = np.array([time.time() - start]) - #%% # ------------------------------------------------------- # Read data # ------------------------------------------------------- @@ -485,7 +476,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): number_of_samples_per_channel=nidaqmx.constants.READ_ALL_AVAILABLE ) - #%% # ------------------------------------------------------- # Process data # ------------------------------------------------------- @@ -524,7 +514,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): experiment, readings, dtype=np.array ) - #%% # ------------------------------------------------------- # Saving # ------------------------------------------------------- @@ -542,7 +531,7 @@ def print_if_must(keys, *args, conds=None, **kwargs): # 'Temperature from Resistance [deg C]': wire_temperature_from_resistance, # 'Wire Temperature (corrected) [deg C]': wire_temperature_corrected, } - n_values = min([local.size for local in local_data.values()]) + n_values = min(local.size for local in local_data.values()) # Time measurement if n_values > 1: @@ -569,6 +558,7 @@ def print_if_must(keys, *args, conds=None, **kwargs): # TODO: here if measure_loop_time: + previous_elapsed_time = None if first: loop_time = np.zeros(n_values) else: @@ -608,7 +598,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): time.sleep(sleeping_time) continue - #%% # ------------------------------------------------------- # Writing to file # ------------------------------------------------------- @@ -626,7 +615,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): print_if_must(('anything', 'writing'), '>> Done') - #%% """ Printing ------------------------------------------------------- """ @@ -670,7 +658,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): # print_if_must(('anything', 'temperature from resistance'), f'>> Temperature from Resistance [deg C]: {wire_temperature_from_resistance}', conds=[wire_temperature_from_resistance.size > 0]) # print_if_must(('anything', 'wire temperature corrected'), f'>> Wire Temperature (corrected) [deg C]: {wire_temperature_corrected}', conds=[wire_temperature_corrected.size > 0]) - #%% # ------------------------------------------------------- # Plotting # ------------------------------------------------------- @@ -748,7 +735,6 @@ def print_if_must(keys, *args, conds=None, **kwargs): QtGui.QApplication.processEvents() - #%% # ------------------------------------------------------- # Finish iteration # ------------------------------------------------------- @@ -764,11 +750,10 @@ def print_if_must(keys, *args, conds=None, **kwargs): ################## -#%% # ------------------------------------------------------- # Plot Results # ------------------------------------------------------- -matplotlib.use('Agg') +mpl.use('Agg') # datatype = [ # ('index', np.float32), diff --git a/sandbox/sandbox.py b/sandbox/sandbox.py deleted file mode 100644 index 7ffbf843..00000000 --- a/sandbox/sandbox.py +++ /dev/null @@ -1,228 +0,0 @@ - -#%% -# Playing with annotations: -if True: - def hi(arg: str) -> bool: - return True - -print(hi.__annotations__) -print(hi(1)) - -#%% -# Accessing member variables -class X: - def __init__(self, value=0): - self.value = value - -def change_value(x): - x.value += 1 - -def change(x): - X(5) - -x = X(3) -change_value(x) -print(x.value) - -change(x) -print(x.value) - -import datetime -#%% -# time and datetime -import time - -now = time.time() -now_date = datetime.datetime.fromtimestamp(now) -print(now_date) - -# Merging dicts -def merge_dicts(*dicts, inplace=False): - if len(dicts): - if inplace: - for d in dicts: - dicts[0].update(d) - return dicts[0] - else: - return merge_dicts({}, *dicts, inplace=True) - else: - return {} - - -a = { - 'p': 0, - 'y': 1 -} -b = { - 't': 0, - 'p': 3 -} -c = {} -d = { - 'h': -1 -} - -print(f'{merge_dicts({})}') -print(f'{merge_dicts(a, b)}') -print(f'{merge_dicts(a, c, b, d, d)}') -print(f'{merge_dicts(a, d, inplace=True)}') -print(f'{merge_dicts(a, b)}') - -#%% -import numpy as np - -wire_diameter = 0.518e-3 # m -wire_length = 6.5e-2 # m -wire_surface_area = np.pi * wire_diameter * wire_length -range_power = np.array([5, 15, 35, 75, 100]) -heat_fluxes = range_power / wire_surface_area -print('Heat fluxes:', heat_fluxes, 'W/m^2') -print('Heat fluxes:', heat_fluxes/100**2, 'W/cm^2') - -#%% -# Relative paths -def common_values(lhs, rhs): - for lvalue, rvalue in zip(lhs, rhs): - if lvalue == rvalue: - yield lvalue - else: - return - -def common_list(lhs, rhs): - return list(common_values(lhs, rhs)) - -def common_path(lhs, rhs): - resolved_lhs = lhs.resolve() - resolved_rhs = rhs.resolve() - return common_list( - reversed(resolved_lhs.parents), - reversed(resolved_rhs.parents))[-1] - -# def relative_path(origin, destination): -# common = common_path(origin, destination) - -# upwards_path = origin.relative_to(common) -# upwards = Path('/'.join(['..'] * len(upwards_path.parts))) - -# downwards = destination.relative_to(common) - -# return upwards.joinpath(downwards) - -def relative_path(origin, destination): - from os.path import relpath - return relpath(destination, start=origin) - -from pathlib import Path - -origin = Path('middle-earth/gondor/minas-tirith/castle').resolve() -destination = Path('middle-earth/gondor/osgiliath/tower').absolute() - -import os.path - -# result = Path('../../also/in/another/place') -result = relative_path(origin, destination) - -print(os.path.relpath(destination, start=origin)) - -def absolute_file_path_to_relative(start_file_path, destination_file_path): - return (start_file_path.count("/") + start_file_path.count("\\")) * (".." + ((start_file_path.find("/") > -1) and "/" or "\\")) + destination_file_path - -def relative_file_path(start, dest): - count = len(start.parts) - upwards = Path('/'.join(['..']*count)) - return upwards / dest - -# result = absolute_file_path_to_relative(str(origin), str(destination)) -print(absolute_file_path_to_relative(str(origin), str(destination))) - -result = relative_file_path(origin, destination) - -print(f'Origin: {origin}') -print(f'Destination: {destination}') -print(f'Result: {result}') -print(f'Got to: {origin.joinpath(result).resolve()}') -print(f'Works: {origin.joinpath(result).resolve() == destination}') -# print(f'Equal: {expected_result.resolve() == result.resolve()}') - -#%% -import numpy as np - - -def relu(x, grad=False): - numpy_x= np.array(x) - if grad: - return np.where(numpy_x <= 0, 0, 1) - return np.maximum(0, numpy_x) - -a=np.array([[1,2,3],[2,3,4]]) -print(np.apply_along_axis(relu, 1, a)) -print(np.apply_along_axis(lambda x: relu(x, grad=True), 1, a)) - -print(relu(a)) -print(relu(a, True)) - -#%% - -# def multi_split(*arrays, sizes=None, **options): -# from sklearn.model_selection import train_test_split - -# if sizes is None: -# return train_test_split(*arrays, **options) - -# if all(isinstance(size, int) for size in sizes): -# split_arrays = [] -# for size in sizes: -# if 'train_size' in previous_size: - -# elif all(isinstance(size, float) for size in sizes): -# split_arrays = [] -# previous_size = 1 -# for size in sizes: -# train_size = -# split_arrays.append( -# train_test_split(*arrays, train_size=train_size, **options) -# ) - -# else: -# TypeError(f'there is no support for sizes={sizes}') - -#%% -x = 3 -try: - print(x(6)) -except TypeError: - print('Success: TypeError raised') - -#%% -try: - from math import prod -except ImportError: - def prod(iterable, start=1.0): - import operator - from functools import reduce - - return start*reduce(operator.mul, iterable, 1) - -import numpy as np - -dims = [4, 4, 4, 4] -a = np.arange(prod(dims)) -a = a.reshape(dims) -print(a) - -print('Sliced a') -print( - a.__getitem__( - tuple([slice(1, 4), slice(0, 2)]) - ) -) - -#%% - -wire_diameter = 0.518e-3 # m -wire_length = 6.5e-2 # m -wire_surface_area = np.pi * wire_diameter * wire_length - -#%% -print('Hello, {name}'.format(name='Frodo', surname='Baggins')) -print('{name}' in 'Hello, {name}') \ No newline at end of file