Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[MNT, DOC] Accelerating deep testing #1904

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

[MNT, DOC] Accelerating deep testing #1904

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d138dda
fix test clustering
hadifawaz1999 Aug 4, 2024
bead7e3
fix bug naming test
hadifawaz1999 Aug 4, 2024
f0e2f86
make classif testing faster
hadifawaz1999 Aug 4, 2024
c29da6a
try fix encoder network
hadifawaz1999 Aug 4, 2024
2eb10a3
use self attention now
hadifawaz1999 Aug 4, 2024
dc6c597
now regression test
hadifawaz1999 Aug 4, 2024
c32ecf1
Merge branch 'main' of https://github.com/aeon-toolkit/aeon
hadifawaz1999 Aug 5, 2024
f982e24
Merge branch 'main' into aif/deep-cltr-testing
hadifawaz1999 Aug 5, 2024
8e582c5
add dummy cltr option use it in test
hadifawaz1999 Aug 5, 2024
4050ae3
add assert error messages
hadifawaz1999 Aug 8, 2024
10a270b
Merge branch 'main' of https://github.com/aeon-toolkit/aeon into aif/…
hadifawaz1999 Aug 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions aeon/classification/deep_learning/_cnn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,9 +332,12 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"avg_pool_size": 4,
"n_layers": 1,
"n_filters": 1,
"kernel_size": 2,
"avg_pool_size": 2,
}

test_params = [param1]
Expand Down Expand Up @@ -652,9 +655,12 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"avg_pool_size": 4,
"n_layers": 1,
"n_filters": 1,
"kernel_size": 2,
"avg_pool_size": 2,
}

test_params = [param1]
Expand Down
2 changes: 2 additions & 0 deletions aeon/classification/deep_learning/_encoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,8 @@ def get_test_params(cls, parameter_set="default"):
"n_epochs": 8,
"batch_size": 4,
"use_bias": False,
"n_filters": [2],
"kernel_size": [2],
"fc_units": 8,
"strides": 2,
"dropout_proba": 0,
Expand Down
2 changes: 1 addition & 1 deletion aeon/classification/deep_learning/_fcn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"use_bias": False,
"n_layers": 1,
Expand Down
21 changes: 15 additions & 6 deletions aeon/classification/deep_learning/_inception_time.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,11 +364,16 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_classifiers": 1,
"n_epochs": 10,
"n_classifiers": 2,
"n_epochs": 2,
"batch_size": 4,
"kernel_size": 4,
"depth": 1,
"kernel_size": 2,
"n_filters": 1,
"n_conv_per_layer": 1,
"use_residual": False,
"use_bottleneck": False,
"use_max_pooling": False,
"depth": 1,
"use_custom_filters": False,
}
Expand Down Expand Up @@ -747,11 +752,15 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"kernel_size": 4,
"depth": 1,
"kernel_size": 2,
"n_filters": 1,
"n_conv_per_layer": 1,
"use_residual": False,
"use_bottleneck": True,
"use_bottleneck": False,
"use_max_pooling": False,
"depth": 1,
"use_custom_filters": False,
}
Expand Down
2 changes: 1 addition & 1 deletion aeon/classification/deep_learning/_mlp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"use_bias": False,
}
Expand Down
2 changes: 1 addition & 1 deletion aeon/classification/deep_learning/_resnet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"n_residual_blocks": 1,
"n_filters": 5,
Expand Down
7 changes: 5 additions & 2 deletions aeon/classification/deep_learning/tests/test_random_state_deep_learning.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,15 @@ def test_random_state_deep_learning_cls(deep_cls):

X, y = make_example_3d_numpy(random_state=random_state)

deep_cls1 = deep_cls(random_state=random_state, n_epochs=4)
test_params = deep_cls.get_test_params()[0]
test_params["random_state"] = random_state

deep_cls1 = deep_cls(**test_params)
deep_cls1.fit(X, y)

layers1 = deep_cls1.training_model_.layers[1:]

deep_cls2 = deep_cls(random_state=random_state, n_epochs=4)
deep_cls2 = deep_cls(**test_params)
deep_cls2.fit(X, y)

layers2 = deep_cls2.training_model_.layers[1:]
Expand Down
26 changes: 13 additions & 13 deletions aeon/classification/deep_learning/tests/test_saving_loading_deep_learning_cls.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,35 +46,35 @@ def test_saving_loading_deep_learning_cls(deep_cls):

X, y = make_example_3d_numpy()

deep_cls_train = deep_cls(
n_epochs=2,
save_best_model=True,
save_last_model=True,
save_init_model=True,
best_file_name=best_file_name,
last_file_name=last_file_name,
init_file_name=init_file_name,
file_path=tmp,
)
test_params = deep_cls.get_test_params()[0]
test_params["save_best_model"] = True
test_params["save_last_model"] = True
test_params["save_init_model"] = True
test_params["best_file_name"] = best_file_name
test_params["last_file_name"] = last_file_name
test_params["init_file_name"] = init_file_name
test_params["file_path"] = tmp

deep_cls_train = deep_cls(**test_params)
deep_cls_train.fit(X, y)

deep_cls_best = deep_cls()
deep_cls_best = deep_cls(**test_params)
deep_cls_best.load_model(
model_path=os.path.join(tmp, best_file_name + ".keras"),
classes=np.unique(y),
)
ypred_best = deep_cls_best.predict(X)
assert len(ypred_best) == len(y)

deep_cls_last = deep_cls()
deep_cls_last = deep_cls(**test_params)
deep_cls_last.load_model(
model_path=os.path.join(tmp, last_file_name + ".keras"),
classes=np.unique(y),
)
ypred_last = deep_cls_last.predict(X)
assert len(ypred_last) == len(y)

deep_cls_init = deep_cls()
deep_cls_init = deep_cls(**test_params)
deep_cls_init.load_model(
model_path=os.path.join(tmp, init_file_name + ".keras"),
classes=np.unique(y),
Expand Down
13 changes: 5 additions & 8 deletions aeon/clustering/deep_learning/_ae_fcn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,16 +331,13 @@ def get_test_params(cls, parameter_set="default"):
"batch_size": 4,
"use_bias": False,
"n_layers": 1,
"n_filters": 5,
"kernel_size": 3,
"n_filters": 4,
"kernel_size": 2,
"padding": "same",
"strides": 1,
"clustering_params": {
"distance": "euclidean",
"averaging_method": "mean",
"n_init": 1,
"max_iter": 30,
},
"latent_space_dim": 4,
"clustering_algorithm": "dummy",
"clustering_params": {"strategy": "random"},
}

return [param1]
13 changes: 6 additions & 7 deletions aeon/clustering/deep_learning/_ae_resnet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class AEResNetClusterer(BaseDeepClusterer):
The number of convolution filters for all the convolution layers in the same
residual block, if not a list, the same number of filters is used in all
convolutions of all residual blocks.
kernel_sizes : int or list of int, default = [8, 5, 3]
kernel_size : int or list of int, default = [8, 5, 3]
The kernel size of all the convolution layers in one residual block, if not
a list, the same kernel size is used in all convolution layers.
strides : int or list of int, default = 1
Expand Down Expand Up @@ -352,12 +352,11 @@ def get_test_params(cls, parameter_set="default"):
"batch_size": 4,
"n_residual_blocks": 1,
"n_conv_per_residual_block": 1,
"clustering_params": {
"distance": "euclidean",
"averaging_method": "mean",
"n_init": 1,
"max_iter": 30,
},
"n_filters": 1,
"kernel_size": 2,
"use_bias": False,
"clustering_algorithm": "dummy",
"clustering_params": {"strategy": "random"},
}

test_params = [param]
Expand Down
9 changes: 7 additions & 2 deletions aeon/clustering/deep_learning/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from aeon.clustering._k_medoids import TimeSeriesKMedoids
from aeon.clustering._k_shapes import TimeSeriesKShapes
from aeon.clustering.base import BaseClusterer
from aeon.clustering.dummy import DummyClusterer


class BaseDeepClusterer(BaseClusterer, ABC):
Expand All @@ -17,7 +18,7 @@ class BaseDeepClusterer(BaseClusterer, ABC):
----------
n_clusters : int, default=None
Number of clusters for the deep learning model.
clustering_algorithm : str, {'kmeans', 'kshape', 'kmedoids'},
clustering_algorithm : str, {'kmeans', 'kshape', 'kmedoids', 'dummy'},
default="kmeans"
The clustering algorithm used in the latent space.
Options include:
Expand Down Expand Up @@ -115,7 +116,11 @@ def _fit_clustering(self, X):
else:
clustering_params_ = self.clustering_params
# clustering_params_["n_clusters"] = self.n_clusters
if self.clustering_algorithm == "kmeans":
if self.clustering_algorithm == "dummy":
self.clusterer = DummyClusterer(
n_clusters=self.n_clusters, **clustering_params_
)
elif self.clustering_algorithm == "kmeans":
Comment on lines +119 to +123
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this not just accept any BaseClusterer? Creating a useless option solely for testing is not a great way to resolve this IMO.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i wanted to change that for accepting an estimator input instead of string, but thought it might be a lot for the PR, but to keep the PR for testing purpose this can be done, if you think its ok to get all in one PR i dont mind can do the changes here

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't mind if you do it here. The dummy option is not a good addition IMO.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will add the changes then

self.clusterer = TimeSeriesKMeans(
n_clusters=self.n_clusters, **clustering_params_
)
Expand Down
64 changes: 39 additions & 25 deletions aeon/clustering/deep_learning/tests/test_random_state_deep_learning_cluster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,49 +7,63 @@

from aeon.clustering import deep_learning
from aeon.testing.data_generation import make_example_3d_numpy
from aeon.utils.validation._dependencies import _check_soft_dependencies

__maintainer__ = ["hadifawaz1999"]

_deep_clr_classes = [
member[1] for member in inspect.getmembers(deep_learning, inspect.isclass)
]


@pytest.mark.skipif(
# not _check_soft_dependencies("tensorflow", severity="none"),
# See Issue #1761
True,
not _check_soft_dependencies("tensorflow", severity="none"),
reason="skip test if required soft dependency not available",
)
def test_random_state_deep_learning_clr():
@pytest.mark.parametrize("deep_clr", _deep_clr_classes)
def test_random_state_deep_learning_clr(deep_clr):
"""Test Deep Clusterer seeding."""
random_state = 42

X, _ = make_example_3d_numpy(random_state=random_state)
if not (deep_clr.__name__ in ["BaseDeepClusterer"]):
random_state = 42

deep_clr_classes = [
member[1] for member in inspect.getmembers(deep_learning, inspect.isclass)
]
X, _ = make_example_3d_numpy(random_state=random_state)

for i in range(len(deep_clr_classes)):
if "BaseDeepClusterer" in str(deep_clr_classes[i]):
continue
test_params = deep_clr.get_test_params()[0]
test_params["random_state"] = random_state

deep_clr1 = deep_clr_classes[i](
n_clusters=2, random_state=random_state, n_epochs=4
)
deep_clr1 = deep_clr(**test_params)
deep_clr1.fit(X)

layers1 = deep_clr1.training_model_.layers[1:]
encoder1 = deep_clr1.training_model_.layers[1]
decoder1 = deep_clr1.training_model_.layers[2]
encoder_layers1 = encoder1.layers[1:]
decoder_layers1 = decoder1.layers[1:]

deep_clr2 = deep_clr_classes[i](
n_clusters=2, random_state=random_state, n_epochs=4
)
deep_clr2 = deep_clr(**test_params)
deep_clr2.fit(X)

layers2 = deep_clr2.training_model_.layers[1:]
encoder2 = deep_clr2.training_model_.layers[1]
decoder2 = deep_clr2.training_model_.layers[2]
encoder_layers2 = encoder2.layers[1:]
decoder_layers2 = decoder2.layers[1:]

# test encoders
for i in range(len(encoder_layers1)):
weights1 = encoder_layers1[i].get_weights()
weights2 = encoder_layers2[i].get_weights()

assert len(weights1) == len(weights2)

for j in range(len(weights1)):
_weight1 = np.asarray(weights1[j])
_weight2 = np.asarray(weights2[j])

assert len(layers1) == len(layers2)
np.testing.assert_almost_equal(_weight1, _weight2, 4)

for i in range(len(layers1)):
weights1 = layers1[i].get_weights()
weights2 = layers2[i].get_weights()
# test decoders
for i in range(len(decoder_layers1)):
weights1 = decoder_layers1[i].get_weights()
weights2 = decoder_layers2[i].get_weights()

assert len(weights1) == len(weights2)

Expand Down
7 changes: 7 additions & 0 deletions aeon/networks/_ae_resnet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,36 +138,43 @@ def build_network(self, input_shape, **kwargs):
self._kernel_size_ = [8, 5, 3] if self.kernel_size is None else self.kernel_size

if isinstance(self._n_filters_, list):
assert len(self._n_filters_) == self.n_residual_blocks
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

raise an actual error with a message instead of asserting

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will do, we should raise an issue to do that all over the networks module, my code my bad ! never thought about raising a message

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be better as a ValueError IMO

self._n_filters = self._n_filters_
else:
self._n_filters = [self._n_filters_] * self.n_residual_blocks

if isinstance(self._kernel_size_, list):
assert len(self._kernel_size_) == self.n_conv_per_residual_block
self._kernel_size = self._kernel_size_
else:
self._kernel_size = [self._kernel_size_] * self.n_conv_per_residual_block

if isinstance(self.strides, list):
assert len(self.strides) == self.n_conv_per_residual_block
self._strides = self.strides
else:
self._strides = [self.strides] * self.n_conv_per_residual_block

if isinstance(self.dilation_rate, list):
assert len(self.dilation_rate) == self.n_conv_per_residual_block
self._dilation_rate = self.dilation_rate
else:
self._dilation_rate = [self.dilation_rate] * self.n_conv_per_residual_block

if isinstance(self.padding, list):
assert len(self.padding) == self.n_conv_per_residual_block
self._padding = self.padding
else:
self._padding = [self.padding] * self.n_conv_per_residual_block

if isinstance(self.activation, list):
assert len(self.activation) == self.n_conv_per_residual_block
self._activation = self.activation
else:
self._activation = [self.activation] * self.n_conv_per_residual_block

if isinstance(self.use_bias, list):
assert len(self.use_bias) == self.n_conv_per_residual_block
self._use_bias = self.use_bias
else:
self._use_bias = [self.use_bias] * self.n_conv_per_residual_block
Expand Down
Loading