diff --git a/cases/credit_scoring/credit_scoring_problem_multiobj.py b/cases/credit_scoring/credit_scoring_problem_multiobj.py index 01fccbb658..4a104531cb 100644 --- a/cases/credit_scoring/credit_scoring_problem_multiobj.py +++ b/cases/credit_scoring/credit_scoring_problem_multiobj.py @@ -15,7 +15,7 @@ from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.operation_types_repository import get_operations_for_task -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, ComplexityMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum, ComplexityMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import set_random_seed diff --git a/cases/evo_operators_comparison/evo_operators_comparison.py b/cases/evo_operators_comparison/evo_operators_comparison.py index c9554b7cea..1a89fe81be 100644 --- a/cases/evo_operators_comparison/evo_operators_comparison.py +++ b/cases/evo_operators_comparison/evo_operators_comparison.py @@ -16,7 +16,7 @@ from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.repository.operation_types_repository import get_operations_for_task -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import fedot_project_root diff --git a/cases/river_levels_prediction/river_level_case_composer.py b/cases/river_levels_prediction/river_level_case_composer.py index 138dd1efc9..b60f51e3cc 100644 --- a/cases/river_levels_prediction/river_level_case_composer.py +++ b/cases/river_levels_prediction/river_level_case_composer.py @@ -16,7 +16,7 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import \ +from fedot.core.repository.metrics_repository import \ RegressionMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum diff --git a/cases/river_levels_prediction/river_level_case_manual.py b/cases/river_levels_prediction/river_level_case_manual.py index d63431e0ac..a85308ce73 100644 --- a/cases/river_levels_prediction/river_level_case_manual.py +++ b/cases/river_levels_prediction/river_level_case_manual.py @@ -11,7 +11,7 @@ from fedot.core.pipelines.node import PipelineNode from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum warnings.filterwarnings('ignore') diff --git a/docs/source/advanced/hyperparameters_tuning.rst b/docs/source/advanced/hyperparameters_tuning.rst index 65484a2a50..0a6c377130 100644 --- a/docs/source/advanced/hyperparameters_tuning.rst +++ b/docs/source/advanced/hyperparameters_tuning.rst @@ -378,7 +378,7 @@ Example for ``SimultaneousTuner``: from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder - from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum + from fedot.core.repository.metrics_repository import ClassificationMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task task = Task(TaskTypesEnum.classification) @@ -466,7 +466,7 @@ Example for ``IOptTuner``: from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder - from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum + from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task task = Task(TaskTypesEnum.regression) @@ -517,7 +517,7 @@ Example for ``OptunaTuner``: from fedot.core.data.data import InputData from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder - from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum + from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task task = Task(TaskTypesEnum.regression) @@ -568,7 +568,7 @@ and obtain a list of tuned pipelines representing a pareto front after tuning. from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder - from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum + from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task task = Task(TaskTypesEnum.regression) @@ -603,7 +603,7 @@ Sequential tuning from fedot.core.data.data import InputData from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder - from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum + from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task, TsForecastingParams task = Task(TaskTypesEnum.ts_forecasting, TsForecastingParams(forecast_length=10)) @@ -663,7 +663,7 @@ Tuning of a node from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder - from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum + from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task from test.integration.quality.test_synthetic_tasks import get_regression_data diff --git a/docs/source/api/repository.rst b/docs/source/api/repository.rst index 962d106259..1f2d3b58b4 100644 --- a/docs/source/api/repository.rst +++ b/docs/source/api/repository.rst @@ -16,7 +16,7 @@ Operation Types Quality metrics --------------- -.. automodule:: fedot.core.repository.quality_metrics_repository +.. automodule:: fedot.core.repository.metrics_repository :members: :no-undoc-members: diff --git a/examples/advanced/decompose/classification_refinement_example.py b/examples/advanced/decompose/classification_refinement_example.py index 5dc05ec479..1806571eb3 100644 --- a/examples/advanced/decompose/classification_refinement_example.py +++ b/examples/advanced/decompose/classification_refinement_example.py @@ -5,7 +5,7 @@ from fedot.core.pipelines.node import PipelineNode from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import set_random_seed diff --git a/examples/advanced/decompose/regression_refinement_example.py b/examples/advanced/decompose/regression_refinement_example.py index c7a3c93d81..e371375937 100644 --- a/examples/advanced/decompose/regression_refinement_example.py +++ b/examples/advanced/decompose/regression_refinement_example.py @@ -11,7 +11,7 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import fedot_project_root diff --git a/examples/advanced/multitask_classification_regression.py b/examples/advanced/multitask_classification_regression.py index 9d3dda5719..e4f31ee7c5 100644 --- a/examples/advanced/multitask_classification_regression.py +++ b/examples/advanced/multitask_classification_regression.py @@ -12,7 +12,7 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task from fedot.core.utils import fedot_project_root diff --git a/examples/advanced/pipeline_sensitivity.py b/examples/advanced/pipeline_sensitivity.py index fe6ada2a7c..4ea82fb952 100644 --- a/examples/advanced/pipeline_sensitivity.py +++ b/examples/advanced/pipeline_sensitivity.py @@ -11,7 +11,7 @@ from fedot.core.data.data import InputData from fedot.core.data.data_split import train_test_data_setup from fedot.core.repository.operation_types_repository import get_operations_for_task -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, MetricsRepository, \ +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum, MetricsRepository, \ RegressionMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import default_fedot_data_dir, fedot_project_root @@ -123,7 +123,7 @@ def run_class_scoring_case(is_composed: bool, path_to_save=None): train_data, test_data = get_scoring_data() task = Task(TaskTypesEnum.classification) # the choice of the metric for the pipeline quality assessment during composition - metric_function = MetricsRepository().metric_by_id(ClassificationMetricsEnum.ROCAUC_penalty) + metric_function = MetricsRepository.get_metric(ClassificationMetricsEnum.ROCAUC_penalty) if is_composed: case = 'scoring_composed' @@ -141,7 +141,7 @@ def run_class_kc2_case(is_composed: bool = False, path_to_save=None): train_data, test_data = get_kc2_data() task = Task(TaskTypesEnum.classification) # the choice of the metric for the pipeline quality assessment during composition - metric_function = MetricsRepository().metric_by_id(ClassificationMetricsEnum.ROCAUC_penalty) + metric_function = MetricsRepository.get_metric(ClassificationMetricsEnum.ROCAUC_penalty) if is_composed: case = 'kc2_composed' @@ -159,7 +159,7 @@ def run_regr_case(is_composed: bool = False, path_to_save=None): train_data, test_data = get_cholesterol_data() task = Task(TaskTypesEnum.regression) # the choice of the metric for the pipeline quality assessment during composition - metric_function = MetricsRepository().metric_by_id(RegressionMetricsEnum.RMSE) + metric_function = MetricsRepository.get_metric(RegressionMetricsEnum.RMSE) if is_composed: case = 'cholesterol_composed' diff --git a/examples/advanced/structural_analysis/structural_analysis_example.py b/examples/advanced/structural_analysis/structural_analysis_example.py index 1a58e5c0f7..482f92b538 100644 --- a/examples/advanced/structural_analysis/structural_analysis_example.py +++ b/examples/advanced/structural_analysis/structural_analysis_example.py @@ -1,15 +1,15 @@ import os from copy import deepcopy from functools import partial +from typing import Callable, Dict, List, Optional, Tuple from golem.core.dag.graph_verifier import GraphVerifier -from golem.core.dag.verification_rules import has_one_root, has_no_cycle, has_no_isolated_components, \ - has_no_isolated_nodes, has_no_self_cycled_nodes +from golem.core.dag.verification_rules import (has_no_cycle, has_no_isolated_components, has_no_isolated_nodes, + has_no_self_cycled_nodes, has_one_root) from golem.core.optimisers.graph import OptGraph from golem.core.optimisers.objective import Objective from golem.structural_analysis.graph_sa.graph_structural_analysis import GraphStructuralAnalysis from golem.structural_analysis.graph_sa.sa_requirements import StructuralAnalysisRequirements -from typing import Callable, Dict, Any, Optional, Tuple, List from examples.advanced.structural_analysis.dataset_access import get_scoring_data from examples.advanced.structural_analysis.pipelines_access import get_three_depth_manual_class_pipeline @@ -20,8 +20,9 @@ from fedot.core.pipelines.pipeline_advisor import PipelineChangeAdvisor from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.pipelines.pipeline_node_factory import PipelineOptNodeFactory -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, QualityMetricsEnum, \ - MetricsRepository +from fedot.core.repository.metrics_repository import (ClassificationMetricsEnum, ComplexityMetricCallable, + ComplexityMetricsEnum, MetricsRepository, + QualityMetricCallable, QualityMetricsEnum) from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.utilities.project_import_export import DEFAULT_PATH @@ -31,19 +32,20 @@ class SAObjective(Objective): This objective has to evaluate pipeline in __call__ method and have 'metrics' field to identify which metrics are optimized. """ + def __init__(self, objective: Callable, - quality_metrics: Dict[Any, Callable], - complexity_metrics: Optional[Dict[Any, Callable]] = None, + quality_metrics: Dict[QualityMetricsEnum, QualityMetricCallable], + complexity_metrics: Optional[Dict[ComplexityMetricsEnum, ComplexityMetricCallable]] = None, is_multi_objective: bool = False, ): self.objective = objective super().__init__(quality_metrics=quality_metrics, complexity_metrics=complexity_metrics, is_multi_objective=is_multi_objective) - def __call__(self, graph: OptGraph) -> float: + def __call__(self, graph: OptGraph, **kwargs) -> float: pip = PipelineAdapter().restore(graph) - return self.objective(pip) + return self.objective(pip, **kwargs) def structural_analysis_set_up(train_data: InputData, test_data: InputData, @@ -53,11 +55,12 @@ def structural_analysis_set_up(train_data: InputData, test_data: InputData, -> Tuple[PipelineOptNodeFactory, SAObjective, SAObjective]: """ Build initial infrastructure for performing SA: node factory, objectives. Can be reused for other SA applications, appropriate parameters must be specified then. """ + def _construct_objective(data: InputData, metric: QualityMetricsEnum) -> SAObjective: """ Build objective function with fit and predict functions inside. """ - metric_func = MetricsRepository.metric_by_id(metric) + metric_func = MetricsRepository.get_metric(metric) get_value = partial(metric_func, reference_data=data) - metrics_ = {metric: data} + metrics_ = {metric: metric_func} data_producer = DataSourceSplitter().build(data=data) objective_function = PipelineObjectiveEvaluate(objective=Objective(quality_metrics=get_value), diff --git a/examples/advanced/time_series_forecasting/composing_pipelines.py b/examples/advanced/time_series_forecasting/composing_pipelines.py index f5d25cda85..a2cacd3e20 100644 --- a/examples/advanced/time_series_forecasting/composing_pipelines.py +++ b/examples/advanced/time_series_forecasting/composing_pipelines.py @@ -16,7 +16,7 @@ from fedot.core.data.data_split import train_test_data_setup from fedot.core.pipelines.pipeline import Pipeline from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import \ +from fedot.core.repository.metrics_repository import \ RegressionMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot.core.utils import fedot_project_root diff --git a/examples/advanced/time_series_forecasting/custom_model_tuning.py b/examples/advanced/time_series_forecasting/custom_model_tuning.py index e8b97f19ad..32f0c5d078 100644 --- a/examples/advanced/time_series_forecasting/custom_model_tuning.py +++ b/examples/advanced/time_series_forecasting/custom_model_tuning.py @@ -12,7 +12,7 @@ from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task, TsForecastingParams diff --git a/examples/advanced/time_series_forecasting/multi_ts_arctic_forecasting.py b/examples/advanced/time_series_forecasting/multi_ts_arctic_forecasting.py index 36aa166bec..9c4f309cab 100644 --- a/examples/advanced/time_series_forecasting/multi_ts_arctic_forecasting.py +++ b/examples/advanced/time_series_forecasting/multi_ts_arctic_forecasting.py @@ -15,7 +15,7 @@ from fedot.core.composer.composer_builder import ComposerBuilder from fedot.core.composer.gp_composer.specific_operators import parameter_change_mutation from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import \ +from fedot.core.repository.metrics_repository import \ RegressionMetricsEnum diff --git a/examples/advanced/time_series_forecasting/sparse_lagged_tuning.py b/examples/advanced/time_series_forecasting/sparse_lagged_tuning.py index d3d0d77690..ddbf36e46e 100644 --- a/examples/advanced/time_series_forecasting/sparse_lagged_tuning.py +++ b/examples/advanced/time_series_forecasting/sparse_lagged_tuning.py @@ -12,7 +12,7 @@ from fedot.core.data.data import InputData from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot.core.utils import fedot_project_root diff --git a/examples/simple/classification/classification_with_tuning.py b/examples/simple/classification/classification_with_tuning.py index 1eef76d04c..c1b024c91f 100644 --- a/examples/simple/classification/classification_with_tuning.py +++ b/examples/simple/classification/classification_with_tuning.py @@ -7,7 +7,7 @@ from fedot.core.data.data import InputData from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import set_random_seed from fedot.utilities.synth_dataset_generator import classification_dataset diff --git a/examples/simple/classification/multiclass_prediction.py b/examples/simple/classification/multiclass_prediction.py index cb0a69457c..eca3ca3897 100644 --- a/examples/simple/classification/multiclass_prediction.py +++ b/examples/simple/classification/multiclass_prediction.py @@ -16,7 +16,7 @@ from fedot.core.data.data import InputData from fedot.core.pipelines.pipeline import Pipeline from fedot.core.repository.operation_types_repository import OperationTypesRepository -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import ( ensure_directory_exists, diff --git a/examples/simple/classification/resample_example.py b/examples/simple/classification/resample_example.py index 27259974c3..fdf505526b 100644 --- a/examples/simple/classification/resample_example.py +++ b/examples/simple/classification/resample_example.py @@ -12,7 +12,7 @@ from fedot.core.data.data import InputData from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task from fedot.core.utils import fedot_project_root diff --git a/examples/simple/pipeline_tune.py b/examples/simple/pipeline_tune.py index f6902ee9e8..66ce2608d8 100644 --- a/examples/simple/pipeline_tune.py +++ b/examples/simple/pipeline_tune.py @@ -9,7 +9,7 @@ from fedot.core.data.data import InputData from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum def get_case_train_test_data(): diff --git a/examples/simple/pipeline_tuning_with_iopt.py b/examples/simple/pipeline_tuning_with_iopt.py index 1665514c81..dee9153183 100644 --- a/examples/simple/pipeline_tuning_with_iopt.py +++ b/examples/simple/pipeline_tuning_with_iopt.py @@ -7,7 +7,7 @@ from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum, Task from fedot.core.utils import fedot_project_root diff --git a/examples/simple/regression/regression_with_tuning.py b/examples/simple/regression/regression_with_tuning.py index 665a6725bd..88acfbb903 100644 --- a/examples/simple/regression/regression_with_tuning.py +++ b/examples/simple/regression/regression_with_tuning.py @@ -9,7 +9,7 @@ from fedot.core.data.data import InputData from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import set_random_seed from fedot.utilities.synth_dataset_generator import regression_dataset diff --git a/examples/simple/time_series_forecasting/tuning_pipelines.py b/examples/simple/time_series_forecasting/tuning_pipelines.py index 18d12ec7a9..4127dc6b4f 100644 --- a/examples/simple/time_series_forecasting/tuning_pipelines.py +++ b/examples/simple/time_series_forecasting/tuning_pipelines.py @@ -10,7 +10,7 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot.core.utils import fedot_project_root diff --git a/fedot/api/api_utils/api_composer.py b/fedot/api/api_utils/api_composer.py index 7bb1f2dabf..5a0545a85b 100644 --- a/fedot/api/api_utils/api_composer.py +++ b/fedot/api/api_utils/api_composer.py @@ -17,12 +17,12 @@ from fedot.core.data.data import InputData from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import MetricType +from fedot.core.repository.metrics_repository import MetricIDType class ApiComposer: - def __init__(self, api_params: ApiParams, metrics: Union[str, MetricType, Sequence]): + def __init__(self, api_params: ApiParams, metrics: Union[MetricIDType, Sequence[MetricIDType]]): self.log = default_log(self) self.params = api_params self.metrics = metrics diff --git a/fedot/api/builder.py b/fedot/api/builder.py index 449b076833..52b8f85ca8 100644 --- a/fedot/api/builder.py +++ b/fedot/api/builder.py @@ -1,12 +1,12 @@ from __future__ import annotations -from typing import Any, Callable, Dict, List, Optional, Type, Union +from typing import Any, Dict, List, Optional, Sequence, Type, Union from golem.core.optimisers.optimizer import GraphOptimizer from fedot.api.main import Fedot from fedot.core.pipelines.pipeline import Pipeline -from fedot.core.repository.quality_metrics_repository import MetricsEnum +from fedot.core.repository.metrics_repository import MetricIDType from fedot.core.repository.tasks import TaskParams @@ -274,7 +274,7 @@ def setup_pipeline_structure( def setup_pipeline_evaluation( self, - metric: Union[str, Callable, MetricsEnum, List[Union[str, Callable, MetricsEnum]]] = DEFAULT_VALUE, + metric: Union[MetricIDType, Sequence[MetricIDType]] = DEFAULT_VALUE, cv_folds: int = DEFAULT_VALUE, max_pipeline_fit_time: Optional[int] = DEFAULT_VALUE, collect_intermediate_metric: bool = DEFAULT_VALUE, @@ -287,19 +287,19 @@ def setup_pipeline_evaluation( .. details:: Default value depends on a given task: - - ``roc_auc`` -> for classification + - ``roc_auc_pen`` -> for classification - ``rmse`` -> for regression & time series forecasting .. details:: Available metrics are listed in the following enumerations: - classification -> \ - :class:`~fedot.core.repository.quality_metrics_repository.ClassificationMetricsEnum` + :class:`~fedot.core.repository.metrics_repository.ClassificationMetricsEnum` - regression -> \ - :class:`~fedot.core.repository.quality_metrics_repository.RegressionMetricsEnum` + :class:`~fedot.core.repository.metrics_repository.RegressionMetricsEnum` - time series forcasting -> \ - :class:`~fedot.core.repository.quality_metrics_repository.TimeSeriesForecastingMetricsEnum` + :class:`~fedot.core.repository.metrics_repository.TimeSeriesForecastingMetricsEnum` - pipeline complexity (task-independent) -> \ - :class:`~fedot.core.repository.quality_metrics_repository.ComplexityMetricsEnum` + :class:`~fedot.core.repository.metrics_repository.ComplexityMetricsEnum` cv_folds: number of folds for cross-validation. diff --git a/fedot/api/main.py b/fedot/api/main.py index f1e6718adc..2c19b86f5f 100644 --- a/fedot/api/main.py +++ b/fedot/api/main.py @@ -1,6 +1,6 @@ import logging from copy import deepcopy -from typing import Any, Callable, List, Optional, Sequence, Tuple, Union +from typing import Any, List, Optional, Sequence, Tuple, Union import numpy as np import pandas as pd @@ -26,6 +26,7 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.ts_wrappers import convert_forecast_to_output, out_of_sample_ts_forecast from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder +from fedot.core.repository.metrics_repository import MetricCallable from fedot.core.repository.tasks import TaskParams, TaskTypesEnum from fedot.core.utils import set_random_seed from fedot.explainability.explainer_template import Explainer @@ -187,7 +188,7 @@ def fit(self, def tune(self, input_data: Optional[InputData] = None, - metric_name: Optional[Union[str, Callable]] = None, + metric_name: Optional[Union[str, MetricCallable]] = None, iterations: int = DEFAULT_TUNING_ITERATIONS_NUMBER, timeout: Optional[float] = None, cv_folds: Optional[int] = None, diff --git a/fedot/core/composer/composer_builder.py b/fedot/core/composer/composer_builder.py index a3eae7e1cf..73217e2575 100644 --- a/fedot/core/composer/composer_builder.py +++ b/fedot/core/composer/composer_builder.py @@ -6,8 +6,8 @@ from golem.core.log import LoggerAdapter, default_log from golem.core.optimisers.genetic.gp_optimizer import EvoGraphOptimizer from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters -from golem.core.optimisers.initial_graphs_generator import InitialPopulationGenerator, GenerationFunction -from golem.core.optimisers.optimizer import GraphOptimizer, AlgorithmParameters, GraphGenerationParams +from golem.core.optimisers.initial_graphs_generator import GenerationFunction, InitialPopulationGenerator +from golem.core.optimisers.optimizer import AlgorithmParameters, GraphGenerationParams, GraphOptimizer from golem.utilities.data_structures import ensure_wrapped_in_sequence from fedot.core.caching.pipelines_cache import OperationsCache @@ -20,11 +20,7 @@ from fedot.core.pipelines.pipeline_graph_generation_params import get_pipeline_generation_params from fedot.core.pipelines.verification import rules_by_task from fedot.core.repository.operation_types_repository import get_operations_for_task -from fedot.core.repository.quality_metrics_repository import ( - ComplexityMetricsEnum, - MetricsEnum, - MetricType -) +from fedot.core.repository.metrics_repository import ComplexityMetricsEnum, MetricIDType, MetricsEnum from fedot.core.repository.tasks import Task from fedot.remote.remote_evaluator import RemoteEvaluator from fedot.utilities.define_metric_by_task import MetricByTask @@ -88,7 +84,7 @@ def with_graph_generation_param(self, graph_generation_params: GraphGenerationPa self.graph_generation_params = graph_generation_params return self - def with_metrics(self, metrics: Union[MetricType, List[MetricType]]): + def with_metrics(self, metrics: Union[MetricIDType, Sequence[MetricIDType]]): self.metrics = ensure_wrapped_in_sequence(metrics) return self diff --git a/fedot/core/composer/metrics.py b/fedot/core/composer/metrics.py index f0eb97f886..20c6fdd395 100644 --- a/fedot/core/composer/metrics.py +++ b/fedot/core/composer/metrics.py @@ -1,7 +1,9 @@ import os.path import sys from abc import abstractmethod +from functools import wraps from pathlib import Path +from typing import Optional, Tuple from uuid import uuid4 import numpy as np @@ -14,13 +16,13 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.ts_wrappers import in_sample_ts_forecast from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.tasks import TaskTypesEnum from fedot.core.utils import default_fedot_data_dir from fedot.utilities.custom_errors import AbstractMethodNotImplementError from fedot.utilities.debug import is_analytic_mode def from_maximised_metric(metric_func): + @wraps(metric_func) def wrapper(*args, **kwargs): return -metric_func(*args, **kwargs) @@ -38,23 +40,14 @@ def smape(y_true: np.ndarray, y_pred: np.ndarray) -> float: class Metric: - output_mode = 'default' - default_value = 0 - @classmethod @abstractmethod - def get_value(cls, pipeline: 'Pipeline', reference_data: InputData, - validation_blocks: int) -> float: - """ Get metrics values based on pipeline and InputData for validation """ + def get_value(cls, **kwargs) -> float: + """ Get metrics value based on pipeline and other optional arguments. """ raise AbstractMethodNotImplementError - @staticmethod - @abstractmethod - def metric(reference: InputData, predicted: OutputData) -> float: - raise AbstractMethodNotImplementError - -class QualityMetric: +class QualityMetric(Metric): max_penalty_part = 0.01 output_mode = 'default' default_value = 0 @@ -65,13 +58,20 @@ def metric(reference: InputData, predicted: OutputData) -> float: raise AbstractMethodNotImplementError @classmethod - def get_value(cls, pipeline: 'Pipeline', reference_data: InputData, - validation_blocks: int = None) -> float: + def get_value(cls, pipeline: Pipeline, reference_data: InputData, + validation_blocks: Optional[int] = None) -> float: + """ Get metric value based on pipeline, reference data, and number of validation blocks. + Args: + pipeline: a :class:`Pipeline` instance for evaluation. + reference_data: :class:`InputData` for evaluation. + validation_blocks: number of validation blocks. Used only for time series forecasting. + If ``None``, data separation is not performed. + """ metric = cls.default_value try: if validation_blocks is None: # Time series or regression classical hold-out validation - results, reference_data = cls._simple_prediction(pipeline, reference_data) + reference_data, results = cls._simple_prediction(pipeline, reference_data) else: # Perform time series in-sample validation reference_data, results = cls._in_sample_prediction(pipeline, reference_data, validation_blocks) @@ -95,43 +95,14 @@ def get_value(cls, pipeline: 'Pipeline', reference_data: InputData, return metric @classmethod - def _simple_prediction(cls, pipeline: 'Pipeline', reference_data: InputData): - """ Method prepares data for metric evaluation and perform simple validation """ - results = pipeline.predict(reference_data, output_mode=cls.output_mode) - - # Define conditions for target and predictions transforming - is_regression = reference_data.task.task_type == TaskTypesEnum.regression - is_multi_target = len(np.array(results.predict).shape) > 1 - is_multi_target_regression = is_regression and is_multi_target - - # Time series forecasting - is_ts_forecasting = reference_data.task.task_type == TaskTypesEnum.ts_forecasting - if is_ts_forecasting or is_multi_target_regression: - results, reference_data = cls.flatten_convert(results, reference_data) - - return results, reference_data - - @staticmethod - def flatten_convert(results, reference_data): - """ Transform target and predictions by converting them into - one-dimensional array - - :param results: output from pipeline - :param reference_data: actual data for validation - """ - # Predictions convert into uni-variate array - forecast_values = np.ravel(np.array(results.predict)) - results.predict = forecast_values - # Target convert into uni-variate array - target_values = np.ravel(np.array(reference_data.target)) - reference_data.target = target_values - - return results, reference_data + def _simple_prediction(cls, pipeline: Pipeline, reference_data: InputData) -> Tuple[InputData, OutputData]: + """ Method calls pipeline.predict() and returns the result. """ + return reference_data, pipeline.predict(reference_data, output_mode=cls.output_mode) @classmethod - def get_value_with_penalty(cls, pipeline: 'Pipeline', reference_data: InputData, - validation_blocks: int = None) -> float: - quality_metric = cls.get_value(pipeline, reference_data) + def get_value_with_penalty(cls, pipeline: Pipeline, reference_data: InputData, + validation_blocks: Optional[int] = None) -> float: + quality_metric = cls.get_value(pipeline, reference_data, validation_blocks) structural_metric = StructuralComplexity.get_value(pipeline) penalty = abs(structural_metric * quality_metric * cls.max_penalty_part) @@ -140,7 +111,8 @@ def get_value_with_penalty(cls, pipeline: 'Pipeline', reference_data: InputData, return metric_with_penalty @staticmethod - def _in_sample_prediction(pipeline: 'Pipeline', data: InputData, validation_blocks: int): + def _in_sample_prediction(pipeline: Pipeline, data: InputData, validation_blocks: int + ) -> Tuple[InputData, OutputData]: """ Performs in-sample pipeline validation for time series prediction """ horizon = int(validation_blocks * data.task.task_params.forecast_length) @@ -160,6 +132,14 @@ def _in_sample_prediction(pipeline: 'Pipeline', data: InputData, validation_bloc return reference_data, results + @staticmethod + def _get_least_frequent_val(array: np.ndarray): + """ Returns the least frequent value in a flattened numpy array. """ + unique_vals, count = np.unique(np.ravel(array), return_counts=True) + least_frequent_idx = np.argmin(count) + least_frequent_val = unique_vals[least_frequent_idx] + return least_frequent_val + class RMSE(QualityMetric): default_value = sys.maxsize @@ -214,16 +194,12 @@ class F1(QualityMetric): @staticmethod @from_maximised_metric def metric(reference: InputData, predicted: OutputData) -> float: - n_classes = reference.num_classes - if n_classes > 2: - additional_params = {'average': F1.multiclass_averaging_mode} + if reference.num_classes == 2: + pos_label = QualityMetric._get_least_frequent_val(reference.target) + additional_params = dict(average=F1.binary_averaging_mode, pos_label=pos_label) else: - u, count = np.unique(np.ravel(reference.target), return_counts=True) - count_sort_ind = np.argsort(count) - pos_label = u[count_sort_ind[0]].item() - additional_params = {'average': F1.binary_averaging_mode, 'pos_label': pos_label} - return f1_score(y_true=reference.target, y_pred=predicted.predict, - **additional_params) + additional_params = dict(average=F1.multiclass_averaging_mode) + return f1_score(y_true=reference.target, y_pred=predicted.predict, **additional_params) class MAE(QualityMetric): @@ -259,15 +235,13 @@ class ROCAUC(QualityMetric): def metric(reference: InputData, predicted: OutputData) -> float: n_classes = reference.num_classes if n_classes > 2: - additional_params = {'multi_class': 'ovr', 'average': 'macro'} + additional_params = dict(multi_class='ovr', average='macro') else: - additional_params = {} + additional_params = dict() - score = round(roc_auc_score(y_score=predicted.predict, - y_true=reference.target, - **additional_params), 3) - - return score + return roc_auc_score(y_score=predicted.predict, + y_true=reference.target, + **additional_params) @staticmethod def roc_curve(target: np.ndarray, predict: np.ndarray, pos_label=None): @@ -281,20 +255,19 @@ def auc(cls, fpr, tpr): class Precision(QualityMetric): output_mode = 'labels' + binary_averaging_mode = 'binary' + multiclass_averaging_mode = 'macro' @staticmethod @from_maximised_metric def metric(reference: InputData, predicted: OutputData) -> float: n_classes = reference.num_classes if n_classes > 2: - return precision_score(y_true=reference.target, y_pred=predicted.predict) + additional_params = dict(average=Precision.multiclass_averaging_mode) else: - u, count = np.unique(np.ravel(reference.target), return_counts=True) - count_sort_ind = np.argsort(count) - pos_label = u[count_sort_ind[0]].item() - additional_params = {'pos_label': pos_label} - return precision_score(y_true=reference.target, y_pred=predicted.predict, - **additional_params) + pos_label = QualityMetric._get_least_frequent_val(reference.target) + additional_params = dict(pos_label=pos_label, average=Precision.binary_averaging_mode) + return precision_score(y_true=reference.target, y_pred=predicted.predict, **additional_params) class Logloss(QualityMetric): @@ -324,21 +297,41 @@ def metric(reference: InputData, predicted: OutputData) -> float: return silhouette_score(reference.features, labels=predicted.predict) -class StructuralComplexity(Metric): +class ComplexityMetric(Metric): + default_value = 0 + norm_constant = 1 + + @classmethod + def get_value(cls, pipeline: Pipeline, **kwargs) -> float: + """ Get metric value and apply norm_constant to it. """ + return cls.metric(pipeline, **kwargs) / cls.norm_constant + + @classmethod + @abstractmethod + def metric(cls, pipeline: Pipeline, **kwargs) -> float: + """ Get metrics value based on pipeline. """ + raise AbstractMethodNotImplementError + + +class StructuralComplexity(ComplexityMetric): + norm_constant = 30 + @classmethod - def get_value(cls, pipeline: 'Pipeline', **args) -> float: - norm_constant = 30 - return (pipeline.depth ** 2 + pipeline.length) / norm_constant + def metric(cls, pipeline: Pipeline, **kwargs) -> float: + return pipeline.depth ** 2 + pipeline.length + +class NodeNum(ComplexityMetric): + norm_constant = 10 -class NodeNum(Metric): @classmethod - def get_value(cls, pipeline: 'Pipeline', **args) -> float: - norm_constant = 10 - return pipeline.length / norm_constant + def metric(cls, pipeline: Pipeline, **kwargs) -> float: + return pipeline.length -class ComputationTime(Metric): +class ComputationTime(ComplexityMetric): + default_value = sys.maxsize + @classmethod - def get_value(cls, pipeline: 'Pipeline', **args) -> float: + def metric(cls, pipeline: Pipeline, **kwargs) -> float: return pipeline.computation_time diff --git a/fedot/core/operations/atomized_model.py b/fedot/core/operations/atomized_model.py index 5fba8f6d44..3fce78bc22 100644 --- a/fedot/core/operations/atomized_model.py +++ b/fedot/core/operations/atomized_model.py @@ -2,18 +2,18 @@ from datetime import timedelta from functools import reduce from operator import and_, or_ -from typing import Callable, Union, Optional, Set, List, Any, Dict +from typing import Any, Callable, Dict, List, Optional, Set, Union -from fedot.core.pipelines.node import PipelineNode from golem.core.tuning.simultaneous import SimultaneousTuner from fedot.core.data.data import InputData, OutputData from fedot.core.operations.operation import Operation from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.pipelines.node import PipelineNode from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.operation_types_repository import OperationMetaInfo, \ - atomized_model_type +from fedot.core.repository.metrics_repository import MetricCallable +from fedot.core.repository.operation_types_repository import OperationMetaInfo, atomized_model_type class AtomizedModel(Operation): @@ -50,16 +50,16 @@ def predict_for_fit(self, return self.predict(fitted_operation, data, params, output_mode) def fine_tune(self, - metric_function: Callable, + metric_function: MetricCallable, input_data: Optional[InputData] = None, iterations: int = 50, timeout: int = 5) -> 'AtomizedModel': """ Method for tuning hyperparameters """ - tuner = TunerBuilder(input_data.task)\ - .with_tuner(SimultaneousTuner)\ - .with_metric(metric_function)\ - .with_iterations(iterations)\ - .with_timeout(timedelta(minutes=timeout))\ + tuner = TunerBuilder(input_data.task) \ + .with_tuner(SimultaneousTuner) \ + .with_metric(metric_function) \ + .with_iterations(iterations) \ + .with_timeout(timedelta(minutes=timeout)) \ .build(input_data) tuned_pipeline = tuner.tune(self.pipeline) tuned_atomized_model = AtomizedModel(tuned_pipeline) diff --git a/fedot/core/optimisers/objective/metrics_objective.py b/fedot/core/optimisers/objective/metrics_objective.py index c6b54448b0..d7ae719192 100644 --- a/fedot/core/optimisers/objective/metrics_objective.py +++ b/fedot/core/optimisers/objective/metrics_objective.py @@ -3,12 +3,12 @@ from golem.core.optimisers.objective import Objective from golem.utilities.data_structures import ensure_wrapped_in_sequence -from fedot.core.repository.quality_metrics_repository import MetricType, MetricsRepository, ComplexityMetricsEnum +from fedot.core.repository.metrics_repository import MetricIDType, MetricsRepository, ComplexityMetricsEnum class MetricsObjective(Objective): def __init__(self, - metrics: Union[MetricType, Iterable[MetricType]], + metrics: Union[MetricIDType, Iterable[MetricIDType]], is_multi_objective: bool = False): quality_metrics = {} complexity_metrics = {} @@ -18,7 +18,7 @@ def __init__(self, metric_id = str(metric) quality_metrics[metric_id] = metric else: - metric_func = MetricsRepository.metric_by_id(metric) + metric_func = MetricsRepository.get_metric(metric) if metric_func: if ComplexityMetricsEnum.has_value(metric): complexity_metrics[metric] = metric_func diff --git a/fedot/core/pipelines/pipeline.py b/fedot/core/pipelines/pipeline.py index c5a727108c..526fb263ba 100644 --- a/fedot/core/pipelines/pipeline.py +++ b/fedot/core/pipelines/pipeline.py @@ -274,6 +274,9 @@ def predict(self, input_data: Union[InputData, MultiModalData], output_mode: str copied_input_data = self._assign_data_to_nodes(copied_input_data) result = self.root_node.predict(input_data=copied_input_data, output_mode=output_mode) + if input_data.task.task_type == TaskTypesEnum.ts_forecasting: + result.predict = result.predict.ravel() + result = self._postprocess(copied_input_data, result, output_mode) return result diff --git a/fedot/core/pipelines/tuning/tuner_builder.py b/fedot/core/pipelines/tuning/tuner_builder.py index c061768436..379028a959 100644 --- a/fedot/core/pipelines/tuning/tuner_builder.py +++ b/fedot/core/pipelines/tuning/tuner_builder.py @@ -1,5 +1,5 @@ from datetime import timedelta -from typing import Type, Union, Iterable, Sequence +from typing import Iterable, Sequence, Type, Union from golem.core.tuning.optuna_tuner import OptunaTuner from golem.core.tuning.simultaneous import SimultaneousTuner @@ -14,7 +14,7 @@ from fedot.core.pipelines.adapters import PipelineAdapter from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace -from fedot.core.repository.quality_metrics_repository import MetricType, MetricsEnum +from fedot.core.repository.metrics_repository import MetricIDType, MetricsEnum from fedot.core.repository.tasks import Task from fedot.utilities.define_metric_by_task import MetricByTask @@ -55,7 +55,7 @@ def with_n_jobs(self, n_jobs: int): self.n_jobs = n_jobs return self - def with_metric(self, metrics: Union[MetricType, Iterable[MetricType]]): + def with_metric(self, metrics: Union[MetricIDType, Iterable[MetricIDType]]): self.metric = ensure_wrapped_in_sequence(metrics) return self diff --git a/fedot/core/repository/quality_metrics_repository.py b/fedot/core/repository/metrics_repository.py similarity index 60% rename from fedot/core/repository/quality_metrics_repository.py rename to fedot/core/repository/metrics_repository.py index 537d87bd20..7c02d4b934 100644 --- a/fedot/core/repository/quality_metrics_repository.py +++ b/fedot/core/repository/metrics_repository.py @@ -1,15 +1,15 @@ -from numbers import Real -from typing import Callable, Union, TypeVar +from typing import Dict, Optional, Protocol, TypeVar, Union -from golem.core.dag.graph import Graph -from golem.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum -from fedot.core.composer.metrics import (ComputationTime, Accuracy, F1, Logloss, MAE, - MAPE, SMAPE, MSE, MSLE, Metric, NodeNum, Precision, R2, - RMSE, ROCAUC, Silhouette, StructuralComplexity, MASE) +from fedot.core.composer.metrics import (Accuracy, ComplexityMetric, ComputationTime, F1, Logloss, MAE, MAPE, MASE, MSE, + MSLE, NodeNum, Precision, QualityMetric, R2, RMSE, ROCAUC, SMAPE, Silhouette, + StructuralComplexity) +from fedot.core.data.data import InputData +from fedot.core.pipelines.pipeline import Pipeline -class MetricsEnum(Enum): +class MetricsEnum(ComparableEnum): def __str__(self): return self.value @@ -18,13 +18,7 @@ def has_value(cls, value): return value in cls._value2member_map_ -G = TypeVar('G', bound=Graph, covariant=True) -MetricCallable = Callable[[G], Real] -MetricType = Union[MetricCallable, MetricsEnum] - - -class QualityMetricsEnum(MetricsEnum): - pass +class QualityMetricsEnum(MetricsEnum): pass class ComplexityMetricsEnum(MetricsEnum): @@ -69,8 +63,25 @@ class TimeSeriesForecastingMetricsEnum(QualityMetricsEnum): RMSE_penalty = 'rmse_pen' +NumberType = Union[int, float, complex] +PipelineType = TypeVar('PipelineType', bound=Pipeline, covariant=True) + + +class QualityMetricCallable(Protocol): + def __call__(self, pipeline: PipelineType, reference_data: InputData, + validation_blocks: Optional[int] = None) -> NumberType: pass + + +class ComplexityMetricCallable(Protocol): + def __call__(self, pipeline: PipelineType, **kwargs) -> NumberType: pass + + +MetricCallable = Union[QualityMetricCallable, ComplexityMetricCallable] +MetricIDType = Union[str, MetricsEnum, MetricCallable] + + class MetricsRepository: - _metrics_implementations = { + _metrics_implementations: Dict[MetricsEnum, MetricCallable] = { # classification ClassificationMetricsEnum.ROCAUC: ROCAUC.get_value, ClassificationMetricsEnum.ROCAUC_penalty: ROCAUC.get_value_with_penalty, @@ -101,10 +112,13 @@ class MetricsRepository: ComplexityMetricsEnum.computation_time: ComputationTime.get_value } + _metrics_classes = {metric_id: getattr(metric_func, '__self__') + for metric_id, metric_func in _metrics_implementations.items()} + @staticmethod - def metric_by_id(metric_id: MetricsEnum, default_callable: MetricCallable = None) -> MetricCallable: - return MetricsRepository._metrics_implementations.get(metric_id, default_callable) + def get_metric(metric_name: MetricsEnum) -> MetricCallable: + return MetricsRepository._metrics_implementations[metric_name] @staticmethod - def metric_class_by_id(metric_id: MetricsEnum) -> Metric: - return MetricsRepository._metrics_implementations[metric_id].__self__() + def get_metric_class(metric_name: MetricsEnum) -> Union[QualityMetric, ComplexityMetric]: + return MetricsRepository._metrics_classes[metric_name] diff --git a/fedot/utilities/define_metric_by_task.py b/fedot/utilities/define_metric_by_task.py index c168ad1605..b11ca03612 100644 --- a/fedot/utilities/define_metric_by_task.py +++ b/fedot/utilities/define_metric_by_task.py @@ -1,7 +1,7 @@ from typing import List from fedot.core.data.data import InputData, OutputData -from fedot.core.repository.quality_metrics_repository import ( +from fedot.core.repository.metrics_repository import ( MetricsEnum, RegressionMetricsEnum, ClassificationMetricsEnum, @@ -27,7 +27,7 @@ def compute_default_metric(task_type: TaskTypesEnum, true: InputData, predicted: round_up_to: int = 6) -> float: """Returns the value of metric defined by task""" metric_id = MetricByTask.get_default_quality_metrics(task_type)[0] - metric = MetricsRepository.metric_class_by_id(metric_id) + metric = MetricsRepository.get_metric_class(metric_id) try: return round(metric.metric(reference=true, predicted=predicted), round_up_to) except ValueError: diff --git a/test/data/expected_metric_values.json b/test/data/expected_metric_values.json new file mode 100644 index 0000000000..8a293325c6 --- /dev/null +++ b/test/data/expected_metric_values.json @@ -0,0 +1,65 @@ +{ + "complexity": { + "node_number": 0.4, + "structural": 0.43333333333333335, + "computation_time_in_seconds": 0.0 + }, + "binary": { + "roc_auc": -0.9799498746867168, + "precision": -0.9473684210526315, + "f1": -0.9473684210526315, + "neg_log_loss": 0.19864695688257933, + "roc_auc_pen": -0.9757034252297411, + "accuracy": -0.95 + }, + "multiclass": { + "roc_auc": -0.9832500832500832, + "precision": -0.9777777777777779, + "f1": -0.9719701552732407, + "neg_log_loss": 0.17094588819131074, + "roc_auc_pen": -0.9789893328893329, + "accuracy": -0.9722222222222222 + }, + "regression": { + "rmse": 52.5400204534369, + "mse": 2760.4537492475683, + "neg_mean_squared_log_error": 0.11968905129112402, + "mape": 0.32791432529967834, + "smape": 49.48675123994897, + "mae": 41.55089094096007, + "r2": 0.389822739375963, + "rmse_pen": 52.64510049434378 + }, + "multitarget": { + "rmse": 15.753366859480218, + "mse": 377.5025166058113, + "neg_mean_squared_log_error": 0.030627538521796293, + "mape": 0.15337090733886807, + "smape": 14.144394353302935, + "mae": 13.50645038033778, + "r2": -2.9713973901034954, + "rmse_pen": 15.784873593199178 + }, + "ts": { + "mase": 0.6080909603204148, + "rmse": 6.977694399080989, + "mse": 48.6882191269662, + "neg_mean_squared_log_error": 0.005433112046490622, + "mape": 0.05897445640267307, + "smape": 5.602921894471363, + "mae": 4.960742044719172, + "r2": 0.8502177471021775, + "rmse_pen": 6.991649787879151 + }, + "multits": { + "mase": 0.6080543658437002, + "rmse": 6.977578875807477, + "mse": 48.68660696811473, + "neg_mean_squared_log_error": 0.005434854312738277, + "mape": 0.05898285612710732, + "smape": 5.603672314933528, + "mae": 4.960443510830185, + "r2": 0.8502227066753377, + "rmse_pen": 6.991534033559091 + } +} \ No newline at end of file diff --git a/test/integration/composer/test_composer.py b/test/integration/composer/test_composer.py index a30d2a933b..2cb174c4bf 100644 --- a/test/integration/composer/test_composer.py +++ b/test/integration/composer/test_composer.py @@ -24,7 +24,7 @@ from fedot.core.pipelines.pipeline_graph_generation_params import get_pipeline_generation_params from fedot.core.repository.dataset_types import DataTypesEnum from fedot.core.repository.operation_types_repository import OperationTypesRepository, get_operations_for_task -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, ComplexityMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum, ComplexityMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import fedot_project_root, set_random_seed from test.unit.pipelines.test_pipeline_comparison import pipeline_first, pipeline_second diff --git a/test/integration/composer/test_history.py b/test/integration/composer/test_history.py index 70807c14ad..c266ba7539 100644 --- a/test/integration/composer/test_history.py +++ b/test/integration/composer/test_history.py @@ -3,8 +3,6 @@ import numpy as np import pytest - -from fedot.core.repository.tasks import TaskTypesEnum from golem.core.dag.graph import Graph from golem.core.optimisers.fitness import SingleObjFitness from golem.core.optimisers.genetic.evaluation import MultiprocessingDispatcher @@ -20,8 +18,9 @@ from fedot.core.pipelines.node import PipelineNode from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.pipeline_graph_generation_params import get_pipeline_generation_params -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, \ - RegressionMetricsEnum, MetricType +from fedot.core.repository.metrics_repository import (ClassificationMetricsEnum, MetricIDType, + RegressionMetricsEnum) +from fedot.core.repository.tasks import TaskTypesEnum from fedot.core.utils import fedot_project_root from test.unit.tasks.test_forecasting import get_ts_data from test.unit.validation.test_table_cv import get_classification_data @@ -116,7 +115,7 @@ def assert_intermediate_metrics(pipeline: Graph): get_ts_data()[0], RegressionMetricsEnum.RMSE), ]) -def test_collect_intermediate_metric(pipeline: Pipeline, input_data: InputData, metric: MetricType): +def test_collect_intermediate_metric(pipeline: Pipeline, input_data: InputData, metric: MetricIDType): graph_gen_params = get_pipeline_generation_params() metrics = [metric] diff --git a/test/integration/optimizer/test_evaluation.py b/test/integration/optimizer/test_evaluation.py index 52a5a616c8..1629736e9d 100644 --- a/test/integration/optimizer/test_evaluation.py +++ b/test/integration/optimizer/test_evaluation.py @@ -10,7 +10,7 @@ from fedot.core.optimisers.objective.metrics_objective import MetricsObjective from fedot.core.pipelines.adapters import PipelineAdapter from fedot.core.pipelines.pipeline import Pipeline -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum from test.unit.pipelines.test_node_cache import pipeline_first, pipeline_second, pipeline_third, pipeline_fourth from test.unit.validation.test_table_cv import get_classification_data diff --git a/test/integration/pipelines/tuning/test_pipeline_tuning.py b/test/integration/pipelines/tuning/test_pipeline_tuning.py index be281a92c0..48510f0339 100644 --- a/test/integration/pipelines/tuning/test_pipeline_tuning.py +++ b/test/integration/pipelines/tuning/test_pipeline_tuning.py @@ -23,7 +23,7 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum, ClassificationMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum, ClassificationMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import fedot_project_root, NESTED_PARAMS_LABEL from test.unit.multimodal.data_generators import get_single_task_multimodal_tabular_data, get_multimodal_pipeline diff --git a/test/integration/pipelines/tuning/test_tuner_builder.py b/test/integration/pipelines/tuning/test_tuner_builder.py index 18cb50a3dd..c2bb4ebc7c 100644 --- a/test/integration/pipelines/tuning/test_tuner_builder.py +++ b/test/integration/pipelines/tuning/test_tuner_builder.py @@ -16,13 +16,13 @@ from fedot.core.optimisers.objective.metrics_objective import MetricsObjective from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, MetricType +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum, MetricIDType from test.integration.pipelines.tuning.test_pipeline_tuning import get_not_default_search_space from test.unit.optimizer.test_pipeline_objective_eval import pipeline_first_test from test.unit.validation.test_table_cv import get_classification_data -def get_objective_evaluate(metric: MetricType, data: InputData, +def get_objective_evaluate(metric: MetricIDType, data: InputData, cv_folds: Optional[int] = None) \ -> PipelineObjectiveEvaluate: objective = MetricsObjective(metric) diff --git a/test/integration/validation/test_table_cv.py b/test/integration/validation/test_table_cv.py index b0a0cd09bb..a7ae120ebb 100644 --- a/test/integration/validation/test_table_cv.py +++ b/test/integration/validation/test_table_cv.py @@ -7,7 +7,7 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.repository.operation_types_repository import OperationTypesRepository -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from test.unit.validation.test_table_cv import get_classification_data diff --git a/test/unit/api/test_api_params.py b/test/unit/api/test_api_params.py index 7295ababa9..43479aeb9d 100644 --- a/test/unit/api/test_api_params.py +++ b/test/unit/api/test_api_params.py @@ -11,7 +11,7 @@ from fedot.core.constants import AUTO_PRESET_NAME from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements -from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum +from fedot.core.repository.metrics_repository import RegressionMetricsEnum from fedot.core.repository.tasks import TaskTypesEnum fedot_params_full = dict(parallelization_mode='populational', diff --git a/test/unit/composer/test_metrics.py b/test/unit/composer/test_metrics.py new file mode 100644 index 0000000000..b8b868a9e7 --- /dev/null +++ b/test/unit/composer/test_metrics.py @@ -0,0 +1,227 @@ +import json +import sys +from itertools import product +from typing import Callable, Dict, Tuple, Union + +import numpy as np +import pandas as pd +import pytest +from sklearn.datasets import load_breast_cancer, load_diabetes, load_linnerud, load_wine + +from fedot.core.composer.metrics import QualityMetric, ROCAUC +from fedot.core.data.data import InputData, OutputData +from fedot.core.data.data_split import train_test_data_setup +from fedot.core.pipelines.node import PipelineNode +from fedot.core.pipelines.pipeline import Pipeline +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.metrics_repository import (ClassificationMetricsEnum, ComplexityMetricsEnum, + MetricsRepository, RegressionMetricsEnum, + TimeSeriesForecastingMetricsEnum) +from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams +from fedot.core.utils import fedot_project_root, set_random_seed + + +@pytest.fixture(scope='session') +def data_setup(request): + task_type = request.param + validation_blocks = None + if task_type in ('binary', 'complexity'): + x, y = load_breast_cancer(return_X_y=True) + task = Task(TaskTypesEnum.classification) + data_type = DataTypesEnum.table + elif task_type == 'multiclass': + x, y = load_wine(return_X_y=True) + task = Task(TaskTypesEnum.classification) + data_type = DataTypesEnum.table + elif task_type == 'regression': + x, y = load_diabetes(return_X_y=True) + task = Task(TaskTypesEnum.regression) + data_type = DataTypesEnum.table + elif task_type == 'multitarget': + x, y = load_linnerud(return_X_y=True) + task = Task(TaskTypesEnum.regression) + data_type = DataTypesEnum.table + elif task_type == 'ts': + file_path = fedot_project_root() / 'test/data/short_time_series.csv' + df = pd.read_csv(file_path) + x = y = df['sea_height'].to_numpy() + task = Task(TaskTypesEnum.ts_forecasting, TsForecastingParams(forecast_length=10)) + data_type = DataTypesEnum.ts + validation_blocks = 2 + elif task_type == 'multits': + file_path = fedot_project_root() / 'test/data/short_time_series.csv' + df = pd.read_csv(file_path) + x = df[['sea_height', 'sea_height']].to_numpy() + y = df['sea_height'].to_numpy() + task = Task(TaskTypesEnum.ts_forecasting, TsForecastingParams(forecast_length=10)) + data_type = DataTypesEnum.multi_ts + validation_blocks = 2 + else: + raise ValueError(f'Unsupported task type: {task_type}') + + x, y = x[:200], y[:200] + + # Wrap data into InputData + input_data = InputData(features=x, + target=y, + idx=np.arange(len(x)), + task=task, + data_type=data_type) + # Train test split + train_data, test_data = train_test_data_setup(input_data, validation_blocks=validation_blocks) + return train_data, test_data, task_type, validation_blocks + + +def get_classification_pipeline(): + first = PipelineNode(operation_type='logit') + second = PipelineNode(operation_type='logit', nodes_from=[first]) + third = PipelineNode(operation_type='logit', nodes_from=[first]) + final = PipelineNode(operation_type='logit', nodes_from=[second, third]) + + pipeline = Pipeline(final) + + return pipeline + + +def get_regression_pipeline(): + first = PipelineNode(operation_type='scaling') + final = PipelineNode(operation_type='linear', nodes_from=[first]) + + pipeline = Pipeline(final) + + return pipeline + + +def get_ts_pipeline(window_size=30): + """ Function return pipeline with lagged transformation in it """ + node_lagged = PipelineNode('lagged') + node_lagged.parameters = {'window_size': window_size} + + node_final = PipelineNode('ridge', nodes_from=[node_lagged]) + pipeline = Pipeline(node_final) + return pipeline + + +@pytest.fixture(scope='session') +def expected_values() -> Dict[str, Dict[str, float]]: + with open(fedot_project_root() / 'test/data/expected_metric_values.json', 'r') as f: + return json.load(f) + + +@pytest.mark.parametrize( + 'metric, pipeline_func, data_setup', + [ # TODO: Add binary classification to the test after completion of https://github.com/aimclub/FEDOT/issues/1221. + *product(ComplexityMetricsEnum, [get_classification_pipeline], ['complexity']), + *product(ClassificationMetricsEnum, [get_classification_pipeline], ['multiclass']), + *product(RegressionMetricsEnum, [get_regression_pipeline], ['regression', 'multitarget']), + *product(TimeSeriesForecastingMetricsEnum, [get_ts_pipeline], ['ts', 'multits']) + ], + indirect=['data_setup'] +) +def test_metrics(metric: ClassificationMetricsEnum, pipeline_func: Callable[[], Pipeline], + data_setup: Tuple[InputData, InputData, str, Union[int, None]], + expected_values: Dict[str, Dict[str, float]]): + set_random_seed(0) + update_expected_values: bool = False + + train, test, task_type, validation_blocks = data_setup + + pipeline = pipeline_func() + pipeline.fit(input_data=train) + metric_function = MetricsRepository.get_metric(metric) + metric_class = MetricsRepository.get_metric_class(metric) + metric_value = metric_function(pipeline=pipeline, reference_data=test, validation_blocks=validation_blocks) + + if not update_expected_values: + expected_value = expected_values[task_type][str(metric)] + assert np.isclose(metric_value, expected_value, rtol=0.001, atol=0.001) + assert not np.isclose(metric_value, metric_class.default_value, rtol=0.01, atol=0.01) + else: + with open(fedot_project_root() / 'test/data/expected_metric_values.json', 'w') as f: + expected_values[task_type] = expected_values.get(task_type) or {} + expected_values[task_type][str(metric)] = metric_value + json.dump(expected_values, f, indent=2) + raise ValueError('The value of `update_expected_values` should equal to `False` ' + 'in order for this test to pass.') + + +@pytest.mark.parametrize( + 'metric, pipeline_func, data_setup', + [ + *product(ClassificationMetricsEnum, [get_classification_pipeline], ['binary']), + ], + indirect=['data_setup'] +) +def test_binary_classification(metric: ClassificationMetricsEnum, pipeline_func: Callable[[], Pipeline], + data_setup: Tuple[InputData, InputData, str, Union[int, None]], + expected_values: Dict[str, Dict[str, float]]): + train, test, task_type, validation_blocks = data_setup + + pipeline = pipeline_func() + pipeline.fit(input_data=train) + metric_function = MetricsRepository.get_metric(metric) + metric_class = MetricsRepository.get_metric_class(metric) + metric_value = metric_function(pipeline=pipeline, reference_data=test, validation_blocks=validation_blocks) + + assert not np.isclose(metric_value, metric_class.default_value, rtol=0.01, atol=0.01) + assert 0 < abs(metric_value) < sys.maxsize + + +@pytest.mark.parametrize( + 'metric, pipeline_func, data_setup, validation_blocks', + [ + *product(ClassificationMetricsEnum, [get_classification_pipeline], ['binary', 'multiclass'], [None]), + *product(RegressionMetricsEnum, [get_regression_pipeline], ['regression', 'multitarget'], [None]), + *product(TimeSeriesForecastingMetricsEnum, [get_ts_pipeline], ['ts', 'multits'], [2]), + ], + indirect=['data_setup'] +) +def test_ideal_case_metrics(metric: ClassificationMetricsEnum, pipeline_func: Callable[[], Pipeline], + validation_blocks: Union[int, None], data_setup: Tuple[InputData, InputData, str], + expected_values): + reference, _, task_type, _ = data_setup + metric_class = MetricsRepository.get_metric_class(metric) + predicted = OutputData(idx=reference.idx, task=reference.task, data_type=reference.data_type) + if task_type == 'multiclass' and metric_class.output_mode != 'labels': + label_vals = np.unique(reference.target) + predicted.predict = np.identity(len(label_vals))[reference.target] + else: + predicted.predict = reference.target + if task_type == 'multits': + reference.features = reference.features[:, 0] + + ideal_value = metric_class.metric(reference, predicted) + + assert ideal_value != metric_class.default_value + + +@pytest.mark.parametrize('data_setup', ['multitarget'], indirect=True) +def test_predict_shape_multi_target(data_setup: Tuple[InputData, InputData, str]): + train, test, _, _ = data_setup + simple_pipeline = Pipeline(PipelineNode('linear')) + simple_pipeline.fit(input_data=train) + + target_shape = test.target.shape + # Get converted data + _, results = QualityMetric()._simple_prediction(simple_pipeline, test) + predict_shape = results.predict.shape + assert target_shape == predict_shape + + +def test_roc_auc_multiclass_correct(): + data = InputData(features=np.array([[1, 2], [2, 3], [3, 4], [4, 1]]), + target=np.array([['x'], ['y'], ['z'], ['x']]), + idx=np.arange(4), + task=Task(TaskTypesEnum.classification), + data_type=DataTypesEnum.table) + prediction = OutputData(features=np.array([[1, 2], [2, 3], [3, 4], [4, 1]]), + predict=np.array([[0.4, 0.3, 0.3], + [0.2, 0.5, 0.3], + [0.1, 0.2, 0.7], + [0.8, 0.1, 0.1]]), + idx=np.arange(4), task=Task(TaskTypesEnum.classification), data_type=DataTypesEnum.table) + for i in range(data.num_classes): + fpr, tpr, threshold = ROCAUC.roc_curve(data.target, prediction.predict[:, i], + pos_label=data.class_labels[i]) + roc_auc = ROCAUC.auc(fpr, tpr) + assert roc_auc diff --git a/test/unit/composer/test_quality_metrics.py b/test/unit/composer/test_quality_metrics.py deleted file mode 100644 index 426faef66d..0000000000 --- a/test/unit/composer/test_quality_metrics.py +++ /dev/null @@ -1,119 +0,0 @@ -import os -import sys - -import numpy as np -import pytest -from sklearn.datasets import load_breast_cancer - -from fedot.core.composer.metrics import QualityMetric, ROCAUC -from fedot.core.data.data import InputData, OutputData -from fedot.core.data.data_split import train_test_data_setup -from fedot.core.pipelines.node import PipelineNode -from fedot.core.pipelines.pipeline import Pipeline -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import \ - (ClassificationMetricsEnum, - ComplexityMetricsEnum, - MetricsRepository, - RegressionMetricsEnum) -from fedot.core.repository.tasks import Task, TaskTypesEnum - - -@pytest.fixture() -def data_setup(): - predictors, response = load_breast_cancer(return_X_y=True) - response = response[:100] - predictors = predictors[:100] - - # Wrap data into InputData - input_data = InputData(features=predictors, - target=response, - idx=np.arange(0, len(predictors)), - task=Task(TaskTypesEnum.classification), - data_type=DataTypesEnum.table) - # Train test split - train_data, test_data = train_test_data_setup(input_data) - return train_data, test_data - - -@pytest.fixture() -def multi_target_data_setup(): - test_file_path = str(os.path.dirname(__file__)) - file = '../../data/multi_target_sample.csv' - path = os.path.join(test_file_path, file) - - target_columns = ['1_day', '2_day', '3_day', '4_day', '5_day', '6_day', '7_day'] - task = Task(TaskTypesEnum.regression) - data = InputData.from_csv(path, target_columns=target_columns, - index_col=None, columns_to_drop=['date'], task=task) - train, test = train_test_data_setup(data) - return train, test - - -def default_valid_pipeline(): - first = PipelineNode(operation_type='logit') - second = PipelineNode(operation_type='logit', nodes_from=[first]) - third = PipelineNode(operation_type='logit', nodes_from=[first]) - final = PipelineNode(operation_type='logit', nodes_from=[second, third]) - - pipeline = Pipeline(final) - - return pipeline - - -def test_structural_quality_correct(): - pipeline = default_valid_pipeline() - metric_function = MetricsRepository().metric_by_id(ComplexityMetricsEnum.structural) - expected_metric_value = 13 - actual_metric_value = metric_function(pipeline) - assert actual_metric_value <= expected_metric_value - - -def test_classification_quality_metric(data_setup): - train, _ = data_setup - pipeline = default_valid_pipeline() - pipeline.fit(input_data=train) - - for metric in ClassificationMetricsEnum: - metric_function = MetricsRepository().metric_by_id(metric) - metric_value = metric_function(pipeline=pipeline, reference_data=train) - assert 0 < abs(metric_value) < sys.maxsize - - -def test_regression_quality_metric(data_setup): - train, _ = data_setup - pipeline = default_valid_pipeline() - pipeline.fit(input_data=train) - - for metric in RegressionMetricsEnum: - metric_function = MetricsRepository().metric_by_id(metric) - metric_value = metric_function(pipeline=pipeline, reference_data=train) - assert metric_value > 0 - - -def test_data_preparation_for_multi_target_correct(multi_target_data_setup): - train, test = multi_target_data_setup - simple_pipeline = Pipeline(PipelineNode('linear')) - simple_pipeline.fit(input_data=train) - - source_shape = test.target.shape - # Get converted data - results, new_test = QualityMetric()._simple_prediction(simple_pipeline, test) - number_elements = len(new_test.target) - assert source_shape[0] * source_shape[1] == number_elements - - -def test_roc_auc_multiclass_correct(): - data = InputData(features=[[1, 2], [2, 3], [3, 4], [4, 1]], target=np.array([['x'], ['y'], ['z'], ['x']]), - idx=np.arange(4), - task=Task(TaskTypesEnum.classification), data_type=DataTypesEnum.table) - prediction = OutputData(features=[[1, 2], [2, 3], [3, 4], [4, 1]], predict=np.array([[0.4, 0.3, 0.3], - [0.2, 0.5, 0.3], - [0.1, 0.2, 0.7], - [0.8, 0.1, 0.1]]), - idx=np.arange(4), task=Task(TaskTypesEnum.classification), data_type=DataTypesEnum.table) - for i in range(data.num_classes): - fpr, tpr, threshold = ROCAUC.roc_curve(data.target, prediction.predict[:, i], - pos_label=data.class_labels[i]) - roc_auc = ROCAUC.auc(fpr, tpr) - assert roc_auc diff --git a/test/unit/optimizer/test_pipeline_objective_eval.py b/test/unit/optimizer/test_pipeline_objective_eval.py index 1c1ce688e7..145a28d3db 100644 --- a/test/unit/optimizer/test_pipeline_objective_eval.py +++ b/test/unit/optimizer/test_pipeline_objective_eval.py @@ -13,7 +13,7 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.pipeline_builder import PipelineBuilder from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, MetricsRepository, \ +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum, MetricsRepository, \ RegressionMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from test.integration.models.test_model import classification_dataset, classification_dataset_with_str_labels @@ -51,7 +51,7 @@ def actual_fitness(data_split, pipeline, metric): metric_values = [] for (train_data, test_data) in data_split(): pipeline.fit(train_data) - metric_function = MetricsRepository().metric_by_id(metric, default_callable=metric) + metric_function = MetricsRepository.get_metric(metric) metric_values.append(metric_function(pipeline=pipeline, reference_data=test_data)) mean_metric = np.mean(metric_values, axis=0) return SingleObjFitness(mean_metric) diff --git a/test/unit/pipelines/prediction_intervals/data/pred_ints_model_test.pickle b/test/unit/pipelines/prediction_intervals/data/pred_ints_model_test.pickle index 88fe04d97a..4c89b978be 100644 Binary files a/test/unit/pipelines/prediction_intervals/data/pred_ints_model_test.pickle and b/test/unit/pipelines/prediction_intervals/data/pred_ints_model_test.pickle differ diff --git a/test/unit/pipelines/prediction_intervals/test_solver_mutations.py b/test/unit/pipelines/prediction_intervals/test_solver_mutations.py index e73e1473dc..61d15dee4f 100644 --- a/test/unit/pipelines/prediction_intervals/test_solver_mutations.py +++ b/test/unit/pipelines/prediction_intervals/test_solver_mutations.py @@ -2,14 +2,14 @@ import numpy as np import pytest -from golem.core.log import default_log, Log +from golem.core.log import Log, default_log from fedot.core.data.data import InputData from fedot.core.pipelines.prediction_intervals.params import PredictionIntervalsParams from fedot.core.pipelines.prediction_intervals.solvers.mutation_of_best_pipeline import solver_mutation_of_best_pipeline from fedot.core.pipelines.prediction_intervals.utils import get_last_generations from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.tasks import TsForecastingParams, Task, TaskTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot.core.utils import fedot_project_root @@ -42,9 +42,9 @@ def test_solver_mutation_of_best_pipeline(params): params_default = {'choice': 'different', 'discard': True, 'percentage': 0.66, 'number_mutations': 10, 'message': 'default solver_mutation_of_best_pipeline failed.'} params_with_replacement = {'choice': 'with_replacement', 'discard': True, 'percentage': 0.5, 'number_mutations': 30, - 'message': 'solver_mutation_of_best_pipeline with inapropriate pipelines failed.'} + 'message': 'solver_mutation_of_best_pipeline with inappropriate pipelines failed.'} params_different = {'choice': 'different', 'discard': False, 'percentage': 0.8, 'number_mutations': 10, - 'mesage': 'solver_mutation_of_best_pipeline failed.'} + 'message': 'solver_mutation_of_best_pipeline failed.'} for x in [params_default, params_with_replacement, params_different]: res = solver_mutation_of_best_pipeline(train_input=params['train_input'], diff --git a/test/unit/tasks/test_regression.py b/test/unit/tasks/test_regression.py index e63ba6473c..7489d1c5b6 100644 --- a/test/unit/tasks/test_regression.py +++ b/test/unit/tasks/test_regression.py @@ -1,4 +1,5 @@ import numpy as np +import pytest from sklearn.datasets import make_regression from sklearn.metrics import mean_squared_error as mse @@ -10,9 +11,7 @@ from fedot.core.repository.dataset_types import DataTypesEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from test.unit.common_tests import is_predict_ignores_target -from test.unit.composer.test_quality_metrics import multi_target_data_setup - -_ = multi_target_data_setup +from test.unit.composer.test_metrics import data_setup # noqa def check_predict_correct(pipeline, input_data): @@ -101,9 +100,10 @@ def test_regression_pipeline_with_data_operation_fit_predict_correct(): assert check_predict_correct(pipeline, train_data) -def test_multi_target_regression_composing_correct(multi_target_data_setup): +@pytest.mark.parametrize('data_setup', ['multitarget'], indirect=True) +def test_multi_target_regression_composing_correct(data_setup): # Load simple dataset for multi-target - train, test = multi_target_data_setup + train, test, _, _ = data_setup problem = 'regression' timeout = 0.1 diff --git a/test/unit/validation/test_table_cv.py b/test/unit/validation/test_table_cv.py index 61d8f5bfef..8ec6ce1a16 100644 --- a/test/unit/validation/test_table_cv.py +++ b/test/unit/validation/test_table_cv.py @@ -15,7 +15,7 @@ from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder from fedot.core.repository.operation_types_repository import OperationTypesRepository -from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum +from fedot.core.repository.metrics_repository import ClassificationMetricsEnum from fedot.core.repository.tasks import Task, TaskTypesEnum from fedot.core.utils import fedot_project_root from test.integration.models.test_model import classification_dataset diff --git a/test/unit/validation/test_time_series_cv.py b/test/unit/validation/test_time_series_cv.py index 9f2e99db8f..3acfcc64ce 100644 --- a/test/unit/validation/test_time_series_cv.py +++ b/test/unit/validation/test_time_series_cv.py @@ -14,7 +14,7 @@ from fedot.core.composer.composer_builder import ComposerBuilder from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.quality_metrics_repository import \ +from fedot.core.repository.metrics_repository import \ MetricsRepository, RegressionMetricsEnum from fedot.core.repository.tasks import TsForecastingParams from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace @@ -112,7 +112,7 @@ def test_composer_cv_correct(): ) init_pipeline = get_simple_ts_pipeline() - metric_function = MetricsRepository().metric_by_id(RegressionMetricsEnum.RMSE) + metric_function = MetricsRepository.get_metric(RegressionMetricsEnum.RMSE) builder = ComposerBuilder(task=time_series.task). \ with_optimizer_params(parameters). \ with_requirements(composer_requirements). \