Metrics fixes, tests, refactoring, typing improvement (#1214)

* fix f1 & precision evaluation

* rem flattening from QualityMetric._simple_prediction

* fix metrics hierarchy

* improve typing of QualityMetricCallable

* add tests for all supported types of metrics

* return **kwargs to complexity metrics

* use MetricCallable where it is appropriate

* tests: simplify regression pipeline, type hints & additional assertion

* remove __self__ from MetricCallable definition;
remove default value for metric_by_id

* rename & redo repository methods

* rename MetricType -> MetricIDType

* reorder module

* rename metrics repo module

* rename metrics test module

* update test data pipeline pickle

* correct typos

* adapt documentation

* fix inheritance

* add kwargs and metric() to complexity metrics

* better types for metrics_repository.py

* combine all metrics to a single test; add default values check

* unwrap pipeline predictions for time series

* fix confusing naming

* add fixture for expected values for faster tests

* remove test for binary classification

* add a less strict test for binary classification

* fix random seed for binary classification

* delete simple test for binary classification

* delete binary test from `test_metrics`

* fix structural_analysis_example.py

* make `test_metrics` fall if `update_expected_values=True`

* fix unused imports

* add `str` to `MetricIDType`

---------

Co-authored-by: kasyanovse <[email protected]>

cases/credit_scoring/credit_scoring_problem_multiobj.py

@@ -15,7 +15,7 @@
 from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
 from fedot.core.repository.operation_types_repository import get_operations_for_task
-from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, ComplexityMetricsEnum
+from fedot.core.repository.metrics_repository import ClassificationMetricsEnum, ComplexityMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 from fedot.core.utils import set_random_seed
 

cases/evo_operators_comparison/evo_operators_comparison.py

@@ -16,7 +16,7 @@
 
 from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements
 from fedot.core.repository.operation_types_repository import get_operations_for_task
-from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum
+from fedot.core.repository.metrics_repository import ClassificationMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 from fedot.core.utils import fedot_project_root
 

cases/river_levels_prediction/river_level_case_composer.py

@@ -16,7 +16,7 @@
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-from fedot.core.repository.quality_metrics_repository import \
+from fedot.core.repository.metrics_repository import \
     RegressionMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 

cases/river_levels_prediction/river_level_case_manual.py

@@ -11,7 +11,7 @@
 from fedot.core.pipelines.node import PipelineNode
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+from fedot.core.repository.metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 
 warnings.filterwarnings('ignore')

docs/source/advanced/hyperparameters_tuning.rst

@@ -378,7 +378,7 @@ Example for ``SimultaneousTuner``:
     from fedot.core.pipelines.pipeline_builder import PipelineBuilder
     from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace
     from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-    from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum
+    from fedot.core.repository.metrics_repository import ClassificationMetricsEnum
     from fedot.core.repository.tasks import TaskTypesEnum, Task
 
     task = Task(TaskTypesEnum.classification)
@@ -466,7 +466,7 @@ Example for ``IOptTuner``:
     from fedot.core.pipelines.pipeline_builder import PipelineBuilder
     from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements
     from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-    from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+    from fedot.core.repository.metrics_repository import RegressionMetricsEnum
     from fedot.core.repository.tasks import TaskTypesEnum, Task
 
     task = Task(TaskTypesEnum.regression)
@@ -517,7 +517,7 @@ Example for ``OptunaTuner``:
     from fedot.core.data.data import InputData
     from fedot.core.pipelines.pipeline_builder import PipelineBuilder
     from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-    from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+    from fedot.core.repository.metrics_repository import RegressionMetricsEnum
     from fedot.core.repository.tasks import TaskTypesEnum, Task
 
     task = Task(TaskTypesEnum.regression)
@@ -568,7 +568,7 @@ and obtain a list of tuned pipelines representing a pareto front after tuning.
     from fedot.core.pipelines.pipeline import Pipeline
     from fedot.core.pipelines.pipeline_builder import PipelineBuilder
     from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-    from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+    from fedot.core.repository.metrics_repository import RegressionMetricsEnum
     from fedot.core.repository.tasks import TaskTypesEnum, Task
 
     task = Task(TaskTypesEnum.regression)
@@ -603,7 +603,7 @@ Sequential tuning
     from fedot.core.data.data import InputData
     from fedot.core.pipelines.pipeline_builder import PipelineBuilder
     from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-    from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+    from fedot.core.repository.metrics_repository import RegressionMetricsEnum
     from fedot.core.repository.tasks import TaskTypesEnum, Task, TsForecastingParams
 
     task = Task(TaskTypesEnum.ts_forecasting, TsForecastingParams(forecast_length=10))
@@ -663,7 +663,7 @@ Tuning of a node
     from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements
     from fedot.core.pipelines.pipeline_builder import PipelineBuilder
     from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-    from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+    from fedot.core.repository.metrics_repository import RegressionMetricsEnum
     from fedot.core.repository.tasks import TaskTypesEnum, Task
     from test.integration.quality.test_synthetic_tasks import get_regression_data
 

docs/source/api/repository.rst

@@ -16,7 +16,7 @@ Operation Types
 Quality metrics
 ---------------
 
-.. automodule:: fedot.core.repository.quality_metrics_repository
+.. automodule:: fedot.core.repository.metrics_repository
    :members:
    :no-undoc-members:
 

examples/advanced/decompose/classification_refinement_example.py

@@ -5,7 +5,7 @@
 from fedot.core.pipelines.node import PipelineNode
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum
+from fedot.core.repository.metrics_repository import ClassificationMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 from fedot.core.utils import set_random_seed
 

examples/advanced/decompose/regression_refinement_example.py

@@ -11,7 +11,7 @@
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
 from fedot.core.repository.dataset_types import DataTypesEnum
-from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+from fedot.core.repository.metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 from fedot.core.utils import fedot_project_root
 

examples/advanced/multitask_classification_regression.py

@@ -12,7 +12,7 @@
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
 from fedot.core.repository.dataset_types import DataTypesEnum
-from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+from fedot.core.repository.metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import TaskTypesEnum, Task
 from fedot.core.utils import fedot_project_root
 

examples/advanced/pipeline_sensitivity.py

@@ -11,7 +11,7 @@
 from fedot.core.data.data import InputData
 from fedot.core.data.data_split import train_test_data_setup
 from fedot.core.repository.operation_types_repository import get_operations_for_task
-from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, MetricsRepository, \
+from fedot.core.repository.metrics_repository import ClassificationMetricsEnum, MetricsRepository, \
     RegressionMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 from fedot.core.utils import default_fedot_data_dir, fedot_project_root
@@ -123,7 +123,7 @@ def run_class_scoring_case(is_composed: bool, path_to_save=None):
     train_data, test_data = get_scoring_data()
     task = Task(TaskTypesEnum.classification)
     # the choice of the metric for the pipeline quality assessment during composition
-    metric_function = MetricsRepository().metric_by_id(ClassificationMetricsEnum.ROCAUC_penalty)
+    metric_function = MetricsRepository.get_metric(ClassificationMetricsEnum.ROCAUC_penalty)
 
     if is_composed:
         case = 'scoring_composed'
@@ -141,7 +141,7 @@ def run_class_kc2_case(is_composed: bool = False, path_to_save=None):
     train_data, test_data = get_kc2_data()
     task = Task(TaskTypesEnum.classification)
     # the choice of the metric for the pipeline quality assessment during composition
-    metric_function = MetricsRepository().metric_by_id(ClassificationMetricsEnum.ROCAUC_penalty)
+    metric_function = MetricsRepository.get_metric(ClassificationMetricsEnum.ROCAUC_penalty)
 
     if is_composed:
         case = 'kc2_composed'
@@ -159,7 +159,7 @@ def run_regr_case(is_composed: bool = False, path_to_save=None):
     train_data, test_data = get_cholesterol_data()
     task = Task(TaskTypesEnum.regression)
     # the choice of the metric for the pipeline quality assessment during composition
-    metric_function = MetricsRepository().metric_by_id(RegressionMetricsEnum.RMSE)
+    metric_function = MetricsRepository.get_metric(RegressionMetricsEnum.RMSE)
 
     if is_composed:
         case = 'cholesterol_composed'

examples/advanced/structural_analysis/structural_analysis_example.py

@@ -1,15 +1,15 @@
 import os
 from copy import deepcopy
 from functools import partial
+from typing import Callable, Dict, List, Optional, Tuple
 
 from golem.core.dag.graph_verifier import GraphVerifier
-from golem.core.dag.verification_rules import has_one_root, has_no_cycle, has_no_isolated_components, \
-    has_no_isolated_nodes, has_no_self_cycled_nodes
+from golem.core.dag.verification_rules import (has_no_cycle, has_no_isolated_components, has_no_isolated_nodes,
+                                               has_no_self_cycled_nodes, has_one_root)
 from golem.core.optimisers.graph import OptGraph
 from golem.core.optimisers.objective import Objective
 from golem.structural_analysis.graph_sa.graph_structural_analysis import GraphStructuralAnalysis
 from golem.structural_analysis.graph_sa.sa_requirements import StructuralAnalysisRequirements
-from typing import Callable, Dict, Any, Optional, Tuple, List
 
 from examples.advanced.structural_analysis.dataset_access import get_scoring_data
 from examples.advanced.structural_analysis.pipelines_access import get_three_depth_manual_class_pipeline
@@ -20,8 +20,9 @@
 from fedot.core.pipelines.pipeline_advisor import PipelineChangeAdvisor
 from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements
 from fedot.core.pipelines.pipeline_node_factory import PipelineOptNodeFactory
-from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum, QualityMetricsEnum, \
-    MetricsRepository
+from fedot.core.repository.metrics_repository import (ClassificationMetricsEnum, ComplexityMetricCallable,
+                                                      ComplexityMetricsEnum, MetricsRepository,
+                                                      QualityMetricCallable, QualityMetricsEnum)
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 from fedot.utilities.project_import_export import DEFAULT_PATH
 
@@ -31,19 +32,20 @@ class SAObjective(Objective):
     This objective has to evaluate pipeline in __call__ method and have 'metrics' field to identify
     which metrics are optimized.
     """
+
     def __init__(self,
                  objective: Callable,
-                 quality_metrics: Dict[Any, Callable],
-                 complexity_metrics: Optional[Dict[Any, Callable]] = None,
+                 quality_metrics: Dict[QualityMetricsEnum, QualityMetricCallable],
+                 complexity_metrics: Optional[Dict[ComplexityMetricsEnum, ComplexityMetricCallable]] = None,
                  is_multi_objective: bool = False,
                  ):
         self.objective = objective
         super().__init__(quality_metrics=quality_metrics, complexity_metrics=complexity_metrics,
                          is_multi_objective=is_multi_objective)
 
-    def __call__(self, graph: OptGraph) -> float:
+    def __call__(self, graph: OptGraph, **kwargs) -> float:
         pip = PipelineAdapter().restore(graph)
-        return self.objective(pip)
+        return self.objective(pip, **kwargs)
 
 
 def structural_analysis_set_up(train_data: InputData, test_data: InputData,
@@ -53,11 +55,12 @@ def structural_analysis_set_up(train_data: InputData, test_data: InputData,
         -> Tuple[PipelineOptNodeFactory, SAObjective, SAObjective]:
     """ Build initial infrastructure for performing SA: node factory, objectives.
     Can be reused for other SA applications, appropriate parameters must be specified then. """
+
     def _construct_objective(data: InputData, metric: QualityMetricsEnum) -> SAObjective:
         """ Build objective function with fit and predict functions inside. """
-        metric_func = MetricsRepository.metric_by_id(metric)
+        metric_func = MetricsRepository.get_metric(metric)
         get_value = partial(metric_func, reference_data=data)
-        metrics_ = {metric: data}
+        metrics_ = {metric: metric_func}
 
         data_producer = DataSourceSplitter().build(data=data)
         objective_function = PipelineObjectiveEvaluate(objective=Objective(quality_metrics=get_value),

examples/advanced/time_series_forecasting/composing_pipelines.py

@@ -16,7 +16,7 @@
 from fedot.core.data.data_split import train_test_data_setup
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.repository.dataset_types import DataTypesEnum
-from fedot.core.repository.quality_metrics_repository import \
+from fedot.core.repository.metrics_repository import \
     RegressionMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams
 from fedot.core.utils import fedot_project_root

examples/advanced/time_series_forecasting/custom_model_tuning.py

@@ -12,7 +12,7 @@
 from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
 from fedot.core.repository.dataset_types import DataTypesEnum
-from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+from fedot.core.repository.metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import TaskTypesEnum, Task, TsForecastingParams
 
 

examples/advanced/time_series_forecasting/multi_ts_arctic_forecasting.py

@@ -15,7 +15,7 @@
 from fedot.core.composer.composer_builder import ComposerBuilder
 from fedot.core.composer.gp_composer.specific_operators import parameter_change_mutation
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-from fedot.core.repository.quality_metrics_repository import \
+from fedot.core.repository.metrics_repository import \
     RegressionMetricsEnum
 
 

examples/advanced/time_series_forecasting/sparse_lagged_tuning.py

@@ -12,7 +12,7 @@
 from fedot.core.data.data import InputData
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
 from fedot.core.repository.dataset_types import DataTypesEnum
-from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+from fedot.core.repository.metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams
 from fedot.core.utils import fedot_project_root
 

examples/simple/classification/classification_with_tuning.py

@@ -7,7 +7,7 @@
 from fedot.core.data.data import InputData
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
 from fedot.core.repository.dataset_types import DataTypesEnum
-from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum
+from fedot.core.repository.metrics_repository import ClassificationMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 from fedot.core.utils import set_random_seed
 from fedot.utilities.synth_dataset_generator import classification_dataset

examples/simple/classification/multiclass_prediction.py

@@ -16,7 +16,7 @@
 from fedot.core.data.data import InputData
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.repository.operation_types_repository import OperationTypesRepository
-from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum
+from fedot.core.repository.metrics_repository import ClassificationMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 from fedot.core.utils import (
     ensure_directory_exists,

examples/simple/classification/resample_example.py

@@ -12,7 +12,7 @@
 from fedot.core.data.data import InputData
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
 from fedot.core.repository.dataset_types import DataTypesEnum
-from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+from fedot.core.repository.metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import TaskTypesEnum, Task
 from fedot.core.utils import fedot_project_root
 

examples/simple/pipeline_tune.py

@@ -9,7 +9,7 @@
 from fedot.core.data.data import InputData
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum
+from fedot.core.repository.metrics_repository import ClassificationMetricsEnum
 
 
 def get_case_train_test_data():

examples/simple/pipeline_tuning_with_iopt.py

@@ -7,7 +7,7 @@
 from fedot.core.pipelines.pipeline_builder import PipelineBuilder
 from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+from fedot.core.repository.metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import TaskTypesEnum, Task
 from fedot.core.utils import fedot_project_root
 

examples/simple/regression/regression_with_tuning.py

@@ -9,7 +9,7 @@
 from fedot.core.data.data import InputData
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
 from fedot.core.repository.dataset_types import DataTypesEnum
-from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+from fedot.core.repository.metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum
 from fedot.core.utils import set_random_seed
 from fedot.utilities.synth_dataset_generator import regression_dataset

examples/simple/time_series_forecasting/tuning_pipelines.py

@@ -10,7 +10,7 @@
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
 from fedot.core.repository.dataset_types import DataTypesEnum
-from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
+from fedot.core.repository.metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams
 from fedot.core.utils import fedot_project_root
 

fedot/api/api_utils/api_composer.py

@@ -17,12 +17,12 @@
 from fedot.core.data.data import InputData
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder
-from fedot.core.repository.quality_metrics_repository import MetricType
+from fedot.core.repository.metrics_repository import MetricIDType
 
 
 class ApiComposer:
 
-    def __init__(self, api_params: ApiParams, metrics: Union[str, MetricType, Sequence]):
+    def __init__(self, api_params: ApiParams, metrics: Union[MetricIDType, Sequence[MetricIDType]]):
         self.log = default_log(self)
         self.params = api_params
         self.metrics = metrics