Add agent type

fedot/api/api_utils/api_params_repository.py

@@ -2,6 +2,8 @@
 from functools import partial
 from typing import Sequence
 
+from golem.core.optimisers.adaptive.context_agents import ContextAgentTypeEnum
+from golem.core.optimisers.adaptive.operator_agent import MutationAgentTypeEnum
 from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum
 from golem.core.optimisers.genetic.operators.mutation import MutationTypesEnum
 
@@ -68,7 +70,9 @@ def default_params_for_task(task_type: TaskTypesEnum) -> dict:
             cache_dir=default_fedot_data_dir(),
             keep_history=True,
             history_dir=default_fedot_data_dir(),
-            with_tuning=False
+            with_tuning=False,
+            adaptive_mutation_type=MutationAgentTypeEnum.default,
+            context_agent_type=ContextAgentTypeEnum.nodes_num
         )
         return default_param_values_dict
 
@@ -115,6 +119,9 @@ def get_params_for_gp_algorithm_params(self, params: dict) -> dict:
         """ Returns dict with parameters suitable for ``GPAlgorithmParameters``"""
         gp_algorithm_params = {'pop_size': params.get('pop_size'),
                                'genetic_scheme_type': GeneticSchemeTypesEnum.parameter_free}
+
+        gp_algorithm_params = self._get_adaptive_optimization_params(params, gp_algorithm_params)
+
         if params.get('genetic_scheme') == 'steady_state':
             gp_algorithm_params['genetic_scheme_type'] = GeneticSchemeTypesEnum.steady_state
 
@@ -136,3 +143,22 @@ def _get_default_mutations(task_type: TaskTypesEnum, params) -> Sequence[Mutatio
             mutations.append(add_resample_mutation)
 
         return mutations
+
+    @staticmethod
+    def _get_adaptive_optimization_params(params: dict, gp_algorithm_params: dict) -> dict:
+        """ Extracts params conserned with adaptive optimization including `adaptive_mutation_type`
+        and `context_agent_type` from params. """
+        adaptive_mutation_type = params.get('adaptive_mutation_type')
+        context_agent_type = params.get('context_agent_type')
+
+        if adaptive_mutation_type:
+            adaptive_mutation_type_class = MutationAgentTypeEnum[adaptive_mutation_type] \
+                if isinstance(adaptive_mutation_type, str) else adaptive_mutation_type
+            gp_algorithm_params['adaptive_mutation_type'] = adaptive_mutation_type_class
+
+        if context_agent_type:
+            context_agent_type_class = ContextAgentTypeEnum[context_agent_type] \
+                if isinstance(context_agent_type, str) else context_agent_type
+            gp_algorithm_params['context_agent_type'] = context_agent_type_class
+
+        return gp_algorithm_params

fedot/api/main.py

@@ -8,7 +8,7 @@
 from golem.core.log import Log, default_log
 from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
 from golem.core.tuning.simultaneous import SimultaneousTuner
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 from golem.visualisation.opt_viz_extra import visualise_pareto
 
 from fedot.api.api_utils.api_composer import ApiComposer
@@ -163,6 +163,12 @@ class Fedot:
             Default optimizer is :class:`~golem.core.optimisers.genetic.gp_optimizer.EvoGraphOptimizer`.
             See the `example \
 <https://github.com/aimclub/FEDOT/blob/master/examples/advanced/fedot_based_solutions/external_optimizer.py>`_
+
+        NB! 'adaptive_mutation_type' and 'context_agent_type' can be specified with 'composer_tuner_params'.
+        Both these params are a part of experimental functionality for adaptive evolution.
+        Adaptive evolution uses multi-armed bandits to choose most promising mutation at each iteration
+        of evolution cycle.
+        The full range of context agents can be obtained in GOLEM: golem.core.optimizers.adaptive.context_agents
     """
 
     def __init__(self,

fedot/core/caching/base_cache.py

@@ -1,7 +1,7 @@
 from typing import Union
 
 from golem.core.log import default_log
-from golem.core.utilities.singleton_meta import SingletonMeta
+from golem.utilities.singleton_meta import SingletonMeta
 
 from fedot.core.caching.pipelines_cache_db import OperationsCacheDB
 from fedot.core.caching.preprocessing_cache_db import PreprocessingCacheDB

fedot/core/caching/pipelines_cache.py

@@ -1,7 +1,7 @@
 import sqlite3
 from typing import List, Optional, TYPE_CHECKING, Union
 
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 from fedot.core.caching.base_cache import BaseCache
 from fedot.core.caching.pipelines_cache_db import OperationsCacheDB
@@ -35,8 +35,12 @@ def save_nodes(self, nodes: Union[PipelineNode, List[PipelineNode]], fold_id: Op
             ]
             self._db.add_operations(mapped)
         except Exception as ex:
-            unexpected_exc = not (isinstance(ex, sqlite3.DatabaseError) and 'disk is full' in str(ex))
-            self.log.warning(f'Nodes can not be saved: {ex}. Continue', exc=ex, raise_if_test=unexpected_exc)
+            exc_is_expected = (isinstance(ex, sqlite3.DatabaseError) and 'disk is full' in str(ex))
+            msg = 'Nodes can not be saved'
+            if exc_is_expected:
+                self.log.warning(msg, exc_info=ex)
+            else:
+                self.log.log_or_raise('warning', ValueError(msg))
 
     def save_pipeline(self, pipeline: 'Pipeline', fold_id: Optional[int] = None):
         """
@@ -61,8 +65,8 @@ def try_load_nodes(self, nodes: Union[PipelineNode, List[PipelineNode]], fold_id
                     nodes_lst[idx].fitted_operation = cached_op
                 else:
                     nodes_lst[idx].fitted_operation = None
-        except Exception as ex:
-            self.log.warning(f'Cache can not be loaded: {ex}. Continue.', exc=ex, raise_if_test=True)
+        except Exception:
+            self.log.log_or_raise('info', ValueError('Cache can not be loaded.'))
 
     def try_load_into_pipeline(self, pipeline: 'Pipeline', fold_id: Optional[int] = None):
         """

fedot/core/caching/preprocessing_cache.py

@@ -30,8 +30,8 @@ def try_load_preprocessor(self, pipeline: 'Pipeline', fold_id: Union[int, None])
             processors = self._db.get_preprocessor(structural_id)
             if processors:
                 pipeline.encoder, pipeline.imputer = processors
-        except Exception as ex:
-            self.log.warning(f'Preprocessor search error: {ex}', raise_if_test=True)
+        except Exception:
+            self.log.log_or_raise('warning', ValueError('Preprocessor search error'))
 
     def add_preprocessor(self, pipeline: 'Pipeline', fold_id: Optional[Union[int, None]] = None):
         """

fedot/core/composer/composer_builder.py

@@ -8,7 +8,7 @@
 from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters
 from golem.core.optimisers.initial_graphs_generator import InitialPopulationGenerator, GenerationFunction
 from golem.core.optimisers.optimizer import GraphOptimizer, AlgorithmParameters, GraphGenerationParams
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 from fedot.core.caching.pipelines_cache import OperationsCache
 from fedot.core.caching.preprocessing_cache import PreprocessingCache

fedot/core/composer/metrics.py

@@ -89,7 +89,7 @@ def get_value(cls, pipeline: 'Pipeline', reference_data: InputData,
                               save_path=Path(save_path, 'forecast.png'))
 
         except Exception as ex:
-            pipeline.log.info(f'Metric can not be evaluated because of: {ex}', raise_if_test=True)
+            pipeline.log.log_or_raise('info', ValueError('Metric can not be evaluated'))
 
         return metric
 

fedot/core/data/merge/data_merger.py

@@ -1,7 +1,7 @@
 from typing import List, Iterable, Union
 
 from golem.core.log import default_log
-from golem.core.utilities.data_structures import are_same_length
+from golem.utilities.data_structures import are_same_length
 
 from fedot.core.data.array_utilities import *
 from fedot.core.data.data import OutputData, InputData

fedot/core/optimisers/objective/data_objective_eval.py

@@ -80,8 +80,8 @@ def evaluate(self, graph: Pipeline) -> Fitness:
             if evaluated_fitness.valid:
                 folds_metrics.append(evaluated_fitness.values)
             else:
-                self._log.warning(f'Invalid fitness after objective evaluation. '
-                                  f'Skipping the graph: {graph_id}', raise_if_test=True)
+                self._log.log_or_raise('warning', ValueError(f'Invalid fitness after objective evaluation. '
+                                                             f'Skipping the graph: {graph_id}'))
             if self._do_unfit:
                 graph.unfit()
         if folds_metrics:

fedot/core/optimisers/objective/metrics_objective.py

@@ -1,7 +1,7 @@
 from typing import Union, Iterable, Callable
 
 from golem.core.optimisers.objective import Objective
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 from fedot.core.repository.quality_metrics_repository import MetricType, MetricsRepository, ComplexityMetricsEnum
 

fedot/core/pipelines/pipeline.py

@@ -12,7 +12,7 @@
 from golem.core.log import default_log
 from golem.core.optimisers.timer import Timer
 from golem.core.paths import copy_doc
-from golem.core.utilities.serializable import Serializable
+from golem.utilities.serializable import Serializable
 from golem.visualisation.graph_viz import NodeColorType
 
 from fedot.core.caching.pipelines_cache import OperationsCache

fedot/core/pipelines/pipeline_node_factory.py

@@ -17,6 +17,15 @@ def __init__(self, requirements: PipelineComposerRequirements,
         self.advisor = advisor or PipelineChangeAdvisor()
         self.graph_model_repository = graph_model_repository or self._init_default_graph_model_repo()
 
+    def get_all_available_operations(self) -> List[str]:
+        """ Returns all available models and data operations. """
+        available_operations = []
+        if self.advisor.models:
+            available_operations += self.advisor.models
+        if self.advisor.data_operations:
+            available_operations += self.advisor.data_operations
+        return available_operations
+
     def _init_default_graph_model_repo(self):
         """ Initialize default graph model repository with operations from composer requirements """
         repo = PipelineOperationRepository(operations_by_keys={'primary': self.requirements.primary,

fedot/core/pipelines/tuning/tuner_builder.py

@@ -4,7 +4,7 @@
 from golem.core.tuning.optuna_tuner import OptunaTuner
 from golem.core.tuning.simultaneous import SimultaneousTuner
 from golem.core.tuning.tuner_interface import BaseTuner
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 from fedot.core.constants import DEFAULT_TUNING_ITERATIONS_NUMBER
 from fedot.core.data.data import InputData

fedot/core/repository/dataset_types.py

@@ -1,4 +1,4 @@
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 
 class DataTypesEnum(Enum):

fedot/core/repository/operation_types_repository.py

@@ -6,7 +6,7 @@
 
 import numpy as np
 from golem.core.log import default_log
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 
 from fedot.core.constants import BEST_QUALITY_PRESET_NAME, AUTO_PRESET_NAME
 from fedot.core.repository.dataset_types import DataTypesEnum

fedot/core/repository/quality_metrics_repository.py

@@ -2,7 +2,7 @@
 from typing import Callable, Union, TypeVar
 
 from golem.core.dag.graph import Graph
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 from fedot.core.composer.metrics import (ComputationTime, Accuracy, F1, Logloss, MAE,
                                          MAPE, SMAPE, MSE, MSLE, Metric, NodeNum, Precision, R2,

fedot/core/repository/tasks.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import Any, List, Optional
 
-from golem.core.utilities.data_structures import ComparableEnum as Enum
+from golem.utilities.data_structures import ComparableEnum as Enum
 
 
 @dataclass

fedot/core/utils.py

@@ -7,7 +7,7 @@
 
 import numpy as np
 import pandas as pd
-from golem.core.utilities.random import RandomStateHandler
+from golem.utilities.random import RandomStateHandler
 from sklearn.model_selection import train_test_split
 
 DEFAULT_PARAMS_STUB = 'default_params'

fedot/remote/infrastructure/clients/client.py

@@ -3,7 +3,7 @@
 from typing import Optional, Type, TypeVar
 
 from golem.core.log import default_log
-from golem.core.utilities.serializable import Serializable
+from golem.utilities.serializable import Serializable
 
 from fedot.core.utils import default_fedot_data_dir
 

fedot/remote/remote_evaluator.py

@@ -5,7 +5,7 @@
 import numpy as np
 from golem.core.log import default_log
 from golem.core.optimisers.genetic.evaluation import DelegateEvaluator
-from golem.core.utilities.serializable import Serializable
+from golem.utilities.serializable import Serializable
 
 from fedot.core.data.data import InputData
 from fedot.remote.infrastructure.clients.client import Client

fedot/remote/run_pipeline.py

@@ -4,7 +4,7 @@
 from typing import Union
 
 from golem.core.log import default_log
-from golem.core.utilities.random import RandomStateHandler
+from golem.utilities.random import RandomStateHandler
 
 from fedot.core.data.data import InputData
 from fedot.core.data.multi_modal import MultiModalData

fedot/utilities/golem_imports_transition.py

@@ -21,7 +21,7 @@
     'fedot.core.adapter': 'golem.core.adapter',
     'fedot.core.dag': 'golem.core.dag',
     'fedot.core.optimisers': 'golem.core.optimisers',
-    'fedot.core.utilities': 'golem.core.utilities',
+    'fedot.core.utilities': 'golem.utilities',
 
     'fedot.core.serializers': 'golem.serializers',
     'fedot.core.visualisation': 'golem.visualisation',

requirements.txt

@@ -1,5 +1,5 @@
 # Base framework
-thegolem==0.3.3
+thegolem==0.4.0
 
 # Data
 numpy>=1.16.0, !=1.24.0

test/integration/pipelines/tuning/test_pipeline_tuning.py

@@ -7,7 +7,7 @@
 from golem.core.tuning.optuna_tuner import OptunaTuner
 from golem.core.tuning.sequential import SequentialTuner
 from golem.core.tuning.simultaneous import SimultaneousTuner
-from golem.core.utilities.data_structures import ensure_wrapped_in_sequence
+from golem.utilities.data_structures import ensure_wrapped_in_sequence
 from hyperopt import hp
 from hyperopt.pyll.stochastic import sample as hp_sample
 

test/unit/remote/test_remote_custom.py

@@ -4,7 +4,7 @@
 from typing import Tuple, Union, Optional, Sequence
 
 import pytest
-from golem.core.utilities.serializable import Serializable
+from golem.utilities.serializable import Serializable
 
 from fedot.remote.infrastructure.clients.client import Client
 from fedot.remote.remote_evaluator import RemoteEvaluator, RemoteTaskParams