From dc70c937d78775458eafc7d458180bbb091333a2 Mon Sep 17 00:00:00 2001 From: Kento Nozawa Date: Sat, 25 Jan 2025 20:57:57 +0900 Subject: [PATCH 1/5] Fix links to external modules --- docs/source/conf.py | 1 + docs/source/reference/index.rst | 6 ++--- optuna_integration/botorch/botorch.py | 24 +++++++++---------- optuna_integration/comet/comet.py | 4 ++-- optuna_integration/dask/dask.py | 2 +- optuna_integration/sklearn/sklearn.py | 18 +++++++------- optuna_integration/tensorboard/tensorboard.py | 2 +- optuna_integration/wandb/wandb.py | 2 +- 8 files changed, 29 insertions(+), 30 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 055b6355d..d54811a37 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -167,6 +167,7 @@ "lightgbm": ("https://lightgbm.readthedocs.io/en/latest", None), "matplotlib": ("https://matplotlib.org/stable", None), "numpy": ("https://numpy.org/doc/stable", None), + "optuna": ("https://optuna.readthedocs.io/en/stable", None), "scipy": ("https://docs.scipy.org/doc/scipy", None), "sklearn": ("https://scikit-learn.org/stable", None), "torch": ("https://pytorch.org/docs/stable", None), diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index eba23d058..3253212b9 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -10,11 +10,11 @@ The former is provided for backward compatibility. For most of the ML frameworks supported by Optuna, the corresponding Optuna integration class serves only to implement a callback object and functions, compliant with the framework's specific callback API, to be called with each intermediate step in the model training. The functionality implemented in these callbacks across the different ML frameworks includes: -(1) Reporting intermediate model scores back to the Optuna trial using `optuna.trial.Trial.report `_, -(2) According to the results of `optuna.trial.Trial.should_prune `_, pruning the current model by raising `optuna.TrialPruned `_, and +(1) Reporting intermediate model scores back to the Optuna trial using :meth:`optuna.trial.Trial.report`, +(2) According to the results of :meth:`optuna.trial.Trial.should_prune`, pruning the current model by raising :class:`optuna.TrialPruned`, and (3) Reporting intermediate Optuna data such as the current trial number back to the framework, as done in :class:`~optuna_integration.MLflowCallback`. -For scikit-learn, an integrated :class:`~optuna_integration.OptunaSearchCV` estimator is available that combines scikit-learn BaseEstimator functionality with access to a class-level ``Study`` object. +For scikit-learn, an integrated :class:`~optuna_integration.OptunaSearchCV` estimator is available that combines scikit-learn BaseEstimator functionality with access to a class-level :class:`~optuna.study.Study` object. AllenNLP -------- diff --git a/optuna_integration/botorch/botorch.py b/optuna_integration/botorch/botorch.py index 2d468662f..74b8d77a9 100644 --- a/optuna_integration/botorch/botorch.py +++ b/optuna_integration/botorch/botorch.py @@ -108,27 +108,27 @@ def logei_candidates_func( Args: train_x: - Previous parameter configurations. A ``torch.Tensor`` of shape + Previous parameter configurations. A :class:`torch.Tensor` of shape ``(n_trials, n_params)``. ``n_trials`` is the number of already observed trials and ``n_params`` is the number of parameters. ``n_params`` may be larger than the actual number of parameters if categorical parameters are included in the search space, since these parameters are one-hot encoded. Values are not normalized. train_obj: - Previously observed objectives. A ``torch.Tensor`` of shape + Previously observed objectives. A :class:`torch.Tensor` of shape ``(n_trials, n_objectives)``. ``n_trials`` is identical to that of ``train_x``. ``n_objectives`` is the number of objectives. Observations are not normalized. train_con: - Objective constraints. A ``torch.Tensor`` of shape ``(n_trials, n_constraints)``. + Objective constraints. A class:`torch.Tensor` of shape ``(n_trials, n_constraints)``. ``n_trials`` is identical to that of ``train_x``. ``n_constraints`` is the number of constraints. A constraint is violated if strictly larger than 0. If no constraints are involved in the optimization, this argument will be :obj:`None`. bounds: - Search space bounds. A ``torch.Tensor`` of shape ``(2, n_params)``. ``n_params`` is + Search space bounds. A class:`torch.Tensor` of shape ``(2, n_params)``. ``n_params`` is identical to that of ``train_x``. The first and the second rows correspond to the lower and upper bounds for each parameter respectively. pending_x: - Pending parameter configurations. A ``torch.Tensor`` of shape + Pending parameter configurations. A class:`torch.Tensor` of shape ``(n_pending, n_params)``. ``n_pending`` is the number of the trials which are already suggested all their parameters but have not completed their evaluation, and ``n_params`` is identical to that of ``train_x``. @@ -215,27 +215,27 @@ def qei_candidates_func( Args: train_x: - Previous parameter configurations. A ``torch.Tensor`` of shape + Previous parameter configurations. A class:`torch.Tensor` of shape ``(n_trials, n_params)``. ``n_trials`` is the number of already observed trials and ``n_params`` is the number of parameters. ``n_params`` may be larger than the actual number of parameters if categorical parameters are included in the search space, since these parameters are one-hot encoded. Values are not normalized. train_obj: - Previously observed objectives. A ``torch.Tensor`` of shape + Previously observed objectives. A class:`torch.Tensor` of shape ``(n_trials, n_objectives)``. ``n_trials`` is identical to that of ``train_x``. ``n_objectives`` is the number of objectives. Observations are not normalized. train_con: - Objective constraints. A ``torch.Tensor`` of shape ``(n_trials, n_constraints)``. + Objective constraints. A class:`torch.Tensor` of shape ``(n_trials, n_constraints)``. ``n_trials`` is identical to that of ``train_x``. ``n_constraints`` is the number of constraints. A constraint is violated if strictly larger than 0. If no constraints are involved in the optimization, this argument will be :obj:`None`. bounds: - Search space bounds. A ``torch.Tensor`` of shape ``(2, n_params)``. ``n_params`` is + Search space bounds. A class:`torch.Tensor` of shape ``(2, n_params)``. ``n_params`` is identical to that of ``train_x``. The first and the second rows correspond to the lower and upper bounds for each parameter respectively. pending_x: - Pending parameter configurations. A ``torch.Tensor`` of shape + Pending parameter configurations. A class:`torch.Tensor` of shape ``(n_pending, n_params)``. ``n_pending`` is the number of the trials which are already suggested all their parameters but have not completed their evaluation, and ``n_params`` is identical to that of ``train_x``. @@ -882,8 +882,8 @@ class BoTorchSampler(BaseSampler): candidates_func: An optional function that suggests the next candidates. It must take the training data, the objectives, the constraints, the search space bounds and return the next - candidates. The arguments are of type ``torch.Tensor``. The return value must be a - ``torch.Tensor``. However, if ``constraints_func`` is omitted, constraints will be + candidates. The arguments are of type class:`torch.Tensor`. The return value must be a + class:`torch.Tensor`. However, if ``constraints_func`` is omitted, constraints will be :obj:`None`. For any constraints that failed to compute, the tensor will contain NaN. diff --git a/optuna_integration/comet/comet.py b/optuna_integration/comet/comet.py index fe7924655..a83161b52 100644 --- a/optuna_integration/comet/comet.py +++ b/optuna_integration/comet/comet.py @@ -22,7 +22,7 @@ class CometCallback: A callback for logging Optuna study trials to a Comet ML Experiment. Comet ML must be installed to run. - This callback is intended for use with Optuna's ``study.optimize()`` method. It ensures that + This callback is intended for use with Optuna's :meth:`~optuna.study.Study.optimize` method. It ensures that all trials from an Optuna study are logged to a single Comet Experiment, facilitating organized tracking of hyperparameter optimization. The callback supports both single and multi-objective optimization. @@ -42,7 +42,7 @@ class CometCallback: The workspace in Comet ML where the project resides. project_name: The name of the project in Comet ML where the experiment will be logged. - Defaults to ``general``. + Defaults to ``"general"``. metric_names: A list of the names of your objective metrics. diff --git a/optuna_integration/dask/dask.py b/optuna_integration/dask/dask.py index c8d02970f..cee2bb39e 100644 --- a/optuna_integration/dask/dask.py +++ b/optuna_integration/dask/dask.py @@ -455,7 +455,7 @@ class DaskStorage(BaseStorage): register: Whether or not to register this storage instance with the cluster scheduler. Most common usage of this storage class will not need to specify this argument. - Defaults to ``True``. + Defaults to :obj:`True`. """ diff --git a/optuna_integration/sklearn/sklearn.py b/optuna_integration/sklearn/sklearn.py index 89351637f..dab346efb 100644 --- a/optuna_integration/sklearn/sklearn.py +++ b/optuna_integration/sklearn/sklearn.py @@ -154,7 +154,7 @@ class _Objective: error_score: Value to assign to the score if an error occurs in fitting. If 'raise', the error is raised. If numeric, - ``sklearn.exceptions.FitFailedWarning`` is raised. This does not + :class:`sklearn.exceptions.FitFailedWarning` is raised. This does not affect the refit step, which will always raise the error. fit_params: @@ -420,8 +420,8 @@ class OptunaSearchCV(BaseEstimator): For integer, if ``estimator`` is a classifier and ``y`` is either binary or multiclass, - ``sklearn.model_selection.StratifiedKFold`` is used. otherwise, - ``sklearn.model_selection.KFold`` is used. + :class:`sklearn.model_selection.StratifiedKFold` is used. otherwise, + :class:`sklearn.model_selection.KFold` is used. enable_pruning: If :obj:`True`, pruning is performed in the case where the @@ -430,7 +430,7 @@ class OptunaSearchCV(BaseEstimator): error_score: Value to assign to the score if an error occurs in fitting. If 'raise', the error is raised. If numeric, - ``sklearn.exceptions.FitFailedWarning`` is raised. This does not + :class:`sklearn.exceptions.FitFailedWarning` is raised. This does not affect the refit step, which will always raise the error. max_iter: @@ -457,9 +457,9 @@ class OptunaSearchCV(BaseEstimator): random_state: Seed of the pseudo random number generator. If int, this is the seed used by the random number generator. If - ``numpy.random.RandomState`` object, this is the random number + :class:`numpy.random.RandomState` object, this is the random number generator. If :obj:`None`, the global random state from - ``numpy.random`` is used. + :mod:`numpy.random` is used. refit: If :obj:`True`, refit the estimator with the best found @@ -648,8 +648,7 @@ def inverse_transform( This is available only if the underlying estimator supports ``inverse_transform`` and ``refit`` is set to :obj:`True`. - Please check the following to know more about ``inverse_transform``: - https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.FunctionTransformer.html#sklearn.preprocessing.FunctionTransformer.inverse_transform + Please check the following to know more about :meth:`sklearn.preprocessing.FunctionTransformer.inverse_transform`. """ self._check_is_fitted() @@ -716,8 +715,7 @@ def transform(self, X: TwoDimArrayLikeType, *args: Any, **kwargs: Any) -> TwoDim This is available only if the underlying estimator supports ``transform`` and ``refit`` is set to :obj:`True`. - Please check the following to know more about ``transform``: - https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.FunctionTransformer.html#sklearn.preprocessing.FunctionTransformer.transform + Please check the following to know more about :meth:`sklearn.preprocessing.FunctionTransformer.transform` """ self._check_is_fitted() diff --git a/optuna_integration/tensorboard/tensorboard.py b/optuna_integration/tensorboard/tensorboard.py index 47c921a6e..a856da28c 100644 --- a/optuna_integration/tensorboard/tensorboard.py +++ b/optuna_integration/tensorboard/tensorboard.py @@ -30,7 +30,7 @@ class TensorBoardCallback: Directory to store TensorBoard logs. metric_name: Name of the metric. Since the metric itself is just a number, - `metric_name` can be used to give it a name. So you know later + ``metric_name`` can be used to give it a name. So you know later if it was roc-auc or accuracy. """ diff --git a/optuna_integration/wandb/wandb.py b/optuna_integration/wandb/wandb.py index 5a639e8ee..b7235944d 100644 --- a/optuna_integration/wandb/wandb.py +++ b/optuna_integration/wandb/wandb.py @@ -37,7 +37,7 @@ class WeightsAndBiasesCallback: .. note:: Users who want to run multiple Optuna studies within the same process should call ``wandb.finish()`` between subsequent calls to - ``study.optimize()``. Calling ``wandb.finish()`` is not necessary + :meth:`~optuna.study.Study.optimize`. Calling ``wandb.finish()`` is not necessary if you are running one Optuna study per process. .. note:: From 3e814ba3b5b3eb4fbf6d7d257cc84fa194eed2e3 Mon Sep 17 00:00:00 2001 From: Kento Nozawa Date: Mon, 27 Jan 2025 17:05:18 +0900 Subject: [PATCH 2/5] Apply suggestions from code review Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- optuna_integration/botorch/botorch.py | 20 ++++++++++---------- optuna_integration/wandb/wandb.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/optuna_integration/botorch/botorch.py b/optuna_integration/botorch/botorch.py index 74b8d77a9..6923e7e5d 100644 --- a/optuna_integration/botorch/botorch.py +++ b/optuna_integration/botorch/botorch.py @@ -119,16 +119,16 @@ def logei_candidates_func( ``(n_trials, n_objectives)``. ``n_trials`` is identical to that of ``train_x``. ``n_objectives`` is the number of objectives. Observations are not normalized. train_con: - Objective constraints. A class:`torch.Tensor` of shape ``(n_trials, n_constraints)``. + Objective constraints. A :class:`torch.Tensor` of shape ``(n_trials, n_constraints)``. ``n_trials`` is identical to that of ``train_x``. ``n_constraints`` is the number of constraints. A constraint is violated if strictly larger than 0. If no constraints are involved in the optimization, this argument will be :obj:`None`. bounds: - Search space bounds. A class:`torch.Tensor` of shape ``(2, n_params)``. ``n_params`` is + Search space bounds. A :class:`torch.Tensor` of shape ``(2, n_params)``. ``n_params`` is identical to that of ``train_x``. The first and the second rows correspond to the lower and upper bounds for each parameter respectively. pending_x: - Pending parameter configurations. A class:`torch.Tensor` of shape + Pending parameter configurations. A :class:`torch.Tensor` of shape ``(n_pending, n_params)``. ``n_pending`` is the number of the trials which are already suggested all their parameters but have not completed their evaluation, and ``n_params`` is identical to that of ``train_x``. @@ -215,27 +215,27 @@ def qei_candidates_func( Args: train_x: - Previous parameter configurations. A class:`torch.Tensor` of shape + Previous parameter configurations. A :class:`torch.Tensor` of shape ``(n_trials, n_params)``. ``n_trials`` is the number of already observed trials and ``n_params`` is the number of parameters. ``n_params`` may be larger than the actual number of parameters if categorical parameters are included in the search space, since these parameters are one-hot encoded. Values are not normalized. train_obj: - Previously observed objectives. A class:`torch.Tensor` of shape + Previously observed objectives. A :class:`torch.Tensor` of shape ``(n_trials, n_objectives)``. ``n_trials`` is identical to that of ``train_x``. ``n_objectives`` is the number of objectives. Observations are not normalized. train_con: - Objective constraints. A class:`torch.Tensor` of shape ``(n_trials, n_constraints)``. + Objective constraints. A :class:`torch.Tensor` of shape ``(n_trials, n_constraints)``. ``n_trials`` is identical to that of ``train_x``. ``n_constraints`` is the number of constraints. A constraint is violated if strictly larger than 0. If no constraints are involved in the optimization, this argument will be :obj:`None`. bounds: - Search space bounds. A class:`torch.Tensor` of shape ``(2, n_params)``. ``n_params`` is + Search space bounds. A :class:`torch.Tensor` of shape ``(2, n_params)``. ``n_params`` is identical to that of ``train_x``. The first and the second rows correspond to the lower and upper bounds for each parameter respectively. pending_x: - Pending parameter configurations. A class:`torch.Tensor` of shape + Pending parameter configurations. A :class:`torch.Tensor` of shape ``(n_pending, n_params)``. ``n_pending`` is the number of the trials which are already suggested all their parameters but have not completed their evaluation, and ``n_params`` is identical to that of ``train_x``. @@ -882,8 +882,8 @@ class BoTorchSampler(BaseSampler): candidates_func: An optional function that suggests the next candidates. It must take the training data, the objectives, the constraints, the search space bounds and return the next - candidates. The arguments are of type class:`torch.Tensor`. The return value must be a - class:`torch.Tensor`. However, if ``constraints_func`` is omitted, constraints will be + candidates. The arguments are of type :class:`torch.Tensor`. The return value must be a + :class:`torch.Tensor`. However, if ``constraints_func`` is omitted, constraints will be :obj:`None`. For any constraints that failed to compute, the tensor will contain NaN. diff --git a/optuna_integration/wandb/wandb.py b/optuna_integration/wandb/wandb.py index b7235944d..779e8ae69 100644 --- a/optuna_integration/wandb/wandb.py +++ b/optuna_integration/wandb/wandb.py @@ -37,7 +37,7 @@ class WeightsAndBiasesCallback: .. note:: Users who want to run multiple Optuna studies within the same process should call ``wandb.finish()`` between subsequent calls to - :meth:`~optuna.study.Study.optimize`. Calling ``wandb.finish()`` is not necessary + :meth:`optuna.study.Study.optimize`. Calling ``wandb.finish()`` is not necessary if you are running one Optuna study per process. .. note:: From e2bdf74ac03b18f1ebf132572cf0237e0038afd4 Mon Sep 17 00:00:00 2001 From: Kento Nozawa Date: Mon, 27 Jan 2025 17:05:43 +0900 Subject: [PATCH 3/5] Update optuna_integration/comet/comet.py Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> --- optuna_integration/comet/comet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_integration/comet/comet.py b/optuna_integration/comet/comet.py index a83161b52..530405113 100644 --- a/optuna_integration/comet/comet.py +++ b/optuna_integration/comet/comet.py @@ -22,7 +22,7 @@ class CometCallback: A callback for logging Optuna study trials to a Comet ML Experiment. Comet ML must be installed to run. - This callback is intended for use with Optuna's :meth:`~optuna.study.Study.optimize` method. It ensures that + This callback is intended for use with :meth:`optuna.study.Study.optimize` method. It ensures that all trials from an Optuna study are logged to a single Comet Experiment, facilitating organized tracking of hyperparameter optimization. The callback supports both single and multi-objective optimization. From ffba3e0b3a7b23f5df27b8b68057ad7b8a1bf576 Mon Sep 17 00:00:00 2001 From: Kento Nozawa Date: Wed, 29 Jan 2025 21:33:40 +0900 Subject: [PATCH 4/5] Link to torch.device page. --- optuna_integration/botorch/botorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_integration/botorch/botorch.py b/optuna_integration/botorch/botorch.py index 74b8d77a9..ad0f97b5b 100644 --- a/optuna_integration/botorch/botorch.py +++ b/optuna_integration/botorch/botorch.py @@ -926,7 +926,7 @@ class BoTorchSampler(BaseSampler): seed: Seed for random number generator. device: - A ``torch.device`` to store input and output data of BoTorch. Please set a CUDA device + A :class:`torch.device` to store input and output data of BoTorch. Please set a CUDA device if you fasten sampling. """ From cc5467df4edb2101be5c99181c3746caac591fee Mon Sep 17 00:00:00 2001 From: Kento Nozawa Date: Thu, 30 Jan 2025 01:55:31 +0900 Subject: [PATCH 5/5] few more minor updates --- optuna_integration/pytorch_lightning/pytorch_lightning.py | 4 ++-- optuna_integration/shap/shap.py | 3 +-- optuna_integration/sklearn/sklearn.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/optuna_integration/pytorch_lightning/pytorch_lightning.py b/optuna_integration/pytorch_lightning/pytorch_lightning.py index b2cf3a922..c2b687725 100644 --- a/optuna_integration/pytorch_lightning/pytorch_lightning.py +++ b/optuna_integration/pytorch_lightning/pytorch_lightning.py @@ -52,7 +52,7 @@ class PyTorchLightningPruningCallback(Callback): .. note:: If you would like to use PyTorchLightningPruningCallback in a distributed training - environment, you need to evoke `PyTorchLightningPruningCallback.check_pruned()` + environment, you need to evoke ``PyTorchLightningPruningCallback.check_pruned()`` manually so that :class:`~optuna.exceptions.TrialPruned` is properly handled. """ @@ -148,7 +148,7 @@ def check_pruned(self) -> None: """Raise :class:`optuna.TrialPruned` manually if pruned. Currently, ``intermediate_values`` are not properly propagated between processes due to - storage cache. Therefore, necessary information is kept in trial_system_attrs when the + storage cache. Therefore, necessary information is kept in ``trial.system_attrs`` when the trial runs in a distributed situation. Please call this method right after calling ``lightning.pytorch.Trainer.fit()``. If a callback doesn't have any backend storage for DDP, this method does nothing. diff --git a/optuna_integration/shap/shap.py b/optuna_integration/shap/shap.py index d20d5d135..cccbdb2cf 100644 --- a/optuna_integration/shap/shap.py +++ b/optuna_integration/shap/shap.py @@ -34,8 +34,7 @@ class ShapleyImportanceEvaluator(BaseImportanceEvaluator): This evaluator requires the `sklearn `_ Python package and `SHAP `_. - The model for the SHAP calculation is based on `sklearn.ensemble.RandomForestClassifier - `_. + The model for the SHAP calculation is based on :class:`sklearn.ensemble.RandomForestClassifier`. Args: n_trees: diff --git a/optuna_integration/sklearn/sklearn.py b/optuna_integration/sklearn/sklearn.py index dab346efb..6818ba07a 100644 --- a/optuna_integration/sklearn/sklearn.py +++ b/optuna_integration/sklearn/sklearn.py @@ -153,7 +153,7 @@ class _Objective: error_score: Value to assign to the score if an error occurs in fitting. If - 'raise', the error is raised. If numeric, + ``"raise"``, the error is raised. If numeric, :class:`sklearn.exceptions.FitFailedWarning` is raised. This does not affect the refit step, which will always raise the error. @@ -429,7 +429,7 @@ class OptunaSearchCV(BaseEstimator): error_score: Value to assign to the score if an error occurs in fitting. If - 'raise', the error is raised. If numeric, + ``"raise"``, the error is raised. If numeric, :class:`sklearn.exceptions.FitFailedWarning` is raised. This does not affect the refit step, which will always raise the error.