From 416ecd5a8de1b2b9225ded3c919cb0d40ec0d9bd Mon Sep 17 00:00:00 2001 From: Miguel Trejo Marrufo <49818188+TremaMiguel@users.noreply.github.com> Date: Thu, 21 Apr 2022 22:41:49 -0500 Subject: [PATCH] [python-package] remove 'fobj' in favor of passing custom objective function in params (fixes #3244) (#5052) * feat: support custom metrics in params * feat: support objective in params * test: custom objective and metric * fix: imports are incorrectly sorted * feat: convert eval metrics str and set to list * feat: convert single callable eval_metric to list * test: single callable objective in params Signed-off-by: Miguel Trejo * feat: callable fobj in basic cv function Signed-off-by: Miguel Trejo * test: cv support objective callable Signed-off-by: Miguel Trejo * fix: assert in cv_res Signed-off-by: Miguel Trejo * docs: objective callable in params Signed-off-by: Miguel Trejo * recover test_boost_from_average_with_single_leaf_trees Signed-off-by: Miguel Trejo * linters fail Signed-off-by: Miguel Trejo * remove metrics helper functions Signed-off-by: Miguel Trejo * feat: choose objective through _choose_param_values Signed-off-by: Miguel Trejo * test: test objective through _choose_param_values Signed-off-by: Miguel Trejo * test: test objective is callabe in train Signed-off-by: Miguel Trejo * test: parametrize choose_param_value with objective aliases Signed-off-by: Miguel Trejo * test: cv booster metric is none Signed-off-by: Miguel Trejo * fix: if string and callable choose callable Signed-off-by: Miguel Trejo * test train uses custom objective metrics Signed-off-by: Miguel Trejo * test: cv uses custom objective metrics Signed-off-by: Miguel Trejo * refactor: remove fobj parameter in train and cv Signed-off-by: Miguel Trejo * refactor: objective through params in sklearn API Signed-off-by: Miguel Trejo * custom objective function in advanced_example Signed-off-by: Miguel Trejo * fix whitespackes lint * objective is none not a particular case for predict method Signed-off-by: Miguel Trejo * replace scipy.expit with custom implementation Signed-off-by: Miguel Trejo * test: set num_boost_round value to 20 Signed-off-by: Miguel Trejo * fix: custom objective default_value is none Signed-off-by: Miguel Trejo * refactor: remove self._fobj Signed-off-by: Miguel Trejo * custom_objective default value is None Signed-off-by: Miguel Trejo * refactor: variables name reference dummy_obj Signed-off-by: Miguel Trejo * linter errors * fix: process objective parameter when calling predict Signed-off-by: Miguel Trejo * linter errors * fix: objective is None during predict call Signed-off-by: Miguel Trejo --- examples/python-guide/advanced_example.py | 15 +- python-package/lightgbm/basic.py | 6 +- python-package/lightgbm/engine.py | 128 +++++++-------- python-package/lightgbm/sklearn.py | 8 +- tests/python_package_test/test_basic.py | 32 +++- tests/python_package_test/test_engine.py | 181 +++++++++++++++++----- tests/python_package_test/utils.py | 15 ++ 7 files changed, 270 insertions(+), 115 deletions(-) diff --git a/examples/python-guide/advanced_example.py b/examples/python-guide/advanced_example.py index 54b62cdb1563..6c2a42ce2bf6 100644 --- a/examples/python-guide/advanced_example.py +++ b/examples/python-guide/advanced_example.py @@ -1,4 +1,5 @@ # coding: utf-8 +import copy import json import pickle from pathlib import Path @@ -159,11 +160,14 @@ def binary_error(preds, train_data): return 'error', np.mean(labels != (preds > 0.5)), False -gbm = lgb.train(params, +# Pass custom objective function through params +params_custom_obj = copy.deepcopy(params) +params_custom_obj['objective'] = loglikelihood + +gbm = lgb.train(params_custom_obj, lgb_train, num_boost_round=10, init_model=gbm, - fobj=loglikelihood, feval=binary_error, valid_sets=lgb_eval) @@ -183,11 +187,14 @@ def accuracy(preds, train_data): return 'accuracy', np.mean(labels == (preds > 0.5)), True -gbm = lgb.train(params, +# Pass custom objective function through params +params_custom_obj = copy.deepcopy(params) +params_custom_obj['objective'] = loglikelihood + +gbm = lgb.train(params_custom_obj, lgb_train, num_boost_round=10, init_model=gbm, - fobj=loglikelihood, feval=[binary_error, accuracy], valid_sets=lgb_eval) diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index f136142b61e5..e86ecbbcdf1a 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -3185,7 +3185,7 @@ def eval(self, data, name, feval=None): preds : numpy 1-D array or numpy 2-D array (for multi-class task) The predicted values. For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes]. - If ``fobj`` is specified, predicted values are returned before any transformation, + If custom objective function is used, predicted values are returned before any transformation, e.g. they are raw margin instead of probability of positive class for binary task in this case. eval_data : Dataset A ``Dataset`` to evaluate. @@ -3231,7 +3231,7 @@ def eval_train(self, feval=None): preds : numpy 1-D array or numpy 2-D array (for multi-class task) The predicted values. For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes]. - If ``fobj`` is specified, predicted values are returned before any transformation, + If custom objective function is used, predicted values are returned before any transformation, e.g. they are raw margin instead of probability of positive class for binary task in this case. eval_data : Dataset The training dataset. @@ -3262,7 +3262,7 @@ def eval_valid(self, feval=None): preds : numpy 1-D array or numpy 2-D array (for multi-class task) The predicted values. For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes]. - If ``fobj`` is specified, predicted values are returned before any transformation, + If custom objective function is used, predicted values are returned before any transformation, e.g. they are raw margin instead of probability of positive class for binary task in this case. eval_data : Dataset The validation dataset. diff --git a/python-package/lightgbm/engine.py b/python-package/lightgbm/engine.py index 61ef4494648f..71a9a115d342 100644 --- a/python-package/lightgbm/engine.py +++ b/python-package/lightgbm/engine.py @@ -12,10 +12,6 @@ from .basic import Booster, Dataset, LightGBMError, _choose_param_value, _ConfigAliases, _InnerPredictor, _log_warning from .compat import SKLEARN_INSTALLED, _LGBMGroupKFold, _LGBMStratifiedKFold -_LGBM_CustomObjectiveFunction = Callable[ - [np.ndarray, Dataset], - Tuple[np.ndarray, np.ndarray] -] _LGBM_CustomMetricFunction = Callable[ [np.ndarray, Dataset], Tuple[str, float, bool] @@ -28,7 +24,6 @@ def train( num_boost_round: int = 100, valid_sets: Optional[List[Dataset]] = None, valid_names: Optional[List[str]] = None, - fobj: Optional[_LGBM_CustomObjectiveFunction] = None, feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None, init_model: Optional[Union[str, Path, Booster]] = None, feature_name: Union[List[str], str] = 'auto', @@ -41,7 +36,8 @@ def train( Parameters ---------- params : dict - Parameters for training. + Parameters for training. Values passed through ``params`` take precedence over those + supplied via arguments. train_set : Dataset Data to be trained on. num_boost_round : int, optional (default=100) @@ -50,27 +46,6 @@ def train( List of data to be evaluated on during training. valid_names : list of str, or None, optional (default=None) Names of ``valid_sets``. - fobj : callable or None, optional (default=None) - Customized objective function. - Should accept two parameters: preds, train_data, - and return (grad, hess). - - preds : numpy 1-D array or numpy 2-D array (for multi-class task) - The predicted values. - Predicted values are returned before any transformation, - e.g. they are raw margin instead of probability of positive class for binary task. - train_data : Dataset - The training dataset. - grad : numpy 1-D array or numpy 2-D array (for multi-class task) - The value of the first order derivative (gradient) of the loss - with respect to the elements of preds for each sample point. - hess : numpy 1-D array or numpy 2-D array (for multi-class task) - The value of the second order derivative (Hessian) of the loss - with respect to the elements of preds for each sample point. - - For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes], - and grad and hess should be returned in the same format. - feval : callable, list of callable, or None, optional (default=None) Customized evaluation function. Each evaluation function should accept two parameters: preds, eval_data, @@ -79,7 +54,7 @@ def train( preds : numpy 1-D array or numpy 2-D array (for multi-class task) The predicted values. For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes]. - If ``fobj`` is specified, predicted values are returned before any transformation, + If custom objective function is used, predicted values are returned before any transformation, e.g. they are raw margin instead of probability of positive class for binary task in this case. eval_data : Dataset A ``Dataset`` to evaluate. @@ -118,6 +93,27 @@ def train( List of callback functions that are applied at each iteration. See Callbacks in Python API for more information. + Note + ---- + A custom objective function can be provided for the ``objective`` parameter. + It should accept two parameters: preds, train_data and return (grad, hess). + + preds : numpy 1-D array or numpy 2-D array (for multi-class task) + The predicted values. + Predicted values are returned before any transformation, + e.g. they are raw margin instead of probability of positive class for binary task. + train_data : Dataset + The training dataset. + grad : numpy 1-D array or numpy 2-D array (for multi-class task) + The value of the first order derivative (gradient) of the loss + with respect to the elements of preds for each sample point. + hess : numpy 1-D array or numpy 2-D array (for multi-class task) + The value of the second order derivative (Hessian) of the loss + with respect to the elements of preds for each sample point. + + For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes], + and grad and hess should be returned in the same format. + Returns ------- booster : Booster @@ -125,10 +121,15 @@ def train( """ # create predictor first params = copy.deepcopy(params) - if fobj is not None: - for obj_alias in _ConfigAliases.get("objective"): - params.pop(obj_alias, None) - params['objective'] = 'none' + params = _choose_param_value( + main_param_name='objective', + params=params, + default_value=None + ) + fobj = None + if callable(params["objective"]): + fobj = params["objective"] + params["objective"] = 'none' for alias in _ConfigAliases.get("num_iterations"): if alias in params: num_boost_round = params.pop(alias) @@ -374,7 +375,7 @@ def _agg_cv_result(raw_results): def cv(params, train_set, num_boost_round=100, folds=None, nfold=5, stratified=True, shuffle=True, - metrics=None, fobj=None, feval=None, init_model=None, + metrics=None, feval=None, init_model=None, feature_name='auto', categorical_feature='auto', fpreproc=None, seed=0, callbacks=None, eval_train_metric=False, return_cvbooster=False): @@ -383,7 +384,8 @@ def cv(params, train_set, num_boost_round=100, Parameters ---------- params : dict - Parameters for Booster. + Parameters for training. Values passed through ``params`` take precedence over those + supplied via arguments. train_set : Dataset Data to be trained on. num_boost_round : int, optional (default=100) @@ -403,27 +405,6 @@ def cv(params, train_set, num_boost_round=100, metrics : str, list of str, or None, optional (default=None) Evaluation metrics to be monitored while CV. If not None, the metric in ``params`` will be overridden. - fobj : callable or None, optional (default=None) - Customized objective function. - Should accept two parameters: preds, train_data, - and return (grad, hess). - - preds : numpy 1-D array or numpy 2-D array (for multi-class task) - The predicted values. - Predicted values are returned before any transformation, - e.g. they are raw margin instead of probability of positive class for binary task. - train_data : Dataset - The training dataset. - grad : numpy 1-D array or numpy 2-D array (for multi-class task) - The value of the first order derivative (gradient) of the loss - with respect to the elements of preds for each sample point. - hess : numpy 1-D array or numpy 2-D array (for multi-class task) - The value of the second order derivative (Hessian) of the loss - with respect to the elements of preds for each sample point. - - For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes], - and grad and hess should be returned in the same format. - feval : callable, list of callable, or None, optional (default=None) Customized evaluation function. Each evaluation function should accept two parameters: preds, eval_data, @@ -432,7 +413,7 @@ def cv(params, train_set, num_boost_round=100, preds : numpy 1-D array or numpy 2-D array (for multi-class task) The predicted values. For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes]. - If ``fobj`` is specified, predicted values are returned before any transformation, + If custom objective function is used, predicted values are returned before any transformation, e.g. they are raw margin instead of probability of positive class for binary task in this case. eval_data : Dataset A ``Dataset`` to evaluate. @@ -474,6 +455,27 @@ def cv(params, train_set, num_boost_round=100, return_cvbooster : bool, optional (default=False) Whether to return Booster models trained on each fold through ``CVBooster``. + Note + ---- + A custom objective function can be provided for the ``objective`` parameter. + It should accept two parameters: preds, train_data and return (grad, hess). + + preds : numpy 1-D array or numpy 2-D array (for multi-class task) + The predicted values. + Predicted values are returned before any transformation, + e.g. they are raw margin instead of probability of positive class for binary task. + train_data : Dataset + The training dataset. + grad : numpy 1-D array or numpy 2-D array (for multi-class task) + The value of the first order derivative (gradient) of the loss + with respect to the elements of preds for each sample point. + hess : numpy 1-D array or numpy 2-D array (for multi-class task) + The value of the second order derivative (Hessian) of the loss + with respect to the elements of preds for each sample point. + + For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes], + and grad and hess should be returned in the same format. + Returns ------- eval_hist : dict @@ -486,12 +488,16 @@ def cv(params, train_set, num_boost_round=100, """ if not isinstance(train_set, Dataset): raise TypeError("Training only accepts Dataset object") - params = copy.deepcopy(params) - if fobj is not None: - for obj_alias in _ConfigAliases.get("objective"): - params.pop(obj_alias, None) - params['objective'] = 'none' + params = _choose_param_value( + main_param_name='objective', + params=params, + default_value=None + ) + fobj = None + if callable(params["objective"]): + fobj = params["objective"] + params["objective"] = 'none' for alias in _ConfigAliases.get("num_iterations"): if alias in params: _log_warning(f"Found '{alias}' in params. Will use it instead of 'num_boost_round' argument") diff --git a/python-package/lightgbm/sklearn.py b/python-package/lightgbm/sklearn.py index 7ebba0bc962c..a1301f98323f 100644 --- a/python-package/lightgbm/sklearn.py +++ b/python-package/lightgbm/sklearn.py @@ -596,11 +596,10 @@ def _process_params(self, stage: str) -> Dict[str, Any]: raise ValueError("Unknown LGBMModel type.") if callable(self._objective): if stage == "fit": - self._fobj = _ObjectiveFunctionWrapper(self._objective) - params['objective'] = 'None' # objective = nullptr for unknown objective + params['objective'] = _ObjectiveFunctionWrapper(self._objective) + else: + params['objective'] = 'None' else: - if stage == "fit": - self._fobj = None params['objective'] = self._objective params.pop('importance_type', None) @@ -756,7 +755,6 @@ def _get_meta_data(collection, name, i): num_boost_round=self.n_estimators, valid_sets=valid_sets, valid_names=eval_names, - fobj=self._fobj, feval=eval_metrics_callable, init_model=init_model, feature_name=feature_name, diff --git a/tests/python_package_test/test_basic.py b/tests/python_package_test/test_basic.py index 09946c93178b..4d6c367d8150 100644 --- a/tests/python_package_test/test_basic.py +++ b/tests/python_package_test/test_basic.py @@ -14,7 +14,7 @@ import lightgbm as lgb from lightgbm.compat import PANDAS_INSTALLED, pd_DataFrame, pd_Series -from .utils import load_breast_cancer +from .utils import dummy_obj, load_breast_cancer, mse_obj def test_basic(tmp_path): @@ -513,6 +513,36 @@ def test_choose_param_value(): assert original_params == expected_params +@pytest.mark.parametrize("objective_alias", lgb.basic._ConfigAliases.get("objective")) +def test_choose_param_value_objective(objective_alias): + # If callable is found in objective + params = {objective_alias: dummy_obj} + params = lgb.basic._choose_param_value( + main_param_name="objective", + params=params, + default_value=None + ) + assert params['objective'] == dummy_obj + + # Value in params should be preferred to the default_value passed from keyword arguments + params = {objective_alias: dummy_obj} + params = lgb.basic._choose_param_value( + main_param_name="objective", + params=params, + default_value=mse_obj + ) + assert params['objective'] == dummy_obj + + # None of objective or its aliases in params, but default_value is callable. + params = {} + params = lgb.basic._choose_param_value( + main_param_name="objective", + params=params, + default_value=mse_obj + ) + assert params['objective'] == mse_obj + + @pytest.mark.parametrize('collection', ['1d_np', '2d_np', 'pd_float', 'pd_str', '1d_list', '2d_list']) @pytest.mark.parametrize('dtype', [np.float32, np.float64]) def test_list_to_1d_numpy(collection, dtype): diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py index 1b202b413a2b..df840a768539 100644 --- a/tests/python_package_test/test_engine.py +++ b/tests/python_package_test/test_engine.py @@ -19,14 +19,18 @@ import lightgbm as lgb -from .utils import (load_boston, load_breast_cancer, load_digits, load_iris, make_synthetic_regression, - sklearn_multiclass_custom_objective, softmax) +from .utils import (dummy_obj, load_boston, load_breast_cancer, load_digits, load_iris, logistic_sigmoid, + make_synthetic_regression, mse_obj, sklearn_multiclass_custom_objective, softmax) decreasing_generator = itertools.count(0, -1) -def dummy_obj(preds, train_data): - return np.ones(preds.shape), np.ones(preds.shape) +def logloss_obj(preds, train_data): + y_true = train_data.get_label() + y_pred = logistic_sigmoid(preds) + grad = y_pred - y_true + hess = y_pred * (1.0 - y_pred) + return grad, hess def multi_logloss(y_true, y_pred): @@ -1882,7 +1886,7 @@ def test_metrics(): lgb_valid = lgb.Dataset(X_test, y_test, reference=lgb_train) evals_result = {} - params_verbose = {'verbose': -1} + params_dummy_obj_verbose = {'verbose': -1, 'objective': dummy_obj} params_obj_verbose = {'objective': 'binary', 'verbose': -1} params_obj_metric_log_verbose = {'objective': 'binary', 'metric': 'binary_logloss', 'verbose': -1} params_obj_metric_err_verbose = {'objective': 'binary', 'metric': 'binary_error', 'verbose': -1} @@ -1891,11 +1895,11 @@ def test_metrics(): 'metric': ['binary_logloss', 'binary_error'], 'verbose': -1} params_obj_metric_none_verbose = {'objective': 'binary', 'metric': 'None', 'verbose': -1} - params_metric_log_verbose = {'metric': 'binary_logloss', 'verbose': -1} - params_metric_err_verbose = {'metric': 'binary_error', 'verbose': -1} - params_metric_inv_verbose = {'metric_types': 'invalid_metric', 'verbose': -1} - params_metric_multi_verbose = {'metric': ['binary_logloss', 'binary_error'], 'verbose': -1} - params_metric_none_verbose = {'metric': 'None', 'verbose': -1} + params_dummy_obj_metric_log_verbose = {'objective': dummy_obj, 'metric': 'binary_logloss', 'verbose': -1} + params_dummy_obj_metric_err_verbose = {'metric': 'binary_error', 'objective': dummy_obj, 'verbose': -1} + params_dummy_obj_metric_inv_verbose = {'objective': dummy_obj, 'metric_types': 'invalid_metric', 'verbose': -1} + params_dummy_obj_metric_multi_verbose = {'objective': dummy_obj, 'metric': ['binary_logloss', 'binary_error'], 'verbose': -1} + params_dummy_obj_metric_none_verbose = {'objective': dummy_obj, 'metric': 'None', 'verbose': -1} def get_cv_result(params=params_obj_verbose, **kwargs): return lgb.cv(params, lgb_train, num_boost_round=2, **kwargs) @@ -1959,32 +1963,32 @@ def train_booster(params=params_obj_verbose, **kwargs): # fobj, no feval # no default metric - res = get_cv_result(params=params_verbose, fobj=dummy_obj) + res = get_cv_result(params=params_dummy_obj_verbose) assert len(res) == 0 # metric in params - res = get_cv_result(params=params_metric_err_verbose, fobj=dummy_obj) + res = get_cv_result(params=params_dummy_obj_metric_err_verbose) assert len(res) == 2 assert 'valid binary_error-mean' in res # metric in args - res = get_cv_result(params=params_verbose, fobj=dummy_obj, metrics='binary_error') + res = get_cv_result(params=params_dummy_obj_verbose, metrics='binary_error') assert len(res) == 2 assert 'valid binary_error-mean' in res # metric in args overwrites its' alias in params - res = get_cv_result(params=params_metric_inv_verbose, fobj=dummy_obj, metrics='binary_error') + res = get_cv_result(params=params_dummy_obj_metric_inv_verbose, metrics='binary_error') assert len(res) == 2 assert 'valid binary_error-mean' in res # multiple metrics in params - res = get_cv_result(params=params_metric_multi_verbose, fobj=dummy_obj) + res = get_cv_result(params=params_dummy_obj_metric_multi_verbose) assert len(res) == 4 assert 'valid binary_logloss-mean' in res assert 'valid binary_error-mean' in res # multiple metrics in args - res = get_cv_result(params=params_verbose, fobj=dummy_obj, + res = get_cv_result(params=params_dummy_obj_verbose, metrics=['binary_logloss', 'binary_error']) assert len(res) == 4 assert 'valid binary_logloss-mean' in res @@ -2042,39 +2046,39 @@ def train_booster(params=params_obj_verbose, **kwargs): # fobj, feval # no default metric, only custom one - res = get_cv_result(params=params_verbose, fobj=dummy_obj, feval=constant_metric) + res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric) assert len(res) == 2 assert 'valid error-mean' in res # metric in params with custom one - res = get_cv_result(params=params_metric_err_verbose, fobj=dummy_obj, feval=constant_metric) + res = get_cv_result(params=params_dummy_obj_metric_err_verbose, feval=constant_metric) assert len(res) == 4 assert 'valid binary_error-mean' in res assert 'valid error-mean' in res # metric in args with custom one - res = get_cv_result(params=params_verbose, fobj=dummy_obj, + res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric, metrics='binary_error') assert len(res) == 4 assert 'valid binary_error-mean' in res assert 'valid error-mean' in res # metric in args overwrites one in params, custom one is evaluated too - res = get_cv_result(params=params_metric_inv_verbose, fobj=dummy_obj, + res = get_cv_result(params=params_dummy_obj_metric_inv_verbose, feval=constant_metric, metrics='binary_error') assert len(res) == 4 assert 'valid binary_error-mean' in res assert 'valid error-mean' in res # multiple metrics in params with custom one - res = get_cv_result(params=params_metric_multi_verbose, fobj=dummy_obj, feval=constant_metric) + res = get_cv_result(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric) assert len(res) == 6 assert 'valid binary_logloss-mean' in res assert 'valid binary_error-mean' in res assert 'valid error-mean' in res # multiple metrics in args with custom one - res = get_cv_result(params=params_verbose, fobj=dummy_obj, feval=constant_metric, + res = get_cv_result(params=params_dummy_obj_verbose, feval=constant_metric, metrics=['binary_logloss', 'binary_error']) assert len(res) == 6 assert 'valid binary_logloss-mean' in res @@ -2082,7 +2086,7 @@ def train_booster(params=params_obj_verbose, **kwargs): assert 'valid error-mean' in res # custom metric is evaluated despite 'None' is passed - res = get_cv_result(params=params_metric_none_verbose, fobj=dummy_obj, feval=constant_metric) + res = get_cv_result(params=params_dummy_obj_metric_none_verbose, feval=constant_metric) assert len(res) == 2 assert 'valid error-mean' in res @@ -2116,16 +2120,16 @@ def train_booster(params=params_obj_verbose, **kwargs): # fobj, no feval # no default metric - train_booster(params=params_verbose, fobj=dummy_obj) + train_booster(params=params_dummy_obj_verbose) assert len(evals_result) == 0 # metric in params - train_booster(params=params_metric_log_verbose, fobj=dummy_obj) + train_booster(params=params_dummy_obj_metric_log_verbose) assert len(evals_result['valid_0']) == 1 assert 'binary_logloss' in evals_result['valid_0'] # multiple metrics in params - train_booster(params=params_metric_multi_verbose, fobj=dummy_obj) + train_booster(params=params_dummy_obj_metric_multi_verbose) assert len(evals_result['valid_0']) == 2 assert 'binary_logloss' in evals_result['valid_0'] assert 'binary_error' in evals_result['valid_0'] @@ -2163,25 +2167,25 @@ def train_booster(params=params_obj_verbose, **kwargs): # fobj, feval # no default metric, only custom one - train_booster(params=params_verbose, fobj=dummy_obj, feval=constant_metric) + train_booster(params=params_dummy_obj_verbose, feval=constant_metric) assert len(evals_result['valid_0']) == 1 assert 'error' in evals_result['valid_0'] # metric in params with custom one - train_booster(params=params_metric_log_verbose, fobj=dummy_obj, feval=constant_metric) + train_booster(params=params_dummy_obj_metric_log_verbose, feval=constant_metric) assert len(evals_result['valid_0']) == 2 assert 'binary_logloss' in evals_result['valid_0'] assert 'error' in evals_result['valid_0'] # multiple metrics in params with custom one - train_booster(params=params_metric_multi_verbose, fobj=dummy_obj, feval=constant_metric) + train_booster(params=params_dummy_obj_metric_multi_verbose, feval=constant_metric) assert len(evals_result['valid_0']) == 3 assert 'binary_logloss' in evals_result['valid_0'] assert 'binary_error' in evals_result['valid_0'] assert 'error' in evals_result['valid_0'] # custom metric is evaluated despite 'None' is passed - train_booster(params=params_metric_none_verbose, fobj=dummy_obj, feval=constant_metric) + train_booster(params=params_dummy_obj_metric_none_verbose, feval=constant_metric) assert len(evals_result) == 1 assert 'error' in evals_result['valid_0'] @@ -2190,9 +2194,12 @@ def train_booster(params=params_obj_verbose, **kwargs): obj_multi_aliases = ['multiclass', 'softmax', 'multiclassova', 'multiclass_ova', 'ova', 'ovr'] for obj_multi_alias in obj_multi_aliases: + # Custom objective replaces multiclass params_obj_class_3_verbose = {'objective': obj_multi_alias, 'num_class': 3, 'verbose': -1} - params_obj_class_1_verbose = {'objective': obj_multi_alias, 'num_class': 1, 'verbose': -1} + params_dummy_obj_class_3_verbose = {'objective': dummy_obj, 'num_class': 3, 'verbose': -1} + params_dummy_obj_class_1_verbose = {'objective': dummy_obj, 'num_class': 1, 'verbose': -1} params_obj_verbose = {'objective': obj_multi_alias, 'verbose': -1} + params_dummy_obj_verbose = {'objective': dummy_obj, 'verbose': -1} # multiclass default metric res = get_cv_result(params_obj_class_3_verbose) assert len(res) == 2 @@ -2203,20 +2210,20 @@ def train_booster(params=params_obj_verbose, **kwargs): assert 'valid multi_logloss-mean' in res assert 'valid error-mean' in res # multiclass metric alias with custom one for custom objective - res = get_cv_result(params_obj_class_3_verbose, fobj=dummy_obj, feval=constant_metric) + res = get_cv_result(params_dummy_obj_class_3_verbose, feval=constant_metric) assert len(res) == 2 assert 'valid error-mean' in res # no metric for invalid class_num - res = get_cv_result(params_obj_class_1_verbose, fobj=dummy_obj) + res = get_cv_result(params_dummy_obj_class_1_verbose) assert len(res) == 0 # custom metric for invalid class_num - res = get_cv_result(params_obj_class_1_verbose, fobj=dummy_obj, feval=constant_metric) + res = get_cv_result(params_dummy_obj_class_1_verbose, feval=constant_metric) assert len(res) == 2 assert 'valid error-mean' in res # multiclass metric alias with custom one with invalid class_num with pytest.raises(lgb.basic.LightGBMError): - get_cv_result(params_obj_class_1_verbose, metrics=obj_multi_alias, - fobj=dummy_obj, feval=constant_metric) + get_cv_result(params_dummy_obj_class_1_verbose, metrics=obj_multi_alias, + feval=constant_metric) # multiclass default metric without num_class with pytest.raises(lgb.basic.LightGBMError): get_cv_result(params_obj_verbose) @@ -2237,20 +2244,20 @@ def train_booster(params=params_obj_verbose, **kwargs): with pytest.raises(lgb.basic.LightGBMError): get_cv_result(params_class_3_verbose) # no metric with non-default num_class for custom objective - res = get_cv_result(params_class_3_verbose, fobj=dummy_obj) + res = get_cv_result(params_dummy_obj_class_3_verbose) assert len(res) == 0 for metric_multi_alias in obj_multi_aliases + ['multi_logloss']: # multiclass metric alias for custom objective - res = get_cv_result(params_class_3_verbose, metrics=metric_multi_alias, fobj=dummy_obj) + res = get_cv_result(params_dummy_obj_class_3_verbose, metrics=metric_multi_alias) assert len(res) == 2 assert 'valid multi_logloss-mean' in res # multiclass metric for custom objective - res = get_cv_result(params_class_3_verbose, metrics='multi_error', fobj=dummy_obj) + res = get_cv_result(params_dummy_obj_class_3_verbose, metrics='multi_error') assert len(res) == 2 assert 'valid multi_error-mean' in res # binary metric with non-default num_class for custom objective with pytest.raises(lgb.basic.LightGBMError): - get_cv_result(params_class_3_verbose, metrics='binary_error', fobj=dummy_obj) + get_cv_result(params_dummy_obj_class_3_verbose, metrics='binary_error') def test_multiple_feval_train(): @@ -2278,6 +2285,97 @@ def test_multiple_feval_train(): assert 'decreasing_metric' in evals_result['valid_0'] +def test_objective_callable_train_binary_classification(): + X, y = load_breast_cancer(return_X_y=True) + params = { + 'verbose': -1, + 'objective': logloss_obj, + 'learning_rate': 0.01 + } + train_dataset = lgb.Dataset(X, y) + booster = lgb.train( + params=params, + train_set=train_dataset, + num_boost_round=20 + ) + y_pred = logistic_sigmoid(booster.predict(X)) + logloss_error = log_loss(y, y_pred) + rocauc_error = roc_auc_score(y, y_pred) + assert booster.params['objective'] == 'none' + assert logloss_error == pytest.approx(0.55, 0.1) + assert rocauc_error == pytest.approx(0.99, 0.5) + + +def test_objective_callable_train_regression(): + X, y = make_synthetic_regression() + params = { + 'verbose': -1, + 'objective': mse_obj + } + lgb_train = lgb.Dataset(X, y) + booster = lgb.train( + params, + lgb_train, + num_boost_round=20 + ) + y_pred = booster.predict(X) + mse_error = mean_squared_error(y, y_pred) + assert booster.params['objective'] == 'none' + assert mse_error == pytest.approx(286, 1) + + +def test_objective_callable_cv_binary_classification(): + X, y = load_breast_cancer(return_X_y=True) + params = { + 'verbose': -1, + 'objective': logloss_obj, + 'learning_rate': 0.01 + } + train_dataset = lgb.Dataset(X, y) + cv_res = lgb.cv( + params, + train_dataset, + num_boost_round=20, + nfold=3, + return_cvbooster=True + ) + cv_booster = cv_res['cvbooster'].boosters + cv_logloss_errors = [ + log_loss(y, logistic_sigmoid(cb.predict(X))) < 0.56 for cb in cv_booster + ] + cv_objs = [ + cb.params['objective'] == 'none' for cb in cv_booster + ] + assert all(cv_objs) + assert all(cv_logloss_errors) + + +def test_objective_callable_cv_regression(): + X, y = make_synthetic_regression() + lgb_train = lgb.Dataset(X, y) + params = { + 'verbose': -1, + 'objective': mse_obj + } + cv_res = lgb.cv( + params, + lgb_train, + num_boost_round=20, + nfold=3, + stratified=False, + return_cvbooster=True + ) + cv_booster = cv_res['cvbooster'].boosters + cv_mse_errors = [ + mean_squared_error(y, cb.predict(X)) < 463 for cb in cv_booster + ] + cv_objs = [ + cb.params['objective'] == 'none' for cb in cv_booster + ] + assert all(cv_objs) + assert all(cv_mse_errors) + + def test_multiple_feval_cv(): X, y = load_breast_cancer(return_X_y=True) @@ -2334,7 +2432,8 @@ def custom_obj(y_pred, ds): builtin_obj_bst = lgb.train(params, ds, num_boost_round=10) builtin_obj_preds = builtin_obj_bst.predict(X) - custom_obj_bst = lgb.train(params, ds, num_boost_round=10, fobj=custom_obj) + params['objective'] = custom_obj + custom_obj_bst = lgb.train(params, ds, num_boost_round=10) custom_obj_preds = softmax(custom_obj_bst.predict(X)) np.testing.assert_allclose(builtin_obj_preds, custom_obj_preds, rtol=0.01) diff --git a/tests/python_package_test/utils.py b/tests/python_package_test/utils.py index 63950d471608..472343091566 100644 --- a/tests/python_package_test/utils.py +++ b/tests/python_package_test/utils.py @@ -119,12 +119,27 @@ def make_synthetic_regression(n_samples=100): return sklearn.datasets.make_regression(n_samples, n_features=4, n_informative=2, random_state=42) +def dummy_obj(preds, train_data): + return np.ones(preds.shape), np.ones(preds.shape) + + +def mse_obj(y_pred, dtrain): + y_true = dtrain.get_label() + grad = (y_pred - y_true) + hess = np.ones(len(grad)) + return grad, hess + + def softmax(x): row_wise_max = np.max(x, axis=1).reshape(-1, 1) exp_x = np.exp(x - row_wise_max) return exp_x / np.sum(exp_x, axis=1).reshape(-1, 1) +def logistic_sigmoid(x): + return 1.0 / (1.0 + np.exp(-x)) + + def sklearn_multiclass_custom_objective(y_true, y_pred): num_rows, num_class = y_pred.shape prob = softmax(y_pred)