From dd60dbc5eb7e6d298c5f1506a8e0b7a8bb390052 Mon Sep 17 00:00:00 2001
From: Qingyun Wu <qw2ky@virginia.edu>
Date: Mon, 6 Dec 2021 17:03:43 -0500
Subject: [PATCH] rename training_function (#327)

* rename training_function

* add docstr

* update docstr

* update docstr and comments

Co-authored-by: Qingyun Wu <qxw5138@psu.edu>
---
 flaml/tune/tune.py                        | 25 +++++++++++++++--------
 notebook/flaml_finetune_transformer.ipynb |  2 +-
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index b7726453a2..214d52c205 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -109,7 +109,7 @@ def compute_with_config(config):
 
 
 def run(
-    training_function,
+    evaluation_function,
     config: Optional[dict] = None,
     low_cost_partial_config: Optional[dict] = None,
     cat_hp_cost: Optional[dict] = None,
@@ -122,7 +122,7 @@ def run(
     min_resource: Optional[float] = None,
     max_resource: Optional[float] = None,
     reduction_factor: Optional[float] = None,
-    scheduler: Optional = None,
+    scheduler=None,
     search_alg=None,
     verbose: Optional[int] = 2,
     local_dir: Optional[str] = None,
@@ -162,7 +162,12 @@ def compute_with_config(config):
         print(analysis.trials[-1].last_result)
 
     Args:
-        training_function: A user-defined training function.
+        evaluation_function: A user-defined evaluation function.
+            It takes a configuration as input, outputs a evaluation
+            result (can be a numerical value or a dictionary of string
+            and numerical value pairs) for the input configuration.
+            For machine learning tasks, it usually involves training and
+            scoring a machine learning model, e.g., through validation loss.
         config: A dictionary to specify the search space.
         low_cost_partial_config: A dictionary from a subset of
             controlled dimensions to the initial low-cost values.
@@ -195,7 +200,9 @@ def compute_with_config(config):
             needing to re-compute the trial. Must be the same length as
             points_to_evaluate.
             e.g.,
+
             .. code-block:: python
+
                 points_to_evaluate = [
                     {"b": .99, "cost_related": {"a": 3}},
                     {"b": .99, "cost_related": {"a": 2}},
@@ -204,7 +211,7 @@ def compute_with_config(config):
 
             means that you know the reward for the two configs in
             points_to_evaluate are 3.0 and 1.0 respectively and want to
-            inform run()
+            inform run().
 
         resource_attr: A string to specify the resource dimension used by
             the scheduler via "scheduler".
@@ -217,7 +224,7 @@ def compute_with_config(config):
             in this case when resource_attr is provided, the 'flaml' scheduler will be
             used, otherwise no scheduler will be used. When set 'flaml', an
             authentic scheduler implemented in FLAML will be used. It does not
-            require users to report intermediate results in training_function.
+            require users to report intermediate results in evaluation_function.
             Find more details abuot this scheduler in this paper
             https://arxiv.org/pdf/1911.04706.pdf).
             When set 'asha', the input for arguments "resource_attr",
@@ -225,9 +232,9 @@ def compute_with_config(config):
             to ASHA's "time_attr",  "max_t", "grace_period" and "reduction_factor"
             respectively. You can also provide a self-defined scheduler instance
             of the TrialScheduler class. When 'asha' or self-defined scheduler is
-            used, you usually need to report intermediate results in the training
+            used, you usually need to report intermediate results in the evaluation
             function. Please find examples using different types of schedulers
-            and how to set up the corresponding training functions in
+            and how to set up the corresponding evaluation functions in
             test/tune/test_scheduler.py. TODO: point to notebook examples.
         search_alg: An instance of BlendSearch as the search algorithm
             to be used. The same instance can be used for iterative tuning.
@@ -382,7 +389,7 @@ def compute_with_config(config):
             )
         _use_ray = True
         return tune.run(
-            training_function,
+            evaluation_function,
             metric=metric,
             mode=mode,
             search_alg=search_alg,
@@ -423,7 +430,7 @@ def compute_with_config(config):
             num_trials += 1
             if verbose:
                 logger.info(f"trial {num_trials} config: {trial_to_run.config}")
-            result = training_function(trial_to_run.config)
+            result = evaluation_function(trial_to_run.config)
             if result is not None:
                 if isinstance(result, dict):
                     report(**result)
diff --git a/notebook/flaml_finetune_transformer.ipynb b/notebook/flaml_finetune_transformer.ipynb
index beae8aed33..e02fb2a2ae 100644
--- a/notebook/flaml_finetune_transformer.ipynb
+++ b/notebook/flaml_finetune_transformer.ipynb
@@ -893,7 +893,7 @@
     "        metric=HP_METRIC,\n",
     "        mode=MODE,\n",
     "        low_cost_partial_config={\"num_train_epochs\": 1}),\n",
-    "    # uncomment the following if scheduler = 'auto',\n",
+    "    # uncomment the following if scheduler = 'asha',\n",
     "    # max_resource=max_num_epoch, min_resource=1,\n",
     "    resources_per_trial={\"gpu\": num_gpus, \"cpu\": num_cpus},\n",
     "    local_dir='logs/',\n",