From 0158987390fdcb8b4cbe8931af45214070e07e51 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 01:51:47 +0800
Subject: [PATCH 01/53] Port previous impl.

---
 python-package/xgboost/callback.py | 380 ++++++++++++++++++++++++++++-
 python-package/xgboost/training.py | 115 ++++-----
 2 files changed, 422 insertions(+), 73 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index b390ab336583..f4571b066732 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -1,9 +1,14 @@
 # coding: utf-8
-# pylint: disable=invalid-name, too-many-statements
+# pylint: disable=invalid-name, too-many-statements, no-self-use
+# pylint: disable=too-many-arguments
 """Training Library containing training routines."""
+from abc import ABC
+import collections
+import numpy
 
 from . import rabit
-from .core import EarlyStopException
+from .core import EarlyStopException, DMatrix, CallbackEnv
+from .compat import STRING_TYPES
 
 
 def _get_callback_context(env):
@@ -23,7 +28,7 @@ def _fmt_metric(value, show_stdv=True):
         if show_stdv:
             return  '{0}:{1:.5f}+{2:.5f}'.format(value[0], value[1], value[2])
         return '{0}:{1:.5f}'.format(value[0], value[1])
-    raise ValueError("wrong metric value")
+    raise ValueError("wrong metric value", value)
 
 
 def print_evaluation(period=1, show_stdv=True):
@@ -253,3 +258,372 @@ def callback(env):
                 rabit.tracker_print(msg.format(best_msg))
             raise EarlyStopException(best_iteration)
     return callback
+
+
+#
+# The new implementation of callback functions.
+#
+# TODOs
+# - eval_set
+# - cv
+# - tests
+# - doc
+# - enforced best_xxx
+# - merged functionality of es and mon.
+
+# pylint: disable=unused-argument
+class TrainingCallback(ABC):
+    '''Interface for training callback.
+
+    .. versionadded:: 1.3.0
+
+    '''
+    def __init__(self):
+        self.history = {}
+
+    def before_training(self, model):
+        '''Run before training starts.'''
+
+    def after_training(self, model):
+        '''Run after training is finished.'''
+
+    def before_iteration(self, model, epoch):
+        '''Run before each iteration.'''
+        return False
+
+    def after_iteration(self, model, epoch):
+        '''Run after each iteration.'''
+        return False
+
+
+class CallbackContainer(TrainingCallback):
+    '''A container for list of callbacks.
+
+    .. versionadded:: 1.3.0
+
+    '''
+    def __init__(self, callbacks):
+        self.callbacks = callbacks
+        super().__init__()
+
+    def before_training(self, model):
+        '''Function called before training.'''
+        for c in self.callbacks:
+            c.before_training(model)
+
+    def after_training(self, model):
+        '''Function called after training.'''
+        for c in self.callbacks:
+            c.after_training(model)
+
+    def before_iteration(self, model, epoch):
+        '''Function called before training iteration.'''
+        return any(c.before_iteration(model, epoch)
+                   for c in self.callbacks)
+
+    def after_iteration(self, model, epoch):
+        '''Function called after training iteration.'''
+        return any(c.after_iteration(model, epoch)
+                   for c in self.callbacks)
+
+
+class LearningRateScheduler(TrainingCallback):
+    '''Callback function for scheduling learning rate.
+
+    .. versionadded:: 1.3.0
+
+    Parameters
+    ----------
+
+    learning_rates : callable/collections.Sequence
+        If it's a callable object, then it should accept an integer parameter
+        `epoch` and returns the corresponding learning rate.  Otherwise it
+        shoule be a sequence like list or tuple with the same size of boosting
+        rounds.
+
+    '''
+    def __init__(self, learning_rates):
+        assert callable(learning_rates) or \
+            isinstance(learning_rates, collections.Sequence)
+        if callable(learning_rates):
+            self.learning_rates = learning_rates
+        else:
+            self.learning_rates = lambda epoch: learning_rates[epoch]
+        super().__init__()
+
+    def after_iteration(self, model, epoch):
+        model.set_param('learning_rate', self.learning_rates(epoch))
+
+
+# pylint: disable=too-many-instance-attributes
+class EarlyStopping(TrainingCallback):
+    ''' Callback function for early stopping
+
+    .. versionadded:: 1.3.0
+
+    Parameters
+    ----------
+    data
+        data for evaluation.
+    name : str
+        Name of data.
+    metric : str/callable
+        Name of metric.  Use the default metric if not specified.
+    metric_name : str
+        Name of metric, used when metric is a callable object.
+    rounds : int
+        Early stopping rounds.
+    maximize : bool
+        Whether to maximize evaluation metric.
+    missing : float
+        Same as missing for DMatrix, used when input is not a DMatrix.
+    wegiht
+        Same as label for DMatrix, used when input is not a DMatrix.
+    label
+        Same as weight for DMatrix, used when input is not a DMatrix.
+    '''
+    def __init__(self, data, name, rounds, metric=None, metric_name='metric',
+                 maximize=False, missing=numpy.nan, weight=None, label=None):
+        if callable(metric):
+            self.data = data
+            self.label = label
+            assert weight is None, 'Weight is not supported by custom metric'
+        else:
+            self.data = self._make_dmatrix(data, label, weight, missing)
+            self.label = None
+            self.weight = None
+
+        self.data_id = id(self.data)
+        self.name = name
+        self.metric = metric
+        self.rounds = rounds
+        if callable(self.metric):
+            self.metric_name = metric_name
+        else:
+            self.metric_name = self.metric
+        self.maximize = maximize
+
+        if self.maximize:
+            self.improve_op = lambda x, y: x > y
+        else:
+            self.improve_op = lambda x, y: x < y
+
+        self.current_rounds = 0
+        self.best_scores = {}
+        super().__init__()
+
+    def _make_dmatrix(self, data, label, weight, missing):
+        if not isinstance(data, DMatrix):
+            assert label is not None, 'Label is required to construct DMatrix.'
+            data = DMatrix(data, label=label, weight=weight, missing=missing)
+        else:
+            assert label is None and weight is None and numpy.isnan(missing), (
+                'label, weight and missing are only used when input is not ' +
+                'a DMatrix'
+            )
+        return data
+
+    def before_training(self, model):
+        if not callable(self.metric):
+            model.set_param({'eval_metric': self.metric})
+
+    def after_training(self, model):
+        model.best_iteration = self.rounds
+        model.set_attr(best_iteration=str(self.rounds))
+
+    def _update_rounds(self, scores, model, epoch):
+        assert len(scores) == 1
+        score = scores[0]
+        metric, s = score[0], score[1]
+        if not self.history:    # First round
+            self.current_rounds = 0
+            self.history[self.name] = {}
+            self.history[self.name][metric] = [s]
+            self.best_scores[self.name] = {}
+            self.best_scores[self.name][metric] = [s]
+        elif not self.improve_op(s, self.best_scores[self.name][metric][-1]):
+            # Not improved
+            self.history[self.name][metric].append(s)
+            self.current_rounds += 1
+        else:                   # Improved
+            self.history[self.name][metric].append(s)
+            self.best_scores[self.name][metric].append(s)
+            record = self.history[self.name][metric][-1]
+            model.set_attr(best_score=str(record),
+                           best_iteration=str(epoch))
+            self.current_rounds = 0  # reset
+
+        if self.current_rounds >= self.rounds:
+            return True
+        return False
+
+    def after_iteration(self, model, epoch):
+        assert not rabit.is_distributed(), '''
+Use distributed version instead.  For dask users:
+
+>>>  from xgboost.dask import EarlyStopping
+'''
+        if callable(self.metric):
+            predt = model.inplace_predict(self.data)
+            score = self.metric(self.label, predt)
+            score = [(self.metric_name, score)]
+        else:
+            score = model.eval(self.data)
+            score = [s.split(':') for s in score.split()]
+            score = [(k, float(v)) for k, v in score[1:]]
+
+        return self._update_rounds(score, model, epoch)
+
+
+class EvaluationMonitor(TrainingCallback):
+    '''Print the evaluation result at each iteration.
+
+    .. versionadded:: 1.3.0
+
+    Parameters
+    ----------
+
+    data
+        Data for evaluation.
+    name : str
+        Name of data.
+    metric : str
+        Name of metric
+    rank : int
+        Which worker should be used for printing the result.
+    missing : float
+        Used when data is not a DMatrix.
+    weight
+        Used when data is not a DMatrix.
+    label
+        Used when data is not a DMatrix.
+    '''
+    def __init__(self, data, name,
+                 metric=None, rank=0, missing=numpy.nan,
+                 weight=None, label=None):
+        data = self._make_dmatrix(data, label, weight, missing)
+        self.data = data
+        self.data_id = id(self.data)
+
+        self.name = name
+        self.metric = metric
+        self.label = label
+        self.printer_rank = rank
+        super().__init__()
+
+    def _make_dmatrix(self, data, label, weight, missing):
+        if not isinstance(data, DMatrix):
+            assert label is not None
+            data = DMatrix(data, label=label, weight=weight, missing=missing)
+        else:
+            assert label is None and weight is None and numpy.isnan(missing), (
+                'label, weight and missing are only used when input is not ' +
+                'a DMatrix'
+            )
+        return data
+
+    def before_training(self, model):
+        model.set_param({'eval_metric': self.metric})
+
+    def _update_history(self, score, epoch):
+        score = [s.split(':') for s in score.split()]
+        score = [(k, float(v)) for k, v in score[1:]]
+
+        if rabit.get_rank() == self.printer_rank:
+            msg = _fmt_metric(score[0])
+            rabit.tracker_print('[%d]\t%s\n' % (epoch, msg))
+
+        def metric_name():
+            if self.metric:
+                return self.metric
+            name = score[0][0]
+            pos = name.index('-')
+            name = name[pos+1:]
+            return name
+
+        if not self.history:
+            self.history[metric_name()] = [score[0][1]]
+        else:
+            self.history[metric_name()].append(score[0][1])
+
+        return False
+
+    def after_iteration(self, model, epoch):
+        assert not rabit.is_distributed()
+        score = model.eval(self.data, self.name)
+        return self._update_history(score, epoch)
+
+
+class LegacyCallbacks(TrainingCallback):
+    '''Adapter for legacy callback functions.
+
+    .. versionadded:: 1.3.0
+
+    Parameters
+    ----------
+
+    callbacks : Sequence
+        A sequence of legacy callbacks (callbacks that are not instance of
+        TrainingCallback)
+    start_iteration : int
+        Begining iteration.
+    end_iteration : int
+        End iteration, normally is the number of boosting rounds.
+    evals : Sequence
+        Sequence of evaluation dataset tuples.
+    feval : Custom evaluation metric.
+    '''
+    def __init__(self, callbacks, start_iteration, end_iteration,
+                 evals, feval):
+        self.callbacks_before_iter = [
+            cb for cb in callbacks
+            if cb.__dict__.get('before_iteration', False)]
+        self.callbacks_after_iter = [
+            cb for cb in callbacks
+            if not cb.__dict__.get('before_iteration', False)]
+
+        self.start_iteration = start_iteration
+        self.end_iteration = end_iteration
+
+        self.evals = evals
+        self.feval = feval
+        assert self.feval is None or callable(self.feval)
+
+        super().__init__()
+
+    def before_iteration(self, model, epoch):
+        for cb in self.callbacks_before_iter:
+            rank = rabit.get_rank()
+            cb(CallbackEnv(model=model,
+                           cvfolds=None,
+                           iteration=epoch,
+                           begin_iteration=self.start_iteration,
+                           end_iteration=self.end_iteration,
+                           rank=rank,
+                           evaluation_result_list=None))
+        return False
+
+    def after_iteration(self, model, epoch):
+        evaluation_result_list = []
+        if self.evals:
+            bst_eval_set = model.eval_set(self.evals, epoch, self.feval)
+            if isinstance(bst_eval_set, STRING_TYPES):
+                msg = bst_eval_set
+            else:
+                msg = bst_eval_set.decode()
+            res = [x.split(':') for x in msg.split()]
+            evaluation_result_list = [(k, float(v)) for k, v in res[1:]]
+        try:
+            for cb in self.callbacks_after_iter:
+                rank = rabit.get_rank()
+                cb(CallbackEnv(model=model,
+                               cvfolds=None,
+                               iteration=epoch,
+                               begin_iteration=self.start_iteration,
+                               end_iteration=self.end_iteration,
+                               rank=rank,
+                               evaluation_result_list=evaluation_result_list))
+        except EarlyStopException:
+            return True
+
+        return False
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 7a29d3952570..14f9f7def270 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -13,21 +13,13 @@
 def _train_internal(params, dtrain,
                     num_boost_round=10, evals=(),
                     obj=None, feval=None,
-                    xgb_model=None, callbacks=None):
+                    xgb_model=None, callbacks=None,
+                    evals_result=None, maximize=None,
+                    verbose_eval=None, early_stopping_rounds=None):
     """internal training function"""
     callbacks = [] if callbacks is None else callbacks
     evals = list(evals)
     params = params.copy()
-    if isinstance(params, dict) \
-            and 'eval_metric' in params \
-            and isinstance(params['eval_metric'], list):
-        params = dict((k, v) for k, v in params.items())
-        eval_metrics = params['eval_metric']
-        params.pop("eval_metric", None)
-        params = list(params.items())
-        for eval_metric in eval_metrics:
-            params += [('eval_metric', eval_metric)]
-
     bst = Booster(params, [dtrain] + [d[0] for d in evals])
     nboost = 0
     num_parallel_tree = 1
@@ -49,26 +41,35 @@ def _train_internal(params, dtrain,
     # Distributed code: Load the checkpoint from rabit.
     version = bst.load_rabit_checkpoint()
     assert rabit.get_world_size() != 1 or version == 0
-    rank = rabit.get_rank()
     start_iteration = int(version / 2)
     nboost += start_iteration
 
-    callbacks_before_iter = [
-        cb for cb in callbacks
-        if cb.__dict__.get('before_iteration', False)]
-    callbacks_after_iter = [
-        cb for cb in callbacks
-        if not cb.__dict__.get('before_iteration', False)]
-
+    is_new_callback = [isinstance(c, callback.TrainingCallback)
+                       for c in callbacks]
+    if any(is_new_callback):
+        assert all(is_new_callback), "You can't mix two styles of callbacks."
+        callbacks = callback.CallbackContainer(callbacks)
+    else:
+        # Most of legacy advanced options becomes callbacks
+        if isinstance(verbose_eval, bool) and verbose_eval:
+            callbacks.append(callback.print_evaluation())
+        else:
+            if isinstance(verbose_eval, int):
+                callbacks.append(callback.print_evaluation(verbose_eval))
+
+        if early_stopping_rounds is not None:
+            callbacks.append(callback.early_stop(early_stopping_rounds,
+                                                 maximize=maximize,
+                                                 verbose=bool(verbose_eval)))
+        if evals_result is not None:
+            callbacks.append(callback.record_evaluation(evals_result))
+        callbacks = callback.LegacyCallbacks(
+            callbacks, start_iteration, num_boost_round, evals, feval)
+
+    callbacks.before_training(bst)
     for i in range(start_iteration, num_boost_round):
-        for cb in callbacks_before_iter:
-            cb(CallbackEnv(model=bst,
-                           cvfolds=None,
-                           iteration=i,
-                           begin_iteration=start_iteration,
-                           end_iteration=num_boost_round,
-                           rank=rank,
-                           evaluation_result_list=None))
+        if callbacks.before_iteration(bst, i):
+            break
         # Distributed code: need to resume to this point.
         # Skip the first update if it is a recovery step.
         if version % 2 == 0:
@@ -79,39 +80,24 @@ def _train_internal(params, dtrain,
         assert rabit.get_world_size() == 1 or version == rabit.version_number()
 
         nboost += 1
-        evaluation_result_list = []
         # check evaluation result.
-        if evals:
-            bst_eval_set = bst.eval_set(evals, i, feval)
-            if isinstance(bst_eval_set, STRING_TYPES):
-                msg = bst_eval_set
-            else:
-                msg = bst_eval_set.decode()
-            res = [x.split(':') for x in msg.split()]
-            evaluation_result_list = [(k, float(v)) for k, v in res[1:]]
-        try:
-            for cb in callbacks_after_iter:
-                cb(CallbackEnv(model=bst,
-                               cvfolds=None,
-                               iteration=i,
-                               begin_iteration=start_iteration,
-                               end_iteration=num_boost_round,
-                               rank=rank,
-                               evaluation_result_list=evaluation_result_list))
-        except EarlyStopException:
+        if callbacks.after_iteration(bst, i):
             break
-        # do checkpoint after evaluation, in case evaluation also updates booster.
+        # do checkpoint after evaluation, in case evaluation also updates
+        # booster.
         bst.save_rabit_checkpoint()
         version += 1
 
+    callbacks.after_training(bst)
+
     if bst.attr('best_score') is not None:
         bst.best_score = float(bst.attr('best_score'))
         bst.best_iteration = int(bst.attr('best_iteration'))
     else:
         bst.best_iteration = nboost - 1
     bst.best_ntree_limit = (bst.best_iteration + 1) * num_parallel_tree
-
-    # Copy to serialise and unserialise booster to reset state and free training memory
+    # Copy to serialise and unserialise booster to reset state and free
+    # training memory
     return bst.copy()
 
 
@@ -189,27 +175,16 @@ def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None,
     -------
     Booster : a trained booster model
     """
-    callbacks = [] if callbacks is None else callbacks
-
-    # Most of legacy advanced options becomes callbacks
-    if isinstance(verbose_eval, bool) and verbose_eval:
-        callbacks.append(callback.print_evaluation())
-    else:
-        if isinstance(verbose_eval, int):
-            callbacks.append(callback.print_evaluation(verbose_eval))
-
-    if early_stopping_rounds is not None:
-        callbacks.append(callback.early_stop(early_stopping_rounds,
-                                             maximize=maximize,
-                                             verbose=bool(verbose_eval)))
-    if evals_result is not None:
-        callbacks.append(callback.record_evaluation(evals_result))
-
-    return _train_internal(params, dtrain,
-                           num_boost_round=num_boost_round,
-                           evals=evals,
-                           obj=obj, feval=feval,
-                           xgb_model=xgb_model, callbacks=callbacks)
+    bst = _train_internal(params, dtrain,
+                          num_boost_round=num_boost_round,
+                          evals=evals,
+                          obj=obj, feval=feval,
+                          xgb_model=xgb_model, callbacks=callbacks,
+                          verbose_eval=verbose_eval,
+                          evals_result=evals_result,
+                          maximize=maximize,
+                          early_stopping_rounds=early_stopping_rounds)
+    return bst
 
 
 class CVPack(object):

From 9cebcd2e2e275a9f14986ad9d9d10db441d3da26 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 02:00:01 +0800
Subject: [PATCH 02/53] Check point.

---
 python-package/xgboost/callback.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index f4571b066732..720272c14253 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -4,6 +4,7 @@
 """Training Library containing training routines."""
 from abc import ABC
 import collections
+import os
 import numpy
 
 from . import rabit
@@ -554,6 +555,32 @@ def after_iteration(self, model, epoch):
         return self._update_history(score, epoch)
 
 
+class TrainingCheckPoint(TrainingCallback):
+    '''Checkpointing operation.
+
+    .. versionadded:: 1.3.0
+
+    Parameters
+    ----------
+
+    path : os.PathLike
+        Output model path.
+    iterations : int
+        Interval of checkpointing.
+    '''
+    def __init__(self, path: os.PathLike, iterations=10):
+        self._path = path
+        self._iterations = iterations
+        self._epoch = 0
+
+    def after_iteration(self, model, epoch):
+        self._epoch += 1
+        if self._epoch == 10:
+            self._epoch = 0
+            if rabit.get_rank() == 0:
+                model.save_model(self._path)
+
+
 class LegacyCallbacks(TrainingCallback):
     '''Adapter for legacy callback functions.
 

From 358b227189e54e444daf846944f8f2c03e5a5405 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 02:58:54 +0800
Subject: [PATCH 03/53] Pass in the parameter.

---
 python-package/xgboost/dask.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/python-package/xgboost/dask.py b/python-package/xgboost/dask.py
index b236bfc0bf94..180dd6332d02 100644
--- a/python-package/xgboost/dask.py
+++ b/python-package/xgboost/dask.py
@@ -627,8 +627,8 @@ async def _get_rabit_args(worker_map, client: Client):
 # evaluation history is instead returned.
 
 
-async def _train_async(client, params, dtrain: DaskDMatrix,
-                       *args, evals=(), **kwargs):
+async def _train_async(client, params, dtrain: DaskDMatrix, *args, evals=(),
+                       early_stopping_rounds=None, **kwargs):
     _assert_dask_support()
     client: Client = _xgb_get_client(client)
     if 'evals_result' in kwargs.keys():
@@ -675,6 +675,7 @@ def dispatched_train(worker_addr, dtrain_ref, evals_ref):
                                *args,
                                evals_result=local_history,
                                evals=local_evals,
+                               early_stopping_rounds=early_stopping_rounds,
                                **kwargs)
             ret = {'booster': bst, 'history': local_history}
             if local_dtrain.num_row() == 0:
@@ -694,7 +695,8 @@ def dispatched_train(worker_addr, dtrain_ref, evals_ref):
     return list(filter(lambda ret: ret is not None, results))[0]
 
 
-def train(client, params, dtrain, *args, evals=(), **kwargs):
+def train(client, params, dtrain, *args, evals=(), early_stopping_rounds=None,
+          **kwargs):
     '''Train XGBoost model.
 
     .. versionadded:: 1.0.0
@@ -724,8 +726,9 @@ def train(client, params, dtrain, *args, evals=(), **kwargs):
     '''
     _assert_dask_support()
     client = _xgb_get_client(client)
-    return client.sync(_train_async, client, params,
-                       dtrain=dtrain, *args, evals=evals, **kwargs)
+    return client.sync(
+        _train_async, client, params, dtrain=dtrain, *args, evals=evals,
+        early_stopping_rounds=early_stopping_rounds, **kwargs)
 
 
 async def _direct_predict_impl(client, data, predict_fn):
@@ -1005,6 +1008,7 @@ def fit(self, X, y,
             base_margin=None,
             eval_set=None,
             sample_weight_eval_set=None,
+            early_stopping_rounds=None,
             verbose=True):
         '''Fit the regressor.
 
@@ -1066,6 +1070,7 @@ async def _fit_async(self,
                          base_margin=None,
                          eval_set=None,
                          sample_weight_eval_set=None,
+                         early_stopping_rounds=None,
                          verbose=True):
         dtrain = await DaskDMatrix(
             client=self.client, data=X, label=y, weight=sample_weights,
@@ -1077,7 +1082,8 @@ async def _fit_async(self,
                                            self.missing)
         results = await train(client=self.client, params=params, dtrain=dtrain,
                               num_boost_round=self.get_num_boosting_rounds(),
-                              evals=evals, verbose_eval=verbose)
+                              evals=evals, verbose_eval=verbose,
+                              early_stopping_rounds=early_stopping_rounds)
         self._Booster = results['booster']
         # pylint: disable=attribute-defined-outside-init
         self.evals_result_ = results['history']
@@ -1089,6 +1095,7 @@ def fit(self, X, y,
             base_margin=None,
             eval_set=None,
             sample_weight_eval_set=None,
+            early_stopping_rounds=None,
             verbose=True):
         _assert_dask_support()
         return self.client.sync(
@@ -1161,6 +1168,7 @@ def fit(self, X, y,
             base_margin=None,
             eval_set=None,
             sample_weight_eval_set=None,
+            early_stopping_rounds=None,
             verbose=True):
         _assert_dask_support()
         return self.client.sync(

From 4a5879c3eb989f0973e60c07c0c2c88ceb2bff11 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 03:38:10 +0800
Subject: [PATCH 04/53] Distributed.

---
 python-package/xgboost/callback.py | 89 +++++++++++-------------------
 python-package/xgboost/training.py |  4 +-
 2 files changed, 34 insertions(+), 59 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 720272c14253..16d4f95aef1b 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -288,11 +288,11 @@ def before_training(self, model):
     def after_training(self, model):
         '''Run after training is finished.'''
 
-    def before_iteration(self, model, epoch):
+    def before_iteration(self, model, epoch, dtrain, evals):
         '''Run before each iteration.'''
         return False
 
-    def after_iteration(self, model, epoch):
+    def after_iteration(self, model, epoch, dtrain, evals):
         '''Run after each iteration.'''
         return False
 
@@ -317,14 +317,14 @@ def after_training(self, model):
         for c in self.callbacks:
             c.after_training(model)
 
-    def before_iteration(self, model, epoch):
+    def before_iteration(self, model, epoch, dtrain, evals):
         '''Function called before training iteration.'''
-        return any(c.before_iteration(model, epoch)
+        return any(c.before_iteration(model, epoch, dtrain, evals)
                    for c in self.callbacks)
 
-    def after_iteration(self, model, epoch):
+    def after_iteration(self, model, epoch, dtrain, evals):
         '''Function called after training iteration.'''
-        return any(c.after_iteration(model, epoch)
+        return any(c.after_iteration(model, epoch, dtrain, evals)
                    for c in self.callbacks)
 
 
@@ -352,10 +352,19 @@ def __init__(self, learning_rates):
             self.learning_rates = lambda epoch: learning_rates[epoch]
         super().__init__()
 
-    def after_iteration(self, model, epoch):
+    def after_iteration(self, model, epoch, dtrain, evals):
         model.set_param('learning_rate', self.learning_rates(epoch))
 
 
+def _allreduce_metric(score, maximize):
+    score = numpy.array([score])
+    if maximize:
+        score = rabit.allreduce(score, rabit.Op.MAX)
+    else:
+        score = rabit.allreduce(score, rabit.Op.MIN)
+    return score
+
+
 # pylint: disable=too-many-instance-attributes
 class EarlyStopping(TrainingCallback):
     ''' Callback function for early stopping
@@ -383,18 +392,8 @@ class EarlyStopping(TrainingCallback):
     label
         Same as weight for DMatrix, used when input is not a DMatrix.
     '''
-    def __init__(self, data, name, rounds, metric=None, metric_name='metric',
-                 maximize=False, missing=numpy.nan, weight=None, label=None):
-        if callable(metric):
-            self.data = data
-            self.label = label
-            assert weight is None, 'Weight is not supported by custom metric'
-        else:
-            self.data = self._make_dmatrix(data, label, weight, missing)
-            self.label = None
-            self.weight = None
-
-        self.data_id = id(self.data)
+    def __init__(self, name, rounds, metric=None, metric_name='metric',
+                 maximize=False):
         self.name = name
         self.metric = metric
         self.rounds = rounds
@@ -413,17 +412,6 @@ def __init__(self, data, name, rounds, metric=None, metric_name='metric',
         self.best_scores = {}
         super().__init__()
 
-    def _make_dmatrix(self, data, label, weight, missing):
-        if not isinstance(data, DMatrix):
-            assert label is not None, 'Label is required to construct DMatrix.'
-            data = DMatrix(data, label=label, weight=weight, missing=missing)
-        else:
-            assert label is None and weight is None and numpy.isnan(missing), (
-                'label, weight and missing are only used when input is not ' +
-                'a DMatrix'
-            )
-        return data
-
     def before_training(self, model):
         if not callable(self.metric):
             model.set_param({'eval_metric': self.metric})
@@ -458,18 +446,23 @@ def _update_rounds(self, scores, model, epoch):
             return True
         return False
 
-    def after_iteration(self, model, epoch):
+    def after_iteration(self, model, epoch, dtrain, evals):
         assert not rabit.is_distributed(), '''
 Use distributed version instead.  For dask users:
 
 >>>  from xgboost.dask import EarlyStopping
 '''
+        msg = 'Must have at least 1 validation dataset for early stopping.'
+        assert len(evals) >= 1, msg
+        stopping_data = evals[-1][1]
         if callable(self.metric):
-            predt = model.inplace_predict(self.data)
-            score = self.metric(self.label, predt)
+            label = stopping_data.get_label()
+            predt = model.predict(stopping_data)
+            score = self.metric(label, predt)
+            score = _allreduce_metric(score, self.maximize)
             score = [(self.metric_name, score)]
         else:
-            score = model.eval(self.data)
+            score = model.eval(stopping_data)
             score = [s.split(':') for s in score.split()]
             score = [(k, float(v)) for k, v in score[1:]]
 
@@ -499,30 +492,12 @@ class EvaluationMonitor(TrainingCallback):
     label
         Used when data is not a DMatrix.
     '''
-    def __init__(self, data, name,
-                 metric=None, rank=0, missing=numpy.nan,
-                 weight=None, label=None):
-        data = self._make_dmatrix(data, label, weight, missing)
-        self.data = data
-        self.data_id = id(self.data)
-
+    def __init__(self, name, metric=None, rank=0):
         self.name = name
         self.metric = metric
-        self.label = label
         self.printer_rank = rank
         super().__init__()
 
-    def _make_dmatrix(self, data, label, weight, missing):
-        if not isinstance(data, DMatrix):
-            assert label is not None
-            data = DMatrix(data, label=label, weight=weight, missing=missing)
-        else:
-            assert label is None and weight is None and numpy.isnan(missing), (
-                'label, weight and missing are only used when input is not ' +
-                'a DMatrix'
-            )
-        return data
-
     def before_training(self, model):
         model.set_param({'eval_metric': self.metric})
 
@@ -549,7 +524,7 @@ def metric_name():
 
         return False
 
-    def after_iteration(self, model, epoch):
+    def after_iteration(self, model, epoch, dtrain, evals):
         assert not rabit.is_distributed()
         score = model.eval(self.data, self.name)
         return self._update_history(score, epoch)
@@ -573,7 +548,7 @@ def __init__(self, path: os.PathLike, iterations=10):
         self._iterations = iterations
         self._epoch = 0
 
-    def after_iteration(self, model, epoch):
+    def after_iteration(self, model, epoch, dtrain, evals):
         self._epoch += 1
         if self._epoch == 10:
             self._epoch = 0
@@ -618,7 +593,7 @@ def __init__(self, callbacks, start_iteration, end_iteration,
 
         super().__init__()
 
-    def before_iteration(self, model, epoch):
+    def before_iteration(self, model, epoch, dtrain, evals):
         for cb in self.callbacks_before_iter:
             rank = rabit.get_rank()
             cb(CallbackEnv(model=model,
@@ -630,7 +605,7 @@ def before_iteration(self, model, epoch):
                            evaluation_result_list=None))
         return False
 
-    def after_iteration(self, model, epoch):
+    def after_iteration(self, model, epoch, dtrain, evals):
         evaluation_result_list = []
         if self.evals:
             bst_eval_set = model.eval_set(self.evals, epoch, self.feval)
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 14f9f7def270..7ffce8ecb961 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -68,7 +68,7 @@ def _train_internal(params, dtrain,
 
     callbacks.before_training(bst)
     for i in range(start_iteration, num_boost_round):
-        if callbacks.before_iteration(bst, i):
+        if callbacks.before_iteration(bst, i, dtrain, evals):
             break
         # Distributed code: need to resume to this point.
         # Skip the first update if it is a recovery step.
@@ -81,7 +81,7 @@ def _train_internal(params, dtrain,
 
         nboost += 1
         # check evaluation result.
-        if callbacks.after_iteration(bst, i):
+        if callbacks.after_iteration(bst, i, dtrain, evals):
             break
         # do checkpoint after evaluation, in case evaluation also updates
         # booster.

From 39efda8d4147708a8b6ea89bb7092f87d390254f Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 03:46:38 +0800
Subject: [PATCH 05/53] Port test.

---
 tests/python/test_callback.py | 64 +++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 tests/python/test_callback.py

diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
new file mode 100644
index 000000000000..223a5f06b000
--- /dev/null
+++ b/tests/python/test_callback.py
@@ -0,0 +1,64 @@
+import xgboost as xgb
+import pytest
+import testing as tm
+import numpy as np
+
+
+def verify_booster_early_stop(booster):
+    dump = booster.get_dump(dump_format='json')
+    assert len(dump) == 10      # boosted for 10 rounds.
+
+
+@pytest.mark.skipif(**tm.no_sklearn())
+def test_early_stopping():
+    from sklearn.datasets import load_breast_cancer
+    X, y = load_breast_cancer(return_X_y=True)
+    m = xgb.DMatrix(X, y)
+    booster = xgb.train({'objective': 'binary:logistic'}, m,
+                        evals=[(m, 'Train')],
+                        num_boost_round=1000,
+                        early_stopping_rounds=5,
+                        verbose_eval=False)
+    verify_booster_early_stop(booster)
+
+
+def eval_error_metric(label, predt):
+    r = np.zeros(predt.shape)
+    gt = predt > 0.5
+    r[gt] = 1 - label[gt]
+    le = predt <= 0.5
+    r[le] = label[le]
+    return np.sum(r)
+
+
+@pytest.mark.skipif(**tm.no_sklearn())
+def test_early_stopping_custom_eval():
+    from sklearn.datasets import load_breast_cancer
+    X, y = load_breast_cancer(return_X_y=True)
+    m = xgb.DMatrix(X, y)
+    booster = xgb.train({'objective': 'binary:logistic'}, m,
+                        evals=[(m, 'Train')],
+                        num_boost_round=1000,
+                        early_stopping_rounds=5,
+                        verbose_eval=False)
+    verify_booster_early_stop(booster)
+
+
+@pytest.mark.skipif(**tm.no_sklearn())
+def test_early_stopping_skl():
+    from sklearn.datasets import load_breast_cancer
+    X, y = load_breast_cancer(return_X_y=True)
+    cls = xgb.XGBClassifier()
+    cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=5)
+    booster = cls.get_booster()
+    verify_booster_early_stop(booster)
+
+
+@pytest.mark.skipif(**tm.no_sklearn())
+def test_early_stopping_custom_eval_skl():
+    from sklearn.datasets import load_breast_cancer
+    X, y = load_breast_cancer(return_X_y=True)
+    cls = xgb.XGBClassifier()
+    cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=5)
+    booster = cls.get_booster()
+    verify_booster_early_stop(booster)

From ba1b44a4867d8ded142be373e44f14287d5d55a9 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 04:06:39 +0800
Subject: [PATCH 06/53] Test checkpoint.

---
 python-package/xgboost/callback.py | 28 +++++++++++++++++++-----
 tests/python/test_callback.py      | 35 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 6 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 16d4f95aef1b..e041b7171e6e 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -6,6 +6,7 @@
 import collections
 import os
 import numpy
+import pickle
 
 from . import rabit
 from .core import EarlyStopException, DMatrix, CallbackEnv
@@ -539,21 +540,36 @@ class TrainingCheckPoint(TrainingCallback):
     ----------
 
     path : os.PathLike
-        Output model path.
+        Output model directory.
+    name : name pattern of output model file.
+        Models will be saved as name_0.json, name_1.json, name_2.json ....
+    as_pickle : boolean
+        When set to Ture, all training parameters will be saved in pickle
+        format, instead of saving only the model.
     iterations : int
         Interval of checkpointing.
+
     '''
-    def __init__(self, path: os.PathLike, iterations=10):
-        self._path = path
-        self._iterations = iterations
+    def __init__(self, directory: os.PathLike, name: str = 'model',
+                 as_pickle=False, rounds: int = 10):
+        self._path = directory
+        self._name = name
+        self._as_pickle = as_pickle
+        self._iterations = rounds
         self._epoch = 0
 
     def after_iteration(self, model, epoch, dtrain, evals):
         self._epoch += 1
-        if self._epoch == 10:
+        if self._epoch == self._iterations:
+            path = os.path.join(self._path, self._name + '_' + str(epoch) +
+                                ('.pkl' if self._as_pickle else '.json'))
             self._epoch = 0
             if rabit.get_rank() == 0:
-                model.save_model(self._path)
+                if self._as_pickle:
+                    with open(path, 'wb') as fd:
+                        pickle.dump(model, fd)
+                else:
+                    model.save_model(path)
 
 
 class LegacyCallbacks(TrainingCallback):
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 223a5f06b000..0fb599f02540 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -1,7 +1,9 @@
 import xgboost as xgb
 import pytest
+import os
 import testing as tm
 import numpy as np
+import tempfile
 
 
 def verify_booster_early_stop(booster):
@@ -62,3 +64,36 @@ def test_early_stopping_custom_eval_skl():
     cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=5)
     booster = cls.get_booster()
     verify_booster_early_stop(booster)
+
+
+def test_learning_rate_scheduler():
+    pass
+
+
+def test_check_point():
+    from sklearn.datasets import load_breast_cancer
+    X, y = load_breast_cancer(return_X_y=True)
+    m = xgb.DMatrix(X, y)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
+                                                      rounds=1,
+                                                      name='model')
+        xgb.train({'objective': 'binary:logistic'}, m,
+                  num_boost_round=10,
+                  verbose_eval=False,
+                  callbacks=[check_point])
+        for i in range(0, 10):
+            assert os.path.exists(
+                os.path.join(tmpdir, 'model_' + str(i) + '.json'))
+
+        check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
+                                                      rounds=1,
+                                                      as_pickle=True,
+                                                      name='model')
+        xgb.train({'objective': 'binary:logistic'}, m,
+                  num_boost_round=10,
+                  verbose_eval=False,
+                  callbacks=[check_point])
+        for i in range(0, 10):
+            assert os.path.exists(
+                os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))

From 6a19780d7cbdc30a8a889934a10577f78ca4a823 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 04:13:39 +0800
Subject: [PATCH 07/53] Start writing dask test.

---
 tests/python/test_with_dask.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/python/test_with_dask.py b/tests/python/test_with_dask.py
index 892230cc559b..3e18451cade6 100644
--- a/tests/python/test_with_dask.py
+++ b/tests/python/test_with_dask.py
@@ -648,6 +648,16 @@ def test_hist(self, params, num_rounds, dataset, client):
     def test_approx(self, client, params, num_rounds, dataset):
         self.run_updater_test(client, params, num_rounds, dataset, 'approx')
 
+    def test_early_stopping(self, client):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        m = xgb.dask.DaskDMatrix(X, y)
+        booster = xgb.dask.train({'objective': 'binary:logistic'}, m,
+                                 evals=[(m, 'Train')],
+                                 num_boost_round=1000,
+                                 early_stopping_rounds=5)
+        assert hasattr(booster, 'best_score')
+
     def run_quantile(self, name):
         if sys.platform.startswith("win"):
             pytest.skip("Skipping dask tests on Windows")

From 1b0a6c60b20cf4b4ed4bb970b1033219c5db7444 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 15:19:56 +0800
Subject: [PATCH 08/53] Pass dask test.

---
 tests/python/test_with_dask.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/python/test_with_dask.py b/tests/python/test_with_dask.py
index 3e18451cade6..42327cdc258c 100644
--- a/tests/python/test_with_dask.py
+++ b/tests/python/test_with_dask.py
@@ -651,12 +651,15 @@ def test_approx(self, client, params, num_rounds, dataset):
     def test_early_stopping(self, client):
         from sklearn.datasets import load_breast_cancer
         X, y = load_breast_cancer(return_X_y=True)
-        m = xgb.dask.DaskDMatrix(X, y)
-        booster = xgb.dask.train({'objective': 'binary:logistic'}, m,
+        X, y = da.from_array(X), da.from_array(y)
+        m = xgb.dask.DaskDMatrix(client, X, y)
+        booster = xgb.dask.train(client, {'objective': 'binary:logistic',
+                                          'tree_method': 'hist'}, m,
                                  evals=[(m, 'Train')],
                                  num_boost_round=1000,
-                                 early_stopping_rounds=5)
+                                 early_stopping_rounds=5)['booster']
         assert hasattr(booster, 'best_score')
+        assert booster.best_iteration == 10
 
     def run_quantile(self, name):
         if sys.platform.startswith("win"):

From e1022f5aa927359afc69245522d4d580212f5ec5 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 15:27:31 +0800
Subject: [PATCH 09/53] Define custom metric.

---
 tests/python/test_callback.py  | 11 +----------
 tests/python/test_with_dask.py | 15 +++++++++++++++
 tests/python/testing.py        | 10 ++++++++++
 3 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 0fb599f02540..0e0609f80f41 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -2,7 +2,6 @@
 import pytest
 import os
 import testing as tm
-import numpy as np
 import tempfile
 
 
@@ -24,15 +23,6 @@ def test_early_stopping():
     verify_booster_early_stop(booster)
 
 
-def eval_error_metric(label, predt):
-    r = np.zeros(predt.shape)
-    gt = predt > 0.5
-    r[gt] = 1 - label[gt]
-    le = predt <= 0.5
-    r[le] = label[le]
-    return np.sum(r)
-
-
 @pytest.mark.skipif(**tm.no_sklearn())
 def test_early_stopping_custom_eval():
     from sklearn.datasets import load_breast_cancer
@@ -40,6 +30,7 @@ def test_early_stopping_custom_eval():
     m = xgb.DMatrix(X, y)
     booster = xgb.train({'objective': 'binary:logistic'}, m,
                         evals=[(m, 'Train')],
+                        feval=tm.eval_error_metric,
                         num_boost_round=1000,
                         early_stopping_rounds=5,
                         verbose_eval=False)
diff --git a/tests/python/test_with_dask.py b/tests/python/test_with_dask.py
index 42327cdc258c..bd6ff9c64cfc 100644
--- a/tests/python/test_with_dask.py
+++ b/tests/python/test_with_dask.py
@@ -661,6 +661,21 @@ def test_early_stopping(self, client):
         assert hasattr(booster, 'best_score')
         assert booster.best_iteration == 10
 
+    @pytest.mark.skipif(**tm.no_sklearn())
+    def test_early_stopping_custom_eval(self, client):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        X, y = da.from_array(X), da.from_array(y)
+        m = xgb.dask.DaskDMatrix(client, X, y)
+        booster = xgb.dask.train(client, {'objective': 'binary:logistic',
+                                          'tree_method': 'hist'}, m,
+                                 evals=[(m, 'Train')],
+                                 feval=tm.eval_error_metric,
+                                 num_boost_round=1000,
+                                 early_stopping_rounds=5)['booster']
+        assert hasattr(booster, 'best_score')
+        assert booster.best_iteration == 10
+
     def run_quantile(self, name):
         if sys.platform.startswith("win"):
             pytest.skip("Skipping dask tests on Windows")
diff --git a/tests/python/testing.py b/tests/python/testing.py
index f6a05a5d7d22..452c89ed3eeb 100644
--- a/tests/python/testing.py
+++ b/tests/python/testing.py
@@ -240,6 +240,16 @@ def non_increasing(L, tolerance=1e-4):
     return all((y - x) < tolerance for x, y in zip(L, L[1:]))
 
 
+def eval_error_metric(predt: np.ndarray, dtrain: xgb.DMatrix):
+    label = dtrain.get_label()
+    r = np.zeros(predt.shape)
+    gt = predt > 0.5
+    r[gt] = 1 - label[gt]
+    le = predt <= 0.5
+    r[le] = label[le]
+    return 'PyError', np.sum(r)
+
+
 CURDIR = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
 PROJECT_ROOT = os.path.normpath(
     os.path.join(CURDIR, os.path.pardir, os.path.pardir))

From 94995ca7169ea939eb97e3863f677e0c57508a1f Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 16:20:28 +0800
Subject: [PATCH 10/53] More tests.

---
 python-package/xgboost/callback.py | 11 +++++------
 tests/python/test_callback.py      |  9 ++++++---
 tests/python/test_with_dask.py     | 17 ++++++++++-------
 tests/python/testing.py            |  2 +-
 4 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index e041b7171e6e..efb6454509b5 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -357,12 +357,11 @@ def after_iteration(self, model, epoch, dtrain, evals):
         model.set_param('learning_rate', self.learning_rates(epoch))
 
 
-def _allreduce_metric(score, maximize):
+def _allreduce_metric(score):
     score = numpy.array([score])
-    if maximize:
-        score = rabit.allreduce(score, rabit.Op.MAX)
-    else:
-        score = rabit.allreduce(score, rabit.Op.MIN)
+    world = rabit.get_world_size()
+    assert world != 0
+    score = rabit.allreduce(score, rabit.Op.SUM) / world
     return score
 
 
@@ -526,8 +525,8 @@ def metric_name():
         return False
 
     def after_iteration(self, model, epoch, dtrain, evals):
-        assert not rabit.is_distributed()
         score = model.eval(self.data, self.name)
+        _allreduce_metric(score)
         return self._update_history(score, epoch)
 
 
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 0e0609f80f41..5bf196dde225 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -28,13 +28,16 @@ def test_early_stopping_custom_eval():
     from sklearn.datasets import load_breast_cancer
     X, y = load_breast_cancer(return_X_y=True)
     m = xgb.DMatrix(X, y)
-    booster = xgb.train({'objective': 'binary:logistic'}, m,
+    early_stopping_rounds = 5
+    booster = xgb.train({'objective': 'binary:logistic',
+                         'tree_method': 'hist'}, m,
                         evals=[(m, 'Train')],
                         feval=tm.eval_error_metric,
                         num_boost_round=1000,
-                        early_stopping_rounds=5,
+                        early_stopping_rounds=early_stopping_rounds,
                         verbose_eval=False)
-    verify_booster_early_stop(booster)
+    dump = booster.get_dump(dump_format='json')
+    assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
 
 
 @pytest.mark.skipif(**tm.no_sklearn())
diff --git a/tests/python/test_with_dask.py b/tests/python/test_with_dask.py
index bd6ff9c64cfc..6b7f674b5994 100644
--- a/tests/python/test_with_dask.py
+++ b/tests/python/test_with_dask.py
@@ -667,14 +667,17 @@ def test_early_stopping_custom_eval(self, client):
         X, y = load_breast_cancer(return_X_y=True)
         X, y = da.from_array(X), da.from_array(y)
         m = xgb.dask.DaskDMatrix(client, X, y)
-        booster = xgb.dask.train(client, {'objective': 'binary:logistic',
-                                          'tree_method': 'hist'}, m,
-                                 evals=[(m, 'Train')],
-                                 feval=tm.eval_error_metric,
-                                 num_boost_round=1000,
-                                 early_stopping_rounds=5)['booster']
+        early_stopping_rounds = 5
+        booster = xgb.dask.train(
+            client, {'objective': 'binary:logistic',
+                     'tree_method': 'hist'}, m,
+            evals=[(m, 'Train')],
+            feval=tm.eval_error_metric,
+            num_boost_round=1000,
+            early_stopping_rounds=early_stopping_rounds)['booster']
         assert hasattr(booster, 'best_score')
-        assert booster.best_iteration == 10
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
 
     def run_quantile(self, name):
         if sys.platform.startswith("win"):
diff --git a/tests/python/testing.py b/tests/python/testing.py
index 452c89ed3eeb..1df680adcf13 100644
--- a/tests/python/testing.py
+++ b/tests/python/testing.py
@@ -240,7 +240,7 @@ def non_increasing(L, tolerance=1e-4):
     return all((y - x) < tolerance for x, y in zip(L, L[1:]))
 
 
-def eval_error_metric(predt: np.ndarray, dtrain: xgb.DMatrix):
+def eval_error_metric(predt, dtrain: xgb.DMatrix):
     label = dtrain.get_label()
     r = np.zeros(predt.shape)
     gt = predt > 0.5

From dcc5262b074b55055ebce2313b61e350bfcfcd80 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 17:59:36 +0800
Subject: [PATCH 11/53] Revise monitor.

---
 python-package/xgboost/callback.py | 86 +++++++++++++++---------------
 python-package/xgboost/training.py | 47 ++++++++++------
 tests/python/test_callback.py      | 33 ++++++++++++
 3 files changed, 107 insertions(+), 59 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index efb6454509b5..b42f43f3b871 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -12,6 +12,11 @@
 from .core import EarlyStopException, DMatrix, CallbackEnv
 from .compat import STRING_TYPES
 
+import sys
+def fprint(*args, **kwargs):
+    print(*args, **kwargs)
+    sys.stdout.flush()
+
 
 def _get_callback_context(env):
     """return whether the current callback context is cv or train"""
@@ -325,8 +330,11 @@ def before_iteration(self, model, epoch, dtrain, evals):
 
     def after_iteration(self, model, epoch, dtrain, evals):
         '''Function called after training iteration.'''
-        return any(c.after_iteration(model, epoch, dtrain, evals)
+        ret = any(c.after_iteration(model, epoch, dtrain, evals)
                    for c in self.callbacks)
+        for c in self.callbacks:
+            self.history.update(c.history)
+        return ret
 
 
 class LearningRateScheduler(TrainingCallback):
@@ -362,7 +370,7 @@ def _allreduce_metric(score):
     world = rabit.get_world_size()
     assert world != 0
     score = rabit.allreduce(score, rabit.Op.SUM) / world
-    return score
+    return score[0]
 
 
 # pylint: disable=too-many-instance-attributes
@@ -375,8 +383,6 @@ class EarlyStopping(TrainingCallback):
     ----------
     data
         data for evaluation.
-    name : str
-        Name of data.
     metric : str/callable
         Name of metric.  Use the default metric if not specified.
     metric_name : str
@@ -392,9 +398,7 @@ class EarlyStopping(TrainingCallback):
     label
         Same as weight for DMatrix, used when input is not a DMatrix.
     '''
-    def __init__(self, name, rounds, metric=None, metric_name='metric',
-                 maximize=False):
-        self.name = name
+    def __init__(self, rounds, metric=None, metric_name=None, maximize=False):
         self.metric = metric
         self.rounds = rounds
         if callable(self.metric):
@@ -420,24 +424,24 @@ def after_training(self, model):
         model.best_iteration = self.rounds
         model.set_attr(best_iteration=str(self.rounds))
 
-    def _update_rounds(self, scores, model, epoch):
+    def _update_rounds(self, scores, name, model, epoch):
         assert len(scores) == 1
         score = scores[0]
         metric, s = score[0], score[1]
         if not self.history:    # First round
             self.current_rounds = 0
-            self.history[self.name] = {}
-            self.history[self.name][metric] = [s]
-            self.best_scores[self.name] = {}
-            self.best_scores[self.name][metric] = [s]
-        elif not self.improve_op(s, self.best_scores[self.name][metric][-1]):
+            self.history[name] = {}
+            self.history[name][metric] = [s]
+            self.best_scores[name] = {}
+            self.best_scores[name][metric] = [s]
+        elif not self.improve_op(s, self.best_scores[name][metric][-1]):
             # Not improved
-            self.history[self.name][metric].append(s)
+            self.history[name][metric].append(s)
             self.current_rounds += 1
         else:                   # Improved
-            self.history[self.name][metric].append(s)
-            self.best_scores[self.name][metric].append(s)
-            record = self.history[self.name][metric][-1]
+            self.history[name][metric].append(s)
+            self.best_scores[name][metric].append(s)
+            record = self.history[name][metric][-1]
             model.set_attr(best_score=str(record),
                            best_iteration=str(epoch))
             self.current_rounds = 0  # reset
@@ -492,41 +496,37 @@ class EvaluationMonitor(TrainingCallback):
     label
         Used when data is not a DMatrix.
     '''
-    def __init__(self, name, metric=None, rank=0):
-        self.name = name
+    def __init__(self, metric=None, rank=0):
         self.metric = metric
         self.printer_rank = rank
         super().__init__()
 
-    def before_training(self, model):
-        model.set_param({'eval_metric': self.metric})
-
     def _update_history(self, score, epoch):
-        score = [s.split(':') for s in score.split()]
-        score = [(k, float(v)) for k, v in score[1:]]
-
-        if rabit.get_rank() == self.printer_rank:
-            msg = _fmt_metric(score[0])
-            rabit.tracker_print('[%d]\t%s\n' % (epoch, msg))
-
-        def metric_name():
-            if self.metric:
-                return self.metric
-            name = score[0][0]
-            pos = name.index('-')
-            name = name[pos+1:]
-            return name
-
-        if not self.history:
-            self.history[metric_name()] = [score[0][1]]
-        else:
-            self.history[metric_name()].append(score[0][1])
+        split_by_data = score.split()[1:]  # remove iteration
+
+        for d in split_by_data:
+            name, s = d.split(':')
+            data_name, metric_name = name.split('-')
+            s = float(s)
+            s = _allreduce_metric(s)
+            if data_name in self.history:
+                data_history = self.history[data_name]
+                if metric_name in data_history:
+                    data_history[metric_name].append(s)
+                else:
+                    data_history[metric_name] = [s]
+            else:
+                self.history[data_name] = {}
+                self.history[data_name][metric_name] = [s]
 
+        if rabit.get_rank() == 0:
+            rabit.tracker_print(score + '\n')
         return False
 
     def after_iteration(self, model, epoch, dtrain, evals):
-        score = model.eval(self.data, self.name)
-        _allreduce_metric(score)
+        evals = [] if evals is None else evals
+        feval = self.metric if callable(self.metric) else None
+        score = model.eval_set(evals, epoch, feval)
         return self._update_history(score, epoch)
 
 
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 7ffce8ecb961..3189d6319452 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -10,6 +10,27 @@
 from . import callback
 
 
+def _configure_deprected_callbacks(
+        verbose_eval, early_stopping_rounds, maximize, start_iteration,
+        num_boost_round, evals, feval, evals_result, callbacks):
+    # Most of legacy advanced options becomes callbacks
+    if isinstance(verbose_eval, bool) and verbose_eval:
+        callbacks.append(callback.print_evaluation())
+    else:
+        if isinstance(verbose_eval, int):
+            callbacks.append(callback.print_evaluation(verbose_eval))
+
+    if early_stopping_rounds is not None:
+        callbacks.append(callback.early_stop(early_stopping_rounds,
+                                             maximize=maximize,
+                                             verbose=bool(verbose_eval)))
+    if evals_result is not None:
+        callbacks.append(callback.record_evaluation(evals_result))
+    callbacks = callback.LegacyCallbacks(
+        callbacks, start_iteration, num_boost_round, evals, feval)
+    return callbacks
+
+
 def _train_internal(params, dtrain,
                     num_boost_round=10, evals=(),
                     obj=None, feval=None,
@@ -46,25 +67,18 @@ def _train_internal(params, dtrain,
 
     is_new_callback = [isinstance(c, callback.TrainingCallback)
                        for c in callbacks]
-    if any(is_new_callback):
+    if any(is_new_callback) or not callbacks:
         assert all(is_new_callback), "You can't mix two styles of callbacks."
+        if verbose_eval:
+            callbacks.append(callback.EvaluationMonitor())
+        if early_stopping_rounds:
+            callbacks.append(callback.EarlyStopping)
         callbacks = callback.CallbackContainer(callbacks)
     else:
-        # Most of legacy advanced options becomes callbacks
-        if isinstance(verbose_eval, bool) and verbose_eval:
-            callbacks.append(callback.print_evaluation())
-        else:
-            if isinstance(verbose_eval, int):
-                callbacks.append(callback.print_evaluation(verbose_eval))
-
-        if early_stopping_rounds is not None:
-            callbacks.append(callback.early_stop(early_stopping_rounds,
-                                                 maximize=maximize,
-                                                 verbose=bool(verbose_eval)))
-        if evals_result is not None:
-            callbacks.append(callback.record_evaluation(evals_result))
-        callbacks = callback.LegacyCallbacks(
-            callbacks, start_iteration, num_boost_round, evals, feval)
+        assert False
+        callbacks = _configure_deprected_callbacks(
+            verbose_eval, early_stopping_rounds, maximize, start_iteration,
+            num_boost_round, evals, feval, evals_result, callbacks)
 
     callbacks.before_training(bst)
     for i in range(start_iteration, num_boost_round):
@@ -90,6 +104,7 @@ def _train_internal(params, dtrain,
 
     callbacks.after_training(bst)
 
+    evals_result.update(callbacks.history)
     if bst.attr('best_score') is not None:
         bst.best_score = float(bst.attr('best_score'))
         bst.best_iteration = int(bst.attr('best_iteration'))
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 5bf196dde225..14e5e8f0f689 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -1,9 +1,42 @@
 import xgboost as xgb
+import unittest
 import pytest
 import os
 import testing as tm
 import tempfile
 
+# We use the dataset for tests.
+pytestmark = pytest.mark.skipif(**tm.no_sklearn())
+
+
+class TestCallbacks(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        cls.X = X
+        cls.y = y
+
+        split = int(X.shape[0]*0.8)
+        cls.X_train = X[: split, ...]
+        cls.y_train = y[: split, ...]
+        cls.X_valid = X[split:, ...]
+        cls.y_valid = y[split:, ...]
+
+    def test_evaluation_monitor(self):
+        D_train = xgb.DMatrix(self.X_train, self.y_train)
+        D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
+        evals_result = {}
+        rounds = 10
+        xgb.train({'objective': 'binary:logistic'}, D_train,
+                  evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+                  num_boost_round=rounds,
+                  evals_result=evals_result,
+                  verbose_eval=True)
+        print('evals_result:', evals_result)
+        assert len(evals_result['Train']['error']) == rounds
+        assert len(evals_result['Valid']['error']) == rounds
+
 
 def verify_booster_early_stop(booster):
     dump = booster.get_dump(dump_format='json')

From e1b6f680a2f91009f3eb4e4e92e3f14da3335443 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 18:17:54 +0800
Subject: [PATCH 12/53] Test for early stopping.

---
 python-package/xgboost/callback.py |  54 ++++-----
 python-package/xgboost/training.py |   3 +-
 tests/python/test_callback.py      | 171 ++++++++++++++---------------
 3 files changed, 107 insertions(+), 121 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index b42f43f3b871..ee296eec8303 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -316,7 +316,7 @@ def __init__(self, callbacks):
     def before_training(self, model):
         '''Function called before training.'''
         for c in self.callbacks:
-            c.before_training(model)
+            c.before_training(model=model)
 
     def after_training(self, model):
         '''Function called after training.'''
@@ -381,31 +381,28 @@ class EarlyStopping(TrainingCallback):
 
     Parameters
     ----------
-    data
-        data for evaluation.
+    rounds : int
+        Early stopping rounds.
     metric : str/callable
         Name of metric.  Use the default metric if not specified.
     metric_name : str
         Name of metric, used when metric is a callable object.
-    rounds : int
-        Early stopping rounds.
     maximize : bool
         Whether to maximize evaluation metric.
-    missing : float
-        Same as missing for DMatrix, used when input is not a DMatrix.
-    wegiht
-        Same as label for DMatrix, used when input is not a DMatrix.
-    label
-        Same as weight for DMatrix, used when input is not a DMatrix.
     '''
-    def __init__(self, rounds, metric=None, metric_name=None, maximize=False):
+    def __init__(self, rounds, metric=None, metric_name=None, maximize=False,
+                 save_best=False):
         self.metric = metric
         self.rounds = rounds
+        self.save_best = save_best
+        assert self.save_best is False, 'save best is not yet supported.'
+
         if callable(self.metric):
             self.metric_name = metric_name
         else:
             self.metric_name = self.metric
         self.maximize = maximize
+        self.stopping_history = {}
 
         if self.maximize:
             self.improve_op = lambda x, y: x > y
@@ -416,49 +413,40 @@ def __init__(self, rounds, metric=None, metric_name=None, maximize=False):
         self.best_scores = {}
         super().__init__()
 
-    def before_training(self, model):
-        if not callable(self.metric):
-            model.set_param({'eval_metric': self.metric})
-
-    def after_training(self, model):
-        model.best_iteration = self.rounds
-        model.set_attr(best_iteration=str(self.rounds))
-
     def _update_rounds(self, scores, name, model, epoch):
         assert len(scores) == 1
         score = scores[0]
         metric, s = score[0], score[1]
-        if not self.history:    # First round
+        if not self.stopping_history:    # First round
             self.current_rounds = 0
-            self.history[name] = {}
-            self.history[name][metric] = [s]
+            self.stopping_history[name] = {}
+            self.stopping_history[name][metric] = [s]
             self.best_scores[name] = {}
             self.best_scores[name][metric] = [s]
         elif not self.improve_op(s, self.best_scores[name][metric][-1]):
             # Not improved
-            self.history[name][metric].append(s)
+            self.stopping_history[name][metric].append(s)
             self.current_rounds += 1
         else:                   # Improved
-            self.history[name][metric].append(s)
+            self.stopping_history[name][metric].append(s)
             self.best_scores[name][metric].append(s)
-            record = self.history[name][metric][-1]
+            record = self.stopping_history[name][metric][-1]
             model.set_attr(best_score=str(record),
                            best_iteration=str(epoch))
             self.current_rounds = 0  # reset
 
         if self.current_rounds >= self.rounds:
+            # Should stop
             return True
         return False
 
     def after_iteration(self, model, epoch, dtrain, evals):
-        assert not rabit.is_distributed(), '''
-Use distributed version instead.  For dask users:
-
->>>  from xgboost.dask import EarlyStopping
-'''
         msg = 'Must have at least 1 validation dataset for early stopping.'
         assert len(evals) >= 1, msg
-        stopping_data = evals[-1][1]
+        stopping_data = evals[-1][0]
+        assert isinstance(stopping_data, DMatrix)
+        stopping_name = evals[-1][1]
+        assert isinstance(stopping_name, str)
         if callable(self.metric):
             label = stopping_data.get_label()
             predt = model.predict(stopping_data)
@@ -470,7 +458,7 @@ def after_iteration(self, model, epoch, dtrain, evals):
             score = [s.split(':') for s in score.split()]
             score = [(k, float(v)) for k, v in score[1:]]
 
-        return self._update_rounds(score, model, epoch)
+        return self._update_rounds(score, stopping_name, model, epoch)
 
 
 class EvaluationMonitor(TrainingCallback):
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 3189d6319452..15cf25b3acb3 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -72,7 +72,8 @@ def _train_internal(params, dtrain,
         if verbose_eval:
             callbacks.append(callback.EvaluationMonitor())
         if early_stopping_rounds:
-            callbacks.append(callback.EarlyStopping)
+            callbacks.append(callback.EarlyStopping(
+                rounds=early_stopping_rounds))
         callbacks = callback.CallbackContainer(callbacks)
     else:
         assert False
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 14e5e8f0f689..70ba48539157 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -37,90 +37,87 @@ def test_evaluation_monitor(self):
         assert len(evals_result['Train']['error']) == rounds
         assert len(evals_result['Valid']['error']) == rounds
 
-
-def verify_booster_early_stop(booster):
-    dump = booster.get_dump(dump_format='json')
-    assert len(dump) == 10      # boosted for 10 rounds.
-
-
-@pytest.mark.skipif(**tm.no_sklearn())
-def test_early_stopping():
-    from sklearn.datasets import load_breast_cancer
-    X, y = load_breast_cancer(return_X_y=True)
-    m = xgb.DMatrix(X, y)
-    booster = xgb.train({'objective': 'binary:logistic'}, m,
-                        evals=[(m, 'Train')],
-                        num_boost_round=1000,
-                        early_stopping_rounds=5,
-                        verbose_eval=False)
-    verify_booster_early_stop(booster)
-
-
-@pytest.mark.skipif(**tm.no_sklearn())
-def test_early_stopping_custom_eval():
-    from sklearn.datasets import load_breast_cancer
-    X, y = load_breast_cancer(return_X_y=True)
-    m = xgb.DMatrix(X, y)
-    early_stopping_rounds = 5
-    booster = xgb.train({'objective': 'binary:logistic',
-                         'tree_method': 'hist'}, m,
-                        evals=[(m, 'Train')],
-                        feval=tm.eval_error_metric,
-                        num_boost_round=1000,
-                        early_stopping_rounds=early_stopping_rounds,
-                        verbose_eval=False)
-    dump = booster.get_dump(dump_format='json')
-    assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
-
-
-@pytest.mark.skipif(**tm.no_sklearn())
-def test_early_stopping_skl():
-    from sklearn.datasets import load_breast_cancer
-    X, y = load_breast_cancer(return_X_y=True)
-    cls = xgb.XGBClassifier()
-    cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=5)
-    booster = cls.get_booster()
-    verify_booster_early_stop(booster)
-
-
-@pytest.mark.skipif(**tm.no_sklearn())
-def test_early_stopping_custom_eval_skl():
-    from sklearn.datasets import load_breast_cancer
-    X, y = load_breast_cancer(return_X_y=True)
-    cls = xgb.XGBClassifier()
-    cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=5)
-    booster = cls.get_booster()
-    verify_booster_early_stop(booster)
-
-
-def test_learning_rate_scheduler():
-    pass
-
-
-def test_check_point():
-    from sklearn.datasets import load_breast_cancer
-    X, y = load_breast_cancer(return_X_y=True)
-    m = xgb.DMatrix(X, y)
-    with tempfile.TemporaryDirectory() as tmpdir:
-        check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
-                                                      rounds=1,
-                                                      name='model')
-        xgb.train({'objective': 'binary:logistic'}, m,
-                  num_boost_round=10,
-                  verbose_eval=False,
-                  callbacks=[check_point])
-        for i in range(0, 10):
-            assert os.path.exists(
-                os.path.join(tmpdir, 'model_' + str(i) + '.json'))
-
-        check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
-                                                      rounds=1,
-                                                      as_pickle=True,
-                                                      name='model')
-        xgb.train({'objective': 'binary:logistic'}, m,
-                  num_boost_round=10,
-                  verbose_eval=False,
-                  callbacks=[check_point])
-        for i in range(0, 10):
-            assert os.path.exists(
-                os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))
+    def test_early_stopping(self):
+        D_train = xgb.DMatrix(self.X_train, self.y_train)
+        D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
+        evals_result = {}
+        rounds = 30
+        early_stopping_rounds = 5
+        booster = xgb.train({'objective': 'binary:logistic'}, D_train,
+                            evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+                            num_boost_round=rounds,
+                            evals_result=evals_result,
+                            verbose_eval=True,
+                            early_stopping_rounds=early_stopping_rounds)
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
+    def test_check_point(self):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        m = xgb.DMatrix(X, y)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
+                                                          rounds=1,
+                                                          name='model')
+            xgb.train({'objective': 'binary:logistic'}, m,
+                      num_boost_round=10,
+                      verbose_eval=False,
+                      callbacks=[check_point])
+            for i in range(0, 10):
+                assert os.path.exists(
+                    os.path.join(tmpdir, 'model_' + str(i) + '.json'))
+
+            check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
+                                                          rounds=1,
+                                                          as_pickle=True,
+                                                          name='model')
+            xgb.train({'objective': 'binary:logistic'}, m,
+                      num_boost_round=10,
+                      verbose_eval=False,
+                      callbacks=[check_point])
+            for i in range(0, 10):
+                assert os.path.exists(
+                    os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))
+
+
+
+# @pytest.mark.skipif(**tm.no_sklearn())
+# def test_early_stopping_custom_eval():
+#     from sklearn.datasets import load_breast_cancer
+#     X, y = load_breast_cancer(return_X_y=True)
+#     m = xgb.DMatrix(X, y)
+#     early_stopping_rounds = 5
+#     booster = xgb.train({'objective': 'binary:logistic',
+#                          'tree_method': 'hist'}, m,
+#                         evals=[(m, 'Train')],
+#                         feval=tm.eval_error_metric,
+#                         num_boost_round=1000,
+#                         early_stopping_rounds=early_stopping_rounds,
+#                         verbose_eval=False)
+#     dump = booster.get_dump(dump_format='json')
+#     assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
+
+# @pytest.mark.skipif(**tm.no_sklearn())
+# def test_early_stopping_skl():
+#     from sklearn.datasets import load_breast_cancer
+#     X, y = load_breast_cancer(return_X_y=True)
+#     cls = xgb.XGBClassifier()
+#     cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=5)
+#     booster = cls.get_booster()
+#     verify_booster_early_stop(booster)
+
+
+# @pytest.mark.skipif(**tm.no_sklearn())
+# def test_early_stopping_custom_eval_skl():
+#     from sklearn.datasets import load_breast_cancer
+#     X, y = load_breast_cancer(return_X_y=True)
+#     cls = xgb.XGBClassifier()
+#     cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=5)
+#     booster = cls.get_booster()
+#     verify_booster_early_stop(booster)
+
+
+# def test_learning_rate_scheduler():
+#     pass

From 888c9ccf4713cc998a55731c6f6fd0b8c96fe55a Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 18:22:28 +0800
Subject: [PATCH 13/53] Extract dask test.

---
 tests/python/test_with_dask.py | 65 ++++++++++++++++++----------------
 1 file changed, 34 insertions(+), 31 deletions(-)

diff --git a/tests/python/test_with_dask.py b/tests/python/test_with_dask.py
index 6b7f674b5994..ab6dd2ac1afe 100644
--- a/tests/python/test_with_dask.py
+++ b/tests/python/test_with_dask.py
@@ -648,37 +648,6 @@ def test_hist(self, params, num_rounds, dataset, client):
     def test_approx(self, client, params, num_rounds, dataset):
         self.run_updater_test(client, params, num_rounds, dataset, 'approx')
 
-    def test_early_stopping(self, client):
-        from sklearn.datasets import load_breast_cancer
-        X, y = load_breast_cancer(return_X_y=True)
-        X, y = da.from_array(X), da.from_array(y)
-        m = xgb.dask.DaskDMatrix(client, X, y)
-        booster = xgb.dask.train(client, {'objective': 'binary:logistic',
-                                          'tree_method': 'hist'}, m,
-                                 evals=[(m, 'Train')],
-                                 num_boost_round=1000,
-                                 early_stopping_rounds=5)['booster']
-        assert hasattr(booster, 'best_score')
-        assert booster.best_iteration == 10
-
-    @pytest.mark.skipif(**tm.no_sklearn())
-    def test_early_stopping_custom_eval(self, client):
-        from sklearn.datasets import load_breast_cancer
-        X, y = load_breast_cancer(return_X_y=True)
-        X, y = da.from_array(X), da.from_array(y)
-        m = xgb.dask.DaskDMatrix(client, X, y)
-        early_stopping_rounds = 5
-        booster = xgb.dask.train(
-            client, {'objective': 'binary:logistic',
-                     'tree_method': 'hist'}, m,
-            evals=[(m, 'Train')],
-            feval=tm.eval_error_metric,
-            num_boost_round=1000,
-            early_stopping_rounds=early_stopping_rounds)['booster']
-        assert hasattr(booster, 'best_score')
-        dump = booster.get_dump(dump_format='json')
-        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
-
     def run_quantile(self, name):
         if sys.platform.startswith("win"):
             pytest.skip("Skipping dask tests on Windows")
@@ -736,3 +705,37 @@ def test_quantile(self):
     @pytest.mark.gtest
     def test_quantile_same_on_all_workers(self):
         self.run_quantile('SameOnAllWorkers')
+
+
+class TestDaskCallbacks:
+    @pytest.mark.skipif(**tm.no_sklearn())
+    def test_early_stopping(self, client):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        X, y = da.from_array(X), da.from_array(y)
+        m = xgb.dask.DaskDMatrix(client, X, y)
+        booster = xgb.dask.train(client, {'objective': 'binary:logistic',
+                                          'tree_method': 'hist'}, m,
+                                 evals=[(m, 'Train')],
+                                 num_boost_round=1000,
+                                 early_stopping_rounds=5)['booster']
+        assert hasattr(booster, 'best_score')
+        assert booster.best_iteration == 10
+
+    @pytest.mark.skipif(**tm.no_sklearn())
+    def test_early_stopping_custom_eval(self, client):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        X, y = da.from_array(X), da.from_array(y)
+        m = xgb.dask.DaskDMatrix(client, X, y)
+        early_stopping_rounds = 5
+        booster = xgb.dask.train(
+            client, {'objective': 'binary:logistic',
+                     'tree_method': 'hist'}, m,
+            evals=[(m, 'Train')],
+            feval=tm.eval_error_metric,
+            num_boost_round=1000,
+            early_stopping_rounds=early_stopping_rounds)['booster']
+        assert hasattr(booster, 'best_score')
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1

From f6a8a2bff22b708b9f7ce4040e41d0c5880352d0 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 18:23:00 +0800
Subject: [PATCH 14/53] Allreduce.

---
 python-package/xgboost/callback.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index ee296eec8303..ff7c03e48f7c 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -417,6 +417,7 @@ def _update_rounds(self, scores, name, model, epoch):
         assert len(scores) == 1
         score = scores[0]
         metric, s = score[0], score[1]
+        s = _allreduce_metric(s)
         if not self.stopping_history:    # First round
             self.current_rounds = 0
             self.stopping_history[name] = {}

From 3129762584da56730f5bf654a6f81e9f4d0e2591 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 18:26:42 +0800
Subject: [PATCH 15/53] Add check point demo.

---
 demo/guide-python/callbacks.py | 41 ++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 demo/guide-python/callbacks.py

diff --git a/demo/guide-python/callbacks.py b/demo/guide-python/callbacks.py
new file mode 100644
index 000000000000..e4a0630782a1
--- /dev/null
+++ b/demo/guide-python/callbacks.py
@@ -0,0 +1,41 @@
+import xgboost as xgb
+import tempfile
+import os
+from sklearn.datasets import load_breast_cancer
+
+
+def check_point_callback():
+    X, y = load_breast_cancer(return_X_y=True)
+    m = xgb.DMatrix(X, y)
+    # Check point to a temporary directory for demo
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # Use callback class from xgboost.callback
+        # Feel free to subclass/customize it to suite your need.
+        check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
+                                                      rounds=2,
+                                                      name='model')
+        xgb.train({'objective': 'binary:logistic'}, m,
+                  num_boost_round=10,
+                  verbose_eval=False,
+                  callbacks=[check_point])
+        for i in range(0, 10):
+            assert os.path.exists(
+                os.path.join(tmpdir, 'model_' + str(i) + '.json'))
+
+        # This version of checkpoint saves everything including parameters and
+        # model.  See: doc/tutorials/saving_model.rst
+        check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
+                                                      rounds=1,
+                                                      as_pickle=True,
+                                                      name='model')
+        xgb.train({'objective': 'binary:logistic'}, m,
+                  num_boost_round=10,
+                  verbose_eval=False,
+                  callbacks=[check_point])
+        for i in range(0, 10):
+            assert os.path.exists(
+                os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))
+
+
+if __name__ == '__main__':
+    check_point_callback()

From 88d5d9bb8f3b556742d0195ed9c24ba05803e422 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 18:32:55 +0800
Subject: [PATCH 16/53] Add test for ES custom feval

---
 python-package/xgboost/training.py |  4 +++-
 tests/python/test_callback.py      | 28 ++++++++++++++--------------
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 15cf25b3acb3..fdcb64c339ff 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -105,7 +105,9 @@ def _train_internal(params, dtrain,
 
     callbacks.after_training(bst)
 
-    evals_result.update(callbacks.history)
+    if evals_result:
+        evals_result.update(callbacks.history)
+
     if bst.attr('best_score') is not None:
         bst.best_score = float(bst.attr('best_score'))
         bst.best_iteration = int(bst.attr('best_iteration'))
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 70ba48539157..16f60da2a811 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -52,6 +52,20 @@ def test_early_stopping(self):
         dump = booster.get_dump(dump_format='json')
         assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
 
+    def test_early_stopping_custom_eval(self):
+        D_train = xgb.DMatrix(self.X_train, self.y_train)
+        D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
+        early_stopping_rounds = 5
+        booster = xgb.train({'objective': 'binary:logistic',
+                             'tree_method': 'hist'}, D_train,
+                            evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+                            feval=tm.eval_error_metric,
+                            num_boost_round=1000,
+                            early_stopping_rounds=early_stopping_rounds,
+                            verbose_eval=False)
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
     def test_check_point(self):
         from sklearn.datasets import load_breast_cancer
         X, y = load_breast_cancer(return_X_y=True)
@@ -83,20 +97,6 @@ def test_check_point(self):
 
 
 # @pytest.mark.skipif(**tm.no_sklearn())
-# def test_early_stopping_custom_eval():
-#     from sklearn.datasets import load_breast_cancer
-#     X, y = load_breast_cancer(return_X_y=True)
-#     m = xgb.DMatrix(X, y)
-#     early_stopping_rounds = 5
-#     booster = xgb.train({'objective': 'binary:logistic',
-#                          'tree_method': 'hist'}, m,
-#                         evals=[(m, 'Train')],
-#                         feval=tm.eval_error_metric,
-#                         num_boost_round=1000,
-#                         early_stopping_rounds=early_stopping_rounds,
-#                         verbose_eval=False)
-#     dump = booster.get_dump(dump_format='json')
-#     assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
 
 
 # @pytest.mark.skipif(**tm.no_sklearn())

From 94f041350e30c205f08e9d445712f4abef030909 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 18:35:25 +0800
Subject: [PATCH 17/53] ES custom eval skl.

---
 tests/python/test_callback.py | 47 ++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 25 deletions(-)

diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 16f60da2a811..4aef217bb432 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -66,6 +66,28 @@ def test_early_stopping_custom_eval(self):
         dump = booster.get_dump(dump_format='json')
         assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
 
+    def test_early_stopping_skl(self):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        cls = xgb.XGBClassifier()
+        early_stopping_rounds = 5
+        cls.fit(X, y, eval_set=[(X, y)],
+                early_stopping_rounds=early_stopping_rounds)
+        booster = cls.get_booster()
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
+    def test_early_stopping_custom_eval_skl(self):
+        from sklearn.datasets import load_breast_cancer
+        X, y = load_breast_cancer(return_X_y=True)
+        cls = xgb.XGBClassifier()
+        early_stopping_rounds = 5
+        cls.fit(X, y, eval_set=[(X, y)],
+                early_stopping_rounds=early_stopping_rounds)
+        booster = cls.get_booster()
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+
     def test_check_point(self):
         from sklearn.datasets import load_breast_cancer
         X, y = load_breast_cancer(return_X_y=True)
@@ -94,30 +116,5 @@ def test_check_point(self):
                 assert os.path.exists(
                     os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))
 
-
-
-# @pytest.mark.skipif(**tm.no_sklearn())
-
-
-# @pytest.mark.skipif(**tm.no_sklearn())
-# def test_early_stopping_skl():
-#     from sklearn.datasets import load_breast_cancer
-#     X, y = load_breast_cancer(return_X_y=True)
-#     cls = xgb.XGBClassifier()
-#     cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=5)
-#     booster = cls.get_booster()
-#     verify_booster_early_stop(booster)
-
-
-# @pytest.mark.skipif(**tm.no_sklearn())
-# def test_early_stopping_custom_eval_skl():
-#     from sklearn.datasets import load_breast_cancer
-#     X, y = load_breast_cancer(return_X_y=True)
-#     cls = xgb.XGBClassifier()
-#     cls.fit(X, y, eval_set=[(X, y)], early_stopping_rounds=5)
-#     booster = cls.get_booster()
-#     verify_booster_early_stop(booster)
-
-
 # def test_learning_rate_scheduler():
 #     pass

From 0a3cca53fc85c074f67f2fd057f4290422345b22 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 18:49:04 +0800
Subject: [PATCH 18/53] Test learning rate scheduling.

* Add breaking.
---
 python-package/xgboost/callback.py |  5 +-
 python-package/xgboost/training.py |  2 +-
 tests/python/test_basic_models.py  | 80 +-------------------------
 tests/python/test_callback.py      | 92 +++++++++++++++++++++++++++++-
 4 files changed, 95 insertions(+), 84 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index ff7c03e48f7c..c334f1d9872f 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -278,6 +278,9 @@ def callback(env):
 # - enforced best_xxx
 # - merged functionality of es and mon.
 
+# Breaking:
+# - reset learning rate no longer accepts total boosting rounds
+
 # pylint: disable=unused-argument
 class TrainingCallback(ABC):
     '''Interface for training callback.
@@ -348,7 +351,7 @@ class LearningRateScheduler(TrainingCallback):
     learning_rates : callable/collections.Sequence
         If it's a callable object, then it should accept an integer parameter
         `epoch` and returns the corresponding learning rate.  Otherwise it
-        shoule be a sequence like list or tuple with the same size of boosting
+        should be a sequence like list or tuple with the same size of boosting
         rounds.
 
     '''
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index fdcb64c339ff..4e8247b6bcb7 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -105,7 +105,7 @@ def _train_internal(params, dtrain,
 
     callbacks.after_training(bst)
 
-    if evals_result:
+    if evals_result is not None:
         evals_result.update(callbacks.history)
 
     if bst.attr('best_score') is not None:
diff --git a/tests/python/test_basic_models.py b/tests/python/test_basic_models.py
index 0b6fb229c076..dc3a2778af02 100644
--- a/tests/python/test_basic_models.py
+++ b/tests/python/test_basic_models.py
@@ -8,7 +8,7 @@
 import locale
 import tempfile
 
-dpath = 'demo/data/'
+dpath = os.path.join(tm.PROJECT_ROOT, 'demo/data/')
 dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')
 dtest = xgb.DMatrix(dpath + 'agaricus.txt.test')
 
@@ -110,84 +110,6 @@ def my_logloss(preds, dtrain):
             for jj in range(ii + 1, len(preds_list)):
                 assert np.sum(np.abs(preds_list[ii] - preds_list[jj])) > 0
 
-    def run_eta_decay(self, tree_method):
-        watchlist = [(dtest, 'eval'), (dtrain, 'train')]
-        num_round = 4
-
-        # learning_rates as a list
-        # init eta with 0 to check whether learning_rates work
-        param = {'max_depth': 2, 'eta': 0, 'verbosity': 0,
-                 'objective': 'binary:logistic', 'eval_metric': 'error',
-                 'tree_method': tree_method}
-        evals_result = {}
-        bst = xgb.train(param, dtrain, num_round, watchlist,
-                        callbacks=[xgb.callback.reset_learning_rate([
-                            0.8, 0.7, 0.6, 0.5
-                        ])],
-                        evals_result=evals_result)
-        eval_errors_0 = list(map(float, evals_result['eval']['error']))
-        assert isinstance(bst, xgb.core.Booster)
-        # validation error should decrease, if eta > 0
-        assert eval_errors_0[0] > eval_errors_0[-1]
-
-        # init learning_rate with 0 to check whether learning_rates work
-        param = {'max_depth': 2, 'learning_rate': 0, 'verbosity': 0,
-                 'objective': 'binary:logistic', 'eval_metric': 'error',
-                 'tree_method': tree_method}
-        evals_result = {}
-        bst = xgb.train(param, dtrain, num_round, watchlist,
-                        callbacks=[xgb.callback.reset_learning_rate(
-                            [0.8, 0.7, 0.6, 0.5])],
-                        evals_result=evals_result)
-        eval_errors_1 = list(map(float, evals_result['eval']['error']))
-        assert isinstance(bst, xgb.core.Booster)
-        # validation error should decrease, if learning_rate > 0
-        assert eval_errors_1[0] > eval_errors_1[-1]
-
-        # check if learning_rates override default value of eta/learning_rate
-        param = {
-            'max_depth': 2, 'verbosity': 0, 'objective': 'binary:logistic',
-            'eval_metric': 'error', 'tree_method': tree_method
-        }
-        evals_result = {}
-        bst = xgb.train(param, dtrain, num_round, watchlist,
-                        callbacks=[xgb.callback.reset_learning_rate(
-                            [0, 0, 0, 0]
-                        )],
-                        evals_result=evals_result)
-        eval_errors_2 = list(map(float, evals_result['eval']['error']))
-        assert isinstance(bst, xgb.core.Booster)
-        # validation error should not decrease, if eta/learning_rate = 0
-        assert eval_errors_2[0] == eval_errors_2[-1]
-
-        # learning_rates as a customized decay function
-        def eta_decay(ithround, num_boost_round):
-            return num_boost_round / (ithround + 1)
-
-        evals_result = {}
-        bst = xgb.train(param, dtrain, num_round, watchlist,
-                        callbacks=[
-                            xgb.callback.reset_learning_rate(eta_decay)
-                        ],
-                        evals_result=evals_result)
-        eval_errors_3 = list(map(float, evals_result['eval']['error']))
-
-        assert isinstance(bst, xgb.core.Booster)
-
-        assert eval_errors_3[0] == eval_errors_2[0]
-
-        for i in range(1, len(eval_errors_0)):
-            assert eval_errors_3[i] != eval_errors_2[i]
-
-    def test_eta_decay_hist(self):
-        self.run_eta_decay('hist')
-
-    def test_eta_decay_approx(self):
-        self.run_eta_decay('approx')
-
-    def test_eta_decay_exact(self):
-        self.run_eta_decay('exact')
-
     def test_boost_from_prediction(self):
         # Re-construct dtrain here to avoid modification
         margined = xgb.DMatrix(dpath + 'agaricus.txt.train')
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 4aef217bb432..1e1b82276373 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -88,6 +88,95 @@ def test_early_stopping_custom_eval_skl(self):
         dump = booster.get_dump(dump_format='json')
         assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
 
+    def run_eta_decay(self, tree_method, deprecated_callback):
+        if deprecated_callback:
+            scheduler = xgb.callback.reset_learning_rate
+        else:
+            scheduler = xgb.callback.LearningRateScheduler
+
+        dpath = os.path.join(tm.PROJECT_ROOT, 'demo/data/')
+        dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')
+        dtest = xgb.DMatrix(dpath + 'agaricus.txt.test')
+        watchlist = [(dtest, 'eval'), (dtrain, 'train')]
+        num_round = 4
+
+        # learning_rates as a list
+        # init eta with 0 to check whether learning_rates work
+        param = {'max_depth': 2, 'eta': 0, 'verbosity': 0,
+                 'objective': 'binary:logistic', 'eval_metric': 'error',
+                 'tree_method': tree_method}
+        evals_result = {}
+        bst = xgb.train(param, dtrain, num_round, watchlist,
+                        callbacks=[scheduler([
+                            0.8, 0.7, 0.6, 0.5
+                        ])],
+                        evals_result=evals_result)
+        eval_errors_0 = list(map(float, evals_result['eval']['error']))
+        assert isinstance(bst, xgb.core.Booster)
+        # validation error should decrease, if eta > 0
+        assert eval_errors_0[0] > eval_errors_0[-1]
+
+        # init learning_rate with 0 to check whether learning_rates work
+        param = {'max_depth': 2, 'learning_rate': 0, 'verbosity': 0,
+                 'objective': 'binary:logistic', 'eval_metric': 'error',
+                 'tree_method': tree_method}
+        evals_result = {}
+        bst = xgb.train(param, dtrain, num_round, watchlist,
+                        callbacks=[scheduler(
+                            [0.8, 0.7, 0.6, 0.5])],
+                        evals_result=evals_result)
+        eval_errors_1 = list(map(float, evals_result['eval']['error']))
+        assert isinstance(bst, xgb.core.Booster)
+        # validation error should decrease, if learning_rate > 0
+        assert eval_errors_1[0] > eval_errors_1[-1]
+
+        # check if learning_rates override default value of eta/learning_rate
+        param = {
+            'max_depth': 2, 'verbosity': 0, 'objective': 'binary:logistic',
+            'eval_metric': 'error', 'tree_method': tree_method
+        }
+        evals_result = {}
+        bst = xgb.train(param, dtrain, num_round, watchlist,
+                        callbacks=[scheduler(
+                            [0, 0, 0, 0]
+                        )],
+                        evals_result=evals_result)
+        eval_errors_2 = list(map(float, evals_result['eval']['error']))
+        assert isinstance(bst, xgb.core.Booster)
+        # validation error should not decrease, if eta/learning_rate = 0
+        assert eval_errors_2[0] == eval_errors_2[-1]
+
+        # learning_rates as a customized decay function
+        def eta_decay(ithround, num_boost_round=num_round):
+            return num_boost_round / (ithround + 1)
+
+        evals_result = {}
+        bst = xgb.train(param, dtrain, num_round, watchlist,
+                        callbacks=[
+                            scheduler(eta_decay)
+                        ],
+                        evals_result=evals_result)
+        eval_errors_3 = list(map(float, evals_result['eval']['error']))
+
+        assert isinstance(bst, xgb.core.Booster)
+
+        assert eval_errors_3[0] == eval_errors_2[0]
+
+        for i in range(1, len(eval_errors_0)):
+            assert eval_errors_3[i] != eval_errors_2[i]
+
+    def test_eta_decay_hist(self):
+        # self.run_eta_decay('hist', True)
+        self.run_eta_decay('hist', False)
+
+    def test_eta_decay_approx(self):
+        # self.run_eta_decay('approx', True)
+        self.run_eta_decay('approx', False)
+
+    def test_eta_decay_exact(self):
+        # self.run_eta_decay('exact', True)
+        self.run_eta_decay('exact', False)
+
     def test_check_point(self):
         from sklearn.datasets import load_breast_cancer
         X, y = load_breast_cancer(return_X_y=True)
@@ -115,6 +204,3 @@ def test_check_point(self):
             for i in range(0, 10):
                 assert os.path.exists(
                     os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))
-
-# def test_learning_rate_scheduler():
-#     pass

From 13332d75b4d2db84fe8420e82f714b5433cb2fd5 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 18:52:25 +0800
Subject: [PATCH 19/53] Lint.

---
 python-package/xgboost/callback.py | 27 +++++++--------------------
 1 file changed, 7 insertions(+), 20 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index c334f1d9872f..6f068538f36c 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -5,18 +5,13 @@
 from abc import ABC
 import collections
 import os
-import numpy
 import pickle
+import numpy
 
 from . import rabit
 from .core import EarlyStopException, DMatrix, CallbackEnv
 from .compat import STRING_TYPES
 
-import sys
-def fprint(*args, **kwargs):
-    print(*args, **kwargs)
-    sys.stdout.flush()
-
 
 def _get_callback_context(env):
     """return whether the current callback context is cv or train"""
@@ -334,7 +329,7 @@ def before_iteration(self, model, epoch, dtrain, evals):
     def after_iteration(self, model, epoch, dtrain, evals):
         '''Function called after training iteration.'''
         ret = any(c.after_iteration(model, epoch, dtrain, evals)
-                   for c in self.callbacks)
+                  for c in self.callbacks)
         for c in self.callbacks:
             self.history.update(c.history)
         return ret
@@ -455,7 +450,7 @@ def after_iteration(self, model, epoch, dtrain, evals):
             label = stopping_data.get_label()
             predt = model.predict(stopping_data)
             score = self.metric(label, predt)
-            score = _allreduce_metric(score, self.maximize)
+            score = _allreduce_metric(score)
             score = [(self.metric_name, score)]
         else:
             score = model.eval(stopping_data)
@@ -473,20 +468,10 @@ class EvaluationMonitor(TrainingCallback):
     Parameters
     ----------
 
-    data
-        Data for evaluation.
-    name : str
-        Name of data.
     metric : str
         Name of metric
     rank : int
         Which worker should be used for printing the result.
-    missing : float
-        Used when data is not a DMatrix.
-    weight
-        Used when data is not a DMatrix.
-    label
-        Used when data is not a DMatrix.
     '''
     def __init__(self, metric=None, rank=0):
         self.metric = metric
@@ -532,8 +517,9 @@ class TrainingCheckPoint(TrainingCallback):
 
     path : os.PathLike
         Output model directory.
-    name : name pattern of output model file.
-        Models will be saved as name_0.json, name_1.json, name_2.json ....
+    name : str
+        pattern of output model file.  Models will be saved as name_0.json,
+        name_1.json, name_2.json ....
     as_pickle : boolean
         When set to Ture, all training parameters will be saved in pickle
         format, instead of saving only the model.
@@ -548,6 +534,7 @@ def __init__(self, directory: os.PathLike, name: str = 'model',
         self._as_pickle = as_pickle
         self._iterations = rounds
         self._epoch = 0
+        super().__init__()
 
     def after_iteration(self, model, epoch, dtrain, evals):
         self._epoch += 1

From 5731802043131b062ca935adafa9ff9409d25b81 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 19:38:04 +0800
Subject: [PATCH 20/53] [ES] Consider specified metric name and data name.

---
 python-package/xgboost/callback.py | 75 ++++++++++++++++++++----------
 python-package/xgboost/training.py |  4 +-
 2 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 6f068538f36c..f57f7772887a 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -384,21 +384,26 @@ class EarlyStopping(TrainingCallback):
     metric : str/callable
         Name of metric.  Use the default metric if not specified.
     metric_name : str
-        Name of metric, used when metric is a callable object.
+        Name of metric that is used for early stopping.
+    data_name: str
+        Name of dataset that is used for early stopping.
     maximize : bool
         Whether to maximize evaluation metric.
     '''
-    def __init__(self, rounds, metric=None, metric_name=None, maximize=False,
+    def __init__(self,
+                 rounds,
+                 metric=None,
+                 metric_name=None,
+                 data_name=None,
+                 maximize=False,
                  save_best=False):
         self.metric = metric
+        self.data = data_name
+        self.metric_name = metric_name
         self.rounds = rounds
         self.save_best = save_best
         assert self.save_best is False, 'save best is not yet supported.'
 
-        if callable(self.metric):
-            self.metric_name = metric_name
-        else:
-            self.metric_name = self.metric
         self.maximize = maximize
         self.stopping_history = {}
 
@@ -412,11 +417,11 @@ def __init__(self, rounds, metric=None, metric_name=None, maximize=False,
         super().__init__()
 
     def _update_rounds(self, scores, name, model, epoch):
-        assert len(scores) == 1
+        assert len(scores) == 1, 'No matching metric name.'
         score = scores[0]
         metric, s = score[0], score[1]
         s = _allreduce_metric(s)
-        if not self.stopping_history:    # First round
+        if not self.stopping_history:  # First round
             self.current_rounds = 0
             self.stopping_history[name] = {}
             self.stopping_history[name][metric] = [s]
@@ -426,12 +431,11 @@ def _update_rounds(self, scores, name, model, epoch):
             # Not improved
             self.stopping_history[name][metric].append(s)
             self.current_rounds += 1
-        else:                   # Improved
+        else:  # Improved
             self.stopping_history[name][metric].append(s)
             self.best_scores[name][metric].append(s)
             record = self.stopping_history[name][metric][-1]
-            model.set_attr(best_score=str(record),
-                           best_iteration=str(epoch))
+            model.set_attr(best_score=str(record), best_iteration=str(epoch))
             self.current_rounds = 0  # reset
 
         if self.current_rounds >= self.rounds:
@@ -442,21 +446,43 @@ def _update_rounds(self, scores, name, model, epoch):
     def after_iteration(self, model, epoch, dtrain, evals):
         msg = 'Must have at least 1 validation dataset for early stopping.'
         assert len(evals) >= 1, msg
-        stopping_data = evals[-1][0]
+        stopping_name = ''
+        stopping_data = None
+        if self.data:
+            for d, name in evals:
+                if name == self.data:
+                    stopping_name = name
+                    stopping_data = d
+            if not stopping_name:
+                raise ValueError('No dataset named:', self.data)
+        else:
+            # Use the last one as default.
+            stopping_name = evals[-1][1]
+            stopping_data = evals[-1][0]
+
         assert isinstance(stopping_data, DMatrix)
-        stopping_name = evals[-1][1]
         assert isinstance(stopping_name, str)
+
         if callable(self.metric):
-            label = stopping_data.get_label()
-            predt = model.predict(stopping_data)
-            score = self.metric(label, predt)
-            score = _allreduce_metric(score)
-            score = [(self.metric_name, score)]
+            score = model.eval_set([(stopping_data, stopping_name)],
+                                   iteration=epoch,
+                                   feval=self.metric)
+            score = [s.split(':') for s in score.split()]
+            score = [(k, _allreduce_metric(float(v))) for k, v in score[1:]]
         else:
-            score = model.eval(stopping_data)
+            score = model.eval_set([(stopping_data, stopping_name)],
+                                   iteration=epoch)
             score = [s.split(':') for s in score.split()]
             score = [(k, float(v)) for k, v in score[1:]]
 
+        # Filter out scores that can not be used for early stopping.
+        if self.metric_name:
+            score = list(
+                filter(lambda s: s[0].split('-')[1] == self.metric_name,
+                       score))
+        else:
+            score = [score[-1]]
+
         return self._update_rounds(score, stopping_name, model, epoch)
 
 
@@ -468,12 +494,14 @@ class EvaluationMonitor(TrainingCallback):
     Parameters
     ----------
 
-    metric : str
-        Name of metric
+    metric : callable
+        Extra user defined metric.
     rank : int
         Which worker should be used for printing the result.
     '''
     def __init__(self, metric=None, rank=0):
+        if metric is not None:
+            assert callable(metric), 'metric must be callable object.'
         self.metric = metric
         self.printer_rank = rank
         super().__init__()
@@ -496,14 +524,13 @@ def _update_history(self, score, epoch):
                 self.history[data_name] = {}
                 self.history[data_name][metric_name] = [s]
 
-        if rabit.get_rank() == 0:
+        if rabit.get_rank() == self.printer_rank:
             rabit.tracker_print(score + '\n')
         return False
 
     def after_iteration(self, model, epoch, dtrain, evals):
         evals = [] if evals is None else evals
-        feval = self.metric if callable(self.metric) else None
-        score = model.eval_set(evals, epoch, feval)
+        score = model.eval_set(evals, epoch, self.metric)
         return self._update_history(score, epoch)
 
 
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 4e8247b6bcb7..ed00b8f439cf 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -70,10 +70,10 @@ def _train_internal(params, dtrain,
     if any(is_new_callback) or not callbacks:
         assert all(is_new_callback), "You can't mix two styles of callbacks."
         if verbose_eval:
-            callbacks.append(callback.EvaluationMonitor())
+            callbacks.append(callback.EvaluationMonitor(metric=feval))
         if early_stopping_rounds:
             callbacks.append(callback.EarlyStopping(
-                rounds=early_stopping_rounds))
+                rounds=early_stopping_rounds, metric=feval))
         callbacks = callback.CallbackContainer(callbacks)
     else:
         assert False

From b24fc131cd6038f5f5e83637d10426439e807ed9 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 19:40:32 +0800
Subject: [PATCH 21/53] Lint.

---
 python-package/xgboost/callback.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index f57f7772887a..8cce07dabe1f 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -501,7 +501,10 @@ class EvaluationMonitor(TrainingCallback):
     '''
     def __init__(self, metric=None, rank=0):
         if metric is not None:
-            assert callable(metric), 'metric must be callable object.'
+            msg = 'metric must be callable object for monitor.  For ' + \
+                'builtin metrics, passing them in training parameter' + \
+                ' will invoke monitor automatically.'
+            assert callable(metric), msg
         self.metric = metric
         self.printer_rank = rank
         super().__init__()

From 5922f2a742510a890fb5bfa88b5591df2843d397 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 20:00:38 +0800
Subject: [PATCH 22/53] Test for customization.

---
 python-package/xgboost/callback.py |  2 +-
 python-package/xgboost/training.py | 10 ++++++++++
 tests/python/test_callback.py      | 21 +++++++++++++++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 8cce07dabe1f..c19d087fac1c 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -419,7 +419,7 @@ def __init__(self,
     def _update_rounds(self, scores, name, model, epoch):
         assert len(scores) == 1, 'No matching metric name.'
         score = scores[0]
-        metric, s = score[0], score[1]
+        metric, s = score[0].split('-')[1], score[1]
         s = _allreduce_metric(s)
         if not self.stopping_history:  # First round
             self.current_rounds = 0
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index ed00b8f439cf..b03b8c0c7a82 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -41,6 +41,16 @@ def _train_internal(params, dtrain,
     callbacks = [] if callbacks is None else callbacks
     evals = list(evals)
     params = params.copy()
+
+    if isinstance(params, dict) and 'eval_metric' in params \
+       and isinstance(params['eval_metric'], list):
+        params = dict((k, v) for k, v in params.items())
+        eval_metrics = params['eval_metric']
+        params.pop("eval_metric", None)
+        params = list(params.items())
+        for eval_metric in eval_metrics:
+            params += [('eval_metric', eval_metric)]
+
     bst = Booster(params, [dtrain] + [d[0] for d in evals])
     nboost = 0
     num_parallel_tree = 1
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 1e1b82276373..836ab497c020 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -66,6 +66,27 @@ def test_early_stopping_custom_eval(self):
         dump = booster.get_dump(dump_format='json')
         assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
 
+    def test_early_stopping_customize(self):
+        D_train = xgb.DMatrix(self.X_train, self.y_train)
+        D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
+        early_stopping_rounds = 5
+        early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
+                                                metric=tm.eval_error_metric,
+                                                metric_name='PyError',
+                                                data_name='Train')
+        # Specify which dataset and which metric should be used for early stopping.
+        booster = xgb.train(
+            {'objective': 'binary:logistic',
+             'eval_metric': ['error', 'rmse'],
+             'tree_method': 'hist'}, D_train,
+            evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+            num_boost_round=1000,
+            callbacks=[early_stop],
+            verbose_eval=False)
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
+        assert len(early_stop.stopping_history['Train']['PyError']) == len(dump)
+
     def test_early_stopping_skl(self):
         from sklearn.datasets import load_breast_cancer
         X, y = load_breast_cancer(return_X_y=True)

From d0e11d7ad0c2eed4b178d45d265dbef3be975d4c Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 20:09:20 +0800
Subject: [PATCH 23/53] Basic doc.

---
 doc/python/callbacks.rst | 64 ++++++++++++++++++++++++++++++++++++++++
 doc/python/index.rst     |  1 +
 2 files changed, 65 insertions(+)
 create mode 100644 doc/python/callbacks.rst

diff --git a/doc/python/callbacks.rst b/doc/python/callbacks.rst
new file mode 100644
index 000000000000..3a4398e7b364
--- /dev/null
+++ b/doc/python/callbacks.rst
@@ -0,0 +1,64 @@
+##################
+Callback Functions
+##################
+
+This document gives a basic walkthrough of callback function used in XGBoost Python
+package.  In XGBoost 1.3, a new callback interface is designed for Python package, which
+provides the flexiablity of designing various extension for training.  Also, XGBoost has a
+number of pre-defined callbacks for supporting early stopping, checkpoints etc.
+
+#######################
+Using builtin callbacks
+#######################
+
+By default, training methods in XGBoost have parameters like ``early_stopping_rounds`` and
+``verbose``/``verbose_eval``, when specified the training procedure will define the
+corresponding callbacks internally.  For example, when ``early_stopping_rounds`` is
+specified, ``EarlyStopping`` callback is invoked inside iteration loop.  You can also pass
+this callback function directly into XGBoost:
+
+.. code-block:: python
+
+    D_train = xgb.DMatrix(X_train, y_train)
+    D_valid = xgb.DMatrix(X_valid, y_valid)
+
+    # Define a custom evaluation metric used for early stopping.
+    def eval_error_metric(predt, dtrain: xgb.DMatrix):
+        label = dtrain.get_label()
+        r = np.zeros(predt.shape)
+        gt = predt > 0.5
+        r[gt] = 1 - label[gt]
+        le = predt <= 0.5
+        r[le] = label[le]
+        return 'PyError', np.sum(r)
+
+    # Specify which dataset and which metric should be used for early stopping.
+    early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
+                                            metric=tm.eval_error_metric,
+                                            metric_name='PyError',
+                                            data_name='Valid')
+    booster = xgb.train(
+        {'objective': 'binary:logistic',
+         'eval_metric': ['error', 'rmse'],
+         'tree_method': 'hist'}, D_train,
+        evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+        num_boost_round=1000,
+        callbacks=[early_stop],
+        verbose_eval=False)
+
+    dump = booster.get_dump(dump_format='json')
+    assert len(early_stop.stopping_history['Valid']['PyError']) == len(dump)
+
+##########################
+Defining your own callback
+##########################
+
+In here we will define a callback for monitoring shap value changes during training.
+First XGBoost provides an interface class: ``xgboost.callback.TrainingCallback``, user
+defined callbacks should inherit this class and override corresponding methods.
+
+.. code-block:: python
+    pass
+
+
+The full example is in.
diff --git a/doc/python/index.rst b/doc/python/index.rst
index 1cbcc451bdd2..7596be247f9b 100644
--- a/doc/python/index.rst
+++ b/doc/python/index.rst
@@ -11,4 +11,5 @@ Contents
 .. toctree::
   python_intro
   python_api
+  callbacks
   Python examples <https://github.com/dmlc/xgboost/tree/master/demo/guide-python>

From 4f4ec6c4956da2033784f27b3c652451019265b1 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 21:21:11 +0800
Subject: [PATCH 24/53] TODO.

---
 python-package/xgboost/callback.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index c19d087fac1c..c6f300daad21 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -272,6 +272,7 @@ def callback(env):
 # - doc
 # - enforced best_xxx
 # - merged functionality of es and mon.
+# - make callbacks a set instead of list.
 
 # Breaking:
 # - reset learning rate no longer accepts total boosting rounds

From 97f678a1ffbf92e73a2657d6176ab5dbad384daa Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 4 Oct 2020 23:15:03 +0800
Subject: [PATCH 25/53] Use evals_log instead of actual data.

---
 python-package/xgboost/callback.py | 145 +++++++++++++----------------
 python-package/xgboost/training.py |   4 +-
 tests/python/test_callback.py      |   2 +-
 3 files changed, 69 insertions(+), 82 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index c6f300daad21..8357b67d6319 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -285,7 +285,7 @@ class TrainingCallback(ABC):
 
     '''
     def __init__(self):
-        self.history = {}
+        pass
 
     def before_training(self, model):
         '''Run before training starts.'''
@@ -293,24 +293,30 @@ def before_training(self, model):
     def after_training(self, model):
         '''Run after training is finished.'''
 
-    def before_iteration(self, model, epoch, dtrain, evals):
+    def before_iteration(self, model, epoch, evals_log):
         '''Run before each iteration.'''
         return False
 
-    def after_iteration(self, model, epoch, dtrain, evals):
+    def after_iteration(self, model, epoch, evals_log):
         '''Run after each iteration.'''
         return False
 
 
-class CallbackContainer(TrainingCallback):
-    '''A container for list of callbacks.
+class CallbackContainer:
+    '''A special callback for invoking a list of callbacks.
 
     .. versionadded:: 1.3.0
 
     '''
-    def __init__(self, callbacks):
+    def __init__(self, callbacks, metric=None):
         self.callbacks = callbacks
-        super().__init__()
+        if metric is not None:
+            msg = 'metric must be callable object for monitor.  For ' + \
+                'builtin metrics, passing them in training parameter' + \
+                ' will invoke monitor automatically.'
+            assert callable(metric), msg
+        self.metric = metric
+        self.history = collections.OrderedDict()
 
     def before_training(self, model):
         '''Function called before training.'''
@@ -324,15 +330,35 @@ def after_training(self, model):
 
     def before_iteration(self, model, epoch, dtrain, evals):
         '''Function called before training iteration.'''
-        return any(c.before_iteration(model, epoch, dtrain, evals)
+        return any(c.before_iteration(model, epoch, self.history)
                    for c in self.callbacks)
 
+    def _update_history(self, score, epoch):
+        split_by_data = score.split()[1:]  # remove iteration
+
+        for d in split_by_data:
+            name, s = d.split(':')
+            data_name, metric_name = name.split('-')
+            s = float(s)
+            s = _allreduce_metric(s)
+            if data_name in self.history:
+                data_history = self.history[data_name]
+                if metric_name in data_history:
+                    data_history[metric_name].append(s)
+                else:
+                    data_history[metric_name] = [s]
+            else:
+                self.history[data_name] = collections.OrderedDict()
+                self.history[data_name][metric_name] = [s]
+        return False
+
     def after_iteration(self, model, epoch, dtrain, evals):
         '''Function called after training iteration.'''
-        ret = any(c.after_iteration(model, epoch, dtrain, evals)
+        evals = [] if evals is None else evals
+        score = model.eval_set(evals, epoch, self.metric)
+        self._update_history(score, epoch)
+        ret = any(c.after_iteration(model, epoch, self.history)
                   for c in self.callbacks)
-        for c in self.callbacks:
-            self.history.update(c.history)
         return ret
 
 
@@ -353,14 +379,14 @@ class LearningRateScheduler(TrainingCallback):
     '''
     def __init__(self, learning_rates):
         assert callable(learning_rates) or \
-            isinstance(learning_rates, collections.Sequence)
+            isinstance(learning_rates, collections.abc.Sequence)
         if callable(learning_rates):
             self.learning_rates = learning_rates
         else:
             self.learning_rates = lambda epoch: learning_rates[epoch]
         super().__init__()
 
-    def after_iteration(self, model, epoch, dtrain, evals):
+    def after_iteration(self, model, epoch, evals_log):
         model.set_param('learning_rate', self.learning_rates(epoch))
 
 
@@ -382,8 +408,6 @@ class EarlyStopping(TrainingCallback):
     ----------
     rounds : int
         Early stopping rounds.
-    metric : str/callable
-        Name of metric.  Use the default metric if not specified.
     metric_name : str
         Name of metric that is used for early stopping.
     data_name: str
@@ -393,12 +417,10 @@ class EarlyStopping(TrainingCallback):
     '''
     def __init__(self,
                  rounds,
-                 metric=None,
                  metric_name=None,
                  data_name=None,
                  maximize=False,
                  save_best=False):
-        self.metric = metric
         self.data = data_name
         self.metric_name = metric_name
         self.rounds = rounds
@@ -417,11 +439,8 @@ def __init__(self,
         self.best_scores = {}
         super().__init__()
 
-    def _update_rounds(self, scores, name, model, epoch):
-        assert len(scores) == 1, 'No matching metric name.'
-        score = scores[0]
-        metric, s = score[0].split('-')[1], score[1]
-        s = _allreduce_metric(s)
+    def _update_rounds(self, score, name, metric, model, epoch):
+        s = _allreduce_metric(score)
         if not self.stopping_history:  # First round
             self.current_rounds = 0
             self.stopping_history[name] = {}
@@ -444,47 +463,32 @@ def _update_rounds(self, scores, name, model, epoch):
             return True
         return False
 
-    def after_iteration(self, model, epoch, dtrain, evals):
+    def after_iteration(self, model, epoch, evals_log):
         msg = 'Must have at least 1 validation dataset for early stopping.'
-        assert len(evals) >= 1, msg
-        stopping_name = ''
-        stopping_data = None
+        assert len(evals_log.keys()) >= 1, msg
+        data_name = ''
         if self.data:
-            for d, name in evals:
-                if name == self.data:
-                    stopping_name = name
-                    stopping_data = d
-            if not stopping_name:
+            for d, _ in evals_log.items():
+                if d == self.data:
+                    data_name = d
+            if not data_name:
                 raise ValueError('No dataset named:', self.data)
         else:
             # Use the last one as default.
-            stopping_name = evals[-1][1]
-            stopping_data = evals[-1][0]
-
-        assert isinstance(stopping_data, DMatrix)
-        assert isinstance(stopping_name, str)
-
-        if callable(self.metric):
-            score = model.eval_set([(stopping_data, stopping_name)],
-                                   iteration=epoch,
-                                   feval=self.metric)
-            score = [s.split(':') for s in score.split()]
-            score = [(k, _allreduce_metric(float(v))) for k, v in score[1:]]
-        else:
-            score = model.eval_set([(stopping_data, stopping_name)],
-                                   iteration=epoch)
-            score = [s.split(':') for s in score.split()]
-            score = [(k, float(v)) for k, v in score[1:]]
+            data_name = list(evals_log.keys())[-1]
+        assert isinstance(data_name, str)
+        data_log = evals_log[data_name]
 
         # Filter out scores that can not be used for early stopping.
         if self.metric_name:
-            score = list(
-                filter(lambda s: s[0].split('-')[1] == self.metric_name,
-                       score))
+            metric_name = self.metric_name
+            score = data_log[self.metric_name][-1]
         else:
-            score = [score[-1]]
-
-        return self._update_rounds(score, stopping_name, model, epoch)
+            # Use last metric by default.
+            assert isinstance(data_log, collections.OrderedDict)
+            metric_name = list(data_log.keys())[-1]
+            score = data_log[metric_name][-1]
+        return self._update_rounds(score, data_name, metric_name, model, epoch)
 
 
 class EvaluationMonitor(TrainingCallback):
@@ -510,33 +514,16 @@ def __init__(self, metric=None, rank=0):
         self.printer_rank = rank
         super().__init__()
 
-    def _update_history(self, score, epoch):
-        split_by_data = score.split()[1:]  # remove iteration
-
-        for d in split_by_data:
-            name, s = d.split(':')
-            data_name, metric_name = name.split('-')
-            s = float(s)
-            s = _allreduce_metric(s)
-            if data_name in self.history:
-                data_history = self.history[data_name]
-                if metric_name in data_history:
-                    data_history[metric_name].append(s)
-                else:
-                    data_history[metric_name] = [s]
-            else:
-                self.history[data_name] = {}
-                self.history[data_name][metric_name] = [s]
-
+    def after_iteration(self, model, epoch, evals_log):
+        msg = f'[{epoch}]'
         if rabit.get_rank() == self.printer_rank:
-            rabit.tracker_print(score + '\n')
+            for data, metric in evals_log.items():
+                for metric_name, log in metric.items():
+                    msg += '\t' + data + '-' + metric_name + ':' + str(log[-1])
+            msg += '\n'
+            rabit.tracker_print(msg)
         return False
 
-    def after_iteration(self, model, epoch, dtrain, evals):
-        evals = [] if evals is None else evals
-        score = model.eval_set(evals, epoch, self.metric)
-        return self._update_history(score, epoch)
-
 
 class TrainingCheckPoint(TrainingCallback):
     '''Checkpointing operation.
@@ -567,7 +554,7 @@ def __init__(self, directory: os.PathLike, name: str = 'model',
         self._epoch = 0
         super().__init__()
 
-    def after_iteration(self, model, epoch, dtrain, evals):
+    def after_iteration(self, model, epoch, evals_log):
         self._epoch += 1
         if self._epoch == self._iterations:
             path = os.path.join(self._path, self._name + '_' + str(epoch) +
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index b03b8c0c7a82..f3b641e1d6c1 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -83,8 +83,8 @@ def _train_internal(params, dtrain,
             callbacks.append(callback.EvaluationMonitor(metric=feval))
         if early_stopping_rounds:
             callbacks.append(callback.EarlyStopping(
-                rounds=early_stopping_rounds, metric=feval))
-        callbacks = callback.CallbackContainer(callbacks)
+                rounds=early_stopping_rounds))
+        callbacks = callback.CallbackContainer(callbacks, metric=feval)
     else:
         assert False
         callbacks = _configure_deprected_callbacks(
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 836ab497c020..98ee58be3012 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -71,7 +71,6 @@ def test_early_stopping_customize(self):
         D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
         early_stopping_rounds = 5
         early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
-                                                metric=tm.eval_error_metric,
                                                 metric_name='PyError',
                                                 data_name='Train')
         # Specify which dataset and which metric should be used for early stopping.
@@ -80,6 +79,7 @@ def test_early_stopping_customize(self):
              'eval_metric': ['error', 'rmse'],
              'tree_method': 'hist'}, D_train,
             evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+            feval=tm.eval_error_metric,
             num_boost_round=1000,
             callbacks=[early_stop],
             verbose_eval=False)

From a2d99b43de140f37b72aa4b18eeec310a4129125 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 00:25:37 +0800
Subject: [PATCH 26/53] Packed booster.

---
 python-package/xgboost/callback.py | 56 ++++++++++++++++++---
 python-package/xgboost/training.py | 80 +++++++++++++++---------------
 2 files changed, 88 insertions(+), 48 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 8357b67d6319..eae028c63f46 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -308,7 +308,7 @@ class CallbackContainer:
     .. versionadded:: 1.3.0
 
     '''
-    def __init__(self, callbacks, metric=None):
+    def __init__(self, callbacks, metric=None, is_cv=False):
         self.callbacks = callbacks
         if metric is not None:
             msg = 'metric must be callable object for monitor.  For ' + \
@@ -317,6 +317,7 @@ def __init__(self, callbacks, metric=None):
             assert callable(metric), msg
         self.metric = metric
         self.history = collections.OrderedDict()
+        self.is_cv = is_cv
 
     def before_training(self, model):
         '''Function called before training.'''
@@ -334,10 +335,8 @@ def before_iteration(self, model, epoch, dtrain, evals):
                    for c in self.callbacks)
 
     def _update_history(self, score, epoch):
-        split_by_data = score.split()[1:]  # remove iteration
-
-        for d in split_by_data:
-            name, s = d.split(':')
+        for d in score:
+            name, s = d[0], d[1]
             data_name, metric_name = name.split('-')
             s = float(s)
             s = _allreduce_metric(s)
@@ -352,11 +351,51 @@ def _update_history(self, score, epoch):
                 self.history[data_name][metric_name] = [s]
         return False
 
+    def aggcv(self, rlist):
+        # pylint: disable=invalid-name
+        """
+        Aggregate cross-validation results.
+
+        If verbose_eval is true, progress is displayed in every call. If
+        verbose_eval is an integer, progress will only be displayed every
+        `verbose_eval` trees, tracked via trial.
+        """
+        cvmap = {}
+        idx = rlist[0].split()[0]
+        for line in rlist:
+            arr = line.split()
+            assert idx == arr[0]
+            for metric_idx, it in enumerate(arr[1:]):
+                if not isinstance(it, STRING_TYPES):
+                    it = it.decode()
+                k, v = it.split(':')
+                if (metric_idx, k) not in cvmap:
+                    cvmap[(metric_idx, k)] = []
+                cvmap[(metric_idx, k)].append(float(v))
+        msg = idx
+        results = []
+        for (metric_idx, k), v in sorted(cvmap.items(), key=lambda x: x[0][0]):
+            v = numpy.array(v)
+            if not isinstance(msg, STRING_TYPES):
+                msg = msg.decode()
+            mean, std = numpy.mean(v), numpy.std(v)
+            results.extend([(k, mean, std)])
+        return results
+
     def after_iteration(self, model, epoch, dtrain, evals):
         '''Function called after training iteration.'''
-        evals = [] if evals is None else evals
-        score = model.eval_set(evals, epoch, self.metric)
-        self._update_history(score, epoch)
+        if self.is_cv:
+            scores = model.eval(epoch, self.metric)
+            scores = self.aggcv(scores)
+            scores = [(s[0], s[1]) for s in scores]  # fliter out std
+            self._update_history(scores, epoch)
+        else:
+            evals = [] if evals is None else evals
+            score = model.eval_set(evals, epoch, self.metric)
+            score = score.split()[1:]  # into datasets
+            # split up `test-error:0.1223`
+            score = [tuple(s.split(':')) for s in score]
+            self._update_history(score, epoch)
         ret = any(c.after_iteration(model, epoch, self.history)
                   for c in self.callbacks)
         return ret
@@ -447,6 +486,7 @@ def _update_rounds(self, score, name, metric, model, epoch):
             self.stopping_history[name][metric] = [s]
             self.best_scores[name] = {}
             self.best_scores[name][metric] = [s]
+            model.set_attr(best_score=str(s), best_iteration=str(epoch))
         elif not self.improve_op(s, self.best_scores[name][metric][-1]):
             # Not improved
             self.stopping_history[name][metric].append(s)
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index f3b641e1d6c1..c60ab24116ab 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -83,7 +83,7 @@ def _train_internal(params, dtrain,
             callbacks.append(callback.EvaluationMonitor(metric=feval))
         if early_stopping_rounds:
             callbacks.append(callback.EarlyStopping(
-                rounds=early_stopping_rounds))
+                rounds=early_stopping_rounds, maximize=maximize))
         callbacks = callback.CallbackContainer(callbacks, metric=feval)
     else:
         assert False
@@ -233,6 +233,28 @@ def eval(self, iteration, feval):
         return self.bst.eval_set(self.watchlist, iteration, feval)
 
 
+class PackedBooster:
+    def __init__(self, cvfolds):
+        self.cvfolds = cvfolds
+
+    def update(self, iteration, obj):
+        for fold in self.cvfolds:
+            fold.update(iteration, obj)
+
+    def eval(self, iteration, feval):
+        result = [f.eval(iteration, feval) for f in self.cvfolds]
+        return result
+
+    def set_attr(self, **kwargs):
+        for f in self.cvfolds:
+            f.bst.set_attr(**kwargs)
+
+    @property
+    def best_iteration(self):
+        ret = self.cvfolds[0].bst.attr('best_iteration')
+        return int(ret)
+
+
 def groups_to_rows(groups, boundaries):
     """
     Given group row boundaries, convert ground indexes to row indexes
@@ -371,7 +393,7 @@ def aggcv(rlist):
 
 def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None,
        metrics=(), obj=None, feval=None, maximize=False, early_stopping_rounds=None,
-       fpreproc=None, as_pandas=True, verbose_eval=None, show_stdv=True,
+       fpreproc=None, as_pandas=True, verbose_eval=True, show_stdv=True,
        seed=0, callbacks=None, shuffle=True):
     # pylint: disable = invalid-name
     """Cross-validation with given parameters.
@@ -470,36 +492,22 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
 
     # setup callbacks
     callbacks = [] if callbacks is None else callbacks
-    if early_stopping_rounds is not None:
-        callbacks.append(callback.early_stop(early_stopping_rounds,
-                                             maximize=maximize,
-                                             verbose=False))
-
-    if isinstance(verbose_eval, bool) and verbose_eval:
-        callbacks.append(callback.print_evaluation(show_stdv=show_stdv))
-    else:
-        if isinstance(verbose_eval, int):
-            callbacks.append(callback.print_evaluation(verbose_eval, show_stdv=show_stdv))
+    if verbose_eval:
+        callbacks.append(callback.EvaluationMonitor(metric=feval))
+    if early_stopping_rounds:
+        callbacks.append(callback.EarlyStopping(
+            rounds=early_stopping_rounds, maximize=maximize))
+    callbacks = callback.CallbackContainer(callbacks, metric=feval, is_cv=True)
+    callbacks.before_training(cvfolds)
 
-    callbacks_before_iter = [
-        cb for cb in callbacks if
-        cb.__dict__.get('before_iteration', False)]
-    callbacks_after_iter = [
-        cb for cb in callbacks if
-        not cb.__dict__.get('before_iteration', False)]
+    booster = PackedBooster(cvfolds)
 
     for i in range(num_boost_round):
-        for cb in callbacks_before_iter:
-            cb(CallbackEnv(model=None,
-                           cvfolds=cvfolds,
-                           iteration=i,
-                           begin_iteration=0,
-                           end_iteration=num_boost_round,
-                           rank=0,
-                           evaluation_result_list=None))
-        for fold in cvfolds:
-            fold.update(i, obj)
-        res = aggcv([f.eval(i, feval) for f in cvfolds])
+        if callbacks.before_iteration(booster, i, dtrain, None):
+            break
+        booster.update(i, obj)
+        evals_log = booster.eval(i, feval)
+        res = aggcv(evals_log)
 
         for key, mean, std in res:
             if key + '-mean' not in results:
@@ -508,18 +516,10 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
                 results[key + '-std'] = []
             results[key + '-mean'].append(mean)
             results[key + '-std'].append(std)
-        try:
-            for cb in callbacks_after_iter:
-                cb(CallbackEnv(model=None,
-                               cvfolds=cvfolds,
-                               iteration=i,
-                               begin_iteration=0,
-                               end_iteration=num_boost_round,
-                               rank=0,
-                               evaluation_result_list=res))
-        except EarlyStopException as e:
+
+        if callbacks.after_iteration(booster, i, dtrain, None):
             for k in results:
-                results[k] = results[k][:(e.best_iteration + 1)]
+                results[k] = results[k][:(booster.best_iteration + 1)]
             break
     if as_pandas:
         try:

From c881417e50d80a0060d653b7c4200c0b70e8b405 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 00:28:57 +0800
Subject: [PATCH 27/53] todos.

---
 python-package/xgboost/callback.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index eae028c63f46..01c5cf7cb245 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -262,17 +262,18 @@ def callback(env):
     return callback
 
 
-#
 # The new implementation of callback functions.
 #
 # TODOs
-# - eval_set
-# - cv
-# - tests
-# - doc
-# - enforced best_xxx
-# - merged functionality of es and mon.
-# - make callbacks a set instead of list.
+# - [x] eval_set
+# - [ ] cv
+# - [ ] tests
+# - [ ] doc
+# - [x] enforced best_xxx
+# - [ ] merged functionality of es and mon.
+# - [ ] make callbacks a set instead of list.
+# - [ ] auto detect maximize.
+# - [ ] Correct printing for cv
 
 # Breaking:
 # - reset learning rate no longer accepts total boosting rounds

From 899add71a4b871ee5c1bcf9fbfc0851a3941048b Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 01:00:33 +0800
Subject: [PATCH 28/53] Auto config.

---
 python-package/xgboost/callback.py | 25 +++++++++++++++++++------
 python-package/xgboost/training.py |  4 ++--
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 01c5cf7cb245..0dda10379207 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -272,7 +272,7 @@ def callback(env):
 # - [x] enforced best_xxx
 # - [ ] merged functionality of es and mon.
 # - [ ] make callbacks a set instead of list.
-# - [ ] auto detect maximize.
+# - [x] auto detect maximize.
 # - [ ] Correct printing for cv
 
 # Breaking:
@@ -459,7 +459,7 @@ def __init__(self,
                  rounds,
                  metric_name=None,
                  data_name=None,
-                 maximize=False,
+                 maximize=None,
                  save_best=False):
         self.data = data_name
         self.metric_name = metric_name
@@ -470,10 +470,11 @@ def __init__(self,
         self.maximize = maximize
         self.stopping_history = {}
 
-        if self.maximize:
-            self.improve_op = lambda x, y: x > y
-        else:
-            self.improve_op = lambda x, y: x < y
+        if self.maximize is not None:
+            if self.maximize:
+                self.improve_op = lambda x, y: x > y
+            else:
+                self.improve_op = lambda x, y: x < y
 
         self.current_rounds = 0
         self.best_scores = {}
@@ -481,6 +482,18 @@ def __init__(self,
 
     def _update_rounds(self, score, name, metric, model, epoch):
         s = _allreduce_metric(score)
+        # Just to be compatibility with old behavior before 1.3.  We should let
+        # user to decide.
+        if self.maximize is None:
+            maximize_metrics = ('auc', 'aucpr', 'map', 'ndcg', 'auc@',
+                                'aucpr@', 'map@', 'ndcg@')
+            if any(metric.startswith(x) for x in maximize_metrics):
+                self.improve_op = lambda x, y: x > y
+                self.maximize = True
+            else:
+                self.improve_op = lambda x, y: x < y
+                self.maximize = False
+
         if not self.stopping_history:  # First round
             self.current_rounds = 0
             self.stopping_history[name] = {}
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index c60ab24116ab..a44f57ebbfe8 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -130,7 +130,7 @@ def _train_internal(params, dtrain,
 
 
 def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None,
-          maximize=False, early_stopping_rounds=None, evals_result=None,
+          maximize=None, early_stopping_rounds=None, evals_result=None,
           verbose_eval=True, xgb_model=None, callbacks=None):
     # pylint: disable=too-many-statements,too-many-branches, attribute-defined-outside-init
     """Train a booster with given parameters.
@@ -392,7 +392,7 @@ def aggcv(rlist):
 
 
 def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None,
-       metrics=(), obj=None, feval=None, maximize=False, early_stopping_rounds=None,
+       metrics=(), obj=None, feval=None, maximize=None, early_stopping_rounds=None,
        fpreproc=None, as_pandas=True, verbose_eval=True, show_stdv=True,
        seed=0, callbacks=None, shuffle=True):
     # pylint: disable = invalid-name

From 268ab02ea2be6bb904253fc3366531b090b3b276 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 01:04:04 +0800
Subject: [PATCH 29/53] Use set.

---
 python-package/xgboost/callback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 0dda10379207..e30edb9a8988 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -310,7 +310,7 @@ class CallbackContainer:
 
     '''
     def __init__(self, callbacks, metric=None, is_cv=False):
-        self.callbacks = callbacks
+        self.callbacks = set(callbacks)
         if metric is not None:
             msg = 'metric must be callable object for monitor.  For ' + \
                 'builtin metrics, passing them in training parameter' + \

From 31403df0805e6f445170c9449a869a2be292b113 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 02:53:03 +0800
Subject: [PATCH 30/53] Minor cleaning.

---
 python-package/xgboost/callback.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index e30edb9a8988..c930aedfa9be 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -28,7 +28,7 @@ def _fmt_metric(value, show_stdv=True):
         return '{0}:{1:.5f}'.format(value[0], value[1])
     if len(value) == 3:
         if show_stdv:
-            return  '{0}:{1:.5f}+{2:.5f}'.format(value[0], value[1], value[2])
+            return '{0}:{1:.5f}+{2:.5f}'.format(value[0], value[1], value[2])
         return '{0}:{1:.5f}'.format(value[0], value[1])
     raise ValueError("wrong metric value", value)
 
@@ -337,9 +337,8 @@ def before_iteration(self, model, epoch, dtrain, evals):
 
     def _update_history(self, score, epoch):
         for d in score:
-            name, s = d[0], d[1]
+            name, s = d[0], float(d[1])
             data_name, metric_name = name.split('-')
-            s = float(s)
             s = _allreduce_metric(s)
             if data_name in self.history:
                 data_history = self.history[data_name]
@@ -394,7 +393,7 @@ def after_iteration(self, model, epoch, dtrain, evals):
             evals = [] if evals is None else evals
             score = model.eval_set(evals, epoch, self.metric)
             score = score.split()[1:]  # into datasets
-            # split up `test-error:0.1223`
+            # split up `test-error:0.1234`
             score = [tuple(s.split(':')) for s in score]
             self._update_history(score, epoch)
         ret = any(c.after_iteration(model, epoch, self.history)
@@ -453,7 +452,7 @@ class EarlyStopping(TrainingCallback):
     data_name: str
         Name of dataset that is used for early stopping.
     maximize : bool
-        Whether to maximize evaluation metric.
+        Whether to maximize evaluation metric.  None means auto (discouraged).
     '''
     def __init__(self,
                  rounds,
@@ -465,6 +464,7 @@ def __init__(self,
         self.metric_name = metric_name
         self.rounds = rounds
         self.save_best = save_best
+        # https://github.com/dmlc/xgboost/issues/5531
         assert self.save_best is False, 'save best is not yet supported.'
 
         self.maximize = maximize
@@ -530,18 +530,17 @@ def after_iteration(self, model, epoch, evals_log):
         else:
             # Use the last one as default.
             data_name = list(evals_log.keys())[-1]
-        assert isinstance(data_name, str)
+        assert isinstance(data_name, str) and data_name
         data_log = evals_log[data_name]
 
         # Filter out scores that can not be used for early stopping.
         if self.metric_name:
             metric_name = self.metric_name
-            score = data_log[self.metric_name][-1]
         else:
             # Use last metric by default.
             assert isinstance(data_log, collections.OrderedDict)
             metric_name = list(data_log.keys())[-1]
-            score = data_log[metric_name][-1]
+        score = data_log[metric_name][-1]
         return self._update_rounds(score, data_name, metric_name, model, epoch)
 
 

From 4fc2369d1db160a38cd0fe82e7c42ace8d9ae779 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 03:11:43 +0800
Subject: [PATCH 31/53] Remove redundant aggcv.

---
 python-package/xgboost/callback.py | 59 ++++++++++++++++++++----------
 python-package/xgboost/training.py | 42 +++------------------
 2 files changed, 45 insertions(+), 56 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index c930aedfa9be..d608db887b0b 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -9,7 +9,7 @@
 import numpy
 
 from . import rabit
-from .core import EarlyStopException, DMatrix, CallbackEnv
+from .core import EarlyStopException, CallbackEnv
 from .compat import STRING_TYPES
 
 
@@ -271,7 +271,7 @@ def callback(env):
 # - [ ] doc
 # - [x] enforced best_xxx
 # - [ ] merged functionality of es and mon.
-# - [ ] make callbacks a set instead of list.
+# - [x] make callbacks a set instead of list.
 # - [x] auto detect maximize.
 # - [ ] Correct printing for cv
 
@@ -320,6 +320,9 @@ def __init__(self, callbacks, metric=None, is_cv=False):
         self.history = collections.OrderedDict()
         self.is_cv = is_cv
 
+        if self.is_cv:
+            self.aggregated_cv = None
+
     def before_training(self, model):
         '''Function called before training.'''
         for c in self.callbacks:
@@ -351,7 +354,7 @@ def _update_history(self, score, epoch):
                 self.history[data_name][metric_name] = [s]
         return False
 
-    def aggcv(self, rlist):
+    def _aggcv(self, rlist):
         # pylint: disable=invalid-name
         """
         Aggregate cross-validation results.
@@ -386,7 +389,8 @@ def after_iteration(self, model, epoch, dtrain, evals):
         '''Function called after training iteration.'''
         if self.is_cv:
             scores = model.eval(epoch, self.metric)
-            scores = self.aggcv(scores)
+            scores = self._aggcv(scores)
+            self.aggregated_cv = scores
             scores = [(s[0], s[1]) for s in scores]  # fliter out std
             self._update_history(scores, epoch)
         else:
@@ -430,6 +434,11 @@ def after_iteration(self, model, epoch, evals_log):
 
 
 def _allreduce_metric(score):
+    '''Helper function for computing customized metric in distributed
+    environment.  Not strictly correct as many functions don't use mean value
+    as final result.
+
+    '''
     score = numpy.array([score])
     world = rabit.get_world_size()
     assert world != 0
@@ -437,6 +446,16 @@ def _allreduce_metric(score):
     return score[0]
 
 
+def _get_latest_log(evals_log: collections.OrderedDict):
+    '''Given the evaluation log, extract the score of latest iteration.'''
+    result = []
+    for data, metric in evals_log.items():
+        for metric_name, log in metric.items():
+            result.append(
+                {'data': data, 'metric': metric_name, 'score': log[-1]})
+    return result
+
+
 # pylint: disable=too-many-instance-attributes
 class EarlyStopping(TrainingCallback):
     ''' Callback function for early stopping
@@ -557,22 +576,23 @@ class EvaluationMonitor(TrainingCallback):
     rank : int
         Which worker should be used for printing the result.
     '''
-    def __init__(self, metric=None, rank=0):
-        if metric is not None:
-            msg = 'metric must be callable object for monitor.  For ' + \
-                'builtin metrics, passing them in training parameter' + \
-                ' will invoke monitor automatically.'
-            assert callable(metric), msg
-        self.metric = metric
+    def __init__(self, rank=0):
         self.printer_rank = rank
         super().__init__()
 
+    def _fmt_metric(self, data, metric, score, std):
+        if std is None:
+            msg = '\t{0}:{1:.5f}'.format(data + '-' + metric, score)
+        else:
+            msg = '\t{0}:{1:.5f}+{2:.5f}'.format(data + '-' + metric, score, std)
+        return msg
+
     def after_iteration(self, model, epoch, evals_log):
         msg = f'[{epoch}]'
         if rabit.get_rank() == self.printer_rank:
             for data, metric in evals_log.items():
                 for metric_name, log in metric.items():
-                    msg += '\t' + data + '-' + metric_name + ':' + str(log[-1])
+                    msg += self._fmt_metric(data, metric_name, log[-1], None)
             msg += '\n'
             rabit.tracker_print(msg)
         return False
@@ -586,20 +606,21 @@ class TrainingCheckPoint(TrainingCallback):
     Parameters
     ----------
 
-    path : os.PathLike
+    directory : os.PathLike
         Output model directory.
     name : str
-        pattern of output model file.  Models will be saved as name_0.json,
-        name_1.json, name_2.json ....
+        pattern of output model file.  Models will be saved as name_0.json, name_1.json,
+        name_2.json ....
     as_pickle : boolean
-        When set to Ture, all training parameters will be saved in pickle
-        format, instead of saving only the model.
+        When set to Ture, all training parameters will be saved in pickle format, instead
+        of saving only the model.
     iterations : int
-        Interval of checkpointing.
+        Interval of checkpointing.  Checkpointing is slow so setting a larger number can
+        reduce performance hit.
 
     '''
     def __init__(self, directory: os.PathLike, name: str = 'model',
-                 as_pickle=False, rounds: int = 10):
+                 as_pickle=False, rounds: int = 100):
         self._path = directory
         self._name = name
         self._as_pickle = as_pickle
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index a44f57ebbfe8..5d03db4cc025 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -80,7 +80,7 @@ def _train_internal(params, dtrain,
     if any(is_new_callback) or not callbacks:
         assert all(is_new_callback), "You can't mix two styles of callbacks."
         if verbose_eval:
-            callbacks.append(callback.EvaluationMonitor(metric=feval))
+            callbacks.append(callback.EvaluationMonitor())
         if early_stopping_rounds:
             callbacks.append(callback.EarlyStopping(
                 rounds=early_stopping_rounds, maximize=maximize))
@@ -359,38 +359,6 @@ def mknfold(dall, nfold, param, seed, evals=(), fpreproc=None, stratified=False,
     return ret
 
 
-def aggcv(rlist):
-    # pylint: disable=invalid-name
-    """
-    Aggregate cross-validation results.
-
-    If verbose_eval is true, progress is displayed in every call. If
-    verbose_eval is an integer, progress will only be displayed every
-    `verbose_eval` trees, tracked via trial.
-    """
-    cvmap = {}
-    idx = rlist[0].split()[0]
-    for line in rlist:
-        arr = line.split()
-        assert idx == arr[0]
-        for metric_idx, it in enumerate(arr[1:]):
-            if not isinstance(it, STRING_TYPES):
-                it = it.decode()
-            k, v = it.split(':')
-            if (metric_idx, k) not in cvmap:
-                cvmap[(metric_idx, k)] = []
-            cvmap[(metric_idx, k)].append(float(v))
-    msg = idx
-    results = []
-    for (metric_idx, k), v in sorted(cvmap.items(), key=lambda x: x[0][0]):
-        v = np.array(v)
-        if not isinstance(msg, STRING_TYPES):
-            msg = msg.decode()
-        mean, std = np.mean(v), np.std(v)
-        results.extend([(k, mean, std)])
-    return results
-
-
 def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None,
        metrics=(), obj=None, feval=None, maximize=None, early_stopping_rounds=None,
        fpreproc=None, as_pandas=True, verbose_eval=True, show_stdv=True,
@@ -493,7 +461,7 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
     # setup callbacks
     callbacks = [] if callbacks is None else callbacks
     if verbose_eval:
-        callbacks.append(callback.EvaluationMonitor(metric=feval))
+        callbacks.append(callback.EvaluationMonitor())
     if early_stopping_rounds:
         callbacks.append(callback.EarlyStopping(
             rounds=early_stopping_rounds, maximize=maximize))
@@ -506,9 +474,9 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
         if callbacks.before_iteration(booster, i, dtrain, None):
             break
         booster.update(i, obj)
-        evals_log = booster.eval(i, feval)
-        res = aggcv(evals_log)
 
+        should_break = callbacks.after_iteration(booster, i, dtrain, None)
+        res = callbacks.aggregated_cv
         for key, mean, std in res:
             if key + '-mean' not in results:
                 results[key + '-mean'] = []
@@ -517,7 +485,7 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
             results[key + '-mean'].append(mean)
             results[key + '-std'].append(std)
 
-        if callbacks.after_iteration(booster, i, dtrain, None):
+        if should_break:
             for k in results:
                 results[k] = results[k][:(booster.best_iteration + 1)]
             break

From c2fa6a1a3c32f8798bcb5182fdbb09769088a857 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 03:54:23 +0800
Subject: [PATCH 32/53] Correct cv print.

---
 python-package/xgboost/callback.py | 60 +++++++++++++++++-------------
 python-package/xgboost/training.py |  2 +-
 2 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index d608db887b0b..4e078e400aab 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -6,6 +6,7 @@
 import collections
 import os
 import pickle
+from typing import Callable, List
 import numpy
 
 from . import rabit
@@ -266,14 +267,13 @@ def callback(env):
 #
 # TODOs
 # - [x] eval_set
-# - [ ] cv
-# - [ ] tests
+# - [x] cv
+# - [x] tests
 # - [ ] doc
 # - [x] enforced best_xxx
-# - [ ] merged functionality of es and mon.
 # - [x] make callbacks a set instead of list.
 # - [x] auto detect maximize.
-# - [ ] Correct printing for cv
+# - [x] Correct printing for cv
 
 # Breaking:
 # - reset learning rate no longer accepts total boosting rounds
@@ -309,7 +309,8 @@ class CallbackContainer:
     .. versionadded:: 1.3.0
 
     '''
-    def __init__(self, callbacks, metric=None, is_cv=False):
+    def __init__(self, callbacks: List[TrainingCallback],
+                 metric: Callable = None, is_cv: bool = False):
         self.callbacks = set(callbacks)
         if metric is not None:
             msg = 'metric must be callable object for monitor.  For ' + \
@@ -341,6 +342,9 @@ def before_iteration(self, model, epoch, dtrain, evals):
     def _update_history(self, score, epoch):
         for d in score:
             name, s = d[0], float(d[1])
+            if self.is_cv:
+                std = float(d[2])
+                s = (s, std)
             data_name, metric_name = name.split('-')
             s = _allreduce_metric(s)
             if data_name in self.history:
@@ -356,12 +360,8 @@ def _update_history(self, score, epoch):
 
     def _aggcv(self, rlist):
         # pylint: disable=invalid-name
-        """
-        Aggregate cross-validation results.
+        """Aggregate cross-validation results.
 
-        If verbose_eval is true, progress is displayed in every call. If
-        verbose_eval is an integer, progress will only be displayed every
-        `verbose_eval` trees, tracked via trial.
         """
         cvmap = {}
         idx = rlist[0].split()[0]
@@ -391,7 +391,6 @@ def after_iteration(self, model, epoch, dtrain, evals):
             scores = model.eval(epoch, self.metric)
             scores = self._aggcv(scores)
             self.aggregated_cv = scores
-            scores = [(s[0], s[1]) for s in scores]  # fliter out std
             self._update_history(scores, epoch)
         else:
             evals = [] if evals is None else evals
@@ -439,9 +438,14 @@ def _allreduce_metric(score):
     as final result.
 
     '''
-    score = numpy.array([score])
     world = rabit.get_world_size()
     assert world != 0
+    if world == 1:
+        return score
+    if isinstance(score, tuple):  # has mean and stdv
+        raise ValueError(
+            'xgboost.cv function should not be used in distributed environment.')
+    score = numpy.array([score])
     score = rabit.allreduce(score, rabit.Op.SUM) / world
     return score[0]
 
@@ -500,7 +504,6 @@ def __init__(self,
         super().__init__()
 
     def _update_rounds(self, score, name, metric, model, epoch):
-        s = _allreduce_metric(score)
         # Just to be compatibility with old behavior before 1.3.  We should let
         # user to decide.
         if self.maximize is None:
@@ -516,17 +519,17 @@ def _update_rounds(self, score, name, metric, model, epoch):
         if not self.stopping_history:  # First round
             self.current_rounds = 0
             self.stopping_history[name] = {}
-            self.stopping_history[name][metric] = [s]
+            self.stopping_history[name][metric] = [score]
             self.best_scores[name] = {}
-            self.best_scores[name][metric] = [s]
-            model.set_attr(best_score=str(s), best_iteration=str(epoch))
-        elif not self.improve_op(s, self.best_scores[name][metric][-1]):
+            self.best_scores[name][metric] = [score]
+            model.set_attr(best_score=str(score), best_iteration=str(epoch))
+        elif not self.improve_op(score, self.best_scores[name][metric][-1]):
             # Not improved
-            self.stopping_history[name][metric].append(s)
+            self.stopping_history[name][metric].append(score)
             self.current_rounds += 1
         else:  # Improved
-            self.stopping_history[name][metric].append(s)
-            self.best_scores[name][metric].append(s)
+            self.stopping_history[name][metric].append(score)
+            self.best_scores[name][metric].append(score)
             record = self.stopping_history[name][metric][-1]
             model.set_attr(best_score=str(record), best_iteration=str(epoch))
             self.current_rounds = 0  # reset
@@ -576,15 +579,16 @@ class EvaluationMonitor(TrainingCallback):
     rank : int
         Which worker should be used for printing the result.
     '''
-    def __init__(self, rank=0):
+    def __init__(self, rank=0, show_stdv=False):
         self.printer_rank = rank
+        self.show_stdv = show_stdv
         super().__init__()
 
     def _fmt_metric(self, data, metric, score, std):
-        if std is None:
-            msg = '\t{0}:{1:.5f}'.format(data + '-' + metric, score)
-        else:
+        if std is not None and self.show_stdv:
             msg = '\t{0}:{1:.5f}+{2:.5f}'.format(data + '-' + metric, score, std)
+        else:
+            msg = '\t{0}:{1:.5f}'.format(data + '-' + metric, score)
         return msg
 
     def after_iteration(self, model, epoch, evals_log):
@@ -592,7 +596,13 @@ def after_iteration(self, model, epoch, evals_log):
         if rabit.get_rank() == self.printer_rank:
             for data, metric in evals_log.items():
                 for metric_name, log in metric.items():
-                    msg += self._fmt_metric(data, metric_name, log[-1], None)
+                    if isinstance(log[-1], tuple):
+                        score = log[-1][0]
+                        stdv = log[-1][1]
+                    else:
+                        score = log[-1]
+                        stdv = None
+                    msg += self._fmt_metric(data, metric_name, score, stdv)
             msg += '\n'
             rabit.tracker_print(msg)
         return False
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 5d03db4cc025..7c418361b865 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -461,7 +461,7 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
     # setup callbacks
     callbacks = [] if callbacks is None else callbacks
     if verbose_eval:
-        callbacks.append(callback.EvaluationMonitor())
+        callbacks.append(callback.EvaluationMonitor(show_stdv=show_stdv))
     if early_stopping_rounds:
         callbacks.append(callback.EarlyStopping(
             rounds=early_stopping_rounds, maximize=maximize))

From e2aa7396e98cd4f5a8190ff82d704db198c104f7 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 04:52:30 +0800
Subject: [PATCH 33/53] Demo.

---
 demo/guide-python/callbacks.py     | 99 +++++++++++++++++++++++++++---
 demo/guide-python/data_iterator.py |  2 +
 doc/python/callbacks.rst           | 19 +++---
 python-package/xgboost/callback.py | 12 +---
 tests/python/test_callback.py      |  4 +-
 5 files changed, 103 insertions(+), 33 deletions(-)

diff --git a/demo/guide-python/callbacks.py b/demo/guide-python/callbacks.py
index e4a0630782a1..3e22ce40edad 100644
--- a/demo/guide-python/callbacks.py
+++ b/demo/guide-python/callbacks.py
@@ -1,10 +1,96 @@
+'''
+Demo for using and defining callback functions.
+
+    .. versionadded:: 1.2.0
+'''
 import xgboost as xgb
 import tempfile
 import os
+import numpy as np
 from sklearn.datasets import load_breast_cancer
+from sklearn.model_selection import train_test_split
+from matplotlib import pyplot as plt
+
+
+class Plotting(xgb.callback.TrainingCallback):
+    '''Plot evaluation result during training.  Only for demonstration purpose as it's quite
+    slow to draw.
+
+    '''
+    def __init__(self, rounds):
+        self.fig = plt.figure()
+        self.ax = self.fig.add_subplot(111)
+        self.rounds = rounds
+        self.has_lines = False
+        self.lines = {}
+        self.fig.show()
+        self.x = np.linspace(0, self.rounds, self.rounds)
+        plt.ion()
+
+    def _get_key(self, data, metric):
+        return f'{data}-{metric}'
+
+    def after_iteration(self, model, epoch, evals_log):
+        '''Update the plot.'''
+        if not self.lines:
+            for data, metric in evals_log.items():
+                for metric_name, log in metric.items():
+                    key = self._get_key(data, metric_name)
+                    expanded = log + [0] * (self.rounds - len(log))
+                    self.lines[key],  = self.ax.plot(self.x, expanded, label=key)
+                    self.ax.legend()
+        else:
+            # https://pythonspot.com/matplotlib-update-plot/
+            for data, metric in evals_log.items():
+                for metric_name, log in metric.items():
+                    key = self._get_key(data, metric_name)
+                    expanded = log + [0] * (self.rounds - len(log))
+                    self.lines[key].set_ydata(expanded)
+            self.fig.canvas.draw()
+        # False to indicate training should not stop.
+        return False
+
+
+def custom_callback():
+    '''Demo for defining a custom callback function that plots evaluation result during
+    training.'''
+    X, y = load_breast_cancer(return_X_y=True)
+    X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state=0)
+
+    D_train = xgb.DMatrix(X_train, y_train)
+    D_valid = xgb.DMatrix(X_valid, y_valid)
+
+    num_boost_round = 100
+    plotting = Plotting(num_boost_round)
+
+    # Pass it to the `callbacks` parameter as a list.
+    xgb.train(
+        {
+            'objective': 'binary:logistic',
+            'eval_metric': ['error', 'rmse'],
+            'tree_method': 'gpu_hist'
+        },
+        D_train,
+        evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+        num_boost_round=num_boost_round,
+        callbacks=[plotting])
 
 
 def check_point_callback():
+    # only for demo, set a larger value (like 100) in practice as checkpointing is quite
+    # slow.
+    rounds = 2
+
+    def check(as_pickle):
+        for i in range(0, 10, rounds):
+            if i == 0:
+                continue
+            if as_pickle:
+                path = os.path.join(tmpdir, 'model_' + str(i) + '.pkl')
+            else:
+                path = os.path.join(tmpdir, 'model_' + str(i) + '.json')
+            assert(os.path.exists(path))
+
     X, y = load_breast_cancer(return_X_y=True)
     m = xgb.DMatrix(X, y)
     # Check point to a temporary directory for demo
@@ -12,30 +98,27 @@ def check_point_callback():
         # Use callback class from xgboost.callback
         # Feel free to subclass/customize it to suite your need.
         check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
-                                                      rounds=2,
+                                                      rounds=rounds,
                                                       name='model')
         xgb.train({'objective': 'binary:logistic'}, m,
                   num_boost_round=10,
                   verbose_eval=False,
                   callbacks=[check_point])
-        for i in range(0, 10):
-            assert os.path.exists(
-                os.path.join(tmpdir, 'model_' + str(i) + '.json'))
+        check(False)
 
         # This version of checkpoint saves everything including parameters and
         # model.  See: doc/tutorials/saving_model.rst
         check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
-                                                      rounds=1,
+                                                      rounds=rounds,
                                                       as_pickle=True,
                                                       name='model')
         xgb.train({'objective': 'binary:logistic'}, m,
                   num_boost_round=10,
                   verbose_eval=False,
                   callbacks=[check_point])
-        for i in range(0, 10):
-            assert os.path.exists(
-                os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))
+        check(True)
 
 
 if __name__ == '__main__':
     check_point_callback()
+    custom_callback()
diff --git a/demo/guide-python/data_iterator.py b/demo/guide-python/data_iterator.py
index c7300d9f6334..dc910c606f6d 100644
--- a/demo/guide-python/data_iterator.py
+++ b/demo/guide-python/data_iterator.py
@@ -1,5 +1,7 @@
 '''A demo for defining data iterator.
 
+    .. versionadded:: 1.2.0
+
 The demo that defines a customized iterator for passing batches of data into
 `xgboost.DeviceQuantileDMatrix` and use this `DeviceQuantileDMatrix` for
 training.  The feature is used primarily designed to reduce the required GPU
diff --git a/doc/python/callbacks.rst b/doc/python/callbacks.rst
index 3a4398e7b364..4f58dc2e7aee 100644
--- a/doc/python/callbacks.rst
+++ b/doc/python/callbacks.rst
@@ -34,14 +34,15 @@ this callback function directly into XGBoost:
 
     # Specify which dataset and which metric should be used for early stopping.
     early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
-                                            metric=tm.eval_error_metric,
-                                            metric_name='PyError',
-                                            data_name='Valid')
+					    metric_name='PyError',
+					    data_name='Train')
+
     booster = xgb.train(
         {'objective': 'binary:logistic',
          'eval_metric': ['error', 'rmse'],
          'tree_method': 'hist'}, D_train,
         evals=[(D_train, 'Train'), (D_valid, 'Valid')],
+	feval=eval_error_metric,
         num_boost_round=1000,
         callbacks=[early_stop],
         verbose_eval=False)
@@ -53,12 +54,6 @@ this callback function directly into XGBoost:
 Defining your own callback
 ##########################
 
-In here we will define a callback for monitoring shap value changes during training.
-First XGBoost provides an interface class: ``xgboost.callback.TrainingCallback``, user
-defined callbacks should inherit this class and override corresponding methods.
-
-.. code-block:: python
-    pass
-
-
-The full example is in.
+XGBoost provides an callback interface class: ``xgboost.callback.TrainingCallback``, user
+defined callbacks should inherit this class and override corresponding methods.  There's a
+working example in `demo/guide-python/callbacks.py <https://github.com/dmlc/xgboost/tree/master/demo/guide-python/callbacks.py>`_
diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 4e078e400aab..b9e57a1be300 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -450,16 +450,6 @@ def _allreduce_metric(score):
     return score[0]
 
 
-def _get_latest_log(evals_log: collections.OrderedDict):
-    '''Given the evaluation log, extract the score of latest iteration.'''
-    result = []
-    for data, metric in evals_log.items():
-        for metric_name, log in metric.items():
-            result.append(
-                {'data': data, 'metric': metric_name, 'score': log[-1]})
-    return result
-
-
 # pylint: disable=too-many-instance-attributes
 class EarlyStopping(TrainingCallback):
     ''' Callback function for early stopping
@@ -639,7 +629,6 @@ def __init__(self, directory: os.PathLike, name: str = 'model',
         super().__init__()
 
     def after_iteration(self, model, epoch, evals_log):
-        self._epoch += 1
         if self._epoch == self._iterations:
             path = os.path.join(self._path, self._name + '_' + str(epoch) +
                                 ('.pkl' if self._as_pickle else '.json'))
@@ -650,6 +639,7 @@ def after_iteration(self, model, epoch, evals_log):
                         pickle.dump(model, fd)
                 else:
                     model.save_model(path)
+        self._epoch += 1
 
 
 class LegacyCallbacks(TrainingCallback):
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 98ee58be3012..98a85bacc92a 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -210,7 +210,7 @@ def test_check_point(self):
                       num_boost_round=10,
                       verbose_eval=False,
                       callbacks=[check_point])
-            for i in range(0, 10):
+            for i in range(1, 10):
                 assert os.path.exists(
                     os.path.join(tmpdir, 'model_' + str(i) + '.json'))
 
@@ -222,6 +222,6 @@ def test_check_point(self):
                       num_boost_round=10,
                       verbose_eval=False,
                       callbacks=[check_point])
-            for i in range(0, 10):
+            for i in range(1, 10):
                 assert os.path.exists(
                     os.path.join(tmpdir, 'model_' + str(i) + '.pkl'))

From bfbe2cf256ed1d7d0686b313f74ce526e272d125 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 04:56:54 +0800
Subject: [PATCH 34/53] Test demo.

---
 demo/guide-python/callbacks.py     | 9 ++++++++-
 python-package/xgboost/callback.py | 2 +-
 python-package/xgboost/training.py | 4 ++--
 tests/python/test_demos.py         | 6 ++++++
 4 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/demo/guide-python/callbacks.py b/demo/guide-python/callbacks.py
index 3e22ce40edad..1df329abc86c 100644
--- a/demo/guide-python/callbacks.py
+++ b/demo/guide-python/callbacks.py
@@ -10,6 +10,7 @@
 from sklearn.datasets import load_breast_cancer
 from sklearn.model_selection import train_test_split
 from matplotlib import pyplot as plt
+import argparse
 
 
 class Plotting(xgb.callback.TrainingCallback):
@@ -120,5 +121,11 @@ def check(as_pickle):
 
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--plot', default=1, type=int)
+    args = parser.parse_args()
+
     check_point_callback()
-    custom_callback()
+
+    if args.plot:
+        custom_callback()
diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index b9e57a1be300..fddfbb3df82e 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -642,7 +642,7 @@ def after_iteration(self, model, epoch, evals_log):
         self._epoch += 1
 
 
-class LegacyCallbacks(TrainingCallback):
+class LegacyCallbacks:
     '''Adapter for legacy callback functions.
 
     .. versionadded:: 1.3.0
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 7c418361b865..aa18d6befc23 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -233,7 +233,7 @@ def eval(self, iteration, feval):
         return self.bst.eval_set(self.watchlist, iteration, feval)
 
 
-class PackedBooster:
+class _PackedBooster:
     def __init__(self, cvfolds):
         self.cvfolds = cvfolds
 
@@ -468,7 +468,7 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
     callbacks = callback.CallbackContainer(callbacks, metric=feval, is_cv=True)
     callbacks.before_training(cvfolds)
 
-    booster = PackedBooster(cvfolds)
+    booster = _PackedBooster(cvfolds)
 
     for i in range(num_boost_round):
         if callbacks.before_iteration(booster, i, dtrain, None):
diff --git a/tests/python/test_demos.py b/tests/python/test_demos.py
index 33e64f7dd40b..bab6b7b83636 100644
--- a/tests/python/test_demos.py
+++ b/tests/python/test_demos.py
@@ -119,6 +119,12 @@ def test_aft_demo():
     os.remove('aft_model.json')
 
 
+def test_callbacks():
+    script = os.path.join(PYTHON_DEMO_DIR, 'callbacks.py')
+    cmd = ['python', script, '--plot=0']
+    subprocess.check_call(cmd)
+
+
 # gpu_acceleration is not tested due to covertype dataset is being too huge.
 # gamma regression is not tested as it requires running a R script first.
 # aft viz is not tested due to ploting is not controled

From a746b529b51f0cb5977260d522da6de00d817752 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 05:02:48 +0800
Subject: [PATCH 35/53] Support old callbacks.

---
 python-package/xgboost/callback.py |  2 ++
 python-package/xgboost/training.py | 35 ++++++++++++++++++++++--------
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index fddfbb3df82e..5d586ba34f4a 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -680,6 +680,7 @@ def __init__(self, callbacks, start_iteration, end_iteration,
         super().__init__()
 
     def before_iteration(self, model, epoch, dtrain, evals):
+        '''Called before each iteration.'''
         for cb in self.callbacks_before_iter:
             rank = rabit.get_rank()
             cb(CallbackEnv(model=model,
@@ -692,6 +693,7 @@ def before_iteration(self, model, epoch, dtrain, evals):
         return False
 
     def after_iteration(self, model, epoch, dtrain, evals):
+        '''Called after each iteration.'''
         evaluation_result_list = []
         if self.evals:
             bst_eval_set = model.eval_set(self.evals, epoch, self.feval)
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index aa18d6befc23..824cbab0e978 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -3,8 +3,7 @@
 # pylint: disable=too-many-branches, too-many-statements
 """Training Library containing training routines."""
 import numpy as np
-from .core import Booster, STRING_TYPES, XGBoostError, CallbackEnv
-from .core import EarlyStopException
+from .core import Booster, XGBoostError
 from .compat import (SKLEARN_INSTALLED, XGBStratifiedKFold)
 from . import rabit
 from . import callback
@@ -86,7 +85,6 @@ def _train_internal(params, dtrain,
                 rounds=early_stopping_rounds, maximize=maximize))
         callbacks = callback.CallbackContainer(callbacks, metric=feval)
     else:
-        assert False
         callbacks = _configure_deprected_callbacks(
             verbose_eval, early_stopping_rounds, maximize, start_iteration,
             num_boost_round, evals, feval, evals_result, callbacks)
@@ -238,19 +236,23 @@ def __init__(self, cvfolds):
         self.cvfolds = cvfolds
 
     def update(self, iteration, obj):
+        '''Iterate through folds for update'''
         for fold in self.cvfolds:
             fold.update(iteration, obj)
 
     def eval(self, iteration, feval):
+        '''Iterate through folds for eval'''
         result = [f.eval(iteration, feval) for f in self.cvfolds]
         return result
 
     def set_attr(self, **kwargs):
+        '''Iterate through folds for setting attributes'''
         for f in self.cvfolds:
             f.bst.set_attr(**kwargs)
 
     @property
     def best_iteration(self):
+        '''Get best_iteration'''
         ret = self.cvfolds[0].bst.attr('best_iteration')
         return int(ret)
 
@@ -460,12 +462,27 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
 
     # setup callbacks
     callbacks = [] if callbacks is None else callbacks
-    if verbose_eval:
-        callbacks.append(callback.EvaluationMonitor(show_stdv=show_stdv))
-    if early_stopping_rounds:
-        callbacks.append(callback.EarlyStopping(
-            rounds=early_stopping_rounds, maximize=maximize))
-    callbacks = callback.CallbackContainer(callbacks, metric=feval, is_cv=True)
+    is_new_callback = [isinstance(c, callback.TrainingCallback)
+                       for c in callbacks]
+    if any(is_new_callback) or not callbacks:
+        if verbose_eval:
+            callbacks.append(callback.EvaluationMonitor(show_stdv=show_stdv))
+        if early_stopping_rounds:
+            callbacks.append(callback.EarlyStopping(
+                rounds=early_stopping_rounds, maximize=maximize))
+        callbacks = callback.CallbackContainer(callbacks, metric=feval, is_cv=True)
+    else:
+        if early_stopping_rounds is not None:
+            callbacks.append(callback.early_stop(early_stopping_rounds,
+                                                 maximize=maximize,
+                                                 verbose=False))
+
+        if isinstance(verbose_eval, bool) and verbose_eval:
+            callbacks.append(callback.print_evaluation(show_stdv=show_stdv))
+        else:
+            if isinstance(verbose_eval, int):
+                callbacks.append(callback.print_evaluation(verbose_eval,
+                                                           show_stdv=show_stdv))
     callbacks.before_training(cvfolds)
 
     booster = _PackedBooster(cvfolds)

From b93a2d71cb9a0bee2e10f373b1440628a6bc7691 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 05:03:31 +0800
Subject: [PATCH 36/53] Remove todo.

---
 python-package/xgboost/callback.py | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 5d586ba34f4a..58c39bc95722 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -264,17 +264,6 @@ def callback(env):
 
 
 # The new implementation of callback functions.
-#
-# TODOs
-# - [x] eval_set
-# - [x] cv
-# - [x] tests
-# - [ ] doc
-# - [x] enforced best_xxx
-# - [x] make callbacks a set instead of list.
-# - [x] auto detect maximize.
-# - [x] Correct printing for cv
-
 # Breaking:
 # - reset learning rate no longer accepts total boosting rounds
 

From c3ad97efd789daf9ce1770137e9a65eaa07cf835 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 05:57:05 +0800
Subject: [PATCH 37/53] Initial batch of fixes.

---
 demo/guide-python/callbacks.py     |  2 +-
 python-package/xgboost/callback.py | 80 +++++++++++++++++-------------
 python-package/xgboost/training.py |  4 +-
 3 files changed, 50 insertions(+), 36 deletions(-)

diff --git a/demo/guide-python/callbacks.py b/demo/guide-python/callbacks.py
index 1df329abc86c..969d97fd8a0f 100644
--- a/demo/guide-python/callbacks.py
+++ b/demo/guide-python/callbacks.py
@@ -1,7 +1,7 @@
 '''
 Demo for using and defining callback functions.
 
-    .. versionadded:: 1.2.0
+    .. versionadded:: 1.3.0
 '''
 import xgboost as xgb
 import tempfile
diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 58c39bc95722..0f449e12d4a0 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -292,6 +292,34 @@ def after_iteration(self, model, epoch, evals_log):
         return False
 
 
+def _aggcv(rlist):
+    # pylint: disable=invalid-name
+    """Aggregate cross-validation results.
+
+    """
+    cvmap = {}
+    idx = rlist[0].split()[0]
+    for line in rlist:
+        arr = line.split()
+        assert idx == arr[0]
+        for metric_idx, it in enumerate(arr[1:]):
+            if not isinstance(it, STRING_TYPES):
+                it = it.decode()
+            k, v = it.split(':')
+            if (metric_idx, k) not in cvmap:
+                cvmap[(metric_idx, k)] = []
+            cvmap[(metric_idx, k)].append(float(v))
+    msg = idx
+    results = []
+    for (metric_idx, k), v in sorted(cvmap.items(), key=lambda x: x[0][0]):
+        v = numpy.array(v)
+        if not isinstance(msg, STRING_TYPES):
+            msg = msg.decode()
+        mean, std = numpy.mean(v), numpy.std(v)
+        results.extend([(k, mean, std)])
+    return results
+
+
 class CallbackContainer:
     '''A special callback for invoking a list of callbacks.
 
@@ -347,38 +375,11 @@ def _update_history(self, score, epoch):
                 self.history[data_name][metric_name] = [s]
         return False
 
-    def _aggcv(self, rlist):
-        # pylint: disable=invalid-name
-        """Aggregate cross-validation results.
-
-        """
-        cvmap = {}
-        idx = rlist[0].split()[0]
-        for line in rlist:
-            arr = line.split()
-            assert idx == arr[0]
-            for metric_idx, it in enumerate(arr[1:]):
-                if not isinstance(it, STRING_TYPES):
-                    it = it.decode()
-                k, v = it.split(':')
-                if (metric_idx, k) not in cvmap:
-                    cvmap[(metric_idx, k)] = []
-                cvmap[(metric_idx, k)].append(float(v))
-        msg = idx
-        results = []
-        for (metric_idx, k), v in sorted(cvmap.items(), key=lambda x: x[0][0]):
-            v = numpy.array(v)
-            if not isinstance(msg, STRING_TYPES):
-                msg = msg.decode()
-            mean, std = numpy.mean(v), numpy.std(v)
-            results.extend([(k, mean, std)])
-        return results
-
     def after_iteration(self, model, epoch, dtrain, evals):
         '''Function called after training iteration.'''
         if self.is_cv:
             scores = model.eval(epoch, self.metric)
-            scores = self._aggcv(scores)
+            scores = _aggcv(scores)
             self.aggregated_cv = scores
             self._update_history(scores, epoch)
         else:
@@ -571,6 +572,8 @@ def _fmt_metric(self, data, metric, score, std):
         return msg
 
     def after_iteration(self, model, epoch, evals_log):
+        if not evals_log:
+            return False
         msg = f'[{epoch}]'
         if rabit.get_rank() == self.printer_rank:
             for data, metric in evals_log.items():
@@ -651,7 +654,7 @@ class LegacyCallbacks:
     feval : Custom evaluation metric.
     '''
     def __init__(self, callbacks, start_iteration, end_iteration,
-                 evals, feval):
+                 feval, cvfolds=None):
         self.callbacks_before_iter = [
             cb for cb in callbacks
             if cb.__dict__.get('before_iteration', False)]
@@ -661,19 +664,27 @@ def __init__(self, callbacks, start_iteration, end_iteration,
 
         self.start_iteration = start_iteration
         self.end_iteration = end_iteration
+        self.cvfolds = cvfolds
 
-        self.evals = evals
         self.feval = feval
         assert self.feval is None or callable(self.feval)
 
         super().__init__()
 
+    def before_training(self, model):
+        '''Nothing to do for legacy callbacks'''
+        pass
+
+    def after_training(self, model):
+        '''Nothing to do for legacy callbacks'''
+        pass
+
     def before_iteration(self, model, epoch, dtrain, evals):
         '''Called before each iteration.'''
         for cb in self.callbacks_before_iter:
             rank = rabit.get_rank()
             cb(CallbackEnv(model=model,
-                           cvfolds=None,
+                           cvfolds=self.cvfolds,
                            iteration=epoch,
                            begin_iteration=self.start_iteration,
                            end_iteration=self.end_iteration,
@@ -684,19 +695,20 @@ def before_iteration(self, model, epoch, dtrain, evals):
     def after_iteration(self, model, epoch, dtrain, evals):
         '''Called after each iteration.'''
         evaluation_result_list = []
-        if self.evals:
-            bst_eval_set = model.eval_set(self.evals, epoch, self.feval)
+        if evals:
+            bst_eval_set = model.eval_set(evals, epoch, self.feval)
             if isinstance(bst_eval_set, STRING_TYPES):
                 msg = bst_eval_set
             else:
                 msg = bst_eval_set.decode()
             res = [x.split(':') for x in msg.split()]
             evaluation_result_list = [(k, float(v)) for k, v in res[1:]]
+
         try:
             for cb in self.callbacks_after_iter:
                 rank = rabit.get_rank()
                 cb(CallbackEnv(model=model,
-                               cvfolds=None,
+                               cvfolds=self.cvfolds,
                                iteration=epoch,
                                begin_iteration=self.start_iteration,
                                end_iteration=self.end_iteration,
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 824cbab0e978..3ae889afe362 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -26,7 +26,7 @@ def _configure_deprected_callbacks(
     if evals_result is not None:
         callbacks.append(callback.record_evaluation(evals_result))
     callbacks = callback.LegacyCallbacks(
-        callbacks, start_iteration, num_boost_round, evals, feval)
+        callbacks, start_iteration, num_boost_round, feval)
     return callbacks
 
 
@@ -483,6 +483,8 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
             if isinstance(verbose_eval, int):
                 callbacks.append(callback.print_evaluation(verbose_eval,
                                                            show_stdv=show_stdv))
+        callbacks = callback.LegacyCallbacks(callbacks, 0, num_boost_round, feval,
+                                             cvfolds=cvfolds)
     callbacks.before_training(cvfolds)
 
     booster = _PackedBooster(cvfolds)

From 3d52ed9b5989408d797584d2033b25a92b76500b Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 06:53:09 +0800
Subject: [PATCH 38/53] Fix cv.

---
 python-package/xgboost/callback.py | 10 ++++++++++
 python-package/xgboost/training.py |  4 ++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 0f449e12d4a0..e7eb62648521 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -669,6 +669,9 @@ def __init__(self, callbacks, start_iteration, end_iteration,
         self.feval = feval
         assert self.feval is None or callable(self.feval)
 
+        if cvfolds is not None:
+            self.aggregated_cv = None
+
         super().__init__()
 
     def before_training(self, model):
@@ -695,7 +698,14 @@ def before_iteration(self, model, epoch, dtrain, evals):
     def after_iteration(self, model, epoch, dtrain, evals):
         '''Called after each iteration.'''
         evaluation_result_list = []
+        if self.cvfolds is not None:
+            scores = model.eval(epoch, self.feval)
+            self.aggregated_cv = _aggcv(scores)
+            evaluation_result_list = self.aggregated_cv
+
         if evals:
+            # When cv is used, evals are embedded into folds.
+            assert self.cvfolds is None
             bst_eval_set = model.eval_set(evals, epoch, self.feval)
             if isinstance(bst_eval_set, STRING_TYPES):
                 msg = bst_eval_set
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 3ae889afe362..38dfdc0f1eae 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -363,7 +363,7 @@ def mknfold(dall, nfold, param, seed, evals=(), fpreproc=None, stratified=False,
 
 def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None,
        metrics=(), obj=None, feval=None, maximize=None, early_stopping_rounds=None,
-       fpreproc=None, as_pandas=True, verbose_eval=True, show_stdv=True,
+       fpreproc=None, as_pandas=True, verbose_eval=None, show_stdv=True,
        seed=0, callbacks=None, shuffle=True):
     # pylint: disable = invalid-name
     """Cross-validation with given parameters.
@@ -465,7 +465,7 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
     is_new_callback = [isinstance(c, callback.TrainingCallback)
                        for c in callbacks]
     if any(is_new_callback) or not callbacks:
-        if verbose_eval:
+        if isinstance(verbose_eval, bool) and verbose_eval:
             callbacks.append(callback.EvaluationMonitor(show_stdv=show_stdv))
         if early_stopping_rounds:
             callbacks.append(callback.EarlyStopping(

From 9dfe80fe934f13b85d540f48eebd9ed82181d5cc Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 07:06:32 +0800
Subject: [PATCH 39/53] Use error.

---
 tests/python/test_callback.py  | 12 ++++++++----
 tests/python/test_with_dask.py |  2 ++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 98a85bacc92a..dec719b8c8bf 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -28,7 +28,8 @@ def test_evaluation_monitor(self):
         D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
         evals_result = {}
         rounds = 10
-        xgb.train({'objective': 'binary:logistic'}, D_train,
+        xgb.train({'objective': 'binary:logistic',
+                   'eval_metric': 'error'}, D_train,
                   evals=[(D_train, 'Train'), (D_valid, 'Valid')],
                   num_boost_round=rounds,
                   evals_result=evals_result,
@@ -43,7 +44,8 @@ def test_early_stopping(self):
         evals_result = {}
         rounds = 30
         early_stopping_rounds = 5
-        booster = xgb.train({'objective': 'binary:logistic'}, D_train,
+        booster = xgb.train({'objective': 'binary:logistic',
+                             'eval_metric': 'error'}, D_train,
                             evals=[(D_train, 'Train'), (D_valid, 'Valid')],
                             num_boost_round=rounds,
                             evals_result=evals_result,
@@ -57,6 +59,7 @@ def test_early_stopping_custom_eval(self):
         D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
         early_stopping_rounds = 5
         booster = xgb.train({'objective': 'binary:logistic',
+                             'eval_metric': 'error',
                              'tree_method': 'hist'}, D_train,
                             evals=[(D_train, 'Train'), (D_valid, 'Valid')],
                             feval=tm.eval_error_metric,
@@ -93,7 +96,7 @@ def test_early_stopping_skl(self):
         cls = xgb.XGBClassifier()
         early_stopping_rounds = 5
         cls.fit(X, y, eval_set=[(X, y)],
-                early_stopping_rounds=early_stopping_rounds)
+                early_stopping_rounds=early_stopping_rounds, eval_metric='error')
         booster = cls.get_booster()
         dump = booster.get_dump(dump_format='json')
         assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
@@ -104,7 +107,8 @@ def test_early_stopping_custom_eval_skl(self):
         cls = xgb.XGBClassifier()
         early_stopping_rounds = 5
         cls.fit(X, y, eval_set=[(X, y)],
-                early_stopping_rounds=early_stopping_rounds)
+                early_stopping_rounds=early_stopping_rounds,
+                eval_metric=tm.eval_error_metric)
         booster = cls.get_booster()
         dump = booster.get_dump(dump_format='json')
         assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
diff --git a/tests/python/test_with_dask.py b/tests/python/test_with_dask.py
index ab6dd2ac1afe..16e9aa625113 100644
--- a/tests/python/test_with_dask.py
+++ b/tests/python/test_with_dask.py
@@ -715,6 +715,7 @@ def test_early_stopping(self, client):
         X, y = da.from_array(X), da.from_array(y)
         m = xgb.dask.DaskDMatrix(client, X, y)
         booster = xgb.dask.train(client, {'objective': 'binary:logistic',
+                                          'eval_metric': 'error',
                                           'tree_method': 'hist'}, m,
                                  evals=[(m, 'Train')],
                                  num_boost_round=1000,
@@ -731,6 +732,7 @@ def test_early_stopping_custom_eval(self, client):
         early_stopping_rounds = 5
         booster = xgb.dask.train(
             client, {'objective': 'binary:logistic',
+                     'eval_metric': 'error',
                      'tree_method': 'hist'}, m,
             evals=[(m, 'Train')],
             feval=tm.eval_error_metric,

From b41e0da77e3fdf23ba2d1b1fd8e43705d812b23f Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 07:37:24 +0800
Subject: [PATCH 40/53] Fix weird metric name.

---
 python-package/xgboost/callback.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index e7eb62648521..5e21d586c88b 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -362,7 +362,9 @@ def _update_history(self, score, epoch):
             if self.is_cv:
                 std = float(d[2])
                 s = (s, std)
-            data_name, metric_name = name.split('-')
+            splited_names = name.split('-')
+            data_name = splited_names[0]
+            metric_name = '-'.join(splited_names[1:])
             s = _allreduce_metric(s)
             if data_name in self.history:
                 data_history = self.history[data_name]
@@ -377,6 +379,8 @@ def _update_history(self, score, epoch):
 
     def after_iteration(self, model, epoch, dtrain, evals):
         '''Function called after training iteration.'''
+        for d, name in evals:
+            assert name.find('-') == -1, 'Dataset name should not contain `-`'
         if self.is_cv:
             scores = model.eval(epoch, self.metric)
             scores = _aggcv(scores)

From 8a9ad135080209ba84a26214dbf20f03793c3593 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 07:40:50 +0800
Subject: [PATCH 41/53] Fix attr.

---
 python-package/xgboost/callback.py | 4 ++--
 python-package/xgboost/training.py | 3 +++
 tests/python/test_demos.py         | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index 5e21d586c88b..dce8d9585a91 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -379,8 +379,6 @@ def _update_history(self, score, epoch):
 
     def after_iteration(self, model, epoch, dtrain, evals):
         '''Function called after training iteration.'''
-        for d, name in evals:
-            assert name.find('-') == -1, 'Dataset name should not contain `-`'
         if self.is_cv:
             scores = model.eval(epoch, self.metric)
             scores = _aggcv(scores)
@@ -388,6 +386,8 @@ def after_iteration(self, model, epoch, dtrain, evals):
             self._update_history(scores, epoch)
         else:
             evals = [] if evals is None else evals
+            for d, name in evals:
+                assert name.find('-') == -1, 'Dataset name should not contain `-`'
             score = model.eval_set(evals, epoch, self.metric)
             score = score.split()[1:]  # into datasets
             # split up `test-error:0.1234`
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 38dfdc0f1eae..3c285498c319 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -250,6 +250,9 @@ def set_attr(self, **kwargs):
         for f in self.cvfolds:
             f.bst.set_attr(**kwargs)
 
+    def attr(self, key):
+        return self.cvfolds[0].bst.attr(key)
+
     @property
     def best_iteration(self):
         '''Get best_iteration'''
diff --git a/tests/python/test_demos.py b/tests/python/test_demos.py
index bab6b7b83636..9ecf3aace61f 100644
--- a/tests/python/test_demos.py
+++ b/tests/python/test_demos.py
@@ -119,7 +119,7 @@ def test_aft_demo():
     os.remove('aft_model.json')
 
 
-def test_callbacks():
+def test_callbacks_demo():
     script = os.path.join(PYTHON_DEMO_DIR, 'callbacks.py')
     cmd = ['python', script, '--plot=0']
     subprocess.check_call(cmd)

From 810be6b5906a627a10b425ff422501b1a53c6beb Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 07:42:33 +0800
Subject: [PATCH 42/53] Lint.

---
 python-package/xgboost/callback.py | 4 +---
 python-package/xgboost/training.py | 5 +++--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index dce8d9585a91..ca536411a7e5 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -386,7 +386,7 @@ def after_iteration(self, model, epoch, dtrain, evals):
             self._update_history(scores, epoch)
         else:
             evals = [] if evals is None else evals
-            for d, name in evals:
+            for _, name in evals:
                 assert name.find('-') == -1, 'Dataset name should not contain `-`'
             score = model.eval_set(evals, epoch, self.metric)
             score = score.split()[1:]  # into datasets
@@ -680,11 +680,9 @@ def __init__(self, callbacks, start_iteration, end_iteration,
 
     def before_training(self, model):
         '''Nothing to do for legacy callbacks'''
-        pass
 
     def after_training(self, model):
         '''Nothing to do for legacy callbacks'''
-        pass
 
     def before_iteration(self, model, epoch, dtrain, evals):
         '''Called before each iteration.'''
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 3c285498c319..e221318e4f9d 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -11,7 +11,7 @@
 
 def _configure_deprected_callbacks(
         verbose_eval, early_stopping_rounds, maximize, start_iteration,
-        num_boost_round, evals, feval, evals_result, callbacks):
+        num_boost_round, feval, evals_result, callbacks):
     # Most of legacy advanced options becomes callbacks
     if isinstance(verbose_eval, bool) and verbose_eval:
         callbacks.append(callback.print_evaluation())
@@ -87,7 +87,7 @@ def _train_internal(params, dtrain,
     else:
         callbacks = _configure_deprected_callbacks(
             verbose_eval, early_stopping_rounds, maximize, start_iteration,
-            num_boost_round, evals, feval, evals_result, callbacks)
+            num_boost_round, feval, evals_result, callbacks)
 
     callbacks.before_training(bst)
     for i in range(start_iteration, num_boost_round):
@@ -251,6 +251,7 @@ def set_attr(self, **kwargs):
             f.bst.set_attr(**kwargs)
 
     def attr(self, key):
+        '''Redirect to booster attr.'''
         return self.cvfolds[0].bst.attr(key)
 
     @property

From 14fe528a6c0301a9df596e92304c24bdd8c04d58 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 12:29:07 +0800
Subject: [PATCH 43/53] Legacy callback.

---
 python-package/xgboost/training.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index e221318e4f9d..3f81cd2fcc2b 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -113,7 +113,7 @@ def _train_internal(params, dtrain,
 
     callbacks.after_training(bst)
 
-    if evals_result is not None:
+    if evals_result is not None and any(is_new_callback):
         evals_result.update(callbacks.history)
 
     if bst.attr('best_score') is not None:

From bc777168efcdf120f913ec37cfbcc25d5003012c Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 12:43:35 +0800
Subject: [PATCH 44/53] Fix dask parameter.

---
 python-package/xgboost/dask.py | 96 +++++++++++++++++++---------------
 tests/python/test_with_dask.py |  7 ++-
 2 files changed, 59 insertions(+), 44 deletions(-)

diff --git a/python-package/xgboost/dask.py b/python-package/xgboost/dask.py
index 180dd6332d02..7650505b3d78 100644
--- a/python-package/xgboost/dask.py
+++ b/python-package/xgboost/dask.py
@@ -1049,7 +1049,7 @@ async def _():
         return self.client.sync(_).__await__()
 
     @property
-    def client(self):
+    def client(self) -> Client:
         '''The dask client used in this model.'''
         client = _xgb_get_client(self._client)
         return client
@@ -1063,26 +1063,25 @@ def client(self, clt):
                    ['estimators', 'model'])
 class DaskXGBRegressor(DaskScikitLearnBase, XGBRegressorBase):
     # pylint: disable=missing-class-docstring
-    async def _fit_async(self,
-                         X,
-                         y,
-                         sample_weights=None,
-                         base_margin=None,
-                         eval_set=None,
-                         sample_weight_eval_set=None,
-                         early_stopping_rounds=None,
-                         verbose=True):
-        dtrain = await DaskDMatrix(
-            client=self.client, data=X, label=y, weight=sample_weights,
-            base_margin=base_margin, missing=self.missing
-        )
+    async def _fit_async(self, X, y, sample_weights, base_margin, eval_set,
+                         sample_weight_eval_set, early_stopping_rounds,
+                         verbose):
+        dtrain = await DaskDMatrix(client=self.client,
+                                   data=X,
+                                   label=y,
+                                   weight=sample_weights,
+                                   base_margin=base_margin,
+                                   missing=self.missing)
         params = self.get_xgb_params()
-        evals = await _evaluation_matrices(self.client,
-                                           eval_set, sample_weight_eval_set,
+        evals = await _evaluation_matrices(self.client, eval_set,
+                                           sample_weight_eval_set,
                                            self.missing)
-        results = await train(client=self.client, params=params, dtrain=dtrain,
+        results = await train(client=self.client,
+                              params=params,
+                              dtrain=dtrain,
                               num_boost_round=self.get_num_boosting_rounds(),
-                              evals=evals, verbose_eval=verbose,
+                              evals=evals,
+                              verbose_eval=verbose,
                               early_stopping_rounds=early_stopping_rounds)
         self._Booster = results['booster']
         # pylint: disable=attribute-defined-outside-init
@@ -1090,7 +1089,9 @@ async def _fit_async(self,
         return self
 
     # pylint: disable=missing-docstring
-    def fit(self, X, y,
+    def fit(self,
+            X,
+            y,
             sample_weights=None,
             base_margin=None,
             eval_set=None,
@@ -1098,10 +1099,15 @@ def fit(self, X, y,
             early_stopping_rounds=None,
             verbose=True):
         _assert_dask_support()
-        return self.client.sync(
-            self._fit_async, X, y, sample_weights, base_margin,
-            eval_set, sample_weight_eval_set, verbose
-        )
+        return self.client.sync(self._fit_async,
+                                X=X,
+                                y=y,
+                                sample_weights=sample_weights,
+                                base_margin=base_margin,
+                                eval_set=eval_set,
+                                sample_weight_eval_set=sample_weight_eval_set,
+                                early_stopping_rounds=early_stopping_rounds,
+                                verbose=verbose)
 
     async def _predict_async(
             self, data, output_margin=False, base_margin=None):
@@ -1121,20 +1127,16 @@ def predict(self, data, output_margin=False, base_margin=None):
                                 output_margin=output_margin,
                                 base_margin=base_margin)
 
-
 @xgboost_model_doc(
     'Implementation of the scikit-learn API for XGBoost classification.',
-    ['estimators', 'model']
-)
+    ['estimators', 'model'])
 class DaskXGBClassifier(DaskScikitLearnBase, XGBClassifierBase):
-    async def _fit_async(self, X, y,
-                         sample_weights=None,
-                         base_margin=None,
-                         eval_set=None,
-                         sample_weight_eval_set=None,
-                         verbose=True):
+    async def _fit_async(self, X, y, sample_weights, base_margin, eval_set,
+                         sample_weight_eval_set, verbose):
         dtrain = await DaskDMatrix(client=self.client,
-                                   data=X, label=y, weight=sample_weights,
+                                   data=X,
+                                   label=y,
+                                   weight=sample_weights,
                                    base_margin=base_margin,
                                    missing=self.missing)
         params = self.get_xgb_params()
@@ -1152,18 +1154,23 @@ async def _fit_async(self, X, y,
         else:
             params["objective"] = "binary:logistic"
 
-        evals = await _evaluation_matrices(self.client,
-                                           eval_set, sample_weight_eval_set,
+        evals = await _evaluation_matrices(self.client, eval_set,
+                                           sample_weight_eval_set,
                                            self.missing)
-        results = await train(client=self.client, params=params, dtrain=dtrain,
+        results = await train(client=self.client,
+                              params=params,
+                              dtrain=dtrain,
                               num_boost_round=self.get_num_boosting_rounds(),
-                              evals=evals, verbose_eval=verbose)
+                              evals=evals,
+                              verbose_eval=verbose)
         self._Booster = results['booster']
         # pylint: disable=attribute-defined-outside-init
         self.evals_result_ = results['history']
         return self
 
-    def fit(self, X, y,
+    def fit(self,
+            X,
+            y,
             sample_weights=None,
             base_margin=None,
             eval_set=None,
@@ -1171,10 +1178,15 @@ def fit(self, X, y,
             early_stopping_rounds=None,
             verbose=True):
         _assert_dask_support()
-        return self.client.sync(
-            self._fit_async, X, y, sample_weights, base_margin, eval_set,
-            sample_weight_eval_set, verbose
-        )
+        return self.client.sync(self._fit_async,
+                                X=X,
+                                y=y,
+                                sample_weights=sample_weights,
+                                base_margin=base_margin,
+                                eval_set=eval_set,
+                                sample_weight_eval_set=sample_weight_eval_set,
+                                early_stopping_rounds=early_stopping_rounds,
+                                verbose=verbose)
 
     async def _predict_proba_async(self, data, output_margin=False,
                                    base_margin=None):
diff --git a/tests/python/test_with_dask.py b/tests/python/test_with_dask.py
index 16e9aa625113..6f12f93a26f7 100644
--- a/tests/python/test_with_dask.py
+++ b/tests/python/test_with_dask.py
@@ -328,7 +328,7 @@ def test_sklearn_grid_search():
             reg.client = client
             model = GridSearchCV(reg, {'max_depth': [2, 4],
                                        'n_estimators': [5, 10]},
-                                 cv=2, verbose=1, iid=True)
+                                 cv=2, verbose=1)
             model.fit(X, y)
             # Expect unique results for each parameter value This confirms
             # sklearn is able to successfully update the parameter
@@ -714,14 +714,17 @@ def test_early_stopping(self, client):
         X, y = load_breast_cancer(return_X_y=True)
         X, y = da.from_array(X), da.from_array(y)
         m = xgb.dask.DaskDMatrix(client, X, y)
+        early_stopping_rounds = 5
         booster = xgb.dask.train(client, {'objective': 'binary:logistic',
                                           'eval_metric': 'error',
                                           'tree_method': 'hist'}, m,
                                  evals=[(m, 'Train')],
                                  num_boost_round=1000,
-                                 early_stopping_rounds=5)['booster']
+                                 early_stopping_rounds=early_stopping_rounds)['booster']
         assert hasattr(booster, 'best_score')
         assert booster.best_iteration == 10
+        dump = booster.get_dump(dump_format='json')
+        assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
 
     @pytest.mark.skipif(**tm.no_sklearn())
     def test_early_stopping_custom_eval(self, client):

From 0625e4026078dac53f20c6a31579eaafd95bd1f7 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 13:35:57 +0800
Subject: [PATCH 45/53] Cleanup.

---
 python-package/xgboost/dask.py     |  4 +-
 python-package/xgboost/training.py | 81 +++++++++++++++---------------
 2 files changed, 44 insertions(+), 41 deletions(-)

diff --git a/python-package/xgboost/dask.py b/python-package/xgboost/dask.py
index 7650505b3d78..53172b19a9e4 100644
--- a/python-package/xgboost/dask.py
+++ b/python-package/xgboost/dask.py
@@ -1132,7 +1132,8 @@ def predict(self, data, output_margin=False, base_margin=None):
     ['estimators', 'model'])
 class DaskXGBClassifier(DaskScikitLearnBase, XGBClassifierBase):
     async def _fit_async(self, X, y, sample_weights, base_margin, eval_set,
-                         sample_weight_eval_set, verbose):
+                         sample_weight_eval_set, early_stopping_rounds,
+                         verbose):
         dtrain = await DaskDMatrix(client=self.client,
                                    data=X,
                                    label=y,
@@ -1162,6 +1163,7 @@ async def _fit_async(self, X, y, sample_weights, base_margin, eval_set,
                               dtrain=dtrain,
                               num_boost_round=self.get_num_boosting_rounds(),
                               evals=evals,
+                              early_stopping_rounds=early_stopping_rounds,
                               verbose_eval=verbose)
         self._Booster = results['booster']
         # pylint: disable=attribute-defined-outside-init
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 3f81cd2fcc2b..3f45c456c5e2 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -11,36 +11,31 @@
 
 def _configure_deprected_callbacks(
         verbose_eval, early_stopping_rounds, maximize, start_iteration,
-        num_boost_round, feval, evals_result, callbacks):
+        num_boost_round, feval, evals_result, callbacks, show_stdv, cvfolds):
     # Most of legacy advanced options becomes callbacks
-    if isinstance(verbose_eval, bool) and verbose_eval:
-        callbacks.append(callback.print_evaluation())
-    else:
-        if isinstance(verbose_eval, int):
-            callbacks.append(callback.print_evaluation(verbose_eval))
-
     if early_stopping_rounds is not None:
         callbacks.append(callback.early_stop(early_stopping_rounds,
                                              maximize=maximize,
                                              verbose=bool(verbose_eval)))
+    if isinstance(verbose_eval, bool) and verbose_eval:
+        callbacks.append(callback.print_evaluation(show_stdv=show_stdv))
+    else:
+        if isinstance(verbose_eval, int):
+            callbacks.append(callback.print_evaluation(verbose_eval,
+                                                       show_stdv=show_stdv))
     if evals_result is not None:
         callbacks.append(callback.record_evaluation(evals_result))
     callbacks = callback.LegacyCallbacks(
-        callbacks, start_iteration, num_boost_round, feval)
+        callbacks, start_iteration, num_boost_round, feval, cvfolds=cvfolds)
     return callbacks
 
 
-def _train_internal(params, dtrain,
-                    num_boost_round=10, evals=(),
-                    obj=None, feval=None,
-                    xgb_model=None, callbacks=None,
-                    evals_result=None, maximize=None,
-                    verbose_eval=None, early_stopping_rounds=None):
-    """internal training function"""
-    callbacks = [] if callbacks is None else callbacks
-    evals = list(evals)
-    params = params.copy()
+def _is_new_callback(callbacks):
+    return any(isinstance(c, callback.TrainingCallback)
+               for c in callbacks) or not callbacks
 
+
+def _configure_metrics(params):
     if isinstance(params, dict) and 'eval_metric' in params \
        and isinstance(params['eval_metric'], list):
         params = dict((k, v) for k, v in params.items())
@@ -49,6 +44,19 @@ def _train_internal(params, dtrain,
         params = list(params.items())
         for eval_metric in eval_metrics:
             params += [('eval_metric', eval_metric)]
+    return params
+
+
+def _train_internal(params, dtrain,
+                    num_boost_round=10, evals=(),
+                    obj=None, feval=None,
+                    xgb_model=None, callbacks=None,
+                    evals_result=None, maximize=None,
+                    verbose_eval=None, early_stopping_rounds=None):
+    """internal training function"""
+    callbacks = [] if callbacks is None else callbacks
+    evals = list(evals)
+    params = _configure_metrics(params.copy())
 
     bst = Booster(params, [dtrain] + [d[0] for d in evals])
     nboost = 0
@@ -74,10 +82,10 @@ def _train_internal(params, dtrain,
     start_iteration = int(version / 2)
     nboost += start_iteration
 
-    is_new_callback = [isinstance(c, callback.TrainingCallback)
-                       for c in callbacks]
-    if any(is_new_callback) or not callbacks:
-        assert all(is_new_callback), "You can't mix two styles of callbacks."
+    is_new_callback = _is_new_callback(callbacks)
+    if is_new_callback:
+        assert all(isinstance(c, callback.TrainingCallback)
+                   for c in callbacks), "You can't mix two styles of callbacks."
         if verbose_eval:
             callbacks.append(callback.EvaluationMonitor())
         if early_stopping_rounds:
@@ -87,7 +95,8 @@ def _train_internal(params, dtrain,
     else:
         callbacks = _configure_deprected_callbacks(
             verbose_eval, early_stopping_rounds, maximize, start_iteration,
-            num_boost_round, feval, evals_result, callbacks)
+            num_boost_round, feval, evals_result, callbacks,
+            show_stdv=False, cvfolds=None)
 
     callbacks.before_training(bst)
     for i in range(start_iteration, num_boost_round):
@@ -113,7 +122,7 @@ def _train_internal(params, dtrain,
 
     callbacks.after_training(bst)
 
-    if evals_result is not None and any(is_new_callback):
+    if evals_result is not None and is_new_callback:
         evals_result.update(callbacks.history)
 
     if bst.attr('best_score') is not None:
@@ -466,9 +475,10 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
 
     # setup callbacks
     callbacks = [] if callbacks is None else callbacks
-    is_new_callback = [isinstance(c, callback.TrainingCallback)
-                       for c in callbacks]
-    if any(is_new_callback) or not callbacks:
+    is_new_callback = _is_new_callback(callbacks)
+    if is_new_callback:
+        assert all(isinstance(c, callback.TrainingCallback)
+                   for c in callbacks), "You can't mix two styles of callbacks."
         if isinstance(verbose_eval, bool) and verbose_eval:
             callbacks.append(callback.EvaluationMonitor(show_stdv=show_stdv))
         if early_stopping_rounds:
@@ -476,19 +486,10 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
                 rounds=early_stopping_rounds, maximize=maximize))
         callbacks = callback.CallbackContainer(callbacks, metric=feval, is_cv=True)
     else:
-        if early_stopping_rounds is not None:
-            callbacks.append(callback.early_stop(early_stopping_rounds,
-                                                 maximize=maximize,
-                                                 verbose=False))
-
-        if isinstance(verbose_eval, bool) and verbose_eval:
-            callbacks.append(callback.print_evaluation(show_stdv=show_stdv))
-        else:
-            if isinstance(verbose_eval, int):
-                callbacks.append(callback.print_evaluation(verbose_eval,
-                                                           show_stdv=show_stdv))
-        callbacks = callback.LegacyCallbacks(callbacks, 0, num_boost_round, feval,
-                                             cvfolds=cvfolds)
+        callbacks = _configure_deprected_callbacks(
+            verbose_eval, early_stopping_rounds, maximize, 0,
+            num_boost_round, feval, None, callbacks,
+            show_stdv=show_stdv, cvfolds=cvfolds)
     callbacks.before_training(cvfolds)
 
     booster = _PackedBooster(cvfolds)

From b991092ff70c100473fa6c9afb3d6a193b574495 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Mon, 5 Oct 2020 14:12:11 +0800
Subject: [PATCH 46/53] Fix moved test.

---
 tests/python-gpu/test_gpu_basic_models.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/python-gpu/test_gpu_basic_models.py b/tests/python-gpu/test_gpu_basic_models.py
index 5705cedc08aa..c3fc259a0653 100644
--- a/tests/python-gpu/test_gpu_basic_models.py
+++ b/tests/python-gpu/test_gpu_basic_models.py
@@ -5,12 +5,12 @@
 import xgboost as xgb
 sys.path.append("tests/python")
 # Don't import the test class, otherwise they will run twice.
-import test_basic_models as test_bm  # noqa
+import test_callback as test_cb  # noqa
 rng = np.random.RandomState(1994)
 
 
 class TestGPUBasicModels(unittest.TestCase):
-    cputest = test_bm.TestModels()
+    cputest = test_cb.TestCallbacks()
 
     def run_cls(self, X, y, deterministic):
         cls = xgb.XGBClassifier(tree_method='gpu_hist',
@@ -36,7 +36,8 @@ def run_cls(self, X, y, deterministic):
         return hash(model_0), hash(model_1)
 
     def test_eta_decay_gpu_hist(self):
-        self.cputest.run_eta_decay('gpu_hist')
+        self.cputest.run_eta_decay('gpu_hist', True)
+        self.cputest.run_eta_decay('gpu_hist', False)
 
     def test_deterministic_gpu_hist(self):
         kRows = 1000

From 60240f925f88bb11d4b2e5de5b6a41252a765cd1 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Tue, 6 Oct 2020 11:17:23 +0800
Subject: [PATCH 47/53] Reviewers' comment.

---
 doc/python/callbacks.rst           | 6 +++---
 python-package/xgboost/training.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/python/callbacks.rst b/doc/python/callbacks.rst
index 4f58dc2e7aee..1c46e4a01f1d 100644
--- a/doc/python/callbacks.rst
+++ b/doc/python/callbacks.rst
@@ -34,15 +34,15 @@ this callback function directly into XGBoost:
 
     # Specify which dataset and which metric should be used for early stopping.
     early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
-					    metric_name='PyError',
-					    data_name='Train')
+                                            metric_name='PyError',
+                                            data_name='Train')
 
     booster = xgb.train(
         {'objective': 'binary:logistic',
          'eval_metric': ['error', 'rmse'],
          'tree_method': 'hist'}, D_train,
         evals=[(D_train, 'Train'), (D_valid, 'Valid')],
-	feval=eval_error_metric,
+        feval=eval_error_metric,
         num_boost_round=1000,
         callbacks=[early_stop],
         verbose_eval=False)
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 3f45c456c5e2..7bf5268b31c1 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -9,7 +9,7 @@
 from . import callback
 
 
-def _configure_deprected_callbacks(
+def _configure_deprecated_callbacks(
         verbose_eval, early_stopping_rounds, maximize, start_iteration,
         num_boost_round, feval, evals_result, callbacks, show_stdv, cvfolds):
     # Most of legacy advanced options becomes callbacks
@@ -93,7 +93,7 @@ def _train_internal(params, dtrain,
                 rounds=early_stopping_rounds, maximize=maximize))
         callbacks = callback.CallbackContainer(callbacks, metric=feval)
     else:
-        callbacks = _configure_deprected_callbacks(
+        callbacks = _configure_deprecated_callbacks(
             verbose_eval, early_stopping_rounds, maximize, start_iteration,
             num_boost_round, feval, evals_result, callbacks,
             show_stdv=False, cvfolds=None)
@@ -486,7 +486,7 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
                 rounds=early_stopping_rounds, maximize=maximize))
         callbacks = callback.CallbackContainer(callbacks, metric=feval, is_cv=True)
     else:
-        callbacks = _configure_deprected_callbacks(
+        callbacks = _configure_deprecated_callbacks(
             verbose_eval, early_stopping_rounds, maximize, 0,
             num_boost_round, feval, None, callbacks,
             show_stdv=show_stdv, cvfolds=cvfolds)

From 2316f8afad6e89581ee6cc450b4b6cc16772b338 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sat, 10 Oct 2020 16:24:50 +0800
Subject: [PATCH 48/53] Reviewers' comments.

---
 demo/guide-python/callbacks.py     | 2 +-
 doc/python/callbacks.rst           | 8 ++++----
 python-package/xgboost/training.py | 6 ++++--
 tests/python/test_callback.py      | 4 ++--
 4 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/demo/guide-python/callbacks.py b/demo/guide-python/callbacks.py
index 969d97fd8a0f..4e5d8b6bac00 100644
--- a/demo/guide-python/callbacks.py
+++ b/demo/guide-python/callbacks.py
@@ -97,7 +97,7 @@ def check(as_pickle):
     # Check point to a temporary directory for demo
     with tempfile.TemporaryDirectory() as tmpdir:
         # Use callback class from xgboost.callback
-        # Feel free to subclass/customize it to suite your need.
+        # Feel free to subclass/customize it to suit your need.
         check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
                                                       rounds=rounds,
                                                       name='model')
diff --git a/doc/python/callbacks.rst b/doc/python/callbacks.rst
index 1c46e4a01f1d..009b4d742fe5 100644
--- a/doc/python/callbacks.rst
+++ b/doc/python/callbacks.rst
@@ -4,7 +4,7 @@ Callback Functions
 
 This document gives a basic walkthrough of callback function used in XGBoost Python
 package.  In XGBoost 1.3, a new callback interface is designed for Python package, which
-provides the flexiablity of designing various extension for training.  Also, XGBoost has a
+provides the flexiblity of designing various extension for training.  Also, XGBoost has a
 number of pre-defined callbacks for supporting early stopping, checkpoints etc.
 
 #######################
@@ -30,11 +30,11 @@ this callback function directly into XGBoost:
         r[gt] = 1 - label[gt]
         le = predt <= 0.5
         r[le] = label[le]
-        return 'PyError', np.sum(r)
+        return 'CustomErr', np.sum(r)
 
     # Specify which dataset and which metric should be used for early stopping.
     early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
-                                            metric_name='PyError',
+                                            metric_name='CustomErr',
                                             data_name='Train')
 
     booster = xgb.train(
@@ -48,7 +48,7 @@ this callback function directly into XGBoost:
         verbose_eval=False)
 
     dump = booster.get_dump(dump_format='json')
-    assert len(early_stop.stopping_history['Valid']['PyError']) == len(dump)
+    assert len(early_stop.stopping_history['Valid']['CustomErr']) == len(dump)
 
 ##########################
 Defining your own callback
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 7bf5268b31c1..c9aee40082c2 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -12,6 +12,8 @@
 def _configure_deprecated_callbacks(
         verbose_eval, early_stopping_rounds, maximize, start_iteration,
         num_boost_round, feval, evals_result, callbacks, show_stdv, cvfolds):
+    link = 'https://xgboost.readthedocs.io/en/latest/python/callbacks.html'
+    raise DeprecationWarning(f'Old style callback is deprecated.  See: {link}')
     # Most of legacy advanced options becomes callbacks
     if early_stopping_rounds is not None:
         callbacks.append(callback.early_stop(early_stopping_rounds,
@@ -85,7 +87,7 @@ def _train_internal(params, dtrain,
     is_new_callback = _is_new_callback(callbacks)
     if is_new_callback:
         assert all(isinstance(c, callback.TrainingCallback)
-                   for c in callbacks), "You can't mix two styles of callbacks."
+                   for c in callbacks), "You can't mix new and old callback styles."
         if verbose_eval:
             callbacks.append(callback.EvaluationMonitor())
         if early_stopping_rounds:
@@ -478,7 +480,7 @@ def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False, folds=None
     is_new_callback = _is_new_callback(callbacks)
     if is_new_callback:
         assert all(isinstance(c, callback.TrainingCallback)
-                   for c in callbacks), "You can't mix two styles of callbacks."
+                   for c in callbacks), "You can't mix new and old callback styles."
         if isinstance(verbose_eval, bool) and verbose_eval:
             callbacks.append(callback.EvaluationMonitor(show_stdv=show_stdv))
         if early_stopping_rounds:
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index dec719b8c8bf..09d90f7e04c1 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -74,7 +74,7 @@ def test_early_stopping_customize(self):
         D_valid = xgb.DMatrix(self.X_valid, self.y_valid)
         early_stopping_rounds = 5
         early_stop = xgb.callback.EarlyStopping(rounds=early_stopping_rounds,
-                                                metric_name='PyError',
+                                                metric_name='CustomErr',
                                                 data_name='Train')
         # Specify which dataset and which metric should be used for early stopping.
         booster = xgb.train(
@@ -88,7 +88,7 @@ def test_early_stopping_customize(self):
             verbose_eval=False)
         dump = booster.get_dump(dump_format='json')
         assert len(dump) - booster.best_iteration == early_stopping_rounds + 1
-        assert len(early_stop.stopping_history['Train']['PyError']) == len(dump)
+        assert len(early_stop.stopping_history['Train']['CustomErr']) == len(dump)
 
     def test_early_stopping_skl(self):
         from sklearn.datasets import load_breast_cancer

From 4a6bb5dbefb27a74f1f3a8f87de1c16b003af67c Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sat, 10 Oct 2020 16:26:19 +0800
Subject: [PATCH 49/53] Redundant attribute in demo.

---
 demo/guide-python/callbacks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/demo/guide-python/callbacks.py b/demo/guide-python/callbacks.py
index 4e5d8b6bac00..88d4b6697526 100644
--- a/demo/guide-python/callbacks.py
+++ b/demo/guide-python/callbacks.py
@@ -22,7 +22,6 @@ def __init__(self, rounds):
         self.fig = plt.figure()
         self.ax = self.fig.add_subplot(111)
         self.rounds = rounds
-        self.has_lines = False
         self.lines = {}
         self.fig.show()
         self.x = np.linspace(0, self.rounds, self.rounds)

From 10c705aee4e29852ed9e547240f6c9bc1984b01a Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sat, 10 Oct 2020 16:34:35 +0800
Subject: [PATCH 50/53] Small fixes in doc and parameter naming.

---
 demo/guide-python/callbacks.py     |  2 +-
 python-package/xgboost/callback.py | 52 ++++++++++++++++--------------
 tests/python/test_callback.py      |  2 +-
 3 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/demo/guide-python/callbacks.py b/demo/guide-python/callbacks.py
index 88d4b6697526..905b92399129 100644
--- a/demo/guide-python/callbacks.py
+++ b/demo/guide-python/callbacks.py
@@ -98,7 +98,7 @@ def check(as_pickle):
         # Use callback class from xgboost.callback
         # Feel free to subclass/customize it to suit your need.
         check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
-                                                      rounds=rounds,
+                                                      iterations=rounds,
                                                       name='model')
         xgb.train({'objective': 'binary:logistic'}, m,
                   num_boost_round=10,
diff --git a/python-package/xgboost/callback.py b/python-package/xgboost/callback.py
index ca536411a7e5..32cc957cb5c9 100644
--- a/python-package/xgboost/callback.py
+++ b/python-package/xgboost/callback.py
@@ -284,11 +284,11 @@ def after_training(self, model):
         '''Run after training is finished.'''
 
     def before_iteration(self, model, epoch, evals_log):
-        '''Run before each iteration.'''
+        '''Run before each iteration.  Return True when training should stop.'''
         return False
 
     def after_iteration(self, model, epoch, evals_log):
-        '''Run after each iteration.'''
+        '''Run after each iteration.  Return True when training should stop.'''
         return False
 
 
@@ -320,8 +320,26 @@ def _aggcv(rlist):
     return results
 
 
+def _allreduce_metric(score):
+    '''Helper function for computing customized metric in distributed
+    environment.  Not strictly correct as many functions don't use mean value
+    as final result.
+
+    '''
+    world = rabit.get_world_size()
+    assert world != 0
+    if world == 1:
+        return score
+    if isinstance(score, tuple):  # has mean and stdv
+        raise ValueError(
+            'xgboost.cv function should not be used in distributed environment.')
+    score = numpy.array([score])
+    score = rabit.allreduce(score, rabit.Op.SUM) / world
+    return score[0]
+
+
 class CallbackContainer:
-    '''A special callback for invoking a list of callbacks.
+    '''A special callback for invoking a list of other callbacks.
 
     .. versionadded:: 1.3.0
 
@@ -330,7 +348,7 @@ def __init__(self, callbacks: List[TrainingCallback],
                  metric: Callable = None, is_cv: bool = False):
         self.callbacks = set(callbacks)
         if metric is not None:
-            msg = 'metric must be callable object for monitor.  For ' + \
+            msg = 'metric must be callable object for monitoring.  For ' + \
                 'builtin metrics, passing them in training parameter' + \
                 ' will invoke monitor automatically.'
             assert callable(metric), msg
@@ -426,24 +444,6 @@ def after_iteration(self, model, epoch, evals_log):
         model.set_param('learning_rate', self.learning_rates(epoch))
 
 
-def _allreduce_metric(score):
-    '''Helper function for computing customized metric in distributed
-    environment.  Not strictly correct as many functions don't use mean value
-    as final result.
-
-    '''
-    world = rabit.get_world_size()
-    assert world != 0
-    if world == 1:
-        return score
-    if isinstance(score, tuple):  # has mean and stdv
-        raise ValueError(
-            'xgboost.cv function should not be used in distributed environment.')
-    score = numpy.array([score])
-    score = rabit.allreduce(score, rabit.Op.SUM) / world
-    return score[0]
-
-
 # pylint: disable=too-many-instance-attributes
 class EarlyStopping(TrainingCallback):
     ''' Callback function for early stopping
@@ -460,6 +460,8 @@ class EarlyStopping(TrainingCallback):
         Name of dataset that is used for early stopping.
     maximize : bool
         Whether to maximize evaluation metric.  None means auto (discouraged).
+    save_best : bool
+        Placeholder, the feature is not yet supported.
     '''
     def __init__(self,
                  rounds,
@@ -562,6 +564,8 @@ class EvaluationMonitor(TrainingCallback):
         Extra user defined metric.
     rank : int
         Which worker should be used for printing the result.
+    show_stdv : bool
+        Used in cv to show standard deviation.  Users should not specify it.
     '''
     def __init__(self, rank=0, show_stdv=False):
         self.printer_rank = rank
@@ -616,11 +620,11 @@ class TrainingCheckPoint(TrainingCallback):
 
     '''
     def __init__(self, directory: os.PathLike, name: str = 'model',
-                 as_pickle=False, rounds: int = 100):
+                 as_pickle=False, iterations: int = 100):
         self._path = directory
         self._name = name
         self._as_pickle = as_pickle
-        self._iterations = rounds
+        self._iterations = iterations
         self._epoch = 0
         super().__init__()
 
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 09d90f7e04c1..9c786b93be0d 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -208,7 +208,7 @@ def test_check_point(self):
         m = xgb.DMatrix(X, y)
         with tempfile.TemporaryDirectory() as tmpdir:
             check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
-                                                          rounds=1,
+                                                          iterations=1,
                                                           name='model')
             xgb.train({'objective': 'binary:logistic'}, m,
                       num_boost_round=10,

From e3ad79a4883106b1a9509acd407f92ac7ceac6e5 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sat, 10 Oct 2020 16:37:57 +0800
Subject: [PATCH 51/53] Fix naming in test.

---
 tests/python/testing.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/python/testing.py b/tests/python/testing.py
index 1df680adcf13..d6ea5ccac7d7 100644
--- a/tests/python/testing.py
+++ b/tests/python/testing.py
@@ -247,7 +247,7 @@ def eval_error_metric(predt, dtrain: xgb.DMatrix):
     r[gt] = 1 - label[gt]
     le = predt <= 0.5
     r[le] = label[le]
-    return 'PyError', np.sum(r)
+    return 'CustomErr', np.sum(r)
 
 
 CURDIR = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))

From f8662cf3d1cac2e59191d70e9df0bcb65abafb0b Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sat, 10 Oct 2020 16:42:51 +0800
Subject: [PATCH 52/53] Pytest deprecated call.

---
 python-package/xgboost/training.py |  3 ++-
 tests/python/test_callback.py      | 11 +++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index c9aee40082c2..c224191ab779 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -2,6 +2,7 @@
 # pylint: disable=too-many-locals, too-many-arguments, invalid-name
 # pylint: disable=too-many-branches, too-many-statements
 """Training Library containing training routines."""
+import warnings
 import numpy as np
 from .core import Booster, XGBoostError
 from .compat import (SKLEARN_INSTALLED, XGBStratifiedKFold)
@@ -13,7 +14,7 @@ def _configure_deprecated_callbacks(
         verbose_eval, early_stopping_rounds, maximize, start_iteration,
         num_boost_round, feval, evals_result, callbacks, show_stdv, cvfolds):
     link = 'https://xgboost.readthedocs.io/en/latest/python/callbacks.html'
-    raise DeprecationWarning(f'Old style callback is deprecated.  See: {link}')
+    warnings.warn(f'Old style callback is deprecated.  See: {link}', DeprecationWarning)
     # Most of legacy advanced options becomes callbacks
     if early_stopping_rounds is not None:
         callbacks.append(callback.early_stop(early_stopping_rounds,
diff --git a/tests/python/test_callback.py b/tests/python/test_callback.py
index 9c786b93be0d..a4b37615a7da 100644
--- a/tests/python/test_callback.py
+++ b/tests/python/test_callback.py
@@ -191,15 +191,18 @@ def eta_decay(ithround, num_boost_round=num_round):
             assert eval_errors_3[i] != eval_errors_2[i]
 
     def test_eta_decay_hist(self):
-        # self.run_eta_decay('hist', True)
+        with pytest.deprecated_call():
+            self.run_eta_decay('hist', True)
         self.run_eta_decay('hist', False)
 
     def test_eta_decay_approx(self):
-        # self.run_eta_decay('approx', True)
+        with pytest.deprecated_call():
+            self.run_eta_decay('approx', True)
         self.run_eta_decay('approx', False)
 
     def test_eta_decay_exact(self):
-        # self.run_eta_decay('exact', True)
+        with pytest.deprecated_call():
+            self.run_eta_decay('exact', True)
         self.run_eta_decay('exact', False)
 
     def test_check_point(self):
@@ -219,7 +222,7 @@ def test_check_point(self):
                     os.path.join(tmpdir, 'model_' + str(i) + '.json'))
 
             check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
-                                                          rounds=1,
+                                                          iterations=1,
                                                           as_pickle=True,
                                                           name='model')
             xgb.train({'objective': 'binary:logistic'}, m,

From 8e4513932c5aa95c5ac9d29e0396d27f84091cb2 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sat, 10 Oct 2020 16:43:30 +0800
Subject: [PATCH 53/53] Fix typo.

---
 demo/guide-python/callbacks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/demo/guide-python/callbacks.py b/demo/guide-python/callbacks.py
index 905b92399129..d4362eeba4d0 100644
--- a/demo/guide-python/callbacks.py
+++ b/demo/guide-python/callbacks.py
@@ -109,7 +109,7 @@ def check(as_pickle):
         # This version of checkpoint saves everything including parameters and
         # model.  See: doc/tutorials/saving_model.rst
         check_point = xgb.callback.TrainingCheckPoint(directory=tmpdir,
-                                                      rounds=rounds,
+                                                      iterations=rounds,
                                                       as_pickle=True,
                                                       name='model')
         xgb.train({'objective': 'binary:logistic'}, m,