diff --git a/include/xgboost/learner.h b/include/xgboost/learner.h
index 32f98c68ea15..34ae5a4d53bb 100644
--- a/include/xgboost/learner.h
+++ b/include/xgboost/learner.h
@@ -8,16 +8,14 @@
 #ifndef XGBOOST_LEARNER_H_
 #define XGBOOST_LEARNER_H_
 
-#include <dmlc/any.h>
 #include <xgboost/base.h>
 #include <xgboost/feature_map.h>
-#include <xgboost/generic_parameters.h>
+#include <xgboost/generic_parameters.h>  // Context
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 #include <xgboost/predictor.h>
 #include <xgboost/task.h>
 
-#include <limits>  // std::numeric_limit
 #include <map>
 #include <memory>
 #include <string>
@@ -275,7 +273,7 @@ class Learner : public Model, public Configurable, public dmlc::Serializable {
   /**
    * \brief Return the context object of this Booster.
    */
-  virtual GenericParameter const* Ctx() const = 0;
+  virtual Context const* Ctx() const = 0;
   /*!
    * \brief Get configuration arguments currently stored by the learner
    * \return Key-value pairs representing configuration arguments
diff --git a/src/common/linalg_op.h b/src/common/linalg_op.h
index 52790e33d859..0de173c8e73f 100644
--- a/src/common/linalg_op.h
+++ b/src/common/linalg_op.h
@@ -4,6 +4,7 @@
 #ifndef XGBOOST_COMMON_LINALG_OP_H_
 #define XGBOOST_COMMON_LINALG_OP_H_
 #include <type_traits>
+#include <cstdint>  // std::int32_t
 
 #include "common.h"
 #include "threading_utils.h"
@@ -60,7 +61,7 @@ void ElementWiseKernel(GenericParameter const* ctx, linalg::TensorView<T, D> t,
 }
 #endif  // !defined(XGBOOST_USE_CUDA)
 
-template <typename T, int32_t kDim>
+template <typename T, std::int32_t kDim>
 auto cbegin(TensorView<T, kDim> v) {  // NOLINT
   auto it = common::MakeIndexTransformIter([&](size_t i) -> std::remove_cv_t<T> const& {
     return linalg::detail::Apply(v, linalg::UnravelIndex(i, v.Shape()));
@@ -68,19 +69,19 @@ auto cbegin(TensorView<T, kDim> v) {  // NOLINT
   return it;
 }
 
-template <typename T, int32_t kDim>
+template <typename T, std::int32_t kDim>
 auto cend(TensorView<T, kDim> v) {  // NOLINT
   return cbegin(v) + v.Size();
 }
 
-template <typename T, int32_t kDim>
+template <typename T, std::int32_t kDim>
 auto begin(TensorView<T, kDim> v) {  // NOLINT
   auto it = common::MakeIndexTransformIter(
       [&](size_t i) -> T& { return linalg::detail::Apply(v, linalg::UnravelIndex(i, v.Shape())); });
   return it;
 }
 
-template <typename T, int32_t kDim>
+template <typename T, std::int32_t kDim>
 auto end(TensorView<T, kDim> v) {  // NOLINT
   return begin(v) + v.Size();
 }
diff --git a/src/learner.cc b/src/learner.cc
index 06d2cf9ec51c..d0ecfcb345b0 100644
--- a/src/learner.cc
+++ b/src/learner.cc
@@ -6,6 +6,7 @@
  */
 #include "xgboost/learner.h"
 
+#include <dmlc/any.h>
 #include <dmlc/io.h>
 #include <dmlc/parameter.h>
 #include <dmlc/thread_local.h>
@@ -13,7 +14,7 @@
 #include <algorithm>
 #include <atomic>
 #include <iomanip>
-#include <limits>
+#include <limits>  // std::numeric_limits
 #include <memory>
 #include <mutex>
 #include <sstream>
@@ -31,7 +32,6 @@
 #include "common/threading_utils.h"
 #include "common/timer.h"
 #include "common/version.h"
-#include "dmlc/any.h"
 #include "xgboost/base.h"
 #include "xgboost/c_api.h"
 #include "xgboost/data.h"
@@ -180,7 +180,7 @@ struct LearnerModelParamLegacy : public dmlc::Parameter<LearnerModelParamLegacy>
   // declare parameters
   DMLC_DECLARE_PARAMETER(LearnerModelParamLegacy) {
     DMLC_DECLARE_FIELD(base_score)
-        .set_default(std::numeric_limits<float>::quiet_NaN())
+        .set_default(ObjFunction::DefaultBaseScore())
         .describe("Global bias of the model.");
     DMLC_DECLARE_FIELD(num_feature)
         .set_default(0)
@@ -252,7 +252,8 @@ void LearnerModelParam::Copy(LearnerModelParam const& that) {
   CHECK(base_score_.Data()->HostCanRead());
 
   num_feature = that.num_feature;
-  num_output_group = that.num_output_group, task = that.task;
+  num_output_group = that.num_output_group;
+  task = that.task;
 }
 
 struct LearnerTrainParam : public XGBoostParameter<LearnerTrainParam> {
@@ -422,15 +423,15 @@ class LearnerConfiguration : public Learner {
 
     CHECK(obj_);
     auto task = obj_->Task();
-    linalg::Tensor<float, 1> base_score({1}, ctx_.gpu_id);
+    linalg::Tensor<float, 1> base_score({1}, Ctx()->gpu_id);
     auto h_base_score = base_score.HostView();
 
     // transform to margin
     h_base_score(0) = obj_->ProbToMargin(mparam_.base_score);
     // move it to model param, which is shared with all other components.
-    learner_model_param_ = LearnerModelParam(&ctx_, mparam_, std::move(base_score), task);
+    learner_model_param_ = LearnerModelParam(Ctx(), mparam_, std::move(base_score), task);
     CHECK(learner_model_param_.Initialized());
-    CHECK_NE(learner_model_param_.BaseScore(&ctx_).Size(), 0);
+    CHECK_NE(learner_model_param_.BaseScore(Ctx()).Size(), 0);
   }
 
  public:
@@ -665,7 +666,7 @@ class LearnerConfiguration : public Learner {
     return cfg_;
   }
 
-  GenericParameter const* Ctx() const override { return &ctx_; }
+  Context const* Ctx() const override { return &ctx_; }
 
  private:
   void ValidateParameters() {