diff --git a/amalgamation/xgboost-all0.cc b/amalgamation/xgboost-all0.cc index ded96bcbab4c..3bc15d05388d 100644 --- a/amalgamation/xgboost-all0.cc +++ b/amalgamation/xgboost-all0.cc @@ -75,19 +75,20 @@ #include "../src/collective/communicator.cc" // common -#include "../src/common/common.cc" -#include "../src/common/column_matrix.cc" -#include "../src/common/random.cc" #include "../src/common/charconv.cc" -#include "../src/common/timer.cc" -#include "../src/common/quantile.cc" -#include "../src/common/host_device_vector.cc" +#include "../src/common/column_matrix.cc" +#include "../src/common/common.cc" #include "../src/common/hist_util.cc" +#include "../src/common/host_device_vector.cc" #include "../src/common/io.cc" #include "../src/common/json.cc" +#include "../src/common/numeric.cc" #include "../src/common/pseudo_huber.cc" +#include "../src/common/quantile.cc" +#include "../src/common/random.cc" #include "../src/common/survival_util.cc" #include "../src/common/threading_utils.cc" +#include "../src/common/timer.cc" #include "../src/common/version.cc" // c_api diff --git a/include/xgboost/learner.h b/include/xgboost/learner.h index 51fefac1365f..34ae5a4d53bb 100644 --- a/include/xgboost/learner.h +++ b/include/xgboost/learner.h @@ -8,10 +8,9 @@ #ifndef XGBOOST_LEARNER_H_ #define XGBOOST_LEARNER_H_ -#include #include #include -#include +#include // Context #include #include #include @@ -274,7 +273,7 @@ class Learner : public Model, public Configurable, public dmlc::Serializable { /** * \brief Return the context object of this Booster. */ - virtual GenericParameter const* Ctx() const = 0; + virtual Context const* Ctx() const = 0; /*! * \brief Get configuration arguments currently stored by the learner * \return Key-value pairs representing configuration arguments @@ -289,7 +288,7 @@ class Learner : public Model, public Configurable, public dmlc::Serializable { /*! \brief The evaluation metrics used to evaluate the model. */ std::vector > metrics_; /*! \brief Training parameter. */ - GenericParameter generic_parameters_; + Context ctx_; }; struct LearnerModelParamLegacy; @@ -298,8 +297,14 @@ struct LearnerModelParamLegacy; * \brief Basic Model Parameters, used to describe the booster. */ struct LearnerModelParam { - /* \brief global bias */ - bst_float base_score { 0.5f }; + private: + /** + * \brief Global bias, this is just a scalar value but can be extended to vector when we + * support multi-class and multi-target. + */ + linalg::Tensor base_score_; + + public: /* \brief number of features */ uint32_t num_feature { 0 }; /* \brief number of classes, if it is multi-class classification */ @@ -310,7 +315,18 @@ struct LearnerModelParam { LearnerModelParam() = default; // As the old `LearnerModelParamLegacy` is still used by binary IO, we keep // this one as an immutable copy. - LearnerModelParam(LearnerModelParamLegacy const& user_param, float base_margin, ObjInfo t); + LearnerModelParam(Context const* ctx, LearnerModelParamLegacy const& user_param, + linalg::Tensor base_margin, ObjInfo t); + LearnerModelParam(LearnerModelParamLegacy const& user_param, ObjInfo t); + LearnerModelParam(bst_feature_t n_features, linalg::Tensor base_margin, + uint32_t n_groups) + : base_score_{std::move(base_margin)}, num_feature{n_features}, num_output_group{n_groups} {} + + linalg::TensorView BaseScore(Context const* ctx) const; + linalg::TensorView BaseScore(int32_t device) const; + + void Copy(LearnerModelParam const& that); + /* \brief Whether this parameter is initialized with LearnerModelParamLegacy. */ bool Initialized() const { return num_feature != 0; } }; diff --git a/include/xgboost/linalg.h b/include/xgboost/linalg.h index 944903ac83e5..3897e89ea1ce 100644 --- a/include/xgboost/linalg.h +++ b/include/xgboost/linalg.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -213,6 +215,22 @@ LINALG_HD decltype(auto) constexpr Apply(Fn &&f, Tup &&t) { constexpr auto kSize = std::tuple_size::value; return Apply(std::forward(f), std::forward(t), std::make_index_sequence{}); } + +/** + * C++ 17 conjunction + */ +template +struct Conjunction : std::true_type {}; +template +struct Conjunction : B1 {}; +template +struct Conjunction : std::conditional_t, B1> {}; + +template +using IsAllIntegral = Conjunction>...>; + +template +using EnableIfIntegral = std::enable_if_t::value>; } // namespace detail /** @@ -406,7 +424,7 @@ class TensorView { * * \endcode */ - template + template * = nullptr> LINALG_HD T &operator()(Index &&...index) { static_assert(sizeof...(index) <= kDim, "Invalid index."); size_t offset = detail::Offset<0ul>(stride_, 0ul, std::forward(index)...); @@ -416,7 +434,7 @@ class TensorView { /** * \brief Index the tensor to obtain a scalar value. */ - template + template * = nullptr> LINALG_HD T const &operator()(Index &&...index) const { static_assert(sizeof...(index) <= kDim, "Invalid index."); size_t offset = detail::Offset<0ul>(stride_, 0ul, std::forward(index)...); @@ -656,7 +674,7 @@ class Tensor { } if (device >= 0) { data_.SetDevice(device); - data_.DevicePointer(); // Pull to device; + data_.ConstDevicePointer(); // Pull to device; } CHECK_EQ(data_.Size(), detail::CalcSize(shape_)); } @@ -702,12 +720,29 @@ class Tensor { } template - explicit Tensor(std::initializer_list data, I const (&shape)[D], int32_t device) { + explicit Tensor(std::initializer_list data, I const (&shape)[D], + int32_t device = Context::kCpuId) { auto &h_vec = data_.HostVector(); h_vec = data; // shape this->Initialize(shape, device); } + /** + * \brief Index operator. Not thread safe, should not be used in performance critical + * region. For more efficient indexing, consider getting a view first. + */ + template + T &operator()(Index &&...idx) { + return this->HostView()(std::forward(idx)...); + } + /** + * \brief Index operator. Not thread safe, should not be used in performance critical + * region. For more efficient indexing, consider getting a view first. + */ + template + T const &operator()(Index &&...idx) const { + return this->HostView()(std::forward(idx)...); + } /** * \brief Get a \ref TensorView for this tensor. @@ -761,7 +796,7 @@ class Tensor { * * If the total size is changed, then data in this tensor is no longer valid. */ - template + template * = nullptr> void Reshape(S &&...s) { static_assert(sizeof...(S) <= kDim, "Invalid shape."); detail::ReshapeImpl<0>(shape_, std::forward(s)...); @@ -777,15 +812,20 @@ class Tensor { * * If the total size is changed, then data in this tensor is no longer valid. */ - template - void Reshape(size_t (&shape)[D]) { + template + void Reshape(common::Span shape) { static_assert(D <= kDim, "Invalid shape."); - std::copy(shape, shape + D, this->shape_); + std::copy(shape.data(), shape.data() + D, this->shape_); std::fill(shape_ + D, shape_ + kDim, 1); auto n = detail::CalcSize(shape_); data_.Resize(n); } + template + void Reshape(size_t (&shape)[D]) { + this->Reshape(common::Span{shape}); + } + /** * \brief Set device ordinal for this tensor. */ diff --git a/include/xgboost/objective.h b/include/xgboost/objective.h index d30f81379f5a..0c0d502bdbfb 100644 --- a/include/xgboost/objective.h +++ b/include/xgboost/objective.h @@ -27,7 +27,10 @@ class RegTree; /*! \brief interface of objective function */ class ObjFunction : public Configurable { protected: - GenericParameter const* ctx_; + Context const* ctx_; + + public: + static constexpr float DefaultBaseScore() { return 0.5f; } public: /*! \brief virtual destructor */ @@ -75,6 +78,13 @@ class ObjFunction : public Configurable { virtual bst_float ProbToMargin(bst_float base_score) const { return base_score; } + /** + * \brief Make initialize estimation of prediction. + * + * \param info MetaInfo that contains label. + * \param base_score Output estimation. + */ + virtual void InitEstimation(MetaInfo const& info, linalg::Tensor* base_score) const; /*! * \brief Return task of this objective. */ diff --git a/include/xgboost/predictor.h b/include/xgboost/predictor.h index 33c695bc19bf..877ff462bf24 100644 --- a/include/xgboost/predictor.h +++ b/include/xgboost/predictor.h @@ -102,13 +102,10 @@ class PredictionContainer { */ class Predictor { protected: - /* - * \brief Runtime parameters. - */ - GenericParameter const* ctx_; + Context const* ctx_; public: - explicit Predictor(GenericParameter const* ctx) : ctx_{ctx} {} + explicit Predictor(Context const* ctx) : ctx_{ctx} {} virtual ~Predictor() = default; diff --git a/src/common/algorithm.h b/src/common/algorithm.h index addcd95cfa24..a5d2d1974eff 100644 --- a/src/common/algorithm.h +++ b/src/common/algorithm.h @@ -1,7 +1,8 @@ /*! * Copyright 2022 by XGBoost Contributors */ -#pragma once +#ifndef XGBOOST_COMMON_ALGORITHM_H_ +#define XGBOOST_COMMON_ALGORITHM_H_ #include // std::upper_bound #include // std::size_t @@ -14,3 +15,4 @@ auto SegmentId(It first, It last, Idx idx) { } } // namespace common } // namespace xgboost +#endif // XGBOOST_COMMON_ALGORITHM_H_ diff --git a/src/common/common.h b/src/common/common.h index 0f21739876b2..1eaf9ae7f4a0 100644 --- a/src/common/common.h +++ b/src/common/common.h @@ -265,6 +265,7 @@ struct OptionalWeights { explicit OptionalWeights(float w) : dft{w} {} XGBOOST_DEVICE float operator[](size_t i) const { return weights.empty() ? dft : weights[i]; } + auto Empty() const { return weights.empty(); } }; /** @@ -276,7 +277,7 @@ XGBOOST_DEVICE size_t LastOf(size_t group, Indexable const &indptr) { } /** - * @brief A CRTP (curiously recurring template pattern) helper function. + * \brief A CRTP (curiously recurring template pattern) helper function. * * https://www.fluentcpp.com/2017/05/19/crtp-helper/ * @@ -284,7 +285,7 @@ XGBOOST_DEVICE size_t LastOf(size_t group, Indexable const &indptr) { * 1. Makes "crtp" explicit in the inheritance structure of a CRTP base class. * 2. Avoids having to `static_cast` in a lot of places. * - * @tparam T The derived class in a CRTP hierarchy. + * \tparam T The derived class in a CRTP hierarchy. */ template struct Crtp { @@ -292,6 +293,13 @@ struct Crtp { T const &Underlying() const { return static_cast(*this); } }; +/** + * \brief C++17 std::as_const + */ +template +typename std::add_const::type &AsConst(T &v) noexcept { // NOLINT(runtime/references) + return v; +} } // namespace common } // namespace xgboost #endif // XGBOOST_COMMON_COMMON_H_ diff --git a/src/common/linalg_op.h b/src/common/linalg_op.h index 05f050772ccc..0de173c8e73f 100644 --- a/src/common/linalg_op.h +++ b/src/common/linalg_op.h @@ -4,6 +4,7 @@ #ifndef XGBOOST_COMMON_LINALG_OP_H_ #define XGBOOST_COMMON_LINALG_OP_H_ #include +#include // std::int32_t #include "common.h" #include "threading_utils.h" @@ -59,6 +60,31 @@ void ElementWiseKernel(GenericParameter const* ctx, linalg::TensorView t, ElementWiseKernelHost(t, ctx->Threads(), fn); } #endif // !defined(XGBOOST_USE_CUDA) + +template +auto cbegin(TensorView v) { // NOLINT + auto it = common::MakeIndexTransformIter([&](size_t i) -> std::remove_cv_t const& { + return linalg::detail::Apply(v, linalg::UnravelIndex(i, v.Shape())); + }); + return it; +} + +template +auto cend(TensorView v) { // NOLINT + return cbegin(v) + v.Size(); +} + +template +auto begin(TensorView v) { // NOLINT + auto it = common::MakeIndexTransformIter( + [&](size_t i) -> T& { return linalg::detail::Apply(v, linalg::UnravelIndex(i, v.Shape())); }); + return it; +} + +template +auto end(TensorView v) { // NOLINT + return begin(v) + v.Size(); +} } // namespace linalg } // namespace xgboost #endif // XGBOOST_COMMON_LINALG_OP_H_ diff --git a/src/common/numeric.cc b/src/common/numeric.cc new file mode 100644 index 000000000000..9740d6af1f8d --- /dev/null +++ b/src/common/numeric.cc @@ -0,0 +1,28 @@ +/*! + * Copyright 2022 by XGBoost Contributors + */ +#include "numeric.h" + +#include // std::accumulate +#include // std::is_same + +#include "threading_utils.h" // MemStackAllocator, ParallelFor, DefaultMaxThreads +#include "xgboost/generic_parameters.h" // Context +#include "xgboost/host_device_vector.h" // HostDeviceVector + +namespace xgboost { +namespace common { +double Reduce(Context const* ctx, HostDeviceVector const& values) { + if (ctx->IsCPU()) { + auto const& h_values = values.ConstHostVector(); + MemStackAllocator result_tloc(ctx->Threads(), 0); + ParallelFor(h_values.size(), ctx->Threads(), + [&](auto i) { result_tloc[omp_get_thread_num()] += h_values[i]; }); + auto result = std::accumulate(result_tloc.cbegin(), result_tloc.cend(), 0.0); + static_assert(std::is_same::value, ""); + return result; + } + return cuda::Reduce(ctx, values); +} +} // namespace common +} // namespace xgboost diff --git a/src/common/numeric.cu b/src/common/numeric.cu new file mode 100644 index 000000000000..faac6ddb56da --- /dev/null +++ b/src/common/numeric.cu @@ -0,0 +1,25 @@ +/*! + * Copyright 2022 by XGBoost Contributors + */ +#include +#include // thrust:plus + +#include "device_helpers.cuh" // dh::Reduce, safe_cuda, dh::XGBCachingDeviceAllocator +#include "numeric.h" +#include "xgboost/generic_parameters.h" // Context +#include "xgboost/host_device_vector.h" // HostDeviceVector + +namespace xgboost { +namespace common { +namespace cuda { +double Reduce(Context const* ctx, HostDeviceVector const& values) { + values.SetDevice(ctx->gpu_id); + auto const d_values = values.ConstDeviceSpan(); + dh::XGBCachingDeviceAllocator alloc; + auto res = dh::Reduce(thrust::cuda::par(alloc), d_values.data(), + d_values.data() + d_values.size(), 0.0, thrust::plus{}); + return res; +} +} // namespace cuda +} // namespace common +} // namespace xgboost diff --git a/src/common/numeric.h b/src/common/numeric.h index ff5ac2242033..cff3e8a12121 100644 --- a/src/common/numeric.h +++ b/src/common/numeric.h @@ -8,8 +8,10 @@ #include // std::iterator_traits #include -#include "threading_utils.h" -#include "xgboost/generic_parameters.h" +#include "common.h" // AssertGPUSupport +#include "threading_utils.h" // MemStackAllocator, DefaultMaxThreads +#include "xgboost/generic_parameters.h" // Context +#include "xgboost/host_device_vector.h" // HostDeviceVector namespace xgboost { namespace common { @@ -18,8 +20,8 @@ namespace common { * \brief Run length encode on CPU, input must be sorted. */ template -void RunLengthEncode(Iter begin, Iter end, std::vector *p_out) { - auto &out = *p_out; +void RunLengthEncode(Iter begin, Iter end, std::vector* p_out) { + auto& out = *p_out; out = std::vector{0}; size_t n = std::distance(begin, end); for (size_t i = 1; i < n; ++i) { @@ -45,7 +47,7 @@ void PartialSum(int32_t n_threads, InIt begin, InIt end, T init, OutIt out_it) { auto n = static_cast(std::distance(begin, end)); const size_t batch_threads = std::max(static_cast(1), std::min(n, static_cast(n_threads))); - common::MemStackAllocator partial_sums(batch_threads); + MemStackAllocator partial_sums(batch_threads); size_t block_size = n / batch_threads; @@ -90,6 +92,20 @@ void PartialSum(int32_t n_threads, InIt begin, InIt end, T init, OutIt out_it) { } exc.Rethrow(); } + +namespace cuda { +double Reduce(Context const* ctx, HostDeviceVector const& values); +#if !defined(XGBOOST_USE_CUDA) +inline double Reduce(Context const*, HostDeviceVector const&) { + AssertGPUSupport(); + return 0; +} +#endif // !defined(XGBOOST_USE_CUDA) +} // namespace cuda +/** + * \brief Reduction with summation. + */ +double Reduce(Context const* ctx, HostDeviceVector const& values); } // namespace common } // namespace xgboost diff --git a/src/common/stats.cu b/src/common/stats.cu new file mode 100644 index 000000000000..dcb04ac4b5de --- /dev/null +++ b/src/common/stats.cu @@ -0,0 +1,47 @@ +/*! + * Copyright 2022 by XGBoost Contributors + */ + +#include // thrust::make_counting_iterator + +#include "common.h" // common::OptionalWeights +#include "device_helpers.cuh" // dh::MakeTransformIterator, tcbegin, tcend +#include "stats.cuh" // common::SegmentedQuantile, common::SegmentedWeightedQuantile +#include "xgboost/generic_parameters.h" // Context +#include "xgboost/host_device_vector.h" // HostDeviceVector +#include "xgboost/linalg.h" // linalg::TensorView, UnravelIndex, Apply + +namespace xgboost { +namespace common { +namespace cuda { +float Median(Context const* ctx, linalg::TensorView t, + common::OptionalWeights weights) { + HostDeviceVector segments{0, t.Size()}; + segments.SetDevice(ctx->gpu_id); + auto d_segments = segments.ConstDeviceSpan(); + auto val_it = dh::MakeTransformIterator( + thrust::make_counting_iterator(0ul), [=] XGBOOST_DEVICE(size_t i) { + return linalg::detail::Apply(t, linalg::UnravelIndex(i, t.Shape())); + }); + + HostDeviceVector quantile{0}; + quantile.SetDevice(ctx->gpu_id); + if (weights.Empty()) { + common::SegmentedQuantile(ctx, 0.5, dh::tcbegin(d_segments), dh::tcend(d_segments), val_it, + val_it + t.Size(), &quantile); + } else { + CHECK_NE(t.Shape(1), 0); + auto w_it = dh::MakeTransformIterator(thrust::make_counting_iterator(0ul), + [=] XGBOOST_DEVICE(size_t i) { + auto sample_idx = i / t.Shape(1); + return weights[sample_idx]; + }); + common::SegmentedWeightedQuantile(ctx, 0.5, dh::tcbegin(d_segments), dh::tcend(d_segments), + val_it, val_it + t.Size(), w_it, w_it + t.Size(), &quantile); + } + CHECK_EQ(quantile.Size(), 1); + return quantile.HostVector().front(); +} +} // namespace cuda +} // namespace common +} // namespace xgboost diff --git a/src/common/stats.h b/src/common/stats.h index 4ad9e4aa770a..c6347c421a9f 100644 --- a/src/common/stats.h +++ b/src/common/stats.h @@ -8,7 +8,8 @@ #include #include -#include "common.h" +#include "common.h" // AssertGPUSupport +#include "xgboost/generic_parameters.h" #include "xgboost/linalg.h" namespace xgboost { @@ -90,6 +91,44 @@ float WeightedQuantile(double alpha, Iter begin, Iter end, WeightIter weights) { idx = std::min(idx, static_cast(n - 1)); return val(idx); } + +namespace cuda { +float Median(Context const* ctx, linalg::TensorView t, + common::OptionalWeights weights); +#if !defined(XGBOOST_USE_CUDA) +inline float Median(Context const*, linalg::TensorView, common::OptionalWeights) { + AssertGPUSupport(); + return 0; +} +#endif // !defined(XGBOOST_USE_CUDA) +} // namespace cuda + +inline float Median(Context const* ctx, linalg::Tensor const& t, + HostDeviceVector const& weights) { + if (!ctx->IsCPU()) { + weights.SetDevice(ctx->gpu_id); + auto opt_weights = OptionalWeights(weights.ConstDeviceSpan()); + auto t_v = t.View(ctx->gpu_id); + return cuda::Median(ctx, t_v, opt_weights); + } + + auto opt_weights = OptionalWeights(weights.ConstHostSpan()); + auto t_v = t.HostView(); + auto iter = common::MakeIndexTransformIter( + [&](size_t i) { return linalg::detail::Apply(t_v, linalg::UnravelIndex(i, t_v.Shape())); }); + float q{0}; + if (opt_weights.Empty()) { + q = common::Quantile(0.5, iter, iter + t_v.Size()); + } else { + CHECK_NE(t_v.Shape(1), 0); + auto w_it = common::MakeIndexTransformIter([&](size_t i) { + auto sample_idx = i / t_v.Shape(1); + return opt_weights[sample_idx]; + }); + q = common::WeightedQuantile(0.5, iter, iter + t_v.Size(), w_it); + } + return q; +} } // namespace common } // namespace xgboost #endif // XGBOOST_COMMON_STATS_H_ diff --git a/src/common/threading_utils.h b/src/common/threading_utils.h index 9d4149d7d07b..656e570ae812 100644 --- a/src/common/threading_utils.h +++ b/src/common/threading_utils.h @@ -8,6 +8,7 @@ #include #include +#include // std::int32_t #include #include // std::is_signed #include @@ -253,7 +254,7 @@ inline int32_t OmpGetNumThreads(int32_t n_threads) { * MaxStackSize, it will be allocated inside the stack. Otherwise, it will be * heap-allocated. */ -template +template class MemStackAllocator { public: explicit MemStackAllocator(size_t required_size) : required_size_(required_size) { @@ -278,11 +279,23 @@ class MemStackAllocator { T& operator[](size_t i) { return ptr_[i]; } T const& operator[](size_t i) const { return ptr_[i]; } + auto data() const { return ptr_; } // NOLINT + auto data() { return ptr_; } // NOLINT + std::size_t size() const { return required_size_; } // NOLINT + + auto cbegin() const { return data(); } // NOLINT + auto cend() const { return data() + size(); } // NOLINT + private: T* ptr_ = nullptr; size_t required_size_; T stack_mem_[MaxStackSize]; }; + +/** + * \brief Constant that can be used for initializing static thread local memory. + */ +std::int32_t constexpr DefaultMaxThreads() { return 128; } } // namespace common } // namespace xgboost diff --git a/src/data/array_interface.h b/src/data/array_interface.h index c646654bef3f..e90473458e1c 100644 --- a/src/data/array_interface.h +++ b/src/data/array_interface.h @@ -345,8 +345,8 @@ struct ToDType { }; #if !defined(XGBOOST_USE_CUDA) -inline void ArrayInterfaceHandler::SyncCudaStream(int64_t stream) { common::AssertGPUSupport(); } -inline bool ArrayInterfaceHandler::IsCudaPtr(void const *ptr) { return false; } +inline void ArrayInterfaceHandler::SyncCudaStream(int64_t) { common::AssertGPUSupport(); } +inline bool ArrayInterfaceHandler::IsCudaPtr(void const *) { return false; } #endif // !defined(XGBOOST_USE_CUDA) /** diff --git a/src/gbm/gblinear.cc b/src/gbm/gblinear.cc index 35de4c70d604..c8cdfeb476b1 100644 --- a/src/gbm/gblinear.cc +++ b/src/gbm/gblinear.cc @@ -161,9 +161,10 @@ class GBLinear : public GradientBooster { uint32_t layer_begin, uint32_t) override { LinearCheckLayer(layer_begin); const int ngroup = model_.learner_model_param->num_output_group; + + auto base_score = learner_model_param_->BaseScore(ctx_); for (int gid = 0; gid < ngroup; ++gid) { - this->Pred(inst, dmlc::BeginPtr(*out_preds), gid, - learner_model_param_->base_score); + this->Pred(inst, dmlc::BeginPtr(*out_preds), gid, base_score(0)); } } @@ -184,6 +185,7 @@ class GBLinear : public GradientBooster { contribs.resize(p_fmat->Info().num_row_ * ncolumns * ngroup); // make sure contributions is zeroed, we could be reusing a previously allocated one std::fill(contribs.begin(), contribs.end(), 0); + auto base_score = learner_model_param_->BaseScore(ctx_); // start collecting the contributions for (const auto &batch : p_fmat->GetBatches()) { // parallel over local batch @@ -202,8 +204,8 @@ class GBLinear : public GradientBooster { } // add base margin to BIAS p_contribs[ncolumns - 1] = - model_.Bias()[gid] + ((base_margin.Size() != 0) ? base_margin(row_idx, gid) - : learner_model_param_->base_score); + model_.Bias()[gid] + + ((base_margin.Size() != 0) ? base_margin(row_idx, gid) : base_score(0)); } }); } @@ -268,10 +270,12 @@ class GBLinear : public GradientBooster { monitor_.Start("PredictBatchInternal"); model_.LazyInitModel(); std::vector &preds = *out_preds; - auto base_margin = p_fmat->Info().base_margin_.View(GenericParameter::kCpuId); + auto base_margin = p_fmat->Info().base_margin_.View(Context::kCpuId); // start collecting the prediction const int ngroup = model_.learner_model_param->num_output_group; preds.resize(p_fmat->Info().num_row_ * ngroup); + + auto base_score = learner_model_param_->BaseScore(Context::kCpuId); for (const auto &page : p_fmat->GetBatches()) { auto const& batch = page.GetView(); // output convention: nrow * k, where nrow is number of rows @@ -285,8 +289,7 @@ class GBLinear : public GradientBooster { const size_t ridx = page.base_rowid + i; // loop over output groups for (int gid = 0; gid < ngroup; ++gid) { - float margin = - (base_margin.Size() != 0) ? base_margin(ridx, gid) : learner_model_param_->base_score; + float margin = (base_margin.Size() != 0) ? base_margin(ridx, gid) : base_score(0); this->Pred(batch[i], &preds[ridx * ngroup], gid, margin); } }); diff --git a/src/gbm/gbtree.cc b/src/gbm/gbtree.cc index 9d1d5404409e..a4106888f240 100644 --- a/src/gbm/gbtree.cc +++ b/src/gbm/gbtree.cc @@ -638,13 +638,12 @@ void GPUDartPredictInc(common::Span out_predts, } #endif -void GPUDartInplacePredictInc(common::Span out_predts, - common::Span predts, float tree_w, - size_t n_rows, float base_score, - bst_group_t n_groups, - bst_group_t group) +void GPUDartInplacePredictInc(common::Span /*out_predts*/, common::Span /*predts*/, + float /*tree_w*/, size_t /*n_rows*/, + linalg::TensorView /*base_score*/, + bst_group_t /*n_groups*/, bst_group_t /*group*/) #if defined(XGBOOST_USE_CUDA) -; // NOLINT + ; // NOLINT #else { common::AssertGPUSupport(); @@ -850,15 +849,17 @@ class Dart : public GBTree { size_t n_rows = p_fmat->Info().num_row_; if (predts.predictions.DeviceIdx() != Context::kCpuId) { p_out_preds->predictions.SetDevice(predts.predictions.DeviceIdx()); + auto base_score = model_.learner_model_param->BaseScore(predts.predictions.DeviceIdx()); GPUDartInplacePredictInc(p_out_preds->predictions.DeviceSpan(), - predts.predictions.DeviceSpan(), w, n_rows, - model_.learner_model_param->base_score, n_groups, group); + predts.predictions.DeviceSpan(), w, n_rows, base_score, n_groups, + group); } else { + auto base_score = model_.learner_model_param->BaseScore(Context::kCpuId); auto& h_predts = predts.predictions.HostVector(); auto& h_out_predts = p_out_preds->predictions.HostVector(); common::ParallelFor(n_rows, ctx_->Threads(), [&](auto ridx) { const size_t offset = ridx * n_groups + group; - h_out_predts[offset] += (h_predts[offset] - model_.learner_model_param->base_score) * w; + h_out_predts[offset] += (h_predts[offset] - base_score(0)) * w; }); } } diff --git a/src/gbm/gbtree.cu b/src/gbm/gbtree.cu index 0b81fff23e5c..12109782d59b 100644 --- a/src/gbm/gbtree.cu +++ b/src/gbm/gbtree.cu @@ -31,13 +31,14 @@ void GPUDartPredictInc(common::Span out_predts, }); } -void GPUDartInplacePredictInc(common::Span out_predts, - common::Span predts, float tree_w, - size_t n_rows, float base_score, - bst_group_t n_groups, bst_group_t group) { +void GPUDartInplacePredictInc(common::Span out_predts, common::Span predts, + float tree_w, size_t n_rows, + linalg::TensorView base_score, bst_group_t n_groups, + bst_group_t group) { + CHECK_EQ(base_score.Size(), 1); dh::LaunchN(n_rows, [=] XGBOOST_DEVICE(size_t ridx) { const size_t offset = ridx * n_groups + group; - out_predts[offset] += (predts[offset] - base_score) * tree_w; + out_predts[offset] += (predts[offset] - base_score(0)) * tree_w; }); } } // namespace gbm diff --git a/src/learner.cc b/src/learner.cc index 0dbf3631a499..2ee83fb71cbf 100644 --- a/src/learner.cc +++ b/src/learner.cc @@ -4,47 +4,48 @@ * \brief Implementation of learning algorithm. * \author Tianqi Chen */ +#include "xgboost/learner.h" + +#include #include #include #include -#include -#include #include +#include #include -#include +#include // std::numeric_limits #include +#include #include -#include #include +#include #include #include -#include "dmlc/any.h" +#include "common/charconv.h" +#include "common/common.h" +#include "common/io.h" +#include "common/linalg_op.h" +#include "common/observer.h" +#include "common/random.h" +#include "common/threading_utils.h" +#include "common/timer.h" +#include "common/version.h" #include "xgboost/base.h" #include "xgboost/c_api.h" #include "xgboost/data.h" -#include "xgboost/model.h" -#include "xgboost/predictor.h" #include "xgboost/feature_map.h" #include "xgboost/gbm.h" #include "xgboost/generic_parameters.h" #include "xgboost/host_device_vector.h" #include "xgboost/json.h" -#include "xgboost/learner.h" #include "xgboost/logging.h" #include "xgboost/metric.h" +#include "xgboost/model.h" #include "xgboost/objective.h" #include "xgboost/parameter.h" - -#include "common/common.h" -#include "common/io.h" -#include "common/observer.h" -#include "common/random.h" -#include "common/timer.h" -#include "common/charconv.h" -#include "common/version.h" -#include "common/threading_utils.h" +#include "xgboost/predictor.h" namespace { @@ -85,26 +86,29 @@ struct LearnerModelParamLegacy : public dmlc::Parameter uint32_t minor_version; uint32_t num_target{1}; + + int32_t base_score_estimated{0}; /*! \brief reserved field */ - int reserved[26]; + int reserved[25]; /*! \brief constructor */ LearnerModelParamLegacy() { std::memset(this, 0, sizeof(LearnerModelParamLegacy)); - base_score = 0.5f; + base_score = ObjFunction::DefaultBaseScore(); num_target = 1; major_version = std::get<0>(Version::Self()); minor_version = std::get<1>(Version::Self()); + base_score_estimated = 0; static_assert(sizeof(LearnerModelParamLegacy) == 136, "Do not change the size of this struct, as it will break binary IO."); } + // Skip other legacy fields. Json ToJson() const { Object obj; char floats[NumericLimits::kToCharsSize]; auto ret = to_chars(floats, floats + NumericLimits::kToCharsSize, base_score); - CHECK(ret.ec == std::errc()); - obj["base_score"] = - std::string{floats, static_cast(std::distance(floats, ret.ptr))}; + CHECK(ret.ec == std::errc{}); + obj["base_score"] = std::string{floats, static_cast(std::distance(floats, ret.ptr))}; char integers[NumericLimits::kToCharsSize]; ret = to_chars(integers, integers + NumericLimits::kToCharsSize, @@ -136,10 +140,14 @@ struct LearnerModelParamLegacy : public dmlc::Parameter } this->Init(m); + std::string str = get(j_param.at("base_score")); from_chars(str.c_str(), str.c_str() + str.size(), base_score); + // It can only be estimated during the first training, we consider it estimated afterward + base_score_estimated = 1; } - inline LearnerModelParamLegacy ByteSwap() const { + + LearnerModelParamLegacy ByteSwap() const { LearnerModelParamLegacy x = *this; dmlc::ByteSwap(&x.base_score, sizeof(x.base_score), 1); dmlc::ByteSwap(&x.num_feature, sizeof(x.num_feature), 1); @@ -149,14 +157,30 @@ struct LearnerModelParamLegacy : public dmlc::Parameter dmlc::ByteSwap(&x.major_version, sizeof(x.major_version), 1); dmlc::ByteSwap(&x.minor_version, sizeof(x.minor_version), 1); dmlc::ByteSwap(&x.num_target, sizeof(x.num_target), 1); + dmlc::ByteSwap(&x.base_score_estimated, sizeof(x.base_score_estimated), 1); dmlc::ByteSwap(x.reserved, sizeof(x.reserved[0]), sizeof(x.reserved) / sizeof(x.reserved[0])); return x; } + template + Args UpdateAllowUnknown(Container const& kwargs) { + // Detect whether user has made their own base score. + if (std::find_if(kwargs.cbegin(), kwargs.cend(), + [](auto const& kv) { return kv.first == "base_score"; }) != kwargs.cend()) { + base_score_estimated = true; + } + if (std::find_if(kwargs.cbegin(), kwargs.cend(), [](auto const& kv) { + return kv.first == "base_score_estimated"; + }) != kwargs.cend()) { + LOG(FATAL) << "`base_score_estimated` cannot be specified as hyper-parameter."; + } + return dmlc::Parameter::UpdateAllowUnknown(kwargs); + } + // declare parameters DMLC_DECLARE_PARAMETER(LearnerModelParamLegacy) { DMLC_DECLARE_FIELD(base_score) - .set_default(0.5f) + .set_default(ObjFunction::DefaultBaseScore()) .describe("Global bias of the model."); DMLC_DECLARE_FIELD(num_feature) .set_default(0) @@ -170,12 +194,12 @@ struct LearnerModelParamLegacy : public dmlc::Parameter .set_default(1) .set_lower_bound(1) .describe("Number of target for multi-target regression."); + DMLC_DECLARE_FIELD(base_score_estimated).set_default(0); } }; -LearnerModelParam::LearnerModelParam(LearnerModelParamLegacy const& user_param, float base_margin, - ObjInfo t) - : base_score{base_margin}, num_feature{user_param.num_feature}, task{t} { +LearnerModelParam::LearnerModelParam(LearnerModelParamLegacy const& user_param, ObjInfo t) + : num_feature{user_param.num_feature}, task{t} { auto n_classes = std::max(static_cast(user_param.num_class), 1u); auto n_targets = user_param.num_target; num_output_group = std::max(n_classes, n_targets); @@ -185,6 +209,53 @@ LearnerModelParam::LearnerModelParam(LearnerModelParamLegacy const& user_param, << ", n_targets:" << n_targets; } +LearnerModelParam::LearnerModelParam(Context const* ctx, LearnerModelParamLegacy const& user_param, + linalg::Tensor base_margin, ObjInfo t) + : LearnerModelParam{user_param, t} { + std::swap(base_score_, base_margin); + // Make sure read access everywhere for thread-safe prediction. + common::AsConst(base_score_).HostView(); + if (!ctx->IsCPU()) { + common::AsConst(base_score_).View(ctx->gpu_id); + } + CHECK(common::AsConst(base_score_).Data()->HostCanRead()); +} + +linalg::TensorView LearnerModelParam::BaseScore(int32_t device) const { + // multi-class is not yet supported. + CHECK_EQ(base_score_.Size(), 1); + if (device == Context::kCpuId) { + // Make sure that we won't run into race condition. + CHECK(base_score_.Data()->HostCanRead()); + return base_score_.HostView(); + } + // Make sure that we won't run into race condition. + CHECK(base_score_.Data()->DeviceCanRead()); + auto v = base_score_.View(device); + CHECK(base_score_.Data()->HostCanRead()); // make sure read access is not removed. + return v; +} + +linalg::TensorView LearnerModelParam::BaseScore(Context const* ctx) const { + return this->BaseScore(ctx->gpu_id); +} + +void LearnerModelParam::Copy(LearnerModelParam const& that) { + base_score_.Reshape(that.base_score_.Shape()); + base_score_.Data()->SetDevice(that.base_score_.DeviceIdx()); + base_score_.Data()->Copy(*that.base_score_.Data()); + common::AsConst(base_score_).HostView(); + if (that.base_score_.DeviceIdx() != Context::kCpuId) { + common::AsConst(base_score_).View(that.base_score_.DeviceIdx()); + } + CHECK_EQ(base_score_.Data()->DeviceCanRead(), that.base_score_.Data()->DeviceCanRead()); + CHECK(base_score_.Data()->HostCanRead()); + + num_feature = that.num_feature; + num_output_group = that.num_output_group; + task = that.task; +} + struct LearnerTrainParam : public XGBoostParameter { // data split mode, can be row, col, or none. DataSplitMode dsplit {DataSplitMode::kAuto}; @@ -308,8 +379,61 @@ class LearnerConfiguration : public Learner { LearnerModelParamLegacy mparam_; LearnerModelParam learner_model_param_; LearnerTrainParam tparam_; + // Initial prediction. std::vector metric_names_; + /** + * \brief Calculate the `base_score` based on input data. + * + * \param p_fmat The training DMatrix used to estimate the base score. + */ + void InitBaseScore(DMatrix const* p_fmat) { + // Before 1.0.0, we save `base_score` into binary as a transformed value by objective. + // After 1.0.0 we save the value provided by user and keep it immutable instead. To + // keep the stability, we initialize it in binary LoadModel instead of configuration. + // Under what condition should we omit the transformation: + // + // - base_score is loaded from old binary model. + // + // What are the other possible conditions: + // + // - model loaded from new binary or JSON. + // - model is created from scratch. + // - model is configured second time due to change of parameter + CHECK(obj_); + if (!mparam_.base_score_estimated) { + if (p_fmat) { + // We estimate it from input data. + linalg::Tensor base_score; + obj_->InitEstimation(p_fmat->Info(), &base_score); + mparam_.base_score = base_score(0); + CHECK(!std::isnan(mparam_.base_score)); + } else { + mparam_.base_score = ObjFunction::DefaultBaseScore(); + } + mparam_.base_score_estimated = true; + // Update the shared model parameter + this->ConfigureModelParam(); + } + } + + // Convert mparam to learner_model_param + void ConfigureModelParam() { + this->ConfigureTargets(); + + CHECK(obj_); + auto task = obj_->Task(); + linalg::Tensor base_score({1}, Ctx()->gpu_id); + auto h_base_score = base_score.HostView(); + + // transform to margin + h_base_score(0) = obj_->ProbToMargin(mparam_.base_score); + // move it to model param, which is shared with all other components. + learner_model_param_ = LearnerModelParam(Ctx(), mparam_, std::move(base_score), task); + CHECK(learner_model_param_.Initialized()); + CHECK_NE(learner_model_param_.BaseScore(Ctx()).Size(), 0); + } + public: explicit LearnerConfiguration(std::vector > cache) : need_configuration_{true} { @@ -329,22 +453,24 @@ class LearnerConfiguration : public Learner { // Configuration before data is known. void Configure() override { // Varient of double checked lock - if (!this->need_configuration_) { return; } + if (!this->need_configuration_) { + return; + } std::lock_guard guard(config_lock_); - if (!this->need_configuration_) { return; } + if (!this->need_configuration_) { + return; + } monitor_.Start("Configure"); auto old_tparam = tparam_; Args args = {cfg_.cbegin(), cfg_.cend()}; tparam_.UpdateAllowUnknown(args); - auto mparam_backup = mparam_; - mparam_.UpdateAllowUnknown(args); - auto initialized = generic_parameters_.GetInitialised(); - auto old_seed = generic_parameters_.seed; - generic_parameters_.UpdateAllowUnknown(args); + auto initialized = ctx_.GetInitialised(); + auto old_seed = ctx_.seed; + ctx_.UpdateAllowUnknown(args); ConsoleLogger::Configure(args); @@ -355,8 +481,8 @@ class LearnerConfiguration : public Learner { } // set seed only before the model is initialized - if (!initialized || generic_parameters_.seed != old_seed) { - common::GlobalRandom().seed(generic_parameters_.seed); + if (!initialized || ctx_.seed != old_seed) { + common::GlobalRandom().seed(ctx_.seed); } // must precede configure gbm since num_features is required for gbm @@ -364,31 +490,15 @@ class LearnerConfiguration : public Learner { args = {cfg_.cbegin(), cfg_.cend()}; // renew this->ConfigureObjective(old_tparam, &args); - auto task = this->ConfigureTargets(); - - // Before 1.0.0, we save `base_score` into binary as a transformed value by objective. - // After 1.0.0 we save the value provided by user and keep it immutable instead. To - // keep the stability, we initialize it in binary LoadModel instead of configuration. - // Under what condition should we omit the transformation: - // - // - base_score is loaded from old binary model. - // - // What are the other possible conditions: - // - // - model loaded from new binary or JSON. - // - model is created from scratch. - // - model is configured second time due to change of parameter - if (!learner_model_param_.Initialized() || mparam_.base_score != mparam_backup.base_score) { - learner_model_param_ = - LearnerModelParam(mparam_, obj_->ProbToMargin(mparam_.base_score), task); - } - + learner_model_param_.task = obj_->Task(); // required by gbm configuration. this->ConfigureGBM(old_tparam, args); - generic_parameters_.ConfigureGpuId(this->gbm_->UseGPU()); + ctx_.ConfigureGpuId(this->gbm_->UseGPU()); + this->ConfigureModelParam(); + this->ConfigureMetrics(args); this->need_configuration_ = false; - if (generic_parameters_.validate_parameters) { + if (ctx_.validate_parameters) { this->ValidateParameters(); } @@ -396,6 +506,11 @@ class LearnerConfiguration : public Learner { monitor_.Stop("Configure"); } + void CheckModelInitialized() const { + CHECK(learner_model_param_.Initialized()) << "Model not yet initialized."; + CHECK_NE(learner_model_param_.BaseScore(this->Ctx()).Size(), 0); + } + virtual PredictionContainer* GetPredictionCache() const { return &((*ThreadLocalPredictionCache::Get())[this]); } @@ -417,7 +532,7 @@ class LearnerConfiguration : public Learner { auto const& objective_fn = learner_parameters.at("objective"); if (!obj_) { - obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_)); + obj_.reset(ObjFunction::Create(tparam_.objective, &ctx_)); } obj_->LoadConfig(objective_fn); learner_model_param_.task = obj_->Task(); @@ -425,7 +540,7 @@ class LearnerConfiguration : public Learner { tparam_.booster = get(gradient_booster["name"]); if (!gbm_) { gbm_.reset(GradientBooster::Create(tparam_.booster, - &generic_parameters_, &learner_model_param_)); + &ctx_, &learner_model_param_)); } gbm_->LoadConfig(gradient_booster); @@ -441,15 +556,15 @@ class LearnerConfiguration : public Learner { } else { metric_names_[i] = get(j_metrics[i]["name"]); } - metrics_[i] = std::unique_ptr(Metric::Create(metric_names_[i], &generic_parameters_)); + metrics_[i] = std::unique_ptr(Metric::Create(metric_names_[i], &ctx_)); if (!old_serialization) { metrics_[i]->LoadConfig(j_metrics[i]); } } - FromJson(learner_parameters.at("generic_param"), &generic_parameters_); + FromJson(learner_parameters.at("generic_param"), &ctx_); // make sure the GPU ID is valid in new environment before start running configure. - generic_parameters_.ConfigureGpuId(false); + ctx_.ConfigureGpuId(false); this->need_configuration_ = true; } @@ -478,7 +593,7 @@ class LearnerConfiguration : public Learner { } learner_parameters["metrics"] = Array(std::move(metrics)); - learner_parameters["generic_param"] = ToJson(generic_parameters_); + learner_parameters["generic_param"] = ToJson(ctx_); } void SetParam(const std::string& key, const std::string& value) override { @@ -551,7 +666,7 @@ class LearnerConfiguration : public Learner { return cfg_; } - GenericParameter const* Ctx() const override { return &generic_parameters_; } + Context const* Ctx() const override { return &ctx_; } private: void ValidateParameters() { @@ -654,7 +769,7 @@ class LearnerConfiguration : public Learner { void ConfigureGBM(LearnerTrainParam const& old, Args const& args) { if (gbm_ == nullptr || old.booster != tparam_.booster) { - gbm_.reset(GradientBooster::Create(tparam_.booster, &generic_parameters_, + gbm_.reset(GradientBooster::Create(tparam_.booster, &ctx_, &learner_model_param_)); } gbm_->Configure(args); @@ -678,7 +793,7 @@ class LearnerConfiguration : public Learner { cfg_["max_delta_step"] = kMaxDeltaStepDefaultValue; } if (obj_ == nullptr || tparam_.objective != old.objective) { - obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_)); + obj_.reset(ObjFunction::Create(tparam_.objective, &ctx_)); } auto& args = *p_args; args = {cfg_.cbegin(), cfg_.cend()}; // renew @@ -691,7 +806,7 @@ class LearnerConfiguration : public Learner { return m->Name() != name; }; if (std::all_of(metrics_.begin(), metrics_.end(), DupCheck)) { - metrics_.emplace_back(std::unique_ptr(Metric::Create(name, &generic_parameters_))); + metrics_.emplace_back(std::unique_ptr(Metric::Create(name, &ctx_))); mparam_.contain_eval_metrics = 1; } } @@ -703,7 +818,7 @@ class LearnerConfiguration : public Learner { /** * Get number of targets from objective function. */ - ObjInfo ConfigureTargets() { + void ConfigureTargets() { CHECK(this->obj_); auto const& cache = this->GetPredictionCache()->Container(); size_t n_targets = 1; @@ -722,7 +837,6 @@ class LearnerConfiguration : public Learner { } else { mparam_.num_target = n_targets; } - return this->obj_->Task(); } }; @@ -754,14 +868,14 @@ class LearnerIO : public LearnerConfiguration { std::string name = get(objective_fn["name"]); tparam_.UpdateAllowUnknown(Args{{"objective", name}}); - obj_.reset(ObjFunction::Create(name, &generic_parameters_)); + obj_.reset(ObjFunction::Create(name, &ctx_)); obj_->LoadConfig(objective_fn); auto const& gradient_booster = learner.at("gradient_booster"); name = get(gradient_booster["name"]); tparam_.UpdateAllowUnknown(Args{{"booster", name}}); gbm_.reset( - GradientBooster::Create(tparam_.booster, &generic_parameters_, &learner_model_param_)); + GradientBooster::Create(tparam_.booster, &ctx_, &learner_model_param_)); gbm_->LoadModel(gradient_booster); auto const& j_attributes = get(learner.at("attributes")); @@ -791,6 +905,7 @@ class LearnerIO : public LearnerConfiguration { void SaveModel(Json* p_out) const override { CHECK(!this->need_configuration_) << "Call Configure before saving model."; + this->CheckModelInitialized(); Version::Save(p_out); Json& out { *p_out }; @@ -826,7 +941,7 @@ class LearnerIO : public LearnerConfiguration { // About to be deprecated by JSON format void LoadModel(dmlc::Stream* fi) override { - generic_parameters_.UpdateAllowUnknown(Args{}); + ctx_.UpdateAllowUnknown(Args{}); tparam_.Init(std::vector>{}); // TODO(tqchen) mark deprecation of old format. common::PeekableInStream fp(fi); @@ -881,8 +996,8 @@ class LearnerIO : public LearnerConfiguration { CHECK(fi->Read(&tparam_.objective)) << "BoostLearner: wrong model format"; CHECK(fi->Read(&tparam_.booster)) << "BoostLearner: wrong model format"; - obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_)); - gbm_.reset(GradientBooster::Create(tparam_.booster, &generic_parameters_, + obj_.reset(ObjFunction::Create(tparam_.objective, &ctx_)); + gbm_.reset(GradientBooster::Create(tparam_.booster, &ctx_, &learner_model_param_)); gbm_->Load(fi); if (mparam_.contain_extra_attrs != 0) { @@ -925,7 +1040,14 @@ class LearnerIO : public LearnerConfiguration { } learner_model_param_ = - LearnerModelParam(mparam_, obj_->ProbToMargin(mparam_.base_score), obj_->Task()); + LearnerModelParam(&ctx_, mparam_, + linalg::Tensor{{std::isnan(mparam_.base_score) + ? std::numeric_limits::quiet_NaN() + : obj_->ProbToMargin(mparam_.base_score)}, + {1}, + Context::kCpuId}, + obj_->Task()); + if (attributes_.find("objective") != attributes_.cend()) { auto obj_str = attributes_.at("objective"); auto j_obj = Json::Load({obj_str.c_str(), obj_str.size()}); @@ -969,6 +1091,8 @@ class LearnerIO : public LearnerConfiguration { // Save model into binary format. The code is about to be deprecated by more robust // JSON serialization format. void SaveModel(dmlc::Stream* fo) const override { + this->CheckModelInitialized(); + LearnerModelParamLegacy mparam = mparam_; // make a copy to potentially modify std::vector > extra_attr; mparam.contain_extra_attrs = 1; @@ -1000,6 +1124,7 @@ class LearnerIO : public LearnerConfiguration { } extra_attr.emplace_back("metrics", os.str()); } + std::string header {"binf"}; fo->Write(header.data(), 4); if (DMLC_IO_NO_ENDIAN_SWAP) { @@ -1022,6 +1147,8 @@ class LearnerIO : public LearnerConfiguration { } void Save(dmlc::Stream* fo) const override { + this->CheckModelInitialized(); + Json memory_snapshot{Object()}; memory_snapshot["Model"] = Object(); auto& model = memory_snapshot["Model"]; @@ -1108,28 +1235,30 @@ class LearnerImpl : public LearnerIO { } } - std::vector DumpModel(const FeatureMap& fmap, - bool with_stats, + std::vector DumpModel(const FeatureMap& fmap, bool with_stats, std::string format) override { this->Configure(); + this->CheckModelInitialized(); + return gbm_->DumpModel(fmap, with_stats, format); } - Learner *Slice(int32_t begin_layer, int32_t end_layer, int32_t step, - bool *out_of_bound) override { + Learner* Slice(int32_t begin_layer, int32_t end_layer, int32_t step, + bool* out_of_bound) override { this->Configure(); + this->CheckModelInitialized(); + CHECK_NE(this->learner_model_param_.num_feature, 0); CHECK_GE(begin_layer, 0); - auto *out_impl = new LearnerImpl({}); - out_impl->learner_model_param_ = this->learner_model_param_; - out_impl->generic_parameters_ = this->generic_parameters_; + auto* out_impl = new LearnerImpl({}); + out_impl->learner_model_param_.Copy(this->learner_model_param_); + out_impl->ctx_ = this->ctx_; auto gbm = std::unique_ptr(GradientBooster::Create( - this->tparam_.booster, &out_impl->generic_parameters_, - &out_impl->learner_model_param_)); + this->tparam_.booster, &out_impl->ctx_, &out_impl->learner_model_param_)); this->gbm_->Slice(begin_layer, end_layer, step, gbm.get(), out_of_bound); out_impl->gbm_ = std::move(gbm); - Json config { Object() }; + Json config{Object()}; this->SaveConfig(&config); out_impl->mparam_ = this->mparam_; out_impl->attributes_ = this->attributes_; @@ -1156,15 +1285,17 @@ class LearnerImpl : public LearnerIO { monitor_.Start("UpdateOneIter"); TrainingObserver::Instance().Update(iter); this->Configure(); - if (generic_parameters_.seed_per_iteration) { - common::GlobalRandom().seed(generic_parameters_.seed * kRandSeedMagic + iter); + this->InitBaseScore(train.get()); + + if (ctx_.seed_per_iteration) { + common::GlobalRandom().seed(ctx_.seed * kRandSeedMagic + iter); } this->CheckDataSplitMode(); this->ValidateDMatrix(train.get(), true); auto local_cache = this->GetPredictionCache(); - auto& predt = local_cache->Cache(train, generic_parameters_.gpu_id); + auto& predt = local_cache->Cache(train, ctx_.gpu_id); monitor_.Start("PredictRaw"); this->PredictRaw(train.get(), &predt, true, 0, 0); @@ -1184,14 +1315,18 @@ class LearnerImpl : public LearnerIO { HostDeviceVector* in_gpair) override { monitor_.Start("BoostOneIter"); this->Configure(); - if (generic_parameters_.seed_per_iteration) { - common::GlobalRandom().seed(generic_parameters_.seed * kRandSeedMagic + iter); + // Should have been set to default in the first prediction. + CHECK(mparam_.base_score_estimated); + + if (ctx_.seed_per_iteration) { + common::GlobalRandom().seed(ctx_.seed * kRandSeedMagic + iter); } this->CheckDataSplitMode(); this->ValidateDMatrix(train.get(), true); + auto local_cache = this->GetPredictionCache(); - local_cache->Cache(train, generic_parameters_.gpu_id); + local_cache->Cache(train, ctx_.gpu_id); gbm_->DoBoost(train.get(), in_gpair, &local_cache->Entry(train.get()), obj_.get()); monitor_.Stop("BoostOneIter"); @@ -1202,23 +1337,24 @@ class LearnerImpl : public LearnerIO { const std::vector& data_names) override { monitor_.Start("EvalOneIter"); this->Configure(); + this->CheckModelInitialized(); std::ostringstream os; os.precision(std::numeric_limits::max_digits10); os << '[' << iter << ']' << std::setiosflags(std::ios::fixed); if (metrics_.size() == 0 && tparam_.disable_default_eval_metric <= 0) { - metrics_.emplace_back(Metric::Create(obj_->DefaultEvalMetric(), &generic_parameters_)); + metrics_.emplace_back(Metric::Create(obj_->DefaultEvalMetric(), &ctx_)); metrics_.back()->Configure({cfg_.begin(), cfg_.end()}); } auto local_cache = this->GetPredictionCache(); for (size_t i = 0; i < data_sets.size(); ++i) { std::shared_ptr m = data_sets[i]; - auto &predt = local_cache->Cache(m, generic_parameters_.gpu_id); + auto &predt = local_cache->Cache(m, ctx_.gpu_id); this->ValidateDMatrix(m.get(), false); this->PredictRaw(m.get(), &predt, false, 0, 0); - auto &out = output_predictions_.Cache(m, generic_parameters_.gpu_id).predictions; + auto &out = output_predictions_.Cache(m, ctx_.gpu_id).predictions; out.Resize(predt.predictions.Size()); out.Copy(predt.predictions); @@ -1241,6 +1377,9 @@ class LearnerImpl : public LearnerIO { static_cast(pred_interactions) + static_cast(pred_contribs); this->Configure(); + this->InitBaseScore(nullptr); + this->CheckModelInitialized(); + CHECK_LE(multiple_predictions, 1) << "Perform one kind of prediction at a time."; if (pred_contribs) { gbm_->PredictContribution(data.get(), out_preds, layer_begin, layer_end, approx_contribs); @@ -1251,10 +1390,10 @@ class LearnerImpl : public LearnerIO { gbm_->PredictLeaf(data.get(), out_preds, layer_begin, layer_end); } else { auto local_cache = this->GetPredictionCache(); - auto& prediction = local_cache->Cache(data, generic_parameters_.gpu_id); + auto& prediction = local_cache->Cache(data, ctx_.gpu_id); this->PredictRaw(data.get(), &prediction, training, layer_begin, layer_end); // Copy the prediction cache to output prediction. out_preds comes from C API - out_preds->SetDevice(generic_parameters_.gpu_id); + out_preds->SetDevice(ctx_.gpu_id); out_preds->Resize(prediction.predictions.Size()); out_preds->Copy(prediction.predictions); if (!output_margin) { @@ -1268,8 +1407,10 @@ class LearnerImpl : public LearnerIO { CHECK(!this->need_configuration_); return this->gbm_->BoostedRounds(); } + uint32_t Groups() const override { CHECK(!this->need_configuration_); + this->CheckModelInitialized(); return this->learner_model_param_.num_output_group; } @@ -1281,6 +1422,9 @@ class LearnerImpl : public LearnerIO { HostDeviceVector** out_preds, uint32_t iteration_begin, uint32_t iteration_end) override { this->Configure(); + this->InitBaseScore(nullptr); + this->CheckModelInitialized(); + auto& out_predictions = this->GetThreadLocal().prediction_entry; this->gbm_->InplacePredict(p_m, missing, &out_predictions, iteration_begin, iteration_end); if (type == PredictionType::kValue) { @@ -1296,6 +1440,8 @@ class LearnerImpl : public LearnerIO { void CalcFeatureScore(std::string const& importance_type, common::Span trees, std::vector* features, std::vector* scores) override { this->Configure(); + this->CheckModelInitialized(); + gbm_->FeatureScore(importance_type, trees, features, scores); } @@ -1315,17 +1461,17 @@ class LearnerImpl : public LearnerIO { void PredictRaw(DMatrix *data, PredictionCacheEntry *out_preds, bool training, unsigned layer_begin, unsigned layer_end) const { CHECK(gbm_ != nullptr) << "Predict must happen after Load or configuration"; + this->CheckModelInitialized(); this->ValidateDMatrix(data, false); gbm_->PredictBatch(data, out_preds, training, layer_begin, layer_end); } void ValidateDMatrix(DMatrix* p_fmat, bool is_training) const { MetaInfo const& info = p_fmat->Info(); - info.Validate(generic_parameters_.gpu_id); + info.Validate(ctx_.gpu_id); auto const row_based_split = [this]() { - return tparam_.dsplit == DataSplitMode::kRow || - tparam_.dsplit == DataSplitMode::kAuto; + return tparam_.dsplit == DataSplitMode::kRow || tparam_.dsplit == DataSplitMode::kAuto; }; if (row_based_split()) { if (is_training) { diff --git a/src/objective/adaptive.h b/src/objective/adaptive.h index 85c041347cb9..00d27a57afef 100644 --- a/src/objective/adaptive.h +++ b/src/objective/adaptive.h @@ -7,6 +7,7 @@ #include #include +#include "../common/common.h" #include "rabit/rabit.h" #include "xgboost/generic_parameters.h" #include "xgboost/host_device_vector.h" diff --git a/src/objective/objective.cc b/src/objective/objective.cc index 5991e918d315..5ba5f87fb8c5 100644 --- a/src/objective/objective.cc +++ b/src/objective/objective.cc @@ -1,10 +1,10 @@ /*! - * Copyright 2015 by Contributors + * Copyright 2015-2022 by Contributors * \file objective.cc * \brief Registry of all objective functions. */ -#include #include +#include #include @@ -31,6 +31,11 @@ ObjFunction* ObjFunction::Create(const std::string& name, GenericParameter const return pobj; } +void ObjFunction::InitEstimation(MetaInfo const&, linalg::Tensor* base_score) const { + CHECK(base_score); + base_score->Reshape(1); + (*base_score)(0) = DefaultBaseScore(); +} } // namespace xgboost namespace xgboost { diff --git a/src/objective/regression_obj.cu b/src/objective/regression_obj.cu index ecd906f699a4..6b2ce6371a6d 100644 --- a/src/objective/regression_obj.cu +++ b/src/objective/regression_obj.cu @@ -15,7 +15,9 @@ #include "../common/common.h" #include "../common/linalg_op.h" +#include "../common/numeric.h" // Reduce #include "../common/pseudo_huber.h" +#include "../common/stats.h" #include "../common/threading_utils.h" #include "../common/transform.h" #include "./regression_loss.h" @@ -37,14 +39,18 @@ namespace xgboost { namespace obj { namespace { -void CheckRegInputs(MetaInfo const& info, HostDeviceVector const& preds) { +void CheckInitInputs(MetaInfo const& info) { CHECK_EQ(info.labels.Shape(0), info.num_row_) << "Invalid shape of labels."; - CHECK_EQ(info.labels.Size(), preds.Size()) << "Invalid shape of labels."; if (!info.weights_.Empty()) { CHECK_EQ(info.weights_.Size(), info.num_row_) << "Number of weights should be equal to number of data points."; } } + +void CheckRegInputs(MetaInfo const& info, HostDeviceVector const& preds) { + CheckInitInputs(info); + CHECK_EQ(info.labels.Size(), preds.Size()) << "Invalid shape of labels."; +} } // anonymous namespace #if defined(XGBOOST_USE_CUDA) @@ -698,6 +704,33 @@ class MeanAbsoluteError : public ObjFunction { }); } + void InitEstimation(MetaInfo const& info, linalg::Tensor* base_margin) const override { + CheckInitInputs(info); + base_margin->Reshape(1); + auto out = base_margin->HostView(); + + double w{0.0}; + if (info.weights_.Empty()) { + w = static_cast(info.num_row_); + } else { + w = common::Reduce(ctx_, info.weights_); + } + + if (info.num_row_ == 0) { + out(0) = 0; + } else { + // weighted avg + out(0) = common::Median(ctx_, info.labels, info.weights_) * w; + } + + // Weighted average base score across all workers + rabit::Allreduce(out.Values().data(), out.Values().size()); + rabit::Allreduce(&w, 1); + + std::transform(linalg::cbegin(out), linalg::cend(out), linalg::begin(out), + [w](float v) { return v / w; }); + } + void UpdateTreeLeaf(HostDeviceVector const& position, MetaInfo const& info, HostDeviceVector const& prediction, RegTree* p_tree) const override { if (ctx_->IsCPU()) { diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc index 0e213b281231..444d1b089d21 100644 --- a/src/predictor/cpu_predictor.cc +++ b/src/predictor/cpu_predictor.cc @@ -429,11 +429,12 @@ class CPUPredictor : public Predictor { } out_preds->resize(model.learner_model_param->num_output_group * (model.param.size_leaf_vector + 1)); + auto base_score = model.learner_model_param->BaseScore(ctx_)(0); // loop over output groups for (uint32_t gid = 0; gid < model.learner_model_param->num_output_group; ++gid) { - (*out_preds)[gid] = PredValue(inst, model.trees, model.tree_info, gid, - &feat_vecs[0], 0, ntree_limit) + - model.learner_model_param->base_score; + (*out_preds)[gid] = + PredValue(inst, model.trees, model.tree_info, gid, &feat_vecs[0], 0, ntree_limit) + + base_score; } } @@ -504,7 +505,8 @@ class CPUPredictor : public Predictor { common::ParallelFor(ntree_limit, n_threads, [&](bst_omp_uint i) { FillNodeMeanValues(model.trees[i].get(), &(mean_values[i])); }); - auto base_margin = info.base_margin_.View(GenericParameter::kCpuId); + auto base_margin = info.base_margin_.View(Context::kCpuId); + auto base_score = model.learner_model_param->BaseScore(Context::kCpuId)(0); // start collecting the contributions for (const auto &batch : p_fmat->GetBatches()) { auto page = batch.GetView(); @@ -548,7 +550,7 @@ class CPUPredictor : public Predictor { CHECK_EQ(base_margin.Shape(1), ngroup); p_contribs[ncolumns - 1] += base_margin(row_idx, gid); } else { - p_contribs[ncolumns - 1] += model.learner_model_param->base_score; + p_contribs[ncolumns - 1] += base_score; } } }); diff --git a/src/predictor/gpu_predictor.cu b/src/predictor/gpu_predictor.cu index 163f7b40f368..2716883303b5 100644 --- a/src/predictor/gpu_predictor.cu +++ b/src/predictor/gpu_predictor.cu @@ -511,7 +511,7 @@ void ExtractPaths( n = d_nodes[n.Parent() + tree_offset]; path_length++; } - return PathInfo{int64_t(idx), path_length, tree_idx}; + return PathInfo{static_cast(idx), path_length, tree_idx}; }); auto end = thrust::copy_if( thrust::cuda::par(alloc), nodes_transform, @@ -859,13 +859,13 @@ class GPUPredictor : public xgboost::Predictor { // Add the base margin term to last column p_fmat->Info().base_margin_.SetDevice(ctx_->gpu_id); const auto margin = p_fmat->Info().base_margin_.Data()->ConstDeviceSpan(); - float base_score = model.learner_model_param->base_score; - dh::LaunchN( - p_fmat->Info().num_row_ * model.learner_model_param->num_output_group, - [=] __device__(size_t idx) { - phis[(idx + 1) * contributions_columns - 1] += - margin.empty() ? base_score : margin[idx]; - }); + + auto base_score = model.learner_model_param->BaseScore(ctx_); + dh::LaunchN(p_fmat->Info().num_row_ * model.learner_model_param->num_output_group, + [=] __device__(size_t idx) { + phis[(idx + 1) * contributions_columns - 1] += + margin.empty() ? base_score(0) : margin[idx]; + }); } void PredictInteractionContributions(DMatrix* p_fmat, @@ -918,17 +918,17 @@ class GPUPredictor : public xgboost::Predictor { // Add the base margin term to last column p_fmat->Info().base_margin_.SetDevice(ctx_->gpu_id); const auto margin = p_fmat->Info().base_margin_.Data()->ConstDeviceSpan(); - float base_score = model.learner_model_param->base_score; + + auto base_score = model.learner_model_param->BaseScore(ctx_); size_t n_features = model.learner_model_param->num_feature; - dh::LaunchN( - p_fmat->Info().num_row_ * model.learner_model_param->num_output_group, - [=] __device__(size_t idx) { - size_t group = idx % ngroup; - size_t row_idx = idx / ngroup; - phis[gpu_treeshap::IndexPhiInteractions( - row_idx, ngroup, group, n_features, n_features, n_features)] += - margin.empty() ? base_score : margin[idx]; - }); + dh::LaunchN(p_fmat->Info().num_row_ * model.learner_model_param->num_output_group, + [=] __device__(size_t idx) { + size_t group = idx % ngroup; + size_t row_idx = idx / ngroup; + phis[gpu_treeshap::IndexPhiInteractions(row_idx, ngroup, group, n_features, + n_features, n_features)] += + margin.empty() ? base_score(0) : margin[idx]; + }); } void PredictInstance(const SparsePage::Inst&, diff --git a/src/predictor/predictor.cc b/src/predictor/predictor.cc index 10d006a832d0..5701ed892f23 100644 --- a/src/predictor/predictor.cc +++ b/src/predictor/predictor.cc @@ -80,14 +80,15 @@ void Predictor::InitOutPredictions(const MetaInfo& info, HostDeviceVectorgpu_id >= 0) { out_preds->SetDevice(ctx_->gpu_id); } - if (base_margin->Size() != 0) { + if (!base_margin->Empty()) { out_preds->Resize(n); ValidateBaseMarginShape(info.base_margin_, info.num_row_, n_classes); out_preds->Copy(*base_margin); } else { - out_preds->Resize(n); // cannot rely on the Resize to fill as it might skip if the size is already correct. - out_preds->Fill(model.learner_model_param->base_score); + out_preds->Resize(n); + auto base_score = model.learner_model_param->BaseScore(Context::kCpuId)(0); + out_preds->Fill(base_score); } } } // namespace xgboost diff --git a/tests/cpp/common/test_numeric.cc b/tests/cpp/common/test_numeric.cc index 5b672585031b..2a91d2d72a78 100644 --- a/tests/cpp/common/test_numeric.cc +++ b/tests/cpp/common/test_numeric.cc @@ -29,5 +29,15 @@ TEST(Numeric, PartialSum) { ASSERT_EQ(sol, result); } } + +TEST(Numeric, Reduce) { + Context ctx; + ASSERT_TRUE(ctx.IsCPU()); + HostDeviceVector values(20); + auto& h_values = values.HostVector(); + std::iota(h_values.begin(), h_values.end(), 0.0f); + auto sum = Reduce(&ctx, values); + ASSERT_EQ(sum, (values.Size() - 1) * values.Size() / 2); +} } // namespace common } // namespace xgboost diff --git a/tests/cpp/common/test_stats.cc b/tests/cpp/common/test_stats.cc index 2a1e375c0f20..79f38ae6a984 100644 --- a/tests/cpp/common/test_stats.cc +++ b/tests/cpp/common/test_stats.cc @@ -54,5 +54,20 @@ TEST(Stats, WeightedQuantile) { q = WeightedQuantile(1.0, beg, end, w); ASSERT_EQ(q, 5); } + +TEST(Stats, Median) { + linalg::Tensor values{{.0f, .0f, 1.f, 2.f}, {4}, Context::kCpuId}; + Context ctx; + HostDeviceVector weights; + auto m = Median(&ctx, values, weights); + ASSERT_EQ(m, .5f); + +#if defined(XGBOOST_USE_CUDA) + ctx.gpu_id = 0; + ASSERT_FALSE(ctx.IsCPU()); + m = Median(&ctx, values, weights); + ASSERT_EQ(m, .5f); +#endif // defined(XGBOOST_USE_CUDA) +} } // namespace common } // namespace xgboost diff --git a/tests/cpp/gbm/test_gblinear.cc b/tests/cpp/gbm/test_gblinear.cc index 61d22f5ea1ff..c53bb08f68ef 100644 --- a/tests/cpp/gbm/test_gblinear.cc +++ b/tests/cpp/gbm/test_gblinear.cc @@ -19,15 +19,11 @@ namespace gbm { TEST(GBLinear, JsonIO) { size_t constexpr kRows = 16, kCols = 16; - LearnerModelParam param; - param.num_feature = kCols; - param.num_output_group = 1; + Context ctx; + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; - GenericParameter gparam; - gparam.Init(Args{}); - - std::unique_ptr gbm { - CreateTrainedGBM("gblinear", Args{}, kRows, kCols, ¶m, &gparam) }; + std::unique_ptr gbm{ + CreateTrainedGBM("gblinear", Args{}, kRows, kCols, &mparam, &ctx)}; Json model { Object() }; gbm->SaveModel(&model); ASSERT_TRUE(IsA(model)); diff --git a/tests/cpp/gbm/test_gbtree.cc b/tests/cpp/gbm/test_gbtree.cc index a5c16f7951d7..13ec23c14906 100644 --- a/tests/cpp/gbm/test_gbtree.cc +++ b/tests/cpp/gbm/test_gbtree.cc @@ -18,15 +18,11 @@ namespace xgboost { TEST(GBTree, SelectTreeMethod) { size_t constexpr kCols = 10; - GenericParameter generic_param; - generic_param.UpdateAllowUnknown(Args{}); - LearnerModelParam mparam; - mparam.base_score = 0.5; - mparam.num_feature = kCols; - mparam.num_output_group = 1; + Context ctx; + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; std::unique_ptr p_gbm { - GradientBooster::Create("gbtree", &generic_param, &mparam)}; + GradientBooster::Create("gbtree", &ctx, &mparam)}; auto& gbtree = dynamic_cast (*p_gbm); // Test if `tree_method` can be set @@ -45,7 +41,7 @@ TEST(GBTree, SelectTreeMethod) { ASSERT_EQ(tparam.updater_seq, "grow_quantile_histmaker"); #ifdef XGBOOST_USE_CUDA - generic_param.UpdateAllowUnknown(Args{{"gpu_id", "0"}}); + ctx.UpdateAllowUnknown(Args{{"gpu_id", "0"}}); gbtree.Configure({{"tree_method", "gpu_hist"}}); ASSERT_EQ(tparam.updater_seq, "grow_gpu_hist"); gbtree.Configure({{"booster", "dart"}, {"tree_method", "gpu_hist"}}); @@ -55,15 +51,11 @@ TEST(GBTree, SelectTreeMethod) { TEST(GBTree, PredictionCache) { size_t constexpr kRows = 100, kCols = 10; - GenericParameter generic_param; - generic_param.UpdateAllowUnknown(Args{}); - LearnerModelParam mparam; - mparam.base_score = 0.5; - mparam.num_feature = kCols; - mparam.num_output_group = 1; + Context ctx; + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; std::unique_ptr p_gbm { - GradientBooster::Create("gbtree", &generic_param, &mparam)}; + GradientBooster::Create("gbtree", &ctx, &mparam)}; auto& gbtree = dynamic_cast (*p_gbm); gbtree.Configure({{"tree_method", "hist"}}); @@ -176,16 +168,11 @@ TEST(GBTree, ChoosePredictor) { TEST(GBTree, JsonIO) { size_t constexpr kRows = 16, kCols = 16; - LearnerModelParam mparam; - mparam.num_feature = kCols; - mparam.num_output_group = 1; - mparam.base_score = 0.5; - - GenericParameter gparam; - gparam.Init(Args{}); + Context ctx; + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; std::unique_ptr gbm { - CreateTrainedGBM("gbtree", Args{}, kRows, kCols, &mparam, &gparam) }; + CreateTrainedGBM("gbtree", Args{}, kRows, kCols, &mparam, &ctx) }; Json model {Object()}; model["model"] = Object(); @@ -215,16 +202,11 @@ TEST(GBTree, JsonIO) { TEST(Dart, JsonIO) { size_t constexpr kRows = 16, kCols = 16; - LearnerModelParam mparam; - mparam.num_feature = kCols; - mparam.base_score = 0.5; - mparam.num_output_group = 1; - - GenericParameter gparam; - gparam.Init(Args{}); + Context ctx; + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; - std::unique_ptr gbm { - CreateTrainedGBM("dart", Args{}, kRows, kCols, &mparam, &gparam) }; + std::unique_ptr gbm{ + CreateTrainedGBM("dart", Args{}, kRows, kCols, &mparam, &ctx)}; Json model {Object()}; model["model"] = Object(); diff --git a/tests/cpp/helpers.h b/tests/cpp/helpers.h index b79ea27187f5..c7f73495c49f 100644 --- a/tests/cpp/helpers.h +++ b/tests/cpp/helpers.h @@ -451,5 +451,16 @@ class RMMAllocator; using RMMAllocatorPtr = std::unique_ptr; RMMAllocatorPtr SetUpRMMResourceForCppTests(int argc, char** argv); +/* + * \brief Make learner model param + */ +inline LearnerModelParam MakeMP(bst_feature_t n_features, float base_score, uint32_t n_groups, + int32_t device = Context::kCpuId) { + size_t shape[1]{1}; + LearnerModelParam mparam(n_features, linalg::Tensor{{base_score}, shape, device}, + n_groups); + return mparam; +} + } // namespace xgboost #endif diff --git a/tests/cpp/linear/test_linear.cc b/tests/cpp/linear/test_linear.cc index f021641a2b23..779c20940598 100644 --- a/tests/cpp/linear/test_linear.cc +++ b/tests/cpp/linear/test_linear.cc @@ -18,10 +18,7 @@ TEST(Linear, Shotgun) { auto p_fmat = xgboost::RandomDataGenerator(kRows, kCols, 0).GenerateDMatrix(); auto lparam = xgboost::CreateEmptyGenericParam(GPUIDX); - LearnerModelParam mparam; - mparam.num_feature = kCols; - mparam.num_output_group = 1; - mparam.base_score = 0.5; + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; { auto updater = std::unique_ptr( @@ -54,10 +51,7 @@ TEST(Linear, coordinate) { auto p_fmat = xgboost::RandomDataGenerator(kRows, kCols, 0).GenerateDMatrix(); auto lparam = xgboost::CreateEmptyGenericParam(GPUIDX); - LearnerModelParam mparam; - mparam.num_feature = kCols; - mparam.num_output_group = 1; - mparam.base_score = 0.5; + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; auto updater = std::unique_ptr( xgboost::LinearUpdater::Create("coord_descent", &lparam)); diff --git a/tests/cpp/linear/test_linear.cu b/tests/cpp/linear/test_linear.cu index c2eea45d166c..193e9b4b21eb 100644 --- a/tests/cpp/linear/test_linear.cu +++ b/tests/cpp/linear/test_linear.cu @@ -13,15 +13,11 @@ TEST(Linear, GPUCoordinate) { size_t constexpr kCols = 10; auto mat = xgboost::RandomDataGenerator(kRows, kCols, 0).GenerateDMatrix(); - auto lparam = CreateEmptyGenericParam(GPUIDX); - - LearnerModelParam mparam; - mparam.num_feature = kCols; - mparam.num_output_group = 1; - mparam.base_score = 0.5; + auto ctx = CreateEmptyGenericParam(GPUIDX); + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; auto updater = std::unique_ptr( - xgboost::LinearUpdater::Create("gpu_coord_descent", &lparam)); + xgboost::LinearUpdater::Create("gpu_coord_descent", &ctx)); updater->Configure({{"eta", "1."}}); xgboost::HostDeviceVector gpair( mat->Info().num_row_, xgboost::GradientPair(-5, 1.0)); @@ -36,4 +32,4 @@ TEST(Linear, GPUCoordinate) { TEST(GPUCoordinate, JsonIO) { TestUpdaterJsonIO("gpu_coord_descent"); } -} // namespace xgboost \ No newline at end of file +} // namespace xgboost diff --git a/tests/cpp/predictor/test_cpu_predictor.cc b/tests/cpp/predictor/test_cpu_predictor.cc index 8ba270083c74..8db605be3bcc 100644 --- a/tests/cpp/predictor/test_cpu_predictor.cc +++ b/tests/cpp/predictor/test_cpu_predictor.cc @@ -21,14 +21,11 @@ TEST(CpuPredictor, Basic) { size_t constexpr kRows = 5; size_t constexpr kCols = 5; - LearnerModelParam param; - param.num_feature = kCols; - param.base_score = 0.0; - param.num_output_group = 1; + LearnerModelParam mparam{MakeMP(kCols, .0, 1)}; GenericParameter ctx; ctx.UpdateAllowUnknown(Args{}); - gbm::GBTreeModel model = CreateTestModel(¶m, &ctx); + gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx); auto dmat = RandomDataGenerator(kRows, kCols, 0).GenerateDMatrix(); @@ -104,14 +101,11 @@ TEST(CpuPredictor, ExternalMemory) { std::unique_ptr cpu_predictor = std::unique_ptr(Predictor::Create("cpu_predictor", &lparam)); - LearnerModelParam param; - param.base_score = 0; - param.num_feature = dmat->Info().num_col_; - param.num_output_group = 1; + LearnerModelParam mparam{MakeMP(dmat->Info().num_col_, .0, 1)}; GenericParameter ctx; ctx.UpdateAllowUnknown(Args{}); - gbm::GBTreeModel model = CreateTestModel(¶m, &ctx); + gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx); // Test predict batch PredictionCacheEntry out_predictions; @@ -201,16 +195,11 @@ TEST(CpuPredictor, InplacePredict) { void TestUpdatePredictionCache(bool use_subsampling) { size_t constexpr kRows = 64, kCols = 16, kClasses = 4; - LearnerModelParam mparam; - mparam.num_feature = kCols; - mparam.num_output_group = kClasses; - mparam.base_score = 0; - - GenericParameter gparam; - gparam.Init(Args{}); + LearnerModelParam mparam{MakeMP(kCols, .0, kClasses)}; + Context ctx; std::unique_ptr gbm; - gbm.reset(static_cast(GradientBooster::Create("gbtree", &gparam, &mparam))); + gbm.reset(static_cast(GradientBooster::Create("gbtree", &ctx, &mparam))); std::map cfg; cfg["tree_method"] = "hist"; cfg["predictor"] = "cpu_predictor"; diff --git a/tests/cpp/predictor/test_gpu_predictor.cu b/tests/cpp/predictor/test_gpu_predictor.cu index 8dacadac5403..2a0b69cbd629 100644 --- a/tests/cpp/predictor/test_gpu_predictor.cu +++ b/tests/cpp/predictor/test_gpu_predictor.cu @@ -1,5 +1,5 @@ /*! - * Copyright 2017-2020 XGBoost contributors + * Copyright 2017-2022 XGBoost contributors */ #include #include @@ -34,14 +34,10 @@ TEST(GPUPredictor, Basic) { int n_row = i, n_col = i; auto dmat = RandomDataGenerator(n_row, n_col, 0).GenerateDMatrix(); - LearnerModelParam param; - param.num_feature = n_col; - param.num_output_group = 1; - param.base_score = 0.5; - - GenericParameter ctx; - ctx.UpdateAllowUnknown(Args{}); - gbm::GBTreeModel model = CreateTestModel(¶m, &ctx); + Context ctx; + ctx.gpu_id = 0; + LearnerModelParam mparam{MakeMP(n_col, .5, 1, ctx.gpu_id)}; + gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx); // Test predict batch PredictionCacheEntry gpu_out_predictions; @@ -93,15 +89,12 @@ TEST(GPUPredictor, ExternalMemoryTest) { std::unique_ptr(Predictor::Create("gpu_predictor", &lparam)); gpu_predictor->Configure({}); - LearnerModelParam param; - param.num_feature = 5; const int n_classes = 3; - param.num_output_group = n_classes; - param.base_score = 0.5; + Context ctx; + ctx.gpu_id = 0; + LearnerModelParam mparam{MakeMP(5, .5, n_classes, ctx.gpu_id)}; - GenericParameter ctx; - ctx.UpdateAllowUnknown(Args{}); - gbm::GBTreeModel model = CreateTestModel(¶m, &ctx, n_classes); + gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx, n_classes); std::vector> dmats; dmats.push_back(CreateSparsePageDMatrix(400)); @@ -171,15 +164,10 @@ TEST(GpuPredictor, LesserFeatures) { TEST(GPUPredictor, ShapStump) { cudaSetDevice(0); - LearnerModelParam param; - param.num_feature = 1; - param.num_output_group = 1; - param.base_score = 0.5; - - GenericParameter ctx; - ctx.UpdateAllowUnknown(Args{}); - - gbm::GBTreeModel model(¶m, &ctx); + Context ctx; + ctx.gpu_id = 0; + LearnerModelParam mparam{MakeMP(1, .5, 1, ctx.gpu_id)}; + gbm::GBTreeModel model(&mparam, &ctx); std::vector> trees; trees.push_back(std::unique_ptr(new RegTree)); @@ -193,24 +181,20 @@ TEST(GPUPredictor, ShapStump) { auto dmat = RandomDataGenerator(3, 1, 0).GenerateDMatrix(); gpu_predictor->PredictContribution(dmat.get(), &predictions, model); auto& phis = predictions.HostVector(); + auto base_score = mparam.BaseScore(Context::kCpuId)(0); EXPECT_EQ(phis[0], 0.0); - EXPECT_EQ(phis[1], param.base_score); + EXPECT_EQ(phis[1], base_score); EXPECT_EQ(phis[2], 0.0); - EXPECT_EQ(phis[3], param.base_score); + EXPECT_EQ(phis[3], base_score); EXPECT_EQ(phis[4], 0.0); - EXPECT_EQ(phis[5], param.base_score); + EXPECT_EQ(phis[5], base_score); } TEST(GPUPredictor, Shap) { - LearnerModelParam param; - param.num_feature = 1; - param.num_output_group = 1; - param.base_score = 0.5; - - GenericParameter ctx; - ctx.UpdateAllowUnknown(Args{}); - - gbm::GBTreeModel model(¶m, &ctx); + Context ctx; + ctx.gpu_id = 0; + LearnerModelParam mparam{MakeMP(1, .5, 1, ctx.gpu_id)}; + gbm::GBTreeModel model(&mparam, &ctx); std::vector> trees; trees.push_back(std::unique_ptr(new RegTree)); @@ -258,14 +242,9 @@ TEST(GPUPredictor, PredictLeafBasic) { std::unique_ptr(Predictor::Create("gpu_predictor", &lparam)); gpu_predictor->Configure({}); - LearnerModelParam param; - param.num_feature = kCols; - param.base_score = 0.0; - param.num_output_group = 1; - - GenericParameter ctx; - ctx.UpdateAllowUnknown(Args{}); - gbm::GBTreeModel model = CreateTestModel(¶m, &ctx); + LearnerModelParam mparam{MakeMP(kCols, .0, 1)}; + Context ctx; + gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx); HostDeviceVector leaf_out_predictions; gpu_predictor->PredictLeaf(dmat.get(), &leaf_out_predictions, model); diff --git a/tests/cpp/predictor/test_predictor.cc b/tests/cpp/predictor/test_predictor.cc index 34c4d48e6dc1..64d2b9a81ea2 100644 --- a/tests/cpp/predictor/test_predictor.cc +++ b/tests/cpp/predictor/test_predictor.cc @@ -210,11 +210,7 @@ void TestCategoricalPrediction(std::string name) { size_t constexpr kCols = 10; PredictionCacheEntry out_predictions; - LearnerModelParam param; - param.num_feature = kCols; - param.num_output_group = 1; - param.base_score = 0.5; - + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; uint32_t split_ind = 3; bst_cat_t split_cat = 4; float left_weight = 1.3f; @@ -222,7 +218,7 @@ void TestCategoricalPrediction(std::string name) { GenericParameter ctx; ctx.UpdateAllowUnknown(Args{}); - gbm::GBTreeModel model(¶m, &ctx); + gbm::GBTreeModel model(&mparam, &ctx); GBTreeModelForTest(&model, split_ind, split_cat, left_weight, right_weight); ctx.UpdateAllowUnknown(Args{{"gpu_id", "0"}}); @@ -237,27 +233,24 @@ void TestCategoricalPrediction(std::string name) { predictor->InitOutPredictions(m->Info(), &out_predictions.predictions, model); predictor->PredictBatch(m.get(), &out_predictions, model, 0); + auto score = mparam.BaseScore(Context::kCpuId)(0); ASSERT_EQ(out_predictions.predictions.Size(), 1ul); ASSERT_EQ(out_predictions.predictions.HostVector()[0], - right_weight + param.base_score); // go to right for matching cat + right_weight + score); // go to right for matching cat row[split_ind] = split_cat + 1; m = GetDMatrixFromData(row, 1, kCols); out_predictions.version = 0; predictor->InitOutPredictions(m->Info(), &out_predictions.predictions, model); predictor->PredictBatch(m.get(), &out_predictions, model, 0); - ASSERT_EQ(out_predictions.predictions.HostVector()[0], - left_weight + param.base_score); + ASSERT_EQ(out_predictions.predictions.HostVector()[0], left_weight + score); } void TestCategoricalPredictLeaf(StringView name) { size_t constexpr kCols = 10; PredictionCacheEntry out_predictions; - LearnerModelParam param; - param.num_feature = kCols; - param.num_output_group = 1; - param.base_score = 0.5; + LearnerModelParam mparam{MakeMP(kCols, .5, 1)}; uint32_t split_ind = 3; bst_cat_t split_cat = 4; @@ -267,7 +260,7 @@ void TestCategoricalPredictLeaf(StringView name) { GenericParameter ctx; ctx.UpdateAllowUnknown(Args{}); - gbm::GBTreeModel model(¶m, &ctx); + gbm::GBTreeModel model(&mparam, &ctx); GBTreeModelForTest(&model, split_ind, split_cat, left_weight, right_weight); ctx.gpu_id = 0; diff --git a/tests/cpp/predictor/test_predictor.h b/tests/cpp/predictor/test_predictor.h index 1ff96096c533..81ee249e250e 100644 --- a/tests/cpp/predictor/test_predictor.h +++ b/tests/cpp/predictor/test_predictor.h @@ -12,11 +12,7 @@ void TestPredictionFromGradientIndex(std::string name, size_t rows, size_t cols, std::shared_ptr p_hist) { constexpr size_t kClasses { 3 }; - LearnerModelParam param; - param.num_feature = cols; - param.num_output_group = kClasses; - param.base_score = 0.5; - + LearnerModelParam mparam{MakeMP(cols, .5, kClasses)}; auto lparam = CreateEmptyGenericParam(0); std::unique_ptr predictor = @@ -25,7 +21,7 @@ void TestPredictionFromGradientIndex(std::string name, size_t rows, size_t cols, GenericParameter ctx; ctx.UpdateAllowUnknown(Args{}); - gbm::GBTreeModel model = CreateTestModel(¶m, &ctx, kClasses); + gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx, kClasses); { auto p_precise = RandomDataGenerator(rows, cols, 0).GenerateDMatrix(); diff --git a/tests/cpp/test_learner.cc b/tests/cpp/test_learner.cc index 4a8214e9c5cd..49c1d9537426 100644 --- a/tests/cpp/test_learner.cc +++ b/tests/cpp/test_learner.cc @@ -3,8 +3,10 @@ */ #include #include +#include // ObjFunction #include +#include // std::stof, std::string #include #include @@ -206,8 +208,7 @@ TEST(Learner, MultiThreadedPredict) { p_dmat->Info().labels.Reshape(kRows); CHECK_NE(p_dmat->Info().num_col_, 0); - std::shared_ptr p_data{ - RandomDataGenerator{kRows, kCols, 0}.GenerateDMatrix()}; + std::shared_ptr p_data{RandomDataGenerator{kRows, kCols, 0}.GenerateDMatrix()}; CHECK_NE(p_data->Info().num_col_, 0); std::shared_ptr learner{Learner::Create({p_dmat})}; @@ -448,4 +449,77 @@ TEST(Learner, MultiTarget) { EXPECT_THROW({ learner->Configure(); }, dmlc::Error); } } + +/** + * Test the model initialization sequence is correctly performed. + */ +TEST(Learner, InitEstimation) { + size_t constexpr kCols = 10; + auto Xy = RandomDataGenerator{10, kCols, 0}.GenerateDMatrix(true); + + { + std::unique_ptr learner{Learner::Create({Xy})}; + learner->SetParam("objective", "reg:absoluteerror"); + learner->Configure(); + HostDeviceVector predt; + learner->Predict(Xy, false, &predt, 0, 0); + + auto h_predt = predt.ConstHostSpan(); + for (auto v : h_predt) { + ASSERT_EQ(v, ObjFunction::DefaultBaseScore()); + } + Json config{Object{}}; + learner->SaveConfig(&config); + auto base_score = + std::stof(get(config["learner"]["learner_model_param"]["base_score"])); + // No base score is estimated yet. + ASSERT_EQ(base_score, ObjFunction::DefaultBaseScore()); + } + + { + std::unique_ptr learner{Learner::Create({Xy})}; + learner->SetParam("objective", "reg:absoluteerror"); + learner->UpdateOneIter(0, Xy); + + HostDeviceVector predt; + learner->Predict(Xy, false, &predt, 0, 0); + auto h_predt = predt.ConstHostSpan(); + for (auto v : h_predt) { + ASSERT_NE(v, ObjFunction::DefaultBaseScore()); + } + + Json config{Object{}}; + learner->SaveConfig(&config); + auto base_score = + std::stof(get(config["learner"]["learner_model_param"]["base_score"])); + ASSERT_NE(base_score, ObjFunction::DefaultBaseScore()); + + ASSERT_THROW( + { + learner->SetParam("base_score_estimated", "1"); + learner->Configure(); + }, + dmlc::Error); + } + + { + std::unique_ptr learner{Learner::Create({Xy})}; + learner->SetParam("objective", "reg:absoluteerror"); + learner->SetParam("base_score", "1.3"); + learner->Configure(); + HostDeviceVector predt; + learner->Predict(Xy, false, &predt, 0, 0); + auto h_predt = predt.ConstHostSpan(); + for (auto v : h_predt) { + ASSERT_FLOAT_EQ(v, 1.3); + } + learner->UpdateOneIter(0, Xy); + Json config{Object{}}; + learner->SaveConfig(&config); + auto base_score = + std::stof(get(config["learner"]["learner_model_param"]["base_score"])); + // no change + ASSERT_FLOAT_EQ(base_score, 1.3); + } +} } // namespace xgboost diff --git a/tests/cpp/test_serialization.cc b/tests/cpp/test_serialization.cc index d80a7442202e..15765f09f29d 100644 --- a/tests/cpp/test_serialization.cc +++ b/tests/cpp/test_serialization.cc @@ -418,6 +418,45 @@ TEST_F(SerializationTest, GPUCoordDescent) { } #endif // defined(XGBOOST_USE_CUDA) +class L1SerializationTest : public SerializationTest {}; + +TEST_F(L1SerializationTest, Exact) { + TestLearnerSerialization({{"booster", "gbtree"}, + {"objective", "reg:absoluteerror"}, + {"seed", "0"}, + {"max_depth", "2"}, + {"tree_method", "exact"}}, + fmap_, p_dmat_); +} + +TEST_F(L1SerializationTest, Approx) { + TestLearnerSerialization({{"booster", "gbtree"}, + {"objective", "reg:absoluteerror"}, + {"seed", "0"}, + {"max_depth", "2"}, + {"tree_method", "approx"}}, + fmap_, p_dmat_); +} + +TEST_F(L1SerializationTest, Hist) { + TestLearnerSerialization({{"booster", "gbtree"}, + {"objective", "reg:absoluteerror"}, + {"seed", "0"}, + {"max_depth", "2"}, + {"tree_method", "hist"}}, + fmap_, p_dmat_); +} + +#if defined(XGBOOST_USE_CUDA) +TEST_F(L1SerializationTest, GpuHist) { + TestLearnerSerialization({{"booster", "gbtree"}, + {"objective", "reg:absoluteerror"}, + {"seed", "0"}, + {"max_depth", "2"}, + {"tree_method", "gpu_hist"}}, + fmap_, p_dmat_); +} +#endif // defined(XGBOOST_USE_CUDA) class LogitSerializationTest : public SerializationTest { protected: diff --git a/tests/python-gpu/test_gpu_updaters.py b/tests/python-gpu/test_gpu_updaters.py index d0e7c5bc883d..7f29a92e6101 100644 --- a/tests/python-gpu/test_gpu_updaters.py +++ b/tests/python-gpu/test_gpu_updaters.py @@ -208,3 +208,8 @@ def test_specified_gpu_id_gpu_update(self, dataset, gpu_id): param = dataset.set_params(param) result = train_result(param, dataset.get_dmat(), 10) assert tm.non_increasing(result['train'][dataset.metric]) + + @pytest.mark.skipif(**tm.no_sklearn()) + @pytest.mark.parametrize("weighted", [True, False]) + def test_adaptive(self, weighted) -> None: + self.cputest.run_adaptive("gpu_hist", weighted) diff --git a/tests/python/test_model_compatibility.py b/tests/python/test_model_compatibility.py index 6f9a184922ab..88549e1f2acb 100644 --- a/tests/python/test_model_compatibility.py +++ b/tests/python/test_model_compatibility.py @@ -102,34 +102,38 @@ def run_scikit_model_check(name, path): @pytest.mark.skipif(**tm.no_sklearn()) def test_model_compatibility(): - '''Test model compatibility, can only be run on CI as others don't + """Test model compatibility, can only be run on CI as others don't have the credentials. - ''' + """ path = os.path.dirname(os.path.abspath(__file__)) - path = os.path.join(path, 'models') + path = os.path.join(path, "models") - zip_path, _ = urllib.request.urlretrieve('https://xgboost-ci-jenkins-artifacts.s3-us-west-2' + - '.amazonaws.com/xgboost_model_compatibility_test.zip') - with zipfile.ZipFile(zip_path, 'r') as z: - z.extractall(path) + if not os.path.exists(path): + zip_path, _ = urllib.request.urlretrieve( + "https://xgboost-ci-jenkins-artifacts.s3-us-west-2" + + ".amazonaws.com/xgboost_model_compatibility_test.zip" + ) + with zipfile.ZipFile(zip_path, "r") as z: + z.extractall(path) models = [ - os.path.join(root, f) for root, subdir, files in os.walk(path) + os.path.join(root, f) + for root, subdir, files in os.walk(path) for f in files - if f != 'version' + if f != "version" ] assert models for path in models: name = os.path.basename(path) - if name.startswith('xgboost-'): + if name.startswith("xgboost-"): booster = xgboost.Booster(model_file=path) run_booster_check(booster, name) # Do full serialization. booster = copy.copy(booster) run_booster_check(booster, name) - elif name.startswith('xgboost_scikit'): + elif name.startswith("xgboost_scikit"): run_scikit_model_check(name, path) else: assert False diff --git a/tests/python/test_updaters.py b/tests/python/test_updaters.py index 3e43b98ff113..e28f173860e7 100644 --- a/tests/python/test_updaters.py +++ b/tests/python/test_updaters.py @@ -1,4 +1,4 @@ -from random import choice +import json from string import ascii_lowercase from typing import Dict, Any import testing as tm @@ -397,3 +397,72 @@ def test_categorical_ames_housing( def test_categorical_missing(self, rows, cols, cats): self.run_categorical_missing(rows, cols, cats, "approx") self.run_categorical_missing(rows, cols, cats, "hist") + + def run_adaptive(self, tree_method, weighted) -> None: + rng = np.random.RandomState(1994) + from sklearn.datasets import make_regression + from sklearn.utils import stats + + n_samples = 256 + X, y = make_regression(n_samples, 16, random_state=rng) + if weighted: + w = rng.normal(size=n_samples) + w -= w.min() + Xy = xgb.DMatrix(X, y, weight=w) + base_score = stats._weighted_percentile(y, w, percentile=50) + else: + Xy = xgb.DMatrix(X, y) + base_score = np.median(y) + + booster_0 = xgb.train( + { + "tree_method": tree_method, + "base_score": base_score, + "objective": "reg:absoluteerror", + }, + Xy, + num_boost_round=1, + ) + booster_1 = xgb.train( + {"tree_method": tree_method, "objective": "reg:absoluteerror"}, + Xy, + num_boost_round=1, + ) + config_0 = json.loads(booster_0.save_config()) + config_1 = json.loads(booster_1.save_config()) + + def get_score(config: Dict) -> float: + return float(config["learner"]["learner_model_param"]["base_score"]) + + assert get_score(config_0) == get_score(config_1) + + raw_booster = booster_1.save_raw(raw_format="deprecated") + booster_2 = xgb.Booster(model_file=raw_booster) + config_2 = json.loads(booster_2.save_config()) + assert get_score(config_1) == get_score(config_2) + + raw_booster = booster_1.save_raw(raw_format="ubj") + booster_2 = xgb.Booster(model_file=raw_booster) + config_2 = json.loads(booster_2.save_config()) + assert get_score(config_1) == get_score(config_2) + + booster_0 = xgb.train( + { + "tree_method": tree_method, + "base_score": base_score + 1.0, + "objective": "reg:absoluteerror", + }, + Xy, + num_boost_round=1, + ) + config_0 = json.loads(booster_0.save_config()) + np.testing.assert_allclose(get_score(config_0), get_score(config_1) + 1) + + @pytest.mark.skipif(**tm.no_sklearn()) + @pytest.mark.parametrize( + "tree_method,weighted", [ + ("approx", False), ("hist", False), ("approx", True), ("hist", True) + ] + ) + def test_adaptive(self, tree_method, weighted) -> None: + self.run_adaptive(tree_method, weighted) diff --git a/tests/python/test_with_dask.py b/tests/python/test_with_dask.py index 7695538923d4..d6eb4f32b9f7 100644 --- a/tests/python/test_with_dask.py +++ b/tests/python/test_with_dask.py @@ -1537,13 +1537,56 @@ def test_quantile(self) -> None: @pytest.mark.skipif(**tm.no_dask()) @pytest.mark.gtest def test_quantile_same_on_all_workers(self) -> None: - self.run_quantile('SameOnAllWorkers') + self.run_quantile("SameOnAllWorkers") + + def test_adaptive(self) -> None: + def get_score(config: Dict) -> float: + return float(config["learner"]["learner_model_param"]["base_score"]) + + def local_test(rabit_args: List[bytes], worker_id: int) -> bool: + with xgb.dask.RabitContext(rabit_args): + if worker_id == 0: + y = np.array([0.0, 0.0, 0.0]) + x = np.array([[0.0]] * 3) + else: + y = np.array([1000.0]) + x = np.array( + [ + [0.0], + ] + ) + + Xy = xgb.DMatrix(x, y) + booster = xgb.train( + {"tree_method": "hist", "objective": "reg:absoluteerror"}, + Xy, + num_boost_round=1, + ) + config = json.loads(booster.save_config()) + base_score = get_score(config) + assert base_score == 250.0 + return True + + with LocalCluster(n_workers=2, dashboard_address=":0") as cluster: + with Client(cluster) as client: + workers = _get_client_workers(client) + rabit_args = client.sync( + xgb.dask._get_rabit_args, len(workers), None, client + ) + futures = [] + for i, _ in enumerate(workers): + f = client.submit(local_test, rabit_args, i) + futures.append(f) + + results = client.gather(futures) + assert all(results) def test_n_workers(self) -> None: with LocalCluster(n_workers=2, dashboard_address=":0") as cluster: with Client(cluster) as client: workers = _get_client_workers(client) from sklearn.datasets import load_breast_cancer + X, y = load_breast_cancer(return_X_y=True) dX = client.submit(da.from_array, X, workers=[workers[0]]).result() dy = client.submit(da.from_array, y, workers=[workers[0]]).result()