diff --git a/.gitignore b/.gitignore
index 20b92c057e1a..15503ad57bf4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -97,8 +97,11 @@ metastore_db
 R-package/src/Makevars
 *.lib
 
-# Visual Studio Code
-/.vscode/
+# Visual Studio
+.vs/
+CMakeSettings.json
+*.ilk
+*.pdb
 
 # IntelliJ/CLion
 .idea
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 000000000000..924f516f95e6
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,28 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+  apt_packages:
+    - graphviz
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: doc/conf.py
+
+# If using Sphinx, optionally build your docs in additional formats such as PDF
+formats:
+   - pdf
+
+# Optionally declare the Python requirements required to build your docs
+python:
+  install:
+   - requirements: doc/requirements.txt
+  system_packages: true
diff --git a/R-package/src/Makevars.win b/R-package/src/Makevars.win
index 381d5a6d3889..f48e41188f6d 100644
--- a/R-package/src/Makevars.win
+++ b/R-package/src/Makevars.win
@@ -30,7 +30,7 @@ $(foreach v, $(XGB_RFLAGS), $(warning $(v)))
 
 PKG_CPPFLAGS=  -I$(PKGROOT)/include -I$(PKGROOT)/dmlc-core/include -I$(PKGROOT)/rabit/include -I$(PKGROOT) $(XGB_RFLAGS)
 PKG_CXXFLAGS= $(SHLIB_OPENMP_CXXFLAGS) $(SHLIB_PTHREAD_FLAGS)
-PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(SHLIB_PTHREAD_FLAGS)
+PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(SHLIB_PTHREAD_FLAGS) -lwsock32 -lws2_32
 OBJECTS= ./xgboost_R.o ./xgboost_custom.o ./xgboost_assert.o ./init.o \
          $(PKGROOT)/amalgamation/xgboost-all0.o $(PKGROOT)/amalgamation/dmlc-minimum0.o \
          $(PKGROOT)/rabit/src/engine.o $(PKGROOT)/rabit/src/rabit_c_api.o \
diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake
index 9c8dc188c735..f28c1f2703e6 100644
--- a/cmake/Utils.cmake
+++ b/cmake/Utils.cmake
@@ -244,7 +244,7 @@ macro(xgboost_target_properties target)
       $<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/utf-8>
       -D_CRT_SECURE_NO_WARNINGS
       -D_CRT_SECURE_NO_DEPRECATE
-      )
+    )
   endif (MSVC)
 
   if (WIN32 AND MINGW)
@@ -314,4 +314,8 @@ macro(xgboost_target_link_libraries target)
   if (RABIT_BUILD_MPI)
     target_link_libraries(${target} PRIVATE MPI::MPI_CXX)
   endif (RABIT_BUILD_MPI)
+
+  if (MINGW)
+    target_link_libraries(${target} PRIVATE wsock32 ws2_32)
+  endif (MINGW)
 endmacro(xgboost_target_link_libraries)
diff --git a/rabit/src/allreduce_base.cc b/rabit/src/allreduce_base.cc
index ffed988efb98..75ba901b2145 100644
--- a/rabit/src/allreduce_base.cc
+++ b/rabit/src/allreduce_base.cc
@@ -5,7 +5,10 @@
  *
  * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou
  */
+#if !defined(NOMINMAX) && defined(_WIN32)
 #define NOMINMAX
+#endif  // !defined(NOMINMAX)
+
 #include "rabit/base.h"
 #include "rabit/internal/rabit-inl.h"
 #include "allreduce_base.h"
diff --git a/src/cli_main.cc b/src/cli_main.cc
index dc99cf4f0451..9e3b9e6c820f 100644
--- a/src/cli_main.cc
+++ b/src/cli_main.cc
@@ -6,7 +6,11 @@
  */
 #define _CRT_SECURE_NO_WARNINGS
 #define _CRT_SECURE_NO_DEPRECATE
+
+#if !defined(NOMINMAX) && defined(_WIN32)
 #define NOMINMAX
+#endif  // !defined(NOMINMAX)
+
 #include <dmlc/timer.h>
 
 #include <xgboost/learner.h>
diff --git a/tests/cpp/common/test_column_matrix.cc b/tests/cpp/common/test_column_matrix.cc
index e2f59c58d8fc..fca5c0c4e316 100644
--- a/tests/cpp/common/test_column_matrix.cc
+++ b/tests/cpp/common/test_column_matrix.cc
@@ -1,7 +1,6 @@
 /*!
  * Copyright 2018-2022 by XGBoost Contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 
 #include "../../../src/common/column_matrix.h"
diff --git a/tests/cpp/common/test_config.cc b/tests/cpp/common/test_config.cc
index 7bf61dcfd729..5807db8caa9a 100644
--- a/tests/cpp/common/test_config.cc
+++ b/tests/cpp/common/test_config.cc
@@ -1,11 +1,13 @@
 /*!
  * Copyright 2019 by Contributors
  */
+#include <gtest/gtest.h>
+
 #include <fstream>
 #include <string>
-#include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
+
 #include "../../../src/common/config.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 namespace xgboost {
diff --git a/tests/cpp/common/test_hist_util.cu b/tests/cpp/common/test_hist_util.cu
index e92e2728eb40..7324531b17f0 100644
--- a/tests/cpp/common/test_hist_util.cu
+++ b/tests/cpp/common/test_hist_util.cu
@@ -1,26 +1,25 @@
 /*!
  * Copyright 2019-2022 by XGBoost Contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
+#include <thrust/device_vector.h>
+#include <xgboost/c_api.h>
+#include <xgboost/data.h>
 
 #include <algorithm>
 #include <cmath>
-#include <thrust/device_vector.h>
-
-#include <xgboost/data.h>
-#include <xgboost/c_api.h>
 
-#include "test_hist_util.h"
-#include "../helpers.h"
-#include "../data/test_array_interface.h"
+#include "../../../include/xgboost/logging.h"
 #include "../../../src/common/device_helpers.cuh"
-#include "../../../src/common/hist_util.h"
 #include "../../../src/common/hist_util.cuh"
-#include "../../../src/data/device_adapter.cuh"
+#include "../../../src/common/hist_util.h"
 #include "../../../src/common/math.h"
+#include "../../../src/data/device_adapter.cuh"
 #include "../../../src/data/simple_dmatrix.h"
-#include "../../../include/xgboost/logging.h"
+#include "../data/test_array_interface.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
+#include "../helpers.h"
+#include "test_hist_util.h"
 
 namespace xgboost {
 namespace common {
diff --git a/tests/cpp/common/test_hist_util.h b/tests/cpp/common/test_hist_util.h
index 37a38d1b3361..c8881c158621 100644
--- a/tests/cpp/common/test_hist_util.h
+++ b/tests/cpp/common/test_hist_util.h
@@ -3,16 +3,17 @@
  */
 #pragma once
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
+
+#include <fstream>
 #include <random>
-#include <vector>
 #include <string>
-#include <fstream>
+#include <vector>
 
-#include "../helpers.h"
 #include "../../../src/common/hist_util.h"
-#include "../../../src/data/simple_dmatrix.h"
 #include "../../../src/data/adapter.h"
+#include "../../../src/data/simple_dmatrix.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
+#include "../helpers.h"
 
 #ifdef __CUDACC__
 #include <xgboost/json.h>
diff --git a/tests/cpp/common/test_io.cc b/tests/cpp/common/test_io.cc
index 974fdefad6c8..feac8bd89934 100644
--- a/tests/cpp/common/test_io.cc
+++ b/tests/cpp/common/test_io.cc
@@ -2,12 +2,12 @@
  * Copyright (c) by XGBoost Contributors 2019
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
 
 #include <fstream>
 
-#include "../helpers.h"
 #include "../../../src/common/io.h"
+#include "../helpers.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 
 namespace xgboost {
 namespace common {
diff --git a/tests/cpp/common/test_json.cc b/tests/cpp/common/test_json.cc
index 7836b23afcae..6397122756c4 100644
--- a/tests/cpp/common/test_json.cc
+++ b/tests/cpp/common/test_json.cc
@@ -2,16 +2,17 @@
  * Copyright (c) by Contributors 2019-2022
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
+
 #include <fstream>
 #include <map>
 
+#include "../../../src/common/charconv.h"
+#include "../../../src/common/io.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
+#include "../helpers.h"
 #include "xgboost/json.h"
-#include "xgboost/logging.h"
 #include "xgboost/json_io.h"
-#include "../helpers.h"
-#include "../../../src/common/io.h"
-#include "../../../src/common/charconv.h"
+#include "xgboost/logging.h"
 
 namespace xgboost {
 
diff --git a/tests/cpp/common/test_quantile.h b/tests/cpp/common/test_quantile.h
index 8118248dc939..d92695f53276 100644
--- a/tests/cpp/common/test_quantile.h
+++ b/tests/cpp/common/test_quantile.h
@@ -1,3 +1,6 @@
+#ifndef XGBOOST_TESTS_CPP_COMMON_TEST_QUANTILE_H_
+#define XGBOOST_TESTS_CPP_COMMON_TEST_QUANTILE_H_
+
 #include <rabit/rabit.h>
 #include <algorithm>
 #include <string>
@@ -62,3 +65,5 @@ template <typename Fn> void RunWithSeedsAndBins(size_t rows, Fn fn) {
 }
 }  // namespace common
 }  // namespace xgboost
+
+#endif  // XGBOOST_TESTS_CPP_COMMON_TEST_QUANTILE_H_
diff --git a/tests/cpp/common/test_version.cc b/tests/cpp/common/test_version.cc
index e6ee030f7b8c..802e04344645 100644
--- a/tests/cpp/common/test_version.cc
+++ b/tests/cpp/common/test_version.cc
@@ -1,18 +1,16 @@
 /*!
  * Copyright 2019 XGBoost contributors
  */
-#include <gtest/gtest.h>
-
-#include <dmlc/filesystem.h>
 #include <dmlc/io.h>
-
-#include <xgboost/version_config.h>
-#include <xgboost/json.h>
+#include <gtest/gtest.h>
 #include <xgboost/base.h>
+#include <xgboost/json.h>
+#include <xgboost/version_config.h>
 
 #include <string>
 
 #include "../../../src/common/version.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 
 namespace xgboost {
 TEST(Version, Basic) {
diff --git a/tests/cpp/data/test_data.cc b/tests/cpp/data/test_data.cc
index 92e94fee8dd1..871a7f49800e 100644
--- a/tests/cpp/data/test_data.cc
+++ b/tests/cpp/data/test_data.cc
@@ -2,13 +2,14 @@
  * Copyright 2019-2022 by XGBoost Contributors
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
+
 #include <fstream>
 #include <memory>
 #include <vector>
 
-#include "xgboost/data.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
+#include "xgboost/data.h"
 
 namespace xgboost {
 TEST(SparsePage, PushCSC) {
diff --git a/tests/cpp/data/test_ellpack_page_raw_format.cu b/tests/cpp/data/test_ellpack_page_raw_format.cu
index d4b5722eabf6..92b4acf4bd96 100644
--- a/tests/cpp/data/test_ellpack_page_raw_format.cu
+++ b/tests/cpp/data/test_ellpack_page_raw_format.cu
@@ -2,12 +2,11 @@
  * Copyright 2021 XGBoost contributors
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
 
-#include "../../../src/data/sparse_page_source.h"
 #include "../../../src/data/ellpack_page.cuh"
-
+#include "../../../src/data/sparse_page_source.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 namespace xgboost {
diff --git a/tests/cpp/data/test_file_iterator.cc b/tests/cpp/data/test_file_iterator.cc
index 12ae9e726d4a..21029620b8ab 100644
--- a/tests/cpp/data/test_file_iterator.cc
+++ b/tests/cpp/data/test_file_iterator.cc
@@ -2,13 +2,13 @@
  * Copyright 2021 XGBoost contributors
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
 
 #include <memory>
 
+#include "../../../src/data/adapter.h"
 #include "../../../src/data/file_iterator.h"
 #include "../../../src/data/proxy_dmatrix.h"
-#include "../../../src/data/adapter.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 namespace xgboost {
diff --git a/tests/cpp/data/test_metainfo.cc b/tests/cpp/data/test_metainfo.cc
index 62146b571fb8..342af77bfba2 100644
--- a/tests/cpp/data/test_metainfo.cc
+++ b/tests/cpp/data/test_metainfo.cc
@@ -2,12 +2,13 @@
 #include "test_metainfo.h"
 
 #include <dmlc/io.h>
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
-#include <string>
+
 #include <memory>
-#include "../../../src/common/version.h"
+#include <string>
 
+#include "../../../src/common/version.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "xgboost/base.h"
 
diff --git a/tests/cpp/data/test_simple_dmatrix.cc b/tests/cpp/data/test_simple_dmatrix.cc
index e709605c9c6d..266115731fd6 100644
--- a/tests/cpp/data/test_simple_dmatrix.cc
+++ b/tests/cpp/data/test_simple_dmatrix.cc
@@ -1,12 +1,13 @@
 // Copyright by Contributors
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
 
 #include <array>
-#include "xgboost/base.h"
-#include "../../../src/data/simple_dmatrix.h"
+
 #include "../../../src/data/adapter.h"
+#include "../../../src/data/simple_dmatrix.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
+#include "xgboost/base.h"
 
 using namespace xgboost;  // NOLINT
 
diff --git a/tests/cpp/data/test_simple_dmatrix.cu b/tests/cpp/data/test_simple_dmatrix.cu
index 19f13b1fddfd..4b020c0a6cda 100644
--- a/tests/cpp/data/test_simple_dmatrix.cu
+++ b/tests/cpp/data/test_simple_dmatrix.cu
@@ -1,5 +1,4 @@
 // Copyright by Contributors
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
 #include "../../../src/data/simple_dmatrix.h"
 
diff --git a/tests/cpp/data/test_sparse_page_dmatrix.cc b/tests/cpp/data/test_sparse_page_dmatrix.cc
index b5ed00fb4f8a..68171932bdc5 100644
--- a/tests/cpp/data/test_sparse_page_dmatrix.cc
+++ b/tests/cpp/data/test_sparse_page_dmatrix.cc
@@ -1,14 +1,16 @@
 // Copyright by Contributors
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/data.h>
-#include <thread>
+
 #include <future>
+#include <thread>
+
 #include "../../../src/common/io.h"
 #include "../../../src/data/adapter.h"
+#include "../../../src/data/file_iterator.h"
 #include "../../../src/data/simple_dmatrix.h"
 #include "../../../src/data/sparse_page_dmatrix.h"
-#include "../../../src/data/file_iterator.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 using namespace xgboost;  // NOLINT
diff --git a/tests/cpp/data/test_sparse_page_dmatrix.cu b/tests/cpp/data/test_sparse_page_dmatrix.cu
index b9e91e6b1ef3..07c86c93f99c 100644
--- a/tests/cpp/data/test_sparse_page_dmatrix.cu
+++ b/tests/cpp/data/test_sparse_page_dmatrix.cu
@@ -1,10 +1,10 @@
 // Copyright by Contributors
 
-#include <dmlc/filesystem.h>
-#include "../helpers.h"
 #include "../../../src/common/compressed_iterator.h"
 #include "../../../src/data/ellpack_page.cuh"
 #include "../../../src/data/sparse_page_dmatrix.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
+#include "../helpers.h"
 
 namespace xgboost {
 
diff --git a/tests/cpp/data/test_sparse_page_raw_format.cc b/tests/cpp/data/test_sparse_page_raw_format.cc
index dc7c5b2be77f..5743c4223a1b 100644
--- a/tests/cpp/data/test_sparse_page_raw_format.cc
+++ b/tests/cpp/data/test_sparse_page_raw_format.cc
@@ -2,10 +2,10 @@
  * Copyright 2021 XGBoost contributors
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
 
 #include "../../../src/data/sparse_page_source.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 namespace xgboost {
diff --git a/tests/cpp/filesystem.h b/tests/cpp/filesystem.h
new file mode 100644
index 000000000000..5410feede316
--- /dev/null
+++ b/tests/cpp/filesystem.h
@@ -0,0 +1,15 @@
+/*!
+ * Copyright (c) 2022 by XGBoost Contributors
+ */
+
+#ifndef XGBOOST_TESTS_CPP_FILESYSTEM_H
+#define XGBOOST_TESTS_CPP_FILESYSTEM_H
+
+// A macro used inside `windows.h` to avoid conflicts with `winsock2.h`
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif  // WIN32_LEAN_AND_MEAN
+
+#include "dmlc/filesystem.h"
+
+#endif  // XGBOOST_TESTS_CPP_FILESYSTEM_H
diff --git a/tests/cpp/gbm/test_gbtree.cc b/tests/cpp/gbm/test_gbtree.cc
index 00201769bc03..a5c16f7951d7 100644
--- a/tests/cpp/gbm/test_gbtree.cc
+++ b/tests/cpp/gbm/test_gbtree.cc
@@ -1,13 +1,13 @@
 /*!
  * Copyright 2019-2022 XGBoost contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/generic_parameters.h>
 
 #include "../../../src/data/adapter.h"
 #include "../../../src/data/proxy_dmatrix.h"
 #include "../../../src/gbm/gbtree.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "xgboost/base.h"
 #include "xgboost/host_device_vector.h"
diff --git a/tests/cpp/helpers.cc b/tests/cpp/helpers.cc
index 17972dc8fc63..0273d964f707 100644
--- a/tests/cpp/helpers.cc
+++ b/tests/cpp/helpers.cc
@@ -3,7 +3,6 @@
  */
 #include "helpers.h"
 
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/gbm.h>
 #include <xgboost/json.h>
@@ -21,6 +20,7 @@
 #include "../../src/data/simple_dmatrix.h"
 #include "../../src/data/sparse_page_dmatrix.h"
 #include "../../src/gbm/gbtree_model.h"
+#include "filesystem.h"  // dmlc::TemporaryDirectory
 #include "xgboost/c_api.h"
 #include "xgboost/predictor.h"
 
diff --git a/tests/cpp/helpers.h b/tests/cpp/helpers.h
index d54e75f2f35c..b79ea27187f5 100644
--- a/tests/cpp/helpers.h
+++ b/tests/cpp/helpers.h
@@ -4,25 +4,24 @@
 #ifndef XGBOOST_TESTS_CPP_HELPERS_H_
 #define XGBOOST_TESTS_CPP_HELPERS_H_
 
-#include <iostream>
-#include <fstream>
-#include <cstdio>
-#include <string>
-#include <memory>
-#include <vector>
+#include <gtest/gtest.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-
-#include <gtest/gtest.h>
-
-#include <dmlc/filesystem.h>
 #include <xgboost/base.h>
-#include <xgboost/json.h>
 #include <xgboost/generic_parameters.h>
+#include <xgboost/json.h>
+
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
 
 #include "../../src/common/common.h"
-#include "../../src/gbm/gbtree_model.h"
 #include "../../src/data/array_interface.h"
+#include "../../src/gbm/gbtree_model.h"
+#include "filesystem.h"  // dmlc::TemporaryDirectory
 
 #if defined(__CUDACC__)
 #define DeclareUnifiedTest(name) GPU ## name
diff --git a/tests/cpp/plugin/test_predictor_oneapi.cc b/tests/cpp/plugin/test_predictor_oneapi.cc
index 61d82d846fe9..1e5df312c694 100755
--- a/tests/cpp/plugin/test_predictor_oneapi.cc
+++ b/tests/cpp/plugin/test_predictor_oneapi.cc
@@ -1,14 +1,14 @@
 /*!
  * Copyright 2017-2020 XGBoost contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/predictor.h>
 
+#include "../../../src/data/adapter.h"
+#include "../../../src/gbm/gbtree_model.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "../predictor/test_predictor.h"
-#include "../../../src/gbm/gbtree_model.h"
-#include "../../../src/data/adapter.h"
 
 namespace xgboost {
 TEST(Plugin, OneAPIPredictorBasic) {
diff --git a/tests/cpp/predictor/test_cpu_predictor.cc b/tests/cpp/predictor/test_cpu_predictor.cc
index ea5dce20c1e5..8ba270083c74 100644
--- a/tests/cpp/predictor/test_cpu_predictor.cc
+++ b/tests/cpp/predictor/test_cpu_predictor.cc
@@ -1,7 +1,6 @@
 /*!
  * Copyright 2017-2022 XGBoost contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/predictor.h>
 
@@ -9,6 +8,7 @@
 #include "../../../src/data/proxy_dmatrix.h"
 #include "../../../src/gbm/gbtree.h"
 #include "../../../src/gbm/gbtree_model.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "test_predictor.h"
 
diff --git a/tests/cpp/predictor/test_gpu_predictor.cu b/tests/cpp/predictor/test_gpu_predictor.cu
index fa849cf51bd9..8dacadac5403 100644
--- a/tests/cpp/predictor/test_gpu_predictor.cu
+++ b/tests/cpp/predictor/test_gpu_predictor.cu
@@ -1,7 +1,6 @@
 /*!
  * Copyright 2017-2020 XGBoost contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/c_api.h>
 #include <xgboost/learner.h>
diff --git a/tests/cpp/test_learner.cc b/tests/cpp/test_learner.cc
index 987626df86bf..4a8214e9c5cd 100644
--- a/tests/cpp/test_learner.cc
+++ b/tests/cpp/test_learner.cc
@@ -2,17 +2,18 @@
  * Copyright 2017-2022 by XGBoost contributors
  */
 #include <gtest/gtest.h>
-#include <vector>
-#include <thread>
-#include "helpers.h"
-#include <dmlc/filesystem.h>
-
 #include <xgboost/learner.h>
 #include <xgboost/version_config.h>
-#include "xgboost/json.h"
+
+#include <thread>
+#include <vector>
+
 #include "../../src/common/io.h"
-#include "../../src/common/random.h"
 #include "../../src/common/linalg_op.h"
+#include "../../src/common/random.h"
+#include "filesystem.h"  // dmlc::TemporaryDirectory
+#include "helpers.h"
+#include "xgboost/json.h"
 
 namespace xgboost {
 TEST(Learner, Basic) {
diff --git a/tests/cpp/test_serialization.cc b/tests/cpp/test_serialization.cc
index bf459cf35dcb..d80a7442202e 100644
--- a/tests/cpp/test_serialization.cc
+++ b/tests/cpp/test_serialization.cc
@@ -1,14 +1,16 @@
 // Copyright (c) 2019-2022 by Contributors
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
-#include <string>
-#include <xgboost/learner.h>
-#include <xgboost/data.h>
 #include <xgboost/base.h>
+#include <xgboost/data.h>
 #include <xgboost/json.h>
-#include "helpers.h"
+#include <xgboost/learner.h>
+
+#include <string>
+
 #include "../../src/common/io.h"
 #include "../../src/common/random.h"
+#include "filesystem.h"  // dmlc::TemporaryDirectory
+#include "helpers.h"
 
 namespace xgboost {
 template <typename Array>
diff --git a/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu b/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu
index 9e8cd19bec74..e211fe70a483 100644
--- a/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu
+++ b/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu
@@ -6,8 +6,8 @@
 #include "../../../../src/data/ellpack_page.cuh"
 #include "../../../../src/tree/gpu_hist/gradient_based_sampler.cuh"
 #include "../../../../src/tree/param.h"
+#include "../../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../../helpers.h"
-#include "dmlc/filesystem.h"
 
 namespace xgboost {
 namespace tree {
diff --git a/tests/cpp/tree/test_gpu_hist.cu b/tests/cpp/tree/test_gpu_hist.cu
index f28f723301af..24fe40f42f58 100644
--- a/tests/cpp/tree/test_gpu_hist.cu
+++ b/tests/cpp/tree/test_gpu_hist.cu
@@ -4,22 +4,22 @@
 #include <gtest/gtest.h>
 #include <thrust/device_vector.h>
 #include <thrust/host_vector.h>
-#include <dmlc/filesystem.h>
 #include <xgboost/base.h>
+
 #include <random>
 #include <string>
 #include <vector>
 
+#include "../../../src/common/common.h"
+#include "../../../src/data/sparse_page_source.h"
+#include "../../../src/tree/constraints.cuh"
+#include "../../../src/tree/updater_gpu_common.cuh"
+#include "../../../src/tree/updater_gpu_hist.cu"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "../histogram_helpers.h"
-
 #include "xgboost/generic_parameters.h"
 #include "xgboost/json.h"
-#include "../../../src/data/sparse_page_source.h"
-#include "../../../src/tree/updater_gpu_hist.cu"
-#include "../../../src/tree/updater_gpu_common.cuh"
-#include "../../../src/common/common.h"
-#include "../../../src/tree/constraints.cuh"
 
 namespace xgboost {
 namespace tree {
diff --git a/tests/cpp/tree/test_tree_model.cc b/tests/cpp/tree/test_tree_model.cc
index fb14e300c0f4..65957255bf38 100644
--- a/tests/cpp/tree/test_tree_model.cc
+++ b/tests/cpp/tree/test_tree_model.cc
@@ -1,11 +1,12 @@
 // Copyright by Contributors
 #include <gtest/gtest.h>
+
+#include "../../../src/common/bitfield.h"
+#include "../../../src/common/categorical.h"
+#include "../filesystem.h"
 #include "../helpers.h"
-#include "dmlc/filesystem.h"
 #include "xgboost/json_io.h"
 #include "xgboost/tree_model.h"
-#include "../../../src/common/bitfield.h"
-#include "../../../src/common/categorical.h"
 
 namespace xgboost {
 TEST(Tree, ModelShape) {