diff --git a/tests/cpp/common/test_column_matrix.cc b/tests/cpp/common/test_column_matrix.cc
index cdd38468a25c..2c662a3f9951 100644
--- a/tests/cpp/common/test_column_matrix.cc
+++ b/tests/cpp/common/test_column_matrix.cc
@@ -1,7 +1,6 @@
 /*!
  * Copyright 2018-2022 by XGBoost Contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 
 #include "../../../src/common/column_matrix.h"
diff --git a/tests/cpp/common/test_config.cc b/tests/cpp/common/test_config.cc
index 7bf61dcfd729..5807db8caa9a 100644
--- a/tests/cpp/common/test_config.cc
+++ b/tests/cpp/common/test_config.cc
@@ -1,11 +1,13 @@
 /*!
  * Copyright 2019 by Contributors
  */
+#include <gtest/gtest.h>
+
 #include <fstream>
 #include <string>
-#include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
+
 #include "../../../src/common/config.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 namespace xgboost {
diff --git a/tests/cpp/common/test_hist_util.cu b/tests/cpp/common/test_hist_util.cu
index e92e2728eb40..7324531b17f0 100644
--- a/tests/cpp/common/test_hist_util.cu
+++ b/tests/cpp/common/test_hist_util.cu
@@ -1,26 +1,25 @@
 /*!
  * Copyright 2019-2022 by XGBoost Contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
+#include <thrust/device_vector.h>
+#include <xgboost/c_api.h>
+#include <xgboost/data.h>
 
 #include <algorithm>
 #include <cmath>
-#include <thrust/device_vector.h>
-
-#include <xgboost/data.h>
-#include <xgboost/c_api.h>
 
-#include "test_hist_util.h"
-#include "../helpers.h"
-#include "../data/test_array_interface.h"
+#include "../../../include/xgboost/logging.h"
 #include "../../../src/common/device_helpers.cuh"
-#include "../../../src/common/hist_util.h"
 #include "../../../src/common/hist_util.cuh"
-#include "../../../src/data/device_adapter.cuh"
+#include "../../../src/common/hist_util.h"
 #include "../../../src/common/math.h"
+#include "../../../src/data/device_adapter.cuh"
 #include "../../../src/data/simple_dmatrix.h"
-#include "../../../include/xgboost/logging.h"
+#include "../data/test_array_interface.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
+#include "../helpers.h"
+#include "test_hist_util.h"
 
 namespace xgboost {
 namespace common {
diff --git a/tests/cpp/common/test_hist_util.h b/tests/cpp/common/test_hist_util.h
index 37a38d1b3361..c8881c158621 100644
--- a/tests/cpp/common/test_hist_util.h
+++ b/tests/cpp/common/test_hist_util.h
@@ -3,16 +3,17 @@
  */
 #pragma once
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
+
+#include <fstream>
 #include <random>
-#include <vector>
 #include <string>
-#include <fstream>
+#include <vector>
 
-#include "../helpers.h"
 #include "../../../src/common/hist_util.h"
-#include "../../../src/data/simple_dmatrix.h"
 #include "../../../src/data/adapter.h"
+#include "../../../src/data/simple_dmatrix.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
+#include "../helpers.h"
 
 #ifdef __CUDACC__
 #include <xgboost/json.h>
diff --git a/tests/cpp/common/test_io.cc b/tests/cpp/common/test_io.cc
index 974fdefad6c8..feac8bd89934 100644
--- a/tests/cpp/common/test_io.cc
+++ b/tests/cpp/common/test_io.cc
@@ -2,12 +2,12 @@
  * Copyright (c) by XGBoost Contributors 2019
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
 
 #include <fstream>
 
-#include "../helpers.h"
 #include "../../../src/common/io.h"
+#include "../helpers.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 
 namespace xgboost {
 namespace common {
diff --git a/tests/cpp/common/test_json.cc b/tests/cpp/common/test_json.cc
index 7836b23afcae..6397122756c4 100644
--- a/tests/cpp/common/test_json.cc
+++ b/tests/cpp/common/test_json.cc
@@ -2,16 +2,17 @@
  * Copyright (c) by Contributors 2019-2022
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
+
 #include <fstream>
 #include <map>
 
+#include "../../../src/common/charconv.h"
+#include "../../../src/common/io.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
+#include "../helpers.h"
 #include "xgboost/json.h"
-#include "xgboost/logging.h"
 #include "xgboost/json_io.h"
-#include "../helpers.h"
-#include "../../../src/common/io.h"
-#include "../../../src/common/charconv.h"
+#include "xgboost/logging.h"
 
 namespace xgboost {
 
diff --git a/tests/cpp/common/test_version.cc b/tests/cpp/common/test_version.cc
index e6ee030f7b8c..802e04344645 100644
--- a/tests/cpp/common/test_version.cc
+++ b/tests/cpp/common/test_version.cc
@@ -1,18 +1,16 @@
 /*!
  * Copyright 2019 XGBoost contributors
  */
-#include <gtest/gtest.h>
-
-#include <dmlc/filesystem.h>
 #include <dmlc/io.h>
-
-#include <xgboost/version_config.h>
-#include <xgboost/json.h>
+#include <gtest/gtest.h>
 #include <xgboost/base.h>
+#include <xgboost/json.h>
+#include <xgboost/version_config.h>
 
 #include <string>
 
 #include "../../../src/common/version.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 
 namespace xgboost {
 TEST(Version, Basic) {
diff --git a/tests/cpp/data/test_data.cc b/tests/cpp/data/test_data.cc
index 92e94fee8dd1..871a7f49800e 100644
--- a/tests/cpp/data/test_data.cc
+++ b/tests/cpp/data/test_data.cc
@@ -2,13 +2,14 @@
  * Copyright 2019-2022 by XGBoost Contributors
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
+
 #include <fstream>
 #include <memory>
 #include <vector>
 
-#include "xgboost/data.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
+#include "xgboost/data.h"
 
 namespace xgboost {
 TEST(SparsePage, PushCSC) {
diff --git a/tests/cpp/data/test_ellpack_page_raw_format.cu b/tests/cpp/data/test_ellpack_page_raw_format.cu
index d4b5722eabf6..92b4acf4bd96 100644
--- a/tests/cpp/data/test_ellpack_page_raw_format.cu
+++ b/tests/cpp/data/test_ellpack_page_raw_format.cu
@@ -2,12 +2,11 @@
  * Copyright 2021 XGBoost contributors
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
 
-#include "../../../src/data/sparse_page_source.h"
 #include "../../../src/data/ellpack_page.cuh"
-
+#include "../../../src/data/sparse_page_source.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 namespace xgboost {
diff --git a/tests/cpp/data/test_file_iterator.cc b/tests/cpp/data/test_file_iterator.cc
index 12ae9e726d4a..21029620b8ab 100644
--- a/tests/cpp/data/test_file_iterator.cc
+++ b/tests/cpp/data/test_file_iterator.cc
@@ -2,13 +2,13 @@
  * Copyright 2021 XGBoost contributors
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
 
 #include <memory>
 
+#include "../../../src/data/adapter.h"
 #include "../../../src/data/file_iterator.h"
 #include "../../../src/data/proxy_dmatrix.h"
-#include "../../../src/data/adapter.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 namespace xgboost {
diff --git a/tests/cpp/data/test_metainfo.cc b/tests/cpp/data/test_metainfo.cc
index 62146b571fb8..342af77bfba2 100644
--- a/tests/cpp/data/test_metainfo.cc
+++ b/tests/cpp/data/test_metainfo.cc
@@ -2,12 +2,13 @@
 #include "test_metainfo.h"
 
 #include <dmlc/io.h>
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
-#include <string>
+
 #include <memory>
-#include "../../../src/common/version.h"
+#include <string>
 
+#include "../../../src/common/version.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "xgboost/base.h"
 
diff --git a/tests/cpp/data/test_simple_dmatrix.cc b/tests/cpp/data/test_simple_dmatrix.cc
index e709605c9c6d..266115731fd6 100644
--- a/tests/cpp/data/test_simple_dmatrix.cc
+++ b/tests/cpp/data/test_simple_dmatrix.cc
@@ -1,12 +1,13 @@
 // Copyright by Contributors
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
 
 #include <array>
-#include "xgboost/base.h"
-#include "../../../src/data/simple_dmatrix.h"
+
 #include "../../../src/data/adapter.h"
+#include "../../../src/data/simple_dmatrix.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
+#include "xgboost/base.h"
 
 using namespace xgboost;  // NOLINT
 
diff --git a/tests/cpp/data/test_simple_dmatrix.cu b/tests/cpp/data/test_simple_dmatrix.cu
index 19f13b1fddfd..4b020c0a6cda 100644
--- a/tests/cpp/data/test_simple_dmatrix.cu
+++ b/tests/cpp/data/test_simple_dmatrix.cu
@@ -1,5 +1,4 @@
 // Copyright by Contributors
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
 #include "../../../src/data/simple_dmatrix.h"
 
diff --git a/tests/cpp/data/test_sparse_page_dmatrix.cc b/tests/cpp/data/test_sparse_page_dmatrix.cc
index b5ed00fb4f8a..68171932bdc5 100644
--- a/tests/cpp/data/test_sparse_page_dmatrix.cc
+++ b/tests/cpp/data/test_sparse_page_dmatrix.cc
@@ -1,14 +1,16 @@
 // Copyright by Contributors
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/data.h>
-#include <thread>
+
 #include <future>
+#include <thread>
+
 #include "../../../src/common/io.h"
 #include "../../../src/data/adapter.h"
+#include "../../../src/data/file_iterator.h"
 #include "../../../src/data/simple_dmatrix.h"
 #include "../../../src/data/sparse_page_dmatrix.h"
-#include "../../../src/data/file_iterator.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 using namespace xgboost;  // NOLINT
diff --git a/tests/cpp/data/test_sparse_page_dmatrix.cu b/tests/cpp/data/test_sparse_page_dmatrix.cu
index b9e91e6b1ef3..07c86c93f99c 100644
--- a/tests/cpp/data/test_sparse_page_dmatrix.cu
+++ b/tests/cpp/data/test_sparse_page_dmatrix.cu
@@ -1,10 +1,10 @@
 // Copyright by Contributors
 
-#include <dmlc/filesystem.h>
-#include "../helpers.h"
 #include "../../../src/common/compressed_iterator.h"
 #include "../../../src/data/ellpack_page.cuh"
 #include "../../../src/data/sparse_page_dmatrix.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
+#include "../helpers.h"
 
 namespace xgboost {
 
diff --git a/tests/cpp/data/test_sparse_page_raw_format.cc b/tests/cpp/data/test_sparse_page_raw_format.cc
index dc7c5b2be77f..5743c4223a1b 100644
--- a/tests/cpp/data/test_sparse_page_raw_format.cc
+++ b/tests/cpp/data/test_sparse_page_raw_format.cc
@@ -2,10 +2,10 @@
  * Copyright 2021 XGBoost contributors
  */
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
 #include <xgboost/data.h>
 
 #include "../../../src/data/sparse_page_source.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 
 namespace xgboost {
diff --git a/tests/cpp/filesystem.h b/tests/cpp/filesystem.h
new file mode 100644
index 000000000000..a017bb57b58c
--- /dev/null
+++ b/tests/cpp/filesystem.h
@@ -0,0 +1,9 @@
+/*!
+ * Copyright (c) 2022 by XGBoost Contributors
+ */
+#pragma once
+
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif  // WIN32_LEAN_AND_MEAN
+#include "dmlc/filesystem.h"
diff --git a/tests/cpp/gbm/test_gbtree.cc b/tests/cpp/gbm/test_gbtree.cc
index 00201769bc03..a5c16f7951d7 100644
--- a/tests/cpp/gbm/test_gbtree.cc
+++ b/tests/cpp/gbm/test_gbtree.cc
@@ -1,13 +1,13 @@
 /*!
  * Copyright 2019-2022 XGBoost contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/generic_parameters.h>
 
 #include "../../../src/data/adapter.h"
 #include "../../../src/data/proxy_dmatrix.h"
 #include "../../../src/gbm/gbtree.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "xgboost/base.h"
 #include "xgboost/host_device_vector.h"
diff --git a/tests/cpp/helpers.cc b/tests/cpp/helpers.cc
index 17972dc8fc63..0273d964f707 100644
--- a/tests/cpp/helpers.cc
+++ b/tests/cpp/helpers.cc
@@ -3,7 +3,6 @@
  */
 #include "helpers.h"
 
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/gbm.h>
 #include <xgboost/json.h>
@@ -21,6 +20,7 @@
 #include "../../src/data/simple_dmatrix.h"
 #include "../../src/data/sparse_page_dmatrix.h"
 #include "../../src/gbm/gbtree_model.h"
+#include "filesystem.h"  // dmlc::TemporaryDirectory
 #include "xgboost/c_api.h"
 #include "xgboost/predictor.h"
 
diff --git a/tests/cpp/helpers.h b/tests/cpp/helpers.h
index d54e75f2f35c..b79ea27187f5 100644
--- a/tests/cpp/helpers.h
+++ b/tests/cpp/helpers.h
@@ -4,25 +4,24 @@
 #ifndef XGBOOST_TESTS_CPP_HELPERS_H_
 #define XGBOOST_TESTS_CPP_HELPERS_H_
 
-#include <iostream>
-#include <fstream>
-#include <cstdio>
-#include <string>
-#include <memory>
-#include <vector>
+#include <gtest/gtest.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-
-#include <gtest/gtest.h>
-
-#include <dmlc/filesystem.h>
 #include <xgboost/base.h>
-#include <xgboost/json.h>
 #include <xgboost/generic_parameters.h>
+#include <xgboost/json.h>
+
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
 
 #include "../../src/common/common.h"
-#include "../../src/gbm/gbtree_model.h"
 #include "../../src/data/array_interface.h"
+#include "../../src/gbm/gbtree_model.h"
+#include "filesystem.h"  // dmlc::TemporaryDirectory
 
 #if defined(__CUDACC__)
 #define DeclareUnifiedTest(name) GPU ## name
diff --git a/tests/cpp/plugin/test_predictor_oneapi.cc b/tests/cpp/plugin/test_predictor_oneapi.cc
index 61d82d846fe9..1e5df312c694 100755
--- a/tests/cpp/plugin/test_predictor_oneapi.cc
+++ b/tests/cpp/plugin/test_predictor_oneapi.cc
@@ -1,14 +1,14 @@
 /*!
  * Copyright 2017-2020 XGBoost contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/predictor.h>
 
+#include "../../../src/data/adapter.h"
+#include "../../../src/gbm/gbtree_model.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "../predictor/test_predictor.h"
-#include "../../../src/gbm/gbtree_model.h"
-#include "../../../src/data/adapter.h"
 
 namespace xgboost {
 TEST(Plugin, OneAPIPredictorBasic) {
diff --git a/tests/cpp/predictor/test_cpu_predictor.cc b/tests/cpp/predictor/test_cpu_predictor.cc
index ea5dce20c1e5..8ba270083c74 100644
--- a/tests/cpp/predictor/test_cpu_predictor.cc
+++ b/tests/cpp/predictor/test_cpu_predictor.cc
@@ -1,7 +1,6 @@
 /*!
  * Copyright 2017-2022 XGBoost contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/predictor.h>
 
@@ -9,6 +8,7 @@
 #include "../../../src/data/proxy_dmatrix.h"
 #include "../../../src/gbm/gbtree.h"
 #include "../../../src/gbm/gbtree_model.h"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "test_predictor.h"
 
diff --git a/tests/cpp/predictor/test_gpu_predictor.cu b/tests/cpp/predictor/test_gpu_predictor.cu
index fa849cf51bd9..8dacadac5403 100644
--- a/tests/cpp/predictor/test_gpu_predictor.cu
+++ b/tests/cpp/predictor/test_gpu_predictor.cu
@@ -1,7 +1,6 @@
 /*!
  * Copyright 2017-2020 XGBoost contributors
  */
-#include <dmlc/filesystem.h>
 #include <gtest/gtest.h>
 #include <xgboost/c_api.h>
 #include <xgboost/learner.h>
diff --git a/tests/cpp/test_learner.cc b/tests/cpp/test_learner.cc
index 987626df86bf..4a8214e9c5cd 100644
--- a/tests/cpp/test_learner.cc
+++ b/tests/cpp/test_learner.cc
@@ -2,17 +2,18 @@
  * Copyright 2017-2022 by XGBoost contributors
  */
 #include <gtest/gtest.h>
-#include <vector>
-#include <thread>
-#include "helpers.h"
-#include <dmlc/filesystem.h>
-
 #include <xgboost/learner.h>
 #include <xgboost/version_config.h>
-#include "xgboost/json.h"
+
+#include <thread>
+#include <vector>
+
 #include "../../src/common/io.h"
-#include "../../src/common/random.h"
 #include "../../src/common/linalg_op.h"
+#include "../../src/common/random.h"
+#include "filesystem.h"  // dmlc::TemporaryDirectory
+#include "helpers.h"
+#include "xgboost/json.h"
 
 namespace xgboost {
 TEST(Learner, Basic) {
diff --git a/tests/cpp/test_serialization.cc b/tests/cpp/test_serialization.cc
index bf459cf35dcb..d80a7442202e 100644
--- a/tests/cpp/test_serialization.cc
+++ b/tests/cpp/test_serialization.cc
@@ -1,14 +1,16 @@
 // Copyright (c) 2019-2022 by Contributors
 #include <gtest/gtest.h>
-#include <dmlc/filesystem.h>
-#include <string>
-#include <xgboost/learner.h>
-#include <xgboost/data.h>
 #include <xgboost/base.h>
+#include <xgboost/data.h>
 #include <xgboost/json.h>
-#include "helpers.h"
+#include <xgboost/learner.h>
+
+#include <string>
+
 #include "../../src/common/io.h"
 #include "../../src/common/random.h"
+#include "filesystem.h"  // dmlc::TemporaryDirectory
+#include "helpers.h"
 
 namespace xgboost {
 template <typename Array>
diff --git a/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu b/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu
index 9e8cd19bec74..e211fe70a483 100644
--- a/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu
+++ b/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu
@@ -6,8 +6,8 @@
 #include "../../../../src/data/ellpack_page.cuh"
 #include "../../../../src/tree/gpu_hist/gradient_based_sampler.cuh"
 #include "../../../../src/tree/param.h"
+#include "../../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../../helpers.h"
-#include "dmlc/filesystem.h"
 
 namespace xgboost {
 namespace tree {
diff --git a/tests/cpp/tree/test_gpu_hist.cu b/tests/cpp/tree/test_gpu_hist.cu
index f28f723301af..24fe40f42f58 100644
--- a/tests/cpp/tree/test_gpu_hist.cu
+++ b/tests/cpp/tree/test_gpu_hist.cu
@@ -4,22 +4,22 @@
 #include <gtest/gtest.h>
 #include <thrust/device_vector.h>
 #include <thrust/host_vector.h>
-#include <dmlc/filesystem.h>
 #include <xgboost/base.h>
+
 #include <random>
 #include <string>
 #include <vector>
 
+#include "../../../src/common/common.h"
+#include "../../../src/data/sparse_page_source.h"
+#include "../../../src/tree/constraints.cuh"
+#include "../../../src/tree/updater_gpu_common.cuh"
+#include "../../../src/tree/updater_gpu_hist.cu"
+#include "../filesystem.h"  // dmlc::TemporaryDirectory
 #include "../helpers.h"
 #include "../histogram_helpers.h"
-
 #include "xgboost/generic_parameters.h"
 #include "xgboost/json.h"
-#include "../../../src/data/sparse_page_source.h"
-#include "../../../src/tree/updater_gpu_hist.cu"
-#include "../../../src/tree/updater_gpu_common.cuh"
-#include "../../../src/common/common.h"
-#include "../../../src/tree/constraints.cuh"
 
 namespace xgboost {
 namespace tree {
diff --git a/tests/cpp/tree/test_tree_model.cc b/tests/cpp/tree/test_tree_model.cc
index fb14e300c0f4..65957255bf38 100644
--- a/tests/cpp/tree/test_tree_model.cc
+++ b/tests/cpp/tree/test_tree_model.cc
@@ -1,11 +1,12 @@
 // Copyright by Contributors
 #include <gtest/gtest.h>
+
+#include "../../../src/common/bitfield.h"
+#include "../../../src/common/categorical.h"
+#include "../filesystem.h"
 #include "../helpers.h"
-#include "dmlc/filesystem.h"
 #include "xgboost/json_io.h"
 #include "xgboost/tree_model.h"
-#include "../../../src/common/bitfield.h"
-#include "../../../src/common/categorical.h"
 
 namespace xgboost {
 TEST(Tree, ModelShape) {