diff --git a/tests/python-gpu/conftest.py b/tests/python-gpu/conftest.py
index 789f96fc5511..c97b1a1668bf 100644
--- a/tests/python-gpu/conftest.py
+++ b/tests/python-gpu/conftest.py
@@ -61,8 +61,8 @@ def pytest_collection_modifyitems(config, items):
     mgpu_mark = pytest.mark.mgpu
     for item in items:
         if item.nodeid.startswith(
-            "python-gpu/test_gpu_with_dask.py"
+            "python-gpu/test_gpu_with_dask/test_gpu_with_dask.py"
         ) or item.nodeid.startswith(
-            "python-gpu/test_spark_with_gpu/test_spark_with_gpu.py"
+            "python-gpu/test_gpu_spark/test_gpu_spark.py"
         ):
             item.add_marker(mgpu_mark)
diff --git a/tests/python-gpu/test_spark_with_gpu/discover_gpu.sh b/tests/python-gpu/test_gpu_spark/discover_gpu.sh
similarity index 100%
rename from tests/python-gpu/test_spark_with_gpu/discover_gpu.sh
rename to tests/python-gpu/test_gpu_spark/discover_gpu.sh
diff --git a/tests/python-gpu/test_spark_with_gpu/test_data.py b/tests/python-gpu/test_gpu_spark/test_data.py
similarity index 100%
rename from tests/python-gpu/test_spark_with_gpu/test_data.py
rename to tests/python-gpu/test_gpu_spark/test_data.py
diff --git a/tests/python-gpu/test_spark_with_gpu/test_spark_with_gpu.py b/tests/python-gpu/test_gpu_spark/test_gpu_spark.py
similarity index 98%
rename from tests/python-gpu/test_spark_with_gpu/test_spark_with_gpu.py
rename to tests/python-gpu/test_gpu_spark/test_gpu_spark.py
index 14b32b8d3511..ef961ab34bd5 100644
--- a/tests/python-gpu/test_spark_with_gpu/test_spark_with_gpu.py
+++ b/tests/python-gpu/test_gpu_spark/test_gpu_spark.py
@@ -17,7 +17,7 @@
 from pyspark.sql import SparkSession
 from xgboost.spark import SparkXGBClassifier, SparkXGBRegressor
 
-gpu_discovery_script_path = "tests/python-gpu/test_spark_with_gpu/discover_gpu.sh"
+gpu_discovery_script_path = "tests/python-gpu/test_gpu_spark/discover_gpu.sh"
 executor_gpu_amount = 4
 executor_cores = 4
 num_workers = executor_gpu_amount
diff --git a/tests/python-gpu/test_gpu_with_dask.py b/tests/python-gpu/test_gpu_with_dask/test_gpu_with_dask.py
similarity index 100%
rename from tests/python-gpu/test_gpu_with_dask.py
rename to tests/python-gpu/test_gpu_with_dask/test_gpu_with_dask.py