From 8596b89106691ea4145072455b701376219870dd Mon Sep 17 00:00:00 2001 From: Weichen Xu Date: Tue, 13 Sep 2022 10:47:40 +0800 Subject: [PATCH] remove debug logging Signed-off-by: Weichen Xu --- python-package/xgboost/spark/core.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/python-package/xgboost/spark/core.py b/python-package/xgboost/spark/core.py index ef46fe2e72d9..35c3f3da0558 100644 --- a/python-package/xgboost/spark/core.py +++ b/python-package/xgboost/spark/core.py @@ -684,16 +684,6 @@ def _fit(self, dataset): num_workers, ) - def log_partition_rows(df, msg): - def count_partition_rows(iter): - yield len(list(iter)) - - result = df.rdd.mapPartitions(count_partition_rows).collect() - get_logger(self.__class__.__name__).warning( - f"debug-repartition: {msg}: {str(list(result))}\n" - ) - - log_partition_rows(dataset, "before-repartition") if self._repartition_needed(dataset) or ( self.isDefined(self.validationIndicatorCol) and self.getOrDefault(self.validationIndicatorCol) @@ -706,7 +696,7 @@ def count_partition_rows(iter): # result unbalance. Directly using `.repartition(N)` might result in some # empty partitions. dataset = dataset.repartition(num_workers, rand(1)) - log_partition_rows(dataset, "after-repartition") + train_params = self._get_distributed_train_params(dataset) booster_params, train_call_kwargs_params = self._get_xgb_train_call_args( train_params