[CI] Fix S3 uploads (#8069)

* [CI] Fix S3 upload issues * Don't launch Docker containers when uploading to S3
dmlc · Jul 13, 2022 · 36cf979 · 36cf979
1 parent abaa593
commit 36cf979
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 8 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -180,7 +180,7 @@ def BuildCPUARM64() {
     if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
       echo 'Uploading Python wheel...'
       sh """
-      ${dockerRun} ${container_type} ${docker_binary} bash -c "source activate aarch64_test && python -m awscli s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress"
+      python3 -m awscli s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress
       """
     }
     stash name: 'xgboost_cli_arm64', includes: 'xgboost'
@@ -234,7 +234,7 @@ def BuildCUDA(args) {
     if (args.cuda_version == ref_cuda_ver && (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release'))) {
       echo 'Uploading Python wheel...'
       sh """
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python -m awscli s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress
+      python3 -m awscli s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress
       """
     }
     echo 'Stashing C++ test executable (testxgboost)...'
@@ -271,7 +271,7 @@ def BuildRPackageWithCUDA(args) {
       """
       echo 'Uploading R tarball...'
       sh """
-      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python -m awscli s3 cp xgboost_r_gpu_linux_*.tar.gz s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress
+      python3 -m awscli s3 cp xgboost_r_gpu_linux_*.tar.gz s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress
       """
     }
     deleteDir()
@@ -329,7 +329,7 @@ def BuildJVMDoc() {
     if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
       echo 'Uploading doc...'
       sh """
-      ${dockerRun} ${container_type} ${docker_binary} python -m awscli s3 cp jvm-packages/${BRANCH_NAME}.tar.bz2 s3://xgboost-docs/${BRANCH_NAME}.tar.bz2 --acl public-read --no-progress
+      python3 -m awscli s3 cp jvm-packages/${BRANCH_NAME}.tar.bz2 s3://xgboost-docs/${BRANCH_NAME}.tar.bz2 --acl public-read --no-progress
       """
     }
     deleteDir()

diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64
@@ -107,8 +107,10 @@ def BuildWin64() {
     stash name: 'xgboost_whl', includes: 'python-package/dist/*.whl'
     if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('release')) {
       echo 'Uploading Python wheel...'
-      path = "${BRANCH_NAME}/"
-      s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', workingDir: 'python-package/dist', includePathPattern:'**/*.whl'
+      bat """
+      cd python-package
+      conda activate && for /R %%i in (dist\\*.whl) DO python -m awscli s3 cp "%%i" s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress
+      """
     }
     echo 'Stashing C++ test executable (testxgboost)...'
     stash name: 'xgboost_cpp_tests', includes: 'build/testxgboost.exe'
@@ -127,8 +129,9 @@ def BuildRPackageWithCUDAWin64() {
       bash tests/ci_build/build_r_pkg_with_cuda_win64.sh ${commit_id}
       """
       echo 'Uploading R tarball...'
-      path = "${BRANCH_NAME}/"
-      s3Upload bucket: 'xgboost-nightly-builds', path: path, acl: 'PublicRead', includePathPattern:'xgboost_r_gpu_win64_*.tar.gz'
+      bat """
+      conda activate && for /R %%i in (xgboost_r_gpu_win64_*.tar.gz) DO python -m awscli s3 cp "%%i" s3://xgboost-nightly-builds/${BRANCH_NAME}/ --acl public-read --no-progress
+      """
     }
     deleteDir()
   }