From f05c92ee101bdc7c7d7f51b56a5d3d3b2c279699 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Tue, 6 Dec 2022 17:42:15 +0800 Subject: [PATCH] [CI] Fix github action mismatched glibcxx. (#8551) * [CI] Fix github action mismatched glibcxx. Split up the Linux test to use the toolchain from conda forge. --- .github/workflows/main.yml | 14 +-- .github/workflows/python_tests.yml | 113 ++++++++++++++++-------- tests/ci_build/conda_env/cpp_test.yml | 11 +++ tests/ci_build/conda_env/sdist_test.yml | 13 +++ 4 files changed, 105 insertions(+), 46 deletions(-) create mode 100644 tests/ci_build/conda_env/cpp_test.yml create mode 100644 tests/ci_build/conda_env/sdist_test.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index abe4b2b71dc8..7b17f9b8c66c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -75,19 +75,18 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'true' - - name: Install system packages - run: | - sudo apt-get install -y --no-install-recommends ninja-build - - uses: conda-incubator/setup-miniconda@v2 + - uses: mamba-org/provision-with-micromamba@f347426e5745fe3dfc13ec5baf20496990d0281f # v14 with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} - activate-environment: test + cache-downloads: true + cache-env: true + environment-name: cpp_test + environment-file: tests/ci_build/conda_env/cpp_test.yml - name: Display Conda env shell: bash -l {0} run: | conda info conda list + - name: Build and install XGBoost static library shell: bash -l {0} run: | @@ -109,6 +108,7 @@ jobs: cd .. rm -rf ./build popd + - name: Build and install XGBoost shared library shell: bash -l {0} run: | diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 7ad742045223..b653362758e6 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -41,12 +41,46 @@ jobs: run: | python tests/ci_build/lint_python.py --format=0 --type-check=0 --pylint=1 + python-sdist-test-on-Linux: + # Mismatched glibcxx version between system and conda forge. + runs-on: ${{ matrix.os }} + name: Test installing XGBoost Python source package on ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + steps: + - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 + with: + submodules: 'true' + - uses: mamba-org/provision-with-micromamba@f347426e5745fe3dfc13ec5baf20496990d0281f # v14 + with: + cache-downloads: true + cache-env: true + environment-name: sdist_test + environment-file: tests/ci_build/conda_env/sdist_test.yml + - name: Display Conda env + shell: bash -l {0} + run: | + conda info + conda list + - name: Build and install XGBoost + shell: bash -l {0} + run: | + cd python-package + python --version + python setup.py sdist + pip install -v ./dist/xgboost-*.tar.gz + cd .. + python -c 'import xgboost' + python-sdist-test: + # Use system toolchain instead of conda toolchain for macos and windows. + # MacOS has linker error if clang++ from conda-forge is used runs-on: ${{ matrix.os }} name: Test installing XGBoost Python source package on ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-11, windows-latest] + os: [macos-11, windows-latest] python-version: ["3.8"] steps: - uses: actions/checkout@v2 @@ -56,11 +90,7 @@ jobs: if: matrix.os == 'macos-11' run: | brew install ninja libomp - - name: Install Ubuntu system dependencies - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get install -y --no-install-recommends ninja-build - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@35d1405e78aa3f784fe3ce9a2eb378d5eeb62169 # v2.1.1 with: auto-update-conda: true python-version: ${{ matrix.python-version }} @@ -80,25 +110,26 @@ jobs: cd .. python -c 'import xgboost' - python-tests-on-win: + python-tests-on-macos: name: Test XGBoost Python package on ${{ matrix.config.os }} runs-on: ${{ matrix.config.os }} + timeout-minutes: 60 strategy: matrix: config: - - {os: windows-latest, python-version: '3.8'} + - {os: macos-11} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 with: submodules: 'true' - - uses: conda-incubator/setup-miniconda@v2 + - uses: mamba-org/provision-with-micromamba@f347426e5745fe3dfc13ec5baf20496990d0281f # v14 with: - auto-update-conda: true - python-version: ${{ matrix.config.python-version }} - activate-environment: win64_env - environment-file: tests/ci_build/conda_env/win64_cpu_test.yml + cache-downloads: true + cache-env: true + environment-name: macos_test + environment-file: tests/ci_build/conda_env/macos_cpu_test.yml - name: Display Conda env shell: bash -l {0} @@ -106,35 +137,43 @@ jobs: conda info conda list - - name: Build XGBoost on Windows + - name: Build XGBoost on macos shell: bash -l {0} run: | - mkdir build_msvc - cd build_msvc - cmake .. -G"Visual Studio 17 2022" -DCMAKE_CONFIGURATION_TYPES="Release" -A x64 -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON - cmake --build . --config Release --parallel $(nproc) + brew install ninja + + mkdir build + cd build + # Set prefix, to use OpenMP library from Conda env + # See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025038228 + # to learn why we don't use libomp from Homebrew. + cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX + ninja - name: Install Python package shell: bash -l {0} run: | cd python-package python --version - python setup.py bdist_wheel --universal - pip install ./dist/*.whl + python setup.py install - name: Test Python package shell: bash -l {0} run: | - pytest -s -v ./tests/python + pytest -s -v -rxXs --durations=0 ./tests/python - python-tests-on-macos: + - name: Test Dask Interface + shell: bash -l {0} + run: | + pytest -s -v -rxXs --durations=0 ./tests/test_distributed/test_with_dask + + python-tests-on-win: name: Test XGBoost Python package on ${{ matrix.config.os }} runs-on: ${{ matrix.config.os }} - timeout-minutes: 90 strategy: matrix: config: - - {os: macos-11, python-version "3.8" } + - {os: windows-latest, python-version: '3.8'} steps: - uses: actions/checkout@v2 @@ -145,8 +184,8 @@ jobs: with: auto-update-conda: true python-version: ${{ matrix.config.python-version }} - activate-environment: macos_test - environment-file: tests/ci_build/conda_env/macos_cpu_test.yml + activate-environment: win64_env + environment-file: tests/ci_build/conda_env/win64_cpu_test.yml - name: Display Conda env shell: bash -l {0} @@ -154,27 +193,23 @@ jobs: conda info conda list - - name: Build XGBoost on macos + - name: Build XGBoost on Windows shell: bash -l {0} run: | - brew install ninja - - mkdir build - cd build - # Set prefix, to use OpenMP library from Conda env - # See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025038228 - # to learn why we don't use libomp from Homebrew. - cmake .. -GNinja -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_PREFIX_PATH=$CONDA_PREFIX - ninja + mkdir build_msvc + cd build_msvc + cmake .. -G"Visual Studio 17 2022" -DCMAKE_CONFIGURATION_TYPES="Release" -A x64 -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON + cmake --build . --config Release --parallel $(nproc) - name: Install Python package shell: bash -l {0} run: | cd python-package python --version - python setup.py install + python setup.py bdist_wheel --universal + pip install ./dist/*.whl - name: Test Python package shell: bash -l {0} run: | - pytest -s -v ./tests/python + pytest -s -v -rxXs --durations=0 ./tests/python diff --git a/tests/ci_build/conda_env/cpp_test.yml b/tests/ci_build/conda_env/cpp_test.yml new file mode 100644 index 000000000000..80ce65cc1cea --- /dev/null +++ b/tests/ci_build/conda_env/cpp_test.yml @@ -0,0 +1,11 @@ +# conda environment for CPP test on Linux distributions +name: cpp_test +channels: +- defaults +- conda-forge +dependencies: +- cmake +- ninja +- c-compiler +- cxx-compiler +- gtest diff --git a/tests/ci_build/conda_env/sdist_test.yml b/tests/ci_build/conda_env/sdist_test.yml new file mode 100644 index 000000000000..fc1e710973b2 --- /dev/null +++ b/tests/ci_build/conda_env/sdist_test.yml @@ -0,0 +1,13 @@ +# conda environment for source distribution test. +name: sdist_test +channels: +- defaults +- conda-forge +dependencies: +- python=3.8 +- pip +- wheel +- cmake +- ninja +- c-compiler +- cxx-compiler