Update to CMake 3.13 for better CUDA support and to enable build conc…

…urrency (#3261)

Signed-off-by: Max H. Gerlach <git@maxgerlach.de>

CHANGELOG.md

@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
+- Moved to CMake version 3.13 with first-class CUDA language support and re-enabled parallelized builds. ([#3261](https://github.com/horovod/horovod/pull/3261))
+
 ### Deprecated
 - Deprecated ElasticRayExecutor APIs in favor of the new RayExecutor API for issue: [#3190](https://github.com/horovod/horovod/issues/3190).
 ### Removed

CMakeLists.txt

@@ -1,19 +1,17 @@
-cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 if(POLICY CMP0074)
     # 1. Introduced with 3.12.4.
     # 2. *_ROOT variables will be checked
     cmake_policy(SET CMP0074 NEW)
 endif()
 
-if(${CMAKE_VERSION} VERSION_GREATER "3.4.0") 
-  find_program(CCACHE_PROGRAM ccache)
-  if(CCACHE_PROGRAM)
+find_program(CCACHE_PROGRAM ccache)
+if(CCACHE_PROGRAM)
     set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
     set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
     set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
-  else()
+else()
     message(STATUS "Could not find CCache. Consider installing CCache to speed up compilation.")
-  endif()
 endif()
 
 project(horovod CXX)
@@ -23,7 +21,10 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
 # Configure path to modules (for find_package)
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake/Modules/")
+set(CMAKE_MODULE_PATH
+        ${CMAKE_MODULE_PATH}
+        "${PROJECT_SOURCE_DIR}/cmake/Modules/"
+        "${PROJECT_SOURCE_DIR}/cmake/upstream/")
 include(cmake/Utilities.cmake)
 
 create_metadata()
@@ -148,14 +149,33 @@ if (NOT "$ENV{HOROVOD_WITHOUT_MPI}" STREQUAL "1")
 endif()
 
 # CUDA and ROCM
+set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+if(NOT DEFINED CMAKE_CUDA_RUNTIME_LIBRARY)
+    set(CMAKE_CUDA_RUNTIME_LIBRARY "Shared")  # Set to "Static" or "Shared"
+endif()
+if (DEFINED ENV{HOROVOD_CUDA_HOME})
+    set(CMAKE_CUDA_COMPILER $ENV{HOROVOD_CUDA_HOME}/bin/nvcc)
+endif()
+include(CheckLanguage)
+check_language(CUDA)
+if (CMAKE_CUDA_COMPILER)
+    if ((CMAKE_CXX_COMPILER_ID MATCHES GNU) AND (CMAKE_SYSTEM_PROCESSOR MATCHES ppc64le))
+        if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0)
+            set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -std=c++11")
+        endif ()
+    endif ()
+    enable_language(CUDA)
+endif ()
+
 macro(ADD_CUDA)
-    if (DEFINED ENV{HOROVOD_CUDA_HOME})
-        set(CUDA_TOOLKIT_ROOT_DIR $ENV{HOROVOD_CUDA_HOME})
+    find_package(CUDAToolkit REQUIRED)
+    include_directories(SYSTEM ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+    string(TOLOWER "${CMAKE_CUDA_RUNTIME_LIBRARY}" lowercase_CMAKE_CUDA_RUNTIME_LIBRARY)
+    if (lowercase_CMAKE_CUDA_RUNTIME_LIBRARY STREQUAL "static")
+        list(APPEND LINKER_LIBS CUDA::cudart_static)
+    elseif (lowercase_CMAKE_CUDA_RUNTIME_LIBRARY STREQUAL "shared")
+        list(APPEND LINKER_LIBS CUDA::cudart)
     endif()
-    option(CUDA_USE_STATIC_CUDA_RUNTIME "Use the static version of the CUDA runtime library if available" OFF)
-    find_package(CUDA REQUIRED)
-    include_directories(SYSTEM ${CUDA_INCLUDE_DIRS})
-    list(APPEND LINKER_LIBS ${CUDA_LIBRARIES})
     list(APPEND SOURCES "${PROJECT_SOURCE_DIR}/horovod/common/ops/cuda_operations.cc"
                         "${PROJECT_SOURCE_DIR}/horovod/common/ops/gpu_operations.cc")
     # CUDA + MPI
@@ -209,7 +229,7 @@ endif()
 if(HOROVOD_GPU_ALLREDUCE STREQUAL "D")
     message(DEPRECATION "DDL backend has been deprecated. Please, start using the NCCL backend by building Horovod with "
                         "'HOROVOD_GPU_OPERATIONS=NCCL'. Will be removed in v0.21.0.")
-    list(APPEND LINKER_LIBS "${CUDA_TOOLKIT_ROOT_DIR}/lib/libddl.so" "${CUDA_TOOLKIT_ROOT_DIR}/lib/libddl_pack.so")
+    list(APPEND LINKER_LIBS "${CUDAToolkit_LIBRARY_ROOT}/lib/libddl.so" "${CUDAToolkit_LIBRARY_ROOT}/lib/libddl_pack.so")
     list(APPEND SOURCES "${PROJECT_SOURCE_DIR}/horovod/common/mpi/ddl_mpi_context_manager.cc"
                         "${PROJECT_SOURCE_DIR}/horovod/common/ops/ddl_operations.cc")
     add_definitions(-DHAVE_DDL=1)

Dockerfile.test.cpu

@@ -33,7 +33,6 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test
 RUN apt-get update -qq && apt-get install -y --no-install-recommends \
         wget \
         ca-certificates \
-        cmake \
         openssh-client \
         openssh-server \
         git \
@@ -52,6 +51,9 @@ RUN ln -s -f /usr/bin/python${PYTHON_VERSION} /usr/bin/python${PYTHON_VERSION/%.
 RUN wget --progress=dot:mega https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm get-pip.py
 RUN pip install --no-cache-dir -U --force pip setuptools requests pytest mock pytest-forked parameterized
 
+# Install recent CMake.
+RUN pip install --no-cache-dir -U cmake~=3.13.0
+
 # Add launch helper scripts
 RUN echo "env SPARK_HOME=/spark SPARK_DRIVER_MEM=512m PYSPARK_PYTHON=/usr/bin/python${PYTHON_VERSION} PYSPARK_DRIVER_PYTHON=/usr/bin/python${PYTHON_VERSION} \"\$@\"" > /spark_env.sh
 RUN echo /spark_env.sh pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.\$1.\${HOROVOD_RANK:-\${OMPI_COMM_WORLD_RANK:-\${PMI_RANK}}}.\$2.xml \${@:2} > /pytest.sh

Dockerfile.test.gpu

@@ -35,7 +35,6 @@ RUN CUDNN_MAJOR=$(cut -d '.' -f 1 <<< "${CUDNN_VERSION}"); \
     apt-get update -qq && apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
         wget \
         ca-certificates \
-        cmake \
         openssh-client \
         openssh-server \
         git \
@@ -57,6 +56,9 @@ RUN ln -s -f /usr/bin/python${PYTHON_VERSION} /usr/bin/python${PYTHON_VERSION/%.
 RUN wget --progress=dot:mega https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm get-pip.py
 RUN pip install --no-cache-dir -U --force pip "setuptools<60.1.0" requests pytest mock pytest-forked parameterized
 
+# Install recent CMake.
+RUN pip install --no-cache-dir -U cmake~=3.13.0
+
 # Add launch helper scripts
 RUN echo "env SPARK_HOME=/spark SPARK_DRIVER_MEM=512m PYSPARK_PYTHON=/usr/bin/python${PYTHON_VERSION} PYSPARK_DRIVER_PYTHON=/usr/bin/python${PYTHON_VERSION} \"\$@\"" > /spark_env.sh
 RUN echo /spark_env.sh pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.\$1.\${HOROVOD_RANK:-\${OMPI_COMM_WORLD_RANK:-\${PMI_RANK}}}.\$2.xml \${@:2} > /pytest.sh

NOTICE

@@ -232,3 +232,38 @@
    The derived work can be found in the files:
 
         - horovod/torch/sync_batch_norm.py
+
+   CMake - Cross Platform Makefile Generator
+   Copyright 2000-2020 Kitware, Inc. and Contributors
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   * Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+
+   * Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+   * Neither the name of Kitware, Inc. nor the names of Contributors
+     may be used to endorse or promote products derived from this
+     software without specific prior written permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+      The derived work can be found in the files:
+
+          - cmake/upstream/FindCUDAToolkit.cmake

cmake/Modules/FindNCCL.cmake

@@ -11,14 +11,14 @@
 #  NCCL_LIBRARIES
 #  NCCL_MAJOR_VERSION
 #
-# The path hints include CUDA_TOOLKIT_ROOT_DIR seeing as some folks
+# The path hints include CUDAToolkit_LIBRARY_ROOT seeing as some folks
 # install NCCL in the same location as the CUDA toolkit.
 
 set(HOROVOD_NCCL_HOME $ENV{HOROVOD_NCCL_HOME} CACHE PATH "Folder contains NVIDIA NCCL")
 set(HOROVOD_NCCL_INCLUDE $ENV{HOROVOD_NCCL_INCLUDE} CACHE PATH "Folder contains NVIDIA NCCL headers")
 set(HOROVOD_NCCL_LIB $ENV{HOROVOD_NCCL_LIB} CACHE PATH "Folder contains NVIDIA NCCL libraries")
 
-list(APPEND NCCL_ROOT ${HOROVOD_NCCL_HOME} ${CUDA_TOOLKIT_ROOT_DIR})
+list(APPEND NCCL_ROOT ${HOROVOD_NCCL_HOME} ${CUDAToolkit_LIBRARY_ROOT})
 # Compatible layer for CMake <3.12. NCCL_ROOT will be accounted in for searching paths and libraries for CMake >=3.12.
 list(APPEND CMAKE_PREFIX_PATH ${NCCL_ROOT})
 

cmake/Modules/FindNVTX.cmake

@@ -10,13 +10,13 @@
 
 set(HOROVOD_NVTX_INCLUDE $ENV{HOROVOD_NVTX_INCLUDE} CACHE PATH "Folder containing NVIDIA NVTX3 headers")
 
-list(APPEND NVTX_ROOT ${CUDA_TOOLKIT_ROOT_DIR})
+list(APPEND NVTX_ROOT ${CUDAToolkit_LIBRARY_ROOT})
 # Compatible layer for CMake <3.12:
 list(APPEND CMAKE_PREFIX_PATH ${NVTX_ROOT})
 
 find_path(NVTX_INCLUDE_DIR
-        NAMES nvtx3/nvToolsExt.h
-        HINTS ${HOROVOD_NVTX_INCLUDE})
+          NAMES nvtx3/nvToolsExt.h
+          HINTS ${HOROVOD_NVTX_INCLUDE})
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(NVTX DEFAULT_MSG NVTX_INCLUDE_DIR)

cmake/Modules/FindTensorflow.cmake

@@ -22,7 +22,7 @@ if (LEN EQUAL "4")
     string(REPLACE " " ";" Tensorflow_LIBRARIES_LIST "${Tensorflow_LIBRARIES}")
     list(GET Tensorflow_LIBRARIES_LIST 0 Tensorflow_LIB_PATH_ARGUMENT)
     string(REGEX REPLACE "^-L" "" Tensorflow_LIB_PATH ${Tensorflow_LIB_PATH_ARGUMENT})
-    if (Tensorflow_VERSION VERSION_GREATER "2.6" OR Tensorflow_VERSION VERSION_EQUAL "2.6")
+    if (Tensorflow_VERSION VERSION_GREATER_EQUAL "2.6")
         # XLA implementations and helpers for resource variables are in _pywrap_tensorflow_internal.so
         set(Tensorflow_LIBRARIES "${Tensorflow_LIBRARIES} ${Tensorflow_LIB_PATH}/python/_pywrap_tensorflow_internal.so")
     endif()

cmake/build_utils.py

@@ -91,7 +91,7 @@ def get_nvcc_bin():
                        'Make sure it is added to your path or in $HOROVOD_CUDA_HOME/bin.')
 
 def get_nvcc_flags():
-    default_flags = ['--std=c++11', '-O3', '-Xcompiler', '-fPIC']
+    default_flags = ['-O3', '-Xcompiler', '-fPIC']
     cc_list_env = os.environ.get('HOROVOD_BUILD_CUDA_CC_LIST')
 
     # Invoke nvcc and extract all supported compute capabilities for CUDA toolkit version