[llvm-core] add support for job pool resource limits from @jusito

recipes/llvm-core/all/conandata.yml

@@ -10,4 +10,11 @@ sources:
     sha256: ce8508e318a01a63d4e8b3090ab2ded3c598a50258cc49e2625b9120d4c03ea5
 
 patches:
+  "13.0.0":
     - patch_file: "patches/13x/00-cmake-target-props.patch"
+    - patch_file: "patches/13x/01-calculate-job-pools.patch"
+  "12.0.0":
+    - patch_file: "patches/12x/00-cmake-target-props.patch"
+    - patch_file: "patches/12x/01-calculate-job-pools.patch"
+  "11.1.0":
+    - patch_file: "patches/11x/00-calculate-job-pools.patch"

recipes/llvm-core/all/conanfile.py

@@ -54,7 +54,10 @@ class LLVMCoreConan(ConanFile):
         'with_terminfo': [True, False],
         'with_zlib': [True, False],
         'with_xml2': [True, False],
+        'with_z3': [True, False],
         'use_llvm_cmake_files': [True, False],
+        'ram_per_compile_job': ['ANY'],
+        'ram_per_link_job': ['ANY'],
     }
     default_options = {
         'shared': False,
@@ -71,10 +74,14 @@ class LLVMCoreConan(ConanFile):
         'use_perf': False,
         'use_sanitizer': 'None',
         'with_ffi': False,
-        'with_zlib': True,
+        'with_terminfo': False, # differs from LLVM default
         'with_xml2': True,
+        'with_z3': True,
+        'with_zlib': True,
         'use_llvm_cmake_files': False,
-        'with_terminfo': False # differs from LLVM default
+        # creating job pools with current free memory
+        'ram_per_compile_job': '2000',
+        'ram_per_link_job': '14000'
     }
 
     @property
@@ -143,7 +150,8 @@ def source(self):
 
     def generate(self):
         tc = CMakeToolchain(self)
-        # See https://releases.llvm.org/13.0.0/docs/CMake.html
+        # https://releases.llvm.org/12.0.0/docs/CMake.html
+        # https://releases.llvm.org/13.0.0/docs/CMake.html
         cmake_definitions = {
             'LLVM_TARGETS_TO_BUILD': self.options.targets,
             'LLVM_BUILD_LLVM_DYLIB': self.options.shared,
@@ -163,11 +171,13 @@ def generate(self):
             'LLVM_ENABLE_EXPENSIVE_CHECKS': self.options.expensive_checks,
             'LLVM_ENABLE_ASSERTIONS': self.settings.build_type,
             'LLVM_USE_PERF': self.options.use_perf,
-            'LLVM_ENABLE_Z3_SOLVER': True,
+            'LLVM_ENABLE_Z3_SOLVER': self.options.with_z3,
             'LLVM_ENABLE_FFI': self.options.with_ffi,
             'LLVM_ENABLE_ZLIB': "FORCE_ON" if self.options.with_zlib else False,
-            'LLVM_ENABLE_LIBXML2': self.options.with_xml2,
-            'LLVM_ENABLE_TERMINFO': self.options.with_terminfo
+            'LLVM_ENABLE_LIBXML2': "FORCE_ON" if self.options.with_xml2 else False,
+            'LLVM_ENABLE_TERMINFO': self.options.with_terminfo,
+            'LLVM_RAM_PER_COMPILE_JOB': self.options.ram_per_compile_job,
+            'LLVM_RAM_PER_LINK_JOB': self.options.ram_per_link_job
         }
         if is_msvc(self):
             build_type = str(self.settings.build_type).upper()
@@ -211,15 +221,17 @@ def build(self):
         cmake.configure(cli_args=["--graphviz=graph/llvm.dot"])
         cmake.build()
 
-    def package_id(self):
-        del self.info.options.use_llvm_cmake_files
-
     @property
     def _is_windows(self):
         return self.settings.os == 'Windows'
 
     def _llvm_components(self):
+        # TODO (@planetmarshall) this is a bit hacky. CMake already has this information, just
+        #  parse the LLVM CMake files.
         # The definitive list of built targets is provided by running `llvm-config --components`
+        non_distributed = {
+            "exegesis"
+        }
         graphviz_folder = Path(self.build_folder) / "graph"
         match_component = re.compile(r"""^llvm.dot.LLVM(.+)\.dependers$""")
         for lib in iglob(str(graphviz_folder / '*')):
@@ -229,7 +241,9 @@ def _llvm_components(self):
             match = match_component.match(lib_name)
             if match:
                 component = match.group(1)
-                yield component.lower(), f"LLVM{component}"
+                component_name = component.lower()
+                if component_name not in non_distributed:
+                    yield component.lower(), f"LLVM{component}"
 
     @property
     def _components_data_file(self):
@@ -256,6 +270,11 @@ def package(self):
         if not self.options.shared:
             self._write_components()
 
+    def package_id(self):
+        del self.info.options.use_llvm_cmake_files
+        del self.info.options.ram_per_compile_job
+        del self.info.options.ram_per_link_job
+
     def package_info(self):
         self.cpp_info.set_property("cmake_file_name", "LLVM")
 

recipes/llvm-core/all/patches/11x/00-calculate-job-pools.patch

@@ -0,0 +1,52 @@
+diff --git a/cmake/modules/HandleLLVMOptions.cmake b/cmake/modules/HandleLLVMOptions.cmake
+index 5ef22eb493b..7078ae8bc86 100644
+--- a/cmake/modules/HandleLLVMOptions.cmake
++++ b/cmake/modules/HandleLLVMOptions.cmake
+@@ -25,8 +25,24 @@ string(TOUPPER "${LLVM_ENABLE_LTO}" uppercase_LLVM_ENABLE_LTO)
+
+ # Ninja Job Pool support
+ # The following only works with the Ninja generator in CMake >= 3.0.
++if (NOT LLVM_RAM_PER_COMPILE_JOB)
++  set(LLVM_RAM_PER_COMPILE_JOB "2000")
++endif()
+ set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
+   "Define the maximum number of concurrent compilation jobs (Ninja only).")
++cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY)
++cmake_host_system_information(RESULT NUMBER_OF_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)
++if(LLVM_RAM_PER_COMPILE_JOB)
++  math(EXPR memory_available_jobs "${AVAILABLE_PHYSICAL_MEMORY} / ${LLVM_RAM_PER_COMPILE_JOB}" OUTPUT_FORMAT DECIMAL)
++  if (memory_available_jobs LESS 1)
++    set(memory_available_jobs 1)
++  endif()
++  if (memory_available_jobs LESS NUMBER_OF_LOGICAL_CORES)
++    set(LLVM_PARALLEL_COMPILE_JOBS "${memory_available_jobs}")
++  else()
++    set(LLVM_PARALLEL_COMPILE_JOBS "${NUMBER_OF_LOGICAL_CORES}")
++  endif()
++endif()
+ if(LLVM_PARALLEL_COMPILE_JOBS)
+   if(NOT CMAKE_GENERATOR STREQUAL "Ninja")
+     message(WARNING "Job pooling is only available with Ninja generators.")
+@@ -36,8 +52,22 @@ if(LLVM_PARALLEL_COMPILE_JOBS)
+   endif()
+ endif()
+
++if (NOT LLVM_RAM_PER_LINK_JOB)
++  set(LLVM_RAM_PER_LINK_JOB "14000")
++endif()
+ set(LLVM_PARALLEL_LINK_JOBS "" CACHE STRING
+   "Define the maximum number of concurrent link jobs (Ninja only).")
++if(LLVM_RAM_PER_LINK_JOB)
++  math(EXPR memory_available_jobs "${AVAILABLE_PHYSICAL_MEMORY} / ${LLVM_RAM_PER_LINK_JOB}" OUTPUT_FORMAT DECIMAL)
++  if (memory_available_jobs LESS 1)
++    set(memory_available_jobs 1)
++  endif()
++  if (memory_available_jobs LESS NUMBER_OF_LOGICAL_CORES)
++    set(LLVM_PARALLEL_LINK_JOBS "${memory_available_jobs}")
++  else()
++    set(LLVM_PARALLEL_LINK_JOBS "${NUMBER_OF_LOGICAL_CORES}")
++  endif()
++endif()
+ if(CMAKE_GENERATOR STREQUAL "Ninja")
+   if(NOT LLVM_PARALLEL_LINK_JOBS AND uppercase_LLVM_ENABLE_LTO STREQUAL "THIN")
+     message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.")

recipes/llvm-core/all/patches/12x/00-cmake-target-props.patch

@@ -0,0 +1,20 @@
+diff --git a/lib/Support/CMakeLists.txt b/lib/Support/CMakeLists.txt
+index cdee11412eb..f32a7b7909d 100644
+--- a/lib/Support/CMakeLists.txt
++++ b/lib/Support/CMakeLists.txt
+@@ -236,14 +236,7 @@ set(llvm_system_libs ${system_libs})
+ # This block is only needed for llvm-config. When we deprecate llvm-config and
+ # move to using CMake export, this block can be removed.
+ if(LLVM_ENABLE_ZLIB)
+-  # CMAKE_BUILD_TYPE is only meaningful to single-configuration generators.
+-  if(CMAKE_BUILD_TYPE)
+-    string(TOUPPER ${CMAKE_BUILD_TYPE} build_type)
+-    get_property(zlib_library TARGET ZLIB::ZLIB PROPERTY LOCATION_${build_type})
+-  endif()
+-  if(NOT zlib_library)
+-    get_property(zlib_library TARGET ZLIB::ZLIB PROPERTY LOCATION)
+-  endif()
++  set(zlib_library ${ZLIB_LIBRARIES})
+   get_library_name(${zlib_library} zlib_library)
+   set(llvm_system_libs ${llvm_system_libs} "${zlib_library}")
+ endif()

recipes/llvm-core/all/patches/12x/01-calculate-job-pools.patch

@@ -0,0 +1,52 @@
+diff --git a/cmake/modules/HandleLLVMOptions.cmake b/cmake/modules/HandleLLVMOptions.cmake
+index 5d4d692a70a..94717899a02 100644
+--- a/cmake/modules/HandleLLVMOptions.cmake
++++ b/cmake/modules/HandleLLVMOptions.cmake
+@@ -25,8 +25,24 @@ string(TOUPPER "${LLVM_ENABLE_LTO}" uppercase_LLVM_ENABLE_LTO)
+
+ # Ninja Job Pool support
+ # The following only works with the Ninja generator in CMake >= 3.0.
++if (NOT LLVM_RAM_PER_COMPILE_JOB)
++  set(LLVM_RAM_PER_COMPILE_JOB "2000")
++endif()
+ set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
+   "Define the maximum number of concurrent compilation jobs (Ninja only).")
++cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY)
++cmake_host_system_information(RESULT NUMBER_OF_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)
++if(LLVM_RAM_PER_COMPILE_JOB)
++  math(EXPR memory_available_jobs "${AVAILABLE_PHYSICAL_MEMORY} / ${LLVM_RAM_PER_COMPILE_JOB}" OUTPUT_FORMAT DECIMAL)
++  if (memory_available_jobs LESS 1)
++    set(memory_available_jobs 1)
++  endif()
++  if (memory_available_jobs LESS NUMBER_OF_LOGICAL_CORES)
++    set(LLVM_PARALLEL_COMPILE_JOBS "${memory_available_jobs}")
++  else()
++    set(LLVM_PARALLEL_COMPILE_JOBS "${NUMBER_OF_LOGICAL_CORES}")
++  endif()
++endif()
+ if(LLVM_PARALLEL_COMPILE_JOBS)
+   if(NOT CMAKE_GENERATOR STREQUAL "Ninja")
+     message(WARNING "Job pooling is only available with Ninja generators.")
+@@ -36,8 +52,22 @@ if(LLVM_PARALLEL_COMPILE_JOBS)
+   endif()
+ endif()
+
++if (NOT LLVM_RAM_PER_LINK_JOB)
++  set(LLVM_RAM_PER_LINK_JOB "14000")
++endif()
+ set(LLVM_PARALLEL_LINK_JOBS "" CACHE STRING
+   "Define the maximum number of concurrent link jobs (Ninja only).")
++if(LLVM_RAM_PER_LINK_JOB)
++  math(EXPR memory_available_jobs "${AVAILABLE_PHYSICAL_MEMORY} / ${LLVM_RAM_PER_LINK_JOB}" OUTPUT_FORMAT DECIMAL)
++  if (memory_available_jobs LESS 1)
++    set(memory_available_jobs 1)
++  endif()
++  if (memory_available_jobs LESS NUMBER_OF_LOGICAL_CORES)
++    set(LLVM_PARALLEL_LINK_JOBS "${memory_available_jobs}")
++  else()
++    set(LLVM_PARALLEL_LINK_JOBS "${NUMBER_OF_LOGICAL_CORES}")
++  endif()
++endif()
+ if(CMAKE_GENERATOR STREQUAL "Ninja")
+   if(NOT LLVM_PARALLEL_LINK_JOBS AND uppercase_LLVM_ENABLE_LTO STREQUAL "THIN")
+     message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.")

recipes/llvm-core/all/patches/13x/01-calculate-job-pools.patch

@@ -0,0 +1,52 @@
+diff --git a/cmake/modules/HandleLLVMOptions.cmake b/cmake/modules/HandleLLVMOptions.cmake
+index 0c3419390c2..8bcd91e1787 100644
+--- a/cmake/modules/HandleLLVMOptions.cmake
++++ b/cmake/modules/HandleLLVMOptions.cmake
+@@ -31,8 +31,24 @@ string(TOUPPER "${LLVM_ENABLE_LTO}" uppercase_LLVM_ENABLE_LTO)
+
+ # Ninja Job Pool support
+ # The following only works with the Ninja generator in CMake >= 3.0.
++if (NOT LLVM_RAM_PER_COMPILE_JOB)
++  set(LLVM_RAM_PER_COMPILE_JOB "2000")
++endif()
+ set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
+   "Define the maximum number of concurrent compilation jobs (Ninja only).")
++cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY)
++cmake_host_system_information(RESULT NUMBER_OF_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)
++if(LLVM_RAM_PER_COMPILE_JOB)
++  math(EXPR memory_available_jobs "${AVAILABLE_PHYSICAL_MEMORY} / ${LLVM_RAM_PER_COMPILE_JOB}" OUTPUT_FORMAT DECIMAL)
++  if (memory_available_jobs LESS 1)
++    set(memory_available_jobs 1)
++  endif()
++  if (memory_available_jobs LESS NUMBER_OF_LOGICAL_CORES)
++    set(LLVM_PARALLEL_COMPILE_JOBS "${memory_available_jobs}")
++  else()
++    set(LLVM_PARALLEL_COMPILE_JOBS "${NUMBER_OF_LOGICAL_CORES}")
++  endif()
++endif()
+ if(LLVM_PARALLEL_COMPILE_JOBS)
+   if(NOT CMAKE_GENERATOR STREQUAL "Ninja")
+     message(WARNING "Job pooling is only available with Ninja generators.")
+@@ -42,8 +58,22 @@ if(LLVM_PARALLEL_COMPILE_JOBS)
+   endif()
+ endif()
+
++if (NOT LLVM_RAM_PER_LINK_JOB)
++  set(LLVM_RAM_PER_LINK_JOB "14000")
++endif()
+ set(LLVM_PARALLEL_LINK_JOBS "" CACHE STRING
+   "Define the maximum number of concurrent link jobs (Ninja only).")
++if(LLVM_RAM_PER_LINK_JOB)
++  math(EXPR memory_available_jobs "${AVAILABLE_PHYSICAL_MEMORY} / ${LLVM_RAM_PER_LINK_JOB}" OUTPUT_FORMAT DECIMAL)
++  if (memory_available_jobs LESS 1)
++    set(memory_available_jobs 1)
++  endif()
++  if (memory_available_jobs LESS NUMBER_OF_LOGICAL_CORES)
++    set(LLVM_PARALLEL_LINK_JOBS "${memory_available_jobs}")
++  else()
++    set(LLVM_PARALLEL_LINK_JOBS "${NUMBER_OF_LOGICAL_CORES}")
++  endif()
++endif()
+ if(CMAKE_GENERATOR STREQUAL "Ninja")
+   if(NOT LLVM_PARALLEL_LINK_JOBS AND uppercase_LLVM_ENABLE_LTO STREQUAL "THIN")
+     message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.")