Merge pull request #2142 from larskanis/cross-darwin

Cross compile for x86_64-darwin and arm64-darwin per rake-compiler-dock

---

This requires latest rake-compiler-dock from master branch.

**What problem is this PR intended to solve?**

Fat binary gems for MacOS are currently built on MacOS native. This complicates the build and release cycle. It is desirable to build all binary gems per cross compiler.

Related to #2079

**Have you included adequate test coverage?**

No CI, but I tested the resulting `x86_64-darwin` gem on MacOS-10.14.4 with both the system ruby-2.3.7 (after extending the `required_ruby_version` of nokogiri) and ruby-2.7.2 installed per rvm. The Mac is running on https://github.com/foxlet/macOS-Simple-KVM .

I'm unable to execute the `arm64-darwin` on any computer. I just did some static inspection of the built `nokogiri.bundle` file.

**Does this change affect the behavior of either the C or the Java implementations?**

C only.

.cross_rubies

@@ -3,18 +3,22 @@
 3.0.0:i686-linux-gnu
 3.0.0:x86_64-linux-gnu
 3.0.0:x86_64-darwin
+3.0.0:arm64-darwin
 2.7.0:i686-w64-mingw32
 2.7.0:x86_64-w64-mingw32
 2.7.0:i686-linux-gnu
 2.7.0:x86_64-linux-gnu
 2.7.0:x86_64-darwin
+2.7.0:arm64-darwin
 2.6.0:i686-w64-mingw32
 2.6.0:x86_64-w64-mingw32
 2.6.0:i686-linux-gnu
 2.6.0:x86_64-linux-gnu
 2.6.0:x86_64-darwin
+2.6.0:arm64-darwin
 2.5.0:i686-w64-mingw32
 2.5.0:x86_64-w64-mingw32
 2.5.0:i686-linux-gnu
 2.5.0:x86_64-linux-gnu
 2.5.0:x86_64-darwin
+2.5.0:arm64-darwin

Gemfile

@@ -18,7 +18,7 @@ gem "minitest", "~>5.8", :group => [:development, :test]
 gem "minitest-reporters", "~>1.4", :group => [:development, :test]
 gem "rake", "~>13.0", :group => [:development, :test]
 gem "rake-compiler", "~>1.1", :group => [:development, :test]
-gem "rake-compiler-dock", "~>1.0", :group => [:development, :test]
+gem "rake-compiler-dock", "~>1.1", :group => [:development, :test]
 gem "rexical", "~>1.0.5", :group => [:development, :test]
 gem "rubocop", "~>0.88", :group => [:development, :test]
 gem "simplecov", "~>0.17.0", :group => [:development, :test]

Rakefile

@@ -70,7 +70,7 @@ HOE = Hoe.spec "nokogiri" do |hoe|
     ["minitest-reporters", "~> 1.4"],
     ["rake", "~> 13.0"],
     ["rake-compiler", "~> 1.1"],
-    ["rake-compiler-dock", "~> 1.0"],
+    ["rake-compiler-dock", "~> 1.1"],
     ["rexical", "~> 1.0.5"],
     ["rubocop", "~> 0.88"],
     ["simplecov", "~> 0.17.0"], # locked on 2020-08-28 due to https://github.com/codeclimate/test-reporter/issues/413

ext/nokogiri/extconf.rb

@@ -33,6 +33,10 @@
       --disable-clean
           Do not clean out intermediate files after successful build
 
+      --prevent-strip
+          Take steps to prevent stripping the symbol table and debugging info from the shared
+          library, potentially overriding RbConfig's CFLAGS/LDFLAGS/DLDFLAGS.
+
 
     Flags only used when using system libraries:
 
@@ -81,6 +85,7 @@
       --enable-cross-build
           Enable cross-build mode. (You probably do not want to set this manually.)
 
+
     Environment variables used:
 
       NOKOGIRI_USE_SYSTEM_LIBRARIES
@@ -513,10 +518,22 @@ def do_clean
 # use same c compiler for libxml and libxslt
 ENV['CC'] = RbConfig::CONFIG['CC']
 
+if arg_config('--prevent-strip')
+  old_cflags = $CFLAGS.split.join(" ")
+  old_ldflags = $LDFLAGS.split.join(" ")
+  old_dldflags = $DLDFLAGS.split.join(" ")
+  $CFLAGS = $CFLAGS.split.reject { |flag| flag == "-s" }.join(" ")
+  $LDFLAGS = $LDFLAGS.split.reject { |flag| flag == "-s" }.join(" ")
+  $DLDFLAGS = $DLDFLAGS.split.reject { |flag| flag == "-s" }.join(" ")
+  puts "Prevent stripping by removing '-s' from $CFLAGS" if old_cflags != $CFLAGS
+  puts "Prevent stripping by removing '-s' from $LDFLAGS" if old_ldflags != $LDFLAGS
+  puts "Prevent stripping by removing '-s' from $DLDFLAGS" if old_dldflags != $DLDFLAGS
+end
+
 # adopt environment config
-append_cflags(ENV["CFLAGS"].split(/\s+/)) if !ENV["CFLAGS"].nil?
-append_cppflags(ENV["CPPFLAGS"].split(/\s+/)) if !ENV["CPPFLAGS"].nil?
-append_ldflags(ENV["LDFLAGS"].split(/\s+/)) if !ENV["LDFLAGS"].nil?
+append_cflags(ENV["CFLAGS"].split) if !ENV["CFLAGS"].nil?
+append_cppflags(ENV["CPPFLAGS"].split) if !ENV["CPPFLAGS"].nil?
+append_ldflags(ENV["LDFLAGS"].split) if !ENV["LDFLAGS"].nil?
 $LIBS = concat_flags($LIBS, ENV["LIBS"])
 
 append_cflags("-g") # always include debugging information
@@ -605,6 +622,14 @@ def configure
             cflags = concat_flags(ENV["CFLAGS"], "-fPIC", "-g")
             execute "configure", ["env", "CHOST=#{host}", "CFLAGS=#{cflags}", "./configure", "--static", configure_prefix]
           end
+
+          def compile
+            if host=~/darwin/
+              execute "compile", "make AR=#{host}-libtool"
+            else
+              super
+            end
+          end
         end
       end
     end
@@ -667,7 +692,7 @@ def configure
       recipe.configure_options += iconv_configure_flags
     end
 
-    if darwin?
+    if darwin? && !cross_build_p
       recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"]
     end
 
@@ -689,7 +714,7 @@ def configure
 
     cflags = concat_flags(ENV["CFLAGS"], "-O2", "-U_FORTIFY_SOURCE", "-g")
 
-    if darwin?
+    if darwin? && !cross_build_p
       recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"]
     end
 

patches/libxml2/0009-avoid-isnan-isinf.patch

@@ -0,0 +1,81 @@
+This patch is a result of rake-compiler-dock using centos 7 (manylinux2014) to cross-compile.
+
+Centos, for reasons I have not been able to discern, implements `isnan` and `isinf` as a function
+and not as a macro. Debian knows how to resolve that function at dynamic-link time (despite using a
+macro at compile time), but musl-based systems (like alpine) do not. Running `nm` on nokogiri.so
+created on such a centos system shows:
+
+```
+                 U __isinf@@GLIBC_2.2.5
+                 U __isnan@@GLIBC_2.2.5
+```
+
+(see https://github.com/sparklemotion/nokogiri/pull/2142 for more info)
+
+This patch avoids using glibc's `isnan` and `isinf` calls, instead using libxml2's fallback
+implementation. There's history here, see libxml2 commit 8813f39:
+
+    commit 8813f39
+    Author: Nick Wellnhofer <wellnhofer@aevum.de>
+    Date:   2017-09-21 00:11:26 +0200
+
+        Simplify XPath NaN, inf and -0 handling
+
+        Use C99 macros NAN, INFINITY, isnan, isinf. If they're not available:
+
+        - Assume that (0.0 / 0.0) generates a NaN and !(x == x) tests for NaN.
+        - Use C89's HUGE_VAL for INFINITY.
+
+        Remove manual handling of NaN, infinity and negative zero in functions
+        xmlXPathValueFlipSign and xmlXPathDivValues.
+
+        Remove xmlXPathGetSign. All the tests for negative zero can be replaced
+        with a test for negative or positive zero.
+
+        Simplify xmlXPathRoundFunction.
+
+        Remove Trio dependency.
+
+        This should work on IEEE 754 compliant implementations even if the C99
+        macros aren't available, but will likely break some ancient platforms.
+        If problems arise, my plan is to port the relevant trionan.c solution
+        to xpath.c. Note that non-compliant implementations are impossible
+        to fully support, anyway, since XPath requires IEEE 754.
+
+This patch would be unnecessary if any of the following was true:
+
+* centos implements these as macros, and doesn't generate an unresolved symbol for either in the shared library
+* we had a way to ensure `__isinf` and `__isnan` resolve on musl (e.g., we implement them locally)
+
+diff --git a/xpath.c b/xpath.c
+index 9f64ab9..5b6d999 100644
+--- a/xpath.c
++++ b/xpath.c
+@@ -509,11 +509,7 @@ xmlXPathInit(void) {
+  */
+ int
+ xmlXPathIsNaN(double val) {
+-#ifdef isnan
+-    return isnan(val);
+-#else
+     return !(val == val);
+-#endif
+ }
+
+ /**
+@@ -524,15 +520,11 @@ xmlXPathIsNaN(double val) {
+  */
+ int
+ xmlXPathIsInf(double val) {
+-#ifdef isinf
+-    return isinf(val) ? (val > 0 ? 1 : -1) : 0;
+-#else
+     if (val >= INFINITY)
+         return 1;
+     if (val <= -INFINITY)
+         return -1;
+     return 0;
+-#endif
+ }
+
+ #endif /* SCHEMAS or XPATH */

rakelib/cross-ruby.rake

@@ -48,6 +48,8 @@ CrossRuby = Struct.new(:version, :host) do
         "x86-linux"
       when /\Ax86_64-darwin/
         "x86_64-darwin"
+      when /\Aarm64-darwin/
+        "arm64-darwin"
       else
         raise "CrossRuby.platform: unsupported host: #{host}"
       end
@@ -60,11 +62,13 @@ CrossRuby = Struct.new(:version, :host) do
       when "x86-mingw32"
         "i686-w64-mingw32-"
       when "x86_64-linux"
-        "x86_64-linux-gnu-"
+        "x86_64-redhat-linux-"
       when "x86-linux"
-        "i686-linux-gnu-"
-      # when /darwin/
-      #   ""
+        "i686-redhat-linux-"
+      when /x86_64.*darwin/
+        "x86_64-apple-darwin-"
+      when /a.*64.*darwin/
+        "aarch64-apple-darwin-"
       else
         raise "CrossRuby.tool: unmatched platform: #{platform}"
       end) + name
@@ -82,6 +86,8 @@ CrossRuby = Struct.new(:version, :host) do
       "elf32-i386"
     when "x86_64-darwin"
       "Mach-O 64-bit x86-64" # hmm
+    when "arm64-darwin"
+      "Mach-O arm64"
     else
       raise "CrossRuby.target_file_format: unmatched platform: #{platform}"
     end
@@ -113,10 +119,8 @@ CrossRuby = Struct.new(:version, :host) do
         "kernel32.dll",
         "msvcrt.dll",
         "ws2_32.dll",
-        *(case
-          when ver >= "2.0.0"
-            "user32.dll"
-          end),
+        "user32.dll",
+        "advapi32.dll",
         libruby_dll,
       ]
     when LINUX_PLATFORM_REGEX
@@ -127,6 +131,7 @@ CrossRuby = Struct.new(:version, :host) do
           "libpthread.so.0"
         end),
         "libc.so.6",
+        "libdl.so.2", # on old dists only - now in libc
       ]
     when DARWIN_PLATFORM_REGEX
       [
@@ -201,27 +206,27 @@ def verify_dll(dll, cross_ruby)
     end
 
   elsif cross_ruby.darwin?
-    dump = `#{["env", "LANG=C", "objdump", "-p", dll].shelljoin}`
-    nm = `#{["env", "LANG=C", "nm", "-g", dll].shelljoin}`
+    dump = `#{["env", "LANG=C", cross_ruby.tool("objdump"), "-p", dll].shelljoin}`
+    nm = `#{["env", "LANG=C", cross_ruby.tool("nm"), "-g", dll].shelljoin}`
 
     raise "unexpected file format for generated dll #{dll}" unless /file format #{Regexp.quote(cross_ruby.target_file_format)}\s/ === dump
     raise "export function Init_nokogiri not in dll #{dll}" unless / T _?Init_nokogiri/ === nm
 
     # if liblzma is being referenced, let's make sure it's referring
     # to the system-installed file and not the homebrew-installed file.
-    ldd = `#{["env", "LANG=C", "otool", "-L", dll].shelljoin}`
+    ldd = `#{["env", "LANG=C", cross_ruby.tool("otool"), "-L", dll].shelljoin}`
     if liblzma_refs = ldd.scan(/^\t([^ ]+) /).map(&:first).uniq.grep(/liblzma/)
       liblzma_refs.each do |ref|
         new_ref = File.join("/usr/lib", File.basename(ref))
-        sh ["env", "LANG=C", "install_name_tool", "-change", ref, new_ref, dll].shelljoin
+        sh ["env", "LANG=C", cross_ruby.tool("install_name_tool"), "-change", ref, new_ref, dll].shelljoin
       end
 
       # reload!
-      ldd = `#{["env", "LANG=C", "otool", "-L", dll].shelljoin}`
+      ldd = `#{["env", "LANG=C", cross_ruby.tool("otool"), "-L", dll].shelljoin}`
     end
 
     # Verify that the DLL dependencies are all allowed.
-    ldd = `#{["env", "LANG=C", "otool", "-L", dll].shelljoin}`
+    ldd = `#{["env", "LANG=C", cross_ruby.tool("otool"), "-L", dll].shelljoin}`
     actual_imports = ldd.scan(/^\t([^ ]+) /).map(&:first).uniq
     if !(actual_imports - allowed_imports).empty?
       raise "unallowed so imports #{actual_imports.inspect} in #{dll} (allowed #{allowed_imports.inspect})"
@@ -243,13 +248,13 @@ namespace "gem" do
     Rake::Task["pkg/#{HOE.spec.full_name}-#{Gem::Platform.new(plat).to_s}.gem"].invoke
   end
 
-  CROSS_RUBIES.find_all { |cr| cr.windows? || cr.linux? }.map(&:platform).uniq.each do |plat|
+  CROSS_RUBIES.find_all { |cr| cr.windows? || cr.linux? || cr.darwin? }.map(&:platform).uniq.each do |plat|
     desc "build native gem for #{plat} platform"
     task plat do
       RakeCompilerDock.sh <<~EOT, platform: plat
         gem install bundler --no-document &&
         bundle &&
-        bundle exec rake gem:#{plat}:builder MAKE='nice make -j`nproc`' FORCE_CROSS_COMPILING=true
+        bundle exec rake gem:#{plat}:builder MAKE='nice make -j`nproc`'
       EOT
     end
 
@@ -262,21 +267,6 @@ namespace "gem" do
     end
   end
 
-  CROSS_RUBIES.find_all { |cr| cr.darwin? }.map(&:platform).uniq.each do |plat|
-    desc "build native gem for #{plat} platform"
-    task plat do
-      sh "rake gem:#{plat}:builder MAKE='nice make -j`nproc`' FORCE_CROSS_COMPILING=true"
-    end
-
-    namespace plat do
-      desc "build native gem for #{plat} platform (child process)"
-      task "builder" do
-        gem_builder(plat)
-      end
-      task "guest" => "builder" # TODO: remove me after this code is on master, temporary backwards compat for CI
-    end
-  end
-
   desc "build a jruby gem"
   task "jruby" do
     RakeCompilerDock.sh("gem install bundler --no-document && bundle && bundle exec rake java gem",
@@ -303,8 +293,8 @@ if java?
 
     ext.ext_dir = 'ext/java'
     ext.lib_dir = 'lib/nokogiri'
-    ext.source_version = '1.6'
-    ext.target_version = '1.6'
+    ext.source_version = '1.7'
+    ext.target_version = '1.7'
     ext.classpath = jars.map { |x| File.expand_path x }.join ':'
     ext.debug = true if ENV['JAVA_DEBUG']
   end
@@ -339,7 +329,6 @@ else
   Rake::ExtensionTask.new("nokogiri", HOE.spec) do |ext|
     ext.lib_dir = File.join(*['lib', 'nokogiri', ENV['FAT_DIR']].compact)
     ext.config_options << ENV['EXTOPTS']
-    ext.no_native = (ENV["FORCE_CROSS_COMPILING"] == "true")
     ext.cross_compile  = true
     ext.cross_platform = CROSS_RUBIES.map(&:platform).uniq
     ext.cross_config_options << "--enable-cross-build"

scripts/build-gems

@@ -3,28 +3,39 @@
 #  script to build gems for all relevant platforms
 #
 set -o errexit
-set -u -x
+set -o nounset
+set -x
 
 rm -rf tmp pkg gems
 mkdir -p gems
 
-# MRI et al (standard gem)
+bundle update
+bundle package
+
 bundle exec rake clean
 bundle exec rake compile test
+
+# MRI et al (standard gem)
+bundle exec rake clean
 bundle exec rake gem
 cp -v pkg/nokogiri*.gem gems
 
 # jruby
 bundle exec rake clean
 bundle exec rake gem:jruby
-cp -v pkg/nokogiri*java.gem gems
+cp -v pkg/nokogiri-*java*.gem gems
 
-# windows and linux fat binary gems
+# precompiled native gems ("fat binary")
 bundle exec rake clean
+
 bundle exec rake gem:windows
+cp -v pkg/nokogiri-*mingw*.gem gems
+
 bundle exec rake gem:linux
-cp -v pkg/nokogiri-*{x86,x64}*.gem gems
+cp -v pkg/nokogiri-*linux*.gem gems
+
+bundle exec rake gem:darwin
+cp -v pkg/nokogiri-*darwin*.gem gems
 
-for f in gems/*.gem ; do
-  ./scripts/test-gem-file-contents $f
-done
+# test those gem files!
+$(dirname $0)/test-gem-files gems/*.gem