WIP: Cross compile for x86_64-darwin and arm64-darwin per rake-compiler-dock

.cross_rubies

@@ -3,18 +3,22 @@
 3.0.0:i686-linux-gnu
 3.0.0:x86_64-linux-gnu
 3.0.0:x86_64-darwin
+3.0.0:arm64-darwin
 2.7.0:i686-w64-mingw32
 2.7.0:x86_64-w64-mingw32
 2.7.0:i686-linux-gnu
 2.7.0:x86_64-linux-gnu
 2.7.0:x86_64-darwin
+2.7.0:arm64-darwin
 2.6.0:i686-w64-mingw32
 2.6.0:x86_64-w64-mingw32
 2.6.0:i686-linux-gnu
 2.6.0:x86_64-linux-gnu
 2.6.0:x86_64-darwin
+2.6.0:arm64-darwin
 2.5.0:i686-w64-mingw32
 2.5.0:x86_64-w64-mingw32
 2.5.0:i686-linux-gnu
 2.5.0:x86_64-linux-gnu
 2.5.0:x86_64-darwin
+2.5.0:arm64-darwin

Gemfile

@@ -18,7 +18,7 @@ gem "minitest", "~>5.8", :group => [:development, :test]
 gem "minitest-reporters", "~>1.4", :group => [:development, :test]
 gem "rake", "~>13.0", :group => [:development, :test]
 gem "rake-compiler", "~>1.1", :group => [:development, :test]
-gem "rake-compiler-dock", "~>1.0", :group => [:development, :test]
+gem "rake-compiler-dock", "~>1.1", :group => [:development, :test]
 gem "rexical", "~>1.0.5", :group => [:development, :test]
 gem "rubocop", "~>0.88", :group => [:development, :test]
 gem "simplecov", "~>0.17.0", :group => [:development, :test]

Rakefile

@@ -70,7 +70,7 @@ HOE = Hoe.spec "nokogiri" do |hoe|
     ["minitest-reporters", "~> 1.4"],
     ["rake", "~> 13.0"],
     ["rake-compiler", "~> 1.1"],
-    ["rake-compiler-dock", "~> 1.0"],
+    ["rake-compiler-dock", "~> 1.1"],
     ["rexical", "~> 1.0.5"],
     ["rubocop", "~> 0.88"],
     ["simplecov", "~> 0.17.0"], # locked on 2020-08-28 due to https://github.com/codeclimate/test-reporter/issues/413

ext/nokogiri/extconf.rb

@@ -33,6 +33,10 @@
       --disable-clean
           Do not clean out intermediate files after successful build
 
+      --prevent-strip
+          Take steps to prevent stripping the symbol table and debugging info from the shared
+          library, potentially overriding RbConfig's CFLAGS/LDFLAGS/DLDFLAGS.
+
 
     Flags only used when using system libraries:
 
@@ -81,6 +85,7 @@
       --enable-cross-build
           Enable cross-build mode. (You probably do not want to set this manually.)
 
+
     Environment variables used:
 
       NOKOGIRI_USE_SYSTEM_LIBRARIES
@@ -513,10 +518,22 @@ def do_clean
 # use same c compiler for libxml and libxslt
 ENV['CC'] = RbConfig::CONFIG['CC']
 
+if arg_config('--prevent-strip')
+  old_cflags = $CFLAGS.split.join(" ")
+  old_ldflags = $LDFLAGS.split.join(" ")
+  old_dldflags = $DLDFLAGS.split.join(" ")
+  $CFLAGS = $CFLAGS.split.reject { |flag| flag == "-s" }.join(" ")
+  $LDFLAGS = $LDFLAGS.split.reject { |flag| flag == "-s" }.join(" ")
+  $DLDFLAGS = $DLDFLAGS.split.reject { |flag| flag == "-s" }.join(" ")
+  puts "Prevent stripping by removing '-s' from $CFLAGS" if old_cflags != $CFLAGS
+  puts "Prevent stripping by removing '-s' from $LDFLAGS" if old_ldflags != $LDFLAGS
+  puts "Prevent stripping by removing '-s' from $DLDFLAGS" if old_dldflags != $DLDFLAGS
+end
+
 # adopt environment config
-append_cflags(ENV["CFLAGS"].split(/\s+/)) if !ENV["CFLAGS"].nil?
-append_cppflags(ENV["CPPFLAGS"].split(/\s+/)) if !ENV["CPPFLAGS"].nil?
-append_ldflags(ENV["LDFLAGS"].split(/\s+/)) if !ENV["LDFLAGS"].nil?
+append_cflags(ENV["CFLAGS"].split) if !ENV["CFLAGS"].nil?
+append_cppflags(ENV["CPPFLAGS"].split) if !ENV["CPPFLAGS"].nil?
+append_ldflags(ENV["LDFLAGS"].split) if !ENV["LDFLAGS"].nil?
 $LIBS = concat_flags($LIBS, ENV["LIBS"])
 
 append_cflags("-g") # always include debugging information
@@ -605,6 +622,14 @@ def configure
             cflags = concat_flags(ENV["CFLAGS"], "-fPIC", "-g")
             execute "configure", ["env", "CHOST=#{host}", "CFLAGS=#{cflags}", "./configure", "--static", configure_prefix]
           end
+
+          def compile
+            if host=~/darwin/
+              execute "compile", "make AR=#{host}-libtool"
+            else
+              super
+            end
+          end
         end
       end
     end
@@ -667,7 +692,7 @@ def configure
       recipe.configure_options += iconv_configure_flags
     end
 
-    if darwin?
+    if darwin? && !cross_build_p
       recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"]
     end
 
@@ -689,7 +714,7 @@ def configure
 
     cflags = concat_flags(ENV["CFLAGS"], "-O2", "-U_FORTIFY_SOURCE", "-g")
 
-    if darwin?
+    if darwin? && !cross_build_p
       recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"]
     end
 

patches/libxml2/0009-avoid-isnan-isinf.patch

@@ -0,0 +1,81 @@
+This patch is a result of rake-compiler-dock using centos 7 (manylinux2014) to cross-compile.
+
+Centos, for reasons I have not been able to discern, implements `isnan` and `isinf` as a function
+and not as a macro. Debian knows how to resolve that function at dynamic-link time (despite using a
+macro at compile time), but musl-based systems (like alpine) do not. Running `nm` on nokogiri.so
+created on such a centos system shows:
+
+```
+                 U __isinf@@GLIBC_2.2.5
+                 U __isnan@@GLIBC_2.2.5
+```
+
+(see https://github.com/sparklemotion/nokogiri/pull/2142 for more info)
+
+This patch avoids using glibc's `isnan` and `isinf` calls, instead using libxml2's fallback
+implementation. There's history here, see libxml2 commit 8813f39:
+
+    commit 8813f39
+    Author: Nick Wellnhofer <wellnhofer@aevum.de>
+    Date:   2017-09-21 00:11:26 +0200
+
+        Simplify XPath NaN, inf and -0 handling
+
+        Use C99 macros NAN, INFINITY, isnan, isinf. If they're not available:
+
+        - Assume that (0.0 / 0.0) generates a NaN and !(x == x) tests for NaN.
+        - Use C89's HUGE_VAL for INFINITY.
+
+        Remove manual handling of NaN, infinity and negative zero in functions
+        xmlXPathValueFlipSign and xmlXPathDivValues.
+
+        Remove xmlXPathGetSign. All the tests for negative zero can be replaced
+        with a test for negative or positive zero.
+
+        Simplify xmlXPathRoundFunction.
+
+        Remove Trio dependency.
+
+        This should work on IEEE 754 compliant implementations even if the C99
+        macros aren't available, but will likely break some ancient platforms.
+        If problems arise, my plan is to port the relevant trionan.c solution
+        to xpath.c. Note that non-compliant implementations are impossible
+        to fully support, anyway, since XPath requires IEEE 754.
+
+This patch would be unnecessary if any of the following was true:
+
+* centos implements these as macros, and doesn't generate an unresolved symbol for either in the shared library
+* we had a way to ensure `__isinf` and `__isnan` resolve on musl (e.g., we implement them locally)
+
+diff --git a/xpath.c b/xpath.c
+index 9f64ab9..5b6d999 100644
+--- a/xpath.c
++++ b/xpath.c
+@@ -509,11 +509,7 @@ xmlXPathInit(void) {
+  */
+ int
+ xmlXPathIsNaN(double val) {
+-#ifdef isnan
+-    return isnan(val);
+-#else
+     return !(val == val);
+-#endif
+ }
+
+ /**
+@@ -524,15 +520,11 @@ xmlXPathIsNaN(double val) {
+  */
+ int
+ xmlXPathIsInf(double val) {
+-#ifdef isinf
+-    return isinf(val) ? (val > 0 ? 1 : -1) : 0;
+-#else
+     if (val >= INFINITY)
+         return 1;
+     if (val <= -INFINITY)
+         return -1;
+     return 0;
+-#endif
+ }
+
+ #endif /* SCHEMAS or XPATH */

rakelib/cross-ruby.rake

@@ -48,6 +48,8 @@ CrossRuby = Struct.new(:version, :host) do
         "x86-linux"
       when /\Ax86_64-darwin/
         "x86_64-darwin"
+      when /\Aarm64-darwin/
+        "arm64-darwin"
       else
         raise "CrossRuby.platform: unsupported host: #{host}"
       end
@@ -60,11 +62,13 @@ CrossRuby = Struct.new(:version, :host) do
       when "x86-mingw32"
         "i686-w64-mingw32-"
       when "x86_64-linux"
-        "x86_64-linux-gnu-"
+        "x86_64-redhat-linux-"
       when "x86-linux"
-        "i686-linux-gnu-"
-      # when /darwin/
-      #   ""
+        "i686-redhat-linux-"
+      when /x86_64.*darwin/
+        "x86_64-apple-darwin-"
+      when /a.*64.*darwin/
+        "aarch64-apple-darwin-"
       else
         raise "CrossRuby.tool: unmatched platform: #{platform}"
       end) + name
@@ -82,6 +86,8 @@ CrossRuby = Struct.new(:version, :host) do
       "elf32-i386"
     when "x86_64-darwin"
       "Mach-O 64-bit x86-64" # hmm
+    when "arm64-darwin"
+      "Mach-O arm64"
     else
       raise "CrossRuby.target_file_format: unmatched platform: #{platform}"
     end
@@ -113,10 +119,8 @@ CrossRuby = Struct.new(:version, :host) do
         "kernel32.dll",
         "msvcrt.dll",
         "ws2_32.dll",
-        *(case
-          when ver >= "2.0.0"
-            "user32.dll"
-          end),
+        "user32.dll",
+        "advapi32.dll",
         libruby_dll,
       ]
     when LINUX_PLATFORM_REGEX
@@ -127,6 +131,7 @@ CrossRuby = Struct.new(:version, :host) do
           "libpthread.so.0"
         end),
         "libc.so.6",
+        "libdl.so.2", # on old dists only - now in libc
       ]
     when DARWIN_PLATFORM_REGEX
       [
@@ -201,27 +206,27 @@ def verify_dll(dll, cross_ruby)
     end
 
   elsif cross_ruby.darwin?
-    dump = `#{["env", "LANG=C", "objdump", "-p", dll].shelljoin}`
-    nm = `#{["env", "LANG=C", "nm", "-g", dll].shelljoin}`
+    dump = `#{["env", "LANG=C", cross_ruby.tool("objdump"), "-p", dll].shelljoin}`
+    nm = `#{["env", "LANG=C", cross_ruby.tool("nm"), "-g", dll].shelljoin}`
 
     raise "unexpected file format for generated dll #{dll}" unless /file format #{Regexp.quote(cross_ruby.target_file_format)}\s/ === dump
     raise "export function Init_nokogiri not in dll #{dll}" unless / T _?Init_nokogiri/ === nm
 
     # if liblzma is being referenced, let's make sure it's referring
     # to the system-installed file and not the homebrew-installed file.
-    ldd = `#{["env", "LANG=C", "otool", "-L", dll].shelljoin}`
+    ldd = `#{["env", "LANG=C", cross_ruby.tool("otool"), "-L", dll].shelljoin}`
     if liblzma_refs = ldd.scan(/^\t([^ ]+) /).map(&:first).uniq.grep(/liblzma/)
       liblzma_refs.each do |ref|
         new_ref = File.join("/usr/lib", File.basename(ref))
-        sh ["env", "LANG=C", "install_name_tool", "-change", ref, new_ref, dll].shelljoin
+        sh ["env", "LANG=C", cross_ruby.tool("install_name_tool"), "-change", ref, new_ref, dll].shelljoin
       end
 
       # reload!
-      ldd = `#{["env", "LANG=C", "otool", "-L", dll].shelljoin}`
+      ldd = `#{["env", "LANG=C", cross_ruby.tool("otool"), "-L", dll].shelljoin}`
     end
 
     # Verify that the DLL dependencies are all allowed.
-    ldd = `#{["env", "LANG=C", "otool", "-L", dll].shelljoin}`
+    ldd = `#{["env", "LANG=C", cross_ruby.tool("otool"), "-L", dll].shelljoin}`
     actual_imports = ldd.scan(/^\t([^ ]+) /).map(&:first).uniq
     if !(actual_imports - allowed_imports).empty?
       raise "unallowed so imports #{actual_imports.inspect} in #{dll} (allowed #{allowed_imports.inspect})"
@@ -243,13 +248,13 @@ namespace "gem" do
     Rake::Task["pkg/#{HOE.spec.full_name}-#{Gem::Platform.new(plat).to_s}.gem"].invoke
   end
 
-  CROSS_RUBIES.find_all { |cr| cr.windows? || cr.linux? }.map(&:platform).uniq.each do |plat|
+  CROSS_RUBIES.find_all { |cr| cr.windows? || cr.linux? || cr.darwin? }.map(&:platform).uniq.each do |plat|
     desc "build native gem for #{plat} platform"
     task plat do
       RakeCompilerDock.sh <<~EOT, platform: plat
         gem install bundler --no-document &&
         bundle &&
-        bundle exec rake gem:#{plat}:builder MAKE='nice make -j`nproc`' FORCE_CROSS_COMPILING=true
+        bundle exec rake gem:#{plat}:builder MAKE='nice make -j`nproc`'
       EOT
     end
 
@@ -262,21 +267,6 @@ namespace "gem" do
     end
   end
 
-  CROSS_RUBIES.find_all { |cr| cr.darwin? }.map(&:platform).uniq.each do |plat|
-    desc "build native gem for #{plat} platform"
-    task plat do
-      sh "rake gem:#{plat}:builder MAKE='nice make -j`nproc`' FORCE_CROSS_COMPILING=true"
-    end
-
-    namespace plat do
-      desc "build native gem for #{plat} platform (child process)"
-      task "builder" do
-        gem_builder(plat)
-      end
-      task "guest" => "builder" # TODO: remove me after this code is on master, temporary backwards compat for CI
-    end
-  end
-
   desc "build a jruby gem"
   task "jruby" do
     RakeCompilerDock.sh("gem install bundler --no-document && bundle && bundle exec rake java gem",
@@ -303,8 +293,8 @@ if java?
 
     ext.ext_dir = 'ext/java'
     ext.lib_dir = 'lib/nokogiri'
-    ext.source_version = '1.6'
-    ext.target_version = '1.6'
+    ext.source_version = '1.7'
+    ext.target_version = '1.7'
     ext.classpath = jars.map { |x| File.expand_path x }.join ':'
     ext.debug = true if ENV['JAVA_DEBUG']
   end
@@ -339,7 +329,6 @@ else
   Rake::ExtensionTask.new("nokogiri", HOE.spec) do |ext|
     ext.lib_dir = File.join(*['lib', 'nokogiri', ENV['FAT_DIR']].compact)
     ext.config_options << ENV['EXTOPTS']
-    ext.no_native = (ENV["FORCE_CROSS_COMPILING"] == "true")
     ext.cross_compile  = true
     ext.cross_platform = CROSS_RUBIES.map(&:platform).uniq
     ext.cross_config_options << "--enable-cross-build"

scripts/build-gems

@@ -3,28 +3,39 @@
 #  script to build gems for all relevant platforms
 #
 set -o errexit
-set -u -x
+set -o nounset
+set -x
 
 rm -rf tmp pkg gems
 mkdir -p gems
 
-# MRI et al (standard gem)
+bundle update
+bundle package
+
 bundle exec rake clean
 bundle exec rake compile test
+
+# MRI et al (standard gem)
+bundle exec rake clean
 bundle exec rake gem
 cp -v pkg/nokogiri*.gem gems
 
 # jruby
 bundle exec rake clean
 bundle exec rake gem:jruby
-cp -v pkg/nokogiri*java.gem gems
+cp -v pkg/nokogiri-*java*.gem gems
 
-# windows and linux fat binary gems
+# precompiled native gems ("fat binary")
 bundle exec rake clean
+
 bundle exec rake gem:windows
+cp -v pkg/nokogiri-*mingw*.gem gems
+
 bundle exec rake gem:linux
-cp -v pkg/nokogiri-*{x86,x64}*.gem gems
+cp -v pkg/nokogiri-*linux*.gem gems
+
+bundle exec rake gem:darwin
+cp -v pkg/nokogiri-*darwin*.gem gems
 
-for f in gems/*.gem ; do
-  ./scripts/test-gem-file-contents $f
-done
+# test those gem files!
+$(dirname $0)/test-gem-files gems/*.gem