diff --git a/.cross_rubies b/.cross_rubies index e3d9a87588..234b75d57f 100644 --- a/.cross_rubies +++ b/.cross_rubies @@ -3,18 +3,22 @@ 3.0.0:i686-linux-gnu 3.0.0:x86_64-linux-gnu 3.0.0:x86_64-darwin +3.0.0:arm64-darwin 2.7.0:i686-w64-mingw32 2.7.0:x86_64-w64-mingw32 2.7.0:i686-linux-gnu 2.7.0:x86_64-linux-gnu 2.7.0:x86_64-darwin +2.7.0:arm64-darwin 2.6.0:i686-w64-mingw32 2.6.0:x86_64-w64-mingw32 2.6.0:i686-linux-gnu 2.6.0:x86_64-linux-gnu 2.6.0:x86_64-darwin +2.6.0:arm64-darwin 2.5.0:i686-w64-mingw32 2.5.0:x86_64-w64-mingw32 2.5.0:i686-linux-gnu 2.5.0:x86_64-linux-gnu 2.5.0:x86_64-darwin +2.5.0:arm64-darwin diff --git a/Gemfile b/Gemfile index 3ee94bd36b..b27affa2e4 100644 --- a/Gemfile +++ b/Gemfile @@ -18,7 +18,7 @@ gem "minitest", "~>5.8", :group => [:development, :test] gem "minitest-reporters", "~>1.4", :group => [:development, :test] gem "rake", "~>13.0", :group => [:development, :test] gem "rake-compiler", "~>1.1", :group => [:development, :test] -gem "rake-compiler-dock", "~>1.0", :group => [:development, :test] +gem "rake-compiler-dock", "~>1.1", :group => [:development, :test] gem "rexical", "~>1.0.5", :group => [:development, :test] gem "rubocop", "~>0.88", :group => [:development, :test] gem "simplecov", "~>0.17.0", :group => [:development, :test] diff --git a/Rakefile b/Rakefile index 19b261bff4..1fb82f427d 100644 --- a/Rakefile +++ b/Rakefile @@ -70,7 +70,7 @@ HOE = Hoe.spec "nokogiri" do |hoe| ["minitest-reporters", "~> 1.4"], ["rake", "~> 13.0"], ["rake-compiler", "~> 1.1"], - ["rake-compiler-dock", "~> 1.0"], + ["rake-compiler-dock", "~> 1.1"], ["rexical", "~> 1.0.5"], ["rubocop", "~> 0.88"], ["simplecov", "~> 0.17.0"], # locked on 2020-08-28 due to https://github.com/codeclimate/test-reporter/issues/413 diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index f3f07bdeae..8dbbd66ca8 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -33,6 +33,10 @@ --disable-clean Do not clean out intermediate files after successful build + --prevent-strip + Take steps to prevent stripping the symbol table and debugging info from the shared + library, potentially overriding RbConfig's CFLAGS/LDFLAGS/DLDFLAGS. + Flags only used when using system libraries: @@ -81,6 +85,7 @@ --enable-cross-build Enable cross-build mode. (You probably do not want to set this manually.) + Environment variables used: NOKOGIRI_USE_SYSTEM_LIBRARIES @@ -513,10 +518,22 @@ def do_clean # use same c compiler for libxml and libxslt ENV['CC'] = RbConfig::CONFIG['CC'] +if arg_config('--prevent-strip') + old_cflags = $CFLAGS.split.join(" ") + old_ldflags = $LDFLAGS.split.join(" ") + old_dldflags = $DLDFLAGS.split.join(" ") + $CFLAGS = $CFLAGS.split.reject { |flag| flag == "-s" }.join(" ") + $LDFLAGS = $LDFLAGS.split.reject { |flag| flag == "-s" }.join(" ") + $DLDFLAGS = $DLDFLAGS.split.reject { |flag| flag == "-s" }.join(" ") + puts "Prevent stripping by removing '-s' from $CFLAGS" if old_cflags != $CFLAGS + puts "Prevent stripping by removing '-s' from $LDFLAGS" if old_ldflags != $LDFLAGS + puts "Prevent stripping by removing '-s' from $DLDFLAGS" if old_dldflags != $DLDFLAGS +end + # adopt environment config -append_cflags(ENV["CFLAGS"].split(/\s+/)) if !ENV["CFLAGS"].nil? -append_cppflags(ENV["CPPFLAGS"].split(/\s+/)) if !ENV["CPPFLAGS"].nil? -append_ldflags(ENV["LDFLAGS"].split(/\s+/)) if !ENV["LDFLAGS"].nil? +append_cflags(ENV["CFLAGS"].split) if !ENV["CFLAGS"].nil? +append_cppflags(ENV["CPPFLAGS"].split) if !ENV["CPPFLAGS"].nil? +append_ldflags(ENV["LDFLAGS"].split) if !ENV["LDFLAGS"].nil? $LIBS = concat_flags($LIBS, ENV["LIBS"]) append_cflags("-g") # always include debugging information @@ -605,6 +622,14 @@ def configure cflags = concat_flags(ENV["CFLAGS"], "-fPIC", "-g") execute "configure", ["env", "CHOST=#{host}", "CFLAGS=#{cflags}", "./configure", "--static", configure_prefix] end + + def compile + if host=~/darwin/ + execute "compile", "make AR=#{host}-libtool" + else + super + end + end end end end @@ -667,7 +692,7 @@ def configure recipe.configure_options += iconv_configure_flags end - if darwin? + if darwin? && !cross_build_p recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] end @@ -689,7 +714,7 @@ def configure cflags = concat_flags(ENV["CFLAGS"], "-O2", "-U_FORTIFY_SOURCE", "-g") - if darwin? + if darwin? && !cross_build_p recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] end diff --git a/patches/libxml2/0009-avoid-isnan-isinf.patch b/patches/libxml2/0009-avoid-isnan-isinf.patch new file mode 100644 index 0000000000..ffd82cba6c --- /dev/null +++ b/patches/libxml2/0009-avoid-isnan-isinf.patch @@ -0,0 +1,81 @@ +This patch is a result of rake-compiler-dock using centos 7 (manylinux2014) to cross-compile. + +Centos, for reasons I have not been able to discern, implements `isnan` and `isinf` as a function +and not as a macro. Debian knows how to resolve that function at dynamic-link time (despite using a +macro at compile time), but musl-based systems (like alpine) do not. Running `nm` on nokogiri.so +created on such a centos system shows: + +``` + U __isinf@@GLIBC_2.2.5 + U __isnan@@GLIBC_2.2.5 +``` + +(see https://github.com/sparklemotion/nokogiri/pull/2142 for more info) + +This patch avoids using glibc's `isnan` and `isinf` calls, instead using libxml2's fallback +implementation. There's history here, see libxml2 commit 8813f39: + + commit 8813f39 + Author: Nick Wellnhofer + Date: 2017-09-21 00:11:26 +0200 + + Simplify XPath NaN, inf and -0 handling + + Use C99 macros NAN, INFINITY, isnan, isinf. If they're not available: + + - Assume that (0.0 / 0.0) generates a NaN and !(x == x) tests for NaN. + - Use C89's HUGE_VAL for INFINITY. + + Remove manual handling of NaN, infinity and negative zero in functions + xmlXPathValueFlipSign and xmlXPathDivValues. + + Remove xmlXPathGetSign. All the tests for negative zero can be replaced + with a test for negative or positive zero. + + Simplify xmlXPathRoundFunction. + + Remove Trio dependency. + + This should work on IEEE 754 compliant implementations even if the C99 + macros aren't available, but will likely break some ancient platforms. + If problems arise, my plan is to port the relevant trionan.c solution + to xpath.c. Note that non-compliant implementations are impossible + to fully support, anyway, since XPath requires IEEE 754. + +This patch would be unnecessary if any of the following was true: + +* centos implements these as macros, and doesn't generate an unresolved symbol for either in the shared library +* we had a way to ensure `__isinf` and `__isnan` resolve on musl (e.g., we implement them locally) + +diff --git a/xpath.c b/xpath.c +index 9f64ab9..5b6d999 100644 +--- a/xpath.c ++++ b/xpath.c +@@ -509,11 +509,7 @@ xmlXPathInit(void) { + */ + int + xmlXPathIsNaN(double val) { +-#ifdef isnan +- return isnan(val); +-#else + return !(val == val); +-#endif + } + + /** +@@ -524,15 +520,11 @@ xmlXPathIsNaN(double val) { + */ + int + xmlXPathIsInf(double val) { +-#ifdef isinf +- return isinf(val) ? (val > 0 ? 1 : -1) : 0; +-#else + if (val >= INFINITY) + return 1; + if (val <= -INFINITY) + return -1; + return 0; +-#endif + } + + #endif /* SCHEMAS or XPATH */ diff --git a/rakelib/cross-ruby.rake b/rakelib/cross-ruby.rake index 8a939961f0..38e6da9caf 100644 --- a/rakelib/cross-ruby.rake +++ b/rakelib/cross-ruby.rake @@ -48,6 +48,8 @@ CrossRuby = Struct.new(:version, :host) do "x86-linux" when /\Ax86_64-darwin/ "x86_64-darwin" + when /\Aarm64-darwin/ + "arm64-darwin" else raise "CrossRuby.platform: unsupported host: #{host}" end @@ -60,11 +62,13 @@ CrossRuby = Struct.new(:version, :host) do when "x86-mingw32" "i686-w64-mingw32-" when "x86_64-linux" - "x86_64-linux-gnu-" + "x86_64-redhat-linux-" when "x86-linux" - "i686-linux-gnu-" - # when /darwin/ - # "" + "i686-redhat-linux-" + when /x86_64.*darwin/ + "x86_64-apple-darwin-" + when /a.*64.*darwin/ + "aarch64-apple-darwin-" else raise "CrossRuby.tool: unmatched platform: #{platform}" end) + name @@ -82,6 +86,8 @@ CrossRuby = Struct.new(:version, :host) do "elf32-i386" when "x86_64-darwin" "Mach-O 64-bit x86-64" # hmm + when "arm64-darwin" + "Mach-O arm64" else raise "CrossRuby.target_file_format: unmatched platform: #{platform}" end @@ -113,10 +119,8 @@ CrossRuby = Struct.new(:version, :host) do "kernel32.dll", "msvcrt.dll", "ws2_32.dll", - *(case - when ver >= "2.0.0" - "user32.dll" - end), + "user32.dll", + "advapi32.dll", libruby_dll, ] when LINUX_PLATFORM_REGEX @@ -127,6 +131,7 @@ CrossRuby = Struct.new(:version, :host) do "libpthread.so.0" end), "libc.so.6", + "libdl.so.2", # on old dists only - now in libc ] when DARWIN_PLATFORM_REGEX [ @@ -201,27 +206,27 @@ def verify_dll(dll, cross_ruby) end elsif cross_ruby.darwin? - dump = `#{["env", "LANG=C", "objdump", "-p", dll].shelljoin}` - nm = `#{["env", "LANG=C", "nm", "-g", dll].shelljoin}` + dump = `#{["env", "LANG=C", cross_ruby.tool("objdump"), "-p", dll].shelljoin}` + nm = `#{["env", "LANG=C", cross_ruby.tool("nm"), "-g", dll].shelljoin}` raise "unexpected file format for generated dll #{dll}" unless /file format #{Regexp.quote(cross_ruby.target_file_format)}\s/ === dump raise "export function Init_nokogiri not in dll #{dll}" unless / T _?Init_nokogiri/ === nm # if liblzma is being referenced, let's make sure it's referring # to the system-installed file and not the homebrew-installed file. - ldd = `#{["env", "LANG=C", "otool", "-L", dll].shelljoin}` + ldd = `#{["env", "LANG=C", cross_ruby.tool("otool"), "-L", dll].shelljoin}` if liblzma_refs = ldd.scan(/^\t([^ ]+) /).map(&:first).uniq.grep(/liblzma/) liblzma_refs.each do |ref| new_ref = File.join("/usr/lib", File.basename(ref)) - sh ["env", "LANG=C", "install_name_tool", "-change", ref, new_ref, dll].shelljoin + sh ["env", "LANG=C", cross_ruby.tool("install_name_tool"), "-change", ref, new_ref, dll].shelljoin end # reload! - ldd = `#{["env", "LANG=C", "otool", "-L", dll].shelljoin}` + ldd = `#{["env", "LANG=C", cross_ruby.tool("otool"), "-L", dll].shelljoin}` end # Verify that the DLL dependencies are all allowed. - ldd = `#{["env", "LANG=C", "otool", "-L", dll].shelljoin}` + ldd = `#{["env", "LANG=C", cross_ruby.tool("otool"), "-L", dll].shelljoin}` actual_imports = ldd.scan(/^\t([^ ]+) /).map(&:first).uniq if !(actual_imports - allowed_imports).empty? raise "unallowed so imports #{actual_imports.inspect} in #{dll} (allowed #{allowed_imports.inspect})" @@ -243,13 +248,13 @@ namespace "gem" do Rake::Task["pkg/#{HOE.spec.full_name}-#{Gem::Platform.new(plat).to_s}.gem"].invoke end - CROSS_RUBIES.find_all { |cr| cr.windows? || cr.linux? }.map(&:platform).uniq.each do |plat| + CROSS_RUBIES.find_all { |cr| cr.windows? || cr.linux? || cr.darwin? }.map(&:platform).uniq.each do |plat| desc "build native gem for #{plat} platform" task plat do RakeCompilerDock.sh <<~EOT, platform: plat gem install bundler --no-document && bundle && - bundle exec rake gem:#{plat}:builder MAKE='nice make -j`nproc`' FORCE_CROSS_COMPILING=true + bundle exec rake gem:#{plat}:builder MAKE='nice make -j`nproc`' EOT end @@ -262,21 +267,6 @@ namespace "gem" do end end - CROSS_RUBIES.find_all { |cr| cr.darwin? }.map(&:platform).uniq.each do |plat| - desc "build native gem for #{plat} platform" - task plat do - sh "rake gem:#{plat}:builder MAKE='nice make -j`nproc`' FORCE_CROSS_COMPILING=true" - end - - namespace plat do - desc "build native gem for #{plat} platform (child process)" - task "builder" do - gem_builder(plat) - end - task "guest" => "builder" # TODO: remove me after this code is on master, temporary backwards compat for CI - end - end - desc "build a jruby gem" task "jruby" do RakeCompilerDock.sh("gem install bundler --no-document && bundle && bundle exec rake java gem", @@ -303,8 +293,8 @@ if java? ext.ext_dir = 'ext/java' ext.lib_dir = 'lib/nokogiri' - ext.source_version = '1.6' - ext.target_version = '1.6' + ext.source_version = '1.7' + ext.target_version = '1.7' ext.classpath = jars.map { |x| File.expand_path x }.join ':' ext.debug = true if ENV['JAVA_DEBUG'] end @@ -339,7 +329,6 @@ else Rake::ExtensionTask.new("nokogiri", HOE.spec) do |ext| ext.lib_dir = File.join(*['lib', 'nokogiri', ENV['FAT_DIR']].compact) ext.config_options << ENV['EXTOPTS'] - ext.no_native = (ENV["FORCE_CROSS_COMPILING"] == "true") ext.cross_compile = true ext.cross_platform = CROSS_RUBIES.map(&:platform).uniq ext.cross_config_options << "--enable-cross-build" diff --git a/scripts/build-gems b/scripts/build-gems index d2ca50b300..1c0a8f3181 100755 --- a/scripts/build-gems +++ b/scripts/build-gems @@ -3,28 +3,39 @@ # script to build gems for all relevant platforms # set -o errexit -set -u -x +set -o nounset +set -x rm -rf tmp pkg gems mkdir -p gems -# MRI et al (standard gem) +bundle update +bundle package + bundle exec rake clean bundle exec rake compile test + +# MRI et al (standard gem) +bundle exec rake clean bundle exec rake gem cp -v pkg/nokogiri*.gem gems # jruby bundle exec rake clean bundle exec rake gem:jruby -cp -v pkg/nokogiri*java.gem gems +cp -v pkg/nokogiri-*java*.gem gems -# windows and linux fat binary gems +# precompiled native gems ("fat binary") bundle exec rake clean + bundle exec rake gem:windows +cp -v pkg/nokogiri-*mingw*.gem gems + bundle exec rake gem:linux -cp -v pkg/nokogiri-*{x86,x64}*.gem gems +cp -v pkg/nokogiri-*linux*.gem gems + +bundle exec rake gem:darwin +cp -v pkg/nokogiri-*darwin*.gem gems -for f in gems/*.gem ; do - ./scripts/test-gem-file-contents $f -done +# test those gem files! +$(dirname $0)/test-gem-files gems/*.gem diff --git a/scripts/setup-osx-native-builders b/scripts/setup-osx-native-builders deleted file mode 100755 index 3f9b237017..0000000000 --- a/scripts/setup-osx-native-builders +++ /dev/null @@ -1,58 +0,0 @@ -#! /usr/bin/env bash - -# for inspiration / related cross-ruby building, see: -# - https://github.com/rake-compiler/rake-compiler/blob/master/tasks/bin/cross-ruby.rake -# - https://github.com/apolcyn/grpc/blob/master/tools/distrib/build_ruby_environment_macos.sh - -# see test/test_gem_platform.rb for context -gem_platform_name="x86_64-darwin" - -# chosen so chruby will see it, should you be a user of chruby -RUBIES_DIR=${HOME}/.rubies - -# prerequisites on OSX -if [[ -z "$(which ruby-install)" ]] ; then - echo "ERROR: ruby-install is not installed, please install it: https://github.com/postmodern/ruby-install" - exit 1 -fi - -set -o errexit -set -o pipefail - -CROSS_FILE=".cross_rubies" -RUBIES=$(cat $CROSS_FILE | fgrep darwin | cut -d. -f1,2 | sort -u) - -RAKE_COMPILER_CONFIG_DIR=${HOME}/.rake-compiler -RAKE_COMPILER_CONFIG=${RAKE_COMPILER_CONFIG_DIR}/config.yml -TMP_CONFIG=$(mktemp /tmp/rake-compiler-config.XXXXXXXX) - -echo "---" > $TMP_CONFIG - -for ruby in $RUBIES ; do - ruby_fullname="native-builder-${ruby}" - ruby_minor="${ruby}.0" - - if [[ $ruby == "3.0" ]] ; then - ruby="3.0.0-preview1" - fi - - rbconfig=$(echo ${RUBIES_DIR}/${ruby_fullname}/lib/ruby/${ruby_minor}/*/rbconfig.rb) - if [[ ! -e $rbconfig ]] ; then - echo "installing $ruby_fullname ..." - ruby-install -i ${RUBIES_DIR}/${ruby_fullname} ruby ${ruby} \ - -- \ - --enable-static \ - --disable-shared \ - --without-gmp \ - --disable-install-doc - rbconfig=$(echo $rbconfig) - else - echo "ruby $ruby_fullname is installed" - fi - - echo "rbconfig-${gem_platform_name}-${ruby_minor}: \"${rbconfig}\"" >> $TMP_CONFIG -done - -mkdir -p $RAKE_COMPILER_CONFIG_DIR -cp $TMP_CONFIG $RAKE_COMPILER_CONFIG -cat $RAKE_COMPILER_CONFIG diff --git a/scripts/test-gem-file-contents b/scripts/test-gem-file-contents index 014ec56600..c1a2670a62 100755 --- a/scripts/test-gem-file-contents +++ b/scripts/test-gem-file-contents @@ -86,6 +86,7 @@ describe File.basename(gemfile) do describe "all platforms" do it "contains every ruby file in lib/" do expected = %x(git ls-files lib).split("\n").grep(/\.rb$/).sort + skip "looks like this isn't a git repository" if expected.empty? actual = gemfile_contents.grep(%r{^lib/}).grep(/\.rb$/).sort assert_equal(expected, actual) end @@ -162,12 +163,23 @@ describe File.basename(gemfile) do assert_operator(gemspec.extra_rdoc_files.grep(%r{ext/nokogiri/.*\.c$}).length, :>, 10) end - it "contains expected .so files " do + it "contains expected shared library files " do native_ruby_versions.each do |version| - assert_includes(gemfile_contents, "lib/nokogiri/#{version}/nokogiri.so") + actual = gemfile_contents.find do |p| + File.fnmatch?("lib/nokogiri/#{version}/nokogiri.{so,bundle}", p, File::FNM_EXTGLOB) + end + assert(actual, "expected to find shared library file for ruby #{version}") end - refute_includes(gemfile_contents, "lib/nokogiri/nokogiri.so") - assert_equal(native_ruby_versions.length, Dir.glob("lib/nokogiri/**/*.so").length) + + actual = gemfile_contents.find do |p| + File.fnmatch?("lib/nokogiri/nokogiri.{so,bundle}", p, File::FNM_EXTGLOB) + end + refute(actual, "did not expect to find shared library file in lib/nokogiri") + + actual = gemfile_contents.find_all do |p| + File.fnmatch?("lib/nokogiri/**/*.{so,bundle}", p, File::FNM_EXTGLOB) + end + assert_equal(native_ruby_versions.length, actual.length, "did not expect extra shared library files") end end if gemspec.platform.is_a?(Gem::Platform) && gemspec.platform.cpu diff --git a/scripts/test-gem-set b/scripts/test-gem-set new file mode 100755 index 0000000000..253a792d44 --- /dev/null +++ b/scripts/test-gem-set @@ -0,0 +1,48 @@ +#! /usr/bin/env bash +# +# script to test a set of gem files +# - test-gem-file-contents +# - conditionally, if the local system can do it, test-gem-installation +# +source "$HOME/.rvm/scripts/rvm" + +set -o errexit +set -o pipefail + +gem_platform_local=`ruby -e "puts Gem::Platform.local.to_s"` + +function remove_all_nokogiris { + yes | gem uninstall --force nokogiri || true +} + +function install_and_test { + gem=$1 + if [[ $gem =~ "java" ]] ; then + rvm use jruby + else + rvm use default + fi + remove_all_nokogiris + gem install --local $gem + ./scripts/test-gem-installation + + if [[ $gem =~ nokogiri-[^-]*\.gem ]] ; then + remove_all_nokogiris + NOKOGIRI_USE_SYSTEM_LIBRARIES=t gem install --local $gem + ./scripts/test-gem-installation + fi +} + +gems=$* + +rvm use default + +for gem in $gems ; do + ./scripts/test-gem-file-contents $gem +done + +for gem in $gems ; do + if [[ $gem =~ nokogiri-[^-]+(-(${gem_platform_local}|java))?\.gem$ ]] ; then + install_and_test $gem + fi +done