diff --git a/.hoerc b/.hoerc index 913d1b035b..54c813cf5f 100644 --- a/.hoerc +++ b/.hoerc @@ -4,12 +4,13 @@ exclude: !ruby/regexp '/ (^\.\/ ((\.git + |.vagrant |.yardoc |concourse |gems |ports |suppressions - |tasks + |rakelib |test |tmp )\/) diff --git a/CHANGELOG.md b/CHANGELOG.md index c631eaf619..ac77fd9dc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ This release ends support for: * Improve performance of some namespace operations. [[#1916](https://github.com/sparklemotion/nokogiri/issues/1916)] (Thanks, [@ashmaroli](https://github.com/ashmaroli)!) * Remove unnecessary array allocations from Node serialization methods [[#1911](https://github.com/sparklemotion/nokogiri/issues/1911)] (Thanks, [@ashmaroli](https://github.com/ashmaroli)!) * Avoid creation of unnecessary zero-length String objects. [[#1970](https://github.com/sparklemotion/nokogiri/issues/1970)] (Thanks, [@ashmaroli](https://github.com/ashmaroli)!) +* Always compile libxml2 and libxslt with '-O2' [[#2022](https://github.com/sparklemotion/nokogiri/issues/2022), [#2100](https://github.com/sparklemotion/nokogiri/issues/2100)] (Thanks, [@ilyazub](https://github.com/ilyazub)!) * [JRuby] Lots of code cleanup and performance improvements. [[#1934](https://github.com/sparklemotion/nokogiri/issues/1934)] (Thanks, [@kares](https://github.com/kares)!) * [JRuby] Clean up deprecated calls into JRuby. [[#2027](https://github.com/sparklemotion/nokogiri/issues/2027)] (Thanks, [@headius](https://github.com/headius)!) diff --git a/Rakefile b/Rakefile index 577c684900..2323ee15cf 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,4 @@ -# -*- ruby -*- -require 'hoe' +require "hoe" Hoe.plugin :bundler Hoe.plugin :debugging @@ -7,22 +6,24 @@ Hoe.plugin :gemspec Hoe.plugin :git Hoe.plugin :markdown -require 'shellwords' +require_relative "rakelib/util" -require_relative "tasks/util" +HOE = Hoe.spec "nokogiri" do |hoe| + hoe.author = [ + "Mike Dalessio", + "Aaron Patterson", + "John Shahid", + "Yoko Harada", + "Akinori MUSHA", + "Lars Kanis", + "Tim Elliott", + ] -HOE = Hoe.spec 'nokogiri' do - developer 'Aaron Patterson', 'aaronp@rubyforge.org' - developer 'Mike Dalessio', 'mike.dalessio@gmail.com' - developer 'Yoko Harada', 'yokolet@gmail.com' - developer 'Tim Elliott', 'tle@holymonkey.com' - developer 'Akinori MUSHA', 'knu@idaemons.org' - developer 'John Shahid', 'jvshahid@gmail.com' - developer 'Lars Kanis', 'lars@greiz-reinsdorf.de' + hoe.email = "nokogiri-talk@googlegroups.com" - license "MIT" + hoe.license "MIT" - self.urls = { + hoe.urls = { "home" => "https://nokogiri.org", "bugs" => "https://github.com/sparklemotion/nokogiri/issues", "doco" => "https://nokogiri.org/rdoc/index.html", @@ -30,23 +31,23 @@ HOE = Hoe.spec 'nokogiri' do "code" => "https://github.com/sparklemotion/nokogiri", } - self.markdown_linkify_files = FileList["*.md"] - self.extra_rdoc_files = FileList['ext/nokogiri/*.c'] + hoe.markdown_linkify_files = FileList["*.md"] + hoe.extra_rdoc_files = FileList["ext/nokogiri/*.c"] - self.clean_globs += [ - 'nokogiri.gemspec', - 'lib/nokogiri/nokogiri.{bundle,jar,rb,so}', - 'lib/nokogiri/[0-9].[0-9]', + hoe.clean_globs += [ + "nokogiri.gemspec", + "lib/nokogiri/nokogiri.{bundle,jar,rb,so}", + "lib/nokogiri/[0-9].[0-9]", ] - self.clean_globs += Dir.glob("ports/*").reject { |d| d =~ %r{/archives$} } + hoe.clean_globs += Dir.glob("ports/*").reject { |d| d =~ %r{/archives$} } unless java? - self.extra_deps += [ + hoe.extra_deps += [ ["mini_portile2", "~> 2.5.0"], # keep version in sync with extconf.rb ] end - self.extra_dev_deps += [ + hoe.extra_dev_deps += [ ["concourse", "~> 0.38"], ["hoe", ["~> 3.22", ">= 3.22.1"]], ["hoe-bundler", "~> 1.2"], @@ -64,26 +65,15 @@ HOE = Hoe.spec 'nokogiri' do ["simplecov", "~> 0.17.0"], # locked due to https://github.com/codeclimate/test-reporter/issues/413 ] - self.spec_extras = { + hoe.spec_extras = { :extensions => ["ext/nokogiri/extconf.rb"], - :required_ruby_version => '>= 2.4.0' + :required_ruby_version => ">= 2.4.0" } - self.testlib = :minitest - self.test_prelude = 'require "helper"' # ensure simplecov gets loaded before anything else + hoe.testlib = :minitest + hoe.test_prelude = %q(require "helper") # ensure simplecov gets loaded before anything else end -require_relative "tasks/cross-ruby" -require_relative "tasks/concourse" -require_relative "tasks/css-generate" -require_relative "tasks/debug" -require_relative "tasks/docker" -require_relative "tasks/docs-linkify" -require_relative "tasks/rubocop" -require_relative "tasks/set-version-to-timestamp" - # work around Hoe's inflexibility about the default tasks Rake::Task[:default].prerequisites.unshift("compile") Rake::Task[:default].prerequisites.unshift("rubocop") - -# vim: syntax=Ruby diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 89c6ec56d8..466b406f69 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -1,12 +1,25 @@ # :stopdoc: ENV['RC_ARCHS'] = '' if RUBY_PLATFORM =~ /darwin/ -require 'mkmf' +require "mkmf" +require "rbconfig" +require "fileutils" +require "shellwords" +require "pathname" -ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) +# +# helpful constants +# +PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) +REQUIRED_LIBXML_VERSION = "2.6.21" +RECOMMENDED_LIBXML_VERSION = "2.9.3" + +# The gem version constraint in the Rakefile is not respected at install time. +# Keep this version in sync with the one in the Rakefile ! +REQUIRED_MINI_PORTILE_VERSION = "~> 2.5.0" # -# functions +# utility functions # def windows? RbConfig::CONFIG['target_os'] =~ /mingw32|mswin/ @@ -32,181 +45,132 @@ def nix? ! (windows? || solaris? || darwin?) end -def sh_export_path path - # because libxslt 1.1.29 configure.in uses AC_PATH_TOOL which treats ":" - # as a $PATH separator, we need to convert windows paths from - # - # C:/path/to/foo - # - # to - # - # /C/path/to/foo - # - # which is sh-compatible, in order to find things properly during - # configuration - if windows? - match = Regexp.new("^([A-Z]):(/.*)").match(path) - if match && match.length == 3 - return File.join("/", match[1], match[2]) - end - end - path +def concat_flags *args + args.compact.join(" ") end -def do_help - print < + <<~SRC + #include SRC else version_int = sprintf "%d%2.2d%2.2d", *(version.split(".")) - <<-SRC -#include -#if LIBXML_VERSION < #{version_int} -#error libxml2 is older than #{version} -#endif + <<~SRC + #include + #if LIBXML_VERSION < #{version_int} + # error libxml2 is older than #{version} + #endif SRC end try_cpp source end -def add_cflags(flags) - print "checking if the C compiler accepts #{flags}... " - with_cflags("#{$CFLAGS} #{flags}") do - if nokogiri_try_compile - puts 'yes' - true - else - puts 'no' - false - end - end -end - -def preserving_globals - values = [ - $arg_config, - $CFLAGS, $CPPFLAGS, - $LDFLAGS, $LIBPATH, $libs - ].map(&:dup) - yield -ensure - $arg_config, - $CFLAGS, $CPPFLAGS, - $LDFLAGS, $LIBPATH, $libs = - values -end - -def asplode(lib) - abort "-----\n#{lib} is missing. Please locate mkmf.log to investigate how it is failing.\n-----" -end - -def have_iconv?(using = nil) +def try_link_iconv(using = nil) checking_for(using ? "iconv using #{using}" : 'iconv') do ['', '-liconv'].any? do |opt| preserving_globals do yield if block_given? - try_link(<<-'SRC', opt) -#include -#include - -int main(void) -{ - iconv_t cd = iconv_open("", ""); - iconv(cd, NULL, NULL, NULL, NULL); - return EXIT_SUCCESS; -} + try_link(<<~'SRC', opt) + #include + #include + int main(void) + { + iconv_t cd = iconv_open("", ""); + iconv(cd, NULL, NULL, NULL, NULL); + return EXIT_SUCCESS; + } SRC end end @@ -214,11 +178,10 @@ def have_iconv?(using = nil) end def iconv_configure_flags - # If --with-iconv-dir or --with-opt-dir is given, it should be - # the first priority - %w[iconv opt].each do |name| - if (config = preserving_globals { dir_config(name) }).any? && - have_iconv?("--with-#{name}-* flags") { dir_config(name) } + # give --with-iconv-dir and --with-opt-dir first priority + ["iconv", "opt"].each do |target| + config = preserving_globals { dir_config(target) } + if config.any? && try_link_iconv("--with-#{target}-* flags") { dir_config(target) } idirs, ldirs = config.map do |dirs| Array(dirs).flat_map do |dir| dir.split(File::PATH_SEPARATOR) @@ -233,12 +196,12 @@ def iconv_configure_flags end end - if have_iconv? + if try_link_iconv return ['--with-iconv=yes'] end - if (config = preserving_globals { package_config('libiconv') }) && - have_iconv?('pkg-config libiconv') { package_config('libiconv') } + config = preserving_globals { package_config('libiconv') } + if config && try_link_iconv('pkg-config libiconv') { package_config('libiconv') } cflags, ldflags, libs = config return [ @@ -249,26 +212,21 @@ def iconv_configure_flags ] end - asplode "libiconv" -end - -# When using rake-compiler-dock on Windows, the underlying Virtualbox shared -# folders don't support symlinks, but libiconv expects it for a build on -# Linux. We work around this limitation by using the temp dir for cooking. -def chdir_for_build - build_dir = ENV['RCD_HOST_RUBY_PLATFORM'].to_s =~ /mingw|mswin|cygwin/ ? '/tmp' : '.' - Dir.chdir(build_dir) do - yield - end + abort_could_not_find_library "libiconv" end def process_recipe(name, version, static_p, cross_p) + require 'rubygems' + gem 'mini_portile2', REQUIRED_MINI_PORTILE_VERSION + require 'mini_portile2' + message "Using mini_portile version #{MiniPortile::VERSION}\n" + MiniPortile.new(name, version).tap do |recipe| - recipe.target = File.join(ROOT, "ports") + recipe.target = File.join(PACKAGE_ROOT_DIR, "ports") # Prefer host_alias over host in order to use i586-mingw32msvc as # correct compiler prefix for cross build, but use host if not set. recipe.host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"] - recipe.patch_files = Dir[File.join(ROOT, "patches", name, "*.patch")].sort + recipe.patch_files = Dir[File.join(PACKAGE_ROOT_DIR, "patches", name, "*.patch")].sort recipe.configure_options << "--libdir=#{File.join(recipe.path, "lib")}" yield recipe @@ -282,7 +240,11 @@ def process_recipe(name, version, static_p, cross_p) recipe.configure_options.delete_if do |option| case option when /\A(\w+)=(.*)\z/ - env[$1] = $2 + if env.key?($1) + env[$1] = concat_flags(env[$1], $2) + else + env[$1] = $2 + end true else false @@ -294,7 +256,7 @@ def process_recipe(name, version, static_p, cross_p) "--disable-shared", "--enable-static", ] - env['CFLAGS'] = "-fPIC #{env['CFLAGS']}" + env["CFLAGS"] = concat_flags(env["CFLAGS"], "-fPIC") else recipe.configure_options += [ "--enable-shared", @@ -312,59 +274,50 @@ def process_recipe(name, version, static_p, cross_p) if RbConfig::CONFIG['target_cpu'] == 'universal' %w[CFLAGS LDFLAGS].each do |key| unless env[key].include?('-arch') - env[key] += ' ' + RbConfig::CONFIG['ARCH_FLAG'] + env[key] = concat_flags(env[key], RbConfig::CONFIG['ARCH_FLAG']) end end end recipe.configure_options += env.map do |key, value| - "#{key}=#{value}" + "#{key}=#{value.strip}" end - message <<-"EOS" -************************************************************************ -IMPORTANT NOTICE: - -Building Nokogiri with a packaged version of #{name}-#{version}#{'.' if recipe.patch_files.empty?} - EOS + checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" + if File.exist?(checkpoint) + message "Building Nokogiri with a packaged version of #{name}-#{version}.\n" + else + message <<~EOM + ---------- IMPORTANT NOTICE ---------- + Building Nokogiri with a packaged version of #{name}-#{version}. + Configuration options: #{recipe.configure_options.shelljoin} + EOM - unless recipe.patch_files.empty? - message "with the following patches applied:\n" + unless recipe.patch_files.empty? + message "The following patches are being applied:\n" - recipe.patch_files.each do |patch| - message "\t- %s\n" % File.basename(patch) + recipe.patch_files.each do |patch| + message " - %s\n" % File.basename(patch) + end end - end - message <<-"EOS" + message <<~EOM -Team Nokogiri will keep on doing their best to provide security -updates in a timely manner, but if this is a concern for you and want -to use the system library instead; abort this installation process and -reinstall nokogiri as follows: + The Nokogiri maintainers intend to provide timely security updates, but if + this is a concern for you and want to use your OS/distro system library + instead, then abort this installation process and install nokogiri as + instructed at: - gem install nokogiri -- --use-system-libraries - [--with-xml2-config=/path/to/xml2-config] - [--with-xslt-config=/path/to/xslt-config] + https://nokogiri.org/tutorials/installing_nokogiri.html#install-with-system-libraries -If you are using Bundler, tell it to use the option: + EOM - bundle config build.nokogiri --use-system-libraries - bundle install - EOS + message <<~EOM if name == 'libxml2' + Note, however, that nokogiri cannot guarantee compatiblity with every + version of libxml2 that may be provided by OS/package vendors. - message <<-"EOS" if name == 'libxml2' + EOM -Note, however, that nokogiri is not fully compatible with arbitrary -versions of libxml2 provided by OS/package vendors. - EOS - - message <<-"EOS" -************************************************************************ - EOS - - checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" - unless File.exist?(checkpoint) chdir_for_build do recipe.cook end @@ -374,117 +327,134 @@ def process_recipe(name, version, static_p, cross_p) end end -def lib_a(ldflag) - case ldflag - when /\A-l(.+)/ - "lib#{$1}.#{$LIBEXT}" - end -end +def do_help + print <<~HELP + usage: ruby #{$0} [options] -def using_system_libraries? - # NOTE: TruffleRuby uses this env var as it does not support using static libraries yet. - arg_config('--use-system-libraries', !!ENV['NOKOGIRI_USE_SYSTEM_LIBRARIES']) -end + --disable-clean + Do not clean out intermediate files after successful build. -# -# main -# + --disable-static + Do not statically link bundled libraries. + + --with-iconv-dir=DIR + Use the iconv library placed under DIR. + + --with-zlib-dir=DIR + Use the zlib library placed under DIR. + + --use-system-libraries + Use system libraries instead of building and using the bundled + libraries. + + --with-xml2-dir=DIR / --with-xml2-config=CONFIG + --with-xslt-dir=DIR / --with-xslt-config=CONFIG + --with-exslt-dir=DIR / --with-exslt-config=CONFIG + Use libxml2/libxslt/libexslt as specified. -case -when arg_config('--help') - do_help -when arg_config('--clean') - do_clean + --enable-cross-build + Do cross-build. + HELP + exit! 0 end -if darwin? - ENV['CFLAGS'] = "#{ENV['CFLAGS']} -I /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libxml2" +def do_clean + root = Pathname(PACKAGE_ROOT_DIR) + pwd = Pathname(Dir.pwd) + + # Skip if this is a development work tree + unless (root + '.git').exist? + message "Cleaning files only used during build.\n" + + # (root + 'tmp') cannot be removed at this stage because + # nokogiri.so is yet to be copied to lib. + + # clean the ports build directory + Pathname.glob(pwd.join('tmp', '*', 'ports')) do |dir| + FileUtils.rm_rf(dir, verbose: true) + end + + if enable_config('static') + # ports installation can be safely removed if statically linked. + FileUtils.rm_rf(root + 'ports', verbose: true) + else + FileUtils.rm_rf(root + 'ports' + 'archives', verbose: true) + end + end + + exit! 0 end +# +# main +# +do_help if arg_config('--help') +do_clean if arg_config('--clean') + if openbsd? && !using_system_libraries? if `#{ENV['CC'] || '/usr/bin/cc'} -v 2>&1` !~ /clang/ ENV['CC'] ||= find_executable('egcc') or abort "Please install gcc 4.9+ from ports using `pkg_add -v gcc`" end - ENV['CFLAGS'] = "#{ENV['CFLAGS']} -I /usr/local/include" + append_cppflags "-I /usr/local/include" end if ENV['CC'] RbConfig::CONFIG['CC'] = RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] end + # use same c compiler for libxml and libxslt ENV['CC'] = RbConfig::CONFIG['CC'] -$LIBS << " #{ENV["LIBS"]}" +# adopt environment config +append_cflags(ENV["CFLAGS"].split(/\s+/)) if !ENV["CFLAGS"].nil? +append_cppflags(ENV["CPPFLAGS"].split(/\s+/)) if !ENV["CPPFLAGS"].nil? +append_ldflags(ENV["LDFLAGS"].split(/\s+/)) if !ENV["LDFLAGS"].nil? +$LIBS = concat_flags($LIBS, ENV["LIBS"]) -# Read CFLAGS from ENV and make sure compiling works. -add_cflags(ENV["CFLAGS"]) +append_cflags("-g") # always include debugging information +append_cflags("-Winline") # we use at least one inline function in the C extension +append_cflags("-Wmissing-noreturn") # good to have no matter what Ruby was compiled with +append_cflags("-Wno-error=unused-command-line-argument-hard-error-in-future") if darwin? +# append_cflags(["-Wcast-qual", "-Wwrite-strings"]) # these tend to be noisy, but on occasion useful during development -if windows? - $CFLAGS << " -DXP_WIN -DXP_WIN32 -DUSE_INCLUDED_VASPRINTF" +# Add SDK-specific include path for macOS and brew versions before v2.2.12 (2020-04-08) [#1851, #1801] +macos_mojave_sdk_include_path = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libxml2" +if using_system_libraries? && darwin? && Dir.exist?(macos_mojave_sdk_include_path) + append_cppflags("-I #{macos_mojave_sdk_include_path}") end -if solaris? || aix? - $CFLAGS << " -DUSE_INCLUDED_VASPRINTF" -end +# Work around a character escaping bug in MSYS by passing an arbitrary +# double quoted parameter to gcc. See https://sourceforge.net/p/mingw/bugs/2142 +append_cppflags(' "-Idummypath"') if windows? -if darwin? - # Let Apple LLVM/clang 5.1 ignore unknown compiler flags - add_cflags("-Wno-error=unused-command-line-argument-hard-error-in-future") -end - -if nix? - $CFLAGS << " -g -DXP_UNIX" -end - -if RUBY_PLATFORM =~ /mingw/i - # Work around a character escaping bug in MSYS by passing an arbitrary - # double quoted parameter to gcc. See https://sourceforge.net/p/mingw/bugs/2142 - $CPPFLAGS << ' "-Idummypath"' -end - -if RbConfig::CONFIG['CC'] =~ /gcc/ - $CFLAGS << " -O3" unless $CFLAGS[/-O\d/] - $CFLAGS << " -Wall -Wcast-qual -Wwrite-strings -Wmissing-noreturn -Winline" -end - -case -when using_system_libraries? +if using_system_libraries? message "Building nokogiri using system libraries.\n" + # Using system libraries means we rely on the system libxml2 for iconv support (or not) dir_config('zlib') - - # Using system libraries means we rely on the system libxml2 with - # regard to the iconv support. - dir_config('xml2').any? or package_config('libxml-2.0') dir_config('xslt').any? or package_config('libxslt') dir_config('exslt').any? or package_config('libexslt') - check_libxml_version or abort "ERROR: cannot discover where libxml2 is located on your system. please make sure `pkg-config` is installed." - check_libxml_version("2.6.21") or abort "ERROR: libxml2 version 2.6.21 or later is required!" - check_libxml_version("2.9.3") or warn "WARNING: libxml2 version 2.9.3 or later is highly recommended, but proceeding anyway." + have_libxml_headers? or + abort "ERROR: cannot discover where libxml2 is located on your system. please make sure `pkg-config` is installed." + have_libxml_headers?(REQUIRED_LIBXML_VERSION) or + abort "ERROR: libxml2 version #{REQUIRED_LIBXML_VERSION} or later is required!" + have_libxml_headers?(RECOMMENDED_LIBXML_VERSION) or + warn "WARNING: libxml2 version #{RECOMMENDED_LIBXML_VERSION} or later is highly recommended, but proceeding anyway." else message "Building nokogiri using packaged libraries.\n" - # The gem version constraint in the Rakefile is not respected at install time. - # Keep this version in sync with the one in the Rakefile ! - require 'rubygems' - gem 'mini_portile2', '~> 2.5.0' - require 'mini_portile2' - message "Using mini_portile version #{MiniPortile::VERSION}\n" + static_p = enable_config('static', true) or message "Static linking is disabled.\n" + cross_build_p = enable_config("cross-build") require 'yaml' - - static_p = enable_config('static', true) or - message "Static linking is disabled.\n" + dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, "dependencies.yml")) dir_config('zlib') - dependencies = YAML.load_file(File.join(ROOT, "dependencies.yml")) - - cross_build_p = enable_config("cross-build") if cross_build_p || windows? zlib_recipe = process_recipe("zlib", dependencies["zlib"]["version"], static_p, cross_build_p) do |recipe| recipe.files = [{ @@ -526,7 +496,8 @@ def install else class << recipe def configure - execute "configure", ["env", "CHOST=#{host}", "CFLAGS=-fPIC #{ENV['CFLAGS']}", "./configure", "--static", configure_prefix] + cflags = concat_flags(ENV["CFLAGS"], "-fPIC", "-g") + execute "configure", ["env", "CHOST=#{host}", "CFLAGS=#{cflags}", "./configure", "--static", configure_prefix] end end end @@ -538,35 +509,37 @@ def configure url: "http://ftp.gnu.org/pub/gnu/libiconv/#{recipe.name}-#{recipe.version}.tar.gz", sha256: dependencies["libiconv"]["sha256"] }] + + cflags = concat_flags(ENV["CFLAGS"], "-O2", "-U_FORTIFY_SOURCE", "-g") + recipe.configure_options += [ "CPPFLAGS=-Wall", - "CFLAGS=-O2 -g", - "CXXFLAGS=-O2 -g", + "CFLAGS=#{cflags}", + "CXXFLAGS=#{cflags}", "LDFLAGS=" ] end end else if darwin? && !have_header('iconv.h') - abort <<'EOM'.chomp ------ -The file "iconv.h" is missing in your build environment, -which means you haven't installed Xcode Command Line Tools properly. - -To install Command Line Tools, try running `xcode-select --install` on -terminal and follow the instructions. If it fails, open Xcode.app, -select from the menu "Xcode" - "Open Developer Tool" - "More Developer -Tools" to open the developer site, download the installer for your OS -version and run it. ------ -EOM + abort <<~EOM.chomp + ----- + The file "iconv.h" is missing in your build environment, + which means you haven't installed Xcode Command Line Tools properly. + + To install Command Line Tools, try running `xcode-select --install` on + terminal and follow the instructions. If it fails, open Xcode.app, + select from the menu "Xcode" - "Open Developer Tool" - "More Developer + Tools" to open the developer site, download the installer for your OS + version and run it. + ----- + EOM end end unless windows? - preserving_globals { - have_library('z', 'gzdopen', 'zlib.h') - } or abort 'zlib is missing; necessary for building libxml2' + preserving_globals { have_library('z', 'gzdopen', 'zlib.h') } or + abort 'zlib is missing; necessary for building libxml2' end libxml2_recipe = process_recipe("libxml2", dependencies["libxml2"]["version"], static_p, cross_build_p) do |recipe| @@ -574,15 +547,31 @@ def configure url: "http://xmlsoft.org/sources/#{recipe.name}-#{recipe.version}.tar.gz", sha256: dependencies["libxml2"]["sha256"] }] + + cflags = concat_flags(ENV["CFLAGS"], "-O2", "-U_FORTIFY_SOURCE", "-g") + + if zlib_recipe + recipe.configure_options << "--with-zlib=#{zlib_recipe.path}" + cflags = concat_flags(cflags, "-I#{zlib_recipe.path}/include") + end + + if libiconv_recipe + recipe.configure_options << "--with-iconv=#{libiconv_recipe.path}" + else + recipe.configure_options += iconv_configure_flags + end + + if darwin? + recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] + end + recipe.configure_options += [ "--without-python", "--without-readline", - *(zlib_recipe ? ["--with-zlib=#{zlib_recipe.path}", "CFLAGS=-I#{zlib_recipe.path}/include"] : []), - *(libiconv_recipe ? "--with-iconv=#{libiconv_recipe.path}" : iconv_configure_flags), "--with-c14n", "--with-debug", "--with-threads", - *(darwin? ? ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] : "") + "CFLAGS=#{cflags}", ] end @@ -591,22 +580,27 @@ def configure url: "http://xmlsoft.org/sources/#{recipe.name}-#{recipe.version}.tar.gz", sha256: dependencies["libxslt"]["sha256"] }] + + cflags = concat_flags(ENV["CFLAGS"], "-O2", "-U_FORTIFY_SOURCE", "-g") + + if darwin? + recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] + end + recipe.configure_options += [ "--without-python", "--without-crypto", "--with-debug", "--with-libxml-prefix=#{sh_export_path(libxml2_recipe.path)}", - *(darwin? ? ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] : "") + "CFLAGS=#{cflags}", ] end - $CFLAGS << ' ' << '-DNOKOGIRI_USE_PACKAGED_LIBRARIES' + $CFLAGS = concat_flags($CFLAGS, "-DNOKOGIRI_USE_PACKAGED_LIBRARIES") $LIBPATH = ["#{zlib_recipe.path}/lib"] | $LIBPATH if zlib_recipe $LIBPATH = ["#{libiconv_recipe.path}/lib"] | $LIBPATH if libiconv_recipe - have_lzma = preserving_globals { - have_library('lzma') - } + have_lzma = preserving_globals { have_library('lzma') } $libs = $libs.shellsplit.tap do |libs| [libxml2_recipe, libxslt_recipe].each do |recipe| @@ -618,7 +612,7 @@ def configure case arg when /\A-L(.+)\z/ # Prioritize ports' directories - if $1.start_with?(ROOT + '/') + if $1.start_with?(PACKAGE_ROOT_DIR + '/') $LIBPATH = [$1] | $LIBPATH else $LIBPATH = $LIBPATH | [$1] @@ -654,9 +648,9 @@ def configure $libs = $libs.shellsplit.map do |arg| case arg when '-lxml2' - File.join(libxml2_recipe.path, 'lib', lib_a(arg)) + File.join(libxml2_recipe.path, 'lib', libflag_to_filename(arg)) when '-lxslt', '-lexslt' - File.join(libxslt_recipe.path, 'lib', lib_a(arg)) + File.join(libxslt_recipe.path, 'lib', libflag_to_filename(arg)) else arg end @@ -664,36 +658,39 @@ def configure end end -{ - "xml2" => ['xmlParseDoc', 'libxml/parser.h'], - "xslt" => ['xsltParseStylesheetDoc', 'libxslt/xslt.h'], - "exslt" => ['exsltFuncRegister', 'libexslt/exslt.h'], -}.each do |lib, (func, header)| - have_func(func, header) || - have_library(lib, func, header) || - have_library("lib#{lib}", func, header) or - asplode("lib#{lib}") +have_func('vasprintf') + +[ + ["xml2", "xmlParseDoc", "libxml/parser.h"], + ["xslt", "xsltParseStylesheetDoc", "libxslt/xslt.h"], + ["exslt", "exsltFuncRegister", "libexslt/exslt.h"] +].each do |lib, func, header| + checking_for "lib#{lib}" do + have_func(func, header) or + have_library(lib, func, header) or + have_library("lib#{lib}", func, header) or + abort_could_not_find_library("lib#{lib}") + end end -have_func('xmlHasFeature') or abort "xmlHasFeature() is missing." -have_func('xmlFirstElementChild') -have_func('xmlRelaxNGSetParserStructuredErrors') -have_func('xmlRelaxNGSetParserStructuredErrors') -have_func('xmlRelaxNGSetValidStructuredErrors') -have_func('xmlSchemaSetValidStructuredErrors') -have_func('xmlSchemaSetParserStructuredErrors') +have_func('xmlHasFeature') or abort "xmlHasFeature() is missing." # introduced in libxml 2.6.21 +have_func('xmlFirstElementChild') # introduced in libxml 2.7.3 +have_func('xmlRelaxNGSetParserStructuredErrors') # introduced in libxml 2.6.24 +have_func('xmlRelaxNGSetValidStructuredErrors') # introduced in libxml 2.6.21 +have_func('xmlSchemaSetValidStructuredErrors') # introduced in libxml 2.6.23 +have_func('xmlSchemaSetParserStructuredErrors') # introduced in libxml 2.6.23 create_makefile('nokogiri/nokogiri') if enable_config('clean', true) # Do not clean if run in a development work tree. File.open('Makefile', 'at') do |mk| - mk.print < -#include -#include +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif /* WIN32_LEAN_AND_MEAN */ + +# ifndef WIN32 +# define WIN32 +# endif /* WIN32 */ + +# include +# include +# include #endif #include #include #include #include - -#ifdef USE_INCLUDED_VASPRINTF -int vasprintf (char **strp, const char *fmt, va_list ap); -#else - -#define _GNU_SOURCE -# include -#undef _GNU_SOURCE - -#endif +#include #include #include @@ -43,6 +36,7 @@ int vasprintf (char **strp, const char *fmt, va_list ap); #include #include #include + #include #include #include @@ -106,25 +100,27 @@ extern VALUE mNokogiriHtml ; extern VALUE mNokogiriHtmlSax ; extern VALUE mNokogiriXslt ; +int vasprintf(char **strp, const char *fmt, va_list ap); + void nokogiri_root_node(xmlNodePtr); void nokogiri_root_nsdef(xmlNsPtr, xmlDocPtr); #ifdef DEBUG -#define NOKOGIRI_DEBUG_START(p) if (getenv("NOKOGIRI_NO_FREE")) return ; if (getenv("NOKOGIRI_DEBUG")) fprintf(stderr,"nokogiri: %s:%d %p start\n", __FILE__, __LINE__, p); -#define NOKOGIRI_DEBUG_END(p) if (getenv("NOKOGIRI_DEBUG")) fprintf(stderr,"nokogiri: %s:%d %p end\n", __FILE__, __LINE__, p); +# define NOKOGIRI_DEBUG_START(p) if (getenv("NOKOGIRI_NO_FREE")) return ; if (getenv("NOKOGIRI_DEBUG")) fprintf(stderr,"nokogiri: %s:%d %p start\n", __FILE__, __LINE__, p); +# define NOKOGIRI_DEBUG_END(p) if (getenv("NOKOGIRI_DEBUG")) fprintf(stderr,"nokogiri: %s:%d %p end\n", __FILE__, __LINE__, p); #else -#define NOKOGIRI_DEBUG_START(p) -#define NOKOGIRI_DEBUG_END(p) +# define NOKOGIRI_DEBUG_START(p) +# define NOKOGIRI_DEBUG_END(p) #endif #ifndef __builtin_expect -# if defined(__GNUC__) -# define __builtin_expect(expr, c) __builtin_expect((long)(expr), (long)(c)) -# endif +# if defined(__GNUC__) +# define __builtin_expect(expr, c) __builtin_expect((long)(expr), (long)(c)) +# endif #endif #define XMLNS_PREFIX "xmlns" diff --git a/ext/nokogiri/xml_node.c b/ext/nokogiri/xml_node.c index 3cde12ad67..e0beb8b36e 100644 --- a/ext/nokogiri/xml_node.c +++ b/ext/nokogiri/xml_node.c @@ -301,7 +301,7 @@ static VALUE reparent_node_with(VALUE pivot_obj, VALUE reparentee_obj, pivot_rep * issue #391, where new node's prefix may become the string "default" * see libxml2 tree.c xmlNewReconciliedNs which implements this behavior. */ - xmlFree(reparentee->ns->prefix); + xmlFree((xmlChar*)reparentee->ns->prefix); reparentee->ns->prefix = NULL; } } diff --git a/ext/nokogiri/xml_sax_parser.c b/ext/nokogiri/xml_sax_parser.c index 1a5f6c5f51..0c7d45ec30 100644 --- a/ext/nokogiri/xml_sax_parser.c +++ b/ext/nokogiri/xml_sax_parser.c @@ -1,8 +1,5 @@ #include -int vasprintf (char **strp, const char *fmt, va_list ap); -void vasprintf_free (void *p); - static ID id_start_document, id_end_document, id_start_element, id_end_element; static ID id_start_element_namespace, id_end_element_namespace; static ID id_comment, id_characters, id_xmldecl, id_error, id_warning; @@ -206,7 +203,7 @@ static void warning_func(void * ctx, const char *msg, ...) va_end(args); ruby_message = NOKOGIRI_STR_NEW2(message); - vasprintf_free(message); + free(message); rb_funcall(doc, id_warning, 1, ruby_message); } @@ -223,7 +220,7 @@ static void error_func(void * ctx, const char *msg, ...) va_end(args); ruby_message = NOKOGIRI_STR_NEW2(message); - vasprintf_free(message); + free(message); rb_funcall(doc, id_error, 1, ruby_message); } diff --git a/ext/nokogiri/xml_xpath_context.c b/ext/nokogiri/xml_xpath_context.c index cb1be24b72..94f47f640b 100644 --- a/ext/nokogiri/xml_xpath_context.c +++ b/ext/nokogiri/xml_xpath_context.c @@ -1,7 +1,5 @@ #include -int vasprintf (char **strp, const char *fmt, va_list ap); - static void deallocate(xmlXPathContextPtr ctx) { NOKOGIRI_DEBUG_START(ctx); diff --git a/ext/nokogiri/xslt_stylesheet.c b/ext/nokogiri/xslt_stylesheet.c index 779294badd..73b975044e 100644 --- a/ext/nokogiri/xslt_stylesheet.c +++ b/ext/nokogiri/xslt_stylesheet.c @@ -7,9 +7,6 @@ VALUE xslt; -int vasprintf (char **strp, const char *fmt, va_list ap); -void vasprintf_free (void *p); - static void mark(nokogiriXsltStylesheetTuple *wrapper) { rb_gc_mark(wrapper->func_instances); @@ -37,7 +34,7 @@ static void xslt_generic_error_handler(void * ctx, const char *msg, ...) rb_str_cat2((VALUE)ctx, message); - vasprintf_free(message); + free(message); } VALUE Nokogiri_wrap_xslt_stylesheet(xsltStylesheetPtr ss) diff --git a/tasks/concourse.rb b/rakelib/concourse.rake similarity index 100% rename from tasks/concourse.rb rename to rakelib/concourse.rake diff --git a/tasks/cross-ruby.rb b/rakelib/cross-ruby.rake similarity index 98% rename from tasks/cross-ruby.rb rename to rakelib/cross-ruby.rake index 27a1599536..a40c482783 100644 --- a/tasks/cross-ruby.rb +++ b/rakelib/cross-ruby.rake @@ -1,3 +1,6 @@ +require "rbconfig" +require "shellwords" + CrossRuby = Struct.new(:version, :host) do WINDOWS_PLATFORM_REGEX = /mingw|mswin/ MINGW32_PLATFORM_REGEX = /mingw32/ @@ -238,7 +241,7 @@ def verify_dll(dll, cross_ruby) desc "build native gem for #{plat} platform (host)" task plat do # TODO remove `find` after https://github.com/rake-compiler/rake-compiler/pull/171 is shipped - RakeCompilerDock.sh <<-EOT, platform: plat + RakeCompilerDock.sh <<~EOT, platform: plat gem install bundler --no-document && bundle && find /usr/local/rvm/gems -name extensiontask.rb | while read f ; do sudo sed -i 's/callback.call(spec) if callback/@cross_compiling.call(spec) if @cross_compiling/' $f ; done && @@ -330,10 +333,10 @@ def verify_dll(dll, cross_ruby) ext.cross_config_options << "--enable-cross-build" ext.cross_compiling do |spec| libs = dependencies.map { |name, dep| "#{name}-#{dep["version"]}" }.join(', ') - - spec.post_install_message = <<-EOS -Nokogiri is built with the packaged libraries: #{libs}. + spec.post_install_message = <<~EOS + Nokogiri is built with the packaged libraries: #{libs}. EOS + spec.files.reject! { |path| File.fnmatch?('ports/*', path) } spec.dependencies.reject! { |dep| dep.name=='mini_portile2' } end diff --git a/tasks/css-generate.rb b/rakelib/css-generate.rake similarity index 100% rename from tasks/css-generate.rb rename to rakelib/css-generate.rake diff --git a/tasks/debug.rb b/rakelib/debug.rake similarity index 100% rename from tasks/debug.rb rename to rakelib/debug.rake diff --git a/tasks/docker.rb b/rakelib/docker.rake similarity index 100% rename from tasks/docker.rb rename to rakelib/docker.rake diff --git a/tasks/rubocop.rb b/rakelib/rubocop.rake similarity index 100% rename from tasks/rubocop.rb rename to rakelib/rubocop.rake diff --git a/tasks/set-version-to-timestamp.rb b/rakelib/set-version-to-timestamp.rake similarity index 100% rename from tasks/set-version-to-timestamp.rb rename to rakelib/set-version-to-timestamp.rake diff --git a/tasks/util.rb b/rakelib/util.rb similarity index 100% rename from tasks/util.rb rename to rakelib/util.rb diff --git a/tasks/docs-linkify.rb b/tasks/docs-linkify.rb deleted file mode 100644 index 200eeed5b6..0000000000 --- a/tasks/docs-linkify.rb +++ /dev/null @@ -1,53 +0,0 @@ -linkify_tasks = [] - -namespace "docs:linkify" do - CHANGELOG_PATH = File.expand_path(File.join(File.dirname(__FILE__), "..", HOE.history_file)) - ROADMAP_PATH = File.expand_path(File.join(File.dirname(__FILE__), "..", "ROADMAP.md")) - - [CHANGELOG_PATH, ROADMAP_PATH].each do |docfile_path| - docfile_name = File.basename(docfile_path) - taskname = docfile_name.downcase.split(".").first - - # this task is idempotent - desc "link issues and usernames in #{docfile_name}" - task taskname do - puts "linkifying #{docfile_path} ..." - docs = File.read(docfile_path) - - github_issue_regex = / - \#([[:digit:]]+) # issue number, like '#1234' - (?!\]\() # not already in a markdown hyperlink - (?![[[:digit:]]]) # don't truncate the issue number to meet the previous negative lookahead - /x - docs.gsub!(github_issue_regex, "[#\\1](#{HOE.urls["bugs"]}/\\1)") - - github_link_regex = %r{ - (? linkify_tasks