From f4663b342ee1c5f2e63f71a5394c9cdf42b8766f Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 13 Oct 2020 08:06:31 -0400 Subject: [PATCH 01/33] cleanup: no need to check for HAVE_RUBY_UTIL_H in C extension which was introduced in ruby v1_9_0_3 --- ext/nokogiri/nokogiri.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ext/nokogiri/nokogiri.c b/ext/nokogiri/nokogiri.c index 740b30d5a0..292ffee358 100644 --- a/ext/nokogiri/nokogiri.c +++ b/ext/nokogiri/nokogiri.c @@ -33,11 +33,7 @@ void vasprintf_free (void *p) free(p); } -#ifdef HAVE_RUBY_UTIL_H #include "ruby/util.h" -#else -#include "util.h" -#endif void nokogiri_root_node(xmlNodePtr node) { From d2165e6024cd9a9152631a08e04fa2f96b0b2826 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 12 Oct 2020 16:10:56 -0400 Subject: [PATCH 02/33] info: add libxml2 version information to extconf have_func calls this will be useful to do some cleanup when we eventually decide to bump our libxml2 dependency. --- ext/nokogiri/extconf.rb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 89c6ec56d8..7c8b447f72 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -675,13 +675,12 @@ def configure asplode("lib#{lib}") end -have_func('xmlHasFeature') or abort "xmlHasFeature() is missing." -have_func('xmlFirstElementChild') -have_func('xmlRelaxNGSetParserStructuredErrors') -have_func('xmlRelaxNGSetParserStructuredErrors') -have_func('xmlRelaxNGSetValidStructuredErrors') -have_func('xmlSchemaSetValidStructuredErrors') -have_func('xmlSchemaSetParserStructuredErrors') +have_func('xmlHasFeature') or abort "xmlHasFeature() is missing." # introduced in libxml 2.6.21 +have_func('xmlFirstElementChild') # introduced in libxml 2.7.3 +have_func('xmlRelaxNGSetParserStructuredErrors') # introduced in libxml 2.6.24 +have_func('xmlRelaxNGSetValidStructuredErrors') # introduced in libxml 2.6.21 +have_func('xmlSchemaSetValidStructuredErrors') # introduced in libxml 2.6.23 +have_func('xmlSchemaSetParserStructuredErrors') # introduced in libxml 2.6.23 create_makefile('nokogiri/nokogiri') From 3d9a23a0ee7a2b5854ede170e645a0825306aa84 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 12 Oct 2020 16:30:53 -0400 Subject: [PATCH 03/33] cleanup: use have_func to check for vasprintf and - remove platform-specific "-D" flags - remove `vasprintf_free` - change calls to `vasprintf_free` to call `free` - whitespace --- ext/nokogiri/extconf.rb | 7 ++--- ext/nokogiri/nokogiri.c | 10 ++----- ext/nokogiri/nokogiri.h | 48 +++++++++++++++----------------- ext/nokogiri/xml_sax_parser.c | 7 ++--- ext/nokogiri/xml_xpath_context.c | 2 -- ext/nokogiri/xslt_stylesheet.c | 5 +--- 6 files changed, 29 insertions(+), 50 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 7c8b447f72..861dd6545a 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -421,11 +421,7 @@ def using_system_libraries? add_cflags(ENV["CFLAGS"]) if windows? - $CFLAGS << " -DXP_WIN -DXP_WIN32 -DUSE_INCLUDED_VASPRINTF" -end - -if solaris? || aix? - $CFLAGS << " -DUSE_INCLUDED_VASPRINTF" + $CFLAGS << " -DXP_WIN -DXP_WIN32" end if darwin? @@ -675,6 +671,7 @@ def configure asplode("lib#{lib}") end +have_func('vasprintf') have_func('xmlHasFeature') or abort "xmlHasFeature() is missing." # introduced in libxml 2.6.21 have_func('xmlFirstElementChild') # introduced in libxml 2.7.3 have_func('xmlRelaxNGSetParserStructuredErrors') # introduced in libxml 2.6.24 diff --git a/ext/nokogiri/nokogiri.c b/ext/nokogiri/nokogiri.c index 292ffee358..b68cd5866a 100644 --- a/ext/nokogiri/nokogiri.c +++ b/ext/nokogiri/nokogiri.c @@ -7,12 +7,11 @@ VALUE mNokogiriXslt ; VALUE mNokogiriXmlSax ; VALUE mNokogiriHtmlSax ; -#ifdef USE_INCLUDED_VASPRINTF +#ifndef HAVE_VASPRINTF /* - * I srsly hate windows. it doesn't have vasprintf. * Thank you Geoffroy Couprie for this implementation of vasprintf! */ -int vasprintf (char **strp, const char *fmt, va_list ap) +int vasprintf(char **strp, const char *fmt, va_list ap) { /* Mingw32/64 have a broken vsnprintf implementation that fails when * using a zero-byte limit in order to retrieve the required size for malloc. @@ -28,11 +27,6 @@ int vasprintf (char **strp, const char *fmt, va_list ap) } #endif -void vasprintf_free (void *p) -{ - free(p); -} - #include "ruby/util.h" void nokogiri_root_node(xmlNodePtr node) diff --git a/ext/nokogiri/nokogiri.h b/ext/nokogiri/nokogiri.h index 2dad57633b..f4e9177b2c 100644 --- a/ext/nokogiri/nokogiri.h +++ b/ext/nokogiri/nokogiri.h @@ -2,31 +2,24 @@ #define NOKOGIRI_NATIVE #if _MSC_VER -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif /* WIN32_LEAN_AND_MEAN */ -#ifndef WIN32 -#define WIN32 -#endif /* WIN32 */ -#include -#include -#include +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif /* WIN32_LEAN_AND_MEAN */ + +# ifndef WIN32 +# define WIN32 +# endif /* WIN32 */ + +# include +# include +# include #endif #include #include #include #include - -#ifdef USE_INCLUDED_VASPRINTF -int vasprintf (char **strp, const char *fmt, va_list ap); -#else - -#define _GNU_SOURCE -# include -#undef _GNU_SOURCE - -#endif +#include #include #include @@ -43,6 +36,7 @@ int vasprintf (char **strp, const char *fmt, va_list ap); #include #include #include + #include #include #include @@ -106,25 +100,27 @@ extern VALUE mNokogiriHtml ; extern VALUE mNokogiriHtmlSax ; extern VALUE mNokogiriXslt ; +int vasprintf(char **strp, const char *fmt, va_list ap); + void nokogiri_root_node(xmlNodePtr); void nokogiri_root_nsdef(xmlNsPtr, xmlDocPtr); #ifdef DEBUG -#define NOKOGIRI_DEBUG_START(p) if (getenv("NOKOGIRI_NO_FREE")) return ; if (getenv("NOKOGIRI_DEBUG")) fprintf(stderr,"nokogiri: %s:%d %p start\n", __FILE__, __LINE__, p); -#define NOKOGIRI_DEBUG_END(p) if (getenv("NOKOGIRI_DEBUG")) fprintf(stderr,"nokogiri: %s:%d %p end\n", __FILE__, __LINE__, p); +# define NOKOGIRI_DEBUG_START(p) if (getenv("NOKOGIRI_NO_FREE")) return ; if (getenv("NOKOGIRI_DEBUG")) fprintf(stderr,"nokogiri: %s:%d %p start\n", __FILE__, __LINE__, p); +# define NOKOGIRI_DEBUG_END(p) if (getenv("NOKOGIRI_DEBUG")) fprintf(stderr,"nokogiri: %s:%d %p end\n", __FILE__, __LINE__, p); #else -#define NOKOGIRI_DEBUG_START(p) -#define NOKOGIRI_DEBUG_END(p) +# define NOKOGIRI_DEBUG_START(p) +# define NOKOGIRI_DEBUG_END(p) #endif #ifndef __builtin_expect -# if defined(__GNUC__) -# define __builtin_expect(expr, c) __builtin_expect((long)(expr), (long)(c)) -# endif +# if defined(__GNUC__) +# define __builtin_expect(expr, c) __builtin_expect((long)(expr), (long)(c)) +# endif #endif #define XMLNS_PREFIX "xmlns" diff --git a/ext/nokogiri/xml_sax_parser.c b/ext/nokogiri/xml_sax_parser.c index 1a5f6c5f51..0c7d45ec30 100644 --- a/ext/nokogiri/xml_sax_parser.c +++ b/ext/nokogiri/xml_sax_parser.c @@ -1,8 +1,5 @@ #include -int vasprintf (char **strp, const char *fmt, va_list ap); -void vasprintf_free (void *p); - static ID id_start_document, id_end_document, id_start_element, id_end_element; static ID id_start_element_namespace, id_end_element_namespace; static ID id_comment, id_characters, id_xmldecl, id_error, id_warning; @@ -206,7 +203,7 @@ static void warning_func(void * ctx, const char *msg, ...) va_end(args); ruby_message = NOKOGIRI_STR_NEW2(message); - vasprintf_free(message); + free(message); rb_funcall(doc, id_warning, 1, ruby_message); } @@ -223,7 +220,7 @@ static void error_func(void * ctx, const char *msg, ...) va_end(args); ruby_message = NOKOGIRI_STR_NEW2(message); - vasprintf_free(message); + free(message); rb_funcall(doc, id_error, 1, ruby_message); } diff --git a/ext/nokogiri/xml_xpath_context.c b/ext/nokogiri/xml_xpath_context.c index cb1be24b72..94f47f640b 100644 --- a/ext/nokogiri/xml_xpath_context.c +++ b/ext/nokogiri/xml_xpath_context.c @@ -1,7 +1,5 @@ #include -int vasprintf (char **strp, const char *fmt, va_list ap); - static void deallocate(xmlXPathContextPtr ctx) { NOKOGIRI_DEBUG_START(ctx); diff --git a/ext/nokogiri/xslt_stylesheet.c b/ext/nokogiri/xslt_stylesheet.c index 779294badd..73b975044e 100644 --- a/ext/nokogiri/xslt_stylesheet.c +++ b/ext/nokogiri/xslt_stylesheet.c @@ -7,9 +7,6 @@ VALUE xslt; -int vasprintf (char **strp, const char *fmt, va_list ap); -void vasprintf_free (void *p); - static void mark(nokogiriXsltStylesheetTuple *wrapper) { rb_gc_mark(wrapper->func_instances); @@ -37,7 +34,7 @@ static void xslt_generic_error_handler(void * ctx, const char *msg, ...) rb_str_cat2((VALUE)ctx, message); - vasprintf_free(message); + free(message); } VALUE Nokogiri_wrap_xslt_stylesheet(xsltStylesheetPtr ss) From 0655f76400a174bf8241d0b77e64f1cb0434d83c Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 12 Oct 2020 16:41:02 -0400 Subject: [PATCH 04/33] cleanup: fix xml_node.c type cast warning --- ext/nokogiri/xml_node.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/nokogiri/xml_node.c b/ext/nokogiri/xml_node.c index 3cde12ad67..e0beb8b36e 100644 --- a/ext/nokogiri/xml_node.c +++ b/ext/nokogiri/xml_node.c @@ -301,7 +301,7 @@ static VALUE reparent_node_with(VALUE pivot_obj, VALUE reparentee_obj, pivot_rep * issue #391, where new node's prefix may become the string "default" * see libxml2 tree.c xmlNewReconciliedNs which implements this behavior. */ - xmlFree(reparentee->ns->prefix); + xmlFree((xmlChar*)reparentee->ns->prefix); reparentee->ns->prefix = NULL; } } From a76b93effa4b9754e0705b1e1c6f7f0d53eb5383 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 14:47:18 -0400 Subject: [PATCH 05/33] cleanup(extconf): document sdk cppflag on macos and clean up the logic to only do this when using system libraries and the directory exists. --- ext/nokogiri/extconf.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 861dd6545a..c2451c0b07 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -397,8 +397,10 @@ def using_system_libraries? do_clean end -if darwin? - ENV['CFLAGS'] = "#{ENV['CFLAGS']} -I /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libxml2" +# Add SDK-specific include path for macOS and brew versions before v2.2.12 (2020-04-08) [#1851, #1801] +macos_mojave_sdk_include_path = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libxml2" +if using_system_libraries? && darwin? && Dir.exist?(macos_mojave_sdk_include_path) + ENV['CFLAGS'] = "#{ENV['CFLAGS']} -I #{macos_mojave_sdk_include_path}" end if openbsd? && !using_system_libraries? From 2b9d3e2f14c85647f703ddd92177c66bb824913f Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 15:24:43 -0400 Subject: [PATCH 06/33] cleanup(extconf): omit unnecessary try_compile wrapper --- ext/nokogiri/extconf.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index c2451c0b07..bedc7b9e00 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -138,10 +138,6 @@ def package_config pkg, options={} end end -def nokogiri_try_compile - try_compile "int main() {return 0;}", "", {werror: true} -end - def check_libxml_version version=nil source = if version.nil? <<-SRC @@ -163,7 +159,7 @@ def check_libxml_version version=nil def add_cflags(flags) print "checking if the C compiler accepts #{flags}... " with_cflags("#{$CFLAGS} #{flags}") do - if nokogiri_try_compile + if try_compile("int main() {return 0;}", "", {werror: true}) puts 'yes' true else From 426fd687534bee9c0c6bdb1d51f2a92c1edc87ce Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 16:13:19 -0400 Subject: [PATCH 07/33] cleanup(extconf): rename ROOT to PACKAGE_ROOT_DIR --- ext/nokogiri/extconf.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index bedc7b9e00..b326a63b53 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -3,11 +3,9 @@ require 'mkmf' -ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) +PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) -# -# functions -# +# utility functions def windows? RbConfig::CONFIG['target_os'] =~ /mingw32|mswin/ end @@ -88,7 +86,7 @@ def do_clean require 'pathname' require 'fileutils' - root = Pathname(ROOT) + root = Pathname(PACKAGE_ROOT_DIR) pwd = Pathname(Dir.pwd) # Skip if this is a development work tree @@ -260,11 +258,11 @@ def chdir_for_build def process_recipe(name, version, static_p, cross_p) MiniPortile.new(name, version).tap do |recipe| - recipe.target = File.join(ROOT, "ports") + recipe.target = File.join(PACKAGE_ROOT_DIR, "ports") # Prefer host_alias over host in order to use i586-mingw32msvc as # correct compiler prefix for cross build, but use host if not set. recipe.host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"] - recipe.patch_files = Dir[File.join(ROOT, "patches", name, "*.patch")].sort + recipe.patch_files = Dir[File.join(PACKAGE_ROOT_DIR, "patches", name, "*.patch")].sort recipe.configure_options << "--libdir=#{File.join(recipe.path, "lib")}" yield recipe @@ -476,7 +474,7 @@ def using_system_libraries? dir_config('zlib') - dependencies = YAML.load_file(File.join(ROOT, "dependencies.yml")) + dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, "dependencies.yml")) cross_build_p = enable_config("cross-build") if cross_build_p || windows? @@ -612,7 +610,7 @@ def configure case arg when /\A-L(.+)\z/ # Prioritize ports' directories - if $1.start_with?(ROOT + '/') + if $1.start_with?(PACKAGE_ROOT_DIR + '/') $LIBPATH = [$1] | $LIBPATH else $LIBPATH = $LIBPATH | [$1] From 652c6fdfcaf2cf6bb10746eda8b7e4598a407689 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 16:17:41 -0400 Subject: [PATCH 08/33] cleanup(extconf): extract constants for libxml required versions and rename and reformat the helper function --- ext/nokogiri/extconf.rb | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index b326a63b53..3e1b01d824 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -4,6 +4,8 @@ require 'mkmf' PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) +REQUIRED_LIBXML_VERSION = "2.6.21" +RECOMMENDED_LIBXML_VERSION = "2.9.3" # utility functions def windows? @@ -136,18 +138,18 @@ def package_config pkg, options={} end end -def check_libxml_version version=nil +def have_libxml_headers?(version=nil) source = if version.nil? - <<-SRC -#include + <<~SRC + #include SRC else version_int = sprintf "%d%2.2d%2.2d", *(version.split(".")) - <<-SRC -#include -#if LIBXML_VERSION < #{version_int} -#error libxml2 is older than #{version} -#endif + <<~SRC + #include + #if LIBXML_VERSION < #{version_int} + # error libxml2 is older than #{version} + #endif SRC end @@ -453,10 +455,12 @@ def using_system_libraries? dir_config('xslt').any? or package_config('libxslt') dir_config('exslt').any? or package_config('libexslt') - check_libxml_version or abort "ERROR: cannot discover where libxml2 is located on your system. please make sure `pkg-config` is installed." - check_libxml_version("2.6.21") or abort "ERROR: libxml2 version 2.6.21 or later is required!" - check_libxml_version("2.9.3") or warn "WARNING: libxml2 version 2.9.3 or later is highly recommended, but proceeding anyway." - + have_libxml_headers? or + abort "ERROR: cannot discover where libxml2 is located on your system. please make sure `pkg-config` is installed." + have_libxml_headers?(REQUIRED_LIBXML_VERSION) or + abort "ERROR: libxml2 version #{REQUIRED_LIBXML_VERSION} or later is required!" + have_libxml_headers?(RECOMMENDED_LIBXML_VERSION) or + warn "WARNING: libxml2 version #{RECOMMENDED_LIBXML_VERSION} or later is highly recommended, but proceeding anyway." else message "Building nokogiri using packaged libraries.\n" From 8c17803caa60a2b77452346491d543f2840b1122 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 18:07:34 -0400 Subject: [PATCH 09/33] cleanup(extconf): extract mini_portile required version --- ext/nokogiri/extconf.rb | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 3e1b01d824..e311752c1c 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -3,11 +3,20 @@ require 'mkmf' +# +# helpful constants +# PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) REQUIRED_LIBXML_VERSION = "2.6.21" RECOMMENDED_LIBXML_VERSION = "2.9.3" -# utility functions +# The gem version constraint in the Rakefile is not respected at install time. +# Keep this version in sync with the one in the Rakefile ! +REQUIRED_MINI_PORTILE_VERSION = "~> 2.5.0" + +# +# utility functions +# def windows? RbConfig::CONFIG['target_os'] =~ /mingw32|mswin/ end @@ -464,10 +473,8 @@ def using_system_libraries? else message "Building nokogiri using packaged libraries.\n" - # The gem version constraint in the Rakefile is not respected at install time. - # Keep this version in sync with the one in the Rakefile ! require 'rubygems' - gem 'mini_portile2', '~> 2.5.0' + gem 'mini_portile2', REQUIRED_MINI_PORTILE_VERSION require 'mini_portile2' message "Using mini_portile version #{MiniPortile::VERSION}\n" From 0f093e8ef80c285171331c250683b22d920dd1d2 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 16:20:23 -0400 Subject: [PATCH 10/33] cleanup(extconf): indent all heredocs with `<<~` --- ext/nokogiri/extconf.rb | 140 ++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 77 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index e311752c1c..ff68242275 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -63,33 +63,33 @@ def sh_export_path path end def do_help - print < -#include - -int main(void) -{ - iconv_t cd = iconv_open("", ""); - iconv(cd, NULL, NULL, NULL, NULL); - return EXIT_SUCCESS; -} + try_link(<<~'SRC', opt) + #include + #include + int main(void) + { + iconv_t cd = iconv_open("", ""); + iconv(cd, NULL, NULL, NULL, NULL); + return EXIT_SUCCESS; + } SRC end end @@ -326,47 +325,35 @@ def process_recipe(name, version, static_p, cross_p) "#{key}=#{value}" end - message <<-"EOS" -************************************************************************ -IMPORTANT NOTICE: - -Building Nokogiri with a packaged version of #{name}-#{version}#{'.' if recipe.patch_files.empty?} - EOS + message <<~EOM + ---------- IMPORTANT NOTICE ---------- + Building Nokogiri with a packaged version of #{name}-#{version}#{'.' if recipe.patch_files.empty?} + EOM unless recipe.patch_files.empty? message "with the following patches applied:\n" recipe.patch_files.each do |patch| - message "\t- %s\n" % File.basename(patch) + message " - %s\n" % File.basename(patch) end end - message <<-"EOS" + message <<~EOM -Team Nokogiri will keep on doing their best to provide security -updates in a timely manner, but if this is a concern for you and want -to use the system library instead; abort this installation process and -reinstall nokogiri as follows: + The Nokogiri maintainers intend to provide timely security updates, but if + this is a concern for you and want to use your OS/distro system library + instead, then abort this installation process and install nokogiri as + instructed at: - gem install nokogiri -- --use-system-libraries - [--with-xml2-config=/path/to/xml2-config] - [--with-xslt-config=/path/to/xslt-config] + https://nokogiri.org/tutorials/installing_nokogiri.html#install-with-system-libraries -If you are using Bundler, tell it to use the option: + EOM - bundle config build.nokogiri --use-system-libraries - bundle install - EOS + message <<~EOM if name == 'libxml2' + Note, however, that nokogiri cannot guarantee compatiblity with every + version of libxml2 that may be provided by OS/package vendors. - message <<-"EOS" if name == 'libxml2' - -Note, however, that nokogiri is not fully compatible with arbitrary -versions of libxml2 provided by OS/package vendors. - EOS - - message <<-"EOS" -************************************************************************ - EOS + EOM checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" unless File.exist?(checkpoint) @@ -451,8 +438,7 @@ def using_system_libraries? $CFLAGS << " -Wall -Wcast-qual -Wwrite-strings -Wmissing-noreturn -Winline" end -case -when using_system_libraries? +if using_system_libraries? message "Building nokogiri using system libraries.\n" dir_config('zlib') @@ -551,18 +537,18 @@ def configure end else if darwin? && !have_header('iconv.h') - abort <<'EOM'.chomp ------ -The file "iconv.h" is missing in your build environment, -which means you haven't installed Xcode Command Line Tools properly. - -To install Command Line Tools, try running `xcode-select --install` on -terminal and follow the instructions. If it fails, open Xcode.app, -select from the menu "Xcode" - "Open Developer Tool" - "More Developer -Tools" to open the developer site, download the installer for your OS -version and run it. ------ -EOM + abort <<~EOM.chomp + ----- + The file "iconv.h" is missing in your build environment, + which means you haven't installed Xcode Command Line Tools properly. + + To install Command Line Tools, try running `xcode-select --install` on + terminal and follow the instructions. If it fails, open Xcode.app, + select from the menu "Xcode" - "Open Developer Tool" - "More Developer + Tools" to open the developer site, download the installer for your OS + version and run it. + ----- + EOM end end @@ -691,12 +677,12 @@ def configure if enable_config('clean', true) # Do not clean if run in a development work tree. File.open('Makefile', 'at') do |mk| - mk.print < Date: Mon, 19 Oct 2020 18:08:14 -0400 Subject: [PATCH 11/33] cleanup(extconf): refactor sh_export_path --- ext/nokogiri/extconf.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index ff68242275..571f4801ba 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -53,12 +53,13 @@ def sh_export_path path # # which is sh-compatible, in order to find things properly during # configuration - if windows? - match = Regexp.new("^([A-Z]):(/.*)").match(path) - if match && match.length == 3 - return File.join("/", match[1], match[2]) - end + return path if !windows? + + match = Regexp.new("^([A-Z]):(/.*)").match(path) + if match && match.length == 3 + return File.join("/", match[1], match[2]) end + path end From 19489f173dd690e3a429f8c344fe1f19dd7f436c Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 18:10:36 -0400 Subject: [PATCH 12/33] cleanup(extconf): comment and refactor package_config --- ext/nokogiri/extconf.rb | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 571f4801ba..02c53a6c67 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -125,27 +125,37 @@ def do_clean end def package_config pkg, options={} + # use MakeMakefile#pkg_config, which uses the system utility `pkg-config`. package = pkg_config(pkg) return package if package + # only if that utility isn't found do we fall back to the pkg-config gem, and + # wrap it with the same logic as MakeMakefile#pkg_config begin require 'rubygems' gem 'pkg-config', (gem_ver='~> 1.1') require 'pkg-config' and message("Using pkg-config gem version #{PKGConfig::VERSION}\n") rescue LoadError - message "pkg-config could not be used to find #{pkg}\nPlease install either `pkg-config` or the pkg-config gem per\n\n gem install pkg-config -v #{gem_ver.inspect}\n\n" - else - return nil unless PKGConfig.have_package(pkg) - - cflags = PKGConfig.cflags(pkg) - ldflags = PKGConfig.libs_only_L(pkg) - libs = PKGConfig.libs_only_l(pkg) + message <<~EOM + pkg-config could not be used to find #{pkg} + Please install either `pkg-config` or the pkg-config gem per - Logging::message "PKGConfig package configuration for %s\n", pkg - Logging::message "cflags: %s\nldflags: %s\nlibs: %s\n\n", cflags, ldflags, libs + gem install pkg-config -v #{gem_ver.inspect} - [cflags, ldflags, libs] + EOM + return nil end + + return nil unless PKGConfig.have_package(pkg) + + cflags = PKGConfig.cflags(pkg) + ldflags = PKGConfig.libs_only_L(pkg) + libs = PKGConfig.libs_only_l(pkg) + + Logging::message "PKGConfig package configuration for %s\n", pkg + Logging::message "cflags: %s\nldflags: %s\nlibs: %s\n\n", cflags, ldflags, libs + + [cflags, ldflags, libs] end def have_libxml_headers?(version=nil) From 90acd2e77aec5aa315e0a5b875ccb1dc59daf835 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 18:11:21 -0400 Subject: [PATCH 13/33] cleanup(extconf): neaten preserving_globals --- ext/nokogiri/extconf.rb | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 02c53a6c67..f3d58d7305 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -190,17 +190,10 @@ def add_cflags(flags) end def preserving_globals - values = [ - $arg_config, - $CFLAGS, $CPPFLAGS, - $LDFLAGS, $LIBPATH, $libs - ].map(&:dup) + values = [$arg_config, $CFLAGS, $CPPFLAGS, $LDFLAGS, $LIBPATH, $libs].map(&:dup) yield ensure - $arg_config, - $CFLAGS, $CPPFLAGS, - $LDFLAGS, $LIBPATH, $libs = - values + $arg_config, $CFLAGS, $CPPFLAGS, $LDFLAGS, $LIBPATH, $libs = values end def asplode(lib) From b011e658032b29bd68705468f55ddf5c9a43225e Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 18:12:27 -0400 Subject: [PATCH 14/33] cleanup(extconf): neaten iconv_configure_flags --- ext/nokogiri/extconf.rb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index f3d58d7305..d9c5bee94e 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -222,11 +222,10 @@ def have_iconv?(using = nil) end def iconv_configure_flags - # If --with-iconv-dir or --with-opt-dir is given, it should be - # the first priority - %w[iconv opt].each do |name| - if (config = preserving_globals { dir_config(name) }).any? && - have_iconv?("--with-#{name}-* flags") { dir_config(name) } + # give --with-iconv-dir and --with-opt-dir first priority + ["iconv", "opt"].each do |target| + config = preserving_globals { dir_config(target) } + if config.any? && have_iconv?("--with-#{target}-* flags") { dir_config(target) } idirs, ldirs = config.map do |dirs| Array(dirs).flat_map do |dir| dir.split(File::PATH_SEPARATOR) @@ -245,8 +244,8 @@ def iconv_configure_flags return ['--with-iconv=yes'] end - if (config = preserving_globals { package_config('libiconv') }) && - have_iconv?('pkg-config libiconv') { package_config('libiconv') } + config = preserving_globals { package_config('libiconv') } + if config && have_iconv?('pkg-config libiconv') { package_config('libiconv') } cflags, ldflags, libs = config return [ From 0e55b14e962c91fac2393bed4d9b9644d6286fbf Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 18:14:30 -0400 Subject: [PATCH 15/33] cleanup(extconf): newlines, rename lib_a to libflag_to_filename --- ext/nokogiri/extconf.rb | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index d9c5bee94e..ea50dcb349 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -259,10 +259,10 @@ def iconv_configure_flags asplode "libiconv" end -# When using rake-compiler-dock on Windows, the underlying Virtualbox shared -# folders don't support symlinks, but libiconv expects it for a build on -# Linux. We work around this limitation by using the temp dir for cooking. def chdir_for_build + # When using rake-compiler-dock on Windows, the underlying Virtualbox shared + # folders don't support symlinks, but libiconv expects it for a build on + # Linux. We work around this limitation by using the temp dir for cooking. build_dir = ENV['RCD_HOST_RUBY_PLATFORM'].to_s =~ /mingw|mswin|cygwin/ ? '/tmp' : '.' Dir.chdir(build_dir) do yield @@ -369,7 +369,7 @@ def process_recipe(name, version, static_p, cross_p) end end -def lib_a(ldflag) +def libflag_to_filename(ldflag) case ldflag when /\A-l(.+)/ "lib#{$1}.#{$LIBEXT}" @@ -444,11 +444,8 @@ def using_system_libraries? if using_system_libraries? message "Building nokogiri using system libraries.\n" + # Using system libraries means we rely on the system libxml2 for iconv support (or not) dir_config('zlib') - - # Using system libraries means we rely on the system libxml2 with - # regard to the iconv support. - dir_config('xml2').any? or package_config('libxml-2.0') dir_config('xslt').any? or package_config('libxslt') dir_config('exslt').any? or package_config('libexslt') @@ -459,6 +456,7 @@ def using_system_libraries? abort "ERROR: libxml2 version #{REQUIRED_LIBXML_VERSION} or later is required!" have_libxml_headers?(RECOMMENDED_LIBXML_VERSION) or warn "WARNING: libxml2 version #{RECOMMENDED_LIBXML_VERSION} or later is highly recommended, but proceeding anyway." + else message "Building nokogiri using packaged libraries.\n" @@ -556,9 +554,8 @@ def configure end unless windows? - preserving_globals { - have_library('z', 'gzdopen', 'zlib.h') - } or abort 'zlib is missing; necessary for building libxml2' + preserving_globals { have_library('z', 'gzdopen', 'zlib.h') } or + abort 'zlib is missing; necessary for building libxml2' end libxml2_recipe = process_recipe("libxml2", dependencies["libxml2"]["version"], static_p, cross_build_p) do |recipe| @@ -596,9 +593,7 @@ def configure $LIBPATH = ["#{zlib_recipe.path}/lib"] | $LIBPATH if zlib_recipe $LIBPATH = ["#{libiconv_recipe.path}/lib"] | $LIBPATH if libiconv_recipe - have_lzma = preserving_globals { - have_library('lzma') - } + have_lzma = preserving_globals { have_library('lzma') } $libs = $libs.shellsplit.tap do |libs| [libxml2_recipe, libxslt_recipe].each do |recipe| @@ -646,9 +641,9 @@ def configure $libs = $libs.shellsplit.map do |arg| case arg when '-lxml2' - File.join(libxml2_recipe.path, 'lib', lib_a(arg)) + File.join(libxml2_recipe.path, 'lib', libflag_to_filename(arg)) when '-lxslt', '-lexslt' - File.join(libxslt_recipe.path, 'lib', lib_a(arg)) + File.join(libxslt_recipe.path, 'lib', libflag_to_filename(arg)) else arg end From 6d6e0173d6c1e45b3b14889abb18099a1ce4f4e7 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 18:16:00 -0400 Subject: [PATCH 16/33] =?UTF-8?q?cleanup(extconf):=20rename=20asplode=20?= =?UTF-8?q?=E2=86=92=20abort=5Fcould=5Fnot=5Ffind=5Flibrary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit and add verbosity to our have_func/have_library checks for libxml2, libxslt, and libexslt. --- ext/nokogiri/extconf.rb | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index ea50dcb349..884e8324a7 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -196,7 +196,7 @@ def preserving_globals $arg_config, $CFLAGS, $CPPFLAGS, $LDFLAGS, $LIBPATH, $libs = values end -def asplode(lib) +def abort_could_not_find_library(lib) abort "-----\n#{lib} is missing. Please locate mkmf.log to investigate how it is failing.\n-----" end @@ -256,7 +256,7 @@ def iconv_configure_flags ] end - asplode "libiconv" + abort_could_not_find_library "libiconv" end def chdir_for_build @@ -651,18 +651,21 @@ def configure end end -{ - "xml2" => ['xmlParseDoc', 'libxml/parser.h'], - "xslt" => ['xsltParseStylesheetDoc', 'libxslt/xslt.h'], - "exslt" => ['exsltFuncRegister', 'libexslt/exslt.h'], -}.each do |lib, (func, header)| - have_func(func, header) || - have_library(lib, func, header) || - have_library("lib#{lib}", func, header) or - asplode("lib#{lib}") +have_func('vasprintf') + +[ + ["xml2", "xmlParseDoc", "libxml/parser.h"], + ["xslt", "xsltParseStylesheetDoc", "libxslt/xslt.h"], + ["exslt", "exsltFuncRegister", "libexslt/exslt.h"] +].each do |lib, func, header| + checking_for "lib#{lib}" do + have_func(func, header) or + have_library(lib, func, header) or + have_library("lib#{lib}", func, header) or + abort_could_not_find_library("lib#{lib}") + end end -have_func('vasprintf') have_func('xmlHasFeature') or abort "xmlHasFeature() is missing." # introduced in libxml 2.6.21 have_func('xmlFirstElementChild') # introduced in libxml 2.7.3 have_func('xmlRelaxNGSetParserStructuredErrors') # introduced in libxml 2.6.24 From bd7dacc1a13a154371e6f42deb2396764caaa388 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 18:17:33 -0400 Subject: [PATCH 17/33] cleanup(extconf): simplify help/clean arg handling --- ext/nokogiri/extconf.rb | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 884e8324a7..6d15f71c4d 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -382,21 +382,16 @@ def using_system_libraries? end # -# main +# main # -case -when arg_config('--help') - do_help -when arg_config('--clean') - do_clean -end - # Add SDK-specific include path for macOS and brew versions before v2.2.12 (2020-04-08) [#1851, #1801] macos_mojave_sdk_include_path = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libxml2" if using_system_libraries? && darwin? && Dir.exist?(macos_mojave_sdk_include_path) ENV['CFLAGS'] = "#{ENV['CFLAGS']} -I #{macos_mojave_sdk_include_path}" end +do_help if arg_config('--help') +do_clean if arg_config('--clean') if openbsd? && !using_system_libraries? if `#{ENV['CC'] || '/usr/bin/cc'} -v 2>&1` !~ /clang/ @@ -409,6 +404,7 @@ def using_system_libraries? if ENV['CC'] RbConfig::CONFIG['CC'] = RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] end + # use same c compiler for libxml and libxslt ENV['CC'] = RbConfig::CONFIG['CC'] From 3050fee6fe8ace67b448f084e09bdf61c61a3296 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 18:23:34 -0400 Subject: [PATCH 18/33] cleanup(extconf): use append_*flags instead of add_cflags or globals Also note that this commit removes -Wcast-qual and -Wwrite-strings because I personally find them to be low-value and noisy. --- ext/nokogiri/extconf.rb | 48 ++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 6d15f71c4d..a1a9659306 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -384,12 +384,6 @@ def using_system_libraries? # # main # - -# Add SDK-specific include path for macOS and brew versions before v2.2.12 (2020-04-08) [#1851, #1801] -macos_mojave_sdk_include_path = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libxml2" -if using_system_libraries? && darwin? && Dir.exist?(macos_mojave_sdk_include_path) - ENV['CFLAGS'] = "#{ENV['CFLAGS']} -I #{macos_mojave_sdk_include_path}" -end do_help if arg_config('--help') do_clean if arg_config('--clean') @@ -398,7 +392,7 @@ def using_system_libraries? ENV['CC'] ||= find_executable('egcc') or abort "Please install gcc 4.9+ from ports using `pkg_add -v gcc`" end - ENV['CFLAGS'] = "#{ENV['CFLAGS']} -I /usr/local/include" + append_cppflags "-I /usr/local/include" end if ENV['CC'] @@ -408,34 +402,28 @@ def using_system_libraries? # use same c compiler for libxml and libxslt ENV['CC'] = RbConfig::CONFIG['CC'] +append_cflags(ENV["CFLAGS"].split(/\s+/)) if !ENV["CFLAGS"].nil? +append_cppflags(ENV["CPPFLAGS"].split(/\s+/)) if !ENV["CPPFLAGS"].nil? +append_ldflags(ENV["LDFLAGS"].split(/\s+/)) if !ENV["LDFLAGS"].nil? $LIBS << " #{ENV["LIBS"]}" -# Read CFLAGS from ENV and make sure compiling works. -add_cflags(ENV["CFLAGS"]) - -if windows? - $CFLAGS << " -DXP_WIN -DXP_WIN32" -end +append_cflags("-g") # always include debugging information +append_cflags("-Winline") # we use at least one inline function in the C extension +append_cflags("-Wmissing-noreturn") # good to have no matter what Ruby was compiled with +# append_cflags(["-Wcast-qual", "-Wwrite-strings"]) # these tend to be noisy, but on occasion useful during development +append_cflags("-Wno-error=unused-command-line-argument-hard-error-in-future") if darwin? +append_cppflags("-DXP_UNIX") if nix? +append_cppflags(["-DXP_WIN", "-DXP_WIN32"]) if windows? -if darwin? - # Let Apple LLVM/clang 5.1 ignore unknown compiler flags - add_cflags("-Wno-error=unused-command-line-argument-hard-error-in-future") -end - -if nix? - $CFLAGS << " -g -DXP_UNIX" -end - -if RUBY_PLATFORM =~ /mingw/i - # Work around a character escaping bug in MSYS by passing an arbitrary - # double quoted parameter to gcc. See https://sourceforge.net/p/mingw/bugs/2142 - $CPPFLAGS << ' "-Idummypath"' +# Add SDK-specific include path for macOS and brew versions before v2.2.12 (2020-04-08) [#1851, #1801] +macos_mojave_sdk_include_path = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libxml2" +if using_system_libraries? && darwin? && Dir.exist?(macos_mojave_sdk_include_path) + append_cppflags "-I #{macos_mojave_sdk_include_path}" end -if RbConfig::CONFIG['CC'] =~ /gcc/ - $CFLAGS << " -O3" unless $CFLAGS[/-O\d/] - $CFLAGS << " -Wall -Wcast-qual -Wwrite-strings -Wmissing-noreturn -Winline" -end +# Work around a character escaping bug in MSYS by passing an arbitrary +# double quoted parameter to gcc. See https://sourceforge.net/p/mingw/bugs/2142 +append_cppflags(' "-Idummypath"') if windows? if using_system_libraries? message "Building nokogiri using system libraries.\n" From 5c383ecf58fcdb69b1af53230f6d889b2043da96 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 19 Oct 2020 18:24:33 -0400 Subject: [PATCH 19/33] cleanup(extconf): remove unnecessary add_cflags method --- ext/nokogiri/extconf.rb | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index a1a9659306..3f55a2be3a 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -176,19 +176,6 @@ def have_libxml_headers?(version=nil) try_cpp source end -def add_cflags(flags) - print "checking if the C compiler accepts #{flags}... " - with_cflags("#{$CFLAGS} #{flags}") do - if try_compile("int main() {return 0;}", "", {werror: true}) - puts 'yes' - true - else - puts 'no' - false - end - end -end - def preserving_globals values = [$arg_config, $CFLAGS, $CPPFLAGS, $LDFLAGS, $LIBPATH, $libs].map(&:dup) yield From 433eeda403b914e6f33f4be76d31eb5ef398e769 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 20 Oct 2020 12:18:32 -0400 Subject: [PATCH 20/33] cleanup(extconf): rearrange methods and code --- ext/nokogiri/extconf.rb | 248 ++++++++++++++++++++-------------------- 1 file changed, 124 insertions(+), 124 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 3f55a2be3a..199fce9a3e 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -41,89 +41,6 @@ def nix? ! (windows? || solaris? || darwin?) end -def sh_export_path path - # because libxslt 1.1.29 configure.in uses AC_PATH_TOOL which treats ":" - # as a $PATH separator, we need to convert windows paths from - # - # C:/path/to/foo - # - # to - # - # /C/path/to/foo - # - # which is sh-compatible, in order to find things properly during - # configuration - return path if !windows? - - match = Regexp.new("^([A-Z]):(/.*)").match(path) - if match && match.length == 3 - return File.join("/", match[1], match[2]) - end - - path -end - -def do_help - print <<~HELP - usage: ruby #{$0} [options] - - --disable-clean - Do not clean out intermediate files after successful build. - - --disable-static - Do not statically link bundled libraries. - - --with-iconv-dir=DIR - Use the iconv library placed under DIR. - - --with-zlib-dir=DIR - Use the zlib library placed under DIR. - - --use-system-libraries - Use system libraries instead of building and using the bundled - libraries. - - --with-xml2-dir=DIR / --with-xml2-config=CONFIG - --with-xslt-dir=DIR / --with-xslt-config=CONFIG - --with-exslt-dir=DIR / --with-exslt-config=CONFIG - Use libxml2/libxslt/libexslt as specified. - - --enable-cross-build - Do cross-build. - HELP - exit! 0 -end - -def do_clean - require 'pathname' - require 'fileutils' - - root = Pathname(PACKAGE_ROOT_DIR) - pwd = Pathname(Dir.pwd) - - # Skip if this is a development work tree - unless (root + '.git').exist? - message "Cleaning files only used during build.\n" - - # (root + 'tmp') cannot be removed at this stage because - # nokogiri.so is yet to be copied to lib. - - # clean the ports build directory - Pathname.glob(pwd.join('tmp', '*', 'ports')) do |dir| - FileUtils.rm_rf(dir, verbose: true) - end - - if enable_config('static') - # ports installation can be safely removed if statically linked. - FileUtils.rm_rf(root + 'ports', verbose: true) - else - FileUtils.rm_rf(root + 'ports' + 'archives', verbose: true) - end - end - - exit! 0 -end - def package_config pkg, options={} # use MakeMakefile#pkg_config, which uses the system utility `pkg-config`. package = pkg_config(pkg) @@ -158,6 +75,61 @@ def package_config pkg, options={} [cflags, ldflags, libs] end +def preserving_globals + values = [$arg_config, $CFLAGS, $CPPFLAGS, $LDFLAGS, $LIBPATH, $libs].map(&:dup) + yield +ensure + $arg_config, $CFLAGS, $CPPFLAGS, $LDFLAGS, $LIBPATH, $libs = values +end + +def abort_could_not_find_library(lib) + abort "-----\n#{lib} is missing. Please locate mkmf.log to investigate how it is failing.\n-----" +end + +def chdir_for_build + # When using rake-compiler-dock on Windows, the underlying Virtualbox shared + # folders don't support symlinks, but libiconv expects it for a build on + # Linux. We work around this limitation by using the temp dir for cooking. + build_dir = ENV['RCD_HOST_RUBY_PLATFORM'].to_s =~ /mingw|mswin|cygwin/ ? '/tmp' : '.' + Dir.chdir(build_dir) do + yield + end +end + +def sh_export_path path + # because libxslt 1.1.29 configure.in uses AC_PATH_TOOL which treats ":" + # as a $PATH separator, we need to convert windows paths from + # + # C:/path/to/foo + # + # to + # + # /C/path/to/foo + # + # which is sh-compatible, in order to find things properly during + # configuration + return path if !windows? + + match = Regexp.new("^([A-Z]):(/.*)").match(path) + if match && match.length == 3 + return File.join("/", match[1], match[2]) + end + + path +end + +def libflag_to_filename(ldflag) + case ldflag + when /\A-l(.+)/ + "lib#{$1}.#{$LIBEXT}" + end +end + +def using_system_libraries? + # NOTE: TruffleRuby uses this env var as it does not support using static libraries yet. + arg_config('--use-system-libraries', !!ENV['NOKOGIRI_USE_SYSTEM_LIBRARIES']) +end + def have_libxml_headers?(version=nil) source = if version.nil? <<~SRC @@ -176,18 +148,7 @@ def have_libxml_headers?(version=nil) try_cpp source end -def preserving_globals - values = [$arg_config, $CFLAGS, $CPPFLAGS, $LDFLAGS, $LIBPATH, $libs].map(&:dup) - yield -ensure - $arg_config, $CFLAGS, $CPPFLAGS, $LDFLAGS, $LIBPATH, $libs = values -end - -def abort_could_not_find_library(lib) - abort "-----\n#{lib} is missing. Please locate mkmf.log to investigate how it is failing.\n-----" -end - -def have_iconv?(using = nil) +def try_link_iconv(using = nil) checking_for(using ? "iconv using #{using}" : 'iconv') do ['', '-liconv'].any? do |opt| preserving_globals do @@ -212,7 +173,7 @@ def iconv_configure_flags # give --with-iconv-dir and --with-opt-dir first priority ["iconv", "opt"].each do |target| config = preserving_globals { dir_config(target) } - if config.any? && have_iconv?("--with-#{target}-* flags") { dir_config(target) } + if config.any? && try_link_iconv("--with-#{target}-* flags") { dir_config(target) } idirs, ldirs = config.map do |dirs| Array(dirs).flat_map do |dir| dir.split(File::PATH_SEPARATOR) @@ -227,12 +188,12 @@ def iconv_configure_flags end end - if have_iconv? + if try_link_iconv return ['--with-iconv=yes'] end config = preserving_globals { package_config('libiconv') } - if config && have_iconv?('pkg-config libiconv') { package_config('libiconv') } + if config && try_link_iconv('pkg-config libiconv') { package_config('libiconv') } cflags, ldflags, libs = config return [ @@ -246,16 +207,6 @@ def iconv_configure_flags abort_could_not_find_library "libiconv" end -def chdir_for_build - # When using rake-compiler-dock on Windows, the underlying Virtualbox shared - # folders don't support symlinks, but libiconv expects it for a build on - # Linux. We work around this limitation by using the temp dir for cooking. - build_dir = ENV['RCD_HOST_RUBY_PLATFORM'].to_s =~ /mingw|mswin|cygwin/ ? '/tmp' : '.' - Dir.chdir(build_dir) do - yield - end -end - def process_recipe(name, version, static_p, cross_p) MiniPortile.new(name, version).tap do |recipe| recipe.target = File.join(PACKAGE_ROOT_DIR, "ports") @@ -356,16 +307,65 @@ def process_recipe(name, version, static_p, cross_p) end end -def libflag_to_filename(ldflag) - case ldflag - when /\A-l(.+)/ - "lib#{$1}.#{$LIBEXT}" - end +def do_help + print <<~HELP + usage: ruby #{$0} [options] + + --disable-clean + Do not clean out intermediate files after successful build. + + --disable-static + Do not statically link bundled libraries. + + --with-iconv-dir=DIR + Use the iconv library placed under DIR. + + --with-zlib-dir=DIR + Use the zlib library placed under DIR. + + --use-system-libraries + Use system libraries instead of building and using the bundled + libraries. + + --with-xml2-dir=DIR / --with-xml2-config=CONFIG + --with-xslt-dir=DIR / --with-xslt-config=CONFIG + --with-exslt-dir=DIR / --with-exslt-config=CONFIG + Use libxml2/libxslt/libexslt as specified. + + --enable-cross-build + Do cross-build. + HELP + exit! 0 end -def using_system_libraries? - # NOTE: TruffleRuby uses this env var as it does not support using static libraries yet. - arg_config('--use-system-libraries', !!ENV['NOKOGIRI_USE_SYSTEM_LIBRARIES']) +def do_clean + require 'pathname' + require 'fileutils' + + root = Pathname(PACKAGE_ROOT_DIR) + pwd = Pathname(Dir.pwd) + + # Skip if this is a development work tree + unless (root + '.git').exist? + message "Cleaning files only used during build.\n" + + # (root + 'tmp') cannot be removed at this stage because + # nokogiri.so is yet to be copied to lib. + + # clean the ports build directory + Pathname.glob(pwd.join('tmp', '*', 'ports')) do |dir| + FileUtils.rm_rf(dir, verbose: true) + end + + if enable_config('static') + # ports installation can be safely removed if statically linked. + FileUtils.rm_rf(root + 'ports', verbose: true) + else + FileUtils.rm_rf(root + 'ports' + 'archives', verbose: true) + end + end + + exit! 0 end # @@ -389,6 +389,7 @@ def using_system_libraries? # use same c compiler for libxml and libxslt ENV['CC'] = RbConfig::CONFIG['CC'] +# adopt environment config append_cflags(ENV["CFLAGS"].split(/\s+/)) if !ENV["CFLAGS"].nil? append_cppflags(ENV["CPPFLAGS"].split(/\s+/)) if !ENV["CPPFLAGS"].nil? append_ldflags(ENV["LDFLAGS"].split(/\s+/)) if !ENV["LDFLAGS"].nil? @@ -397,15 +398,16 @@ def using_system_libraries? append_cflags("-g") # always include debugging information append_cflags("-Winline") # we use at least one inline function in the C extension append_cflags("-Wmissing-noreturn") # good to have no matter what Ruby was compiled with -# append_cflags(["-Wcast-qual", "-Wwrite-strings"]) # these tend to be noisy, but on occasion useful during development append_cflags("-Wno-error=unused-command-line-argument-hard-error-in-future") if darwin? +# append_cflags(["-Wcast-qual", "-Wwrite-strings"]) # these tend to be noisy, but on occasion useful during development + append_cppflags("-DXP_UNIX") if nix? append_cppflags(["-DXP_WIN", "-DXP_WIN32"]) if windows? # Add SDK-specific include path for macOS and brew versions before v2.2.12 (2020-04-08) [#1851, #1801] macos_mojave_sdk_include_path = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libxml2" if using_system_libraries? && darwin? && Dir.exist?(macos_mojave_sdk_include_path) - append_cppflags "-I #{macos_mojave_sdk_include_path}" + append_cppflags("-I #{macos_mojave_sdk_include_path}") end # Work around a character escaping bug in MSYS by passing an arbitrary @@ -431,21 +433,19 @@ def using_system_libraries? else message "Building nokogiri using packaged libraries.\n" + static_p = enable_config('static', true) or message "Static linking is disabled.\n" + cross_build_p = enable_config("cross-build") + require 'rubygems' gem 'mini_portile2', REQUIRED_MINI_PORTILE_VERSION require 'mini_portile2' message "Using mini_portile version #{MiniPortile::VERSION}\n" require 'yaml' - - static_p = enable_config('static', true) or - message "Static linking is disabled.\n" + dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, "dependencies.yml")) dir_config('zlib') - dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, "dependencies.yml")) - - cross_build_p = enable_config("cross-build") if cross_build_p || windows? zlib_recipe = process_recipe("zlib", dependencies["zlib"]["version"], static_p, cross_build_p) do |recipe| recipe.files = [{ From a95f0ebb3814ade579abd8e0d26558b88d66bb61 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 20 Oct 2020 13:54:22 -0400 Subject: [PATCH 21/33] cleanup(extconf): remove phantom preprocessor macros these date back to 2009, were hardly ever used, and aren't used at all now --- ext/nokogiri/extconf.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 199fce9a3e..ce25b253f2 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -401,9 +401,6 @@ def do_clean append_cflags("-Wno-error=unused-command-line-argument-hard-error-in-future") if darwin? # append_cflags(["-Wcast-qual", "-Wwrite-strings"]) # these tend to be noisy, but on occasion useful during development -append_cppflags("-DXP_UNIX") if nix? -append_cppflags(["-DXP_WIN", "-DXP_WIN32"]) if windows? - # Add SDK-specific include path for macOS and brew versions before v2.2.12 (2020-04-08) [#1851, #1801] macos_mojave_sdk_include_path = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/libxml2" if using_system_libraries? && darwin? && Dir.exist?(macos_mojave_sdk_include_path) From afa2e803b0ee7b9fef4e09ed6699a3217db90106 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 21 Oct 2020 17:13:59 -0400 Subject: [PATCH 22/33] cleanup(extconf): move mini_portile require into process_recipe --- ext/nokogiri/extconf.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index ce25b253f2..31d6f248f8 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -208,6 +208,11 @@ def iconv_configure_flags end def process_recipe(name, version, static_p, cross_p) + require 'rubygems' + gem 'mini_portile2', REQUIRED_MINI_PORTILE_VERSION + require 'mini_portile2' + message "Using mini_portile version #{MiniPortile::VERSION}\n" + MiniPortile.new(name, version).tap do |recipe| recipe.target = File.join(PACKAGE_ROOT_DIR, "ports") # Prefer host_alias over host in order to use i586-mingw32msvc as @@ -433,11 +438,6 @@ def do_clean static_p = enable_config('static', true) or message "Static linking is disabled.\n" cross_build_p = enable_config("cross-build") - require 'rubygems' - gem 'mini_portile2', REQUIRED_MINI_PORTILE_VERSION - require 'mini_portile2' - message "Using mini_portile version #{MiniPortile::VERSION}\n" - require 'yaml' dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, "dependencies.yml")) From 4645abb4b91da0c2ed8793add37bdbd99d8304dd Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 21 Oct 2020 17:15:48 -0400 Subject: [PATCH 23/33] cleanup(rake): indent heredocs with `<<~` --- tasks/cross-ruby.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/cross-ruby.rb b/tasks/cross-ruby.rb index 27a1599536..cdf444b029 100644 --- a/tasks/cross-ruby.rb +++ b/tasks/cross-ruby.rb @@ -238,7 +238,7 @@ def verify_dll(dll, cross_ruby) desc "build native gem for #{plat} platform (host)" task plat do # TODO remove `find` after https://github.com/rake-compiler/rake-compiler/pull/171 is shipped - RakeCompilerDock.sh <<-EOT, platform: plat + RakeCompilerDock.sh <<~EOT, platform: plat gem install bundler --no-document && bundle && find /usr/local/rvm/gems -name extensiontask.rb | while read f ; do sudo sed -i 's/callback.call(spec) if callback/@cross_compiling.call(spec) if @cross_compiling/' $f ; done && @@ -330,10 +330,10 @@ def verify_dll(dll, cross_ruby) ext.cross_config_options << "--enable-cross-build" ext.cross_compiling do |spec| libs = dependencies.map { |name, dep| "#{name}-#{dep["version"]}" }.join(', ') - - spec.post_install_message = <<-EOS -Nokogiri is built with the packaged libraries: #{libs}. + spec.post_install_message = <<~EOS + Nokogiri is built with the packaged libraries: #{libs}. EOS + spec.files.reject! { |path| File.fnmatch?('ports/*', path) } spec.dependencies.reject! { |dep| dep.name=='mini_portile2' } end From fd2ffbaebb55163a3f0b1e4171ec04f201d69b30 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 07:40:09 -0400 Subject: [PATCH 24/33] cleanup(dev): explicit requires --- Rakefile | 5 ----- ext/nokogiri/extconf.rb | 9 +++++---- tasks/cross-ruby.rb | 3 +++ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Rakefile b/Rakefile index 577c684900..8125faf0a3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,4 +1,3 @@ -# -*- ruby -*- require 'hoe' Hoe.plugin :bundler @@ -7,8 +6,6 @@ Hoe.plugin :gemspec Hoe.plugin :git Hoe.plugin :markdown -require 'shellwords' - require_relative "tasks/util" HOE = Hoe.spec 'nokogiri' do @@ -85,5 +82,3 @@ require_relative "tasks/set-version-to-timestamp" # work around Hoe's inflexibility about the default tasks Rake::Task[:default].prerequisites.unshift("compile") Rake::Task[:default].prerequisites.unshift("rubocop") - -# vim: syntax=Ruby diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 31d6f248f8..2ac4ce2c29 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -1,7 +1,11 @@ # :stopdoc: ENV['RC_ARCHS'] = '' if RUBY_PLATFORM =~ /darwin/ -require 'mkmf' +require "mkmf" +require "rbconfig" +require "fileutils" +require "shellwords" +require "pathname" # # helpful constants @@ -344,9 +348,6 @@ def do_help end def do_clean - require 'pathname' - require 'fileutils' - root = Pathname(PACKAGE_ROOT_DIR) pwd = Pathname(Dir.pwd) diff --git a/tasks/cross-ruby.rb b/tasks/cross-ruby.rb index cdf444b029..a40c482783 100644 --- a/tasks/cross-ruby.rb +++ b/tasks/cross-ruby.rb @@ -1,3 +1,6 @@ +require "rbconfig" +require "shellwords" + CrossRuby = Struct.new(:version, :host) do WINDOWS_PLATFORM_REGEX = /mingw|mswin/ MINGW32_PLATFORM_REGEX = /mingw32/ From 5f05028f03495e8f040dcb9770dfd5d02881716d Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 08:01:05 -0400 Subject: [PATCH 25/33] cleanup(dev): groom the Rakefile --- Rakefile | 52 ++++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/Rakefile b/Rakefile index 8125faf0a3..67f3e898b0 100644 --- a/Rakefile +++ b/Rakefile @@ -1,4 +1,4 @@ -require 'hoe' +require "hoe" Hoe.plugin :bundler Hoe.plugin :debugging @@ -8,18 +8,22 @@ Hoe.plugin :markdown require_relative "tasks/util" -HOE = Hoe.spec 'nokogiri' do - developer 'Aaron Patterson', 'aaronp@rubyforge.org' - developer 'Mike Dalessio', 'mike.dalessio@gmail.com' - developer 'Yoko Harada', 'yokolet@gmail.com' - developer 'Tim Elliott', 'tle@holymonkey.com' - developer 'Akinori MUSHA', 'knu@idaemons.org' - developer 'John Shahid', 'jvshahid@gmail.com' - developer 'Lars Kanis', 'lars@greiz-reinsdorf.de' +HOE = Hoe.spec "nokogiri" do |hoe| + hoe.author = [ + "Mike Dalessio", + "Aaron Patterson", + "John Shahid", + "Yoko Harada", + "Akinori MUSHA", + "Lars Kanis", + "Tim Elliott", + ] + + hoe.email = "nokogiri-talk@googlegroups.com" - license "MIT" + hoe.license "MIT" - self.urls = { + hoe.urls = { "home" => "https://nokogiri.org", "bugs" => "https://github.com/sparklemotion/nokogiri/issues", "doco" => "https://nokogiri.org/rdoc/index.html", @@ -27,23 +31,23 @@ HOE = Hoe.spec 'nokogiri' do "code" => "https://github.com/sparklemotion/nokogiri", } - self.markdown_linkify_files = FileList["*.md"] - self.extra_rdoc_files = FileList['ext/nokogiri/*.c'] + hoe.markdown_linkify_files = FileList["*.md"] + hoe.extra_rdoc_files = FileList["ext/nokogiri/*.c"] - self.clean_globs += [ - 'nokogiri.gemspec', - 'lib/nokogiri/nokogiri.{bundle,jar,rb,so}', - 'lib/nokogiri/[0-9].[0-9]', + hoe.clean_globs += [ + "nokogiri.gemspec", + "lib/nokogiri/nokogiri.{bundle,jar,rb,so}", + "lib/nokogiri/[0-9].[0-9]", ] - self.clean_globs += Dir.glob("ports/*").reject { |d| d =~ %r{/archives$} } + hoe.clean_globs += Dir.glob("ports/*").reject { |d| d =~ %r{/archives$} } unless java? - self.extra_deps += [ + hoe.extra_deps += [ ["mini_portile2", "~> 2.5.0"], # keep version in sync with extconf.rb ] end - self.extra_dev_deps += [ + hoe.extra_dev_deps += [ ["concourse", "~> 0.38"], ["hoe", ["~> 3.22", ">= 3.22.1"]], ["hoe-bundler", "~> 1.2"], @@ -61,13 +65,13 @@ HOE = Hoe.spec 'nokogiri' do ["simplecov", "~> 0.17.0"], # locked due to https://github.com/codeclimate/test-reporter/issues/413 ] - self.spec_extras = { + hoe.spec_extras = { :extensions => ["ext/nokogiri/extconf.rb"], - :required_ruby_version => '>= 2.4.0' + :required_ruby_version => ">= 2.4.0" } - self.testlib = :minitest - self.test_prelude = 'require "helper"' # ensure simplecov gets loaded before anything else + hoe.testlib = :minitest + hoe.test_prelude = %q(require "helper") # ensure simplecov gets loaded before anything else end require_relative "tasks/cross-ruby" From 28f12039d5504d599471169c8512a8d2b9d23391 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 08:01:46 -0400 Subject: [PATCH 26/33] dev: Remove old markdown tasks, which were replaced by hoe-markdown (This should have been done as part of ec4cf38.) --- Rakefile | 1 - tasks/docs-linkify.rb | 53 ------------------------------------------- 2 files changed, 54 deletions(-) delete mode 100644 tasks/docs-linkify.rb diff --git a/Rakefile b/Rakefile index 67f3e898b0..c0c2946504 100644 --- a/Rakefile +++ b/Rakefile @@ -79,7 +79,6 @@ require_relative "tasks/concourse" require_relative "tasks/css-generate" require_relative "tasks/debug" require_relative "tasks/docker" -require_relative "tasks/docs-linkify" require_relative "tasks/rubocop" require_relative "tasks/set-version-to-timestamp" diff --git a/tasks/docs-linkify.rb b/tasks/docs-linkify.rb deleted file mode 100644 index 200eeed5b6..0000000000 --- a/tasks/docs-linkify.rb +++ /dev/null @@ -1,53 +0,0 @@ -linkify_tasks = [] - -namespace "docs:linkify" do - CHANGELOG_PATH = File.expand_path(File.join(File.dirname(__FILE__), "..", HOE.history_file)) - ROADMAP_PATH = File.expand_path(File.join(File.dirname(__FILE__), "..", "ROADMAP.md")) - - [CHANGELOG_PATH, ROADMAP_PATH].each do |docfile_path| - docfile_name = File.basename(docfile_path) - taskname = docfile_name.downcase.split(".").first - - # this task is idempotent - desc "link issues and usernames in #{docfile_name}" - task taskname do - puts "linkifying #{docfile_path} ..." - docs = File.read(docfile_path) - - github_issue_regex = / - \#([[:digit:]]+) # issue number, like '#1234' - (?!\]\() # not already in a markdown hyperlink - (?![[[:digit:]]]) # don't truncate the issue number to meet the previous negative lookahead - /x - docs.gsub!(github_issue_regex, "[#\\1](#{HOE.urls["bugs"]}/\\1)") - - github_link_regex = %r{ - (? linkify_tasks From 9095706c14f7511b35cdca362005472fd4ccf552 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 08:25:24 -0400 Subject: [PATCH 27/33] cleanup(dev): move rake tasks into /rakelib --- .hoerc | 3 ++- Rakefile | 10 +--------- tasks/concourse.rb => rakelib/concourse.rake | 0 tasks/cross-ruby.rb => rakelib/cross-ruby.rake | 0 tasks/css-generate.rb => rakelib/css-generate.rake | 0 tasks/debug.rb => rakelib/debug.rake | 0 tasks/docker.rb => rakelib/docker.rake | 0 tasks/rubocop.rb => rakelib/rubocop.rake | 0 .../set-version-to-timestamp.rake | 0 {tasks => rakelib}/util.rb | 0 10 files changed, 3 insertions(+), 10 deletions(-) rename tasks/concourse.rb => rakelib/concourse.rake (100%) rename tasks/cross-ruby.rb => rakelib/cross-ruby.rake (100%) rename tasks/css-generate.rb => rakelib/css-generate.rake (100%) rename tasks/debug.rb => rakelib/debug.rake (100%) rename tasks/docker.rb => rakelib/docker.rake (100%) rename tasks/rubocop.rb => rakelib/rubocop.rake (100%) rename tasks/set-version-to-timestamp.rb => rakelib/set-version-to-timestamp.rake (100%) rename {tasks => rakelib}/util.rb (100%) diff --git a/.hoerc b/.hoerc index 913d1b035b..54c813cf5f 100644 --- a/.hoerc +++ b/.hoerc @@ -4,12 +4,13 @@ exclude: !ruby/regexp '/ (^\.\/ ((\.git + |.vagrant |.yardoc |concourse |gems |ports |suppressions - |tasks + |rakelib |test |tmp )\/) diff --git a/Rakefile b/Rakefile index c0c2946504..2323ee15cf 100644 --- a/Rakefile +++ b/Rakefile @@ -6,7 +6,7 @@ Hoe.plugin :gemspec Hoe.plugin :git Hoe.plugin :markdown -require_relative "tasks/util" +require_relative "rakelib/util" HOE = Hoe.spec "nokogiri" do |hoe| hoe.author = [ @@ -74,14 +74,6 @@ HOE = Hoe.spec "nokogiri" do |hoe| hoe.test_prelude = %q(require "helper") # ensure simplecov gets loaded before anything else end -require_relative "tasks/cross-ruby" -require_relative "tasks/concourse" -require_relative "tasks/css-generate" -require_relative "tasks/debug" -require_relative "tasks/docker" -require_relative "tasks/rubocop" -require_relative "tasks/set-version-to-timestamp" - # work around Hoe's inflexibility about the default tasks Rake::Task[:default].prerequisites.unshift("compile") Rake::Task[:default].prerequisites.unshift("rubocop") diff --git a/tasks/concourse.rb b/rakelib/concourse.rake similarity index 100% rename from tasks/concourse.rb rename to rakelib/concourse.rake diff --git a/tasks/cross-ruby.rb b/rakelib/cross-ruby.rake similarity index 100% rename from tasks/cross-ruby.rb rename to rakelib/cross-ruby.rake diff --git a/tasks/css-generate.rb b/rakelib/css-generate.rake similarity index 100% rename from tasks/css-generate.rb rename to rakelib/css-generate.rake diff --git a/tasks/debug.rb b/rakelib/debug.rake similarity index 100% rename from tasks/debug.rb rename to rakelib/debug.rake diff --git a/tasks/docker.rb b/rakelib/docker.rake similarity index 100% rename from tasks/docker.rb rename to rakelib/docker.rake diff --git a/tasks/rubocop.rb b/rakelib/rubocop.rake similarity index 100% rename from tasks/rubocop.rb rename to rakelib/rubocop.rake diff --git a/tasks/set-version-to-timestamp.rb b/rakelib/set-version-to-timestamp.rake similarity index 100% rename from tasks/set-version-to-timestamp.rb rename to rakelib/set-version-to-timestamp.rake diff --git a/tasks/util.rb b/rakelib/util.rb similarity index 100% rename from tasks/util.rb rename to rakelib/util.rb From fe314f82daa5eb507e88619c2986b87d5525850f Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 10:36:40 -0400 Subject: [PATCH 28/33] cleanup(extconf): CFLAGS handling in recipes --- ext/nokogiri/extconf.rb | 42 ++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 2ac4ce2c29..a0dc31801c 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -248,7 +248,7 @@ def process_recipe(name, version, static_p, cross_p) "--disable-shared", "--enable-static", ] - env['CFLAGS'] = "-fPIC #{env['CFLAGS']}" + env["CFLAGS"] = [env["CFLAGS"], "-fPIC"].join(" ") else recipe.configure_options += [ "--enable-shared", @@ -485,7 +485,8 @@ def install else class << recipe def configure - execute "configure", ["env", "CHOST=#{host}", "CFLAGS=-fPIC #{ENV['CFLAGS']}", "./configure", "--static", configure_prefix] + cflags = [ENV["CFLAGS"], "-fPIC", "-g"].join(" ") + execute "configure", ["env", "CHOST=#{host}", "CFLAGS=#{cflags}", "./configure", "--static", configure_prefix] end end end @@ -497,10 +498,13 @@ def configure url: "http://ftp.gnu.org/pub/gnu/libiconv/#{recipe.name}-#{recipe.version}.tar.gz", sha256: dependencies["libiconv"]["sha256"] }] + + cflags = [ENV["CFLAGS"], "-O2", "-g"].join(" ") + recipe.configure_options += [ "CPPFLAGS=-Wall", - "CFLAGS=-O2 -g", - "CXXFLAGS=-O2 -g", + "CFLAGS=#{cflags}", + "CXXFLAGS=#{cflags}", "LDFLAGS=" ] end @@ -532,15 +536,31 @@ def configure url: "http://xmlsoft.org/sources/#{recipe.name}-#{recipe.version}.tar.gz", sha256: dependencies["libxml2"]["sha256"] }] + + cflags = ENV["CFLAGS"] + + if zlib_recipe + recipe.configure_options << "--with-zlib=#{zlib_recipe.path}" + cflags = [cflags, "-I#{zlib_recipe.path}/include"].join(" ") + end + + if libiconv_recipe + recipe.configure_options << "--with-iconv=#{libiconv_recipe.path}" + else + recipe.configure_options << iconv_configure_flags + end + + if darwin? + recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] + end + recipe.configure_options += [ "--without-python", "--without-readline", - *(zlib_recipe ? ["--with-zlib=#{zlib_recipe.path}", "CFLAGS=-I#{zlib_recipe.path}/include"] : []), - *(libiconv_recipe ? "--with-iconv=#{libiconv_recipe.path}" : iconv_configure_flags), "--with-c14n", "--with-debug", "--with-threads", - *(darwin? ? ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] : "") + "CFLAGS=#{cflags}" ] end @@ -549,16 +569,20 @@ def configure url: "http://xmlsoft.org/sources/#{recipe.name}-#{recipe.version}.tar.gz", sha256: dependencies["libxslt"]["sha256"] }] + + if darwin? + recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] + end + recipe.configure_options += [ "--without-python", "--without-crypto", "--with-debug", "--with-libxml-prefix=#{sh_export_path(libxml2_recipe.path)}", - *(darwin? ? ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] : "") ] end - $CFLAGS << ' ' << '-DNOKOGIRI_USE_PACKAGED_LIBRARIES' + $CFLAGS = [$CFLAGS, "-DNOKOGIRI_USE_PACKAGED_LIBRARIES"].join(" ") $LIBPATH = ["#{zlib_recipe.path}/lib"] | $LIBPATH if zlib_recipe $LIBPATH = ["#{libiconv_recipe.path}/lib"] | $LIBPATH if libiconv_recipe From 511828967882183fa8b119f424be2de22ae5e581 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 13:21:40 -0400 Subject: [PATCH 29/33] cleanup(extconf): configuration parameter handling - extract helper concat_flags - strip spaces from configuration params --- ext/nokogiri/extconf.rb | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index a0dc31801c..26d62f1196 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -45,6 +45,10 @@ def nix? ! (windows? || solaris? || darwin?) end +def concat_flags *args + args.compact.join(" ") +end + def package_config pkg, options={} # use MakeMakefile#pkg_config, which uses the system utility `pkg-config`. package = pkg_config(pkg) @@ -248,7 +252,7 @@ def process_recipe(name, version, static_p, cross_p) "--disable-shared", "--enable-static", ] - env["CFLAGS"] = [env["CFLAGS"], "-fPIC"].join(" ") + env["CFLAGS"] = concat_flags(env["CFLAGS"], "-fPIC") else recipe.configure_options += [ "--enable-shared", @@ -266,13 +270,13 @@ def process_recipe(name, version, static_p, cross_p) if RbConfig::CONFIG['target_cpu'] == 'universal' %w[CFLAGS LDFLAGS].each do |key| unless env[key].include?('-arch') - env[key] += ' ' + RbConfig::CONFIG['ARCH_FLAG'] + env[key] = concat_flags(env[key], RbConfig::CONFIG['ARCH_FLAG']) end end end recipe.configure_options += env.map do |key, value| - "#{key}=#{value}" + "#{key}=#{value.strip}" end message <<~EOM @@ -399,7 +403,7 @@ def do_clean append_cflags(ENV["CFLAGS"].split(/\s+/)) if !ENV["CFLAGS"].nil? append_cppflags(ENV["CPPFLAGS"].split(/\s+/)) if !ENV["CPPFLAGS"].nil? append_ldflags(ENV["LDFLAGS"].split(/\s+/)) if !ENV["LDFLAGS"].nil? -$LIBS << " #{ENV["LIBS"]}" +$LIBS = concat_flags($LIBS, ENV["LIBS"]) append_cflags("-g") # always include debugging information append_cflags("-Winline") # we use at least one inline function in the C extension @@ -485,7 +489,7 @@ def install else class << recipe def configure - cflags = [ENV["CFLAGS"], "-fPIC", "-g"].join(" ") + cflags = concat_flags(ENV["CFLAGS"], "-fPIC", "-g") execute "configure", ["env", "CHOST=#{host}", "CFLAGS=#{cflags}", "./configure", "--static", configure_prefix] end end @@ -541,13 +545,13 @@ def configure if zlib_recipe recipe.configure_options << "--with-zlib=#{zlib_recipe.path}" - cflags = [cflags, "-I#{zlib_recipe.path}/include"].join(" ") + cflags = concat_flags(cflags, "-I#{zlib_recipe.path}/include") end if libiconv_recipe recipe.configure_options << "--with-iconv=#{libiconv_recipe.path}" else - recipe.configure_options << iconv_configure_flags + recipe.configure_options += iconv_configure_flags end if darwin? @@ -560,7 +564,7 @@ def configure "--with-c14n", "--with-debug", "--with-threads", - "CFLAGS=#{cflags}" + "CFLAGS=#{cflags}", ] end @@ -582,7 +586,7 @@ def configure ] end - $CFLAGS = [$CFLAGS, "-DNOKOGIRI_USE_PACKAGED_LIBRARIES"].join(" ") + $CFLAGS = concat_flags($CFLAGS, "-DNOKOGIRI_USE_PACKAGED_LIBRARIES") $LIBPATH = ["#{zlib_recipe.path}/lib"] | $LIBPATH if zlib_recipe $LIBPATH = ["#{libiconv_recipe.path}/lib"] | $LIBPATH if libiconv_recipe From a93cf67dc94bec64a9d3848f8968341fc05f054b Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 13:22:29 -0400 Subject: [PATCH 30/33] cleanup(extconf): add configuration params to build output and skip printing the big notice if the library is already build --- ext/nokogiri/extconf.rb | 45 ++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 26d62f1196..d3f00d761d 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -279,38 +279,41 @@ def process_recipe(name, version, static_p, cross_p) "#{key}=#{value.strip}" end - message <<~EOM - ---------- IMPORTANT NOTICE ---------- - Building Nokogiri with a packaged version of #{name}-#{version}#{'.' if recipe.patch_files.empty?} - EOM + checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" + if File.exist?(checkpoint) + message "Building Nokogiri with a packaged version of #{name}-#{version}.\n" + else + message <<~EOM + ---------- IMPORTANT NOTICE ---------- + Building Nokogiri with a packaged version of #{name}-#{version}. + Configuration options: #{recipe.configure_options.shelljoin} + EOM - unless recipe.patch_files.empty? - message "with the following patches applied:\n" + unless recipe.patch_files.empty? + message "The following patches are being applied:\n" - recipe.patch_files.each do |patch| - message " - %s\n" % File.basename(patch) + recipe.patch_files.each do |patch| + message " - %s\n" % File.basename(patch) + end end - end - message <<~EOM + message <<~EOM - The Nokogiri maintainers intend to provide timely security updates, but if - this is a concern for you and want to use your OS/distro system library - instead, then abort this installation process and install nokogiri as - instructed at: + The Nokogiri maintainers intend to provide timely security updates, but if + this is a concern for you and want to use your OS/distro system library + instead, then abort this installation process and install nokogiri as + instructed at: https://nokogiri.org/tutorials/installing_nokogiri.html#install-with-system-libraries - EOM + EOM - message <<~EOM if name == 'libxml2' - Note, however, that nokogiri cannot guarantee compatiblity with every - version of libxml2 that may be provided by OS/package vendors. + message <<~EOM if name == 'libxml2' + Note, however, that nokogiri cannot guarantee compatiblity with every + version of libxml2 that may be provided by OS/package vendors. - EOM + EOM - checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" - unless File.exist?(checkpoint) chdir_for_build do recipe.cook end From 4f5c4d6bf0d9545d1cd55c0aee07fcab68976329 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 13:42:06 -0400 Subject: [PATCH 31/33] cleanup(extconf): ensure config variables don't get clobbered if, for example, CFLAGS appeared multiple times then it would get clobbered. this change appends all the values to a single set of flags. --- ext/nokogiri/extconf.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index d3f00d761d..359a7491d6 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -240,7 +240,11 @@ def process_recipe(name, version, static_p, cross_p) recipe.configure_options.delete_if do |option| case option when /\A(\w+)=(.*)\z/ - env[$1] = $2 + if env.key?($1) + env[$1] = concat_flags(env[$1], $2) + else + env[$1] = $2 + end true else false From 0b49210cdfd2c1089b7af6fd91ff5399481303cd Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 13:44:44 -0400 Subject: [PATCH 32/33] perf: Always compile libxml2 and libxslt with '-O2 -g' This is the default for those libraries, but we were overriding CFLAGS in some cases. Additionally, we need to compile with -U_FORTIFY_SOURCE to avoid Ubuntu's convention of setting -D_FORTIFY_SOURCE=2 when -O2 is set. This leads to problems when running precompiled libraries on musl (see #2100 for details). Closes #2022. --- ext/nokogiri/extconf.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 359a7491d6..466b406f69 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -510,7 +510,7 @@ def configure sha256: dependencies["libiconv"]["sha256"] }] - cflags = [ENV["CFLAGS"], "-O2", "-g"].join(" ") + cflags = concat_flags(ENV["CFLAGS"], "-O2", "-U_FORTIFY_SOURCE", "-g") recipe.configure_options += [ "CPPFLAGS=-Wall", @@ -548,7 +548,7 @@ def configure sha256: dependencies["libxml2"]["sha256"] }] - cflags = ENV["CFLAGS"] + cflags = concat_flags(ENV["CFLAGS"], "-O2", "-U_FORTIFY_SOURCE", "-g") if zlib_recipe recipe.configure_options << "--with-zlib=#{zlib_recipe.path}" @@ -581,6 +581,8 @@ def configure sha256: dependencies["libxslt"]["sha256"] }] + cflags = concat_flags(ENV["CFLAGS"], "-O2", "-U_FORTIFY_SOURCE", "-g") + if darwin? recipe.configure_options += ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] end @@ -590,6 +592,7 @@ def configure "--without-crypto", "--with-debug", "--with-libxml-prefix=#{sh_export_path(libxml2_recipe.path)}", + "CFLAGS=#{cflags}", ] end From ed2aa3b65f24d2de5908417f37614fad4f9b8f60 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 26 Oct 2020 14:39:50 -0400 Subject: [PATCH 33/33] doc: update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c631eaf619..ac77fd9dc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ This release ends support for: * Improve performance of some namespace operations. [[#1916](https://github.com/sparklemotion/nokogiri/issues/1916)] (Thanks, [@ashmaroli](https://github.com/ashmaroli)!) * Remove unnecessary array allocations from Node serialization methods [[#1911](https://github.com/sparklemotion/nokogiri/issues/1911)] (Thanks, [@ashmaroli](https://github.com/ashmaroli)!) * Avoid creation of unnecessary zero-length String objects. [[#1970](https://github.com/sparklemotion/nokogiri/issues/1970)] (Thanks, [@ashmaroli](https://github.com/ashmaroli)!) +* Always compile libxml2 and libxslt with '-O2' [[#2022](https://github.com/sparklemotion/nokogiri/issues/2022), [#2100](https://github.com/sparklemotion/nokogiri/issues/2100)] (Thanks, [@ilyazub](https://github.com/ilyazub)!) * [JRuby] Lots of code cleanup and performance improvements. [[#1934](https://github.com/sparklemotion/nokogiri/issues/1934)] (Thanks, [@kares](https://github.com/kares)!) * [JRuby] Clean up deprecated calls into JRuby. [[#2027](https://github.com/sparklemotion/nokogiri/issues/2027)] (Thanks, [@headius](https://github.com/headius)!)