Skip to content

Commit

Permalink
Merge pull request #847 from joshcooper/mac_cross_compile
Browse files Browse the repository at this point in the history
Correctly patch host ruby 3.2.3 and 3.2.4 when cross compiling
  • Loading branch information
cthorn42 committed May 14, 2024
2 parents 8caf9c6 + 9c19360 commit 15f7f1e
Show file tree
Hide file tree
Showing 5 changed files with 192 additions and 86 deletions.
5 changes: 3 additions & 2 deletions configs/components/_base-rubygem.rb
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,14 @@

# If a gem needs more command line options to install set the :gem_install_options
# in its component file rubygem-<compoment>, before the instance_eval of this file.
if settings[:gem_install_options].nil?
gem_install_options = settings["#{pkg.get_name}_gem_install_options".to_sym]
if gem_install_options.nil?
pkg.install do
"#{settings[:gem_install]} #{name}-#{version}.gem"
end
else
pkg.install do
"#{settings[:gem_install]} #{name}-#{version}.gem #{settings[:gem_install_options]}"
"#{settings[:gem_install]} #{name}-#{version}.gem #{gem_install_options}"
end
end

71 changes: 17 additions & 54 deletions configs/components/pl-ruby-patch.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,70 +8,33 @@
# This component should also be present in the puppet-agent project
component "pl-ruby-patch" do |pkg, settings, platform|
if platform.is_cross_compiled?
if platform.is_macos?
pkg.build_requires 'gnu-sed'
pkg.environment "PATH", "/usr/local/opt/gnu-sed/libexec/gnubin:$(PATH)"
end

ruby_api_version = settings[:ruby_version].gsub(/\.\d*$/, '.0')
ruby_version_y = settings[:ruby_version].gsub(/(\d+)\.(\d+)\.(\d+)/, '\1.\2')

base_ruby = case platform.name
when /solaris-10/
"/opt/csw/lib/ruby/2.0.0"
when /osx/
"/usr/local/opt/ruby@#{ruby_version_y}/lib/ruby/#{ruby_api_version}"
else
"/opt/pl-build-tools/lib/ruby/2.1.0"
end
pkg.add_source("file://resources/files/ruby/patch-hostruby.rb")

# The `target_triple` determines which directory native extensions are stored in the
# compiled ruby and must match ruby's naming convention.
#
# solaris 10 uses ruby 2.0 which doesn't install native extensions based on architecture
unless platform.name =~ /solaris-10/
# weird architecture naming conventions...
target_triple = if platform.architecture =~ /ppc64el|ppc64le/
"powerpc64le-linux"
elsif platform.name == 'solaris-11-sparc'
"sparc-solaris-2.11"
elsif platform.is_macos?
if ruby_version_y.start_with?('2')
"aarch64-darwin"
else
"arm64-darwin"
end
# weird architecture naming conventions...
target_triple = if platform.architecture =~ /ppc64el|ppc64le/
"powerpc64le-linux"
elsif platform.name == 'solaris-11-sparc'
"sparc-solaris-2.11"
elsif platform.name =~ /solaris-10/
"sparc-solaris"
elsif platform.is_macos?
if ruby_version_y.start_with?('2')
"aarch64-darwin"
else
"#{platform.architecture}-linux"
"arm64-darwin"
end
else
"#{platform.architecture}-linux"
end

pkg.build do
[
%(#{platform[:sed]} -i 's/Gem::Platform.local.to_s/"#{target_triple}"/' #{base_ruby}/rubygems/basic_specification.rb),
%(#{platform[:sed]} -i 's/Gem.extension_api_version/"#{ruby_api_version}"/' #{base_ruby}/rubygems/basic_specification.rb)
]
end
end

# make rubygems use our target rbconfig when installing gems
case File.basename(base_ruby)
when '2.0.0', '2.1.0'
sed_command = %(s|Gem.ruby|&, '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{settings[:ruby_version]}-orig.rb'|)
else
sed_command = %(s|Gem.ruby.shellsplit|& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{settings[:ruby_version]}-orig.rb'|)
end

# rubygems switched which file has the command we need to patch starting in rubygems 3.4.10, which we install in our formula
# for ruby in homebrew-puppet
if Gem::Version.new(settings[:ruby_version]) >= Gem::Version.new('3.2.2') || platform.is_macos? && ruby_version_y.start_with?('2')
filename = 'builder.rb'
else
filename = 'ext_conf_builder.rb'
end

pkg.build do
pkg.install do
[
%(#{platform[:sed]} -i "#{sed_command}" #{base_ruby}/rubygems/ext/#{filename})
"#{settings[:host_ruby]} patch-hostruby.rb #{settings[:ruby_version]} #{target_triple}"
]
end
end
Expand Down
48 changes: 23 additions & 25 deletions configs/components/rubygem-ffi.rb
Original file line number Diff line number Diff line change
Expand Up @@ -15,38 +15,16 @@
pkg.sha256sum '6f2ed2fa68047962d6072b964420cba91d82ce6fa8ee251950c17fca6af3c2a0'
end

instance_eval File.read('configs/components/_base-rubygem.rb')

rb_major_minor_version = settings[:ruby_version].to_f

# Prior to ruby 3.2, both ruby and the ffi gem vendored a version of libffi.
# If libffi happened to be installed in /usr/lib, then the ffi gem preferred
# that instead of building libffi itself. To ensure consistency, we use
# --disable-system-ffi so that the ffi gem *always* builds libffi, then
# builds the ffi_c native extension and links it against libffi.so.
#
# In ruby 3.2 and up, libffi is no longer vendored. So we created a separate
# libffi vanagon component which is built before ruby. The ffi gem still
# vendors libffi, so we use the --enable-system-ffi option to ensure the ffi
# gem *always* uses the libffi.so we already built. Note the term "system" is
# misleading, because we override PKG_CONFIG_PATH below so that our libffi.so
# is preferred, not the one in /usr/lib.
if rb_major_minor_version > 2.7
pkg.install do
"#{settings[:gem_install]} ffi-#{pkg.get_version}.gem -- --enable-system-ffi"
end
else
pkg.install do
"#{settings[:gem_install]} ffi-#{pkg.get_version}.gem -- --disable-system-ffi"
end
end

# Windows versions of the FFI gem have custom filenames, so we overwite the
# defaults that _base-rubygem provides here, just for Windows for Ruby < 3.2
if platform.is_windows? && rb_major_minor_version < 3.2
# Pin this if lower than Ruby 2.7
pkg.version '1.9.25' if rb_major_minor_version < 2.7

instance_eval File.read('configs/components/_base-rubygem.rb')

# Vanagon's `pkg.mirror` is additive, and the _base_rubygem sets the
# non-Windows gem as the first mirror, which is incorrect. We need to unset
# the list of mirrors before adding the Windows-appropriate ones here:
Expand Down Expand Up @@ -81,6 +59,26 @@
pkg.install do
"#{settings[:gem_install]} ffi-#{pkg.get_version}-#{platform.architecture}-mingw32.gem"
end
else
# Prior to ruby 3.2, both ruby and the ffi gem vendored a version of libffi.
# If libffi happened to be installed in /usr/lib, then the ffi gem preferred
# that instead of building libffi itself. To ensure consistency, we use
# --disable-system-libffi so that the ffi gem *always* builds libffi, then
# builds the ffi_c native extension and links it against libffi.so.
#
# In ruby 3.2 and up, libffi is no longer vendored. So we created a separate
# libffi vanagon component which is built before ruby. The ffi gem still
# vendors libffi, so we use the --enable-system-libffi option to ensure the ffi
# gem *always* uses the libffi.so we already built. Note the term "system" is
# misleading, because we override PKG_CONFIG_PATH below so that our libffi.so
# is preferred, not the one in /usr/lib.
settings["#{pkg.get_name}_gem_install_options".to_sym] =
if rb_major_minor_version > 2.7
"-- --enable-system-libffi"
else
"-- --disable-system-libffi"
end
instance_eval File.read('configs/components/_base-rubygem.rb')
end

# due to contrib/make_sunver.pl missing on solaris 11 we cannot compile libffi, so we provide the opencsw library
Expand Down Expand Up @@ -157,4 +155,4 @@
%(#{platform[:sed]} -i '0,/ensure_required_ruby_version_met/b; /ensure_required_ruby_version_met/d' #{base_ruby}/rubygems/installer.rb)
end
end
end
end
9 changes: 4 additions & 5 deletions configs/components/rubygem-nokogiri.rb
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
component 'rubygem-nokogiri' do |pkg, _settings, _platform|
component 'rubygem-nokogiri' do |pkg, settings, _platform|
pkg.version '1.14.2'
pkg.sha256sum 'c765a74aac6cf430a710bb0b6038b8ee11f177393cd6ae8dadc7a44a6e2658b6'
# On macOS when we are not cross compiling we need to use runtime's libxml2 and libxslt
if platform.is_macos? && !platform.is_cross_compiled?
settings[:gem_install_options] = "-- --use-system-libraries \

settings["#{pkg.get_name}_gem_install_options".to_sym] = "--platform=ruby -- \
--use-system-libraries \
--with-xml2-lib=#{settings[:libdir]} \
--with-xml2-include=#{settings[:includedir]}/libxml2 \
--with-xslt-lib=#{settings[:libdir]} \
--with-xslt-include=#{settings[:includedir]}"
end
instance_eval File.read('configs/components/_base-rubygem.rb')
pkg.build_requires 'rubygem-mini_portile2'
gem_home = settings[:gem_home]
Expand Down
145 changes: 145 additions & 0 deletions resources/files/ruby/patch-hostruby.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
# When cross compiling we need to run gem install using the host ruby, but
# force ruby to use our overridden rbconfig.rb. To do that, we insert a
# require statement between the ruby executable and it's first argument,
# thereby hooking the ruby process.
#
# In the future we could use the --target-rbconfig=<path> option to point
# to our rbconfig.rb. But that option is only available in newer ruby versions.
require 'rbconfig'
require 'tempfile'

if ARGV.length < 2
warn <<USAGE
USAGE: patch-hostruby.rb <target_ruby_version> <target_triple>
example: patch-hostruby.rb 3.2.2 arm64-darwin
USAGE
exit(1)
end

# target ruby versions (what we're trying to build)
target_ruby_version = ARGV[0]
target_triple = ARGV[1]
target_api_version = target_ruby_version.gsub(/\.\d*$/, '.0')

# host ruby (the ruby we execute to build the target)
host_rubylibdir = RbConfig::CONFIG['rubylibdir']
GEM_VERSION = Gem::Version.new(Gem::VERSION)

# Rewrite the file in-place securely, yielding each line to the caller
def rewrite(file)
# create temp file in the same directory as the file we're patching,
# so rename doesn't cross filesystems
tmpfile = Tempfile.new(File.basename(file), File.dirname(file))
begin
File.open("#{file}.orig", "w") do |orig|
File.open(file, 'r').readlines.each do |line|
orig.write(line)
yield line
tmpfile.write(line)
end
end
ensure
tmpfile.close
File.unlink(file)
File.rename(tmpfile.path, file)
tmpfile.unlink
end
end

# Based on the RUBYGEMS version of the host ruby, the line and file that needs patching is different
# Note the RUBY version doesn't matter (for either the host or target ruby).
#
# Here we define different intervals. For each interval, we specify the regexp to match, what to
# replace it with, and which file to edit in-place. Note `\&` is a placeholder for whatever the regexp
# was, that way we can easily append to it. And since it's in a double quoted string, it's escaped
# as `\\&`
#
if GEM_VERSION <= Gem::Version.new('2.0.0')
# $ git show v2.0.0:lib/rubygems/ext/ext_conf_builder.rb
# cmd = "#{Gem.ruby} #{File.basename extension}"
regexp = /{Gem\.ruby}/
replace = "\\& -r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb"
builder = 'rubygems/ext/ext_conf_builder.rb'
elsif GEM_VERSION < Gem::Version.new('3.0.0') # there weren't any tags between >= 2.7.11 and < 3.0.0
# $ git show v2.0.1:lib/rubygems/ext/ext_conf_builder.rb
# cmd = [Gem.ruby, File.basename(extension), *args].join ' '
#
# $ git show v2.7.11:lib/rubygems/ext/ext_conf_builder.rb
# cmd = [Gem.ruby, "-r", get_relative_path(siteconf.path), File.basename(extension), *args].join ' '
regexp = /Gem\.ruby/
replace = "\\&, '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'"
builder = 'rubygems/ext/ext_conf_builder.rb'
elsif GEM_VERSION <= Gem::Version.new('3.4.8')
# $ git show v3.0.0:lib/rubygems/ext/ext_conf_builder.rb
# cmd = Gem.ruby.shellsplit << "-I" << File.expand_path("../../..", __FILE__) <<
#
# $ git show v3.4.8:lib/rubygems/ext/ext_conf_builder.rb
# cmd = Gem.ruby.shellsplit << "-I" << File.expand_path("../..", __dir__) << File.basename(extension)
regexp = /Gem\.ruby\.shellsplit/
replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'"
builder = 'rubygems/ext/ext_conf_builder.rb'
elsif GEM_VERSION <= Gem::Version.new('3.4.14')
# NOTE: rubygems 3.4.9 moved the code to builder.rb
#
# $ git show v3.4.9:lib/rubygems/ext/builder.rb
# cmd = Gem.ruby.shellsplit
#
# $ git show v3.4.14:lib/rubygems/ext/builder.rb
# cmd = Gem.ruby.shellsplit
regexp = /Gem\.ruby\.shellsplit/
replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'"
builder = 'rubygems/ext/builder.rb'
elsif GEM_VERSION <= Gem::Version.new('3.5.10')
# $ git show v3.4.9:lib/rubygems/ext/builder.rb
# cmd = Shellwords.split(Gem.ruby)
#
# $ git show v3.5.10:lib/rubygems/ext/builder.rb
# cmd = Shellwords.split(Gem.ruby)
regexp = /Shellwords\.split\(Gem\.ruby\)/
replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'"
builder = 'rubygems/ext/builder.rb'
else
raise "We don't know how to patch rubygems #{GEM_VERSION}"
end

# path to the builder file on the HOST ruby
builder = File.join(host_rubylibdir, builder)

raise "We can't patch #{builder} because it doesn't exist" unless File.exist?(builder)

# hook rubygems builder so it loads our rbconfig when building native gems
patched = false
rewrite(builder) do |line|
if line.gsub!(regexp, replace)
patched = true
end
end

raise "Failed to patch rubygems hook, because we couldn't match #{regexp} in #{builder}" unless patched

puts "Patched '#{regexp.inspect}' in #{builder}"

# solaris 10 uses ruby 2.0 which doesn't install native extensions based on architecture
if RUBY_PLATFORM !~ /solaris2\.10$/ || RUBY_VERSION != '2.0.0'
# ensure native extensions are written to a directory that matches the
# architecture of the target ruby we're building for. To do that we
# patch the host ruby to pretend to be the target architecture.
triple_patched = false
api_version_patched = false
spec_file = "#{host_rubylibdir}/rubygems/basic_specification.rb"
rewrite(spec_file) do |line|
if line.gsub!(/Gem::Platform\.local\.to_s/, "'#{target_triple}'")
triple_patched = true
end
if line.gsub!(/Gem\.extension_api_version/, "'#{target_api_version}'")
api_version_patched = true
end
end

raise "Failed to patch '#{target_triple}' in #{spec_file}" unless triple_patched
puts "Patched '#{target_triple}' in #{spec_file}"

raise "Failed to patch '#{target_api_version}' in #{spec_file}" unless api_version_patched
puts "Patched '#{target_api_version}' in #{spec_file}"
end

0 comments on commit 15f7f1e

Please sign in to comment.