Merge pull request #3991 from rubygems/fileutils_double_load

Fix fileutils double load when using `bundler/inline`

(cherry picked from commit 6ebda1d)

Manifest.txt

@@ -229,6 +229,7 @@ bundler/lib/bundler/vendor/thor/lib/thor/shell/color.rb
 bundler/lib/bundler/vendor/thor/lib/thor/shell/html.rb
 bundler/lib/bundler/vendor/thor/lib/thor/util.rb
 bundler/lib/bundler/vendor/thor/lib/thor/version.rb
+bundler/lib/bundler/vendor/tmpdir/lib/tmpdir.rb
 bundler/lib/bundler/vendor/uri/lib/uri.rb
 bundler/lib/bundler/vendor/uri/lib/uri/common.rb
 bundler/lib/bundler/vendor/uri/lib/uri/file.rb
@@ -246,6 +247,7 @@ bundler/lib/bundler/vendored_fileutils.rb
 bundler/lib/bundler/vendored_molinillo.rb
 bundler/lib/bundler/vendored_persistent.rb
 bundler/lib/bundler/vendored_thor.rb
+bundler/lib/bundler/vendored_tmpdir.rb
 bundler/lib/bundler/vendored_uri.rb
 bundler/lib/bundler/version.rb
 bundler/lib/bundler/version_ranges.rb

bundler/Rakefile

@@ -175,13 +175,30 @@ Automatiek::RakeTask.new("thor") do |lib|
   lib.vendor_lib = "lib/bundler/vendor/thor"
 end
 
-desc "Vendor a specific version of fileutils"
-Automatiek::RakeTask.new("fileutils") do |lib|
-  lib.version = "v1.4.1"
-  lib.download = { :github => "https://github.com/ruby/fileutils" }
-  lib.namespace = "FileUtils"
+# We currently include the official version as of
+# https://github.com/ruby/tmpdir/tree/a3e06bd49829dc49c346aa10f8b91116fd7e17ad,
+# with the following changes on top:
+# * require fileutils relatively to use our vendored version.
+# * Inherit from `Dir` so that code assuming we're inside the
+#   `Dir` class still works. Also change the `systmpdir` class variable to an
+#   instance variable since otherwise inheriting from dir doesn't work.
+# * Remove a "block variable shadowing outer variable" warning on older rubies
+#   that was breaking some specs.
+desc "Vendor a specific version of tmpdir"
+Automatiek::RakeTask.new("tmpdir") do |lib|
+  lib.version = "master"
+  lib.download = { :github => "https://github.com/ruby/tmpdir" }
+  lib.namespace = "Dir"
   lib.prefix = "Bundler"
-  lib.vendor_lib = "lib/bundler/vendor/fileutils"
+  lib.vendor_lib = "lib/bundler/vendor/tmpdir"
+
+  lib.dependency("fileutils") do |sublib|
+    sublib.version = "v1.4.1"
+    sublib.download = { :github => "https://github.com/ruby/fileutils" }
+    sublib.namespace = "FileUtils"
+    sublib.prefix = "Bundler"
+    sublib.vendor_lib = "lib/bundler/vendor/fileutils"
+  end
 end
 
 # We currently include the following changes over the official version:

bundler/lib/bundler/compact_index_client/updater.rb

@@ -22,13 +22,13 @@ def message
 
       def initialize(fetcher)
         @fetcher = fetcher
-        require "tmpdir"
+        require_relative "../vendored_tmpdir"
       end
 
       def update(local_path, remote_path, retrying = nil)
         headers = {}
 
-        Dir.mktmpdir("bundler-compact-index-") do |local_temp_dir|
+        Bundler::Dir.mktmpdir("bundler-compact-index-") do |local_temp_dir|
           local_temp_path = Pathname.new(local_temp_dir).join(local_path.basename)
 
           # first try to fetch any new bytes on the existing file

bundler/lib/bundler/vendor/tmpdir/lib/tmpdir.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+#
+# tmpdir - retrieve temporary directory path
+#
+# $Id$
+#
+
+require_relative '../../fileutils/lib/fileutils'
+begin
+  require 'etc.so'
+rescue LoadError # rescue LoadError for miniruby
+end
+
+class Bundler::Dir < Dir
+
+  @systmpdir ||= defined?(Etc.systmpdir) ? Etc.systmpdir : '/tmp'
+
+  ##
+  # Returns the operating system's temporary file path.
+
+  def self.tmpdir
+    tmp = nil
+    ['TMPDIR', 'TMP', 'TEMP', ['system temporary path', @systmpdir], ['/tmp']*2, ['.']*2].each do |name, dir = ENV[name]|
+      next if !dir
+      dir = File.expand_path(dir)
+      stat = File.stat(dir) rescue next
+      case
+      when !stat.directory?
+        warn "#{name} is not a directory: #{dir}"
+      when !stat.writable?
+        warn "#{name} is not writable: #{dir}"
+      when stat.world_writable? && !stat.sticky?
+        warn "#{name} is world-writable: #{dir}"
+      else
+        tmp = dir
+        break
+      end
+    end
+    raise ArgumentError, "could not find a temporary directory" unless tmp
+    tmp
+  end
+
+  # Bundler::Dir.mktmpdir creates a temporary directory.
+  #
+  # The directory is created with 0700 permission.
+  # Application should not change the permission to make the temporary directory accessible from other users.
+  #
+  # The prefix and suffix of the name of the directory is specified by
+  # the optional first argument, <i>prefix_suffix</i>.
+  # - If it is not specified or nil, "d" is used as the prefix and no suffix is used.
+  # - If it is a string, it is used as the prefix and no suffix is used.
+  # - If it is an array, first element is used as the prefix and second element is used as a suffix.
+  #
+  #  Bundler::Dir.mktmpdir {|dir| dir is ".../d..." }
+  #  Bundler::Dir.mktmpdir("foo") {|dir| dir is ".../foo..." }
+  #  Bundler::Dir.mktmpdir(["foo", "bar"]) {|dir| dir is ".../foo...bar" }
+  #
+  # The directory is created under Bundler::Dir.tmpdir or
+  # the optional second argument <i>tmpdir</i> if non-nil value is given.
+  #
+  #  Bundler::Dir.mktmpdir {|dir| dir is "#{Bundler::Dir.tmpdir}/d..." }
+  #  Bundler::Dir.mktmpdir(nil, "/var/tmp") {|dir| dir is "/var/tmp/d..." }
+  #
+  # If a block is given,
+  # it is yielded with the path of the directory.
+  # The directory and its contents are removed
+  # using Bundler::FileUtils.remove_entry before Bundler::Dir.mktmpdir returns.
+  # The value of the block is returned.
+  #
+  #  Bundler::Dir.mktmpdir {|dir|
+  #    # use the directory...
+  #    open("#{dir}/foo", "w") { ... }
+  #  }
+  #
+  # If a block is not given,
+  # The path of the directory is returned.
+  # In this case, Bundler::Dir.mktmpdir doesn't remove the directory.
+  #
+  #  dir = Bundler::Dir.mktmpdir
+  #  begin
+  #    # use the directory...
+  #    open("#{dir}/foo", "w") { ... }
+  #  ensure
+  #    # remove the directory.
+  #    Bundler::FileUtils.remove_entry dir
+  #  end
+  #
+  def self.mktmpdir(prefix_suffix=nil, *rest, **options)
+    base = nil
+    path = Tmpname.create(prefix_suffix || "d", *rest, **options) {|p, _, _, d|
+      base = d
+      mkdir(p, 0700)
+    }
+    if block_given?
+      begin
+        yield path.dup
+      ensure
+        unless base
+          stat = File.stat(File.dirname(path))
+          if stat.world_writable? and !stat.sticky?
+            raise ArgumentError, "parent directory is world writable but not sticky"
+          end
+        end
+        Bundler::FileUtils.remove_entry path
+      end
+    else
+      path
+    end
+  end
+
+  module Tmpname # :nodoc:
+    module_function
+
+    def tmpdir
+      Bundler::Dir.tmpdir
+    end
+
+    UNUSABLE_CHARS = [File::SEPARATOR, File::ALT_SEPARATOR, File::PATH_SEPARATOR, ":"].uniq.join("").freeze
+
+    class << (RANDOM = Random.new)
+      MAX = 36**6 # < 0x100000000
+      def next
+        rand(MAX).to_s(36)
+      end
+    end
+    private_constant :RANDOM
+
+    def create(basename, tmpdir=nil, max_try: nil, **opts)
+      origdir = tmpdir
+      tmpdir ||= tmpdir()
+      n = nil
+      prefix, suffix = basename
+      prefix = (String.try_convert(prefix) or
+                raise ArgumentError, "unexpected prefix: #{prefix.inspect}")
+      prefix = prefix.delete(UNUSABLE_CHARS)
+      suffix &&= (String.try_convert(suffix) or
+                  raise ArgumentError, "unexpected suffix: #{suffix.inspect}")
+      suffix &&= suffix.delete(UNUSABLE_CHARS)
+      begin
+        t = Time.now.strftime("%Y%m%d")
+        path = "#{prefix}#{t}-#{$$}-#{RANDOM.next}"\
+               "#{n ? %[-#{n}] : ''}#{suffix||''}"
+        path = File.join(tmpdir, path)
+        yield(path, n, opts, origdir)
+      rescue Errno::EEXIST
+        n ||= 0
+        n += 1
+        retry if !max_try or n < max_try
+        raise "cannot generate temporary name using `#{basename}' under `#{tmpdir}'"
+      end
+      path
+    end
+  end
+end

bundler/lib/bundler/vendored_tmpdir.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+module Bundler; end
+require_relative "vendor/tmpdir/lib/tmpdir"

bundler/spec/bundler/compact_index_client/updater_spec.rb

@@ -3,6 +3,7 @@
 require "net/http"
 require "bundler/compact_index_client"
 require "bundler/compact_index_client/updater"
+require "tmpdir"
 
 RSpec.describe Bundler::CompactIndexClient::Updater do
   let(:fetcher) { double(:fetcher) }
@@ -40,7 +41,7 @@
   context "when bundler doesn't have permissions on Dir.tmpdir" do
     it "Errno::EACCES is raised" do
       local_path # create local path before stubbing mktmpdir
-      allow(Dir).to receive(:mktmpdir) { raise Errno::EACCES }
+      allow(Bundler::Dir).to receive(:mktmpdir) { raise Errno::EACCES }
 
       expect do
         updater.update(local_path, remote_path)

bundler/spec/install/gemfile/path_spec.rb

@@ -718,11 +718,11 @@
       expect(bar_file).not_to be_file
 
       build_lib "foo" do |s|
-        s.write("lib/rubygems_plugin.rb", "FileUtils.touch('#{foo_file}')")
+        s.write("lib/rubygems_plugin.rb", "require 'fileutils'; FileUtils.touch('#{foo_file}')")
       end
 
       build_git "bar" do |s|
-        s.write("lib/rubygems_plugin.rb", "FileUtils.touch('#{bar_file}')")
+        s.write("lib/rubygems_plugin.rb", "require 'fileutils'; FileUtils.touch('#{bar_file}')")
       end
 
       install_gemfile <<-G

bundler/spec/plugins/source/example_spec.rb

@@ -33,6 +33,7 @@ def fetch_gemspec_files
 
               def install(spec, opts)
                 mkdir_p(install_path.parent)
+                require 'fileutils'
                 FileUtils.cp_r(path, install_path)
 
                 spec_path = install_path.join("\#{spec.full_name}.gemspec")

bundler/spec/runtime/inline_spec.rb

@@ -369,4 +369,34 @@ def confirm(msg, newline = nil)
     expect(out).to eq("WIN")
     expect(err).to be_empty
   end
+
+  it "when requiring fileutils after does not show redefinition warnings" do
+    dependency_installer_loads_fileutils = ruby "require 'rubygems/dependency_installer'; puts $LOADED_FEATURES.grep(/fileutils/)", :raise_on_error => false
+    skip "does not work if rubygems/dependency_installer loads fileutils, which happens until rubygems 3.2.0" unless dependency_installer_loads_fileutils.empty?
+
+    skip "does not work on ruby 3.0 because it changes the path to look for default gems, tsort is a default gem there, and we can't install it either like we do with fiddle because it doesn't yet exist" unless RUBY_VERSION < "3.0.0"
+
+    Dir.mkdir tmp("path_without_gemfile")
+
+    default_fileutils_version = ruby "gem 'fileutils', '< 999999'; require 'fileutils'; puts FileUtils::VERSION", :raise_on_error => false
+    skip "fileutils isn't a default gem" if default_fileutils_version.empty?
+
+    realworld_system_gems "fileutils --version 1.4.1"
+
+    realworld_system_gems "fiddle" # not sure why, but this is needed on Windows to boot rubygems succesfully
+
+    realworld_system_gems "timeout uri" # this spec uses net/http which requires these default gems
+
+    script <<-RUBY, :dir => tmp("path_without_gemfile"), :env => { "BUNDLER_GEM_DEFAULT_DIR" => system_gem_path.to_s }
+      require "bundler/inline"
+
+      gemfile(true) do
+        source "#{file_uri_for(gem_repo2)}"
+      end
+
+      require "fileutils"
+    RUBY
+
+    expect(err).to eq("The Gemfile specifies no dependencies")
+  end
 end

lib/rubygems/ext/builder.rb

@@ -201,6 +201,7 @@ def build_extensions
 
     dest_path = @spec.extension_dir
 
+    require "fileutils"
     FileUtils.rm_f @spec.gem_build_complete_path
 
     @spec.extensions.each do |extension|

lib/rubygems/installer.rb

@@ -12,7 +12,6 @@
 require 'rubygems/package'
 require 'rubygems/ext'
 require 'rubygems/user_interaction'
-require 'fileutils'
 
 ##
 # The installer installs the files contained in the .gem into the Gem.home.
@@ -492,7 +491,11 @@ def generate_bin # :nodoc:
 
       mode = File.stat(bin_path).mode
       dir_mode = options[:prog_mode] || (mode | 0111)
-      FileUtils.chmod dir_mode, bin_path unless dir_mode == mode
+
+      unless dir_mode == mode
+        require 'fileutils'
+        FileUtils.chmod dir_mode, bin_path
+      end
 
       check_executable_overwrite filename
 

lib/rubygems/security.rb

@@ -6,7 +6,6 @@
 #++
 
 require 'rubygems/exceptions'
-require 'fileutils'
 require_relative 'openssl'
 
 ##