[deliver][pilot] use altool instead of using iTMSTransporter for Xcode 14

deliver/lib/deliver/runner.rb

@@ -206,7 +206,7 @@ def upload_binary
       pkg_path = options[:pkg]
 
       platform = options[:platform]
-      transporter = transporter_for_selected_team
+      transporter = transporter_for_selected_team(upload: true)
 
       case platform
       when "ios", "appletvos"
@@ -216,15 +216,15 @@ def upload_binary
           package_path: "/tmp",
           platform: platform
         )
-        result = transporter.upload(package_path: package_path, asset_path: ipa_path)
+        result = transporter.upload(package_path: package_path, asset_path: ipa_path, platform: platform)
       when "osx"
         package_path = FastlaneCore::PkgUploadPackageBuilder.new.generate(
           app_id: Deliver.cache[:app].id,
           pkg_path: pkg_path,
           package_path: "/tmp",
           platform: platform
         )
-        result = transporter.upload(package_path: package_path, asset_path: pkg_path)
+        result = transporter.upload(package_path: package_path, asset_path: pkg_path, platform: platform)
       else
         UI.user_error!("No suitable file found for upload for platform: #{options[:platform]}")
       end
@@ -270,25 +270,25 @@ def submit_for_review
     # If itc_provider was explicitly specified, use it.
     # If there are multiple teams, infer the provider from the selected team name.
     # If there are fewer than two teams, don't infer the provider.
-    def transporter_for_selected_team
+    def transporter_for_selected_team(upload: false)
       # Use JWT auth
       api_token = Spaceship::ConnectAPI.token
       unless api_token.nil?
         api_token.refresh! if api_token.expired?
-        return FastlaneCore::ItunesTransporter.new(nil, nil, false, nil, api_token.text)
+        return FastlaneCore::ItunesTransporter.new(nil, nil, false, nil, api_token.text, upload: upload, api_key: options[:api_key])
       end
 
       tunes_client = Spaceship::ConnectAPI.client.tunes_client
 
-      generic_transporter = FastlaneCore::ItunesTransporter.new(options[:username], nil, false, options[:itc_provider])
+      generic_transporter = FastlaneCore::ItunesTransporter.new(options[:username], nil, false, options[:itc_provider], upload: upload, api_key: options[:api_key])
       return generic_transporter unless options[:itc_provider].nil? && tunes_client.teams.count > 1
 
       begin
         team = tunes_client.teams.find { |t| t['providerId'].to_s == tunes_client.team_id }
         name = team['name']
         provider_id = generic_transporter.provider_ids[name]
         UI.verbose("Inferred provider id #{provider_id} for team #{name}.")
-        return FastlaneCore::ItunesTransporter.new(options[:username], nil, false, provider_id)
+        return FastlaneCore::ItunesTransporter.new(options[:username], nil, false, provider_id, upload: upload, api_key: options[:api_key])
       rescue => ex
         UI.verbose("Couldn't infer a provider short name for team with id #{tunes_client.team_id} automatically: #{ex}. Proceeding without provider short name.")
         return generic_transporter

deliver/spec/runner_spec.rb

@@ -63,7 +63,7 @@ def provider_ids
         expect_any_instance_of(FastlaneCore::IpaUploadPackageBuilder).to receive(:generate)
           .with(app_id: 'YI8C2AS', ipa_path: 'ACME.ipa', package_path: '/tmp', platform: 'ios')
           .and_return('path')
-        expect(transporter).to receive(:upload).with(package_path: 'path', asset_path: 'ACME.ipa').and_return(true)
+        expect(transporter).to receive(:upload).with(package_path: 'path', asset_path: 'ACME.ipa', platform: 'ios').and_return(true)
         runner.upload_binary
       end
     end
@@ -77,7 +77,7 @@ def provider_ids
         expect_any_instance_of(FastlaneCore::IpaUploadPackageBuilder).to receive(:generate)
           .with(app_id: 'YI8C2AS', ipa_path: 'ACME.ipa', package_path: '/tmp', platform: 'appletvos')
           .and_return('path')
-        expect(transporter).to receive(:upload).with(package_path: 'path', asset_path: 'ACME.ipa').and_return(true)
+        expect(transporter).to receive(:upload).with(package_path: 'path', asset_path: 'ACME.ipa', platform: 'appletvos').and_return(true)
         runner.upload_binary
       end
     end
@@ -93,7 +93,7 @@ def provider_ids
         expect_any_instance_of(FastlaneCore::PkgUploadPackageBuilder).to receive(:generate)
           .with(app_id: 'YI8C2AS', pkg_path: 'ACME.pkg', package_path: '/tmp', platform: 'osx')
           .and_return('path')
-        expect(transporter).to receive(:upload).with(package_path: 'path', asset_path: 'ACME.pkg').and_return(true)
+        expect(transporter).to receive(:upload).with(package_path: 'path', asset_path: 'ACME.pkg', platform: 'osx').and_return(true)
         runner.upload_binary
       end
     end

fastlane_core/lib/fastlane_core/itunes_transporter.rb

@@ -180,9 +180,122 @@ def additional_upload_parameters
     end
   end
 
+  # Generates commands and executes the altool.
+  class AltoolTransporterExecutor < TransporterExecutor
+    ERROR_REGEX = /\*\*\* Error:\s+(.+)/
+
+    private_constant :ERROR_REGEX
+
+    def execute(command, hide_output)
+      if Helper.test?
+        yield(nil) if block_given?
+        command
+      end
+
+      @errors = []
+      @all_lines = []
+
+      if hide_output
+        # Show a one time message instead
+        UI.success("Waiting for App Store Connect transporter to be finished.")
+        UI.success("Application Loader progress... this might take a few minutes...")
+      end
+
+      begin
+        exit_status = FastlaneCore::FastlanePty.spawn(command) do |command_stdout, command_stdin, pid|
+          command_stdout.each do |line|
+            @all_lines << line
+            parse_line(line, hide_output) # this is where the parsing happens
+          end
+        end
+      rescue => ex
+        # FastlanePty adds exit_status on to StandardError so every error will have a status code
+        exit_status = ex.exit_status
+        @errors << ex.to_s
+      end
+
+      @errors << "The call to the altool completed with a non-zero exit status: #{exit_status}. This indicates a failure." unless exit_status.zero?
+
+      unless @errors.empty? || @all_lines.empty?
+        # Print the last lines that appear after the last error from the logs
+        # If error text is not detected, it will be 20 lines
+        # This is key for non-verbose mode
+        error_line_index = @all_lines.rindex { |line| ERROR_REGEX.match?(line) }
+        @all_lines[(error_line_index || -20)..-1].each do |line|
+          UI.important("[altool] #{line}")
+        end
+        UI.message("Application Loader output above ^")
+        @errors.each { |error| UI.error(error) }
+      end
+
+      yield(@all_lines) if block_given?
+      exit_status.zero?
+    end
+
+    def build_upload_command(username, password, source = "/tmp", provider_short_name = "", jwt = nil, platform = nil, api_key = nil)
+      use_api_key = !api_key.nil?
+      [
+        ("API_PRIVATE_KEYS_DIR=#{Dir.tmpdir}" if use_api_key),
+        "xcrun altool",
+        "--upload-app",
+        ("-u #{username.shellescape}" unless use_api_key),
+        ("-p #{password.shellescape}" unless use_api_key),
+        ("--apiKey #{api_key[:key_id]}" if use_api_key),
+        ("--apiIssuer #{api_key[:issuer_id]}" if use_api_key),
+        platform_option(platform),
+        file_upload_option(source),
+        additional_upload_parameters,
+        "-k 100000"
+      ].compact.join(' ')
+    end
+
+    def additional_upload_parameters
+      env_deliver_additional_params = ENV["DELIVER_ALTOOL_ADDITIONAL_UPLOAD_PARAMETERS"]
+      if env_deliver_additional_params.to_s.strip.empty?
+        return nil
+      end
+
+      env_deliver_additional_params.to_s.strip
+    end
+
+    def handle_error(password)
+      UI.error("Could not download/upload from App Store Connect!")
+    end
+
+    def displayable_errors
+      @errors.map { |error| "[Application Loader Error Output]: #{error}" }.join("\n")
+    end
+
+    private
+
+    def file_upload_option(source)
+      "-f #{source.shellescape}"
+    end
+
+    def platform_option(platform)
+      "-t #{platform == 'osx' ? 'macos' : platform}"
+    end
+
+    def parse_line(line, hide_output)
+      output_done = false
+
+      if line =~ ERROR_REGEX
+        @errors << $1
+        output_done = true
+      end
+
+      unless hide_output
+        # General logging for debug purposes
+        unless output_done
+          UI.verbose("[altool]: #{line}")
+        end
+      end
+    end
+  end
+
   # Generates commands and executes the iTMSTransporter through the shell script it provides by the same name
   class ShellScriptTransporterExecutor < TransporterExecutor
-    def build_upload_command(username, password, source = "/tmp", provider_short_name = "", jwt = nil)
+    def build_upload_command(username, password, source = "/tmp", provider_short_name = "", jwt = nil, platform = nil, api_key = nil)
       use_jwt = !jwt.to_s.empty?
       [
         '"' + Helper.transporter_path + '"',
@@ -278,7 +391,7 @@ def shell_escaped_password(password)
   # Generates commands and executes the iTMSTransporter by invoking its Java app directly, to avoid the crazy parameter
   # escaping problems in its accompanying shell script.
   class JavaTransporterExecutor < TransporterExecutor
-    def build_upload_command(username, password, source = "/tmp", provider_short_name = "", jwt = nil)
+    def build_upload_command(username, password, source = "/tmp", provider_short_name = "", jwt = nil, platform = nil, api_key = nil)
       use_jwt = !jwt.to_s.empty?
       if !Helper.user_defined_itms_path? && Helper.mac? && Helper.xcode_at_least?(11)
         [
@@ -476,7 +589,7 @@ def self.hide_transporter_output?
     #                            see: https://github.com/fastlane/fastlane/issues/1524#issuecomment-196370628
     #                            for more information about how to use the iTMSTransporter to list your provider
     #                            short names
-    def initialize(user = nil, password = nil, use_shell_script = false, provider_short_name = nil, jwt = nil)
+    def initialize(user = nil, password = nil, use_shell_script = false, provider_short_name = nil, jwt = nil, upload: false, api_key: nil)
       # Xcode 6.x doesn't have the same iTMSTransporter Java setup as later Xcode versions, so
       # we can't default to using the newer direct Java invocation strategy for those versions.
       use_shell_script ||= Helper.is_mac? && Helper.xcode_version.start_with?('6.')
@@ -489,8 +602,16 @@ def initialize(user = nil, password = nil, use_shell_script = false, provider_sh
       end
 
       @jwt = jwt
+      @api_key = api_key
+
+      if should_use_altool?(upload, use_shell_script)
+        UI.verbose("Using altool as transporter.")
+        @transporter_executor = AltoolTransporterExecutor.new
+      else
+        UI.verbose("Using iTMSTransporter as transporter.")
+        @transporter_executor = use_shell_script ? ShellScriptTransporterExecutor.new : JavaTransporterExecutor.new
+      end
 
-      @transporter_executor = use_shell_script ? ShellScriptTransporterExecutor.new : JavaTransporterExecutor.new
       @provider_short_name = provider_short_name
     end
 
@@ -539,7 +660,7 @@ def download(app_id, dir = nil)
     # @return (Bool) True if everything worked fine
     # @raise [Deliver::TransporterTransferError] when something went wrong
     #   when transferring
-    def upload(app_id = nil, dir = nil, package_path: nil, asset_path: nil)
+    def upload(app_id = nil, dir = nil, package_path: nil, asset_path: nil, platform: nil)
       raise "app_id and dir are required or package_path or asset_path is required" if (app_id.nil? || dir.nil?) && package_path.nil? && asset_path.nil?
 
       # Transport can upload .ipa, .dmg, and .pkg files directly with -assetFile
@@ -569,14 +690,33 @@ def upload(app_id = nil, dir = nil, package_path: nil, asset_path: nil)
       password_placeholder = @jwt.nil? ? 'YourPassword' : nil
       jwt_placeholder = @jwt.nil? ? nil : 'YourJWT'
 
-      command = @transporter_executor.build_upload_command(@user, @password, actual_dir, @provider_short_name, @jwt)
-      UI.verbose(@transporter_executor.build_upload_command(@user, password_placeholder, actual_dir, @provider_short_name, jwt_placeholder))
+      # Handle AppStore Connect API
+      use_api_key = !@api_key.nil?
+      api_key_placeholder = nil unless use_api_key
+      api_key_placeholder = { key_id: "YourKeyID", issuer_id: "YourIssuerID" } if use_api_key
+
+      if use_api_key
+        # Specified p8 needs to be generated to call altool
+        tmp_p8_file_path = File.join(Dir.tmpdir, "AuthKey_#{@api_key[:key_id]}.p8")
+        File.open(tmp_p8_file_path, "w") do |p8|
+          key_content = @api_key[:is_key_content_base64] ? Base64.decode64(@api_key[:key]) : @api_key[:key]
+          p8.write(key_content)
+        end
+      end
+
+      command = @transporter_executor.build_upload_command(@user, @password, actual_dir, @provider_short_name, @jwt, platform, @api_key)
+      UI.verbose(@transporter_executor.build_upload_command(@user, password_placeholder, actual_dir, @provider_short_name, jwt_placeholder, platform, api_key_placeholder))
 
       begin
         result = @transporter_executor.execute(command, ItunesTransporter.hide_transporter_output?)
       rescue TransporterRequiresApplicationSpecificPasswordError => ex
         handle_two_step_failure(ex)
         return upload(app_id, dir, package_path: package_path, asset_path: asset_path)
+      ensure
+        if use_api_key
+          tmp_p8_file_path = File.join(Dir.tmpdir, "AuthKey_#{@api_key[:key_id]}.p8")
+          FileUtils.rm_f(tmp_p8_file_path)
+        end
       end
 
       if result
@@ -657,6 +797,12 @@ def provider_ids
 
     TWO_FACTOR_ENV_VARIABLE = "FASTLANE_APPLE_APPLICATION_SPECIFIC_PASSWORD"
 
+    # Returns whether altool should be used or ItunesTransporter should be used
+    def should_use_altool?(upload, use_shell_script)
+      # Xcode 14 no longer supports iTMSTransporter. Use altool instead
+      !use_shell_script && upload && !Helper.user_defined_itms_path? && Helper.mac? && Helper.xcode_at_least?(14)
+    end
+
     # Returns the password to be used with the transporter
     def load_password_for_transporter
       # 3 different sources for the password

fastlane_core/spec/helper_spec.rb

@@ -144,7 +144,9 @@
       end
 
       it "#transporter_path", requires_xcode: true do
-        expect(FastlaneCore::Helper.transporter_path).to match(%r{/Applications/Xcode.*.app/Contents/Applications/Application Loader.app/Contents/itms/bin/iTMSTransporter|/Applications/Xcode.*.app/Contents/SharedFrameworks/ContentDeliveryServices.framework/Versions/A/itms/bin/iTMSTransporter})
+        unless FastlaneCore::Helper.xcode_at_least?("14")
+          expect(FastlaneCore::Helper.transporter_path).to match(%r{/Applications/Xcode.*.app/Contents/Applications/Application Loader.app/Contents/itms/bin/iTMSTransporter|/Applications/Xcode.*.app/Contents/SharedFrameworks/ContentDeliveryServices.framework/Versions/A/itms/bin/iTMSTransporter})
+        end
       end
 
       it "#xcode_version", requires_xcode: true do
@@ -178,7 +180,13 @@
       context "#itms_path" do
         it "default", requires_xcode: true do
           stub_const('ENV', { 'FASTLANE_ITUNES_TRANSPORTER_PATH' => nil })
-          expect(FastlaneCore::Helper.itms_path).to match(/itms/)
+
+          if FastlaneCore::Helper.xcode_at_least?("14")
+            expect(FastlaneCore::UI).to receive(:user_error!).with(/Could not find transporter/)
+            FastlaneCore::Helper.itms_path
+          else
+            expect(FastlaneCore::Helper.itms_path).to match(/itms/)
+          end
         end
 
         it "uses FASTLANE_ITUNES_TRANSPORTER_PATH", requires_xcode: true do

fastlane_core/spec/itunes_transporter_spec.rb

@@ -1110,6 +1110,7 @@ def xcrun_download_command(provider_short_name = nil, jwt: nil)
     describe "with `FASTLANE_ITUNES_TRANSPORTER_USE_SHELL_SCRIPT` set" do
       before(:each) do
         ENV["FASTLANE_ITUNES_TRANSPORTER_USE_SHELL_SCRIPT"] = "1"
+        allow(FastlaneCore::Helper).to receive(:itms_path).and_return('/tmp')
         allow(File).to receive(:exist?).with("C:/Program Files (x86)/itms").and_return(true) if FastlaneCore::Helper.windows?
       end
 
@@ -1139,6 +1140,7 @@ def xcrun_download_command(provider_short_name = nil, jwt: nil)
 
     describe "with no special configuration" do
       before(:each) do
+        allow(FastlaneCore::Helper).to receive(:itms_path).and_return('/tmp')
         allow(File).to receive(:exist?).and_return(true) unless FastlaneCore::Helper.mac?
         ENV.delete("FASTLANE_ITUNES_TRANSPORTER_USE_SHELL_SCRIPT")
       end
@@ -1273,6 +1275,7 @@ def xcrun_download_command(provider_short_name = nil, jwt: nil)
     describe "with simulated no-test environment" do
       before(:each) do
         allow(FastlaneCore::Helper).to receive(:test?).and_return(false)
+        allow(FastlaneCore::Helper).to receive(:itms_path).and_return('/tmp')
         @transporter = FastlaneCore::ItunesTransporter.new(email, password, false)
       end
 

pilot/lib/pilot/build_manager.rb

@@ -47,7 +47,7 @@ def upload(options)
       end
 
       transporter = transporter_for_selected_team(options)
-      result = transporter.upload(package_path: package_path, asset_path: asset_path)
+      result = transporter.upload(package_path: package_path, asset_path: asset_path, platform: platform)
 
       unless result
         transporter_errors = transporter.displayable_errors
@@ -391,13 +391,13 @@ def transporter_for_selected_team(options)
       api_token = Spaceship::ConnectAPI.token
       unless api_token.nil?
         api_token.refresh! if api_token.expired?
-        return FastlaneCore::ItunesTransporter.new(nil, nil, false, nil, api_token.text)
+        return FastlaneCore::ItunesTransporter.new(nil, nil, false, nil, api_token.text, upload: true, api_key: options[:api_key])
       end
 
       # Otherwise use username and password
       tunes_client = Spaceship::ConnectAPI.client ? Spaceship::ConnectAPI.client.tunes_client : nil
 
-      generic_transporter = FastlaneCore::ItunesTransporter.new(options[:username], nil, false, options[:itc_provider])
+      generic_transporter = FastlaneCore::ItunesTransporter.new(options[:username], nil, false, options[:itc_provider], upload: true, api_key: options[:api_key])
       return generic_transporter if options[:itc_provider] || tunes_client.nil?
       return generic_transporter unless tunes_client.teams.count > 1
 
@@ -406,7 +406,7 @@ def transporter_for_selected_team(options)
         name = team['name']
         provider_id = generic_transporter.provider_ids[name]
         UI.verbose("Inferred provider id #{provider_id} for team #{name}.")
-        return FastlaneCore::ItunesTransporter.new(options[:username], nil, false, provider_id)
+        return FastlaneCore::ItunesTransporter.new(options[:username], nil, false, provider_id, upload: true, api_key: options[:api_key])
       rescue => ex
         STDERR.puts(ex.to_s)
         UI.verbose("Couldn't infer a provider short name for team with id #{tunes_client.team_id} automatically: #{ex}. Proceeding without provider short name.")