Update test_integration files per PR #1956 (#1965)

* Update test_integration files per PR #1956

test_integration_cluster.rb

Request handling during server TERM - two tests

`#test_term_closes_listeners_tcp`
`#test_term_closes_listeners_unix`

using `#term_closes_listeners`

Send requests 10 per second.  Send 10, then :TERM server, then send another 30.
No more than 10 should throw Errno::ECONNRESET.

Request handling during phased restart - two tests

`#test_usr1_all_respond_tcp`
`#test_usr1_all_respond_unix`

using `#usr1_all_respond`

Send requests 1 per second.  Send 1, then :USR1 server, then send another 24.
All should be responded to, and at least three workers should be used

Stuck worker tests - two tests

`#test_stuck_external_term_spawn`
Tests whether externally TERM'd 'stuck' workers are proper re-spawned.

`#test_stuck_phased_restart`
Tests whether 'stuck' workers are properly shutdown during phased-restart.

helper files/methods changes

1. helper file changes to allow binding to TCP or UNIX, see kwarg unix:
2. Skip on Windows for signal TERM

* Misc updates, debug output, cleanup

* Add comments

* fix test_int_signal_with_background_thread_in_jruby per review

* TestIntegrationCluster#term_closes_listeners - add interleaved assert

* cluster.rb - remove duplicate Worker#term? method

lib/puma/cluster.rb

@@ -92,10 +92,6 @@ def term?
         @term
       end
 
-      def term?
-        @term
-      end
-
       def ping!(status)
         @last_checkin = Time.now
         @last_status = status

test/config/worker_shutdown_timeout_2.rb

@@ -0,0 +1 @@
+worker_shutdown_timeout 2

test/helper.rb

@@ -21,6 +21,9 @@
 $LOAD_PATH << File.expand_path("../../lib", __FILE__)
 Thread.abort_on_exception = true
 
+$debugging_info = ''.dup
+$debugging_hold = false    # needed for TestCLI#test_control_clustered
+
 require "puma"
 require "puma/events"
 require "puma/detect"
@@ -84,10 +87,12 @@ module TestSkips
   UNIX_SKT_EXIST = Object.const_defined? :UNIXSocket
   UNIX_SKT_MSG = "UnixSockets aren't available on the #{RUBY_PLATFORM} platform"
 
+  SIGNAL_LIST = Signal.list.keys.map(&:to_sym) - (Puma.windows? ? [:INT, :TERM] : [])
+
   # usage: skip_unless_signal_exist? :USR2
   def skip_unless_signal_exist?(sig, bt: caller)
-    signal = sig.to_s
-    unless Signal.list.key? signal
+    signal = sig.to_s.sub(/\ASIG/, '').to_sym
+    unless SIGNAL_LIST.include? signal
       skip "Signal #{signal} isn't available on the #{RUBY_PLATFORM} platform", bt
     end
   end
@@ -130,4 +135,19 @@ def self.run(reporter, options = {}) # :nodoc:
     prove_it!
     super
   end
+
+  def full_name
+    "#{self.class.name}##{name}"
+  end
+end
+
+Minitest.after_run do
+  # needed for TestCLI#test_control_clustered
+  unless $debugging_hold
+    out = $debugging_info.strip
+    unless out.empty?
+      puts "", " Debugging Info".rjust(75, '-'),
+        out, '-' * 75, ""
+    end
+  end
 end

test/helpers/integration.rb

@@ -15,60 +15,68 @@ class TestIntegration < Minitest::Test
 
   def setup
     @ios_to_close = []
+    @bind_path    = "test/#{name}_server.sock"
   end
 
   def teardown
     if defined?(@server) && @server
-      begin
-        Process.kill "INT", @server.pid
-      rescue
-        Errno::ESRCH
-      end
-      begin
-        Process.wait @server.pid
-      rescue Errno::ECHILD
-      end
-      @server.close unless @server.closed?
-      @server = nil
+      stop_server @pid, signal: :INT
     end
 
     @ios_to_close.each do |io|
       io.close if io.is_a?(IO) && !io.closed?
       io = nil
     end
+    refute File.exist?(@bind_path), "Bind path must be removed after stop"
+    File.unlink(@bind_path) rescue nil
+
+    # wait until the end for OS buffering?
+    if defined?(@server) && @server
+      @server.close unless @server.closed?
+      @server = nil
+    end
   end
 
   private
 
-  def cli_server(argv, bind = nil)
-    if bind
-      cmd = "#{BASE} bin/puma -b #{bind} #{argv}"
+  def cli_server(argv, unix: false)
+    if unix
+      cmd = "#{BASE} bin/puma -b unix://#{@bind_path} #{argv}"
     else
       @tcp_port = UniquePort.call
       cmd = "#{BASE} bin/puma -b tcp://#{HOST}:#{@tcp_port} #{argv}"
     end
     @server = IO.popen(cmd, "r")
     wait_for_server_to_boot
+    @pid = @server.pid
     @server
   end
 
-  def send_term_to_server(pid)
-    Process.kill(:TERM, pid)
-    sleep 1
-    Process.wait2(pid)
+  # rescue statements are just in case method is called with a server
+  # that is already stopped/killed, especially since Process.wait2 is
+  # blocking
+  def stop_server(pid = @pid, signal: :TERM)
+    begin
+      Process.kill signal, pid
+    rescue Errno::ESRCH
+    end
+    begin
+      Process.wait2 pid
+    rescue Errno::ECHILD
+    end
   end
 
   def restart_server_and_listen(argv)
-    cli_server(argv)
+    cli_server argv
     connection = connect
     initial_reply = read_body(connection)
-    restart_server(connection)
+    restart_server connection
     [initial_reply, read_body(connect)]
   end
 
   # reuses an existing connection to make sure that works
   def restart_server(connection)
-    Process.kill :USR2, @server.pid
+    Process.kill :USR2, @pid
     connection.write "GET / HTTP/1.1\r\n\r\n" # trigger it to start by sending a new request
     wait_for_server_to_boot
   end
@@ -77,8 +85,8 @@ def wait_for_server_to_boot
     true while @server.gets !~ /Ctrl-C/ # wait for server to say it booted
   end
 
-  def connect(path = nil)
-    s = TCPSocket.new HOST, @tcp_port
+  def connect(path = nil, unix: false)
+    s = unix ? UNIXSocket.new(@bind_path) : TCPSocket.new(HOST, @tcp_port)
     @ios_to_close << s
     s << "GET /#{path} HTTP/1.1\r\n\r\n"
     true until s.gets == "\r\n"

test/rackup/sleep_pid.ru

@@ -0,0 +1,8 @@
+# call with "GET /sleep<d> HTTP/1.1\r\n\r\n", where <d> is the number of
+# seconds to sleep, returns process pid
+
+run lambda { |env|
+  dly = (env['REQUEST_PATH'][/\/sleep(\d+)/,1] || '0').to_i
+  sleep dly
+  [200, {"Content-Type" => "text/plain"}, ["Slept #{dly} #{Process.pid}"]]
+}

test/rackup/sleep_step.ru

@@ -0,0 +1,10 @@
+# call with "GET /sleep<d>-<s> HTTP/1.1\r\n\r\n", where <d> is the number of
+# seconds to sleep and <s> is the step
+
+run lambda { |env|
+  p = env['REQUEST_PATH']
+  dly = (p[/\/sleep(\d+)/,1] || '0').to_i
+  step = p[/(\d+)\z/,1].to_i
+  sleep dly
+  [200, {"Content-Type" => "text/plain"}, ["Slept #{dly} #{step}"]]
+}

test/test_cli.rb

@@ -81,13 +81,17 @@ def test_control_clustered
                          "--control-token", "",
                          "test/rackup/lobster.ru"], @events
 
+    # without this, Minitest.after_run will trigger on this test ?
+    $debugging_hold = true
+
     t = Thread.new { cli.run }
 
     wait_booted
 
     s = UNIXSocket.new @tmp_path
     s << "GET /stats HTTP/1.0\r\n\r\n"
     body = s.read
+    s.close
 
     require 'json'
     status = JSON.parse(body.split("\n").last)
@@ -99,10 +103,23 @@ def test_control_clustered
     s = UNIXSocket.new @tmp_path
     s << "GET /stats HTTP/1.0\r\n\r\n"
     body = s.read
-    assert_match(/\{ "started_at": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", "workers": 2, "phase": 0, "booted_workers": 2, "old_workers": 0, "worker_status": \[\{ "started_at": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", "pid": \d+, "index": 0, "phase": 0, "booted": true, "last_checkin": "[^"]+", "last_status": \{ "backlog":0, "running":2, "pool_capacity":2, "max_threads": 2 \} \},\{ "started_at": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", "pid": \d+, "index": 1, "phase": 0, "booted": true, "last_checkin": "[^"]+", "last_status": \{ "backlog":0, "running":2, "pool_capacity":2, "max_threads": 2 \} \}\] \}/, body.split("\r\n").last)
+    s.close
 
-    cli.launcher.stop
-    t.join
+    assert_match(/\{ "started_at": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", "workers": 2, "phase": 0, "booted_workers": 2, "old_workers": 0, "worker_status": \[\{ "started_at": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", "pid": \d+, "index": 0, "phase": 0, "booted": true, "last_checkin": "[^"]+", "last_status": \{ "backlog":0, "running":2, "pool_capacity":2, "max_threads": 2 \} \},\{ "started_at": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", "pid": \d+, "index": 1, "phase": 0, "booted": true, "last_checkin": "[^"]+", "last_status": \{ "backlog":0, "running":2, "pool_capacity":2, "max_threads": 2 \} \}\] \}/, body.split("\r\n").last)
+  ensure
+    if UNIX_SKT_EXIST && HAS_FORK
+      cli.launcher.stop
+
+      done = nil
+      until done
+        @events.stdout.rewind
+        log = @events.stdout.readlines.join ''
+        done = log[/ - Goodbye!/]
+      end
+
+      t.join
+      $debugging_hold = false
+    end
   end
 
   def test_control
@@ -121,11 +138,14 @@ def test_control
     s = UNIXSocket.new @tmp_path
     s << "GET /stats HTTP/1.0\r\n\r\n"
     body = s.read
+    s.close
 
     assert_match(/{ "started_at": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", "backlog": 0, "running": 0, "pool_capacity": 16, "max_threads": 16 }/, body.split("\r\n").last)
-
-    cli.launcher.stop
-    t.join
+  ensure
+    if UNIX_SKT_EXIST
+      cli.launcher.stop
+      t.join
+    end
   end
 
   def test_control_stop
@@ -144,10 +164,11 @@ def test_control_stop
     s = UNIXSocket.new @tmp_path
     s << "GET /stop HTTP/1.0\r\n\r\n"
     body = s.read
+    s.close
 
     assert_equal '{ "status": "ok" }', body.split("\r\n").last
-
-    t.join
+  ensure
+    t.join if UNIX_SKT_EXIST
   end
 
   def control_gc_stats(uri, cntl)