Adds two tests for worker SIGTERM/respawn and phased-restart

test_worker_spawn_external_term - sends SIGTERM to workers, checks respawn, etc

test_worker_phased_restart - checking worker handling during phased-restart

test/test_integration.rb

@@ -13,8 +13,8 @@ class TestIntegration < Minitest::Test
 
   def setup
     unique = UniquePort.call
-    @state_path = "test/test_#{unique}_puma.state"
-    @bind_path = "test/test_#{unique}_server.sock"
+    @state_path   = "test/test_#{unique}_puma.state"
+    @bind_path    = "test/test_#{unique}_server.sock"
     @control_path = "test/test_#{unique}_control.sock"
     @token = "xxyyzz"
 
@@ -27,8 +27,8 @@ def setup
   end
 
   def teardown
-    File.unlink @state_path rescue nil
-    File.unlink @bind_path rescue nil
+    File.unlink @state_path   rescue nil
+    File.unlink @bind_path    rescue nil
     File.unlink @control_path rescue nil
 
     @wait.close
@@ -389,7 +389,207 @@ def test_no_zombie_children
     worker_pids.each { |pid| Process.kill :TERM, pid }
     sleep 2
 
+    # should be empty if all old workers removed
+    waited_pids = worker_pids.map { |pid|
+      begin
+        Process.wait pid, Process::WNOHANG
+        pid
+      rescue Errno::ECHILD
+        nil # child is already reaped/removed
+      end
+    }.compact
+
     # Should return nil if Puma has correctly cleaned up
-    assert_nil Process.waitpid(-1, Process::WNOHANG)
+    assert_empty waited_pids
+  end
+
+  def test_worker_spawn_external_term
+    skip NO_FORK_MSG unless HAS_FORK
+
+    host = '127.0.0.1'
+    port = UniquePort.call
+    workers_booted = 0
+
+    conf = Puma::Configuration.new do |c|
+      c.bind "tcp://#{host}:#{port}"
+      c.threads 1, 1
+      c.workers 2
+      c.worker_shutdown_timeout 2
+      c.app TestApps::SLEEP
+      c.after_worker_fork { |idx| workers_booted += 1 }
+    end
+
+    l = Puma::Launcher.new conf, :events => @events
+
+    t = Thread.new do
+      Thread.current.abort_on_exception = true
+      l.run
+    end
+
+    wait_booted
+
+    # make sure two workers have booted
+    time = 0
+    until workers_booted >= 2 || time >= 10
+      sleep 2
+      time += 2
+    end
+
+    cluster = l.instance_variable_get(:@runner)
+
+    http0 = Net::HTTP.new host, port
+    http1 = Net::HTTP.new host, port
+
+    worker0 = Thread.new do
+      begin
+        req0 = Net::HTTP::Get.new '/sleep35'
+        http0.start.request(req0) { |rep0| rep0.body }
+      rescue
+      end
+    end
+
+    worker1 = Thread.new do
+      begin
+        req1 = Net::HTTP::Get.new '/sleep40'
+        http1.start.request(req1) { |rep1| rep1.body }
+      rescue
+      end
+    end
+
+    old_pids = cluster.instance_variable_get(:@workers).map(&:pid)
+
+    start_time = Time.now.to_f
+    old_pids.each { |p| Process.kill :TERM, p }
+
+    # make sure four workers have booted
+    time = 0
+    until workers_booted >= 4 || time >= 30
+      sleep 2
+      time += 2
+    end
+
+    new_pids = cluster.instance_variable_get(:@workers).map(&:pid)
+
+    # should be empty if all old workers removed
+    old_waited = old_pids.map { |pid|
+      begin
+        Process.wait(pid, Process::WNOHANG)
+        pid
+      rescue Errno::ECHILD
+        nil # child is already terminated
+      end
+    }.compact
+
+    Thread.kill worker0
+    Thread.kill worker1
+    http0   = nil
+    http1   = nil
+    cluster = nil
+
+    l.stop
+    assert_kind_of Thread, t.join, "server didn't stop"
+
+    assert_operator (Time.now.to_f - start_time).round(2), :<, 32
+
+    msg = "old_pids #{old_pids.inspect}  new_pids #{new_pids.inspect}  old_waited #{old_waited.inspect}"
+    assert_equal 2, new_pids.length, msg
+    assert_equal 2, old_pids.length, msg
+    assert_equal new_pids, (new_pids - old_pids), "#{msg}\nBoth workers should be replaced"
+    assert_empty old_waited, msg
+  end
+
+  def test_worker_phased_restart
+    skip NO_FORK_MSG unless HAS_FORK
+
+    host = '127.0.0.1'
+    port = UniquePort.call
+    workers_booted = 0
+
+    conf = Puma::Configuration.new do |c|
+      c.bind "tcp://#{host}:#{port}"
+      c.threads 1, 1
+      c.workers 2
+      c.worker_shutdown_timeout 2
+      c.app TestApps::SLEEP
+      c.after_worker_fork { |idx| workers_booted += 1 }
+    end
+
+    l = Puma::Launcher.new conf, :events => @events
+
+    t = Thread.new do
+      Thread.current.abort_on_exception = true
+      l.run
+    end
+
+    wait_booted
+
+    # make sure two workers have booted
+    time = 0
+    until workers_booted >= 2 || time >= 10
+      sleep 2
+      time += 2
+    end
+
+    cluster = l.instance_variable_get :@runner
+
+    http0 = Net::HTTP.new host, port
+    http1 = Net::HTTP.new host, port
+
+    worker0 = Thread.new do
+      begin
+        req0 = Net::HTTP::Get.new "/sleep35", {}
+        http0.start.request(req0) { |rep0| rep0.body }
+      rescue
+      end
+    end
+
+    worker1 = Thread.new do
+      begin
+        req1 = Net::HTTP::Get.new "/sleep40", {}
+        http1.start.request(req1) { |rep1| rep1.body }
+      rescue
+      end
+    end
+
+    old_pids = cluster.instance_variable_get(:@workers).map(&:pid)
+
+    start_time = Time.now.to_f
+    l.phased_restart
+
+    # make sure four workers have booted
+    time = 0
+    until workers_booted >= 4 || time >= 30
+      sleep 2
+      time += 2
+    end
+
+    new_pids = cluster.instance_variable_get(:@workers).map(&:pid)
+
+    # should be empty if all old workers removed
+    old_waited = old_pids.map { |pid|
+      begin
+        Process.wait(pid, Process::WNOHANG)
+        pid
+      rescue Errno::ECHILD
+        nil # child is already terminated
+      end
+    }.compact
+
+    Thread.kill worker0
+    Thread.kill worker1
+    http0   = nil
+    http1   = nil
+    cluster = nil
+
+    l.stop
+    assert_kind_of Thread, t.join, "server didn't stop"
+
+    assert_operator (Time.now.to_f - start_time).round(2), :<, 32
+
+    msg = "old_pids #{old_pids.inspect}  new_pids #{new_pids.inspect}  old_waited #{old_waited.inspect}"
+    assert_equal 2, new_pids.length, msg
+    assert_equal 2, old_pids.length, msg
+    assert_equal new_pids, (new_pids - old_pids), "#{msg}\nBoth workers should be replaced"
+    assert_empty old_waited, msg
   end
 end