Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add worker_check_interval configuration option #2759

Merged
merged 1 commit into from Dec 11, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/puma/cluster.rb
Expand Up @@ -135,7 +135,7 @@ def all_workers_booted?
def check_workers
return if @next_check >= Time.now

@next_check = Time.now + Const::WORKER_CHECK_INTERVAL
@next_check = Time.now + @options[:worker_check_interval]

timeout_workers
wait_workers
Expand Down
2 changes: 1 addition & 1 deletion lib/puma/cluster/worker.rb
Expand Up @@ -130,7 +130,7 @@ def run
Puma::Util.purge_interrupt_queue
break
end
sleep Const::WORKER_CHECK_INTERVAL
sleep @options[:worker_check_interval]
end
end
server_thread.join
Expand Down
2 changes: 2 additions & 0 deletions lib/puma/configuration.rb
Expand Up @@ -11,6 +11,7 @@ module ConfigDefault

DefaultTCPHost = "0.0.0.0"
DefaultTCPPort = 9292
DefaultWorkerCheckInterval = 5
DefaultWorkerTimeout = 60
DefaultWorkerShutdownTimeout = 30
end
Expand Down Expand Up @@ -195,6 +196,7 @@ def puma_default_options
:workers => Integer(ENV['WEB_CONCURRENCY'] || 0),
:silence_single_worker_warning => false,
:mode => :http,
:worker_check_interval => DefaultWorkerCheckInterval,
:worker_timeout => DefaultWorkerTimeout,
:worker_boot_timeout => DefaultWorkerTimeout,
:worker_shutdown_timeout => DefaultWorkerShutdownTimeout,
Expand Down
3 changes: 0 additions & 3 deletions lib/puma/const.rb
Expand Up @@ -235,9 +235,6 @@ module Const

EARLY_HINTS = "rack.early_hints".freeze

# Minimum interval to checks worker health
WORKER_CHECK_INTERVAL = 5

# Illegal character in the key or value of response header
DQUOTE = "\"".freeze
HTTP_HEADER_DELIMITER = Regexp.escape("(),/:;<=>?@[]{}\\").freeze
Expand Down
15 changes: 14 additions & 1 deletion lib/puma/dsl.rb
Expand Up @@ -736,6 +736,19 @@ def tag(string)
@options[:tag] = string.to_s
end

# Change the default interval for checking workers.
#
# The default value is 5 seconds.
#
# @note Cluster mode only.
# @example
# worker_check_interval 5
# @see Puma::Cluster#check_workers
#
def worker_check_interval(interval)
@options[:worker_check_interval] = Integer(interval)
end

# Verifies that all workers have checked in to the master process within
# the given timeout. If not the worker process will be restarted. This is
# not a request timeout, it is to protect against a hung or dead process.
Expand All @@ -750,7 +763,7 @@ def tag(string)
#
def worker_timeout(timeout)
timeout = Integer(timeout)
min = Const::WORKER_CHECK_INTERVAL
min = @options.fetch(:worker_check_interval, Puma::ConfigDefault::DefaultWorkerCheckInterval)

if timeout <= min
raise "The minimum worker_timeout must be greater than the worker reporting interval (#{min})"
Expand Down
6 changes: 6 additions & 0 deletions test/helpers/integration.rb
@@ -1,6 +1,7 @@
# frozen_string_literal: true

require "puma/control_cli"
require "json"
require "open3"
require_relative 'tmp_path'

Expand Down Expand Up @@ -205,6 +206,11 @@ def cli_pumactl(argv, unix: false)
r
end

def get_stats
read_pipe = cli_pumactl "stats"
JSON.parse(read_pipe.readlines.last)
end

def hot_restart_does_not_drop_connections(num_threads: 1, total_requests: 500)
skipped = true
skip_if :jruby, suffix: <<-MSG
Expand Down
19 changes: 18 additions & 1 deletion test/test_integration_cluster.rb
@@ -1,6 +1,8 @@
require_relative "helper"
require_relative "helpers/integration"

require "time"

class TestIntegrationCluster < TestIntegration
parallelize_me! if ::Puma.mri?

Expand Down Expand Up @@ -146,14 +148,29 @@ def test_stuck_phased_restart
worker_respawn { |phase0_worker_pids| Process.kill :USR1, @pid }
end

def test_worker_check_interval
@control_tcp_port = UniquePort.call
worker_check_interval = 1

cli_server "-w 1 -t 1:1 --control-url tcp://#{HOST}:#{@control_tcp_port} --control-token #{TOKEN} test/rackup/hello.ru", config: "worker_check_interval #{worker_check_interval}"

sleep worker_check_interval + 1
last_checkin_1 = Time.parse(get_stats["worker_status"].first["last_checkin"])

sleep worker_check_interval + 1
last_checkin_2 = Time.parse(get_stats["worker_status"].first["last_checkin"])

assert(last_checkin_2 > last_checkin_1)
end

def test_worker_boot_timeout
timeout = 1
worker_timeout(timeout, 2, "worker failed to boot within \\\d+ seconds", "worker_boot_timeout #{timeout}; on_worker_boot { sleep #{timeout + 1} }")
end

def test_worker_timeout
skip 'Thread#name not available' unless Thread.current.respond_to?(:name)
timeout = Puma::Const::WORKER_CHECK_INTERVAL + 1
timeout = Puma::ConfigDefault::DefaultWorkerCheckInterval + 1
worker_timeout(timeout, 1, "worker failed to check in within \\\d+ seconds", <<RUBY)
worker_timeout #{timeout}
on_worker_boot do
Expand Down
2 changes: 1 addition & 1 deletion test/test_launcher.rb
Expand Up @@ -148,7 +148,7 @@ def test_puma_stats_clustered
end
launcher = launcher(conf)
Thread.new do
sleep Puma::Const::WORKER_CHECK_INTERVAL + 1
sleep Puma::ConfigDefault::DefaultWorkerCheckInterval + 1
status = Puma.stats_hash[:worker_status].first[:last_status]
Puma::Server::STAT_METHODS.each do |stat|
assert_includes status, stat
Expand Down