Commit
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -153,6 +153,8 @@ def ❤ | |
end | ||
end | ||
|
||
rtt = check_rtt | ||
|
||
fails = procd = 0 | ||
kb = memory_usage(::Process.pid) | ||
|
||
|
@@ -163,6 +165,7 @@ def ❤ | |
conn.hmset(key, "info", to_json, | ||
"busy", curstate.size, | ||
"beat", Time.now.to_f, | ||
"rtt_us", rtt, | ||
"quiet", @done, | ||
"rss", kb) | ||
conn.expire(key, 60) | ||
|
@@ -185,6 +188,29 @@ def ❤ | |
end | ||
end | ||
|
||
RTT_WARNING_LEVEL = 50_000 | ||
|
||
def check_rtt | ||
a = b = 0 | ||
Sidekiq.redis do |x| | ||
This comment has been minimized.
Sorry, something went wrong.
This comment has been minimized.
Sorry, something went wrong.
mperham
Author
Collaborator
|
||
a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond) | ||
x.ping | ||
b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond) | ||
end | ||
rtt = b - a | ||
# Ideal RTT for Redis is < 1000µs | ||
# Workable is < 10,000µs | ||
# Log a warning if it's a disaster. | ||
if rtt > RTT_WARNING_LEVEL | ||
Sidekiq.logger.warn <<-EOM | ||
Your Redis network connection is performing extremely poorly. | ||
Current RTT is #{rtt} µs, ideally this should be < 1000. | ||
Ensure Redis is running in the same AZ or datacenter as Sidekiq. | ||
EOM | ||
end | ||
rtt | ||
end | ||
|
||
MEMORY_GRABBER = case RUBY_PLATFORM | ||
when /linux/ | ||
->(pid) { | ||
|
@mperham Because the
ping
releases the GVL and other Sidekiq threads can do work in the background between lines 196 and 198, a fully-utilized Sidekiq server will constantly complain about its Redis RTT. I think this is misleading and could cause people to spend a lot of time debugging their Redis install when it's really just the case that their Sidekiq threads are completely loaded.This can be reproduced by any program that causes a Sidekiq process' CPU utilization to hit ~100%.