Skip to content

Commit

Permalink
merge master
Browse files Browse the repository at this point in the history
  • Loading branch information
mperham committed Mar 1, 2019
1 parent db64467 commit 653c58f
Show file tree
Hide file tree
Showing 35 changed files with 2,409 additions and 2,328 deletions.
6 changes: 6 additions & 0 deletions Changes.md
Expand Up @@ -23,6 +23,12 @@ get the old behavior. [#3968]
arguments to Sidekiq. Use a proper process supervisor (e.g. systemd or
foreman) to manage Sidekiq.

HEAD
---------

- Better handling of malformed job arguments in payload [#4095]
- add back in bootstap's dropdown css component [#4099, urkle]

5.2.5
---------

Expand Down
8 changes: 7 additions & 1 deletion Ent-Changes.md
Expand Up @@ -7,14 +7,20 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t
HEAD
-------------

- Fix excessive lock reclaims with concurrent limiter [#4105]
- Add ES translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.

1.8.0
-------------

- Require Sidekiq Pro 4.0 and Sidekiq 5.2.
- Refactor historical metrics API to use revamped Statsd support in Sidekiq Pro
- Add a gauge to historical metrics for `default` queue latency [#4079]

1.7.2
-------------

- Add PT and JA translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.
- Add PT and JA translations
- Fix elapsed time calculations to use monotonic clock [#4000, sj26]
- Fix edge case where flapping leadership would cause old periodic
jobs to be fired once [#3974]
Expand Down
7 changes: 6 additions & 1 deletion Pro-Changes.md
Expand Up @@ -7,10 +7,15 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t
HEAD
---------

- Add ES translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.

4.0.5
---------

- Increase super\_fetch retriever thread count from 1 to 2 to make it
less sensitive to Redis latency.
- Better handling of invalid job JSON by reliable scheduler [#4053]
- Added ZH, PT, JA and RU translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.
- Added ZH, PT, JA and RU translations.

4.0.4
---------
Expand Down
150 changes: 4 additions & 146 deletions bin/sidekiqctl
Expand Up @@ -2,152 +2,10 @@

require 'fileutils'
require 'sidekiq/api'

class Sidekiqctl
CMD = File.basename($0)

def self.print_usage
puts "#{CMD} - control Sidekiq from the command line."
puts
puts "Usage: #{CMD} status <section>"
puts
puts " <section> (optional) view a specific section of the status output"
puts " Valid sections are: #{Sidekiqctl::Status::VALID_SECTIONS.join(', ')}"
puts
end

class Status
VALID_SECTIONS = %w[all version overview processes queues]
def display(section = nil)
section ||= 'all'
unless VALID_SECTIONS.include? section
puts "I don't know how to check the status of '#{section}'!"
puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
return
end
send(section)
rescue StandardError => e
puts "Couldn't get status: #{e}"
end

def all
version
puts
overview
puts
processes
puts
queues
end

def version
puts "Sidekiq #{Sidekiq::VERSION}"
puts Time.now
end

def overview
puts '---- Overview ----'
puts " Processed: #{delimit stats.processed}"
puts " Failed: #{delimit stats.failed}"
puts " Busy: #{delimit stats.workers_size}"
puts " Enqueued: #{delimit stats.enqueued}"
puts " Retries: #{delimit stats.retry_size}"
puts " Scheduled: #{delimit stats.scheduled_size}"
puts " Dead: #{delimit stats.dead_size}"
end

def processes
puts "---- Processes (#{process_set.size}) ----"
process_set.each_with_index do |process, index|
puts "#{process['identity']} #{tags_for(process)}"
puts " Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
puts " Threads: #{process['concurrency']} (#{process['busy']} busy)"
puts " Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
puts '' unless (index+1) == process_set.size
end
end

COL_PAD = 2
def queues
puts "---- Queues (#{queue_data.size}) ----"
columns = {
name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
}
columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
puts
queue_data.each do |q|
columns.each do |col, (dir, width)|
print q.send(col).public_send(dir, width)
end
puts
end
end

private

def delimit(number)
number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
end

def split_multiline(values, opts = {})
return 'none' unless values
pad = opts[:pad] || 0
max_length = opts[:max_length] || (80 - pad)
out = []
line = ''
values.each do |value|
if (line.length + value.length) > max_length
out << line
line = ' ' * pad
end
line << value + ', '
end
out << line[0..-3]
out.join("\n")
end

def tags_for(process)
tags = [
process['tag'],
process['labels'],
(process['quiet'] == 'true' ? 'quiet' : nil)
].flatten.compact
tags.any? ? "[#{tags.join('] [')}]" : nil
end

def time_ago(timestamp)
seconds = Time.now - Time.at(timestamp)
return 'just now' if seconds < 60
return 'a minute ago' if seconds < 120
return "#{seconds.floor / 60} minutes ago" if seconds < 3600
return 'an hour ago' if seconds < 7200
"#{seconds.floor / 60 / 60} hours ago"
end

QUEUE_STRUCT = Struct.new(:name, :size, :latency)
def queue_data
@queue_data ||= Sidekiq::Queue.all.map do |q|
QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
end
end

def process_set
@process_set ||= Sidekiq::ProcessSet.new
end

def stats
@stats ||= Sidekiq::Stats.new
end
end
end
require 'sidekiq/ctl'

if ARGV[0] == 'status'
Sidekiqctl::Status.new.display(ARGV[1])
exit
end

if ARGV.length < 2
Sidekiqctl.print_usage
Sidekiq::Ctl::Status.new.display(ARGV[1])
else
Sidekiq::Ctl.print_usage
end
2 changes: 1 addition & 1 deletion lib/sidekiq/client.rb
Expand Up @@ -198,7 +198,7 @@ def atomic_push(conn, payloads)
q = payloads.first['queue']
now = Time.now.to_f
to_push = payloads.map do |entry|
entry['enqueued_at'] = now
entry['enqueued_at'] ||= now
Sidekiq.dump_json(entry)
end
conn.sadd('queues', q)
Expand Down
147 changes: 147 additions & 0 deletions lib/sidekiq/ctl.rb
@@ -0,0 +1,147 @@
#!/usr/bin/env ruby

require 'fileutils'
require 'sidekiq/api'

class Sidekiq::Ctl
CMD = File.basename($0)

attr_reader :stage

def self.print_usage
puts "#{CMD} - control Sidekiq from the command line."
puts
puts "Usage: #{CMD} status <section>"
puts
puts " <section> (optional) view a specific section of the status output"
puts " Valid sections are: #{Sidekiqctl::Status::VALID_SECTIONS.join(', ')}"
puts
end

class Status
VALID_SECTIONS = %w[all version overview processes queues]
COL_PAD = 2

def display(section = nil)
section ||= 'all'
unless VALID_SECTIONS.include? section
puts "I don't know how to check the status of '#{section}'!"
puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
return
end
send(section)
rescue StandardError => e
puts "Couldn't get status: #{e}"
end

def all
version
puts
overview
puts
processes
puts
queues
end

def version
puts "Sidekiq #{Sidekiq::VERSION}"
puts Time.now
end

def overview
puts '---- Overview ----'
puts " Processed: #{delimit stats.processed}"
puts " Failed: #{delimit stats.failed}"
puts " Busy: #{delimit stats.workers_size}"
puts " Enqueued: #{delimit stats.enqueued}"
puts " Retries: #{delimit stats.retry_size}"
puts " Scheduled: #{delimit stats.scheduled_size}"
puts " Dead: #{delimit stats.dead_size}"
end

def processes
puts "---- Processes (#{process_set.size}) ----"
process_set.each_with_index do |process, index|
puts "#{process['identity']} #{tags_for(process)}"
puts " Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
puts " Threads: #{process['concurrency']} (#{process['busy']} busy)"
puts " Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
puts '' unless (index+1) == process_set.size
end
end

def queues
puts "---- Queues (#{queue_data.size}) ----"
columns = {
name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
}
columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
puts
queue_data.each do |q|
columns.each do |col, (dir, width)|
print q.send(col).public_send(dir, width)
end
puts
end
end

private

def delimit(number)
number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
end

def split_multiline(values, opts = {})
return 'none' unless values
pad = opts[:pad] || 0
max_length = opts[:max_length] || (80 - pad)
out = []
line = ''
values.each do |value|
if (line.length + value.length) > max_length
out << line
line = ' ' * pad
end
line << value + ', '
end
out << line[0..-3]
out.join("\n")
end

def tags_for(process)
tags = [
process['tag'],
process['labels'],
(process['quiet'] == 'true' ? 'quiet' : nil)
].flatten.compact
tags.any? ? "[#{tags.join('] [')}]" : nil
end

def time_ago(timestamp)
seconds = Time.now - Time.at(timestamp)
return 'just now' if seconds < 60
return 'a minute ago' if seconds < 120
return "#{seconds.floor / 60} minutes ago" if seconds < 3600
return 'an hour ago' if seconds < 7200
"#{seconds.floor / 60 / 60} hours ago"
end

QUEUE_STRUCT = Struct.new(:name, :size, :latency)
def queue_data
@queue_data ||= Sidekiq::Queue.all.map do |q|
QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
end
end

def process_set
@process_set ||= Sidekiq::ProcessSet.new
end

def stats
@stats ||= Sidekiq::Stats.new
end
end
end
13 changes: 10 additions & 3 deletions lib/sidekiq/web/helpers.rb
Expand Up @@ -207,9 +207,16 @@ def truncate(text, truncate_after_chars = 2000)
end

def display_args(args, truncate_after_chars = 2000)
args.map do |arg|
h(truncate(to_display(arg), truncate_after_chars))
end.join(", ")
return "Invalid job payload, args is nil" if args == nil
return "Invalid job payload, args must be an Array, not #{args.class.name}" if !args.is_a?(Array)

begin
args.map do |arg|
h(truncate(to_display(arg), truncate_after_chars))
end.join(", ")
rescue
"Illegal job arguments: #{h args.inspect}"
end
end

def csrf_tag
Expand Down

0 comments on commit 653c58f

Please sign in to comment.