From 7b262bb83d07254acc8dd7e48f3056bc6a478a8d Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Thu, 29 Feb 2024 16:20:28 -0800 Subject: [PATCH] Add support for a new flavor of json serialization configuration, fixes #6208 --- Gemfile | 1 + lib/sidekiq.rb | 18 ++++++--- lib/sidekiq/api.rb | 2 +- lib/sidekiq/job_util.rb | 31 +-------------- lib/sidekiq/json.rb | 86 +++++++++++++++++++++++++++++++++++++++++ test/helper.rb | 5 +++ 6 files changed, 106 insertions(+), 37 deletions(-) create mode 100644 lib/sidekiq/json.rb diff --git a/Gemfile b/Gemfile index 7fde758b4b..6f9ca9966a 100644 --- a/Gemfile +++ b/Gemfile @@ -10,6 +10,7 @@ gem "activejob", RAILS_VERSION gem "activerecord", RAILS_VERSION gem "railties", RAILS_VERSION gem "redis-client" +gem "benchmark-ips" # gem "bumbler" # gem "debug" diff --git a/lib/sidekiq.rb b/lib/sidekiq.rb index c997b46e66..912c1fc0ea 100644 --- a/lib/sidekiq.rb +++ b/lib/sidekiq.rb @@ -27,6 +27,7 @@ rescue LoadError end +require "sidekiq/json" require "sidekiq/config" require "sidekiq/logger" require "sidekiq/client" @@ -35,8 +36,6 @@ require "sidekiq/worker_compatibility_alias" require "sidekiq/redis_client_adapter" -require "json" - module Sidekiq NAME = "Sidekiq" LICENSE = "See LICENSE and the LGPL-3.0 for licensing details." @@ -49,12 +48,19 @@ def self.server? defined?(Sidekiq::CLI) end - def self.load_json(string) - JSON.parse(string) + PARSE_OPTIONS = {} + GENERATE_OPTIONS = {} + def self.parse_json(string, options = PARSE_OPTIONS) + ::JSON.parse(string, options) end - def self.dump_json(object) - JSON.generate(object) + def self.generate_json(object, options = GENERATE_OPTIONS) + ::JSON.generate(object, options) + end + # backwards compatibility + class << self + alias_method :load_json, :parse_json + alias_method :dump_json, :generate_json end def self.pro? diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index 437f5261de..64217957d1 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -355,7 +355,7 @@ def initialize(item, queue_name = nil) # @api private def parse(item) Sidekiq.load_json(item) - rescue JSON::ParserError + rescue ::JSON::ParserError # If the job payload in Redis is invalid JSON, we'll load # the item as an empty hash and store the invalid JSON as # the job 'args' for display in the Web UI. diff --git a/lib/sidekiq/job_util.rb b/lib/sidekiq/job_util.rb index 30ed91f81e..910aae417f 100644 --- a/lib/sidekiq/job_util.rb +++ b/lib/sidekiq/job_util.rb @@ -71,37 +71,8 @@ def normalized_hash(item_class) private - RECURSIVE_JSON_UNSAFE = { - Integer => ->(val) {}, - Float => ->(val) {}, - TrueClass => ->(val) {}, - FalseClass => ->(val) {}, - NilClass => ->(val) {}, - String => ->(val) {}, - Array => ->(val) { - val.each do |e| - unsafe_item = RECURSIVE_JSON_UNSAFE[e.class].call(e) - return unsafe_item unless unsafe_item.nil? - end - nil - }, - Hash => ->(val) { - val.each do |k, v| - return k unless String === k - - unsafe_item = RECURSIVE_JSON_UNSAFE[v.class].call(v) - return unsafe_item unless unsafe_item.nil? - end - nil - } - } - - RECURSIVE_JSON_UNSAFE.default = ->(val) { val } - RECURSIVE_JSON_UNSAFE.compare_by_identity - private_constant :RECURSIVE_JSON_UNSAFE - def json_unsafe?(item) - RECURSIVE_JSON_UNSAFE[item.class].call(item) + Sidekiq::JSON::RULES[item.class].call(item) end end end diff --git a/lib/sidekiq/json.rb b/lib/sidekiq/json.rb new file mode 100644 index 0000000000..e0580f0eea --- /dev/null +++ b/lib/sidekiq/json.rb @@ -0,0 +1,86 @@ +# Sidekiq does not add a serialization step to job processing. +# All job serialization is expected to work with `JSON.parse/generate` +# but since the `json` gem does support optional extensions for core +# Ruby types, we can enable those extensions for the user in order +# to make transition from `perform_async(args)` -> `perform(args)` +# a little smoother. +# +# !!!!!!!!!!!!!!!!!! PLEASE NOTE !!!!!!!!!!!!!!!!!!! +# +# Symbols are not legal keys in JSON hashes so there's still +# effectively no way to support Symbols as Hash keys without a much +# more complex serialization step like ActiveJob implements. +# +# Good, supported types: +# perform_async(:foo, [:foo, 123], { "mike" => :foo }) +# +# Bad, unsupported: +# perform_async(foo: 1, { :foo => 123 }) +# +# Clean, easy serialization of Symbol'd keys remains an unsolved problem. +# + +require "json" + +module Sidekiq + module JSON + RULES = { + Integer => ->(val) {}, + Float => ->(val) {}, + TrueClass => ->(val) {}, + FalseClass => ->(val) {}, + NilClass => ->(val) {}, + String => ->(val) {}, + Array => ->(val) { + val.each do |e| + unsafe_item = RULES[e.class].call(e) + return unsafe_item unless unsafe_item.nil? + end + nil + }, + Hash => ->(val) { + val.each do |k, v| + return k unless String === k + + unsafe_item = RULES[v.class].call(v) + return unsafe_item unless unsafe_item.nil? + end + nil + } + } + + RULES.default = ->(val) { val } + RULES.compare_by_identity + + DEFAULT_VERSION = :v7 + CURRENT_VERSION = DEFAULT_VERSION + + # Activate the given JSON flavor globally. + def self.flavor!(ver = DEFAULT_VERSION) + return ver if ver == CURRENT_VERSION + raise ArgumentError, "Once set, Sidekiq's JSON flavor cannot be changed" if DEFAULT_VERSION != CURRENT_VERSION + raise ArgumentError, "Unknown JSON flavor `#{ver}`" unless ver == :v7 || ver == :v8 + + if ver == :v8 + # this cannot be reverted; once v8 is activated in a process + # you cannot go back to v7. + require "json/add/core" + require "json/add/complex" + require "json/add/set" + require "json/add/rational" + require "json/add/bigdecimal" + Sidekiq::GENERATE_OPTIONS[:create_additions] = true + Sidekiq::PARSE_OPTIONS[:create_additions] = true + # Mark the core types as safe + [::Date, ::DateTime, ::Exception, ::Range, ::Regexp, + ::Struct, ::Symbol, ::Time, ::Complex, ::Set, + ::Rational, ::BigDecimal].each do |klass| + RULES[klass] = ->(_) {} + end + end + + remove_const(:CURRENT_VERSION) + const_set(:CURRENT_VERSION, ver) + end + end +end diff --git a/test/helper.rb b/test/helper.rb index b841b22931..a84f3ef05b 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -57,6 +57,11 @@ def capture_logging(cfg, lvl = Logger::INFO) end end +def global_change(&block) + pid = fork(&block) + Process.wait(pid) if pid +end + Signal.trap("TTIN") do Thread.list.each do |thread| puts "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"