From f255d4e91b3d7af2f6d9968c6a8b349ea92b17c6 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 12:47:26 +0200 Subject: [PATCH 01/17] Update ruby versions on travis --- .travis.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e88d12857..fc880915a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,22 +25,21 @@ script: after_script: - if [[ "${COV}" = "true" ]]; then ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT; fi; - rvm: - - 2.6.1 + - 2.6.2 - 2.4.5 - 2.3.8 - - jruby-9.2.5.0 + - jruby-9.2.7.0 matrix: fast_finish: true include: - - rvm: 2.5.3 + - rvm: 2.5.5 gemfile: gemfiles/sidekiq_develop.gemfile env: COV=true - - rvm: 2.6.0 + - rvm: 2.6.2 gemfile: gemfiles/sidekiq_6.0.gemfile - - rvm: 2.5.3 + - rvm: 2.5.5 gemfile: gemfiles/sidekiq_6.0.gemfile gemfile: From 2a32028b886fa81c11b66c4ca9500bd7454a23e1 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 12:47:42 +0200 Subject: [PATCH 02/17] Beautify documentation --- lib/sidekiq_unique_jobs/locksmith.rb | 29 +++++++++++++++++++--- lib/sidekiq_unique_jobs/scripts.rb | 37 +++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/lib/sidekiq_unique_jobs/locksmith.rb b/lib/sidekiq_unique_jobs/locksmith.rb index e104e42c5..1267f1c39 100644 --- a/lib/sidekiq_unique_jobs/locksmith.rb +++ b/lib/sidekiq_unique_jobs/locksmith.rb @@ -23,7 +23,10 @@ def initialize(item, redis_pool = nil) @redis_pool = redis_pool end + # # Deletes the lock unless it has a ttl set + # + # def delete return if ttl @@ -39,11 +42,14 @@ def delete! ) end + # # Create a lock for the item + # # @param [Integer] timeout the number of seconds to wait for a lock. - # nil means wait indefinitely - # @yield the block to execute if a lock is successful - # @return the Sidekiq job_id (jid) + # + # @return [String] the Sidekiq job_id (jid) + # + # def lock(timeout = nil, &block) Scripts.call(:lock, redis_pool, keys: [exists_key, grabbed_key, available_key, UNIQUE_SET, unique_digest], @@ -56,9 +62,12 @@ def lock(timeout = nil, &block) end alias wait lock + # # Removes the lock keys from Redis if locked by the provided jid/token + # # @return [false] unless locked? # @return [String] Sidekiq job_id (jid) if successful + # def unlock(token = nil) token ||= jid return false unless locked?(token) @@ -66,9 +75,14 @@ def unlock(token = nil) unlock!(token) end + # # Removes the lock keys from Redis + # + # @param [String] token the token to unlock (defaults to jid) + # # @return [false] unless locked? # @return [String] Sidekiq job_id (jid) if successful + # def unlock!(token = nil) token ||= jid @@ -80,10 +94,17 @@ def unlock!(token = nil) ) end - # Checks if this instance is considered locked + # # @param [String] token the unique token to check for a lock. # nil will default to the jid provided in the initializer # @return [true, false] + # + # Checks if this instance is considered locked + # + # @param [] token + # + # @return [] + # def locked?(token = nil) token ||= jid diff --git a/lib/sidekiq_unique_jobs/scripts.rb b/lib/sidekiq_unique_jobs/scripts.rb index 5a6002fef..3ac441d19 100644 --- a/lib/sidekiq_unique_jobs/scripts.rb +++ b/lib/sidekiq_unique_jobs/scripts.rb @@ -16,14 +16,20 @@ module Scripts module_function + # # Call a lua script with the provided file_name + # + # @note this method is recursive if we need to load a lua script + # that wasn't previously loaded. + # # @param [Symbol] file_name the name of the lua script # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection # @param [Hash] options arguments to pass to the script file # @option options [Array] :keys the array of keys to pass to the script # @option options [Array] :argv the array of arguments to pass to the script - # @note this method is recursive if we need to load a lua script - # that wasn't previously loaded. + # + # @return value from script + # def call(file_name, redis_pool, options = {}) execute_script(file_name, redis_pool, options) rescue Redis::CommandError => ex @@ -32,12 +38,17 @@ def call(file_name, redis_pool, options = {}) end end + # # Execute the script file + # # @param [Symbol] file_name the name of the lua script # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection # @param [Hash] options arguments to pass to the script file # @option options [Array] :keys the array of keys to pass to the script # @option options [Array] :argv the array of arguments to pass to the script + # + # @return value from script (evalsha) + # def execute_script(file_name, redis_pool, options = {}) redis(redis_pool) do |conn| sha = script_sha(conn, file_name) @@ -45,10 +56,15 @@ def execute_script(file_name, redis_pool, options = {}) end end + # # Return sha of already loaded lua script or load it and return the sha + # # @param [Sidekiq::RedisConnection] conn the redis connection # @param [Symbol] file_name the name of the lua script # @return [String] sha of the script file + # + # @return [String] the sha of the script + # def script_sha(conn, file_name) if (sha = SCRIPT_SHAS.get(file_name)) return sha @@ -59,10 +75,15 @@ def script_sha(conn, file_name) sha end - # Return sha of already loaded lua script or load it and return the sha + # + # Handle errors to allow retrying errors that need retrying + # # @param [Redis::CommandError] ex exception to handle # @param [Symbol] file_name the name of the lua script - # @raise [ScriptError] when the error isn't handled + # + # @return [void] + # + # @yieldreturn [void] yields back to the caller when NOSCRIPT is raised def handle_error(ex, file_name) if ex.message == "NOSCRIPT No matching script. Please use EVAL." SCRIPT_SHAS.delete(file_name) @@ -72,16 +93,24 @@ def handle_error(ex, file_name) raise ScriptError, file_name: file_name, source_exception: ex end + # # Reads the lua file from disk + # # @param [Symbol] file_name the name of the lua script + # # @return [String] the content of the lua file + # def script_source(file_name) script_path(file_name).read end + # # Construct a Pathname to a lua script + # # @param [Symbol] file_name the name of the lua script + # # @return [Pathname] the full path to the gems lua script + # def script_path(file_name) LUA_PATHNAME.join("#{file_name}.lua") end From 75f75047a2829dccc14c948dbc8394e509a2b205 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 12:47:54 +0200 Subject: [PATCH 03/17] Bump rubocop versions and config --- .rubocop.yml | 4 ++++ Gemfile | 1 + 2 files changed, 5 insertions(+) diff --git a/.rubocop.yml b/.rubocop.yml index ece036a48..29a48ba7e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,5 +1,6 @@ require: - rubocop-rspec + - rubocop-performance inherit_mode: merge: @@ -64,6 +65,9 @@ Naming/FileName: Exclude: - lib/sidekiq-unique-jobs.rb +Naming/RescuedExceptionsVariableName: + PreferredName: ex + Naming/UncommunicativeMethodParamName: AllowedNames: - ex diff --git a/Gemfile b/Gemfile index d0fcb7eba..42885d69f 100644 --- a/Gemfile +++ b/Gemfile @@ -21,6 +21,7 @@ platforms :mri_25 do gem "pry" gem "reek", ">= 5.3" gem "rubocop" + gem "rubocop-performance" gem "rubocop-rspec" gem "simplecov-json" gem "travis" From 7095002cdb48f7537b6601088e244cf27451ba26 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 15:09:27 +0200 Subject: [PATCH 04/17] More documentation layout improvements --- lib/sidekiq_unique_jobs/lock/while_executing.rb | 1 + lib/sidekiq_unique_jobs/server/middleware.rb | 12 +++++++++--- spec/unit/sidekiq_unique_jobs/unlockable_spec.rb | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/sidekiq_unique_jobs/lock/while_executing.rb b/lib/sidekiq_unique_jobs/lock/while_executing.rb index 9955253d6..52e05060a 100644 --- a/lib/sidekiq_unique_jobs/lock/while_executing.rb +++ b/lib/sidekiq_unique_jobs/lock/while_executing.rb @@ -16,6 +16,7 @@ class WhileExecuting < BaseLock # @param [Hash] item the Sidekiq job hash # @param [Proc] callback callback to call after unlock # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection + # def initialize(item, callback, redis_pool = nil) super(item, callback, redis_pool) append_unique_key_suffix diff --git a/lib/sidekiq_unique_jobs/server/middleware.rb b/lib/sidekiq_unique_jobs/server/middleware.rb index 18c928c48..048f30137 100644 --- a/lib/sidekiq_unique_jobs/server/middleware.rb +++ b/lib/sidekiq_unique_jobs/server/middleware.rb @@ -9,13 +9,19 @@ class Middleware include Logging include OptionsWithFallback - # Runs the server middleware - # Used from Sidekiq::Processor#process + # + # + # Runs the server middleware (used from Sidekiq::Processor#process) + # # @param [Sidekiq::Worker] worker_class # @param [Hash] item a sidekiq job hash # @param [String] queue name of the queue + # + # @see https://github.com/mperham/sidekiq/wiki/Job-Format + # @see https://github.com/mperham/sidekiq/wiki/Middleware + # # @yield when uniqueness is disabled - # @yield when the lock class executes successfully + # @yield when the lock is acquired def call(worker_class, item, queue) @worker_class = worker_class @item = item diff --git a/spec/unit/sidekiq_unique_jobs/unlockable_spec.rb b/spec/unit/sidekiq_unique_jobs/unlockable_spec.rb index 440f024fd..b23ffac11 100644 --- a/spec/unit/sidekiq_unique_jobs/unlockable_spec.rb +++ b/spec/unit/sidekiq_unique_jobs/unlockable_spec.rb @@ -6,6 +6,7 @@ def item_with_digest SidekiqUniqueJobs::UniqueArgs.digest(item) item end + let(:item) do { "class" => MyUniqueJob, "queue" => "customqueue", From 6c11912f5fb6b9cc4a6dea72cab7f10de24fac53 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 15:10:06 +0200 Subject: [PATCH 05/17] Attempt some type of benchmarks --- bin/bench | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bin/bench b/bin/bench index d1086cc4e..7de5afc5a 100755 --- a/bin/bench +++ b/bin/bench @@ -10,11 +10,22 @@ require "benchmark/ips" require "sidekiq-unique-jobs" ITERATIONS ||= 10_000 +LOCK_TYPES = %w[ + until_executed + until_expired + until_and_while_executing + until_executing + while_executing +] Benchmark.ips do |x| - x.config(time: 5, warmup: 2) - x.report("new_shit") do |_times| - SidekiqUniqueJobs::Scripts::AcquireLock.execute(nil, SecureRandom.hex, SecureRandom.hex) + x.config(time: 180, warmup: 2) + x.report("locksmith lock and unlock") do |_times| + item = { "ttl" => rand(10), "jid" => SecureRandom.hex, "unique_digest" => SecureRandom.hex, lock: LOCK_TYPES.sample } + locksmith = SidekiqUniqueJobs::Locksmith.new(item) + locksmith.lock + locksmith.unlock end + x.compare! end From f85f44f54c8fab2b942627c483385f9ed335835f Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 15:10:38 +0200 Subject: [PATCH 06/17] Initiative to easy testing various lock --- spec/support/simulate_lock.rb | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 spec/support/simulate_lock.rb diff --git a/spec/support/simulate_lock.rb b/spec/support/simulate_lock.rb new file mode 100644 index 000000000..a81404a20 --- /dev/null +++ b/spec/support/simulate_lock.rb @@ -0,0 +1,43 @@ +require "concurrent/array" + +module SimulateLock + extend self + @items = Concurrent::Array.new + + def lock_until_executed(digest, jid, ttl = nil) + lock() + end + + def lock_until_expired(digest, jid, ttl) + item = get_item(digest: digest, jid: jid, lock_type: :until_expired, expiration: ttl) + lock(item) + end + + def lock_until_and_while_executing(digest, jid, ttl) + item = get_item(digest: digest, jid: jid, lock_type: :until_expired, expiration: ttl) + lock(item) + end + + def lock_while_executing(digest, jid, ttl = nil) + lock(digest: digest, jid: "randomjid", lock_type: :while_executing, expiration: nil) + end + + def runtime_lock(digest, jid, ttl = nil) + + end + + def lock(item) + Locksmith.new(item).lock + end + + def get_item(digest: "randomdigest", jid: "randomjid", lock_type: :until_executed, expiration: nil) + item = { + UNIQUE_DIGEST_KEY => digest, + JID_KEY => jid, + LOCK_EXPIRATION_KEY => expiration, + LOCK_KEY => lock_type, + } + @items << item + item + end +end From 5b3c2f85788458dd277a8e80927d65c517578e5c Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 19:17:20 +0200 Subject: [PATCH 07/17] Add a couple of more test workers --- rails_example/app/workers/sequence_action_worker.rb | 10 ++++++++++ rails_example/app/workers/test_uniq_job.rb | 8 ++++++++ 2 files changed, 18 insertions(+) create mode 100644 rails_example/app/workers/sequence_action_worker.rb create mode 100644 rails_example/app/workers/test_uniq_job.rb diff --git a/rails_example/app/workers/sequence_action_worker.rb b/rails_example/app/workers/sequence_action_worker.rb new file mode 100644 index 000000000..35c3d090c --- /dev/null +++ b/rails_example/app/workers/sequence_action_worker.rb @@ -0,0 +1,10 @@ +class SequenceActionWorker + include Sidekiq::Worker + + sidekiq_options lock: :until_and_while_executing + + def perform(id) + sleep 1.minute + p id + end +end diff --git a/rails_example/app/workers/test_uniq_job.rb b/rails_example/app/workers/test_uniq_job.rb new file mode 100644 index 000000000..55d0b542f --- /dev/null +++ b/rails_example/app/workers/test_uniq_job.rb @@ -0,0 +1,8 @@ +class TestUniqJob + include Sidekiq::Worker + sidekiq_options queue: :default, retry: false, unique: :until_executed + + def perform + sleep 3.minutes + end +end From 3f7df9cfbd90e1e9c50ecf2f596351bfa9366202 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 19:20:26 +0200 Subject: [PATCH 08/17] Update README --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7c21a091e..c90be74d0 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ - [Introduction](#introduction) - [Documentation](#documentation) - [Requirements](#requirements) + - [ActiveJob](#activejob) + - [redis-namespace](#redis-namespace) - [Installation](#installation) - [Support Me](#support-me) - [General Information](#general-information) @@ -56,9 +58,17 @@ Below are links to the latest major versions (4 & 5): ## Requirements -See [Sidekiq requirements][] for what is required. Starting from 5.0.0 only sidekiq >= 4 is supported and support for MRI <= 2.1 is dropped. ActiveJob is not supported +See [Sidekiq requirements][] for what is required. Starting from 5.0.0 only sidekiq >= 4 and MRI >= 2.2. ActiveJob is not supported -Version 6 requires Redis >= 3 and pure Sidekiq, no ActiveJob supported anymore. See [About ActiveJob](https://github.com/mhenrixon/sidekiq-unique-jobs/wiki/About-ActiveJob) for why. +### ActiveJob + +Version 6 requires Redis >= 3 and pure Sidekiq, no ActiveJob supported anymore. See [About ActiveJob](https://github.com/mhenrixon/sidekiq-unique-jobs/wiki/About-ActiveJob) for why. It simply is too complex and generates more issues than I can handle given how little timer I have to spend on this project. + +### redis-namespace + +Will not be officially supported anymore. Since Mike [won't support redis-namespace](https://github.com/mperham/sidekiq/issues/3366#issuecomment-284270120) neither will I. + +[Read this](http://www.mikeperham.com/2017/04/10/migrating-from-redis-namespace/) for how to migrate away from namespacing. ## Installation From 09df403a907859f55cba0cf65583e47ad3230358 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 19:21:32 +0200 Subject: [PATCH 09/17] Run rubocop in example project --- .rubocop.yml | 2 +- rails_example/.rubocop.yml | 9 +++++++++ rails_example/Gemfile | 5 ++++- rails_example/config/database.yml | 3 --- rails_example/spec/factories/posts.rb | 6 +++--- rails_example/spec/models/post_spec.rb | 3 ++- rails_example/spec/rails_helper.rb | 14 +++++++------- rails_example/spec/spec_helper.rb | 2 +- rails_example/spec/workers/simple_worker_spec.rb | 3 ++- 9 files changed, 29 insertions(+), 18 deletions(-) create mode 100644 rails_example/.rubocop.yml diff --git a/.rubocop.yml b/.rubocop.yml index 29a48ba7e..49105039e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,6 +1,6 @@ require: - - rubocop-rspec - rubocop-performance + - rubocop-rspec inherit_mode: merge: diff --git a/rails_example/.rubocop.yml b/rails_example/.rubocop.yml new file mode 100644 index 000000000..709f7ad1c --- /dev/null +++ b/rails_example/.rubocop.yml @@ -0,0 +1,9 @@ +inherit_from: ../.rubocop.yml + +require: + - rubocop-performance + - rubocop-rspec + +inherit_mode: + merge: + - Exclude diff --git a/rails_example/Gemfile b/rails_example/Gemfile index 5e1aaf6ab..3539eb517 100644 --- a/rails_example/Gemfile +++ b/rails_example/Gemfile @@ -2,7 +2,7 @@ source "https://rubygems.org" -ruby "2.5.3" +ruby "2.5.5" gem "bigdecimal" gem "json" @@ -24,4 +24,7 @@ group :development, :test do gem "pry-rails" gem "rspec-eventually" gem "rspec-rails" + gem "rubocop" + gem "rubocop-performance" + gem "rubocop-rspec" end diff --git a/rails_example/config/database.yml b/rails_example/config/database.yml index 826b3c7af..e6a8ae5f6 100644 --- a/rails_example/config/database.yml +++ b/rails_example/config/database.yml @@ -5,9 +5,6 @@ defaults: &defaults port: <%= ENV.fetch('DB_PORT') { 5432 } %> username: <%= ENV.fetch('DB_USERNAME') { `whoami` } %> password: <%= ENV.fetch('DB_PASSWORD') { nil } %> - pool: <%= ENV.fetch('DB_POOL') { 25 } %> - timeout: <%= ENV.fetch('DB_TIMEOUT') { 5000 } %> - reaping_frequency: <%= ENV.fetch('DB_REAPING_FREQUENCY') { 5000 } %> development: <<: *defaults diff --git a/rails_example/spec/factories/posts.rb b/rails_example/spec/factories/posts.rb index a0e0a6ecc..cec6f1ac5 100644 --- a/rails_example/spec/factories/posts.rb +++ b/rails_example/spec/factories/posts.rb @@ -2,9 +2,9 @@ FactoryBot.define do factory :post do - title { 'MyString' } - body { 'MyText' } - excerpt { 'MyString' } + title { "MyString" } + body { "MyText" } + excerpt { "MyString" } read_count { 1 } end end diff --git a/rails_example/spec/models/post_spec.rb b/rails_example/spec/models/post_spec.rb index 7dd4423f3..16fbd7400 100644 --- a/rails_example/spec/models/post_spec.rb +++ b/rails_example/spec/models/post_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true -require 'rails_helper' +require "rails_helper" describe Post, type: :model do + it { "behaves" } end diff --git a/rails_example/spec/rails_helper.rb b/rails_example/spec/rails_helper.rb index 1d055d64b..8f3934e8c 100644 --- a/rails_example/spec/rails_helper.rb +++ b/rails_example/spec/rails_helper.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true # This file is copied to spec/ when you run 'rails generate rspec:install' -ENV['RAILS_ENV'] ||= 'development' -require File.expand_path('../../config/environment', __FILE__) +ENV["RAILS_ENV"] ||= "development" +require File.expand_path("../config/environment", __dir__) # Prevent database truncation if the environment is production -abort('The Rails environment is running in production mode!') if Rails.env.production? -require 'spec_helper' +abort("The Rails environment is running in production mode!") if Rails.env.production? +require "spec_helper" -require 'rspec/rails' -require 'pry-byebug' +require "rspec/rails" +require "pry-byebug" -Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f } +Dir[Rails.root.join("spec", "support", "**", "*.rb")].each { |f| require f } ActiveRecord::Migration.maintain_test_schema! diff --git a/rails_example/spec/spec_helper.rb b/rails_example/spec/spec_helper.rb index 0cefc7bb1..99b5c11c6 100644 --- a/rails_example/spec/spec_helper.rb +++ b/rails_example/spec/spec_helper.rb @@ -18,7 +18,7 @@ config.example_status_persistence_file_path = ".rspec_status" # config.disable_monkey_patching! - config.default_formatter = 'doc' if config.files_to_run.one? + config.default_formatter = "doc" if config.files_to_run.one? config.profile_examples = 10 config.order = :random diff --git a/rails_example/spec/workers/simple_worker_spec.rb b/rails_example/spec/workers/simple_worker_spec.rb index 2064ac782..f1d85d3d1 100644 --- a/rails_example/spec/workers/simple_worker_spec.rb +++ b/rails_example/spec/workers/simple_worker_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true -require 'rails_helper' +require "rails_helper" describe SimpleWorker do + it { "behaves" } end From a06022ba9b98d8dd2dd06f9049f5f55c0b542b1a Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 19:23:08 +0200 Subject: [PATCH 10/17] Keep working away at lock simulation --- .../sidekiq_unique_jobs/locksmith_spec.rb | 1 + spec/support/simulate_lock.rb | 24 ++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/spec/integration/sidekiq_unique_jobs/locksmith_spec.rb b/spec/integration/sidekiq_unique_jobs/locksmith_spec.rb index 8ae56f1c7..12bfb848e 100644 --- a/spec/integration/sidekiq_unique_jobs/locksmith_spec.rb +++ b/spec/integration/sidekiq_unique_jobs/locksmith_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "spec_helper" + RSpec.describe SidekiqUniqueJobs::Locksmith, redis: :redis do let(:locksmith_one) { described_class.new(item_one) } let(:locksmith_two) { described_class.new(item_two) } diff --git a/spec/support/simulate_lock.rb b/spec/support/simulate_lock.rb index a81404a20..1a10c21e7 100644 --- a/spec/support/simulate_lock.rb +++ b/spec/support/simulate_lock.rb @@ -5,36 +5,44 @@ module SimulateLock @items = Concurrent::Array.new def lock_until_executed(digest, jid, ttl = nil) - lock() + item = get_item(digest: digest, jid: jid, lock_type: :until_executed, ttl: ttl) + lock(item) end def lock_until_expired(digest, jid, ttl) - item = get_item(digest: digest, jid: jid, lock_type: :until_expired, expiration: ttl) + item = get_item(digest: digest, jid: jid, lock_type: :until_expired, ttl: ttl) lock(item) end - def lock_until_and_while_executing(digest, jid, ttl) - item = get_item(digest: digest, jid: jid, lock_type: :until_expired, expiration: ttl) + def lock_until_and_while_executing(digest, jid, ttl = nil) + item = get_item(digest: digest, jid: jid, lock_type: :until_expired, ttl: ttl) lock(item) end def lock_while_executing(digest, jid, ttl = nil) - lock(digest: digest, jid: "randomjid", lock_type: :while_executing, expiration: nil) + item = get_item(digest: digest, jid: "randomjid", lock_type: :while_executing, ttl: nil) + lock(item) end def runtime_lock(digest, jid, ttl = nil) - + item = get_item(digest: digest, jid: "randomjid", lock_type: :while_executing, ttl: nil) + item = get_item(digest: "#{digest}:RUN", jid: "randomjid", lock_type: :while_executing, ttl: nil) + lock(item) end def lock(item) Locksmith.new(item).lock end - def get_item(digest: "randomdigest", jid: "randomjid", lock_type: :until_executed, expiration: nil) + def unlock(item) + Locksmith.new(item).unlock + end + + def get_item(digest: "randomdigest", jid: "randomjid", lock_type: :until_executed, ttl: nil) item = { UNIQUE_DIGEST_KEY => digest, JID_KEY => jid, - LOCK_EXPIRATION_KEY => expiration, + LOCK_EXPIRATION_KEY => ttl, LOCK_KEY => lock_type, } @items << item From 7d4e967ce24419bc2d7cd7f33520c664207e64e0 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sat, 13 Apr 2019 19:52:23 +0200 Subject: [PATCH 11/17] Adding more real world worker code/gems --- rails_example/Gemfile | 2 ++ rails_example/app/workers/hard_worker.rb | 15 ++++++++++++ .../app/workers/sequence_action_worker.rb | 1 + rails_example/app/workers/simple_worker.rb | 1 + .../workers/slow_until_executing_worker.rb | 1 + .../app/workers/spawn_simple_worker.rb | 1 + rails_example/app/workers/test_uniq_job.rb | 1 + .../while_executing_with_timeout_worker.rb | 1 + .../app/workers/while_executing_worker.rb | 1 + .../app/workers/without_args_worker.rb | 2 ++ .../app/workers/without_argument_worker.rb | 2 ++ rails_example/config/initializers/sidekiq.rb | 23 ++++++++++++++++++- rails_example/config/routes.rb | 1 + rails_example/config/schedule.yml | 6 +++++ 14 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 rails_example/app/workers/hard_worker.rb create mode 100644 rails_example/config/schedule.yml diff --git a/rails_example/Gemfile b/rails_example/Gemfile index 3539eb517..73e391df8 100644 --- a/rails_example/Gemfile +++ b/rails_example/Gemfile @@ -11,6 +11,8 @@ gem "puma" gem "rack-protection" gem "rails", "~> 5.2" gem "sidekiq" +gem 'sidekiq-cron' +gem 'sidekiq-status' gem "sidekiq-unique-jobs", path: ".." gem "sinatra" gem "slim-rails" diff --git a/rails_example/app/workers/hard_worker.rb b/rails_example/app/workers/hard_worker.rb new file mode 100644 index 000000000..e37d31e2b --- /dev/null +++ b/rails_example/app/workers/hard_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class HardWorker + include Sidekiq::Worker + include Sidekiq::Status::Worker + sidekiq_options lock: :until_executed, + queue: :default + + def perform + SidekiqUniqueJobs.with_context(self.class.name) do + SidekiqUniqueJobs.logger.debug { "#{__method__}" } + end + sleep 1 + end +end diff --git a/rails_example/app/workers/sequence_action_worker.rb b/rails_example/app/workers/sequence_action_worker.rb index 35c3d090c..af1bf19a0 100644 --- a/rails_example/app/workers/sequence_action_worker.rb +++ b/rails_example/app/workers/sequence_action_worker.rb @@ -1,5 +1,6 @@ class SequenceActionWorker include Sidekiq::Worker + include Sidekiq::Status::Worker sidekiq_options lock: :until_and_while_executing diff --git a/rails_example/app/workers/simple_worker.rb b/rails_example/app/workers/simple_worker.rb index 62692a4d4..31de250f7 100644 --- a/rails_example/app/workers/simple_worker.rb +++ b/rails_example/app/workers/simple_worker.rb @@ -2,6 +2,7 @@ class SimpleWorker include Sidekiq::Worker + include Sidekiq::Status::Worker sidekiq_options lock: :until_executed, queue: :default, unique_args: (lambda do |args| diff --git a/rails_example/app/workers/slow_until_executing_worker.rb b/rails_example/app/workers/slow_until_executing_worker.rb index 674864b47..75085d9ed 100644 --- a/rails_example/app/workers/slow_until_executing_worker.rb +++ b/rails_example/app/workers/slow_until_executing_worker.rb @@ -2,6 +2,7 @@ class SlowUntilExecutingWorker include Sidekiq::Worker + include Sidekiq::Status::Worker sidekiq_options lock: :until_executing, queue: :default, unique_args: (lambda do |args| diff --git a/rails_example/app/workers/spawn_simple_worker.rb b/rails_example/app/workers/spawn_simple_worker.rb index 111d3dc74..90aa9b9b8 100644 --- a/rails_example/app/workers/spawn_simple_worker.rb +++ b/rails_example/app/workers/spawn_simple_worker.rb @@ -2,6 +2,7 @@ class SpawnSimpleWorker include Sidekiq::Worker + include Sidekiq::Status::Worker def perform(spawn_arg) SidekiqUniqueJobs.with_context(self.class.name) do diff --git a/rails_example/app/workers/test_uniq_job.rb b/rails_example/app/workers/test_uniq_job.rb index 55d0b542f..94529f703 100644 --- a/rails_example/app/workers/test_uniq_job.rb +++ b/rails_example/app/workers/test_uniq_job.rb @@ -1,5 +1,6 @@ class TestUniqJob include Sidekiq::Worker + include Sidekiq::Status::Worker sidekiq_options queue: :default, retry: false, unique: :until_executed def perform diff --git a/rails_example/app/workers/while_executing_with_timeout_worker.rb b/rails_example/app/workers/while_executing_with_timeout_worker.rb index c68c237f4..d865eee1e 100644 --- a/rails_example/app/workers/while_executing_with_timeout_worker.rb +++ b/rails_example/app/workers/while_executing_with_timeout_worker.rb @@ -2,6 +2,7 @@ class WhileExecutingWithTimeoutWorker include Sidekiq::Worker + include Sidekiq::Status::Worker sidekiq_options unique: :while_executing, lock_timeout: 5, unique_args: ->(args) { args.second } diff --git a/rails_example/app/workers/while_executing_worker.rb b/rails_example/app/workers/while_executing_worker.rb index d1513d950..8e30dcf4f 100644 --- a/rails_example/app/workers/while_executing_worker.rb +++ b/rails_example/app/workers/while_executing_worker.rb @@ -2,6 +2,7 @@ class WhileExecutingWorker include Sidekiq::Worker + include Sidekiq::Status::Worker sidekiq_options unique: :while_executing, lock_timeout: 15 diff --git a/rails_example/app/workers/without_args_worker.rb b/rails_example/app/workers/without_args_worker.rb index 7c61fa20c..c5582d34c 100644 --- a/rails_example/app/workers/without_args_worker.rb +++ b/rails_example/app/workers/without_args_worker.rb @@ -2,6 +2,8 @@ class WithoutArgsWorker include Sidekiq::Worker + include Sidekiq::Status::Worker + sidekiq_options queue: :default, retry: true, backtrace: true, diff --git a/rails_example/app/workers/without_argument_worker.rb b/rails_example/app/workers/without_argument_worker.rb index 11c3739f2..694413de0 100644 --- a/rails_example/app/workers/without_argument_worker.rb +++ b/rails_example/app/workers/without_argument_worker.rb @@ -2,6 +2,8 @@ class WithoutArgumentWorker include Sidekiq::Worker + include Sidekiq::Status::Worker + sidekiq_options unique: :until_executed, log_duplicate_payload: true diff --git a/rails_example/config/initializers/sidekiq.rb b/rails_example/config/initializers/sidekiq.rb index f7d687c26..52e517c21 100644 --- a/rails_example/config/initializers/sidekiq.rb +++ b/rails_example/config/initializers/sidekiq.rb @@ -7,9 +7,30 @@ Sidekiq.configure_server do |config| config.redis = { url: ENV['REDIS_URL'] } - # config.error_handlers << Proc.new { |ex, context| Airbrake.notify_or_ignore(ex, parameters: context) } + config.error_handlers << Proc.new {|ex,ctx_hash| p ex, ctx_hash } + + config.death_handlers << ->(job, _ex) do + SidekiqUniqueJobs::Digests.del(digest: job['unique_digest']) if job['unique_digest'] + end + + # accepts :expiration (optional) + Sidekiq::Status.configure_server_middleware config, expiration: 30.minutes + + # accepts :expiration (optional) + Sidekiq::Status.configure_client_middleware config, expiration: 30.minutes + + schedule_file = "config/schedule.yml" + + if File.exist?(schedule_file) + Sidekiq::Cron::Job.load_from_hash YAML.load_file(schedule_file) + end end Sidekiq.configure_client do |config| config.redis = { url: ENV['REDIS_URL'] } + # accepts :expiration (optional) + Sidekiq::Status.configure_client_middleware config, expiration: 30.minutes end + +Sidekiq.log_format = :json if Sidekiq.respond_to?(:log_format) +SidekiqUniqueJobs.logger.level = Object.const_get("Logger::#{ENV.fetch('LOGLEVEL') { 'debug' }.upcase}") diff --git a/rails_example/config/routes.rb b/rails_example/config/routes.rb index 7f9c89929..9489c468b 100644 --- a/rails_example/config/routes.rb +++ b/rails_example/config/routes.rb @@ -2,6 +2,7 @@ Rails.application.routes.draw do require 'sidekiq_unique_jobs/web' + require 'sidekiq-status/web' mount Sidekiq::Web, at: '/sidekiq' get 'work' => 'work#index' diff --git a/rails_example/config/schedule.yml b/rails_example/config/schedule.yml new file mode 100644 index 000000000..e50c88e26 --- /dev/null +++ b/rails_example/config/schedule.yml @@ -0,0 +1,6 @@ +#config/schedule.yml + +my_first_job: + cron: "*/2 * * * *" + class: "HardWorker" + queue: default From 88b5c75ebacc1bed0d585872d437891d4365ad48 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sun, 14 Apr 2019 17:57:34 +0200 Subject: [PATCH 12/17] Delete runtime lock when worker raises error --- .../lock/until_and_while_executing.rb | 1 - .../lock/while_executing.rb | 7 ++- .../lock/while_executing_reject.rb | 8 --- .../lock/until_and_while_executing_spec.rb | 11 ++++ .../lock/while_executing_reject_spec.rb | 56 +++++++++++-------- .../lock/while_executing_spec.rb | 37 +++++++----- .../lock/while_executing_reject_spec.rb | 6 +- .../on_conflict/replace_spec.rb | 1 + 8 files changed, 76 insertions(+), 51 deletions(-) diff --git a/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb b/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb index be604a094..d622662c8 100644 --- a/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +++ b/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb @@ -19,7 +19,6 @@ def execute return unless locked? unlock - runtime_lock.execute { yield } end diff --git a/lib/sidekiq_unique_jobs/lock/while_executing.rb b/lib/sidekiq_unique_jobs/lock/while_executing.rb index 52e05060a..78e9b2ec2 100644 --- a/lib/sidekiq_unique_jobs/lock/while_executing.rb +++ b/lib/sidekiq_unique_jobs/lock/while_executing.rb @@ -35,7 +35,12 @@ def lock def execute return strategy.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY]) - with_cleanup { yield } + yield + rescue Exception + delete! + raise + else + unlock_with_callback end private diff --git a/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb b/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb index 95fa880d8..f017112c5 100644 --- a/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +++ b/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb @@ -11,14 +11,6 @@ class Lock # # @author Mikael Henriksson class WhileExecutingReject < WhileExecuting - # Executes in the Sidekiq server process - # @yield to the worker class perform method - def execute - return strategy.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY]) - - with_cleanup { yield } - end - # Overridden with a forced {OnConflict::Reject} strategy # @return [OnConflict::Reject] a reject strategy def strategy diff --git a/spec/integration/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb b/spec/integration/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb index 31e5d9530..cc7023cea 100644 --- a/spec/integration/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb +++ b/spec/integration/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb @@ -70,5 +70,16 @@ expect(unset).to eq(true) end end + + context "when worker raises error in runtime lock" do + it "always unlocks" do + process_one.lock + + expect { process_one.execute { raise "Hell" } } + .to raise_error(RuntimeError, "Hell") + + expect(runtime_one.locked?).to eq(false) + end + end end end diff --git a/spec/integration/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb b/spec/integration/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb index 609268eb8..2d1aa069e 100644 --- a/spec/integration/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb +++ b/spec/integration/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb @@ -29,7 +29,7 @@ "args" => args } end - describe "#execute" do + describe "#lock" do it "does not lock jobs" do expect(process_one.lock).to eq(true) expect(process_one).not_to be_locked @@ -37,39 +37,49 @@ expect(process_two.lock).to eq(true) expect(process_two).not_to be_locked end + end - context "when job is executing" do - it "locks the process" do - process_one.execute do - expect(process_one).to be_locked - end + describe "#execute" do + it "locks the process" do + process_one.execute do + expect(process_one).to be_locked end + end - shared_examples "rejects job to deadset" do - it "moves subsequent jobs to dead queue" do - process_one.execute do - expect(dead_count).to eq(0) - expect { process_two.execute {} } - .to change { dead_count }.from(0).to(1) - end + shared_examples "rejects job to deadset" do + it "moves subsequent jobs to dead queue" do + process_one.execute do + expect(dead_count).to eq(0) + expect { process_two.execute {} } + .to change { dead_count }.from(0).to(1) end end + end + + it_behaves_like "rejects job to deadset" + + context "when Sidekiq::DeadSet respond to kill" do it_behaves_like "rejects job to deadset" + end - context "when Sidekiq::DeadSet respond to kill" do - it_behaves_like "rejects job to deadset" + context "when Sidekiq::DeadSet does not respond to kill" do + let(:strategy) { SidekiqUniqueJobs::OnConflict::Reject.new(item_two) } + + before do + allow(strategy).to receive(:deadset_kill?).and_return(false) + allow(process_two).to receive(:strategy).and_return(strategy) end - context "when Sidekiq::DeadSet does not respond to kill" do - let(:strategy) { SidekiqUniqueJobs::OnConflict::Reject.new(item_two) } + it_behaves_like "rejects job to deadset" + end + end - before do - allow(strategy).to receive(:deadset_kill?).and_return(false) - allow(process_two).to receive(:strategy).and_return(strategy) - end + context "when worker raises error" do + it "always unlocks" do + expect { process_one.execute { raise "Hell" } } + .to raise_error(RuntimeError, "Hell") - it_behaves_like "rejects job to deadset" - end + expect(process_one.locked?).to eq(false) end end end diff --git a/spec/integration/sidekiq_unique_jobs/lock/while_executing_spec.rb b/spec/integration/sidekiq_unique_jobs/lock/while_executing_spec.rb index bff6aa301..6387e22d6 100644 --- a/spec/integration/sidekiq_unique_jobs/lock/while_executing_spec.rb +++ b/spec/integration/sidekiq_unique_jobs/lock/while_executing_spec.rb @@ -44,26 +44,33 @@ end describe "#execute" do - context "when executing" do - it "locks the process" do - process_one.execute do - expect(process_one).to be_locked - end + it "locks the process" do + process_one.execute do + expect(process_one).to be_locked end + end + + it "calls back" do + process_one.execute {} + expect(callback).to have_received(:call) + end - it "calls back" do - process_one.execute {} - expect(callback).to have_received(:call) + it "prevents other processes from executing" do + process_one.execute do + unset = true + process_two.execute { unset = false } + expect(unset).to eq(true) end - it "prevents other processes from executing" do - process_one.execute do - unset = true - process_two.execute { unset = false } - expect(unset).to eq(true) - end + expect(callback).to have_received(:call).once + end + + context "when worker raises error" do + it "always unlocks" do + expect { process_one.execute { raise "Hell" } } + .to raise_error(RuntimeError, "Hell") - expect(callback).to have_received(:call).once + expect(process_one.locked?).to eq(false) end end end diff --git a/spec/unit/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb b/spec/unit/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb index e50121654..3a7b3aa58 100644 --- a/spec/unit/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb +++ b/spec/unit/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb @@ -29,7 +29,7 @@ before do allow(locksmith).to receive(:lock).with(0).and_return(token) - allow(lock).to receive(:with_cleanup).and_yield + allow(lock).to receive(:unlock_with_callback) end context "when lock succeeds" do @@ -37,7 +37,7 @@ it "processes the job" do execute - expect(lock).to have_received(:with_cleanup) + expect(lock).to have_received(:unlock_with_callback) end end @@ -47,7 +47,7 @@ it "rejects the job" do execute - expect(lock).not_to have_received(:with_cleanup) + expect(lock).not_to have_received(:unlock_with_callback) end end end diff --git a/spec/unit/sidekiq_unique_jobs/on_conflict/replace_spec.rb b/spec/unit/sidekiq_unique_jobs/on_conflict/replace_spec.rb index 690b07ae8..d30330104 100644 --- a/spec/unit/sidekiq_unique_jobs/on_conflict/replace_spec.rb +++ b/spec/unit/sidekiq_unique_jobs/on_conflict/replace_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "spec_helper" + RSpec.describe SidekiqUniqueJobs::OnConflict::Replace, redis: :redis do let(:strategy) { described_class.new(item) } let(:unique_digest) { "uniquejobs:56c68cab5038eb57959538866377560d" } From 41984564a9256ccdb827f13cc998347d13abd144 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sun, 14 Apr 2019 17:57:59 +0200 Subject: [PATCH 13/17] Mandatory rubocop commit --- bin/bench | 2 +- rails_example/Gemfile | 4 ++-- spec/support/simulate_lock.rb | 8 +++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/bin/bench b/bin/bench index 7de5afc5a..a0ca4de1e 100755 --- a/bin/bench +++ b/bin/bench @@ -16,7 +16,7 @@ LOCK_TYPES = %w[ until_and_while_executing until_executing while_executing -] +].freeze Benchmark.ips do |x| x.config(time: 180, warmup: 2) diff --git a/rails_example/Gemfile b/rails_example/Gemfile index 73e391df8..e179a883e 100644 --- a/rails_example/Gemfile +++ b/rails_example/Gemfile @@ -11,8 +11,8 @@ gem "puma" gem "rack-protection" gem "rails", "~> 5.2" gem "sidekiq" -gem 'sidekiq-cron' -gem 'sidekiq-status' +gem "sidekiq-cron" +gem "sidekiq-status" gem "sidekiq-unique-jobs", path: ".." gem "sinatra" gem "slim-rails" diff --git a/spec/support/simulate_lock.rb b/spec/support/simulate_lock.rb index 1a10c21e7..b037f2f94 100644 --- a/spec/support/simulate_lock.rb +++ b/spec/support/simulate_lock.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "concurrent/array" module SimulateLock @@ -20,13 +22,13 @@ def lock_until_and_while_executing(digest, jid, ttl = nil) end def lock_while_executing(digest, jid, ttl = nil) - item = get_item(digest: digest, jid: "randomjid", lock_type: :while_executing, ttl: nil) + item = get_item(digest: digest, jid: jid, lock_type: :while_executing, ttl: ttl) lock(item) end def runtime_lock(digest, jid, ttl = nil) - item = get_item(digest: digest, jid: "randomjid", lock_type: :while_executing, ttl: nil) - item = get_item(digest: "#{digest}:RUN", jid: "randomjid", lock_type: :while_executing, ttl: nil) + item = get_item(digest: digest, jid: jid, lock_type: :while_executing, ttl: ttl) + item = get_item(digest: "#{digest}:RUN", jid: "randomjid", lock_type: :while_executing, ttl: ttl) lock(item) end From d641b860c393cca42e5aa1a84b5db556881dc253 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sun, 14 Apr 2019 18:16:50 +0200 Subject: [PATCH 14/17] Update appraisals --- gemfiles/sidekiq_4.0.gemfile | 1 + gemfiles/sidekiq_4.1.gemfile | 1 + gemfiles/sidekiq_4.2.gemfile | 1 + gemfiles/sidekiq_5.0.gemfile | 1 + gemfiles/sidekiq_5.1.gemfile | 1 + gemfiles/sidekiq_5.2.gemfile | 1 + gemfiles/sidekiq_6.0.gemfile | 1 + gemfiles/sidekiq_develop.gemfile | 1 + 8 files changed, 8 insertions(+) diff --git a/gemfiles/sidekiq_4.0.gemfile b/gemfiles/sidekiq_4.0.gemfile index 2dd2a1d8d..14a006fc5 100644 --- a/gemfiles/sidekiq_4.0.gemfile +++ b/gemfiles/sidekiq_4.0.gemfile @@ -20,6 +20,7 @@ platforms :mri_25 do gem "pry" gem "reek", ">= 5.3" gem "rubocop" + gem "rubocop-performance" gem "rubocop-rspec" gem "simplecov-json" gem "travis" diff --git a/gemfiles/sidekiq_4.1.gemfile b/gemfiles/sidekiq_4.1.gemfile index f81e47baa..9fbdbf205 100644 --- a/gemfiles/sidekiq_4.1.gemfile +++ b/gemfiles/sidekiq_4.1.gemfile @@ -20,6 +20,7 @@ platforms :mri_25 do gem "pry" gem "reek", ">= 5.3" gem "rubocop" + gem "rubocop-performance" gem "rubocop-rspec" gem "simplecov-json" gem "travis" diff --git a/gemfiles/sidekiq_4.2.gemfile b/gemfiles/sidekiq_4.2.gemfile index 52a5add22..e9d0e9802 100644 --- a/gemfiles/sidekiq_4.2.gemfile +++ b/gemfiles/sidekiq_4.2.gemfile @@ -20,6 +20,7 @@ platforms :mri_25 do gem "pry" gem "reek", ">= 5.3" gem "rubocop" + gem "rubocop-performance" gem "rubocop-rspec" gem "simplecov-json" gem "travis" diff --git a/gemfiles/sidekiq_5.0.gemfile b/gemfiles/sidekiq_5.0.gemfile index 1349c8fdc..2e0f2c13d 100644 --- a/gemfiles/sidekiq_5.0.gemfile +++ b/gemfiles/sidekiq_5.0.gemfile @@ -20,6 +20,7 @@ platforms :mri_25 do gem "pry" gem "reek", ">= 5.3" gem "rubocop" + gem "rubocop-performance" gem "rubocop-rspec" gem "simplecov-json" gem "travis" diff --git a/gemfiles/sidekiq_5.1.gemfile b/gemfiles/sidekiq_5.1.gemfile index e018fc1c1..a7cf79031 100644 --- a/gemfiles/sidekiq_5.1.gemfile +++ b/gemfiles/sidekiq_5.1.gemfile @@ -20,6 +20,7 @@ platforms :mri_25 do gem "pry" gem "reek", ">= 5.3" gem "rubocop" + gem "rubocop-performance" gem "rubocop-rspec" gem "simplecov-json" gem "travis" diff --git a/gemfiles/sidekiq_5.2.gemfile b/gemfiles/sidekiq_5.2.gemfile index c9f431303..3b2b05e16 100644 --- a/gemfiles/sidekiq_5.2.gemfile +++ b/gemfiles/sidekiq_5.2.gemfile @@ -20,6 +20,7 @@ platforms :mri_25 do gem "pry" gem "reek", ">= 5.3" gem "rubocop" + gem "rubocop-performance" gem "rubocop-rspec" gem "simplecov-json" gem "travis" diff --git a/gemfiles/sidekiq_6.0.gemfile b/gemfiles/sidekiq_6.0.gemfile index 83556109e..6c1261b1f 100644 --- a/gemfiles/sidekiq_6.0.gemfile +++ b/gemfiles/sidekiq_6.0.gemfile @@ -20,6 +20,7 @@ platforms :mri_25 do gem "pry" gem "reek", ">= 5.3" gem "rubocop" + gem "rubocop-performance" gem "rubocop-rspec" gem "simplecov-json" gem "travis" diff --git a/gemfiles/sidekiq_develop.gemfile b/gemfiles/sidekiq_develop.gemfile index 098aea720..812f60a42 100644 --- a/gemfiles/sidekiq_develop.gemfile +++ b/gemfiles/sidekiq_develop.gemfile @@ -20,6 +20,7 @@ platforms :mri_25 do gem "pry" gem "reek", ">= 5.3" gem "rubocop" + gem "rubocop-performance" gem "rubocop-rspec" gem "simplecov-json" gem "travis" From abafee984eef34e29eb6e4da05f8a81a6d5417e1 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sun, 14 Apr 2019 18:36:47 +0200 Subject: [PATCH 15/17] Move redcarpet into Gemfile to avoid jruby --- Gemfile | 1 + gemfiles/sidekiq_4.0.gemfile | 1 + gemfiles/sidekiq_4.1.gemfile | 1 + gemfiles/sidekiq_4.2.gemfile | 1 + gemfiles/sidekiq_5.0.gemfile | 1 + gemfiles/sidekiq_5.1.gemfile | 1 + gemfiles/sidekiq_5.2.gemfile | 1 + gemfiles/sidekiq_6.0.gemfile | 1 + gemfiles/sidekiq_develop.gemfile | 1 + sidekiq-unique-jobs.gemspec | 2 +- 10 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 42885d69f..e02942e48 100644 --- a/Gemfile +++ b/Gemfile @@ -20,6 +20,7 @@ platforms :mri_25 do gem "memory_profiler" gem "pry" gem "reek", ">= 5.3" + gem "redcarpet", "~> 3.4" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/gemfiles/sidekiq_4.0.gemfile b/gemfiles/sidekiq_4.0.gemfile index 14a006fc5..ed6ff8dee 100644 --- a/gemfiles/sidekiq_4.0.gemfile +++ b/gemfiles/sidekiq_4.0.gemfile @@ -19,6 +19,7 @@ platforms :mri_25 do gem "memory_profiler" gem "pry" gem "reek", ">= 5.3" + gem "redcarpet", "~> 3.4" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/gemfiles/sidekiq_4.1.gemfile b/gemfiles/sidekiq_4.1.gemfile index 9fbdbf205..3598594b1 100644 --- a/gemfiles/sidekiq_4.1.gemfile +++ b/gemfiles/sidekiq_4.1.gemfile @@ -19,6 +19,7 @@ platforms :mri_25 do gem "memory_profiler" gem "pry" gem "reek", ">= 5.3" + gem "redcarpet", "~> 3.4" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/gemfiles/sidekiq_4.2.gemfile b/gemfiles/sidekiq_4.2.gemfile index e9d0e9802..bca2049f1 100644 --- a/gemfiles/sidekiq_4.2.gemfile +++ b/gemfiles/sidekiq_4.2.gemfile @@ -19,6 +19,7 @@ platforms :mri_25 do gem "memory_profiler" gem "pry" gem "reek", ">= 5.3" + gem "redcarpet", "~> 3.4" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/gemfiles/sidekiq_5.0.gemfile b/gemfiles/sidekiq_5.0.gemfile index 2e0f2c13d..b8bb8bd08 100644 --- a/gemfiles/sidekiq_5.0.gemfile +++ b/gemfiles/sidekiq_5.0.gemfile @@ -19,6 +19,7 @@ platforms :mri_25 do gem "memory_profiler" gem "pry" gem "reek", ">= 5.3" + gem "redcarpet", "~> 3.4" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/gemfiles/sidekiq_5.1.gemfile b/gemfiles/sidekiq_5.1.gemfile index a7cf79031..4a7bc08f3 100644 --- a/gemfiles/sidekiq_5.1.gemfile +++ b/gemfiles/sidekiq_5.1.gemfile @@ -19,6 +19,7 @@ platforms :mri_25 do gem "memory_profiler" gem "pry" gem "reek", ">= 5.3" + gem "redcarpet", "~> 3.4" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/gemfiles/sidekiq_5.2.gemfile b/gemfiles/sidekiq_5.2.gemfile index 3b2b05e16..659adeeff 100644 --- a/gemfiles/sidekiq_5.2.gemfile +++ b/gemfiles/sidekiq_5.2.gemfile @@ -19,6 +19,7 @@ platforms :mri_25 do gem "memory_profiler" gem "pry" gem "reek", ">= 5.3" + gem "redcarpet", "~> 3.4" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/gemfiles/sidekiq_6.0.gemfile b/gemfiles/sidekiq_6.0.gemfile index 6c1261b1f..9036bfc28 100644 --- a/gemfiles/sidekiq_6.0.gemfile +++ b/gemfiles/sidekiq_6.0.gemfile @@ -19,6 +19,7 @@ platforms :mri_25 do gem "memory_profiler" gem "pry" gem "reek", ">= 5.3" + gem "redcarpet", "~> 3.4" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/gemfiles/sidekiq_develop.gemfile b/gemfiles/sidekiq_develop.gemfile index 812f60a42..edd1dfc4a 100644 --- a/gemfiles/sidekiq_develop.gemfile +++ b/gemfiles/sidekiq_develop.gemfile @@ -19,6 +19,7 @@ platforms :mri_25 do gem "memory_profiler" gem "pry" gem "reek", ">= 5.3" + gem "redcarpet", "~> 3.4" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/sidekiq-unique-jobs.gemspec b/sidekiq-unique-jobs.gemspec index 78a2c27ea..997ce07f4 100644 --- a/sidekiq-unique-jobs.gemspec +++ b/sidekiq-unique-jobs.gemspec @@ -55,7 +55,7 @@ Gem::Specification.new do |spec| # ===== Documentation ===== spec.add_development_dependency "github-markup", "~> 3.0" spec.add_development_dependency "github_changelog_generator", "~> 1.14" - spec.add_development_dependency "redcarpet", "~> 3.4" + # spec.add_development_dependency "redcarpet", "~> 3.4" spec.add_development_dependency "yard", "~> 0.9.18" # ===== Release Management ===== From f9d8f43aa6adb9f59d216ae3f9b2d2fd30c58cae Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sun, 14 Apr 2019 18:36:53 +0200 Subject: [PATCH 16/17] Rubocop --- lib/sidekiq_unique_jobs/lock/while_executing.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sidekiq_unique_jobs/lock/while_executing.rb b/lib/sidekiq_unique_jobs/lock/while_executing.rb index 78e9b2ec2..9ed89d4ca 100644 --- a/lib/sidekiq_unique_jobs/lock/while_executing.rb +++ b/lib/sidekiq_unique_jobs/lock/while_executing.rb @@ -36,7 +36,7 @@ def execute return strategy.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY]) yield - rescue Exception + rescue Exception # rubocop:disable Lint/RescueException delete! raise else From 455207020d177a70a1b67f83b88be6da305daf16 Mon Sep 17 00:00:00 2001 From: Mika Hel Date: Sun, 14 Apr 2019 18:56:32 +0200 Subject: [PATCH 17/17] Rubocop --- Gemfile | 2 +- bin/bench | 7 ++++++- spec/support/simulate_lock.rb | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index e02942e48..76ae980e7 100644 --- a/Gemfile +++ b/Gemfile @@ -19,8 +19,8 @@ platforms :mri_25 do gem "guard-rubocop" gem "memory_profiler" gem "pry" - gem "reek", ">= 5.3" gem "redcarpet", "~> 3.4" + gem "reek", ">= 5.3" gem "rubocop" gem "rubocop-performance" gem "rubocop-rspec" diff --git a/bin/bench b/bin/bench index a0ca4de1e..c2d7e4bfb 100755 --- a/bin/bench +++ b/bin/bench @@ -21,7 +21,12 @@ LOCK_TYPES = %w[ Benchmark.ips do |x| x.config(time: 180, warmup: 2) x.report("locksmith lock and unlock") do |_times| - item = { "ttl" => rand(10), "jid" => SecureRandom.hex, "unique_digest" => SecureRandom.hex, lock: LOCK_TYPES.sample } + item = { + "ttl" => rand(10), + "jid" => SecureRandom.hex, + "unique_digest" => SecureRandom.hex, + "lock" => LOCK_TYPES.sample, + } locksmith = SidekiqUniqueJobs::Locksmith.new(item) locksmith.lock locksmith.unlock diff --git a/spec/support/simulate_lock.rb b/spec/support/simulate_lock.rb index b037f2f94..e6557dd68 100644 --- a/spec/support/simulate_lock.rb +++ b/spec/support/simulate_lock.rb @@ -28,6 +28,7 @@ def lock_while_executing(digest, jid, ttl = nil) def runtime_lock(digest, jid, ttl = nil) item = get_item(digest: digest, jid: jid, lock_type: :while_executing, ttl: ttl) + lock(item) item = get_item(digest: "#{digest}:RUN", jid: "randomjid", lock_type: :while_executing, ttl: ttl) lock(item) end