processor.rb

# frozen_string_literal: true

require "sidekiq/util"
require "sidekiq/fetch"
require "sidekiq/job_logger"
require "sidekiq/job_retry"

module Sidekiq
  ##
  # The Processor is a standalone thread which:
  #
  # 1. fetches a job from Redis
  # 2. executes the job
  #   a. instantiate the Worker
  #   b. run the middleware chain
  #   c. call #perform
  #
  # A Processor can exit due to shutdown (processor_stopped)
  # or due to an error during job execution (processor_died)
  #
  # If an error occurs in the job execution, the
  # Processor calls the Manager to create a new one
  # to replace itself and exits.
  #
  class Processor
    include Util

    attr_reader :thread
    attr_reader :job

    def initialize(mgr)
      @mgr = mgr
      @down = false
      @done = false
      @job = nil
      @thread = nil
      @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
      @reloader = Sidekiq.options[:reloader]
      @job_logger = (mgr.options[:job_logger] || Sidekiq::JobLogger).new
      @retrier = Sidekiq::JobRetry.new
    end

    def terminate(wait = false)
      @done = true
      return unless @thread
      @thread.value if wait
    end

    def kill(wait = false)
      @done = true
      return unless @thread
      # unlike the other actors, terminate does not wait
      # for the thread to finish because we don't know how
      # long the job will take to finish.  Instead we
      # provide a `kill` method to call after the shutdown
      # timeout passes.
      @thread.raise ::Sidekiq::Shutdown
      @thread.value if wait
    end

    def start
      @thread ||= safe_thread("processor", &method(:run))
    end

    private unless $TESTING

    def run
      process_one until @done
      @mgr.processor_stopped(self)
    rescue Sidekiq::Shutdown
      @mgr.processor_stopped(self)
    rescue Exception => ex
      @mgr.processor_died(self, ex)
    end

    def process_one
      @job = fetch
      process(@job) if @job
      @job = nil
    end

    def get_one
      work = @strategy.retrieve_work
      if @down
        logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
        @down = nil
      end
      work
    rescue Sidekiq::Shutdown
    rescue => ex
      handle_fetch_exception(ex)
    end

    def fetch
      j = get_one
      if j && @done
        j.requeue
        nil
      else
        j
      end
    end

    def handle_fetch_exception(ex)
      unless @down
        @down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
        logger.error("Error fetching job: #{ex}")
        handle_exception(ex)
      end
      sleep(1)
      nil
    end

    def dispatch(job_hash, queue)
      # since middleware can mutate the job hash
      # we clone here so we report the original
      # job structure to the Web UI
      pristine = cloned(job_hash)

      @job_logger.prepare(job_hash) do
        @retrier.global(pristine, queue) do
          @job_logger.call(job_hash, queue) do
            stats(pristine, queue) do
              # Rails 5 requires a Reloader to wrap code execution.  In order to
              # constantize the worker and instantiate an instance, we have to call
              # the Reloader.  It handles code loading, db connection management, etc.
              # Effectively this block denotes a "unit of work" to Rails.
              @reloader.call do
                klass = constantize(job_hash["class"])
                worker = klass.new
                worker.jid = job_hash["jid"]
                @retrier.local(worker, pristine, queue) do
                  yield worker
                end
              end
            end
          end
        end
      end
    end

    def process(work)
      jobstr = work.job
      queue = work.queue_name

      # Treat malformed JSON as a special case: job goes straight to the morgue.
      job_hash = nil
      begin
        job_hash = Sidekiq.load_json(jobstr)
      rescue => ex
        handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
        # we can't notify because the job isn't a valid hash payload.
        DeadSet.new.kill(jobstr, notify_failure: false)
        return work.acknowledge
      end

      ack = false
      begin
        dispatch(job_hash, queue) do |worker|
          Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
            execute_job(worker, cloned(job_hash["args"]))
          end
        end
        ack = true
      rescue Sidekiq::Shutdown
        # Had to force kill this job because it didn't finish
        # within the timeout.  Don't acknowledge the work since
        # we didn't properly finish it.
      rescue Sidekiq::JobRetry::Handled => h
        # this is the common case: job raised error and Sidekiq::JobRetry::Handled
        # signals that we created a retry successfully.  We can acknowlege the job.
        ack = true
        e = h.cause || h
        handle_exception(e, {context: "Job raised exception", job: job_hash, jobstr: jobstr})
        raise e
      rescue Exception => ex
        # Unexpected error!  This is very bad and indicates an exception that got past
        # the retry subsystem (e.g. network partition).  We won't acknowledge the job
        # so it can be rescued when using Sidekiq Pro.
        handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
        raise e
      ensure
        if ack
          # We don't want a shutdown signal to interrupt job acknowledgment.
          Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
            work.acknowledge
          end
        end
      end
    end

    def execute_job(worker, cloned_args)
      worker.perform(*cloned_args)
    end

    # Ruby doesn't provide atomic counters out of the box so we'll
    # implement something simple ourselves.
    # https://bugs.ruby-lang.org/issues/14706
    class Counter
      def initialize
        @value = 0
        @lock = Mutex.new
      end

      def incr(amount = 1)
        @lock.synchronize { @value += amount }
      end

      def reset
        @lock.synchronize {
          val = @value
          @value = 0
          val
        }
      end
    end

    # jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
    class SharedWorkerState
      def initialize
        @worker_state = {}
        @lock = Mutex.new
      end

      def set(tid, hash)
        @lock.synchronize { @worker_state[tid] = hash }
      end

      def delete(tid)
        @lock.synchronize { @worker_state.delete(tid) }
      end

      def dup
        @lock.synchronize { @worker_state.dup }
      end

      def size
        @lock.synchronize { @worker_state.size }
      end

      def clear
        @lock.synchronize { @worker_state.clear }
      end
    end

    PROCESSED = Counter.new
    FAILURE = Counter.new
    WORKER_STATE = SharedWorkerState.new

    def stats(job_hash, queue)
      WORKER_STATE.set(tid, {queue: queue, payload: job_hash, run_at: Time.now.to_i})

      begin
        yield
      rescue Exception
        FAILURE.incr
        raise
      ensure
        WORKER_STATE.delete(tid)
        PROCESSED.incr
      end
    end

    # Deep clone the arguments passed to the worker so that if
    # the job fails, what is pushed back onto Redis hasn't
    # been mutated by the worker.
    def cloned(thing)
      deep_dup(thing)
    end

    def constantize(str)
      return Object.const_get(str) unless str.include?("::")

      names = str.split("::")
      names.shift if names.empty? || names.first.empty?

      names.inject(Object) do |constant, name|
        # the false flag limits search for name to under the constant namespace
        #   which mimics Rails' behaviour
        constant.const_get(name, false)
      end
    end

    def deep_dup(obj)
      if Integer === obj || Float === obj || TrueClass === obj || FalseClass === obj || NilClass === obj
        return obj
      elsif String === obj
        return obj.dup
      elsif Array === obj
        duped = Array.new(obj.size)
        obj.each_with_index do |value, index|
          duped[index] = deep_dup(value)
        end
      elsif Hash === obj
        duped = obj.dup
        duped.each_pair do |key, value|
          duped[key] = deep_dup(value)
        end
      else
        duped = obj.dup
      end

      duped
    end
  end
end