Inject scheduling latency

Use conditional variable as it is much more thread and interrupt friendly

History.md

@@ -1,7 +1,7 @@
 ## Master
 
 * Features
-  * Your feature goes here (#Github Number)
+  * Inject small latency to make Puma to prefer workers that are more idle (#2079)
 
 * Bugfixes
   * Your bugfix goes here (#Github Number)

lib/puma/server.rb

@@ -214,6 +214,13 @@ def run_lopez_mode(background=true)
       end
     end
 
+    def wait_for_threads_to_finish
+      @thread_pool.wait_for_threads_to_finish(
+        tick_time: ENV.fetch('PUMA_INJECT_LATENCY', '0.001').to_f,
+        max_wait_ticks: ENV.fetch('PUMA_INJECT_WAIT_TICKS', '3').to_i
+      )
+    end
+
     def handle_servers_lopez_mode
       begin
         check = @check
@@ -386,6 +393,8 @@ def handle_servers
                 break if handle_check
               else
                 begin
+                  wait_for_threads_to_finish
+
                   if io = sock.accept_nonblock
                     client = Client.new io, @binder.env(sock)
                     if remote_addr_value

lib/puma/thread_pool.rb

@@ -75,6 +75,10 @@ def backlog
       @mutex.synchronize { @todo.size }
     end
 
+    def busy
+      spawned - waiting
+    end
+
     def pool_capacity
       waiting + (@max - spawned)
     end
@@ -213,6 +217,22 @@ def wait_until_not_full
       end
     end
 
+    def wait_for_threads_to_finish(tick_time: 0.001, max_wait_ticks: 3)
+      @mutex.synchronize do
+        max_wait_ticks.times do |tick|
+          return if @shutdown
+
+          # if we do wait for more than busy threads
+          # and what is in backlog, stop
+          break if tick >= busy + @todo.size
+
+          # this will be signaled once a request finishes,
+          # which can happen earlier than tick time
+          @not_full.wait @mutex, tick_time
+        end
+      end
+    end
+
     # If too many threads are in the pool, tell one to finish go ahead
     # and exit. If +force+ is true, then a trim request is requested
     # even if all threads are being utilized.