Add support for concurrent append in out_file

lib/fluent/error.rb

@@ -28,6 +28,9 @@ def to_s
   class InvalidRootDirectory < UnrecoverableError
   end
 
+  class InvalidLockDirectory < UnrecoverableError
+  end
+
   # For internal use
   class UncatchableError < Exception
   end

lib/fluent/plugin/base.rb

@@ -51,6 +51,10 @@ def fluentd_worker_id
         @_fluentd_worker_id
       end
 
+      def fluentd_lockdir
+        @_fluentd_lockdir ||= ENV['FLUENTD_LOCKDIR']
+      end
+
       def configure(conf)
         if Fluent::Engine.supervisor_mode || (conf.respond_to?(:for_this_worker?) && conf.for_this_worker?)
           workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
@@ -70,6 +74,19 @@ def multi_workers_ready?
         true
       end
 
+      def acquire_worker_lock(name, &block)
+        if fluentd_lockdir.nil?
+          raise InvalidLockDirectory, "can't acquire lock because FLUENTD_LOCKDIR isn't set"
+        end
+
+        name = name.gsub(/[^a-zA-Z0-9]/, "_")
+        lockfile = "fluentd-#{name}.lock"
+        File.open(File.join(fluentd_lockdir, lockfile), "w") do |f|
+          f.flock(File::LOCK_EX)
+          yield
+        end
+      end
+
       def string_safe_encoding(str)
         unless str.valid_encoding?
           str = str.scrub('?')

lib/fluent/plugin/out_file.rb

@@ -188,6 +188,10 @@ def configure(conf)
         condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
         @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
       end
+
+      if @need_lock && @append && fluentd_lockdir.nil?
+        raise InvalidLockDirectory, "must set FLUENTD_LOCKDIR on multi-worker append mode"
+      end
     end
 
     def multi_workers_ready?
@@ -217,7 +221,13 @@ def write(chunk)
                end
 
       if @append
-        writer.call(path, chunk)
+        if @need_lock
+          acquire_worker_lock(path) do
+            writer.call(path, chunk)
+          end
+        else
+          writer.call(path, chunk)
+        end
       else
         find_filepath_available(path, with_lock: @need_lock) do |actual_path|
           writer.call(actual_path, chunk)

lib/fluent/supervisor.rb

@@ -875,7 +875,11 @@ def supervise
       se = ServerEngine.create(ServerModule, WorkerModule){
         Fluent::Supervisor.load_config(@config_path, params)
       }
-      se.run
+
+      Dir.mktmpdir("fluentd-lock-") do |lockdir|
+        ENV['FLUENTD_LOCKDIR'] = lockdir
+        se.run
+      end
     end
 
     def install_main_process_signal_handlers