Mock empty state task in file settings tests (#108100)

When the file watched by file settings is initially missing, a special
method in reserved state service is called to write a dummy cluster
state entry. In the case of tests, there is no real running master
service, so when the task is submitted, the file watcher thread actually
barfs and the watcher dies, silently. That then causes the test to
timeout as it waits indefinitely but the file watcher is no longer
watching for the test file that was written.

This commit mocks out writing this empty state in the reserved state
service. It also collapses the two tests that check stopping while
blocked in processing works since they were almost exactly the same.

closes #106968

server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java

@@ -8,8 +8,8 @@
 
 package org.elasticsearch.reservedstate.service;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -55,7 +55,6 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106968")
 public class FileSettingsServiceTests extends ESTestCase {
     private Environment env;
     private ClusterService clusterService;
@@ -234,54 +233,11 @@ public void testStopWorksInMiddleOfProcessing() throws Exception {
             return new ReservedStateChunk(Collections.emptyMap(), new ReservedStateVersion(1L, Version.CURRENT));
         }).when(spiedController).parse(any(String.class), any());
 
-        service.start();
-        service.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE));
-        assertTrue(service.watching());
-
-        Files.createDirectories(service.watchedFileDir());
-
-        // Make some fake settings file to cause the file settings service to process it
-        writeTestFile(service.watchedFile(), "{}");
-
-        // we need to wait a bit, on MacOS it may take up to 10 seconds for the Java watcher service to notice the file,
-        // on Linux is instantaneous. Windows is instantaneous too.
-        assertTrue(processFileLatch.await(30, TimeUnit.SECONDS));
-
-        // Stopping the service should interrupt the watcher thread, we should be able to stop
-        service.stop();
-        assertFalse(service.watching());
-        service.close();
-        // let the deadlocked thread end, so we can cleanly exit the test
-        deadThreadLatch.countDown();
-    }
-
-    public void testStopWorksIfProcessingDidntReturnYet() throws Exception {
-        var spiedController = spy(controller);
-        var service = new FileSettingsService(clusterService, spiedController, env);
-
-        CountDownLatch processFileLatch = new CountDownLatch(1);
-        CountDownLatch deadThreadLatch = new CountDownLatch(1);
-
-        doAnswer((Answer<ReservedStateChunk>) invocation -> {
-            // allow the other thread to continue, but hold on a bit to avoid
-            // completing the task immediately in the main watcher loop
-            try {
-                Thread.sleep(1_000);
-            } catch (InterruptedException e) {
-                // pass it on
-                Thread.currentThread().interrupt();
-            }
-            processFileLatch.countDown();
-            new Thread(() -> {
-                // Simulate a thread that never allows the completion to complete
-                try {
-                    deadThreadLatch.await();
-                } catch (InterruptedException e) {
-                    throw new RuntimeException(e);
-                }
-            }).start();
-            return new ReservedStateChunk(Collections.emptyMap(), new ReservedStateVersion(1L, Version.CURRENT));
-        }).when(spiedController).parse(any(String.class), any());
+        doAnswer((Answer<Void>) invocation -> {
+            var completionListener = invocation.getArgument(1, ActionListener.class);
+            completionListener.onResponse(null);
+            return null;
+        }).when(spiedController).initEmpty(any(String.class), any());
 
         service.start();
         service.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE));
@@ -296,7 +252,7 @@ public void testStopWorksIfProcessingDidntReturnYet() throws Exception {
         // on Linux is instantaneous. Windows is instantaneous too.
         assertTrue(processFileLatch.await(30, TimeUnit.SECONDS));
 
-        // Stopping the service should interrupt the watcher thread, allowing the whole thing to exit
+        // Stopping the service should interrupt the watcher thread, we should be able to stop
         service.stop();
         assertFalse(service.watching());
         service.close();