[ML] Make two tests work regardless of failure store feature flag

Two ML unit tests make assertions that are sensitive to whether
the new failure store functionality is available or not. Since
this functionality is still in a state of flux, accept error
messages either with or without the new parts for now.

(Even if the old text was left as an option forever it wouldn't
really harm the ability of the tests to detect regressions as
it's virtually impossible that some other bug would cause the
text to go back to exactly what it was before.)

Fixes elastic#106107
Fixes elastic#106108

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java

@@ -56,6 +56,7 @@
 import static org.elasticsearch.xpack.ml.job.task.OpenJobPersistentTasksExecutorTests.addJobTask;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createDatafeed;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -313,7 +314,6 @@ public void testShardNotAllActive() {
             .checkDatafeedTaskCanBeCreated();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106107")
     public void testIndexDoesntExist() {
         Job job = createScheduledJob("job_id").build(new Date());
         DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("not_foo"));
@@ -335,12 +335,22 @@ public void testIndexDoesntExist() {
         assertNull(result.getExecutorNode());
         assertThat(
             result.getExplanation(),
-            equalTo(
-                "cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and "
-                    + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
-                    + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
-                    + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, include_regular_indices=true, "
-                    + "include_failure_indices=false, allow_failure_indices=true]] with exception [no such index [not_foo]]"
+            anyOf(
+                // TODO remove this first option and only allow the second once the failure store functionality is permanently switched on
+                equalTo(
+                    "cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and "
+                        + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
+                        + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
+                        + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true]] "
+                        + "with exception [no such index [not_foo]]"
+                ),
+                equalTo(
+                    "cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and "
+                        + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
+                        + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
+                        + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, include_regular_indices=true, "
+                        + "include_failure_indices=false, allow_failure_indices=true]] with exception [no such index [not_foo]]"
+                )
             )
         );
 
@@ -357,14 +367,25 @@ public void testIndexDoesntExist() {
         );
         assertThat(
             e.getMessage(),
-            containsString(
-                "No node found to start datafeed [datafeed_id], allocation explanation "
-                    + "[cannot start datafeed [datafeed_id] because it failed resolving "
-                    + "indices given [not_foo] and indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, "
-                    + "expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, "
-                    + "allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, "
-                    + "include_regular_indices=true, include_failure_indices=false, allow_failure_indices=true]] "
-                    + "with exception [no such index [not_foo]]]"
+            anyOf(
+                // TODO remove this first option and only allow the second once the failure store functionality is permanently switched on
+                containsString(
+                    "No node found to start datafeed [datafeed_id], allocation explanation "
+                        + "[cannot start datafeed [datafeed_id] because it failed resolving "
+                        + "indices given [not_foo] and indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, "
+                        + "expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, "
+                        + "allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true"
+                        + "]] with exception [no such index [not_foo]]]"
+                ),
+                containsString(
+                    "No node found to start datafeed [datafeed_id], allocation explanation "
+                        + "[cannot start datafeed [datafeed_id] because it failed resolving "
+                        + "indices given [not_foo] and indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, "
+                        + "expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, "
+                        + "allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, "
+                        + "ignore_throttled=true, include_regular_indices=true, include_failure_indices=false, "
+                        + "allow_failure_indices=true]] with exception [no such index [not_foo]]]"
+                )
             )
         );
     }
@@ -498,7 +519,6 @@ public void testSelectNode_jobTaskStale() {
             .checkDatafeedTaskCanBeCreated();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106108")
     public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() {
         // Here we test that when there are 2 problems, the most critical gets reported first.
         // In this case job is Opening (non-critical) and the index does not exist (critical)
@@ -525,13 +545,24 @@ public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() {
         );
         assertThat(
             e.getMessage(),
-            containsString(
-                "No node found to start datafeed [datafeed_id], allocation explanation "
-                    + "[cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and "
-                    + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
-                    + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
-                    + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, include_regular_indices=true, "
-                    + "include_failure_indices=false, allow_failure_indices=true]] with exception [no such index [not_foo]]]"
+            anyOf(
+                // TODO remove this first option and only allow the second once the failure store functionality is permanently switched on
+                containsString(
+                    "No node found to start datafeed [datafeed_id], allocation explanation "
+                        + "[cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and "
+                        + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
+                        + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
+                        + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true]] "
+                        + "with exception [no such index [not_foo]]]"
+                ),
+                containsString(
+                    "No node found to start datafeed [datafeed_id], allocation explanation "
+                        + "[cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and "
+                        + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
+                        + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
+                        + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, include_regular_indices=true, "
+                        + "include_failure_indices=false, allow_failure_indices=true]] with exception [no such index [not_foo]]]"
+                )
             )
         );
     }