From 29c5abc874918aede3bef24df2f571339ee6bf5d Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 13 Jan 2021 16:18:28 +0000 Subject: [PATCH] Extend the selective muting of memory tests on Debian 8 (#67453) The selective muting implemented for autoscaling in #67159 is extended to the ML tests that also fail when machine memory is reported as 0. Most of the logic to determine when memory will not be accurately reported is now in a utility method in the base class. Relates #66885 Backport of #67422 --- .../elasticsearch/test/ESIntegTestCase.java | 20 +++++++++++++++++++ ...ansportGetAutoscalingCapacityActionIT.java | 10 +--------- .../integration/BasicDistributedJobsIT.java | 4 ++++ .../integration/MlDistributedFailureIT.java | 3 +++ .../xpack/ml/integration/TooManyJobsIT.java | 4 ++++ 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 2d3a5c28a76fc..9abd7fd064827 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -57,6 +57,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.client.ClusterAdminClient; @@ -132,6 +133,7 @@ import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.ingest.IngestMetadata; +import org.elasticsearch.monitor.os.OsInfo; import org.elasticsearch.node.NodeMocksPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; @@ -2411,4 +2413,22 @@ public static String resolveCustomDataPath(String index) { public static boolean inFipsJvm() { return Boolean.parseBoolean(System.getProperty(FIPS_SYSPROP)); } + + /** + * On Debian 8 the "memory" subsystem is not mounted by default + * when cgroups are enabled, and this confuses many versions of + * Java prior to Java 15. Tests that rely on machine memory + * being accurately determined will not work on such setups, + * and can use this method for selective muting. + * See https://github.com/elastic/elasticsearch/issues/67089 + * and https://github.com/elastic/elasticsearch/issues/66885 + */ + protected boolean willSufferDebian8MemoryProblem() { + final NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet(); + final boolean anyDebian8Nodes = response.getNodes() + .stream() + .anyMatch(ni -> ni.getInfo(OsInfo.class).getPrettyName().equals("Debian GNU/Linux 8 (jessie)")); + boolean java15Plus = JavaVersion.current().compareTo(JavaVersion.parse("15")) >= 0; + return anyDebian8Nodes && java15Plus == false; + } } diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityActionIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityActionIT.java index 5a4dd40f2f08b..ad7de56752181 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityActionIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityActionIT.java @@ -6,10 +6,7 @@ package org.elasticsearch.xpack.autoscaling.action; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.monitor.os.OsInfo; import org.elasticsearch.monitor.os.OsProbe; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.autoscaling.AutoscalingIntegTestCase; @@ -26,13 +23,8 @@ public class TransportGetAutoscalingCapacityActionIT extends AutoscalingIntegTestCase { public void testCurrentCapacity() throws Exception { - final NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet(); - final boolean anyDebian8Nodes = response.getNodes() - .stream() - .anyMatch(ni -> ni.getInfo(OsInfo.class).getPrettyName().equals("Debian GNU/Linux 8 (jessie)")); - boolean java15Plus = JavaVersion.current().compareTo(JavaVersion.parse("15")) >= 0; // see: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654 - assumeTrue("cannot run on debian 8 prior to java 15", java15Plus || anyDebian8Nodes == false); + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); assertThat(capacity().results().keySet(), Matchers.empty()); long memory = OsProbe.getInstance().getTotalPhysicalMemorySize(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index eab3ade5c202a..03bb86f417bf2 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -423,6 +423,10 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception { } public void testCloseUnassignedLazyJobAndDatafeed() throws Exception { + + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); + internalCluster().ensureAtLeastNumDataNodes(3); ensureStableCluster(3); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index fa98bd74b427c..1ac229027f01b 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -408,6 +408,9 @@ public void testStopAndForceStopDatafeed() throws Exception { public void testJobRelocationIsMemoryAware() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); + internalCluster().ensureAtLeastNumDataNodes(1); ensureStableCluster(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index 7e1f4c845f90a..14e499117dbc1 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -126,10 +126,14 @@ public void testLazyNodeValidation() throws Exception { } public void testSingleNode() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); verifyMaxNumberOfJobsLimit(1, randomIntBetween(1, 20), randomBoolean()); } public void testMultipleNodes() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); verifyMaxNumberOfJobsLimit(3, randomIntBetween(1, 20), randomBoolean()); }