Skip to content

Commit

Permalink
[7.11] Workaround for JDK bug with total mem on Debian8 (#68542) (#68699
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@danhermann
danhermann committed Feb 8, 2021
1 parent c8e12ec commit 8a20ebf
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 2 deletions.
56 changes: 55 additions & 1 deletion server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
View file
Expand Up @@ -100,11 +100,16 @@ public long getTotalPhysicalMemorySize() {
return 0;
}
try {
final long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean);
long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean);
if (totalMem < 0) {
logger.debug("OS reported a negative total memory value [{}]", totalMem);
return 0;
}
if (totalMem == 0 && isDebian8()) {
// workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654
totalMem = getTotalMemFromProcMeminfo();
}

return totalMem;
} catch (Exception e) {
logger.warn("exception retrieving total physical memory", e);
Expand Down Expand Up @@ -641,6 +646,55 @@ List<String> readOsRelease() throws IOException {
}
}

/**
* Returns the lines from /proc/meminfo as a workaround for JDK bugs that prevent retrieval of total system memory
* on some Linux variants such as Debian8.
*/
@SuppressForbidden(reason = "access /proc/meminfo")
List<String> readProcMeminfo() throws IOException {
final List<String> lines;
if (Files.exists(PathUtils.get("/proc/meminfo"))) {
lines = Files.readAllLines(PathUtils.get("/proc/meminfo"));
assert lines != null && lines.isEmpty() == false;
return lines;
} else {
return Collections.emptyList();
}
}

/**
* Retrieves system total memory in bytes from /proc/meminfo
*/
long getTotalMemFromProcMeminfo() throws IOException {
List<String> meminfoLines = readProcMeminfo();
final List<String> memTotalLines = meminfoLines.stream().filter(line -> line.startsWith("MemTotal")).collect(Collectors.toList());
assert memTotalLines.size() <= 1 : memTotalLines;
if (memTotalLines.size() == 1) {
final String memTotalLine = memTotalLines.get(0);
int beginIdx = memTotalLine.indexOf("MemTotal:");
int endIdx = memTotalLine.lastIndexOf(" kB");
if (beginIdx + 9 < endIdx) {
final String memTotalString = memTotalLine.substring(beginIdx + 9, endIdx).trim();
try {
long memTotalInKb = Long.parseLong(memTotalString);
return memTotalInKb * 1024;
} catch (NumberFormatException e) {
logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]");
return 0;
}
} else {
logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]");
return 0;
}
} else {
return 0;
}
}

boolean isDebian8() throws IOException {
return Constants.LINUX && getPrettyName().equals("Debian GNU/Linux 8 (jessie)");
}

public OsStats osStats() {
final OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage());
final OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize());
Expand Down
83 changes: 82 additions & 1 deletion server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
View file
Expand Up @@ -255,6 +255,71 @@ public void testCgroupProbeWithMissingMemory() {
assertNull(cgroup);
}

public void testGetTotalMemFromProcMeminfo() throws Exception {
// missing MemTotal line
List<String> meminfoLines = Arrays.asList(
"MemFree: 8467692 kB",
"MemAvailable: 39646240 kB",
"Buffers: 4699504 kB",
"Cached: 23290380 kB",
"SwapCached: 0 kB",
"Active: 43637908 kB",
"Inactive: 8130280 kB"
);
OsProbe probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));

// MemTotal line with invalid value
meminfoLines = Arrays.asList(
"MemTotal: invalid kB",
"MemFree: 8467692 kB",
"MemAvailable: 39646240 kB",
"Buffers: 4699504 kB",
"Cached: 23290380 kB",
"SwapCached: 0 kB",
"Active: 43637908 kB",
"Inactive: 8130280 kB"
);
probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));

// MemTotal line with invalid unit
meminfoLines = Arrays.asList(
"MemTotal: 39646240 MB",
"MemFree: 8467692 kB",
"MemAvailable: 39646240 kB",
"Buffers: 4699504 kB",
"Cached: 23290380 kB",
"SwapCached: 0 kB",
"Active: 43637908 kB",
"Inactive: 8130280 kB"
);
probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));

// MemTotal line with random valid value
long memTotalInKb = randomLongBetween(1, Long.MAX_VALUE / 1024L);
meminfoLines = Arrays.asList(
"MemTotal: " + memTotalInKb + " kB",
"MemFree: 8467692 kB",
"MemAvailable: 39646240 kB",
"Buffers: 4699504 kB",
"Cached: 23290380 kB",
"SwapCached: 0 kB",
"Active: 43637908 kB",
"Inactive: 8130280 kB"
);
probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(memTotalInKb * 1024L));
}

public void testGetTotalMemoryOnDebian8() throws Exception {
// tests the workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654
final OsProbe osProbe = new OsProbe();
assumeTrue("runs only on Debian 8", osProbe.isDebian8());
assertThat(osProbe.getTotalPhysicalMemorySize(), greaterThan(0L));
}

private static List<String> getProcSelfGroupLines(String hierarchy) {
return Arrays.asList(
"10:freezer:/",
Expand Down Expand Up @@ -283,12 +348,14 @@ private static OsProbe buildStubOsProbe(final boolean areCgroupStatsAvailable, f
* @param areCgroupStatsAvailable whether or not cgroup data is available. Normally OsProbe establishes this for itself.
* @param hierarchy a mock value used to generate a cgroup hierarchy.
* @param procSelfCgroupLines the lines that will be used as the content of <code>/proc/self/cgroup</code>
* @param procMeminfoLines lines that will be used as the content of <code>/proc/meminfo</code>
* @return a test instance
*/
private static OsProbe buildStubOsProbe(
final boolean areCgroupStatsAvailable,
final String hierarchy,
List<String> procSelfCgroupLines
List<String> procSelfCgroupLines,
List<String> procMeminfoLines
) {
return new OsProbe() {
@Override
Expand Down Expand Up @@ -339,6 +406,20 @@ String readSysFsCgroupMemoryUsageInBytes(String controlGroup) {
boolean areCgroupStatsAvailable() {
return areCgroupStatsAvailable;
}

@Override
List<String> readProcMeminfo() throws IOException {
return procMeminfoLines;
}
};
}

private static OsProbe buildStubOsProbe(
final boolean areCgroupStatsAvailable,
final String hierarchy,
List<String> procSelfCgroupLines
) {
return buildStubOsProbe(areCgroupStatsAvailable, hierarchy, procSelfCgroupLines, org.elasticsearch.common.collect.List.of());
}

}

0 comments on commit 8a20ebf

Please sign in to comment.