Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[7.11] Workaround for JDK bug with total mem on Debian8 (#68542) #68699

Merged
merged 2 commits into from Feb 8, 2021
Merged

[7.11] Workaround for JDK bug with total mem on Debian8 (#68542) #68699

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ac2516
Workaround for JDK bug with total mem on Debian8 (#68542)
danhermann Feb 8, 2021
4e1f352
java8
danhermann Feb 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
56 changes: 55 additions & 1 deletion server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
View file
Open in desktop
Expand Up @@ -100,11 +100,16 @@ public long getTotalPhysicalMemorySize() {
return 0;
}
try {
final long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean);
long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean);
if (totalMem < 0) {
logger.debug("OS reported a negative total memory value [{}]", totalMem);
return 0;
}
if (totalMem == 0 && isDebian8()) {
// workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654
totalMem = getTotalMemFromProcMeminfo();
}

return totalMem;
} catch (Exception e) {
logger.warn("exception retrieving total physical memory", e);
Expand Down Expand Up @@ -641,6 +646,55 @@ List<String> readOsRelease() throws IOException {
}
}

/**
* Returns the lines from /proc/meminfo as a workaround for JDK bugs that prevent retrieval of total system memory
* on some Linux variants such as Debian8.
*/
@SuppressForbidden(reason = "access /proc/meminfo")
List<String> readProcMeminfo() throws IOException {
final List<String> lines;
if (Files.exists(PathUtils.get("/proc/meminfo"))) {
lines = Files.readAllLines(PathUtils.get("/proc/meminfo"));
assert lines != null && lines.isEmpty() == false;
return lines;
} else {
return Collections.emptyList();
}
}

/**
* Retrieves system total memory in bytes from /proc/meminfo
*/
long getTotalMemFromProcMeminfo() throws IOException {
List<String> meminfoLines = readProcMeminfo();
final List<String> memTotalLines = meminfoLines.stream().filter(line -> line.startsWith("MemTotal")).collect(Collectors.toList());
assert memTotalLines.size() <= 1 : memTotalLines;
if (memTotalLines.size() == 1) {
final String memTotalLine = memTotalLines.get(0);
int beginIdx = memTotalLine.indexOf("MemTotal:");
int endIdx = memTotalLine.lastIndexOf(" kB");
if (beginIdx + 9 < endIdx) {
final String memTotalString = memTotalLine.substring(beginIdx + 9, endIdx).trim();
try {
long memTotalInKb = Long.parseLong(memTotalString);
return memTotalInKb * 1024;
} catch (NumberFormatException e) {
logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]");
return 0;
}
} else {
logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]");
return 0;
}
} else {
return 0;
}
}

boolean isDebian8() throws IOException {
return Constants.LINUX && getPrettyName().equals("Debian GNU/Linux 8 (jessie)");
}

public OsStats osStats() {
final OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage());
final OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize());
Expand Down
83 changes: 82 additions & 1 deletion server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
View file
Open in desktop
Expand Up @@ -255,6 +255,71 @@ public void testCgroupProbeWithMissingMemory() {
assertNull(cgroup);
}

public void testGetTotalMemFromProcMeminfo() throws Exception {
// missing MemTotal line
List<String> meminfoLines = Arrays.asList(
"MemFree: 8467692 kB",
"MemAvailable: 39646240 kB",
"Buffers: 4699504 kB",
"Cached: 23290380 kB",
"SwapCached: 0 kB",
"Active: 43637908 kB",
"Inactive: 8130280 kB"
);
OsProbe probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));

// MemTotal line with invalid value
meminfoLines = Arrays.asList(
"MemTotal: invalid kB",
"MemFree: 8467692 kB",
"MemAvailable: 39646240 kB",
"Buffers: 4699504 kB",
"Cached: 23290380 kB",
"SwapCached: 0 kB",
"Active: 43637908 kB",
"Inactive: 8130280 kB"
);
probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));

// MemTotal line with invalid unit
meminfoLines = Arrays.asList(
"MemTotal: 39646240 MB",
"MemFree: 8467692 kB",
"MemAvailable: 39646240 kB",
"Buffers: 4699504 kB",
"Cached: 23290380 kB",
"SwapCached: 0 kB",
"Active: 43637908 kB",
"Inactive: 8130280 kB"
);
probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));

// MemTotal line with random valid value
long memTotalInKb = randomLongBetween(1, Long.MAX_VALUE / 1024L);
meminfoLines = Arrays.asList(
"MemTotal: " + memTotalInKb + " kB",
"MemFree: 8467692 kB",
"MemAvailable: 39646240 kB",
"Buffers: 4699504 kB",
"Cached: 23290380 kB",
"SwapCached: 0 kB",
"Active: 43637908 kB",
"Inactive: 8130280 kB"
);
probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(memTotalInKb * 1024L));
}

public void testGetTotalMemoryOnDebian8() throws Exception {
// tests the workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654
final OsProbe osProbe = new OsProbe();
assumeTrue("runs only on Debian 8", osProbe.isDebian8());
assertThat(osProbe.getTotalPhysicalMemorySize(), greaterThan(0L));
}

private static List<String> getProcSelfGroupLines(String hierarchy) {
return Arrays.asList(
"10:freezer:/",
Expand Down Expand Up @@ -283,12 +348,14 @@ private static OsProbe buildStubOsProbe(final boolean areCgroupStatsAvailable, f
* @param areCgroupStatsAvailable whether or not cgroup data is available. Normally OsProbe establishes this for itself.
* @param hierarchy a mock value used to generate a cgroup hierarchy.
* @param procSelfCgroupLines the lines that will be used as the content of <code>/proc/self/cgroup</code>
* @param procMeminfoLines lines that will be used as the content of <code>/proc/meminfo</code>
* @return a test instance
*/
private static OsProbe buildStubOsProbe(
final boolean areCgroupStatsAvailable,
final String hierarchy,
List<String> procSelfCgroupLines
List<String> procSelfCgroupLines,
List<String> procMeminfoLines
) {
return new OsProbe() {
@Override
Expand Down Expand Up @@ -339,6 +406,20 @@ String readSysFsCgroupMemoryUsageInBytes(String controlGroup) {
boolean areCgroupStatsAvailable() {
return areCgroupStatsAvailable;
}

@Override
List<String> readProcMeminfo() throws IOException {
return procMeminfoLines;
}
};
}

private static OsProbe buildStubOsProbe(
final boolean areCgroupStatsAvailable,
final String hierarchy,
List<String> procSelfCgroupLines
) {
return buildStubOsProbe(areCgroupStatsAvailable, hierarchy, procSelfCgroupLines, org.elasticsearch.common.collect.List.of());
}

}