diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java index 93e803daa35a8..d33fc70407762 100644 --- a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java +++ b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java @@ -100,11 +100,16 @@ public long getTotalPhysicalMemorySize() { return 0; } try { - final long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean); + long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean); if (totalMem < 0) { logger.debug("OS reported a negative total memory value [{}]", totalMem); return 0; } + if (totalMem == 0 && isDebian8()) { + // workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654 + totalMem = getTotalMemFromProcMeminfo(); + } + return totalMem; } catch (Exception e) { logger.warn("exception retrieving total physical memory", e); @@ -641,6 +646,55 @@ List readOsRelease() throws IOException { } } + /** + * Returns the lines from /proc/meminfo as a workaround for JDK bugs that prevent retrieval of total system memory + * on some Linux variants such as Debian8. + */ + @SuppressForbidden(reason = "access /proc/meminfo") + List readProcMeminfo() throws IOException { + final List lines; + if (Files.exists(PathUtils.get("/proc/meminfo"))) { + lines = Files.readAllLines(PathUtils.get("/proc/meminfo")); + assert lines != null && lines.isEmpty() == false; + return lines; + } else { + return Collections.emptyList(); + } + } + + /** + * Retrieves system total memory in bytes from /proc/meminfo + */ + long getTotalMemFromProcMeminfo() throws IOException { + List meminfoLines = readProcMeminfo(); + final List memTotalLines = meminfoLines.stream().filter(line -> line.startsWith("MemTotal")).collect(Collectors.toList()); + assert memTotalLines.size() <= 1 : memTotalLines; + if (memTotalLines.size() == 1) { + final String memTotalLine = memTotalLines.get(0); + int beginIdx = memTotalLine.indexOf("MemTotal:"); + int endIdx = memTotalLine.lastIndexOf(" kB"); + if (beginIdx + 9 < endIdx) { + final String memTotalString = memTotalLine.substring(beginIdx + 9, endIdx).trim(); + try { + long memTotalInKb = Long.parseLong(memTotalString); + return memTotalInKb * 1024; + } catch (NumberFormatException e) { + logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]"); + return 0; + } + } else { + logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]"); + return 0; + } + } else { + return 0; + } + } + + boolean isDebian8() throws IOException { + return Constants.LINUX && getPrettyName().equals("Debian GNU/Linux 8 (jessie)"); + } + public OsStats osStats() { final OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage()); final OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize()); diff --git a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java index 9dd6ff7235fb1..c5b43ec240e48 100644 --- a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java +++ b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java @@ -255,6 +255,71 @@ public void testCgroupProbeWithMissingMemory() { assertNull(cgroup); } + public void testGetTotalMemFromProcMeminfo() throws Exception { + // missing MemTotal line + List meminfoLines = Arrays.asList( + "MemFree: 8467692 kB", + "MemAvailable: 39646240 kB", + "Buffers: 4699504 kB", + "Cached: 23290380 kB", + "SwapCached: 0 kB", + "Active: 43637908 kB", + "Inactive: 8130280 kB" + ); + OsProbe probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines); + assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L)); + + // MemTotal line with invalid value + meminfoLines = Arrays.asList( + "MemTotal: invalid kB", + "MemFree: 8467692 kB", + "MemAvailable: 39646240 kB", + "Buffers: 4699504 kB", + "Cached: 23290380 kB", + "SwapCached: 0 kB", + "Active: 43637908 kB", + "Inactive: 8130280 kB" + ); + probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines); + assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L)); + + // MemTotal line with invalid unit + meminfoLines = Arrays.asList( + "MemTotal: 39646240 MB", + "MemFree: 8467692 kB", + "MemAvailable: 39646240 kB", + "Buffers: 4699504 kB", + "Cached: 23290380 kB", + "SwapCached: 0 kB", + "Active: 43637908 kB", + "Inactive: 8130280 kB" + ); + probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines); + assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L)); + + // MemTotal line with random valid value + long memTotalInKb = randomLongBetween(1, Long.MAX_VALUE / 1024L); + meminfoLines = Arrays.asList( + "MemTotal: " + memTotalInKb + " kB", + "MemFree: 8467692 kB", + "MemAvailable: 39646240 kB", + "Buffers: 4699504 kB", + "Cached: 23290380 kB", + "SwapCached: 0 kB", + "Active: 43637908 kB", + "Inactive: 8130280 kB" + ); + probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines); + assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(memTotalInKb * 1024L)); + } + + public void testGetTotalMemoryOnDebian8() throws Exception { + // tests the workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654 + final OsProbe osProbe = new OsProbe(); + assumeTrue("runs only on Debian 8", osProbe.isDebian8()); + assertThat(osProbe.getTotalPhysicalMemorySize(), greaterThan(0L)); + } + private static List getProcSelfGroupLines(String hierarchy) { return Arrays.asList( "10:freezer:/", @@ -283,12 +348,14 @@ private static OsProbe buildStubOsProbe(final boolean areCgroupStatsAvailable, f * @param areCgroupStatsAvailable whether or not cgroup data is available. Normally OsProbe establishes this for itself. * @param hierarchy a mock value used to generate a cgroup hierarchy. * @param procSelfCgroupLines the lines that will be used as the content of /proc/self/cgroup + * @param procMeminfoLines lines that will be used as the content of /proc/meminfo * @return a test instance */ private static OsProbe buildStubOsProbe( final boolean areCgroupStatsAvailable, final String hierarchy, - List procSelfCgroupLines + List procSelfCgroupLines, + List procMeminfoLines ) { return new OsProbe() { @Override @@ -339,6 +406,20 @@ String readSysFsCgroupMemoryUsageInBytes(String controlGroup) { boolean areCgroupStatsAvailable() { return areCgroupStatsAvailable; } + + @Override + List readProcMeminfo() throws IOException { + return procMeminfoLines; + } }; } + + private static OsProbe buildStubOsProbe( + final boolean areCgroupStatsAvailable, + final String hierarchy, + List procSelfCgroupLines + ) { + return buildStubOsProbe(areCgroupStatsAvailable, hierarchy, procSelfCgroupLines, org.elasticsearch.common.collect.List.of()); + } + }