ES|QL: account for page overhead when calculating memory used by bloc…

…ks (#108347)
elastic · May 9, 2024 · c9b8d72 · c9b8d72
1 parent 155e7c5
commit c9b8d72
Show file tree

Hide file tree

Showing 9 changed files with 56 additions and 46 deletions.
diff --git a/...p-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/...p-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
@@ -269,7 +269,6 @@ public void testManyEval() throws IOException {
         assertMap(map, matchesMap().entry("columns", columns).entry("values", hasSize(10_000)));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108104")
     public void testTooManyEval() throws IOException {
         initManyLongs();
         assertCircuitBreaks(() -> manyEval(490));

diff --git a/...sql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java b/...sql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java
@@ -23,7 +23,9 @@ final class BooleanArrayVector extends AbstractVector implements BooleanVector {
 
     static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanArrayVector.class)
         // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
-        + RamUsageEstimator.shallowSizeOfInstance(BooleanVectorBlock.class);
+        + RamUsageEstimator.shallowSizeOfInstance(BooleanVectorBlock.class)
+        // TODO: remove this if/when we account for memory used by Pages
+        + Block.PAGE_MEM_OVERHEAD_PER_BLOCK;
 
     private final boolean[] values;
 

diff --git a/...ql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java b/...ql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java
@@ -25,7 +25,9 @@ final class BytesRefArrayVector extends AbstractVector implements BytesRefVector
 
     static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefArrayVector.class)
         // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
-        + RamUsageEstimator.shallowSizeOfInstance(BytesRefVectorBlock.class);
+        + RamUsageEstimator.shallowSizeOfInstance(BytesRefVectorBlock.class)
+        // TODO: remove this if/when we account for memory used by Pages
+        + Block.PAGE_MEM_OVERHEAD_PER_BLOCK;
 
     private final BytesRefArray values;
 

diff --git a/...esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java b/...esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java
@@ -23,7 +23,9 @@ final class DoubleArrayVector extends AbstractVector implements DoubleVector {
 
     static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DoubleArrayVector.class)
         // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
-        + RamUsageEstimator.shallowSizeOfInstance(DoubleVectorBlock.class);
+        + RamUsageEstimator.shallowSizeOfInstance(DoubleVectorBlock.class)
+        // TODO: remove this if/when we account for memory used by Pages
+        + Block.PAGE_MEM_OVERHEAD_PER_BLOCK;
 
     private final double[] values;
 

diff --git a/...in/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java b/...in/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java
@@ -23,7 +23,9 @@ final class IntArrayVector extends AbstractVector implements IntVector {
 
     static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IntArrayVector.class)
         // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
-        + RamUsageEstimator.shallowSizeOfInstance(IntVectorBlock.class);
+        + RamUsageEstimator.shallowSizeOfInstance(IntVectorBlock.class)
+        // TODO: remove this if/when we account for memory used by Pages
+        + Block.PAGE_MEM_OVERHEAD_PER_BLOCK;
 
     private final int[] values;
 

diff --git a/...n/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java b/...n/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java
@@ -23,7 +23,9 @@ final class LongArrayVector extends AbstractVector implements LongVector {
 
     static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LongArrayVector.class)
         // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
-        + RamUsageEstimator.shallowSizeOfInstance(LongVectorBlock.class);
+        + RamUsageEstimator.shallowSizeOfInstance(LongVectorBlock.class)
+        // TODO: remove this if/when we account for memory used by Pages
+        + Block.PAGE_MEM_OVERHEAD_PER_BLOCK;
 
     private final long[] values;
 

diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.compute.data;
 
 import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -44,6 +45,17 @@ public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, R
      */
     long MAX_LOOKUP = 100_000;
 
+    /**
+     * We do not track memory for pages directly (only for single blocks),
+     * but the page memory overhead can still be significant, especially for pages containing thousands of blocks.
+     * For now, we approximate this overhead, per block, using this value.
+     *
+     * The exact overhead per block would be (more correctly) {@link RamUsageEstimator#NUM_BYTES_OBJECT_REF},
+     * but we approximate it with {@link RamUsageEstimator#NUM_BYTES_OBJECT_ALIGNMENT} to avoid further alignments
+     * to object size (at the end of the alignment, it would make no practical difference).
+     */
+    int PAGE_MEM_OVERHEAD_PER_BLOCK = RamUsageEstimator.NUM_BYTES_OBJECT_ALIGNMENT;
+
     /**
      * {@return an efficient dense single-value view of this block}.
      * Null, if the block is not dense single-valued. That is, if

diff --git a/...ck/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st b/...ck/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st
@@ -38,7 +38,9 @@ final class $Type$ArrayVector extends AbstractVector implements $Type$Vector {
 
     static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance($Type$ArrayVector.class)
         // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
-        + RamUsageEstimator.shallowSizeOfInstance($Type$VectorBlock.class);
+        + RamUsageEstimator.shallowSizeOfInstance($Type$VectorBlock.class)
+        // TODO: remove this if/when we account for memory used by Pages
+        + Block.PAGE_MEM_OVERHEAD_PER_BLOCK;
 
 $if(BytesRef)$
     private final BytesRefArray values;

diff --git a/...lugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java b/...lugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java
@@ -42,9 +42,8 @@ public class BlockAccountingTests extends ComputeTestCase {
     public void testBooleanVector() {
         BlockFactory blockFactory = blockFactory();
         Vector empty = blockFactory.newBooleanArrayVector(new boolean[] {}, 0);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            BooleanVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(BooleanVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Vector emptyPlusOne = blockFactory.newBooleanArrayVector(new boolean[] { randomBoolean() }, 1);
@@ -62,9 +61,8 @@ public void testBooleanVector() {
     public void testIntVector() {
         BlockFactory blockFactory = blockFactory();
         Vector empty = blockFactory.newIntArrayVector(new int[] {}, 0);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            IntVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(IntVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Vector emptyPlusOne = blockFactory.newIntArrayVector(new int[] { randomInt() }, 1);
@@ -82,9 +80,8 @@ public void testIntVector() {
     public void testLongVector() {
         BlockFactory blockFactory = blockFactory();
         Vector empty = blockFactory.newLongArrayVector(new long[] {}, 0);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            LongVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(LongVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Vector emptyPlusOne = blockFactory.newLongArrayVector(new long[] { randomLong() }, 1);
@@ -103,9 +100,8 @@ public void testLongVector() {
     public void testDoubleVector() {
         BlockFactory blockFactory = blockFactory();
         Vector empty = blockFactory.newDoubleArrayVector(new double[] {}, 0);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            DoubleVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(DoubleVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Vector emptyPlusOne = blockFactory.newDoubleArrayVector(new double[] { randomDouble() }, 1);
@@ -127,9 +123,8 @@ public void testBytesRefVector() {
         var emptyArray = new BytesRefArray(0, blockFactory.bigArrays());
         var arrayWithOne = new BytesRefArray(0, blockFactory.bigArrays());
         Vector emptyVector = blockFactory.newBytesRefArrayVector(emptyArray, 0);
-        long expectedEmptyVectorUsed = RamUsageTester.ramUsed(emptyVector, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            BytesRefVectorBlock.class
-        );
+        long expectedEmptyVectorUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(emptyVector, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(BytesRefVectorBlock.class);
         assertThat(emptyVector.ramBytesUsed(), is(expectedEmptyVectorUsed));
 
         var bytesRef = new BytesRef(randomAlphaOfLengthBetween(1, 16));
@@ -146,9 +141,8 @@ public void testBytesRefVector() {
     public void testBooleanBlock() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new BooleanArrayBlock(new boolean[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            BooleanVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(BooleanVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Block emptyPlusOne = new BooleanArrayBlock(
@@ -194,18 +188,16 @@ public void testBooleanBlockWithNullFirstValues() {
             Block.MvOrdering.UNORDERED,
             blockFactory()
         );
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            BooleanVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(BooleanVectorBlock.class);
         assertThat(empty.ramBytesUsed(), lessThanOrEqualTo(expectedEmptyUsed));
     }
 
     public void testIntBlock() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new IntArrayBlock(new int[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            IntVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(IntVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Block emptyPlusOne = new IntArrayBlock(
@@ -242,18 +234,16 @@ public void testIntBlock() {
     public void testIntBlockWithNullFirstValues() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new IntArrayBlock(new int[] {}, 0, null, BitSet.valueOf(new byte[] { 1 }), Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            IntVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(IntVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
     }
 
     public void testLongBlock() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new LongArrayBlock(new long[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            LongVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(LongVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Block emptyPlusOne = new LongArrayBlock(
@@ -299,18 +289,16 @@ public void testLongBlockWithNullFirstValues() {
             Block.MvOrdering.UNORDERED,
             blockFactory()
         );
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            LongVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(LongVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
     }
 
     public void testDoubleBlock() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new DoubleArrayBlock(new double[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            DoubleVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(DoubleVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Block emptyPlusOne = new DoubleArrayBlock(
@@ -356,9 +344,8 @@ public void testDoubleBlockWithNullFirstValues() {
             Block.MvOrdering.UNORDERED,
             blockFactory()
         );
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
-            DoubleVectorBlock.class
-        );
+        long expectedEmptyUsed = Block.PAGE_MEM_OVERHEAD_PER_BLOCK + RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR)
+            + RamUsageEstimator.shallowSizeOfInstance(DoubleVectorBlock.class);
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
     }