From d280e303bd0b2c42d62ae47093cbe6f392ad558b Mon Sep 17 00:00:00 2001 From: Andrei Tokar Date: Mon, 19 Feb 2024 16:46:06 -0500 Subject: [PATCH 1/3] fix for #3909 --- h2/src/main/org/h2/mvstore/FileStore.java | 7 ++-- .../main/org/h2/mvstore/FreeSpaceBitSet.java | 6 ++-- .../org/h2/mvstore/RandomAccessStore.java | 34 +++++++++++-------- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/h2/src/main/org/h2/mvstore/FileStore.java b/h2/src/main/org/h2/mvstore/FileStore.java index 957ed056c4..0194e7c928 100644 --- a/h2/src/main/org/h2/mvstore/FileStore.java +++ b/h2/src/main/org/h2/mvstore/FileStore.java @@ -423,7 +423,7 @@ public final void setAutoCommitDelay(int millis) { stopBackgroundThread(millis >= 0); // start the background thread if needed if (millis > 0 && mvStore.isOpen()) { - int sleep = Math.max(1, millis / 10); + int sleep = Math.max(1, millis / 5); BackgroundWriterThread t = new BackgroundWriterThread(this, sleep, toString()); if (backgroundWriterThread.compareAndSet(null, t)) { t.start(); @@ -622,7 +622,7 @@ protected final boolean hasPersistentData() { } protected final boolean isIdle() { - return autoCompactLastFileOpCount == getWriteCount() + getReadCount(); + return autoCompactLastFileOpCount >= getWriteCount() + getReadCount(); } protected final void setLastChunk(C last) { @@ -1835,7 +1835,8 @@ void writeInBackground() { mvStore.tryCommit(); } doHousekeeping(mvStore); - autoCompactLastFileOpCount = getWriteCount() + getReadCount(); + // less than 10 I/O operations will still count as "idle" + autoCompactLastFileOpCount = getWriteCount() + getReadCount() + 10; } } catch (InterruptedException ignore) { } catch (Throwable e) { diff --git a/h2/src/main/org/h2/mvstore/FreeSpaceBitSet.java b/h2/src/main/org/h2/mvstore/FreeSpaceBitSet.java index f0fff58cc1..23fd5512bb 100644 --- a/h2/src/main/org/h2/mvstore/FreeSpaceBitSet.java +++ b/h2/src/main/org/h2/mvstore/FreeSpaceBitSet.java @@ -145,10 +145,8 @@ private int allocate(int blocks, int reservedLow, int reservedHigh, boolean allo int freeBlocks = end - start; if (end < 0 || freeBlocks >= blocks) { if ((reservedHigh < 0 || start < reservedHigh) && start + blocks > reservedLow) { // overlap detected - if (reservedHigh < 0) { - start = getAfterLastBlock(); - end = -1; - } else { + if (reservedHigh >= 0) { + freeBlocksTotal += freeBlocks; i = reservedHigh; continue; } diff --git a/h2/src/main/org/h2/mvstore/RandomAccessStore.java b/h2/src/main/org/h2/mvstore/RandomAccessStore.java index e69737f45d..148d801fa9 100644 --- a/h2/src/main/org/h2/mvstore/RandomAccessStore.java +++ b/h2/src/main/org/h2/mvstore/RandomAccessStore.java @@ -44,6 +44,7 @@ public abstract class RandomAccessStore extends FileStore private long reservedLow; private long reservedHigh; private boolean stopIdleHousekeeping; + private int restoreHousekeepingAtRate; public RandomAccessStore(Map config) { super(config); @@ -703,15 +704,15 @@ private void shrinkIfPossible(int minPercent) { @Override protected void doHousekeeping(MVStore mvStore) throws InterruptedException { boolean idle = isIdle(); - if (idle && stopIdleHousekeeping) { + int rewritableChunksFillRate = getRewritableChunksFillRate(); + if (idle && stopIdleHousekeeping && rewritableChunksFillRate > restoreHousekeepingAtRate) { return; } int autoCommitMemory = mvStore.getAutoCommitMemory(); - int fillRate = getFillRate(); - if (isFragmented() && fillRate < getAutoCompactFillRate()) { + if (isFragmented() && getFillRate() < getAutoCompactFillRate()) { mvStore.tryExecuteUnderStoreLock(() -> { int moveSize = 2 * autoCommitMemory; - if (isIdle()) { + if (idle) { moveSize *= 4; } compactMoveChunks(101, moveSize, mvStore); @@ -719,29 +720,34 @@ protected void doHousekeeping(MVStore mvStore) throws InterruptedException { }); } - int chunksFillRate = getRewritableChunksFillRate(); - int adjustedChunksFillRate = 100 - (100 - chunksFillRate) / 2; - int fillRateToCompare = isIdle() ? chunksFillRate : adjustedChunksFillRate; - if (fillRateToCompare < getTargetFillRate()) { + int chunksFillRate = getChunksFillRate(); + int adjustedChunksFillRate = 50 + rewritableChunksFillRate / 2; + int fillRateToCompare = idle ? rewritableChunksFillRate : adjustedChunksFillRate; + if (fillRateToCompare < getTargetFillRate(idle)) { mvStore.tryExecuteUnderStoreLock(() -> { int writeLimit = autoCommitMemory; - if (!isIdle()) { + if (!idle) { writeLimit /= 4; } - if (rewriteChunks(writeLimit, isIdle() ? adjustedChunksFillRate : chunksFillRate)) { + if (rewriteChunks(writeLimit, idle ? adjustedChunksFillRate : rewritableChunksFillRate)) { dropUnusedChunks(); } return true; }); } - stopIdleHousekeeping = idle && getFillRate() <= fillRate && getRewritableChunksFillRate() <= chunksFillRate; + stopIdleHousekeeping = idle && getChunksFillRate() < chunksFillRate; + if (stopIdleHousekeeping) { + // this rate can change with the time, even when database is idle, + // since chunks become older and may become eligible for re-writing + restoreHousekeepingAtRate = getRewritableChunksFillRate() - 2; + } } - private int getTargetFillRate() { + private int getTargetFillRate(boolean idle) { int targetRate = getAutoCompactFillRate(); // use a lower fill rate if there were any file operations since the last time - if (!isIdle()) { - targetRate /= 2; + if (!idle) { + targetRate = targetRate * targetRate / 100; } return targetRate; } From 9faf805a5b915d656245c8e546058112162aa4c3 Mon Sep 17 00:00:00 2001 From: Andrei Tokar Date: Sat, 24 Feb 2024 15:08:46 -0500 Subject: [PATCH 2/3] fix for #3909 --- .../org/h2/mvstore/RandomAccessStore.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/h2/src/main/org/h2/mvstore/RandomAccessStore.java b/h2/src/main/org/h2/mvstore/RandomAccessStore.java index 148d801fa9..4f6a14a1f5 100644 --- a/h2/src/main/org/h2/mvstore/RandomAccessStore.java +++ b/h2/src/main/org/h2/mvstore/RandomAccessStore.java @@ -721,26 +721,34 @@ protected void doHousekeeping(MVStore mvStore) throws InterruptedException { } int chunksFillRate = getChunksFillRate(); - int adjustedChunksFillRate = 50 + rewritableChunksFillRate / 2; - int fillRateToCompare = idle ? rewritableChunksFillRate : adjustedChunksFillRate; + int adjustedUpFillRate = 50 + rewritableChunksFillRate / 2; + int fillRateToCompare = idle ? rewritableChunksFillRate : adjustedUpFillRate; if (fillRateToCompare < getTargetFillRate(idle)) { + int targetFillRate = idle ? adjustedUpFillRate : rewritableChunksFillRate; mvStore.tryExecuteUnderStoreLock(() -> { int writeLimit = autoCommitMemory; if (!idle) { writeLimit /= 4; } - if (rewriteChunks(writeLimit, idle ? adjustedChunksFillRate : rewritableChunksFillRate)) { + if (rewriteChunks(writeLimit, targetFillRate)) { dropUnusedChunks(); } return true; }); } - stopIdleHousekeeping = idle && getChunksFillRate() < chunksFillRate; - if (stopIdleHousekeeping) { - // this rate can change with the time, even when database is idle, - // since chunks become older and may become eligible for re-writing - restoreHousekeepingAtRate = getRewritableChunksFillRate() - 2; - } + stopIdleHousekeeping = false; + if (idle) { + int currentChunksFillRate = getChunksFillRate(); + stopIdleHousekeeping = currentChunksFillRate <= chunksFillRate; + if (stopIdleHousekeeping) { + // this rate can change with the time, even when database is idle, + // since chunks become older and may become eligible for re-writing + rewritableChunksFillRate = getRewritableChunksFillRate(); + restoreHousekeepingAtRate = rewritableChunksFillRate > currentChunksFillRate ? + (currentChunksFillRate + rewritableChunksFillRate) / 2 : + rewritableChunksFillRate - 2; + } + } } private int getTargetFillRate(boolean idle) { From b3c20eb38a2b8fdb193dadd10a54511f7ef0776a Mon Sep 17 00:00:00 2001 From: Andrei Tokar Date: Sat, 23 Mar 2024 15:13:10 -0400 Subject: [PATCH 3/3] fix for #3909 --- h2/src/main/org/h2/mvstore/FileStore.java | 2 +- .../main/org/h2/mvstore/RandomAccessStore.java | 18 ++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/h2/src/main/org/h2/mvstore/FileStore.java b/h2/src/main/org/h2/mvstore/FileStore.java index 0194e7c928..949736503a 100644 --- a/h2/src/main/org/h2/mvstore/FileStore.java +++ b/h2/src/main/org/h2/mvstore/FileStore.java @@ -423,7 +423,7 @@ public final void setAutoCommitDelay(int millis) { stopBackgroundThread(millis >= 0); // start the background thread if needed if (millis > 0 && mvStore.isOpen()) { - int sleep = Math.max(1, millis / 5); + int sleep = Math.max(10, millis / 3); BackgroundWriterThread t = new BackgroundWriterThread(this, sleep, toString()); if (backgroundWriterThread.compareAndSet(null, t)) { t.start(); diff --git a/h2/src/main/org/h2/mvstore/RandomAccessStore.java b/h2/src/main/org/h2/mvstore/RandomAccessStore.java index 4f6a14a1f5..745e3b2af7 100644 --- a/h2/src/main/org/h2/mvstore/RandomAccessStore.java +++ b/h2/src/main/org/h2/mvstore/RandomAccessStore.java @@ -44,7 +44,6 @@ public abstract class RandomAccessStore extends FileStore private long reservedLow; private long reservedHigh; private boolean stopIdleHousekeeping; - private int restoreHousekeepingAtRate; public RandomAccessStore(Map config) { super(config); @@ -705,11 +704,13 @@ private void shrinkIfPossible(int minPercent) { protected void doHousekeeping(MVStore mvStore) throws InterruptedException { boolean idle = isIdle(); int rewritableChunksFillRate = getRewritableChunksFillRate(); - if (idle && stopIdleHousekeeping && rewritableChunksFillRate > restoreHousekeepingAtRate) { + if (idle && stopIdleHousekeeping) { return; } int autoCommitMemory = mvStore.getAutoCommitMemory(); - if (isFragmented() && getFillRate() < getAutoCompactFillRate()) { + int fileFillRate = getFillRate(); + long chunksTotalSize = size() * fileFillRate / 100; + if (isFragmented() && fileFillRate < getAutoCompactFillRate()) { mvStore.tryExecuteUnderStoreLock(() -> { int moveSize = 2 * autoCommitMemory; if (idle) { @@ -739,15 +740,8 @@ protected void doHousekeeping(MVStore mvStore) throws InterruptedException { stopIdleHousekeeping = false; if (idle) { int currentChunksFillRate = getChunksFillRate(); - stopIdleHousekeeping = currentChunksFillRate <= chunksFillRate; - if (stopIdleHousekeeping) { - // this rate can change with the time, even when database is idle, - // since chunks become older and may become eligible for re-writing - rewritableChunksFillRate = getRewritableChunksFillRate(); - restoreHousekeepingAtRate = rewritableChunksFillRate > currentChunksFillRate ? - (currentChunksFillRate + rewritableChunksFillRate) / 2 : - rewritableChunksFillRate - 2; - } + long currentTotalChunksSize = size() * getFillRate() / 100; + stopIdleHousekeeping = currentTotalChunksSize > chunksTotalSize || currentTotalChunksSize == chunksTotalSize && currentChunksFillRate <= chunksFillRate; } }