Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[package management service] fix wrappedBuffer always using the same block of memory #11782

Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -58,11 +58,12 @@ private CompletableFuture<List<LogRecord>> getRecords(InputStream inputStream) {
try {
int read = 0;
while ((read = inputStream.read(readBuffer)) != -1) {
log.info("write something into the ledgers " + offset);
log.info("write something into the ledgers offset: {}, length: {}", offset, read);
ByteBuf writeBuf = Unpooled.wrappedBuffer(readBuffer, 0, read);
freeznet marked this conversation as resolved.
Show resolved Hide resolved
offset += writeBuf.readableBytes();
LogRecord record = new LogRecord(offset, writeBuf);
records.add(record);
readBuffer = new byte[8192];
}
future.complete(records);
} catch (IOException e) {
Expand Down
Expand Up @@ -27,6 +27,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import org.apache.commons.lang3.RandomUtils;
import org.apache.distributedlog.exceptions.LogNotFoundException;
import org.apache.distributedlog.exceptions.ZKException;
import org.apache.pulsar.packages.management.core.PackagesStorage;
Expand All @@ -41,7 +42,7 @@ public class BookKeeperPackagesStorageTest extends BookKeeperClusterTestCase {
private PackagesStorage storage;

public BookKeeperPackagesStorageTest() {
super(1);
super(2);
}

@BeforeMethod()
Expand Down Expand Up @@ -89,6 +90,23 @@ public void testReadWriteOperations() throws ExecutionException, InterruptedExce
assertEquals(testData, readResult);
}

@Test(timeOut = 60000)
public void testReadWriteLargeDataOperations() throws ExecutionException, InterruptedException {
byte[] data = RandomUtils.nextBytes(8192 * 3 + 4096);
ByteArrayInputStream testDataStream = new ByteArrayInputStream(data);
String testPath = "test-large-read-write";

// write some data to the dlog
storage.writeAsync(testPath, testDataStream).get();

// read the data from the dlog
ByteArrayOutputStream readData = new ByteArrayOutputStream();
storage.readAsync(testPath, readData).get();
byte[] readResult = readData.toByteArray();

assertEquals(data, readResult);
}

@Test(timeOut = 60000)
public void testReadNonExistentData() {
String testPath = "non-existent-path";
Expand Down
Expand Up @@ -96,6 +96,19 @@ public void writeBytesArrayData() throws ExecutionException, InterruptedExceptio
verify(dlm, times(1)).asyncClose();
}

@Test
public void writeLongBytesArrayData() throws ExecutionException, InterruptedException {
byte[] data = new byte[8192 * 3 + 4096];
DLOutputStream.openWriterAsync(dlm)
.thenCompose(w -> w.writeAsync(new ByteArrayInputStream(data))
.thenCompose(DLOutputStream::closeAsync)).get();

verify(writer, times(1)).writeBulk(any(List.class));
verify(writer, times(1)).markEndOfStream();
verify(writer, times(1)).asyncClose();
verify(dlm, times(1)).asyncClose();
}

@Test
public void openAsyncLogWriterFailed() {
when(dlm.openAsyncLogWriter()).thenReturn(failedFuture(new Exception("Open writer was failed")));
Expand Down