Skip to content

Commit

Permalink
#22951 Enforcing the new expectations for the 'byteSize' limit in uni…
Browse files Browse the repository at this point in the history
…t tests
  • Loading branch information
nbali committed Nov 16, 2022
1 parent f00c85d commit 5840e55
Showing 1 changed file with 11 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -762,17 +762,23 @@ private static <K> boolean checkBatchByteSizes(
Iterable<KV<K, Iterable<String>>> listToCheck,
SerializableFunction<String, Long> getElementByteSizeFn) {
for (KV<?, Iterable<String>> element : listToCheck) {
List<String> batchElements = Lists.newArrayList(element.getValue());
if (batchElements.size() == 1) {
// if we have a single element that is over the batch size byte limit, we can't do anything
// than to fire it as a 1-element batch
continue;
}
long byteSize = 0;
for (String str : element.getValue()) {
if (byteSize >= BATCH_SIZE_BYTES) {
// We already reached the batch size, so extra elements are not expected.
return false;
}
for (String str : batchElements) {
try {
byteSize += getElementByteSizeFn.apply(str);
} catch (Exception e) {
throw new RuntimeException(e);
}
// size of elements should be less than or equal to the batch size
if (byteSize > BATCH_SIZE_BYTES) {
return false;
}
}
}
return true;
Expand Down

0 comments on commit 5840e55

Please sign in to comment.