From eee330a510565600c0c19852ef69d46852c4e533 Mon Sep 17 00:00:00 2001 From: zhengchenyu001 Date: Wed, 21 Aug 2024 15:32:24 +0800 Subject: [PATCH] TEZ-4577: SortSpan could be created real small, resulting in eventual job failure --- .../tez/runtime/library/common/sort/impl/PipelinedSorter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java index 1815006cd4..d0eeadca06 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java @@ -357,7 +357,7 @@ public void sort() throws IOException { } Preconditions.checkArgument(buffers.get(bufferIndex) != null, "block should not be empty"); //TODO: fix per item being passed. - span = new SortSpan((ByteBuffer)buffers.get(bufferIndex).clear(), items, + span = new SortSpan((ByteBuffer)buffers.get(bufferIndex).clear(), (1024*1024), perItem, ConfigUtils.getIntermediateOutputKeyComparator(this.conf)); } else { // queue up the sort @@ -940,7 +940,7 @@ private final class SortSpan implements IndexedSortable { public SortSpan(ByteBuffer source, int maxItems, int perItem, RawComparator comparator) { capacity = source.remaining(); int metasize = METASIZE*maxItems; - int dataSize = maxItems * perItem; + long dataSize = (long) maxItems * (long) perItem; if(capacity < (metasize+dataSize)) { // try to allocate less meta space, because we have sample data metasize = METASIZE*(capacity/(perItem+METASIZE));