From 5154d04f1ccc21811c0668bf68cfd19f91100907 Mon Sep 17 00:00:00 2001 From: Jiang Yang <37145547+Ted-Jiang@users.noreply.github.com> Date: Mon, 16 Aug 2021 17:23:04 +0800 Subject: [PATCH] PARQUET-2073: Fix estimate remaining row count in ColumnWriteStoreBase. (#922) --- .../org/apache/parquet/column/impl/ColumnWriteStoreBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java index a5e7836b3a..8cfdace8de 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java @@ -237,10 +237,11 @@ private void sizeCheck() { } else { rowCountForNextRowCountCheck = min(rowCountForNextRowCountCheck, writer.getRowsWrittenSoFar() + pageRowCountLimit); } + //estimate remaining row count by previous input for next row count check long rowsToFillPage = usedMem == 0 ? props.getMaxRowCountForPageSizeCheck() - : (long) rows / usedMem * remainingMem; + : rows * remainingMem / usedMem; if (rowsToFillPage < minRecordToWait) { minRecordToWait = rowsToFillPage; }