From 5442296bf728835dabd755cba5729050668d1e79 Mon Sep 17 00:00:00 2001 From: wangjunbo Date: Thu, 22 Feb 2024 18:25:42 +0800 Subject: [PATCH 1/4] [KYUUBI #6070] Performance Improvement for converting rows to thrift rows --- .../engine/result/TRowSetGenerator.scala | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala index 096e45ad8a9..96a1cd22611 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala @@ -18,6 +18,8 @@ package org.apache.kyuubi.engine.result import java.util.{ArrayList => JArrayList} +import scala.collection.JavaConverters._ + import org.apache.kyuubi.shaded.hive.service.rpc.thrift._ trait TRowSetGenerator[SchemaT, RowT, ColumnT] @@ -40,23 +42,17 @@ trait TRowSetGenerator[SchemaT, RowT, ColumnT] } def toRowBasedSet(rows: Seq[RowT], schema: SchemaT): TRowSet = { - val rowSize = rows.length - val tRows = new JArrayList[TRow](rowSize) - var i = 0 - while (i < rowSize) { - val row = rows(i) + val tRows = rows.map { row => + val tRow = new TRow() var j = 0 val columnSize = getColumnSizeFromSchemaType(schema) - val tColumnValues = new JArrayList[TColumnValue](columnSize) while (j < columnSize) { val columnValue = toTColumnValue(row, j, schema) - tColumnValues.add(columnValue) + tRow.addToColVals(columnValue) j += 1 } - i += 1 - val tRow = new TRow(tColumnValues) - tRows.add(tRow) - } + tRow + }.asJava new TRowSet(0, tRows) } @@ -65,13 +61,11 @@ trait TRowSetGenerator[SchemaT, RowT, ColumnT] val tRowSet = new TRowSet(0, new JArrayList[TRow](rowSize)) var i = 0 val columnSize = getColumnSizeFromSchemaType(schema) - val tColumns = new JArrayList[TColumn](columnSize) while (i < columnSize) { val tColumn = toTColumn(rows, i, getColumnType(schema, i)) - tColumns.add(tColumn) + tRowSet.addToColumns(tColumn) i += 1 } - tRowSet.setColumns(tColumns) tRowSet } } From 97db3c99f82afc70321079e15709156098d589bf Mon Sep 17 00:00:00 2001 From: wangjunbo Date: Fri, 23 Feb 2024 14:08:25 +0800 Subject: [PATCH 2/4] fix --- .../org/apache/kyuubi/engine/result/TRowSetGenerator.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala index 96a1cd22611..c59086bdd2f 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala @@ -61,11 +61,13 @@ trait TRowSetGenerator[SchemaT, RowT, ColumnT] val tRowSet = new TRowSet(0, new JArrayList[TRow](rowSize)) var i = 0 val columnSize = getColumnSizeFromSchemaType(schema) + val tColumns = new JArrayList[TColumn](columnSize) while (i < columnSize) { val tColumn = toTColumn(rows, i, getColumnType(schema, i)) - tRowSet.addToColumns(tColumn) + tColumns.add(tColumn) i += 1 } + tRowSet.setColumns(tColumns) tRowSet } } From 90a125635e39953dab2dba433c242dce07826432 Mon Sep 17 00:00:00 2001 From: wangjunbo Date: Fri, 23 Feb 2024 19:45:55 +0800 Subject: [PATCH 3/4] fix --- .../org/apache/kyuubi/engine/result/TRowSetGenerator.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala index c59086bdd2f..e2492036b12 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala @@ -43,15 +43,15 @@ trait TRowSetGenerator[SchemaT, RowT, ColumnT] def toRowBasedSet(rows: Seq[RowT], schema: SchemaT): TRowSet = { val tRows = rows.map { row => - val tRow = new TRow() var j = 0 val columnSize = getColumnSizeFromSchemaType(schema) + val tColumnValues = new JArrayList[TColumnValue](columnSize) while (j < columnSize) { val columnValue = toTColumnValue(row, j, schema) - tRow.addToColVals(columnValue) + tColumnValues.add(columnValue) j += 1 } - tRow + new TRow(tColumnValues) }.asJava new TRowSet(0, tRows) } From 84114f3986eef6260814604348d0f59881d113f8 Mon Sep 17 00:00:00 2001 From: wangjunbo Date: Fri, 23 Feb 2024 20:35:09 +0800 Subject: [PATCH 4/4] fix --- .../apache/kyuubi/engine/result/TRowSetGenerator.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala index e2492036b12..4f5701919eb 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/result/TRowSetGenerator.scala @@ -43,13 +43,13 @@ trait TRowSetGenerator[SchemaT, RowT, ColumnT] def toRowBasedSet(rows: Seq[RowT], schema: SchemaT): TRowSet = { val tRows = rows.map { row => - var j = 0 + var i = 0 val columnSize = getColumnSizeFromSchemaType(schema) val tColumnValues = new JArrayList[TColumnValue](columnSize) - while (j < columnSize) { - val columnValue = toTColumnValue(row, j, schema) + while (i < columnSize) { + val columnValue = toTColumnValue(row, i, schema) tColumnValues.add(columnValue) - j += 1 + i += 1 } new TRow(tColumnValues) }.asJava