Skip to content

Commit

Permalink
added row binary tests for v2
Browse files Browse the repository at this point in the history
  • Loading branch information
chernser committed Mar 6, 2025
1 parent e00a973 commit 856bc9d
Show file tree
Hide file tree
Showing 4 changed files with 85 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import com.clickhouse.client.api.Client;
import com.clickhouse.client.api.enums.Protocol;
import com.clickhouse.client.api.insert.InsertResponse;
import com.clickhouse.client.api.metadata.TableSchema;
import com.clickhouse.data.ClickHouseDataProcessor;
import com.clickhouse.data.ClickHouseFormat;
import com.clickhouse.data.ClickHouseOutputStream;
Expand Down Expand Up @@ -126,7 +127,6 @@ public static void loadClickHouseRecords(String tableName, DataSet dataSet) {
ClickHouseOutputStream.of(new ByteArrayOutputStream()), response.getColumns(), Collections.emptyMap());
assert dataProcessor.getColumns() != null;
dataSet.setClickHouseDataProcessor(dataProcessor);

ArrayList<ClickHouseRecord> records = new ArrayList<>();
for (ClickHouseRecord record : response.records()) {
records.add(record);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@
import com.clickhouse.client.ClickHouseResponseSummary;
import com.clickhouse.client.api.Client;
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter;
import com.clickhouse.client.api.enums.Protocol;
import com.clickhouse.client.api.insert.InsertResponse;
import com.clickhouse.client.api.insert.InsertSettings;
import com.clickhouse.client.api.query.QueryResponse;
import com.clickhouse.client.config.ClickHouseClientOption;
import com.clickhouse.data.ClickHouseColumn;
import com.clickhouse.data.ClickHouseDataProcessor;
import com.clickhouse.data.ClickHouseFormat;
import com.clickhouse.data.ClickHouseRecord;
Expand All @@ -27,6 +29,8 @@
import org.slf4j.LoggerFactory;

import java.math.BigInteger;
import java.util.List;
import java.util.Map;

import static com.clickhouse.client.ClickHouseServerForTest.isCloud;

Expand Down Expand Up @@ -168,4 +172,59 @@ public void insertV1RowBinary(DataState dataState) {
LOGGER.error("Error: ", e);
}
}

@Benchmark
public void insertV2RowBinary(DataState dataState) {
try {
try (InsertResponse response = clientV2.insert(dataState.dataSet.getTableName(), out -> {
RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, dataState.dataSet.getSchema(), ClickHouseFormat.RowBinary);
List<ClickHouseColumn> columns = dataState.dataSet.getSchema().getColumns();
for (Map<String, Object> row : dataState.dataSet.getRows()) {
for (ClickHouseColumn column : columns) {
w.setValue(column.getColumnName(),row.get(column.getColumnName()));
}
w.commitRow();
}
out.flush();

}, ClickHouseFormat.RowBinaryWithDefaults, new InsertSettings()).get()) {
if (response.getWrittenRows() <= 0) {
throw new RuntimeException("Rows written: " + response.getWrittenRows());
}
}
} catch (Exception e) {
LOGGER.error("Error: ", e);
}
}

@Benchmark
public void insertV1WithV2RowBinaryWriter(DataState dataState) {
try {
ClickHouseFormat format = ClickHouseFormat.RowBinary;
try (ClickHouseResponse response = clientV1.read(getServer())
.write()
.option(ClickHouseClientOption.ASYNC, false)
.format(format)
.query("INSERT INTO `" + DB_NAME + "`.`" + dataState.dataSet.getTableName() + "`")
.data(out -> {
RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, dataState.dataSet.getSchema(), ClickHouseFormat.RowBinary);
List<ClickHouseColumn> columns = dataState.dataSet.getSchema().getColumns();
for (Map<String, Object> row : dataState.dataSet.getRows()) {
for (ClickHouseColumn column : columns) {
w.setValue(column.getColumnName(),row.get(column.getColumnName()));
}
w.commitRow();
}
out.flush();
})
.executeAndWait()) {
ClickHouseResponseSummary summary = response.getSummary();
if (summary.getWrittenRows() <= 0) {
throw new RuntimeException("Rows written: " + summary.getWrittenRows());
}
}
} catch ( Exception e) {
LOGGER.error("Error: ", e);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
package com.clickhouse.benchmark.data;

import com.clickhouse.client.api.metadata.TableSchema;
import com.clickhouse.data.ClickHouseColumn;
import com.clickhouse.data.ClickHouseDataProcessor;
import com.clickhouse.data.ClickHouseFormat;
import com.clickhouse.data.ClickHouseRecord;
import com.clickhouse.data.ClickHouseValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.shaded.com.google.common.collect.Table;

import java.io.BufferedReader;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

Expand All @@ -26,6 +30,8 @@ public class FileDataSet implements DataSet{

private List<byte[]> lines =null;

private List<Map<String, Object>> data;

public FileDataSet(String filePath) {
File srcFile = new File(filePath);

Expand Down Expand Up @@ -98,9 +104,10 @@ public String getTrucateTableString() {
return "TRUNCATE TABLE " + getTableName();
}

private TableSchema tableSchema = new TableSchema();
@Override
public TableSchema getSchema() {
return new TableSchema(); //TODO: parse from create statement or get by client
return tableSchema;
}

@Override
Expand All @@ -115,7 +122,7 @@ public List<byte[]> getBytesList(ClickHouseFormat format) {

@Override
public List<Map<String, Object>> getRows() {
return Collections.emptyList();
return data;
}

@Override
Expand All @@ -133,6 +140,18 @@ public List<ClickHouseRecord> getClickHouseRecords() {
@Override
public void setClickHouseRecords(List<ClickHouseRecord> records) {
this.clickHouseRecords = records;
List<ClickHouseColumn> columns = tableSchema.getColumns();
data = new ArrayList<>(records.size());
for (ClickHouseRecord record : records) {
Iterator<ClickHouseValue> vIter = record.iterator();
int i = 0;
Map<String, Object> row = new HashMap<>();
while (vIter.hasNext()) {
ClickHouseValue v = vIter.next();
row.put(columns.get(i++).getColumnName(), v.asObject());
}
data.add(row);
}
}

private ClickHouseDataProcessor dataProcessor;
Expand All @@ -145,6 +164,9 @@ public ClickHouseDataProcessor getClickHouseDataProcessor() {
@Override
public void setClickHouseDataProcessor(ClickHouseDataProcessor dataProcessor) {
this.dataProcessor = dataProcessor;
for (ClickHouseColumn column : dataProcessor.getColumns()) {
tableSchema.addColumn(column.getColumnName(), column.getOriginalTypeName());
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public class SimpleDataSet implements DataSet {

public SimpleDataSet() {
tableName = name + "_dataset_" + UUID.randomUUID().toString().replaceAll("-", "");
size = 10;
size = 100000;

data = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
Expand Down

0 comments on commit 856bc9d

Please sign in to comment.