V1.0.0 - Import & Export topic using Apache parquet file

Use command line

Install

go install github.com/huantt/[email protected]

export PATH=$PATH:$(go env GOPATH)/bin

Export Kafka topics to parquet file

kafka-dump export \
--file=path/to/output/data.parquet \
--kafka-topics=users-activities \
--kafka-group-id=id=kafka-dump.local \
--kafka-servers=localhost:9092 \
--kafka-username=admin \
--kafka-password=admin \
--kafka-security-protocol=SASL_SSL \
--kafka-sasl-mechanism=PLAIN

Count number of rows in parquet file

kafka-dump count-parquet-rows \
--file=path/to/output/data.parquet

Import Kafka topics from parquet file

kafka-dump import \
--file=path/to/input/data.parquet \
--kafka-servers=localhost:9092 \
--kafka-username=admin \
--kafka-password=admin \
--kafka-security-protocol=SASL_SSL \
--kafka-sasl-mechanism=PLAIN

Use Docker

docker run -d --rm \
-v /local-data:/data \
huanttok/kafka-dump:1.0.0 \
./kafka-dump export \
--file=/data/path/to/output/data.parquet \
--kafka-topics=users-activities \
--kafka-group-id=id=kafka-dump.local \
--kafka-servers=localhost:9092 \
--kafka-username=admin \
--kafka-password=admin \
--kafka-security-protocol=SASL_SSL \
--kafka-sasl-mechanism=PLAIN