add zstd compression

.gitignore

@@ -5,7 +5,6 @@
 *.exe
 *.exe~
 *.dll
-*.so
 *.dylib
 
 # Test binary, built with `go test -c`

Dockerfile

@@ -0,0 +1,38 @@
+ARG GO_VERSION=1.21
+ARG RUST_VERSION=nightly-2023-12-03
+ARG CARGO_CHEF_TAG=0.1.41
+
+FROM ubuntu:20.04
+
+RUN apt-get update && ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime
+
+# Install basic packages
+RUN apt-get install -y build-essential
+RUN apt-get install -y curl
+RUN apt-get install -y wget
+RUN apt-get install -y git
+RUN apt-get install -y pkg-config
+
+# Install dev-packages
+RUN apt-get install -y libclang-dev
+RUN apt-get install -y libssl-dev
+RUN apt-get install -y llvm
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+ENV CARGO_HOME=/root/.cargo
+# Add Toolchain
+ARG RUST_VERSION
+RUN rustup toolchain install ${RUST_VERSION}
+ARG CARGO_CHEF_TAG
+RUN cargo install cargo-chef --locked --version ${CARGO_CHEF_TAG} \
+    && rm -rf $CARGO_HOME/registry/
+
+# Install Go
+ARG GO_VERSION
+RUN rm -rf /usr/local/go
+RUN wget https://go.dev/dl/go${GO_VERSION}.1.linux-amd64.tar.gz
+RUN tar -C /usr/local -xzf go${GO_VERSION}.1.linux-amd64.tar.gz
+RUN rm go${GO_VERSION}.1.linux-amd64.tar.gz
+ENV PATH="/usr/local/go/bin:${PATH}"

Makefile

@@ -1,4 +1,4 @@
-.PHONY: fmt lint test
+.PHONY: fmt lint test build run
 
 lint:
 	GOBIN=$(PWD)/build/bin go run ./build/lint.go
@@ -8,5 +8,11 @@ fmt:
 	goimports -w .
 	gofumpt -l -w .
 
+build:
+	docker build -t my-dev-container --platform linux/amd64 .
+
+run: build
+	docker run -it --rm -v "$(PWD):/workspace" -w /workspace my-dev-container
+
 test:
 	go test -v -race -gcflags="-l" -ldflags="-s=false" -coverprofile=coverage.txt -covermode=atomic ./...

README.md

@@ -1 +1,40 @@
-# da-codec
+# da-codec
+
+## Building `libscroll_zstd.so` File.
+
+Follow these steps to build the `.so` file:
+
+1. Build and enter the container:
+    ```
+    make run
+    ```
+
+2. Change directory to rs:
+    ```
+    cd rs
+    ```
+
+3. Build libzstd:
+    ```
+    export CARGO_NET_GIT_FETCH_WITH_CLI=true
+    make libzstd
+    ```
+
+## Running unit tests
+
+Follow these steps to run unit tests:
+
+1. Build and enter the container:
+    ```
+    make run
+    ```
+
+2. Set the directory for shared libraries:
+    ```
+    export LD_LIBRARY_PATH=${PWD}/rs:$LD_LIBRARY_PATH
+    ```
+
+3. Execute the unit tests:
+    ```
+    go test -v -race ./...
+    ```

encoding/codecv1/codecv1.go

@@ -1,5 +1,12 @@
 package codecv1
 
+/*
+#cgo LDFLAGS: -L../../rs -lm -ldl -lscroll_zstd
+#include <stdlib.h>
+#include "../../rs/rs_zstd.h"
+*/
+import "C"
+
 import (
 	"crypto/sha256"
 	"encoding/binary"
@@ -9,6 +16,7 @@ import (
 	"math"
 	"math/big"
 	"strings"
+	"unsafe"
 
 	"github.com/scroll-tech/go-ethereum/accounts/abi"
 	"github.com/scroll-tech/go-ethereum/common"
@@ -619,8 +627,22 @@ func getMaxNumChunks(useCompression bool) int {
 }
 
 func compressScrollBatchBytes(batchBytes []byte) ([]byte, error) {
-	// TODO: replace this function with real implementation.
-	return batchBytes, nil
+	srcSize := C.uint64_t(len(batchBytes))
+	outbuf := make([]byte, len(batchBytes))
+	outbufSize := C.uint64_t(len(batchBytes))
+
+	err := C.compress_scroll_batch_bytes(
+		(*C.uchar)(unsafe.Pointer(&batchBytes[0])),
+		srcSize,
+		(*C.uchar)(unsafe.Pointer(&outbuf[0])),
+		&outbufSize,
+	)
+
+	if err != nil {
+		return nil, fmt.Errorf("compress_scroll_batch_bytes fail: %s", C.GoString(err))
+	}
+
+	return outbuf[:int(outbufSize)], nil
 }
 
 // calculatePaddedBlobSize calculates the required size on blob storage

encoding/codecv1/codecv1_test.go

@@ -772,9 +772,9 @@ func TestCodecV1ChunkAndBatchBlobSizeEstimation(t *testing.T) {
 		traceFile     string
 		expectedSizes map[bool]uint64
 	}{
-		{"../testdata/blockTrace_02.json", map[bool]uint64{false: 302, true: 426}},
-		{"../testdata/blockTrace_03.json", map[bool]uint64{false: 5929, true: 6053}},
-		{"../testdata/blockTrace_04.json", map[bool]uint64{false: 98, true: 221}},
+		{"../testdata/blockTrace_02.json", map[bool]uint64{false: 302, true: 236}},
+		{"../testdata/blockTrace_03.json", map[bool]uint64{false: 5929, true: 2617}},
+		{"../testdata/blockTrace_04.json", map[bool]uint64{false: 98, true: 54}},
 	}
 
 	for _, c := range cases {
@@ -798,8 +798,8 @@ func TestCodecV1ChunkAndBatchBlobSizeEstimation(t *testing.T) {
 		traceFiles    []string
 		expectedSizes map[bool]uint64
 	}{
-		{[]string{"../testdata/blockTrace_02.json", "../testdata/blockTrace_03.json"}, map[bool]uint64{false: 6166, true: 6290}},
-		{[]string{"../testdata/blockTrace_04.json"}, map[bool]uint64{false: 98, true: 221}},
+		{[]string{"../testdata/blockTrace_02.json", "../testdata/blockTrace_03.json"}, map[bool]uint64{false: 6166, true: 2834}},
+		{[]string{"../testdata/blockTrace_04.json"}, map[bool]uint64{false: 98, true: 54}},
 	}
 
 	for _, cc := range combinedCases {

rs/.gitignore

@@ -0,0 +1,2 @@
+/target
+/_obj