feat(codecv2): add zstd compression and increase chunks num per batch…

… post compression (#2)

* feat: add codecv0 and codecv1

* feat: add codecv0 and codecv1

* feat(codecv1): add zstd compression and increase chunks num per batch post compression

* add zstd compression

* remove .so files

* add codecv2

* fix

* fix lint

* add .so files

* add codecv2 type

* tweak

* Apply suggestions from code review

Co-authored-by: Péter Garamvölgyi <[email protected]>

* remove timezone setting

* refactor

* fix

* Apply suggestions from code review

Co-authored-by: georgehao <[email protected]>

* small fix

* refactor

* rename rs to libzstd

* use early return

* change DecodeDABlock to receiver form

* use enum as versions, removing constant

* remove init() and log.Crit()

* add gas estimation

* tweak

* build .so files on-the-fly

* fix CI

* use default path to compile and link

* remove libzstd/rs_zstd.h and fix CI

* rename

* rename

* rename

* revert

* add -L/lib/ -Wl,-rpath=/lib

* temp try

* try

* add CGO_LDFLAGS

* fix

* try fix

* revert

* remove cp to /usr/local/lib

* tweak

* fix

* try deleting LD_LIBRARY_PATH

* revert

* remove Dockerfile

* tweak

* remove txTypeTest

* decrease the 256 bytes extra buffer to 128 bytes

* add a comment

---------

Co-authored-by: Péter Garamvölgyi <[email protected]>
Co-authored-by: georgehao <[email protected]>

.github/workflows/codec.yml

@@ -19,7 +19,7 @@ jobs:
      - name: Install Go
        uses: actions/setup-go@v2
        with:
-         go-version: 1.20.x
+         go-version: 1.21.x
      - name: Checkout code
        uses: actions/checkout@v2
      - name: Lint
@@ -33,7 +33,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Install goimports
@@ -53,7 +53,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Test codec packages

.gitignore

@@ -5,6 +5,7 @@
 *.exe
 *.exe~
 *.dll
+*.a
 *.so
 *.dylib
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: fmt lint test
+.PHONY: fmt lint test build run
 
 lint:
 	GOBIN=$(PWD)/build/bin go run ./build/lint.go
@@ -9,4 +9,4 @@ fmt:
 	gofumpt -l -w .
 
 test:
-	go test -v -race -gcflags="-l" -ldflags="-s=false" -coverprofile=coverage.txt -covermode=atomic ./...
+	./run_test.sh

README.md

@@ -1 +1,13 @@
-# da-codec
+# da-codec
+
+## Running unit tests
+```
+docker pull scrolltech/go-rust-builder:go-1.21-rust-nightly-2023-12-03 --platform linux/amd64
+docker run -it --rm -v "$(PWD):/workspace" -w /workspace scrolltech/go-rust-builder:go-1.21-rust-nightly-2023-12-03
+cd libzstd
+make libzstd
+cd ..
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/libzstd
+export CGO_LDFLAGS="-L$(pwd)/libzstd -Wl,-rpath,$(pwd)/libzstd"
+go test -v -race ./...
+```

build/.golangci.yml

@@ -104,10 +104,12 @@ linters-settings:
     # minimal occurrences count to trigger, 3 by default
     min-occurrences: 3
   depguard:
-    list-type: blacklist
-    include-go-root: false
-    packages:
-      - github.com/davecgh/go-spew/spew
+    rules:
+      main:
+        files:
+          - $all
+        deny:
+          - pkg: "github.com/davecgh/go-spew/spew"
   misspell:
     # Correct spellings using locale preferences for US or UK.
     # Default is to use a neutral variety of English.

build/lint.go

@@ -15,7 +15,7 @@ import (
 
 const (
 	// GolangCIVersion to be used for linting.
-	GolangCIVersion = "github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2"
+	GolangCIVersion = "github.com/golangci/golangci-lint/cmd/golangci-lint@v1.57.2"
 )
 
 // GOBIN environment variable.
@@ -51,7 +51,7 @@ func lint() {
 	}
 
 	cmd = exec.Command(filepath.Join(goBin(), "golangci-lint"))
-	cmd.Args = append(cmd.Args, "run", "--config", "build/.golangci.yml")
+	cmd.Args = append(cmd.Args, "run", "--config", "build/.golangci.yml", "--timeout", "2m")
 
 	if *v {
 		cmd.Args = append(cmd.Args, "-v")

encoding/bitmap.go

@@ -24,6 +24,7 @@ func ConstructSkippedBitmap(batchIndex uint64, chunks []*Chunk, totalL1MessagePo
 				if tx.Type != types.L1MessageTxType {
 					continue
 				}
+
 				currentIndex := tx.Nonce
 
 				if currentIndex < nextIndex {

encoding/codecv0/codecv0.go

@@ -16,9 +16,6 @@ import (
 	"github.com/scroll-tech/da-codec/encoding"
 )
 
-// CodecV0Version denotes the version of the codec.
-const CodecV0Version = 0
-
 // DABlock represents a Data Availability Block.
 type DABlock struct {
 	BlockNumber     uint64
@@ -91,22 +88,20 @@ func (b *DABlock) Encode() []byte {
 	return bytes
 }
 
-// DecodeDABlock takes a byte slice and decodes it into a DABlock.
-func DecodeDABlock(bytes []byte) (*DABlock, error) {
+// Decode populates the fields of a DABlock from a byte slice.
+func (b *DABlock) Decode(bytes []byte) error {
 	if len(bytes) != 60 {
-		return nil, errors.New("block encoding is not 60 bytes long")
+		return errors.New("block encoding is not 60 bytes long")
 	}
 
-	block := &DABlock{
-		BlockNumber:     binary.BigEndian.Uint64(bytes[0:8]),
-		Timestamp:       binary.BigEndian.Uint64(bytes[8:16]),
-		BaseFee:         new(big.Int).SetUint64(binary.BigEndian.Uint64(bytes[40:48])),
-		GasLimit:        binary.BigEndian.Uint64(bytes[48:56]),
-		NumTransactions: binary.BigEndian.Uint16(bytes[56:58]),
-		NumL1Messages:   binary.BigEndian.Uint16(bytes[58:60]),
-	}
+	b.BlockNumber = binary.BigEndian.Uint64(bytes[0:8])
+	b.Timestamp = binary.BigEndian.Uint64(bytes[8:16])
+	b.BaseFee = new(big.Int).SetUint64(binary.BigEndian.Uint64(bytes[40:48]))
+	b.GasLimit = binary.BigEndian.Uint64(bytes[48:56])
+	b.NumTransactions = binary.BigEndian.Uint16(bytes[56:58])
+	b.NumL1Messages = binary.BigEndian.Uint16(bytes[58:60])
 
-	return block, nil
+	return nil
 }
 
 // NewDAChunk creates a new DAChunk from the given encoding.Chunk and the total number of L1 messages popped before.
@@ -160,8 +155,9 @@ func (c *DAChunk) Encode() ([]byte, error) {
 			if txData.Type == types.L1MessageTxType {
 				continue
 			}
+
 			var txLen [4]byte
-			rlpTxData, err := encoding.ConvertTxDataToRLPEncoding(txData)
+			rlpTxData, err := encoding.ConvertTxDataToRLPEncoding(txData, false /* no mock */)
 			if err != nil {
 				return nil, err
 			}
@@ -248,7 +244,7 @@ func NewDABatch(batch *encoding.Batch) (*DABatch, error) {
 	}
 
 	daBatch := DABatch{
-		Version:                CodecV0Version,
+		Version:                uint8(encoding.CodecV0),
 		BatchIndex:             batch.Index,
 		L1MessagePopped:        totalL1MessagePoppedAfter - batch.TotalL1MessagePoppedBefore,
 		TotalL1MessagePopped:   totalL1MessagePoppedAfter,
@@ -298,12 +294,6 @@ func (b *DABatch) Hash() common.Hash {
 	return crypto.Keccak256Hash(bytes)
 }
 
-// DecodeFromCalldata attempts to decode a DABatch and an array of DAChunks from the provided calldata byte slice.
-func DecodeFromCalldata(data []byte) (*DABatch, []*DAChunk, error) {
-	// TODO: implement this function.
-	return nil, nil, nil
-}
-
 // CalldataNonZeroByteGas is the gas consumption per non zero byte in calldata.
 const CalldataNonZeroByteGas = 16
 
@@ -471,7 +461,7 @@ func EstimateBatchL1CommitCalldataSize(b *encoding.Batch) (uint64, error) {
 }
 
 func getTxPayloadLength(txData *types.TransactionData) (uint64, error) {
-	rlpTxData, err := encoding.ConvertTxDataToRLPEncoding(txData)
+	rlpTxData, err := encoding.ConvertTxDataToRLPEncoding(txData, false /* no mock */)
 	if err != nil {
 		return 0, err
 	}