feat: major changes (#3)

* feat: major changes

- migrate from bbolt to pebble
- migrate from httproute to gorilla mux
- nytimes gzip
- improvements all aroud

* chore: disable tests until we have a custom test env with higher end machines

.dockerignore

@@ -12,5 +12,8 @@ shell.nix
 README.md
 minio.yaml
 ingress.yaml
+swag.yaml
+swagger.yaml
+treefmt.toml
+\_db
 Makefile
-treefmt.toml

.github/workflows/test.yml

@@ -1,21 +1,21 @@
-name: Go
+# name: Go
 
-on: [push]
+# on: [push]
 
-jobs:
-  build:
+# jobs:
+#   build:
 
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go-version: [ '1.19' ]
+#     runs-on: ubuntu-latest
+#     strategy:
+#       matrix:
+#         go-version: [ '1.20' ]
 
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go-version }}
-          cache-dependency-path: subdir/go.sum
-      - name: Test
-        run: go test --count=1 ./...
+#     steps:
+#       - uses: actions/checkout@v3
+#       - name: Setup Go ${{ matrix.go-version }}
+#         uses: actions/setup-go@v4
+#         with:
+#           go-version: ${{ matrix.go-version }}
+#           cache-dependency-path: subdir/go.sum
+#       - name: Test
+#         run: go test --count=1 ./...

.gitignore

@@ -23,13 +23,18 @@ bin/
 go.work
 
 .envrc.local
+.vscode
 
 k6/node_modules
 node_modules
 
 *-store
 *.db
 *.db.*
+*_db
+*_db*
+
 
 web/.serve
-tmp
+tmp
+__debug*

Dockerfile

@@ -1,14 +1,22 @@
 # syntax=docker/dockerfile:1
-FROM golang:1.19 as builder
-RUN mkdir /build
-ADD . /build/
+FROM golang:1.20-alpine as builder
+
 WORKDIR /build
-ENV CGO_CFLAGS="-O -D__BLST_PORTABLE__"
-ENV CGO_CFLAGS_ALLOW="-O -D__BLST_PORTABLE__"
-RUN --mount=type=cache,target=/root/.cache/go-build go test ./... && go build -o ./bin/mev-freelay ./cmd/freelay/main.go && go build -o ./bin/purge ./cmd/purge/main.go && go build -o ./bin/backup ./cmd/backup/main.go && go build -o ./bin/restore ./cmd/restore/main.go && go build -o ./bin/migrate ./cmd/migrate/main.go && go build -o ./bin/compact ./cmd/compact/main.go && go build -o ./bin/import ./cmd/import/main.go
+
+COPY go.mod .
+COPY go.sum .
+RUN --mount=type=cache,target=/root/.cache/go-build go mod download
+
+ADD . .
+
+# RUN --mount=type=cache,target=/root/.cache/go-build go test ./...
+
+RUN apk add --no-cache gcc musl-dev git linux-headers
+RUN --mount=type=cache,target=/root/.cache/go-build GOOS=linux go build -trimpath -ldflags "-s -linkmode external -extldflags '-static'" -o ./bin/mev-freelay ./main.go
+
 
 FROM alpine
-RUN apk add --no-cache libgcc libstdc++ libc6-compat
+RUN apk add --no-cache libstdc++ libc6-compat
 RUN mkdir /app
 WORKDIR /app
 
@@ -17,16 +25,5 @@ ENV SHA_VERSION=$SHA_VERSION
 
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder /build/bin/mev-freelay /app/mev-freelay
-COPY --from=builder /build/bin/purge /app/purge
-COPY --from=builder /build/bin/backup /app/backup
-COPY --from=builder /build/bin/restore /app/restore
-COPY --from=builder /build/bin/migrate /app/migrate
-COPY --from=builder /build/bin/compact /app/compact
-COPY --from=builder /build/bin/import /app/import
-COPY --from=builder /build/web /app/web
-EXPOSE 50051
-EXPOSE 50052
-EXPOSE 50053
-EXPOSE 50054
-EXPOSE 9000
+EXPOSE 50051 50052 9000 6060
 ENTRYPOINT ["/app/mev-freelay"]

Makefile

@@ -1,17 +1,18 @@
-clear:
+clean:
 	find . -name \test.*.db -type f -delete
 	find . -name \fake.*.db -type f -delete
+	find . -name \test_*_db -type d -exec rm -r {} +
+	find . -name \fake_*_db -type d -exec rm -r {} +
 
 minio:
-	docker-compose -f minio.yaml up -d
+	docker compose -f minio.yaml up -d
 
 ingress:
-	docker-compose -f ingress.yaml up -d
-
+	docker compose -f ingress.yaml up -d
 
 swag:
-	docker-compose -f swag.yaml down --remove-orphans
-	docker-compose -f swag.yaml up -d
+	docker compose -f swag.yaml down --remove-orphans
+	docker compose -f swag.yaml up -d
 
 fmt:
 	golangci-lint run && treefmt

README.md

@@ -14,49 +14,40 @@ Please note that this relay has not been subjected to a security audit, and as s
 
 # USP (Unique Selling Points)
 
-- Use of a single process and embedded bbolt database for persistence, leading to lower latency and resource usage for operations
+- Use of a single process and embedded pebble database for persistence, leading to lower latency and resource usage for operations
 - Automatic archiving of submitted payloads to S3, which keeps the operation database length constant
 - Simple backup and restore process
 
 # Tech Stack
 
-We aimed for a straightforward technology stack that is easy to manage and deploy. For handling **API** requests, we are using [httprouter](https://github.com/julienschmidt/httprouter). We chose [bbolt](https://github.com/etcd-io/bbolt) as our database due to its fast read speeds and wrapped it with [bolted](https://github.com/draganm/bolted) for greater control over read and write operations. Finally, we used [kartusche](https://github.com/numtide/kartusche) to build the website, which utilizes [mustache](https://mustache.github.io/) templating.
-
-# Build
-
-```
-DOCKER_BUILDKIT=1
-export CGO_CFLAGS_ALLOW="-D__BLST_PORTABLE__"
-export CGO_CFLAGS="-D__BLST_PORTABLE__"
-```
+We aimed for a straightforward technology stack that is easy to manage and deploy. We chose [pebble](https://github.com/cockroachdb/pebble) as our database due to its fast read speeds and multi threaded writes. Finally, we used [go-lean](https://github.com/draganm/go-lean) to build the website, which utilizes [mustache](https://mustache.github.io/) templating.
 
 # Production
 
 ## Using Dockerfile
 
 ```
-DOCKER_BUILDKIT=1 docker build -t freelay .
-docker run freelay -e BEACONS <prysm_beacon_url> -e BLOCK_SIM_URL <geth_url> -e SECRET_KEY <generate with cmd/keys> -e NETWORK goerli -e DB_PREFIX prod -e DB_DIR dbs
+DOCKER_BUILDKIT=1 docker build -t mev-freelay .
+docker run mev-freelay freelay -e BEACONS <prysm_beacon_url> -e BLOCK_SIM_URL <geth_url> -e SECRET_KEY <generate with keys> -e NETWORK goerli -e DB_PTH dbs/prod_db
 ```
 
 ## Using binary
 
 ```
-go build -o freelay cmd/freelay/main.go
-./freelay --beacons <prysm_beacon_url> --block-sim-url <geth_url> --secret-key <generate with cmd/keys> --network goerli --db-prefix prod --db-dir dbs
+go build -o mev-freelay .
+./mev-freelay freelay --beacons <prysm_beacon_url> --block-sim-url <geth_url> --secret-key <generate with keys> --network goerli --db-pth dbs/prod_db
 ```
 
 # Development
 
 ```
-go run cmd/freelay/main.go --beacons <prysm_beacon_url> --block-sim-url <geth_url> --secret-key <generate with cmd/keys> --network goerli --db-prefix prod --db-dir dbs
+go run main.go freerelay --beacons <prysm_beacon_url> --block-sim-url <geth_url> --secret-key <generate with keys> --network goerli --db-pth dbs/prod_db
+```
 
 Relay (Proposer, Builder, Data): http://localhost:50051
-API (Website API): http://localhost:50052
-Website: http://localhost:50053
-SFTP: http://localhost:50054
+API (Website): http://localhost:50052
 Prometheus: http://localhost:9000
-```
+PPROF: http://localhost:6060
 
 ## Minio S3
 
@@ -101,23 +92,23 @@ make swag
 ## Generate Keys
 
 ```
-go run cmd/keys/main.go
+go run main.go keys
 ```
 
 ## Backup Database to S3
 
 It is using default `AWS` credentials. You can override them with `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
 
 ```
-go run cmd/backup/main.go --backup-url http://localhost:50052/backup --bucket <bucket>
+go run main.go backup --backup-url http://localhost:50052/api/backup --bucket <bucket>
 ```
 
 ## Restore Database from S3
 
 It is using default `AWS` credentials. You can override them with `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
 
 ```
-go run cmd/restore/main.go --db-dir <dir> --db-prefix <prefix> --bucket <bucket>
+go run main.go restore --db-pth <db_pth> --bucket <bucket>
 ```
 
 ## Purge Database
@@ -127,25 +118,19 @@ Archives payloads (executed-payloads, submitted, bid-traces) that are older then
 It is using default `AWS` credentials. You can override them with `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
 
 ```
-go run cmd/purge/main.go --archive-url http://localhost:50052/archive --prune-url http://localhost:50052/prune  --bucket <bucket>
-```
-
-## Compact Database
-
-```
-go run cmd/compact/main.go --db-dir <dir> --db-prefix <prefix>
+go run main.go purge --archive-url http://localhost:50052/api/archive --prune-url http://localhost:50052/api/prune --bucket <bucket>
 ```
 
-## Migrate Database
+## Migrate Database (bbolt -> pebble)
 
 ```
-go run cmd/migrate/main.go --db-dir <dir> --db-prefix <prefix>
+go run main.go migrate --bbolt-pth <bbolt_pth> --pebble-pth <pebble_pth>
 ```
 
-## Import Delivered Payloads
+## Import Database (postgres -> pebble)
 
 ```
-go run cmd/import/main.go --db-dir <dir> --db-prefix <prefix> --file <file> --sql-uri <sql-uri> --sql-table <dev_payload_delivered>
+go run main.go import --db-pth <db_pth> --sql-uri <sql_uri> --sql-table-prefix <dev>
 ```
 
 # Builder Specs

cmd/backup/main.go → cmd/backup.go

@@ -8,7 +8,7 @@
 // The above copyright notice and either this complete permission notice or at a minimum a reference to the UPL must be included in all copies or substantial portions of the Software.
 // THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 // This script ensures source code files have copyright license headers. See license.sh for more information.
-package main
+package cmd
 
 import (
 	"compress/gzip"
@@ -17,7 +17,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"os"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -29,10 +28,16 @@ import (
 	"go.uber.org/zap"
 )
 
-func main() {
-	app := &cli.App{
-		Usage: "create a backup of the db and upload it to s3",
+func Backup() *cli.Command {
+	return &cli.Command{
+		Name:  "backup",
+		Usage: `backup db to s3`,
 		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "sha-version",
+				Value:   "unknown",
+				EnvVars: []string{"SHA_VERSION"},
+			},
 			&cli.StringFlag{
 				Name:     "backup-url",
 				EnvVars:  []string{"BACKUP_URL"},
@@ -47,16 +52,13 @@ func main() {
 				Name:    "aws-uri",
 				EnvVars: []string{"AWS_URI"},
 			},
-			&cli.StringFlag{
-				Name:    "sha-version",
-				Value:   "unknown",
-				EnvVars: []string{"SHA_VERSION"},
-			},
 		},
 		Action: func(c *cli.Context) error {
 			defer zap.L().Sync() // nolint:errcheck
 			logger.SetVersion(c.String("sha-version"))
 
+			log := logger.WithValues("cmd", "backup")
+
 			ctx, cancel := context.WithCancel(c.Context)
 			defer cancel()
 
@@ -86,7 +88,7 @@ func main() {
 			sess := s3.NewFromConfig(cfg)
 			uploader := manager.NewUploader(sess)
 
-			logger.Info("creating backup", "url", c.String("backup-url"))
+			log.Info("creating backup", "url", c.String("backup-url"))
 			req, err := http.NewRequestWithContext(ctx, "GET", c.String("backup-url"), nil)
 			if err != nil {
 				return err
@@ -106,7 +108,7 @@ func main() {
 				return errors.New("no content-disposition")
 			}
 			filename := strings.TrimPrefix(cd, "attachment; filename=")
-			logger.Info("filename", "name", filename)
+			log.Info("filename", "name", filename)
 
 			r, w := io.Pipe()
 			defer r.Close() // nolint:errcheck
@@ -119,12 +121,12 @@ func main() {
 				defer gz.Close() // nolint:errcheck
 				_, err := io.Copy(gz, resp.Body)
 				if err != nil {
-					logger.Error(err, "write")
+					log.Error(err, "write")
 					cancel()
 				}
 			}()
 
-			logger.Info("uploading", "filename", filename)
+			log.Info("uploading", "filename", filename)
 			result, err := uploader.Upload(ctx, &s3.PutObjectInput{
 				Bucket: aws.String(c.String("bucket")),
 				Key:    aws.String(fmt.Sprintf("%s.gz", filename)),
@@ -133,14 +135,9 @@ func main() {
 			if err != nil {
 				return err
 			}
-			logger.Info("uploaded to", "location", result.Location)
+			log.Info("uploaded to", "location", result.Location)
 
 			return nil
 		},
 	}
-
-	if err := app.Run(os.Args); err != nil {
-		logger.Error(err, "run")
-		os.Exit(1)
-	}
 }