Skip to content

Commit

Permalink
feat(coordinator): split the coordinator cron to single process (#995)
Browse files Browse the repository at this point in the history
Co-authored-by: maskpp <[email protected]>
Co-authored-by: HAOYUatHZ <[email protected]>
Co-authored-by: HAOYUatHZ <[email protected]>
  • Loading branch information
4 people authored Oct 26, 2023
1 parent 8494ab1 commit 50040a1
Show file tree
Hide file tree
Showing 20 changed files with 217 additions and 65 deletions.
27 changes: 24 additions & 3 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ jobs:
tags: scrolltech/bridgehistoryapi-server:${{github.ref_name}}
# cache-from: type=gha,scope=${{ github.workflow }}
# cache-to: type=gha,scope=${{ github.workflow }}
coordinator:
coordinator-api:
runs-on: ubuntu-latest
steps:
- name: Checkout code
Expand All @@ -127,8 +127,29 @@ jobs:
uses: docker/build-push-action@v2
with:
context: .
file: ./build/dockerfiles/coordinator.Dockerfile
file: ./build/dockerfiles/coordinator-api.Dockerfile
push: true
tags: scrolltech/coordinator:${{github.ref_name}}
tags: scrolltech/coordinator-api:${{github.ref_name}}
# cache-from: type=gha,scope=${{ github.workflow }}
# cache-to: type=gha,scope=${{ github.workflow }}
coordinator-cron:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push coordinator docker
uses: docker/build-push-action@v2
with:
context: .
file: ./build/dockerfiles/coordinator-cron.Dockerfile
push: true
tags: scrolltech/coordinator-cron:${{github.ref_name}}
# cache-from: type=gha,scope=${{ github.workflow }}
# cache-to: type=gha,scope=${{ github.workflow }}
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,15 @@ COPY . .
RUN cp -r ./common/libzkp/interface ./coordinator/internal/logic/verifier/lib
COPY --from=zkp-builder /app/target/release/libzkp.so ./coordinator/internal/logic/verifier/lib/
COPY --from=zkp-builder /app/target/release/libzktrie.so ./coordinator/internal/logic/verifier/lib/
RUN cd ./coordinator && make coordinator_skip_libzkp && mv ./build/bin/coordinator /bin/coordinator && mv internal/logic/verifier/lib /bin/
RUN cd ./coordinator && make coordinator_api_skip_libzkp && mv ./build/bin/coordinator_api /bin/coordinator_api && mv internal/logic/verifier/lib /bin/

# Pull coordinator into a second stage deploy alpine container
FROM ubuntu:20.04
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/src/coordinator/internal/logic/verifier/lib
# ENV CHAIN_ID=534353
RUN mkdir -p /src/coordinator/internal/logic/verifier/lib
COPY --from=builder /bin/lib /src/coordinator/internal/logic/verifier/lib
COPY --from=builder /bin/coordinator /bin/
RUN /bin/coordinator --version
COPY --from=builder /bin/coordinator_api /bin/
RUN /bin/coordinator_api --version

ENTRYPOINT ["/bin/coordinator"]
ENTRYPOINT ["/bin/coordinator_api"]
25 changes: 25 additions & 0 deletions build/dockerfiles/coordinator-cron.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Download Go dependencies
FROM scrolltech/go-alpine-builder:1.19 as base

WORKDIR /src
COPY go.work* ./
COPY ./rollup/go.* ./rollup/
COPY ./common/go.* ./common/
COPY ./coordinator/go.* ./coordinator/
COPY ./database/go.* ./database/
COPY ./prover/go.* ./prover/
COPY ./tests/integration-test/go.* ./tests/integration-test/
COPY ./bridge-history-api/go.* ./bridge-history-api/
RUN go mod download -x

# Build coordinator
FROM base as builder
RUN --mount=target=. \
--mount=type=cache,target=/root/.cache/go-build \
cd /src/coordinator/cmd/cron/ && go build -v -p 4 -o /bin/coordinator_cron

# Pull coordinator into a second stage deploy alpine container
FROM alpine:latest
COPY --from=builder /bin/coordinator_cron /bin/

ENTRYPOINT ["coordinator_cron"]
6 changes: 6 additions & 0 deletions build/dockerfiles/coordinator-cron.Dockerfile.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
assets/
contracts/
docs/
l2geth/
rpc-gateway/
*target/*
6 changes: 4 additions & 2 deletions common/utils/simulation.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,10 @@ var (
// DBCliApp the name of mock database app.
DBCliApp MockAppName = "db_cli-test"

// CoordinatorApp the name of mock coordinator app.
CoordinatorApp MockAppName = "coordinator-test"
// CoordinatorAPIApp the name of mock coordinator app.
CoordinatorAPIApp MockAppName = "coordinator-api-test"
// CoordinatorCronApp the name of mock coordinator cron app.
CoordinatorCronApp MockAppName = "coordinator-cron-test"

// ChunkProverApp the name of mock chunk prover app.
ChunkProverApp MockAppName = "chunkProver-test"
Expand Down
2 changes: 1 addition & 1 deletion common/version/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"runtime/debug"
)

var tag = "v4.3.37"
var tag = "v4.3.38"

var commit = func() string {
if info, ok := debug.ReadBuildInfo(); ok {
Expand Down
25 changes: 16 additions & 9 deletions coordinator/Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
.PHONY: lint docker clean coordinator coordinator_skip_libzkp mock_coordinator

IMAGE_NAME=coordinator
IMAGE_VERSION=latest
REPO_ROOT_DIR=./..

Expand All @@ -22,14 +21,20 @@ libzkp:
rm -rf ./internal/logic/verifier/lib && cp -r ../common/libzkp/interface ./internal/logic/verifier/lib
find ../common | grep libzktrie.so | xargs -I{} cp {} ./internal/logic/verifier/lib

coordinator: libzkp ## Builds the Coordinator instance.
go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator ./cmd
coordinator_api: libzkp ## Builds the Coordinator api instance.
go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_api ./cmd/api

coordinator_skip_libzkp:
go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator ./cmd
coordinator_cron:
go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_cron ./cmd/cron

mock_coordinator: ## Builds the mocked Coordinator instance.
go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator ./cmd
coordinator_api_skip_libzkp:
go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_api ./cmd/api

mock_coordinator_api: ## Builds the mocked Coordinator instance.
go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator_api ./cmd/api

mock_coordinator_cron: ## Builds the mocked Coordinator instance.
go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator_cron ./cmd/cron

test-verifier: libzkp
go test -tags ffi -timeout 0 -v ./internal/logic/verifier
Expand All @@ -45,7 +50,9 @@ clean: ## Empty out the bin folder
@rm -rf build/bin

docker:
DOCKER_BUILDKIT=1 docker build -t scrolltech/${IMAGE_NAME}:${IMAGE_VERSION} ${REPO_ROOT_DIR}/ -f ${REPO_ROOT_DIR}/build/dockerfiles/coordinator.Dockerfile
DOCKER_BUILDKIT=1 docker build -t scrolltech/coordinator-api:${IMAGE_VERSION} ${REPO_ROOT_DIR}/ -f ${REPO_ROOT_DIR}/build/dockerfiles/coordinator-api.Dockerfile
DOCKER_BUILDKIT=1 docker build -t scrolltech/coordinator-cron:${IMAGE_VERSION} ${REPO_ROOT_DIR}/ -f ${REPO_ROOT_DIR}/build/dockerfiles/coordinator-cron.Dockerfile

docker_push:
docker push scrolltech/${IMAGE_NAME}:${IMAGE_VERSION}
docker push scrolltech/coordinator-api:${IMAGE_VERSION}
docker push scrolltech/coordinator-cron:${IMAGE_VERSION}
11 changes: 7 additions & 4 deletions coordinator/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ See [monorepo prerequisites](../README.md#prerequisites).

```bash
make clean
make coordinator
make coordinator_api
make coordinator_cron
```
The built coordinator binary is in the `build/bin` directory.

Expand Down Expand Up @@ -44,13 +45,15 @@ The coordinator behavior can be configured using [`config.json`](config.json). C

* Using default ports and config.json:
```bash
./build/bin/coordinator --http
./build/bin/coordinator_api --http
./build/bin/coordinator_cron
```

* Using manually specified ports and config.json:
```bash
./build/bin/coordinator --config ./config.json --http --http.addr localhost --http.port 8390
./build/bin/coordinator_api --config ./config.json --http --http.addr localhost --http.port 8390
./build/bin/coordinator_cron --config ./config.json
```

* For other flags, refer to [`cmd/app/flags.go`](cmd/app/flags.go).
* For other flags, refer to [`cmd/api/app/flags.go`](cmd/api/app/flags.go).

16 changes: 5 additions & 11 deletions coordinator/cmd/app/app.go → coordinator/cmd/api/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (

"scroll-tech/coordinator/internal/config"
"scroll-tech/coordinator/internal/controller/api"
"scroll-tech/coordinator/internal/controller/cron"
"scroll-tech/coordinator/internal/route"
)

Expand All @@ -41,7 +40,7 @@ func init() {
return utils.LogSetup(ctx)
}
// Register `coordinator-test` app for integration-test.
utils.RegisterSimulation(app, utils.CoordinatorApp)
utils.RegisterSimulation(app, utils.CoordinatorAPIApp)
}

func action(ctx *cli.Context) error {
Expand All @@ -51,28 +50,23 @@ func action(ctx *cli.Context) error {
log.Crit("failed to load config file", "config file", cfgFile, "error", err)
}

subCtx, cancel := context.WithCancel(ctx.Context)
db, err := database.InitDB(cfg.DB)
if err != nil {
log.Crit("failed to init db connection", "err", err)
}

registry := prometheus.DefaultRegisterer
observability.Server(ctx, db)

proofCollector := cron.NewCollector(subCtx, db, cfg, registry)
defer func() {
proofCollector.Stop()
cancel()
if err = database.CloseDB(db); err != nil {
log.Error("can not close db connection", "error", err)
}
}()

registry := prometheus.DefaultRegisterer
observability.Server(ctx, db)

apiSrv := apiServer(ctx, cfg, db, registry)

log.Info(
"coordinator start successfully",
"Start coordinator api successfully.",
"version", version.Version,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
)

func TestRunCoordinator(t *testing.T) {
coordinator := cmd.NewCmd("coordinator-test", "--version")
coordinator := cmd.NewCmd("coordinator-api-test", "--version")
defer coordinator.WaitExit()

// wait result
Expand Down
19 changes: 0 additions & 19 deletions coordinator/cmd/app/flags.go → coordinator/cmd/api/app/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,6 @@ var (
&httpEnabledFlag,
&httpListenAddrFlag,
&httpPortFlag,
// ws flags
&wsEnabledFlag,
&wsListenAddrFlag,
&wsPortFlag,
}
// httpEnabledFlag enable rpc server.
httpEnabledFlag = cli.BoolFlag{
Expand All @@ -31,19 +27,4 @@ var (
Usage: "HTTP-RPC server listening port",
Value: 8390,
}
wsEnabledFlag = cli.BoolFlag{
Name: "ws",
Usage: "Enable the WS-RPC server",
}
wsListenAddrFlag = cli.StringFlag{
Name: "ws.addr",
Usage: "WS-RPC server listening interface",
Value: "localhost",
}
// websocket port
wsPortFlag = cli.IntFlag{
Name: "ws.port",
Usage: "WS-RPC server listening port",
Value: 8391,
}
)
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ func NewCoordinatorApp(base *docker.App, file string) *CoordinatorApp {

// RunApp run coordinator-test child process by multi parameters.
func (c *CoordinatorApp) RunApp(t *testing.T, args ...string) {
c.AppAPI = cmd.NewCmd(string(utils.CoordinatorApp), append(c.args, args...)...)
c.AppAPI.RunApp(func() bool { return c.AppAPI.WaitResult(t, time.Second*20, "Start coordinator successfully") })
c.AppAPI = cmd.NewCmd(string(utils.CoordinatorAPIApp), append(c.args, args...)...)
c.AppAPI.RunApp(func() bool { return c.AppAPI.WaitResult(t, time.Second*20, "Start coordinator api successfully") })
}

// Free stop and release coordinator-test.
Expand Down
7 changes: 7 additions & 0 deletions coordinator/cmd/api/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package main

import "scroll-tech/coordinator/cmd/api/app"

func main() {
app.Run()
}
87 changes: 87 additions & 0 deletions coordinator/cmd/cron/app/app.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package app

import (
"context"
"fmt"
"os"
"os/signal"

"github.com/prometheus/client_golang/prometheus"
"github.com/scroll-tech/go-ethereum/log"
"github.com/urfave/cli/v2"

"scroll-tech/common/database"
"scroll-tech/common/observability"
"scroll-tech/common/utils"
"scroll-tech/common/version"

"scroll-tech/coordinator/internal/config"
"scroll-tech/coordinator/internal/controller/cron"
)

var app *cli.App

func init() {
// Set up coordinator app info.
app = cli.NewApp()
app.Action = action
app.Name = "coordinator cron"
app.Usage = "The Scroll L2 Coordinator cron"
app.Version = version.Version
app.Flags = append(app.Flags, utils.CommonFlags...)
app.Before = func(ctx *cli.Context) error {
return utils.LogSetup(ctx)
}
// Register `coordinator-cron-test` app for integration-cron-test.
utils.RegisterSimulation(app, utils.CoordinatorCronApp)
}

func action(ctx *cli.Context) error {
cfgFile := ctx.String(utils.ConfigFileFlag.Name)
cfg, err := config.NewConfig(cfgFile)
if err != nil {
log.Crit("failed to load config file", "config file", cfgFile, "error", err)
}

subCtx, cancel := context.WithCancel(ctx.Context)
db, err := database.InitDB(cfg.DB)
if err != nil {
log.Crit("failed to init db connection", "err", err)
}

registry := prometheus.DefaultRegisterer
observability.Server(ctx, db)

proofCollector := cron.NewCollector(subCtx, db, cfg, registry)
defer func() {
proofCollector.Stop()
cancel()
if err = database.CloseDB(db); err != nil {
log.Error("can not close db connection", "error", err)
}
}()

log.Info(
"coordinator cron start successfully",
"version", version.Version,
)

// Catch CTRL-C to ensure a graceful shutdown.
interrupt := make(chan os.Signal, 1)
signal.Notify(interrupt, os.Interrupt)

// Wait until the interrupt signal is received from an OS signal.
<-interrupt

log.Info("coordinator cron exiting success")
return nil
}

// Run coordinator.
func Run() {
// RunApp the coordinator.
if err := app.Run(os.Args); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
Loading

0 comments on commit 50040a1

Please sign in to comment.