From 50040a164e2c8032303fdede927d6559b6139b5d Mon Sep 17 00:00:00 2001 From: georgehao Date: Thu, 26 Oct 2023 13:15:44 +0800 Subject: [PATCH] feat(coordinator): split the coordinator cron to single process (#995) Co-authored-by: maskpp Co-authored-by: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com> Co-authored-by: HAOYUatHZ --- .github/workflows/docker.yml | 27 +++++- ....Dockerfile => coordinator-api.Dockerfile} | 8 +- ...> coordinator-api.Dockerfile.dockerignore} | 0 build/dockerfiles/coordinator-cron.Dockerfile | 25 ++++++ .../coordinator-cron.Dockerfile.dockerignore | 6 ++ common/utils/simulation.go | 6 +- common/version/version.go | 2 +- coordinator/Makefile | 25 ++++-- coordinator/README.md | 11 ++- coordinator/cmd/{ => api}/app/app.go | 16 ++-- coordinator/cmd/{ => api}/app/app_test.go | 2 +- coordinator/cmd/{ => api}/app/flags.go | 19 ---- coordinator/cmd/{ => api}/app/mock_app.go | 4 +- coordinator/cmd/api/main.go | 7 ++ coordinator/cmd/cron/app/app.go | 87 +++++++++++++++++++ coordinator/cmd/cron/app/app_test.go | 19 ++++ coordinator/cmd/cron/main.go | 7 ++ coordinator/cmd/main.go | 7 -- .../internal/controller/cron/collect_proof.go | 2 +- tests/integration-test/integration_test.go | 2 +- 20 files changed, 217 insertions(+), 65 deletions(-) rename build/dockerfiles/{coordinator.Dockerfile => coordinator-api.Dockerfile} (85%) rename build/dockerfiles/{coordinator.Dockerfile.dockerignore => coordinator-api.Dockerfile.dockerignore} (100%) create mode 100644 build/dockerfiles/coordinator-cron.Dockerfile create mode 100644 build/dockerfiles/coordinator-cron.Dockerfile.dockerignore rename coordinator/cmd/{ => api}/app/app.go (90%) rename coordinator/cmd/{ => api}/app/app_test.go (83%) rename coordinator/cmd/{ => api}/app/flags.go (60%) rename coordinator/cmd/{ => api}/app/mock_app.go (94%) create mode 100644 coordinator/cmd/api/main.go create mode 100644 coordinator/cmd/cron/app/app.go create mode 100644 coordinator/cmd/cron/app/app_test.go create mode 100644 coordinator/cmd/cron/main.go delete mode 100644 coordinator/cmd/main.go diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 552295b347..7ea01174be 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -111,7 +111,7 @@ jobs: tags: scrolltech/bridgehistoryapi-server:${{github.ref_name}} # cache-from: type=gha,scope=${{ github.workflow }} # cache-to: type=gha,scope=${{ github.workflow }} - coordinator: + coordinator-api: runs-on: ubuntu-latest steps: - name: Checkout code @@ -127,8 +127,29 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./build/dockerfiles/coordinator.Dockerfile + file: ./build/dockerfiles/coordinator-api.Dockerfile push: true - tags: scrolltech/coordinator:${{github.ref_name}} + tags: scrolltech/coordinator-api:${{github.ref_name}} + # cache-from: type=gha,scope=${{ github.workflow }} + # cache-to: type=gha,scope=${{ github.workflow }} + coordinator-cron: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push coordinator docker + uses: docker/build-push-action@v2 + with: + context: . + file: ./build/dockerfiles/coordinator-cron.Dockerfile + push: true + tags: scrolltech/coordinator-cron:${{github.ref_name}} # cache-from: type=gha,scope=${{ github.workflow }} # cache-to: type=gha,scope=${{ github.workflow }} diff --git a/build/dockerfiles/coordinator.Dockerfile b/build/dockerfiles/coordinator-api.Dockerfile similarity index 85% rename from build/dockerfiles/coordinator.Dockerfile rename to build/dockerfiles/coordinator-api.Dockerfile index 0f501f771c..4ebdb14540 100644 --- a/build/dockerfiles/coordinator.Dockerfile +++ b/build/dockerfiles/coordinator-api.Dockerfile @@ -36,7 +36,7 @@ COPY . . RUN cp -r ./common/libzkp/interface ./coordinator/internal/logic/verifier/lib COPY --from=zkp-builder /app/target/release/libzkp.so ./coordinator/internal/logic/verifier/lib/ COPY --from=zkp-builder /app/target/release/libzktrie.so ./coordinator/internal/logic/verifier/lib/ -RUN cd ./coordinator && make coordinator_skip_libzkp && mv ./build/bin/coordinator /bin/coordinator && mv internal/logic/verifier/lib /bin/ +RUN cd ./coordinator && make coordinator_api_skip_libzkp && mv ./build/bin/coordinator_api /bin/coordinator_api && mv internal/logic/verifier/lib /bin/ # Pull coordinator into a second stage deploy alpine container FROM ubuntu:20.04 @@ -44,7 +44,7 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/src/coordinator/internal/logic/verifier/li # ENV CHAIN_ID=534353 RUN mkdir -p /src/coordinator/internal/logic/verifier/lib COPY --from=builder /bin/lib /src/coordinator/internal/logic/verifier/lib -COPY --from=builder /bin/coordinator /bin/ -RUN /bin/coordinator --version +COPY --from=builder /bin/coordinator_api /bin/ +RUN /bin/coordinator_api --version -ENTRYPOINT ["/bin/coordinator"] +ENTRYPOINT ["/bin/coordinator_api"] diff --git a/build/dockerfiles/coordinator.Dockerfile.dockerignore b/build/dockerfiles/coordinator-api.Dockerfile.dockerignore similarity index 100% rename from build/dockerfiles/coordinator.Dockerfile.dockerignore rename to build/dockerfiles/coordinator-api.Dockerfile.dockerignore diff --git a/build/dockerfiles/coordinator-cron.Dockerfile b/build/dockerfiles/coordinator-cron.Dockerfile new file mode 100644 index 0000000000..ae1d700638 --- /dev/null +++ b/build/dockerfiles/coordinator-cron.Dockerfile @@ -0,0 +1,25 @@ +# Download Go dependencies +FROM scrolltech/go-alpine-builder:1.19 as base + +WORKDIR /src +COPY go.work* ./ +COPY ./rollup/go.* ./rollup/ +COPY ./common/go.* ./common/ +COPY ./coordinator/go.* ./coordinator/ +COPY ./database/go.* ./database/ +COPY ./prover/go.* ./prover/ +COPY ./tests/integration-test/go.* ./tests/integration-test/ +COPY ./bridge-history-api/go.* ./bridge-history-api/ +RUN go mod download -x + +# Build coordinator +FROM base as builder +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + cd /src/coordinator/cmd/cron/ && go build -v -p 4 -o /bin/coordinator_cron + +# Pull coordinator into a second stage deploy alpine container +FROM alpine:latest +COPY --from=builder /bin/coordinator_cron /bin/ + +ENTRYPOINT ["coordinator_cron"] \ No newline at end of file diff --git a/build/dockerfiles/coordinator-cron.Dockerfile.dockerignore b/build/dockerfiles/coordinator-cron.Dockerfile.dockerignore new file mode 100644 index 0000000000..e67a9b0efe --- /dev/null +++ b/build/dockerfiles/coordinator-cron.Dockerfile.dockerignore @@ -0,0 +1,6 @@ +assets/ +contracts/ +docs/ +l2geth/ +rpc-gateway/ +*target/* diff --git a/common/utils/simulation.go b/common/utils/simulation.go index 320cdafc0f..bff7ff4acc 100644 --- a/common/utils/simulation.go +++ b/common/utils/simulation.go @@ -24,8 +24,10 @@ var ( // DBCliApp the name of mock database app. DBCliApp MockAppName = "db_cli-test" - // CoordinatorApp the name of mock coordinator app. - CoordinatorApp MockAppName = "coordinator-test" + // CoordinatorAPIApp the name of mock coordinator app. + CoordinatorAPIApp MockAppName = "coordinator-api-test" + // CoordinatorCronApp the name of mock coordinator cron app. + CoordinatorCronApp MockAppName = "coordinator-cron-test" // ChunkProverApp the name of mock chunk prover app. ChunkProverApp MockAppName = "chunkProver-test" diff --git a/common/version/version.go b/common/version/version.go index c32707b1f1..1b0853d562 100644 --- a/common/version/version.go +++ b/common/version/version.go @@ -5,7 +5,7 @@ import ( "runtime/debug" ) -var tag = "v4.3.37" +var tag = "v4.3.38" var commit = func() string { if info, ok := debug.ReadBuildInfo(); ok { diff --git a/coordinator/Makefile b/coordinator/Makefile index b9fe1d35b6..893c51aefd 100644 --- a/coordinator/Makefile +++ b/coordinator/Makefile @@ -1,6 +1,5 @@ .PHONY: lint docker clean coordinator coordinator_skip_libzkp mock_coordinator -IMAGE_NAME=coordinator IMAGE_VERSION=latest REPO_ROOT_DIR=./.. @@ -22,14 +21,20 @@ libzkp: rm -rf ./internal/logic/verifier/lib && cp -r ../common/libzkp/interface ./internal/logic/verifier/lib find ../common | grep libzktrie.so | xargs -I{} cp {} ./internal/logic/verifier/lib -coordinator: libzkp ## Builds the Coordinator instance. - go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator ./cmd +coordinator_api: libzkp ## Builds the Coordinator api instance. + go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_api ./cmd/api -coordinator_skip_libzkp: - go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator ./cmd +coordinator_cron: + go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_cron ./cmd/cron -mock_coordinator: ## Builds the mocked Coordinator instance. - go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator ./cmd +coordinator_api_skip_libzkp: + go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_api ./cmd/api + +mock_coordinator_api: ## Builds the mocked Coordinator instance. + go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator_api ./cmd/api + +mock_coordinator_cron: ## Builds the mocked Coordinator instance. + go build -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator_cron ./cmd/cron test-verifier: libzkp go test -tags ffi -timeout 0 -v ./internal/logic/verifier @@ -45,7 +50,9 @@ clean: ## Empty out the bin folder @rm -rf build/bin docker: - DOCKER_BUILDKIT=1 docker build -t scrolltech/${IMAGE_NAME}:${IMAGE_VERSION} ${REPO_ROOT_DIR}/ -f ${REPO_ROOT_DIR}/build/dockerfiles/coordinator.Dockerfile + DOCKER_BUILDKIT=1 docker build -t scrolltech/coordinator-api:${IMAGE_VERSION} ${REPO_ROOT_DIR}/ -f ${REPO_ROOT_DIR}/build/dockerfiles/coordinator-api.Dockerfile + DOCKER_BUILDKIT=1 docker build -t scrolltech/coordinator-cron:${IMAGE_VERSION} ${REPO_ROOT_DIR}/ -f ${REPO_ROOT_DIR}/build/dockerfiles/coordinator-cron.Dockerfile docker_push: - docker push scrolltech/${IMAGE_NAME}:${IMAGE_VERSION} \ No newline at end of file + docker push scrolltech/coordinator-api:${IMAGE_VERSION} + docker push scrolltech/coordinator-cron:${IMAGE_VERSION} \ No newline at end of file diff --git a/coordinator/README.md b/coordinator/README.md index 7d9e207287..904f5abd7c 100644 --- a/coordinator/README.md +++ b/coordinator/README.md @@ -12,7 +12,8 @@ See [monorepo prerequisites](../README.md#prerequisites). ```bash make clean -make coordinator +make coordinator_api +make coordinator_cron ``` The built coordinator binary is in the `build/bin` directory. @@ -44,13 +45,15 @@ The coordinator behavior can be configured using [`config.json`](config.json). C * Using default ports and config.json: ```bash -./build/bin/coordinator --http +./build/bin/coordinator_api --http +./build/bin/coordinator_cron ``` * Using manually specified ports and config.json: ```bash -./build/bin/coordinator --config ./config.json --http --http.addr localhost --http.port 8390 +./build/bin/coordinator_api --config ./config.json --http --http.addr localhost --http.port 8390 +./build/bin/coordinator_cron --config ./config.json ``` -* For other flags, refer to [`cmd/app/flags.go`](cmd/app/flags.go). +* For other flags, refer to [`cmd/api/app/flags.go`](cmd/api/app/flags.go). diff --git a/coordinator/cmd/app/app.go b/coordinator/cmd/api/app/app.go similarity index 90% rename from coordinator/cmd/app/app.go rename to coordinator/cmd/api/app/app.go index 16d7b385d2..10c6c46c8a 100644 --- a/coordinator/cmd/app/app.go +++ b/coordinator/cmd/api/app/app.go @@ -22,7 +22,6 @@ import ( "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/api" - "scroll-tech/coordinator/internal/controller/cron" "scroll-tech/coordinator/internal/route" ) @@ -41,7 +40,7 @@ func init() { return utils.LogSetup(ctx) } // Register `coordinator-test` app for integration-test. - utils.RegisterSimulation(app, utils.CoordinatorApp) + utils.RegisterSimulation(app, utils.CoordinatorAPIApp) } func action(ctx *cli.Context) error { @@ -51,28 +50,23 @@ func action(ctx *cli.Context) error { log.Crit("failed to load config file", "config file", cfgFile, "error", err) } - subCtx, cancel := context.WithCancel(ctx.Context) db, err := database.InitDB(cfg.DB) if err != nil { log.Crit("failed to init db connection", "err", err) } - - registry := prometheus.DefaultRegisterer - observability.Server(ctx, db) - - proofCollector := cron.NewCollector(subCtx, db, cfg, registry) defer func() { - proofCollector.Stop() - cancel() if err = database.CloseDB(db); err != nil { log.Error("can not close db connection", "error", err) } }() + registry := prometheus.DefaultRegisterer + observability.Server(ctx, db) + apiSrv := apiServer(ctx, cfg, db, registry) log.Info( - "coordinator start successfully", + "Start coordinator api successfully.", "version", version.Version, ) diff --git a/coordinator/cmd/app/app_test.go b/coordinator/cmd/api/app/app_test.go similarity index 83% rename from coordinator/cmd/app/app_test.go rename to coordinator/cmd/api/app/app_test.go index 146fbc863c..b726a10cc3 100644 --- a/coordinator/cmd/app/app_test.go +++ b/coordinator/cmd/api/app/app_test.go @@ -10,7 +10,7 @@ import ( ) func TestRunCoordinator(t *testing.T) { - coordinator := cmd.NewCmd("coordinator-test", "--version") + coordinator := cmd.NewCmd("coordinator-api-test", "--version") defer coordinator.WaitExit() // wait result diff --git a/coordinator/cmd/app/flags.go b/coordinator/cmd/api/app/flags.go similarity index 60% rename from coordinator/cmd/app/flags.go rename to coordinator/cmd/api/app/flags.go index 68ca985e8a..62a30bfadc 100644 --- a/coordinator/cmd/app/flags.go +++ b/coordinator/cmd/api/app/flags.go @@ -8,10 +8,6 @@ var ( &httpEnabledFlag, &httpListenAddrFlag, &httpPortFlag, - // ws flags - &wsEnabledFlag, - &wsListenAddrFlag, - &wsPortFlag, } // httpEnabledFlag enable rpc server. httpEnabledFlag = cli.BoolFlag{ @@ -31,19 +27,4 @@ var ( Usage: "HTTP-RPC server listening port", Value: 8390, } - wsEnabledFlag = cli.BoolFlag{ - Name: "ws", - Usage: "Enable the WS-RPC server", - } - wsListenAddrFlag = cli.StringFlag{ - Name: "ws.addr", - Usage: "WS-RPC server listening interface", - Value: "localhost", - } - // websocket port - wsPortFlag = cli.IntFlag{ - Name: "ws.port", - Usage: "WS-RPC server listening port", - Value: 8391, - } ) diff --git a/coordinator/cmd/app/mock_app.go b/coordinator/cmd/api/app/mock_app.go similarity index 94% rename from coordinator/cmd/app/mock_app.go rename to coordinator/cmd/api/app/mock_app.go index 47f060e589..dafaeba25b 100644 --- a/coordinator/cmd/app/mock_app.go +++ b/coordinator/cmd/api/app/mock_app.go @@ -55,8 +55,8 @@ func NewCoordinatorApp(base *docker.App, file string) *CoordinatorApp { // RunApp run coordinator-test child process by multi parameters. func (c *CoordinatorApp) RunApp(t *testing.T, args ...string) { - c.AppAPI = cmd.NewCmd(string(utils.CoordinatorApp), append(c.args, args...)...) - c.AppAPI.RunApp(func() bool { return c.AppAPI.WaitResult(t, time.Second*20, "Start coordinator successfully") }) + c.AppAPI = cmd.NewCmd(string(utils.CoordinatorAPIApp), append(c.args, args...)...) + c.AppAPI.RunApp(func() bool { return c.AppAPI.WaitResult(t, time.Second*20, "Start coordinator api successfully") }) } // Free stop and release coordinator-test. diff --git a/coordinator/cmd/api/main.go b/coordinator/cmd/api/main.go new file mode 100644 index 0000000000..9b4c40be42 --- /dev/null +++ b/coordinator/cmd/api/main.go @@ -0,0 +1,7 @@ +package main + +import "scroll-tech/coordinator/cmd/api/app" + +func main() { + app.Run() +} diff --git a/coordinator/cmd/cron/app/app.go b/coordinator/cmd/cron/app/app.go new file mode 100644 index 0000000000..92e5e5992e --- /dev/null +++ b/coordinator/cmd/cron/app/app.go @@ -0,0 +1,87 @@ +package app + +import ( + "context" + "fmt" + "os" + "os/signal" + + "github.com/prometheus/client_golang/prometheus" + "github.com/scroll-tech/go-ethereum/log" + "github.com/urfave/cli/v2" + + "scroll-tech/common/database" + "scroll-tech/common/observability" + "scroll-tech/common/utils" + "scroll-tech/common/version" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/controller/cron" +) + +var app *cli.App + +func init() { + // Set up coordinator app info. + app = cli.NewApp() + app.Action = action + app.Name = "coordinator cron" + app.Usage = "The Scroll L2 Coordinator cron" + app.Version = version.Version + app.Flags = append(app.Flags, utils.CommonFlags...) + app.Before = func(ctx *cli.Context) error { + return utils.LogSetup(ctx) + } + // Register `coordinator-cron-test` app for integration-cron-test. + utils.RegisterSimulation(app, utils.CoordinatorCronApp) +} + +func action(ctx *cli.Context) error { + cfgFile := ctx.String(utils.ConfigFileFlag.Name) + cfg, err := config.NewConfig(cfgFile) + if err != nil { + log.Crit("failed to load config file", "config file", cfgFile, "error", err) + } + + subCtx, cancel := context.WithCancel(ctx.Context) + db, err := database.InitDB(cfg.DB) + if err != nil { + log.Crit("failed to init db connection", "err", err) + } + + registry := prometheus.DefaultRegisterer + observability.Server(ctx, db) + + proofCollector := cron.NewCollector(subCtx, db, cfg, registry) + defer func() { + proofCollector.Stop() + cancel() + if err = database.CloseDB(db); err != nil { + log.Error("can not close db connection", "error", err) + } + }() + + log.Info( + "coordinator cron start successfully", + "version", version.Version, + ) + + // Catch CTRL-C to ensure a graceful shutdown. + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + + // Wait until the interrupt signal is received from an OS signal. + <-interrupt + + log.Info("coordinator cron exiting success") + return nil +} + +// Run coordinator. +func Run() { + // RunApp the coordinator. + if err := app.Run(os.Args); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/coordinator/cmd/cron/app/app_test.go b/coordinator/cmd/cron/app/app_test.go new file mode 100644 index 0000000000..97d30e5649 --- /dev/null +++ b/coordinator/cmd/cron/app/app_test.go @@ -0,0 +1,19 @@ +package app + +import ( + "fmt" + "testing" + "time" + + "scroll-tech/common/cmd" + "scroll-tech/common/version" +) + +func TestRunCoordinatorCron(t *testing.T) { + coordinator := cmd.NewCmd("coordinator-cron-test", "--version") + defer coordinator.WaitExit() + + // wait result + coordinator.ExpectWithTimeout(t, true, time.Second*3, fmt.Sprintf("coordinator cron version %s", version.Version)) + coordinator.RunApp(nil) +} diff --git a/coordinator/cmd/cron/main.go b/coordinator/cmd/cron/main.go new file mode 100644 index 0000000000..512aff4ad8 --- /dev/null +++ b/coordinator/cmd/cron/main.go @@ -0,0 +1,7 @@ +package main + +import "scroll-tech/coordinator/cmd/cron/app" + +func main() { + app.Run() +} diff --git a/coordinator/cmd/main.go b/coordinator/cmd/main.go deleted file mode 100644 index 29a0632ca5..0000000000 --- a/coordinator/cmd/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "scroll-tech/coordinator/cmd/app" - -func main() { - app.Run() -} diff --git a/coordinator/internal/controller/cron/collect_proof.go b/coordinator/internal/controller/cron/collect_proof.go index b27f6a6a4f..734995846b 100644 --- a/coordinator/internal/controller/cron/collect_proof.go +++ b/coordinator/internal/controller/cron/collect_proof.go @@ -76,7 +76,7 @@ func NewCollector(ctx context.Context, db *gorm.DB, cfg *config.Config, reg prom go c.checkBatchAllChunkReady() go c.cleanupChallenge() - log.Info("Start coordinator successfully.") + log.Info("Start coordinator cron successfully.") return c } diff --git a/tests/integration-test/integration_test.go b/tests/integration-test/integration_test.go index 996b04c0ec..e15a5cccba 100644 --- a/tests/integration-test/integration_test.go +++ b/tests/integration-test/integration_test.go @@ -17,7 +17,7 @@ import ( "scroll-tech/database/migrate" - capp "scroll-tech/coordinator/cmd/app" + capp "scroll-tech/coordinator/cmd/api/app" "scroll-tech/common/database" "scroll-tech/common/docker"