From 5dbb5c5fb7a2fcdc323cb743e2b54d20ed5454c8 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 14:54:54 +0900 Subject: [PATCH 01/43] extend api for proxy --- coordinator/internal/controller/api/auth.go | 41 +++++++++++++++------ coordinator/internal/logic/auth/login.go | 24 ++++++++---- coordinator/internal/types/prover.go | 4 ++ 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/coordinator/internal/controller/api/auth.go b/coordinator/internal/controller/api/auth.go index b38f0d827a..cdbb8225d8 100644 --- a/coordinator/internal/controller/api/auth.go +++ b/coordinator/internal/controller/api/auth.go @@ -26,21 +26,43 @@ func NewAuthController(db *gorm.DB, cfg *config.Config, vf *verifier.Verifier) * } } -// Login the api controller for login +// Login the api controller for login, used as the Authenticator in JWT +// It can work in two mode: full process for normal login, or if login request +// is posted from proxy, run a simpler process to login a client func (a *AuthController) Login(c *gin.Context) (interface{}, error) { + + // check if the login is post by proxy + var viaProxy bool + if proverType, proverTypeExist := c.Get(types.ProverProviderTypeKey); proverTypeExist { + proverType := uint8(proverType.(float64)) + viaProxy = proverType == types.ProverProviderTypeProxy + } + var login types.LoginParameter if err := c.ShouldBind(&login); err != nil { return "", fmt.Errorf("missing the public_key, err:%w", err) } - // check login parameter's token is equal to bearer token, the Authorization must be existed - // if not exist, the jwt token will intercept it - brearToken := c.GetHeader("Authorization") - if brearToken != "Bearer "+login.Message.Challenge { - return "", errors.New("check challenge failure for the not equal challenge string") + // if not, process with normal login + if !viaProxy { + // check login parameter's token is equal to bearer token, the Authorization must be existed + // if not exist, the jwt token will intercept it + brearToken := c.GetHeader("Authorization") + if brearToken != "Bearer "+login.Message.Challenge { + return "", errors.New("check challenge failure for the not equal challenge string") + } + + if err := a.loginLogic.VerifyMsg(&login); err != nil { + return "", err + } + + // check the challenge is used, if used, return failure + if err := a.loginLogic.InsertChallengeString(c, login.Message.Challenge); err != nil { + return "", fmt.Errorf("login insert challenge string failure:%w", err) + } } - if err := a.loginLogic.Check(&login); err != nil { + if err := a.loginLogic.CompatiblityCheck(&login); err != nil { return "", fmt.Errorf("check the login parameter failure: %w", err) } @@ -49,11 +71,6 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { return "", fmt.Errorf("prover hard fork name failure:%w", err) } - // check the challenge is used, if used, return failure - if err := a.loginLogic.InsertChallengeString(c, login.Message.Challenge); err != nil { - return "", fmt.Errorf("login insert challenge string failure:%w", err) - } - returnData := types.LoginParameterWithHardForkName{ HardForkName: hardForkNames, LoginParameter: login, diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index f5c01a2dfe..0d9222dff6 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -50,13 +50,19 @@ func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) e return l.challengeOrm.InsertChallenge(ctx.Copy(), challenge) } -func (l *LoginLogic) Check(login *types.LoginParameter) error { +// Verify the completeness of login message +func (l *LoginLogic) VerifyMsg(login *types.LoginParameter) error { verify, err := login.Verify() if err != nil || !verify { log.Error("auth message verify failure", "prover_name", login.Message.ProverName, "prover_version", login.Message.ProverVersion, "message", login.Message) return errors.New("auth message verify failure") } + return nil +} + +// Check if the login client is compatible with the setting in coordinator +func (l *LoginLogic) CompatiblityCheck(login *types.LoginParameter) error { if !version.CheckScrollRepoVersion(login.Message.ProverVersion, l.cfg.ProverManager.Verifier.MinProverVersion) { return fmt.Errorf("incompatible prover version. please upgrade your prover, minimum allowed version: %s, actual version: %s", l.cfg.ProverManager.Verifier.MinProverVersion, login.Message.ProverVersion) @@ -80,14 +86,16 @@ func (l *LoginLogic) Check(login *types.LoginParameter) error { } } - if login.Message.ProverProviderType != types.ProverProviderTypeInternal && login.Message.ProverProviderType != types.ProverProviderTypeExternal { + switch login.Message.ProverProviderType { + case types.ProverProviderTypeInternal: + case types.ProverProviderTypeExternal: + case types.ProverProviderTypeProxy: + case types.ProverProviderTypeUndefined: // for backward compatibility, set ProverProviderType as internal - if login.Message.ProverProviderType == types.ProverProviderTypeUndefined { - login.Message.ProverProviderType = types.ProverProviderTypeInternal - } else { - log.Error("invalid prover_provider_type", "value", login.Message.ProverProviderType, "prover name", login.Message.ProverName, "prover version", login.Message.ProverVersion) - return errors.New("invalid prover provider type.") - } + login.Message.ProverProviderType = types.ProverProviderTypeInternal + default: + log.Error("invalid prover_provider_type", "value", login.Message.ProverProviderType, "prover name", login.Message.ProverName, "prover version", login.Message.ProverVersion) + return errors.New("invalid prover provider type.") } return nil diff --git a/coordinator/internal/types/prover.go b/coordinator/internal/types/prover.go index 048fac00a2..4254c673d5 100644 --- a/coordinator/internal/types/prover.go +++ b/coordinator/internal/types/prover.go @@ -64,6 +64,8 @@ func (r ProverProviderType) String() string { return "prover provider type internal" case ProverProviderTypeExternal: return "prover provider type external" + case ProverProviderTypeProxy: + return "prover provider type proxy" default: return fmt.Sprintf("prover provider type: %d", r) } @@ -76,4 +78,6 @@ const ( ProverProviderTypeInternal // ProverProviderTypeExternal is an external prover provider type ProverProviderTypeExternal + // ProverProviderTypeProxy is an proxy prover provider type + ProverProviderTypeProxy = 3 ) From 1f2b85767179ee14d92e59b8debb30f6474fdd50 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 15:35:51 +0900 Subject: [PATCH 02/43] add proxy_login route --- coordinator/internal/route/route.go | 1 + 1 file changed, 1 insertion(+) diff --git a/coordinator/internal/route/route.go b/coordinator/internal/route/route.go index 9e9eef076e..2a0383aa8d 100644 --- a/coordinator/internal/route/route.go +++ b/coordinator/internal/route/route.go @@ -34,6 +34,7 @@ func v1(router *gin.RouterGroup, conf *config.Config) { // need jwt token api r.Use(loginMiddleware.MiddlewareFunc()) { + r.POST("/proxy_login", loginMiddleware.LoginHandler) r.POST("/get_task", api.GetTask.GetTasks) r.POST("/submit_proof", api.SubmitProof.SubmitProof) } From 9796d16f6c68d3811303cedafcf0d11816823591 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 20:32:11 +0900 Subject: [PATCH 03/43] WIP: update login logic and coordinator client --- coordinator/internal/controller/api/auth.go | 4 +- .../internal/controller/proxy/client.go | 214 ++++++++++++++++++ coordinator/internal/logic/auth/login.go | 26 +-- 3 files changed, 229 insertions(+), 15 deletions(-) create mode 100644 coordinator/internal/controller/proxy/client.go diff --git a/coordinator/internal/controller/api/auth.go b/coordinator/internal/controller/api/auth.go index cdbb8225d8..1a3305d2b1 100644 --- a/coordinator/internal/controller/api/auth.go +++ b/coordinator/internal/controller/api/auth.go @@ -22,7 +22,7 @@ type AuthController struct { // NewAuthController returns an LoginController instance func NewAuthController(db *gorm.DB, cfg *config.Config, vf *verifier.Verifier) *AuthController { return &AuthController{ - loginLogic: auth.NewLoginLogic(db, cfg, vf), + loginLogic: auth.NewLoginLogic(db, cfg.ProverManager.Verifier, vf), } } @@ -52,7 +52,7 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { return "", errors.New("check challenge failure for the not equal challenge string") } - if err := a.loginLogic.VerifyMsg(&login); err != nil { + if err := auth.VerifyMsg(&login); err != nil { return "", err } diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go new file mode 100644 index 0000000000..48d768ba80 --- /dev/null +++ b/coordinator/internal/controller/proxy/client.go @@ -0,0 +1,214 @@ +package proxy + +import ( + "bytes" + "crypto/ecdsa" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/crypto" + + "scroll-tech/coordinator/internal/types" +) + +// Client wraps an http client with a preset host for coordinator API calls +type Client struct { + httpClient *http.Client + host string + loginToken string +} + +// NewClient creates a new Client with the specified host +func NewClient(host string) *Client { + return &Client{ + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + host: host, + } +} + +// NewClientWithHTTPClient creates a new Client with a custom http.Client +func NewClientWithHTTPClient(host string, httpClient *http.Client) *Client { + return &Client{ + httpClient: httpClient, + host: host, + } +} + +// FullLogin performs the complete login process: get challenge then login +func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { + // Step 1: Get challenge + url := fmt.Sprintf("%s/v1/challenge", c.host) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create challenge request: %w", err) + } + + challengeResp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get challenge: %w", err) + } + defer challengeResp.Body.Close() + + if challengeResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("challenge request failed with status: %d", challengeResp.StatusCode) + } + + // Step 2: Parse challenge response + var loginSchema types.LoginSchema + if err := json.NewDecoder(challengeResp.Body).Decode(&loginSchema); err != nil { + return nil, fmt.Errorf("failed to parse challenge response: %w", err) + } + + // Step 3: Use the token from challenge as Bearer token for login + url = fmt.Sprintf("%s/v1/login", c.host) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal login parameter: %w", err) + } + + req, err = http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create login request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+loginSchema.Token) + + loginResp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform login request: %w", err) + } + + // Parse login response as LoginSchema and store the token + if loginResp.StatusCode == http.StatusOK { + var loginResult types.LoginSchema + if err := json.NewDecoder(loginResp.Body).Decode(&loginResult); err == nil { + c.loginToken = loginResult.Token + } + // Note: Body is consumed after decoding, caller should not read it again + return &loginResult, nil + } + + return nil, fmt.Errorf("login request failed with status: %d", loginResp.StatusCode) +} + +// ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter +func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) { + url := fmt.Sprintf("%s/v1/proxy_login", c.host) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal proxy login parameter: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create proxy login request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.loginToken) + + return c.httpClient.Do(req) +} + +// GetTask makes a POST request to /v1/get_task with GetTaskParameter +func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Response, error) { + url := fmt.Sprintf("%s/v1/get_task", c.host) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal get task parameter: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create get task request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + return c.httpClient.Do(req) +} + +// SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter +func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*http.Response, error) { + url := fmt.Sprintf("%s/v1/submit_proof", c.host) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal submit proof parameter: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create submit proof request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + return c.httpClient.Do(req) +} + +// transformToValidPrivateKey safely transforms arbitrary bytes into valid private key bytes +func (c *Client) buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { + // Try appending bytes from 0x0 to 0x20 until we get a valid private key + for appendByte := byte(0x0); appendByte <= 0x20; appendByte++ { + // Append the byte to input + extendedBytes := append(inputBytes, appendByte) + + // Calculate 256-bit hash + hash := crypto.Keccak256(extendedBytes) + + // Try to create private key from hash + if k, err := crypto.ToECDSA(hash); err == nil { + return k, nil + } + } + + return nil, fmt.Errorf("failed to generate valid private key from input bytes") +} + +func (c *Client) generateLoginParameter(privateKeyBytes []byte, challenge string) (*types.LoginParameter, error) { + // Generate private key + privKey, err := c.buildPrivateKey(privateKeyBytes) + if err != nil { + return nil, err + } + + // Generate public key string + publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&privKey.PublicKey)) + + // Create login parameter with proxy settings + loginParam := &types.LoginParameter{ + Message: types.Message{ + Challenge: challenge, + ProverName: "proxy", + ProverVersion: "proxy", + ProverProviderType: types.ProverProviderTypeProxy, + ProverTypes: []types.ProverType{}, // Default empty + VKs: []string{}, // Default empty + }, + PublicKey: publicKeyHex, + } + + // Sign the message with the private key + if err := loginParam.SignWithKey(privKey); err != nil { + return nil, fmt.Errorf("failed to sign login parameter: %w", err) + } + + return loginParam, nil +} diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index 0d9222dff6..35350ef1ea 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -19,7 +19,7 @@ import ( // LoginLogic the auth logic type LoginLogic struct { - cfg *config.Config + cfg *config.VerifierConfig challengeOrm *orm.Challenge openVmVks map[string]struct{} @@ -28,30 +28,25 @@ type LoginLogic struct { } // NewLoginLogic new a LoginLogic -func NewLoginLogic(db *gorm.DB, cfg *config.Config, vf *verifier.Verifier) *LoginLogic { +func NewLoginLogic(db *gorm.DB, vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { proverVersionHardForkMap := make(map[string][]string) var hardForks []string - for _, cfg := range cfg.ProverManager.Verifier.Verifiers { + for _, cfg := range vcfg.Verifiers { hardForks = append(hardForks, cfg.ForkName) } - proverVersionHardForkMap[cfg.ProverManager.Verifier.MinProverVersion] = hardForks + proverVersionHardForkMap[vcfg.MinProverVersion] = hardForks return &LoginLogic{ - cfg: cfg, + cfg: vcfg, openVmVks: vf.OpenVMVkMap, challengeOrm: orm.NewChallenge(db), proverVersionHardForkMap: proverVersionHardForkMap, } } -// InsertChallengeString insert and check the challenge string is existed -func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) error { - return l.challengeOrm.InsertChallenge(ctx.Copy(), challenge) -} - // Verify the completeness of login message -func (l *LoginLogic) VerifyMsg(login *types.LoginParameter) error { +func VerifyMsg(login *types.LoginParameter) error { verify, err := login.Verify() if err != nil || !verify { log.Error("auth message verify failure", "prover_name", login.Message.ProverName, @@ -61,11 +56,16 @@ func (l *LoginLogic) VerifyMsg(login *types.LoginParameter) error { return nil } +// InsertChallengeString insert and check the challenge string is existed +func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) error { + return l.challengeOrm.InsertChallenge(ctx.Copy(), challenge) +} + // Check if the login client is compatible with the setting in coordinator func (l *LoginLogic) CompatiblityCheck(login *types.LoginParameter) error { - if !version.CheckScrollRepoVersion(login.Message.ProverVersion, l.cfg.ProverManager.Verifier.MinProverVersion) { - return fmt.Errorf("incompatible prover version. please upgrade your prover, minimum allowed version: %s, actual version: %s", l.cfg.ProverManager.Verifier.MinProverVersion, login.Message.ProverVersion) + if !version.CheckScrollRepoVersion(login.Message.ProverVersion, l.cfg.MinProverVersion) { + return fmt.Errorf("incompatible prover version. please upgrade your prover, minimum allowed version: %s, actual version: %s", l.cfg.MinProverVersion, login.Message.ProverVersion) } vks := make(map[string]struct{}) From 412ad56a64677e9175b8a2d37523d35a9ec4723c Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 20:43:40 +0900 Subject: [PATCH 04/43] extend loginlogic --- coordinator/internal/logic/auth/login.go | 27 +++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index 35350ef1ea..12830d6d4d 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -1,6 +1,7 @@ package auth import ( + "context" "errors" "fmt" "strings" @@ -20,15 +21,35 @@ import ( // LoginLogic the auth logic type LoginLogic struct { cfg *config.VerifierConfig - challengeOrm *orm.Challenge + deduplicator ChallengeDeduplicator openVmVks map[string]struct{} proverVersionHardForkMap map[string][]string } +type ChallengeDeduplicator interface { + InsertChallenge(ctx context.Context, challengeString string) error +} + +type SimpleDeduplicator struct { +} + +func (s *SimpleDeduplicator) InsertChallenge(ctx context.Context, challengeString string) error { + return nil +} + +// NewLoginLogicWithSimpleDEduplicator new a LoginLogic, do not use db to deduplicate challege +func NewLoginLogicWithSimpleDEduplicator(vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { + return newLoginLogic(&SimpleDeduplicator{}, vcfg, vf) +} + // NewLoginLogic new a LoginLogic func NewLoginLogic(db *gorm.DB, vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { + return newLoginLogic(orm.NewChallenge(db), vcfg, vf) +} + +func newLoginLogic(deduplicator ChallengeDeduplicator, vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { proverVersionHardForkMap := make(map[string][]string) var hardForks []string @@ -40,7 +61,7 @@ func NewLoginLogic(db *gorm.DB, vcfg *config.VerifierConfig, vf *verifier.Verifi return &LoginLogic{ cfg: vcfg, openVmVks: vf.OpenVMVkMap, - challengeOrm: orm.NewChallenge(db), + deduplicator: deduplicator, proverVersionHardForkMap: proverVersionHardForkMap, } } @@ -58,7 +79,7 @@ func VerifyMsg(login *types.LoginParameter) error { // InsertChallengeString insert and check the challenge string is existed func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) error { - return l.challengeOrm.InsertChallenge(ctx.Copy(), challenge) + return l.deduplicator.InsertChallenge(ctx.Copy(), challenge) } // Check if the login client is compatible with the setting in coordinator From 3adb2e0a1b123c49179711cf8b89cd8041651750 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 21:18:13 +0900 Subject: [PATCH 05/43] WIP: controller --- coordinator/internal/config/proxy_config.go | 53 +++++++++++++++++++ .../internal/controller/proxy/client.go | 27 ++++------ .../internal/controller/proxy/controller.go | 41 ++++++++++++++ 3 files changed, 104 insertions(+), 17 deletions(-) create mode 100644 coordinator/internal/config/proxy_config.go create mode 100644 coordinator/internal/controller/proxy/controller.go diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go new file mode 100644 index 0000000000..f94ef54c2e --- /dev/null +++ b/coordinator/internal/config/proxy_config.go @@ -0,0 +1,53 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" + + "scroll-tech/common/utils" +) + +// Proxy loads proxy configuration items. +type ProxyManager struct { + // Zk verifier config help to confine the connected prover. + Verifier *VerifierConfig `json:"verifier"` +} + +// Coordinator configuration +type UpStream struct { + BaseUrl string `json:"base_url"` + RetryCount uint `json:"retry_count"` + RetryWaitTime uint `json:"retry_wait_time_sec"` + ConnectionTimeoutSec uint `json:"connection_timeout_sec"` +} + +// Config load configuration items. +type ProxyConfig struct { + ProxyManager *ProxyManager `json:"proxy_manager"` + ProxyName string `json:"proxy_name"` + Auth *Auth `json:"auth"` + Coordinators map[string]*UpStream `json:"coondiators"` +} + +// NewConfig returns a new instance of Config. +func NewProxyConfig(file string) (*ProxyConfig, error) { + buf, err := os.ReadFile(filepath.Clean(file)) + if err != nil { + return nil, err + } + + cfg := &ProxyConfig{} + err = json.Unmarshal(buf, cfg) + if err != nil { + return nil, err + } + + // Override config with environment variables + err = utils.OverrideConfigWithEnv(cfg, "SCROLL_COORDINATOR_PROXY") + if err != nil { + return nil, err + } + + return cfg, nil +} diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 48d768ba80..17c394db68 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -11,38 +11,31 @@ import ( "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto" + "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" ) // Client wraps an http client with a preset host for coordinator API calls type Client struct { httpClient *http.Client - host string + baseURL string loginToken string } // NewClient creates a new Client with the specified host -func NewClient(host string) *Client { +func NewClient(cfg *config.UpStream) *Client { return &Client{ httpClient: &http.Client{ - Timeout: 30 * time.Second, + Timeout: time.Duration(cfg.ConnectionTimeoutSec) * time.Second, }, - host: host, - } -} - -// NewClientWithHTTPClient creates a new Client with a custom http.Client -func NewClientWithHTTPClient(host string, httpClient *http.Client) *Client { - return &Client{ - httpClient: httpClient, - host: host, + baseURL: cfg.BaseUrl, } } // FullLogin performs the complete login process: get challenge then login func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { // Step 1: Get challenge - url := fmt.Sprintf("%s/v1/challenge", c.host) + url := fmt.Sprintf("%s/coordinator/v1/challenge", c.baseURL) req, err := http.NewRequest("GET", url, nil) if err != nil { @@ -66,7 +59,7 @@ func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { } // Step 3: Use the token from challenge as Bearer token for login - url = fmt.Sprintf("%s/v1/login", c.host) + url = fmt.Sprintf("%s/coordinator/v1/login", c.baseURL) jsonData, err := json.Marshal(param) if err != nil { @@ -101,7 +94,7 @@ func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) { - url := fmt.Sprintf("%s/v1/proxy_login", c.host) + url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) if err != nil { @@ -121,7 +114,7 @@ func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) // GetTask makes a POST request to /v1/get_task with GetTaskParameter func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Response, error) { - url := fmt.Sprintf("%s/v1/get_task", c.host) + url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) if err != nil { @@ -143,7 +136,7 @@ func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Resp // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*http.Response, error) { - url := fmt.Sprintf("%s/v1/submit_proof", c.host) + url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) if err != nil { diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go new file mode 100644 index 0000000000..02f6350d7d --- /dev/null +++ b/coordinator/internal/controller/proxy/controller.go @@ -0,0 +1,41 @@ +package proxy + +import ( + "github.com/scroll-tech/go-ethereum/log" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/logic/verifier" +) + +var ( + // GetTask the prover task controller + GetTask *GetTaskController + // SubmitProof the submit proof controller + SubmitProof *SubmitProofController + // Auth the auth controller + Auth *AuthController +) + +// Clients manager a series of thread-safe clients for requesting upstream +// coordinators +type Clients map[string]*Client + +// InitController inits Controller with database +func InitController(cfg *config.ProxyConfig) { + vf, err := verifier.NewVerifier(cfg.ProxyManager.Verifier) + if err != nil { + panic("proof receiver new verifier failure") + } + + log.Info("verifier created", "openVmVerifier", vf.OpenVMVkMap) + + clients := make(map[string]*Client) + + for nm, cfg := range cfg.Coordinators { + clients[nm] = NewClient(cfg) + } + + Auth = NewAuthController(cfg, clients, vf) + // GetTask = NewGetTaskController(cfg, chainCfg, db, vf, reg) + // SubmitProof = NewSubmitProofController(cfg, chainCfg, db, vf, reg) +} From 5c6c225f7628f6ecbc5a9a88a07ede7d7642d8a8 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 22:14:22 +0900 Subject: [PATCH 06/43] WIP: config and client controller --- coordinator/internal/config/proxy_config.go | 19 ++++ .../internal/controller/proxy/client.go | 75 +++------------- .../controller/proxy/client_manager.go | 86 +++++++++++++++++++ .../internal/controller/proxy/controller.go | 15 +++- 4 files changed, 128 insertions(+), 67 deletions(-) create mode 100644 coordinator/internal/controller/proxy/client_manager.go diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index f94ef54c2e..5edb7fc783 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -12,6 +12,25 @@ import ( type ProxyManager struct { // Zk verifier config help to confine the connected prover. Verifier *VerifierConfig `json:"verifier"` + Client *ProxyClient `json:"proxy_cli"` + Auth *Auth `json:"auth"` +} + +func (m *ProxyManager) Normalize() { + if m.Client.Auth == nil { + m.Client.Auth = m.Auth + } + + if m.Client.ProxyVersion == "" { + m.Client.ProxyVersion = m.Verifier.MinProverVersion + } +} + +// Proxy client configuration for connect to upstream as a client +type ProxyClient struct { + ProxyName string `json:"proxy_name"` + ProxyVersion string `json:"proxy_version,omitempty"` + Auth *Auth `json:"auth,omitempty"` } // Coordinator configuration diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 17c394db68..9fe5ea8cb7 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -2,29 +2,28 @@ package proxy import ( "bytes" - "crypto/ecdsa" + "context" "encoding/json" "fmt" "net/http" "time" - "github.com/scroll-tech/go-ethereum/common" - "github.com/scroll-tech/go-ethereum/crypto" + "github.com/gin-gonic/gin" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" ) // Client wraps an http client with a preset host for coordinator API calls -type Client struct { +type upClient struct { httpClient *http.Client baseURL string loginToken string } // NewClient creates a new Client with the specified host -func NewClient(cfg *config.UpStream) *Client { - return &Client{ +func newUpClient(cfg *config.UpStream) *upClient { + return &upClient{ httpClient: &http.Client{ Timeout: time.Duration(cfg.ConnectionTimeoutSec) * time.Second, }, @@ -33,7 +32,7 @@ func NewClient(cfg *config.UpStream) *Client { } // FullLogin performs the complete login process: get challenge then login -func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { +func (c *upClient) Login(ctx context.Context, param types.LoginParameter) (*types.LoginSchema, error) { // Step 1: Get challenge url := fmt.Sprintf("%s/coordinator/v1/challenge", c.baseURL) @@ -93,7 +92,7 @@ func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) { +func (c *upClient) ProxyLogin(ctx *gin.Context, param types.LoginParameter) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) @@ -101,7 +100,7 @@ func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) return nil, fmt.Errorf("failed to marshal proxy login parameter: %w", err) } - req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) if err != nil { return nil, fmt.Errorf("failed to create proxy login request: %w", err) } @@ -113,7 +112,7 @@ func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Response, error) { +func (c *upClient) GetTask(ctx *gin.Context, param types.GetTaskParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) @@ -121,7 +120,7 @@ func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Resp return nil, fmt.Errorf("failed to marshal get task parameter: %w", err) } - req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) if err != nil { return nil, fmt.Errorf("failed to create get task request: %w", err) } @@ -135,7 +134,7 @@ func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Resp } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*http.Response, error) { +func (c *upClient) SubmitProof(ctx *gin.Context, param types.SubmitProofParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) @@ -143,7 +142,7 @@ func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*h return nil, fmt.Errorf("failed to marshal submit proof parameter: %w", err) } - req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) if err != nil { return nil, fmt.Errorf("failed to create submit proof request: %w", err) } @@ -155,53 +154,3 @@ func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*h return c.httpClient.Do(req) } - -// transformToValidPrivateKey safely transforms arbitrary bytes into valid private key bytes -func (c *Client) buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { - // Try appending bytes from 0x0 to 0x20 until we get a valid private key - for appendByte := byte(0x0); appendByte <= 0x20; appendByte++ { - // Append the byte to input - extendedBytes := append(inputBytes, appendByte) - - // Calculate 256-bit hash - hash := crypto.Keccak256(extendedBytes) - - // Try to create private key from hash - if k, err := crypto.ToECDSA(hash); err == nil { - return k, nil - } - } - - return nil, fmt.Errorf("failed to generate valid private key from input bytes") -} - -func (c *Client) generateLoginParameter(privateKeyBytes []byte, challenge string) (*types.LoginParameter, error) { - // Generate private key - privKey, err := c.buildPrivateKey(privateKeyBytes) - if err != nil { - return nil, err - } - - // Generate public key string - publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&privKey.PublicKey)) - - // Create login parameter with proxy settings - loginParam := &types.LoginParameter{ - Message: types.Message{ - Challenge: challenge, - ProverName: "proxy", - ProverVersion: "proxy", - ProverProviderType: types.ProverProviderTypeProxy, - ProverTypes: []types.ProverType{}, // Default empty - VKs: []string{}, // Default empty - }, - PublicKey: publicKeyHex, - } - - // Sign the message with the private key - if err := loginParam.SignWithKey(privKey); err != nil { - return nil, fmt.Errorf("failed to sign login parameter: %w", err) - } - - return loginParam, nil -} diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go new file mode 100644 index 0000000000..302292fb16 --- /dev/null +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -0,0 +1,86 @@ +package proxy + +import ( + "context" + "crypto/ecdsa" + "fmt" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/crypto" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/types" +) + +type Client interface { + Client(context.Context) *upClient +} + +type ClientManager struct { + cliCfg *config.ProxyClient + cfg *config.UpStream + privKey *ecdsa.PrivateKey +} + +// transformToValidPrivateKey safely transforms arbitrary bytes into valid private key bytes +func buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { + // Try appending bytes from 0x0 to 0x20 until we get a valid private key + for appendByte := byte(0x0); appendByte <= 0x20; appendByte++ { + // Append the byte to input + extendedBytes := append(inputBytes, appendByte) + + // Calculate 256-bit hash + hash := crypto.Keccak256(extendedBytes) + + // Try to create private key from hash + if k, err := crypto.ToECDSA(hash); err == nil { + return k, nil + } + } + + return nil, fmt.Errorf("failed to generate valid private key from input bytes") +} + +func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*ClientManager, error) { + + privKey, err := buildPrivateKey([]byte(cliCfg.Auth.Secret)) + if err != nil { + return nil, err + } + + return &ClientManager{ + privKey: privKey, + cfg: cfg, + cliCfg: cliCfg, + }, nil +} + +func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { + return newUpClient(cliMgr.cfg) +} + +func (cliMgr *ClientManager) generateLoginParameter(privKey []byte, challenge string) (*types.LoginParameter, error) { + + // Generate public key string + publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&cliMgr.privKey.PublicKey)) + + // Create login parameter with proxy settings + loginParam := &types.LoginParameter{ + Message: types.Message{ + Challenge: challenge, + ProverName: cliMgr.cliCfg.ProxyName, + ProverVersion: cliMgr.cliCfg.ProxyVersion, + ProverProviderType: types.ProverProviderTypeProxy, + ProverTypes: []types.ProverType{}, // Default empty + VKs: []string{}, // Default empty + }, + PublicKey: publicKeyHex, + } + + // Sign the message with the private key + if err := loginParam.SignWithKey(cliMgr.privKey); err != nil { + return nil, fmt.Errorf("failed to sign login parameter: %w", err) + } + + return loginParam, nil +} diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 02f6350d7d..297f8c68a4 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -18,10 +18,13 @@ var ( // Clients manager a series of thread-safe clients for requesting upstream // coordinators -type Clients map[string]*Client +type Clients map[string]Client // InitController inits Controller with database func InitController(cfg *config.ProxyConfig) { + // normalize cfg + cfg.ProxyManager.Normalize() + vf, err := verifier.NewVerifier(cfg.ProxyManager.Verifier) if err != nil { panic("proof receiver new verifier failure") @@ -29,10 +32,14 @@ func InitController(cfg *config.ProxyConfig) { log.Info("verifier created", "openVmVerifier", vf.OpenVMVkMap) - clients := make(map[string]*Client) + clients := make(map[string]Client) - for nm, cfg := range cfg.Coordinators { - clients[nm] = NewClient(cfg) + for nm, upCfg := range cfg.Coordinators { + cli, err := NewClientManager(cfg.ProxyManager.Client, upCfg) + if err != nil { + panic("create new client fail") + } + clients[nm] = cli } Auth = NewAuthController(cfg, clients, vf) From 76ecdf064a79bac7a645bd9c9afcde50bcf21d07 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 22:14:32 +0900 Subject: [PATCH 07/43] add proxy config sample --- coordinator/conf/config_proxy.json | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 coordinator/conf/config_proxy.json diff --git a/coordinator/conf/config_proxy.json b/coordinator/conf/config_proxy.json new file mode 100644 index 0000000000..39c3ca0a5f --- /dev/null +++ b/coordinator/conf/config_proxy.json @@ -0,0 +1,33 @@ +{ + "proxy_manager": { + "proxy_cli": { + "proxy_name": "proxy_name" + }, + "auth": { + "secret": "proxy secret key", + "challenge_expire_duration_sec": 3600, + "login_expire_duration_sec": 3600 + }, + "verifier": { + "min_prover_version": "v4.4.45", + "verifiers": [ + { + "assets_path": "assets", + "fork_name": "euclidV2" + }, + { + "assets_path": "assets", + "fork_name": "feynman" + } + ] + } + }, + "coordinators": { + "sepolia": { + "base_url": "http://localhost:8555", + "retry_count": 10, + "retry_wait_time_sec": 10, + "connection_timeout_sec": 30 + } + } +} From 0d238d77a6ddb1c0f66030f9154ffb45c9092d80 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 22:32:38 +0900 Subject: [PATCH 08/43] WIP: the structure of client manager --- coordinator/internal/controller/proxy/client.go | 16 ++++++++++++++-- .../internal/controller/proxy/client_manager.go | 8 ++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 9fe5ea8cb7..8850274f1c 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -14,25 +14,32 @@ import ( "scroll-tech/coordinator/internal/types" ) +type ClientHelper interface { + GenLoginParam(string) (*types.LoginParameter, error) + OnError(isUnauth bool) +} + // Client wraps an http client with a preset host for coordinator API calls type upClient struct { httpClient *http.Client baseURL string loginToken string + helper ClientHelper } // NewClient creates a new Client with the specified host -func newUpClient(cfg *config.UpStream) *upClient { +func newUpClient(cfg *config.UpStream, helper ClientHelper) *upClient { return &upClient{ httpClient: &http.Client{ Timeout: time.Duration(cfg.ConnectionTimeoutSec) * time.Second, }, baseURL: cfg.BaseUrl, + helper: helper, } } // FullLogin performs the complete login process: get challenge then login -func (c *upClient) Login(ctx context.Context, param types.LoginParameter) (*types.LoginSchema, error) { +func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { // Step 1: Get challenge url := fmt.Sprintf("%s/coordinator/v1/challenge", c.baseURL) @@ -60,6 +67,11 @@ func (c *upClient) Login(ctx context.Context, param types.LoginParameter) (*type // Step 3: Use the token from challenge as Bearer token for login url = fmt.Sprintf("%s/coordinator/v1/login", c.baseURL) + param, err := c.helper.GenLoginParam(loginSchema.Token) + if err != nil { + return nil, fmt.Errorf("failed to setup login parameter: %w", err) + } + jsonData, err := json.Marshal(param) if err != nil { return nil, fmt.Errorf("failed to marshal login parameter: %w", err) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 302292fb16..399a424fe1 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -56,10 +56,14 @@ func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*Client } func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { - return newUpClient(cliMgr.cfg) + return newUpClient(cliMgr.cfg, cliMgr) } -func (cliMgr *ClientManager) generateLoginParameter(privKey []byte, challenge string) (*types.LoginParameter, error) { +func (cliMgr *ClientManager) OnError(isUnauth bool) { + +} + +func (cliMgr *ClientManager) GenLoginParam(challenge string) (*types.LoginParameter, error) { // Generate public key string publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&cliMgr.privKey.PublicKey)) From 7b3a65b35b029e93c8622ef0c163168a4f0bb796 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 22:41:17 +0900 Subject: [PATCH 09/43] framework for auto login --- .../controller/proxy/client_manager.go | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 399a424fe1..6893f34dd4 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -1,10 +1,11 @@ package proxy import ( - "context" "crypto/ecdsa" "fmt" + "sync" + "github.com/gin-gonic/gin" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto" @@ -13,13 +14,18 @@ import ( ) type Client interface { - Client(context.Context) *upClient + Client(*gin.Context) *upClient } type ClientManager struct { cliCfg *config.ProxyClient cfg *config.UpStream privKey *ecdsa.PrivateKey + + cachedCli struct { + sync.RWMutex + cli *upClient + } } // transformToValidPrivateKey safely transforms arbitrary bytes into valid private key bytes @@ -55,8 +61,26 @@ func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*Client }, nil } -func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { - return newUpClient(cliMgr.cfg, cliMgr) +func (cliMgr *ClientManager) doLogin() *upClient { + loginCli := newUpClient(cliMgr.cfg, cliMgr) + + return loginCli +} + +func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { + cliMgr.cachedCli.RLock() + if cliMgr.cachedCli.cli != nil { + defer cliMgr.cachedCli.RUnlock() + return cliMgr.cachedCli.cli + } + cliMgr.cachedCli.RUnlock() + cliMgr.cachedCli.Lock() + defer cliMgr.cachedCli.Unlock() + if cliMgr.cachedCli.cli != nil { + return cliMgr.cachedCli.cli + } + + return nil } func (cliMgr *ClientManager) OnError(isUnauth bool) { From 4f878d9231404310a8b7519695747599492807bf Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 23:05:56 +0900 Subject: [PATCH 10/43] AI step --- .../controller/proxy/client_manager.go | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 6893f34dd4..b22b9847d5 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -1,6 +1,7 @@ package proxy import ( + "context" "crypto/ecdsa" "fmt" "sync" @@ -24,7 +25,9 @@ type ClientManager struct { cachedCli struct { sync.RWMutex - cli *upClient + cli *upClient + completionCtx context.Context + completionDone context.CancelFunc } } @@ -74,13 +77,49 @@ func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { return cliMgr.cachedCli.cli } cliMgr.cachedCli.RUnlock() + cliMgr.cachedCli.Lock() - defer cliMgr.cachedCli.Unlock() if cliMgr.cachedCli.cli != nil { + defer cliMgr.cachedCli.Unlock() return cliMgr.cachedCli.cli } - return nil + var completionCtx context.Context + // Check if completion context is set + if cliMgr.cachedCli.completionCtx != nil { + completionCtx = cliMgr.cachedCli.completionCtx + } else { + // Set new completion context and launch login goroutine + ctx, completionDone := context.WithCancel(context.TODO()) + cliMgr.cachedCli.completionCtx = ctx + + // Launch login goroutine + go func() { + defer completionDone() + + loginCli := cliMgr.doLogin() + if loginResult, err := loginCli.Login(context.Background()); err == nil { + loginCli.loginToken = loginResult.Token + + cliMgr.cachedCli.Lock() + cliMgr.cachedCli.cli = loginCli + cliMgr.cachedCli.completionCtx = nil + cliMgr.cachedCli.Unlock() + } + }() + } + cliMgr.cachedCli.Unlock() + + // Wait for completion or request cancellation + select { + case <-ctx.Done(): + return nil + case <-completionCtx.Done(): + cliMgr.cachedCli.Lock() + cli := cliMgr.cachedCli.cli + cliMgr.cachedCli.Unlock() + return cli + } } func (cliMgr *ClientManager) OnError(isUnauth bool) { From 624a7a29b843d89a8643daa246a224670b958e74 Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 25 Aug 2025 09:35:10 +0900 Subject: [PATCH 11/43] WIP: AI step --- coordinator/conf/config_proxy.json | 3 +- coordinator/internal/config/proxy_config.go | 6 +- .../internal/controller/proxy/client.go | 2 +- .../controller/proxy/client_manager.go | 116 ++++++++++++++---- 4 files changed, 98 insertions(+), 29 deletions(-) diff --git a/coordinator/conf/config_proxy.json b/coordinator/conf/config_proxy.json index 39c3ca0a5f..886c10bf51 100644 --- a/coordinator/conf/config_proxy.json +++ b/coordinator/conf/config_proxy.json @@ -1,7 +1,8 @@ { "proxy_manager": { "proxy_cli": { - "proxy_name": "proxy_name" + "proxy_name": "proxy_name", + "secret": "client private key" }, "auth": { "secret": "proxy secret key", diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index 5edb7fc783..e2d510a29b 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -17,8 +17,8 @@ type ProxyManager struct { } func (m *ProxyManager) Normalize() { - if m.Client.Auth == nil { - m.Client.Auth = m.Auth + if m.Client.Secret == "" { + m.Client.Secret = m.Auth.Secret } if m.Client.ProxyVersion == "" { @@ -30,7 +30,7 @@ func (m *ProxyManager) Normalize() { type ProxyClient struct { ProxyName string `json:"proxy_name"` ProxyVersion string `json:"proxy_version,omitempty"` - Auth *Auth `json:"auth,omitempty"` + Secret string `json:"secret,omitempty"` } // Coordinator configuration diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 8850274f1c..92b0e5d3c8 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -16,7 +16,7 @@ import ( type ClientHelper interface { GenLoginParam(string) (*types.LoginParameter, error) - OnError(isUnauth bool) + OnResp(*upClient, *http.Response) } // Client wraps an http client with a preset host for coordinator API calls diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index b22b9847d5..bae8954998 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -4,18 +4,20 @@ import ( "context" "crypto/ecdsa" "fmt" + "net/http" "sync" + "time" - "github.com/gin-gonic/gin" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto" + "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" ) type Client interface { - Client(*gin.Context) *upClient + Client(context.Context) *upClient } type ClientManager struct { @@ -25,9 +27,9 @@ type ClientManager struct { cachedCli struct { sync.RWMutex - cli *upClient - completionCtx context.Context - completionDone context.CancelFunc + cli *upClient + completionCtx context.Context + resultChan chan *upClient } } @@ -52,7 +54,7 @@ func buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*ClientManager, error) { - privKey, err := buildPrivateKey([]byte(cliCfg.Auth.Secret)) + privKey, err := buildPrivateKey([]byte(cliCfg.Secret)) if err != nil { return nil, err } @@ -64,13 +66,35 @@ func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*Client }, nil } -func (cliMgr *ClientManager) doLogin() *upClient { - loginCli := newUpClient(cliMgr.cfg, cliMgr) +func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) time.Time { + // Calculate wait time between 2 seconds and cfg.RetryWaitTime + minWait := 2 * time.Second + waitDuration := time.Duration(cliMgr.cfg.RetryWaitTime) * time.Second + if waitDuration < minWait { + waitDuration = minWait + } - return loginCli + for { + log.Info("attempting login to upstream coordinator", "baseURL", cliMgr.cfg.BaseUrl) + loginResult, err := loginCli.Login(ctx) + if err == nil && loginResult != nil { + log.Info("login to upstream coordinator successful", "baseURL", cliMgr.cfg.BaseUrl, "time", loginResult.Time) + return loginResult.Time + } + log.Info("login to upstream coordinator failed, retrying", "baseURL", cliMgr.cfg.BaseUrl, "error", err, "waitDuration", waitDuration) + + timer := time.NewTimer(waitDuration) + select { + case <-ctx.Done(): + timer.Stop() + return time.Now() + case <-timer.C: + // Continue to next retry + } + } } -func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { +func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { cliMgr.cachedCli.RLock() if cliMgr.cachedCli.cli != nil { defer cliMgr.cachedCli.RUnlock() @@ -91,21 +115,54 @@ func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { } else { // Set new completion context and launch login goroutine ctx, completionDone := context.WithCancel(context.TODO()) - cliMgr.cachedCli.completionCtx = ctx + loginCli := newUpClient(cliMgr.cfg, cliMgr) + cliMgr.cachedCli.completionCtx = context.WithValue(ctx, "cli", loginCli) // Launch login goroutine go func() { defer completionDone() + expiredT := cliMgr.doLogin(context.Background(), loginCli) + + cliMgr.cachedCli.Lock() + cliMgr.cachedCli.cli = loginCli + cliMgr.cachedCli.completionCtx = nil + + // Launch waiting thread to clear cached client before expiration + go func() { + now := time.Now() + clearTime := expiredT.Add(-10 * time.Second) // 10s before expiration + + // If clear time is too soon (less than 10s from now), set it to 10s from now + if clearTime.Before(now.Add(10 * time.Second)) { + clearTime = now.Add(10 * time.Second) + log.Error("token expiration time is too close, delaying clear time", + "baseURL", cliMgr.cfg.BaseUrl, + "expiredT", expiredT, + "adjustedClearTime", clearTime) + } + + waitDuration := time.Until(clearTime) + log.Info("token expiration monitor started", + "baseURL", cliMgr.cfg.BaseUrl, + "expiredT", expiredT, + "clearTime", clearTime, + "waitDuration", waitDuration) + + timer := time.NewTimer(waitDuration) + select { + case <-ctx.Done(): + timer.Stop() + log.Info("token expiration monitor cancelled", "baseURL", cliMgr.cfg.BaseUrl) + case <-timer.C: + log.Info("clearing cached client before token expiration", + "baseURL", cliMgr.cfg.BaseUrl, + "expiredT", expiredT) + cliMgr.clearCachedCli(loginCli) + } + }() + + cliMgr.cachedCli.Unlock() - loginCli := cliMgr.doLogin() - if loginResult, err := loginCli.Login(context.Background()); err == nil { - loginCli.loginToken = loginResult.Token - - cliMgr.cachedCli.Lock() - cliMgr.cachedCli.cli = loginCli - cliMgr.cachedCli.completionCtx = nil - cliMgr.cachedCli.Unlock() - } }() } cliMgr.cachedCli.Unlock() @@ -115,15 +172,26 @@ func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { case <-ctx.Done(): return nil case <-completionCtx.Done(): - cliMgr.cachedCli.Lock() - cli := cliMgr.cachedCli.cli - cliMgr.cachedCli.Unlock() + cli := completionCtx.Value("cli").(*upClient) return cli } } -func (cliMgr *ClientManager) OnError(isUnauth bool) { +func (cliMgr *ClientManager) clearCachedCli(cli *upClient) { + cliMgr.cachedCli.Lock() + if cliMgr.cachedCli.cli == cli { + cliMgr.cachedCli.cli = nil + cliMgr.cachedCli.completionCtx = nil + log.Info("cached client cleared due to forbidden response", "baseURL", cliMgr.cfg.BaseUrl) + } + cliMgr.cachedCli.Unlock() +} +func (cliMgr *ClientManager) OnResp(cli *upClient, resp *http.Response) { + if resp.StatusCode == http.StatusForbidden { + log.Info("cached client cleared due to forbidden response", "baseURL", cliMgr.cfg.BaseUrl) + cliMgr.clearCachedCli(cli) + } } func (cliMgr *ClientManager) GenLoginParam(challenge string) (*types.LoginParameter, error) { From 321dd43af8c0d6e74734cbaf676b31d571bacc8b Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 25 Aug 2025 11:43:50 +0900 Subject: [PATCH 12/43] unit test for client --- coordinator/test/proxy_test.go | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 coordinator/test/proxy_test.go diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go new file mode 100644 index 0000000000..b287e41d8b --- /dev/null +++ b/coordinator/test/proxy_test.go @@ -0,0 +1,69 @@ +package test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/controller/proxy" +) + +func testProxyClientCfg() *config.ProxyClient { + + return &config.ProxyClient{ + Secret: "test-secret-key", + ProxyName: "test-proxy", + } +} + +func testProxyUpStreamCfg(coordinatorURL string) *config.UpStream { + + return &config.UpStream{ + BaseUrl: fmt.Sprintf("http://%s", coordinatorURL), + RetryWaitTime: 3, + ConnectionTimeoutSec: 30, + } + +} + +func testProxyClient(t *testing.T) { + + // Setup coordinator and http server. + coordinatorURL := randomURL() + proofCollector, httpHandler := setupCoordinator(t, 1, coordinatorURL) + defer func() { + proofCollector.Stop() + assert.NoError(t, httpHandler.Shutdown(context.Background())) + }() + + cliCfg := testProxyClientCfg() + upCfg := testProxyUpStreamCfg(coordinatorURL) + + clientManager, err := proxy.NewClientManager(cliCfg, upCfg) + assert.NoError(t, err) + assert.NotNil(t, clientManager) + + // Create context with timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Test Client method + client := clientManager.Client(ctx) + + // Client should not be nil if login succeeds + // Note: This might be nil if the coordinator is not properly set up for proxy authentication + // but the test validates that the Client method completes without panic + t.Logf("Client toke: %v", client) + +} + +func TestProxyClient(t *testing.T) { + + // Set up the test environment. + setEnv(t) + t.Run("TestProxyHandshake", testProxyClient) +} From 64ef0f4ec038c1a8a41950a408bc3228cfde7a46 Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 25 Aug 2025 11:52:03 +0900 Subject: [PATCH 13/43] WIP --- .../controller/proxy/client_manager.go | 22 ++++++++++--------- .../internal/controller/proxy/controller.go | 2 +- coordinator/test/proxy_test.go | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index bae8954998..ec57a3b351 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -21,6 +21,7 @@ type Client interface { } type ClientManager struct { + name string cliCfg *config.ProxyClient cfg *config.UpStream privKey *ecdsa.PrivateKey @@ -52,7 +53,7 @@ func buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { return nil, fmt.Errorf("failed to generate valid private key from input bytes") } -func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*ClientManager, error) { +func NewClientManager(name string, cliCfg *config.ProxyClient, cfg *config.UpStream) (*ClientManager, error) { privKey, err := buildPrivateKey([]byte(cliCfg.Secret)) if err != nil { @@ -60,6 +61,7 @@ func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*Client } return &ClientManager{ + name: name, privKey: privKey, cfg: cfg, cliCfg: cliCfg, @@ -75,13 +77,13 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) ti } for { - log.Info("attempting login to upstream coordinator", "baseURL", cliMgr.cfg.BaseUrl) + log.Info("attempting login to upstream coordinator", "name", cliMgr.name) loginResult, err := loginCli.Login(ctx) if err == nil && loginResult != nil { - log.Info("login to upstream coordinator successful", "baseURL", cliMgr.cfg.BaseUrl, "time", loginResult.Time) + log.Info("login to upstream coordinator successful", "name", cliMgr.name, "time", loginResult.Time) return loginResult.Time } - log.Info("login to upstream coordinator failed, retrying", "baseURL", cliMgr.cfg.BaseUrl, "error", err, "waitDuration", waitDuration) + log.Info("login to upstream coordinator failed, retrying", "name", cliMgr.name, "error", err, "waitDuration", waitDuration) timer := time.NewTimer(waitDuration) select { @@ -136,14 +138,14 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { if clearTime.Before(now.Add(10 * time.Second)) { clearTime = now.Add(10 * time.Second) log.Error("token expiration time is too close, delaying clear time", - "baseURL", cliMgr.cfg.BaseUrl, + "name", cliMgr.name, "expiredT", expiredT, "adjustedClearTime", clearTime) } waitDuration := time.Until(clearTime) log.Info("token expiration monitor started", - "baseURL", cliMgr.cfg.BaseUrl, + "name", cliMgr.name, "expiredT", expiredT, "clearTime", clearTime, "waitDuration", waitDuration) @@ -152,10 +154,10 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { select { case <-ctx.Done(): timer.Stop() - log.Info("token expiration monitor cancelled", "baseURL", cliMgr.cfg.BaseUrl) + log.Info("token expiration monitor cancelled", "name", cliMgr.name) case <-timer.C: log.Info("clearing cached client before token expiration", - "baseURL", cliMgr.cfg.BaseUrl, + "name", cliMgr.name, "expiredT", expiredT) cliMgr.clearCachedCli(loginCli) } @@ -182,14 +184,14 @@ func (cliMgr *ClientManager) clearCachedCli(cli *upClient) { if cliMgr.cachedCli.cli == cli { cliMgr.cachedCli.cli = nil cliMgr.cachedCli.completionCtx = nil - log.Info("cached client cleared due to forbidden response", "baseURL", cliMgr.cfg.BaseUrl) + log.Info("cached client cleared due to forbidden response", "name", cliMgr.name) } cliMgr.cachedCli.Unlock() } func (cliMgr *ClientManager) OnResp(cli *upClient, resp *http.Response) { if resp.StatusCode == http.StatusForbidden { - log.Info("cached client cleared due to forbidden response", "baseURL", cliMgr.cfg.BaseUrl) + log.Info("cached client cleared due to forbidden response", "name", cliMgr.name) cliMgr.clearCachedCli(cli) } } diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 297f8c68a4..5c42927685 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -35,7 +35,7 @@ func InitController(cfg *config.ProxyConfig) { clients := make(map[string]Client) for nm, upCfg := range cfg.Coordinators { - cli, err := NewClientManager(cfg.ProxyManager.Client, upCfg) + cli, err := NewClientManager(nm, cfg.ProxyManager.Client, upCfg) if err != nil { panic("create new client fail") } diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index b287e41d8b..670d6548ca 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -43,7 +43,7 @@ func testProxyClient(t *testing.T) { cliCfg := testProxyClientCfg() upCfg := testProxyUpStreamCfg(coordinatorURL) - clientManager, err := proxy.NewClientManager(cliCfg, upCfg) + clientManager, err := proxy.NewClientManager("test_coordinator", cliCfg, upCfg) assert.NoError(t, err) assert.NotNil(t, clientManager) From 5a07a1652bebb8bf450b12a08891e89fb116592b Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 27 Aug 2025 09:43:30 +0900 Subject: [PATCH 14/43] WIP --- coordinator/internal/controller/proxy/auth.go | 46 +++++++++++++++++++ .../internal/controller/proxy/get_task.go | 45 ++++++++++++++++++ .../internal/controller/proxy/submit_proof.go | 29 ++++++++++++ .../internal/middleware/proxy_bearer.go | 1 + 4 files changed, 121 insertions(+) create mode 100644 coordinator/internal/controller/proxy/auth.go create mode 100644 coordinator/internal/controller/proxy/get_task.go create mode 100644 coordinator/internal/controller/proxy/submit_proof.go create mode 100644 coordinator/internal/middleware/proxy_bearer.go diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go new file mode 100644 index 0000000000..97fffc8c76 --- /dev/null +++ b/coordinator/internal/controller/proxy/auth.go @@ -0,0 +1,46 @@ +package proxy + +import ( + "fmt" + + "github.com/gin-gonic/gin" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/controller/api" + "scroll-tech/coordinator/internal/logic/auth" + "scroll-tech/coordinator/internal/logic/verifier" + "scroll-tech/coordinator/internal/types" +) + +// AuthController is login API +type AuthController struct { + *api.AuthController + clients Clients +} + +// NewAuthController returns an LoginController instance +func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier) *AuthController { + + loginLogic := auth.NewLoginLogicWithSimpleDEduplicator(cfg.ProxyManager.Verifier, vf) + auth := api.NewAuthControllerWithLogic(loginLogic) + return &AuthController{ + AuthController: auth, + clients: clients, + } +} + +// Login extended the Login hander in api controller +func (a *AuthController) Login(c *gin.Context) (interface{}, error) { + + ret, err := a.AuthController.Login(c) + if err != nil { + return nil, err + } + loginParam := ret.(types.LoginParameterWithHardForkName) + // band recursive proxy now ... + if loginParam.Message.ProverProviderType == types.ProverProviderTypeProxy { + return nil, fmt.Errorf("do not allow recursive proxy for login %v", loginParam.Message) + } + + return loginParam, nil +} diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go new file mode 100644 index 0000000000..f351389a24 --- /dev/null +++ b/coordinator/internal/controller/proxy/get_task.go @@ -0,0 +1,45 @@ +package proxy + +import ( + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" + "github.com/scroll-tech/go-ethereum/params" + "gorm.io/gorm" + + "scroll-tech/common/types/message" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/logic/provertask" + "scroll-tech/coordinator/internal/logic/verifier" + coordinatorType "scroll-tech/coordinator/internal/types" +) + +// GetTaskController the get prover task api controller +type GetTaskController struct { + proverTasks map[message.ProofType]provertask.ProverTask + + getTaskAccessCounter *prometheus.CounterVec +} + +// NewGetTaskController create a get prover task controller +func NewGetTaskController(cfg *config.Config, chainCfg *params.ChainConfig, db *gorm.DB, verifier *verifier.Verifier, reg prometheus.Registerer) *GetTaskController { + // TODO: implement proxy get task controller initialization + return &GetTaskController{ + proverTasks: make(map[message.ProofType]provertask.ProverTask), + } +} + +func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { + // TODO: implement proxy get task access counter + return nil +} + +// GetTasks get assigned chunk/batch task +func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { + // TODO: implement proxy get tasks logic +} + +func (ptc *GetTaskController) proofType(para *coordinatorType.GetTaskParameter) message.ProofType { + // TODO: implement proxy proof type logic + return message.ProofTypeChunk +} \ No newline at end of file diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go new file mode 100644 index 0000000000..f2afa12945 --- /dev/null +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -0,0 +1,29 @@ +package proxy + +import ( + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" + "github.com/scroll-tech/go-ethereum/params" + "gorm.io/gorm" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/logic/submitproof" + "scroll-tech/coordinator/internal/logic/verifier" +) + +// SubmitProofController the submit proof api controller +type SubmitProofController struct { + submitProofReceiverLogic *submitproof.ProofReceiverLogic +} + +// NewSubmitProofController create the submit proof api controller instance +func NewSubmitProofController(cfg *config.Config, chainCfg *params.ChainConfig, db *gorm.DB, vf *verifier.Verifier, reg prometheus.Registerer) *SubmitProofController { + return &SubmitProofController{ + submitProofReceiverLogic: submitproof.NewSubmitProofReceiverLogic(cfg.ProverManager, chainCfg, db, vf, reg), + } +} + +// SubmitProof prover submit the proof to coordinator +func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { + // TODO: implement proxy submit proof logic +} \ No newline at end of file diff --git a/coordinator/internal/middleware/proxy_bearer.go b/coordinator/internal/middleware/proxy_bearer.go new file mode 100644 index 0000000000..c870d7c164 --- /dev/null +++ b/coordinator/internal/middleware/proxy_bearer.go @@ -0,0 +1 @@ +package middleware From 5614ec3b8633f45c19885fccc5b0841ebe026976 Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 1 Sep 2025 10:12:16 +0900 Subject: [PATCH 15/43] WIP --- coordinator/internal/controller/api/auth.go | 14 +- coordinator/internal/controller/proxy/auth.go | 316 +++++++++++++++++- .../internal/controller/proxy/client.go | 29 +- .../controller/proxy/client_manager.go | 1 - coordinator/internal/middleware/login_jwt.go | 9 + 5 files changed, 346 insertions(+), 23 deletions(-) diff --git a/coordinator/internal/controller/api/auth.go b/coordinator/internal/controller/api/auth.go index 1a3305d2b1..c2abb86f09 100644 --- a/coordinator/internal/controller/api/auth.go +++ b/coordinator/internal/controller/api/auth.go @@ -19,6 +19,12 @@ type AuthController struct { loginLogic *auth.LoginLogic } +func NewAuthControllerWithLogic(loginLogic *auth.LoginLogic) *AuthController { + return &AuthController{ + loginLogic: loginLogic, + } +} + // NewAuthController returns an LoginController instance func NewAuthController(db *gorm.DB, cfg *config.Config, vf *verifier.Verifier) *AuthController { return &AuthController{ @@ -102,10 +108,6 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { c.Set(types.ProverName, proverName) } - if publicKey, ok := claims[types.PublicKey]; ok { - c.Set(types.PublicKey, publicKey) - } - if proverVersion, ok := claims[types.ProverVersion]; ok { c.Set(types.ProverVersion, proverVersion) } @@ -118,5 +120,9 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { c.Set(types.ProverProviderTypeKey, providerType) } + if publicKey, ok := claims[types.PublicKey]; ok { + return publicKey + } + return nil } diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 97fffc8c76..5c293fae3a 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -2,8 +2,14 @@ package proxy import ( "fmt" + "sync" + "context" + "time" + + jwt "github.com/appleboy/gin-jwt/v2" "github.com/gin-gonic/gin" + "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/api" @@ -14,33 +20,319 @@ import ( // AuthController is login API type AuthController struct { - *api.AuthController - clients Clients + apiLogin *api.AuthController + clients Clients + userTokenCache *UserTokenCache + tokenCacheUpdate chan<- *TokenUpdate +} + +type TokenUpdate struct { + PublicKey string + Upstream string + Phase uint + LoginParam types.LoginParameter + CompleteNotify chan<- *types.LoginSchema +} + +type UpstreamTokens struct { + LoginData map[string]*types.LoginSchema + LoginPhase uint + NextLoginPhase uint +} + +type UserTokenCache struct { + sync.RWMutex + data map[string]UpstreamTokens +} + +func newUserTokens() UpstreamTokens { + return UpstreamTokens{ + LoginData: make(map[string]*types.LoginSchema), + } +} + +func newUserCache() *UserTokenCache { + return &UserTokenCache{data: make(map[string]UpstreamTokens)} +} + +// get retrieves UpstreamTokens for a given user key, returns empty if still not exists +func (c *UserTokenCache) Get(userKey string) *UpstreamTokens { + c.RLock() + defer c.RUnlock() + + tokens, exists := c.data[userKey] + if !exists { + return nil + } + + return &tokens +} + +// prepare for a total update via Login request +func (c *UserTokenCache) updatePrepare(userKey string) UpstreamTokens { + c.Lock() + defer c.Unlock() + + if _, exists := c.data[userKey]; !exists { + log.Info("initializing user token cache", "userKey", userKey) + c.data[userKey] = newUserTokens() + } + updated := c.data[userKey] + updated.NextLoginPhase = updated.LoginPhase + 1 + c.data[userKey] = updated + return updated } +// partialSet updates a single entry in upstreamTokens for a given user +func (c *UserTokenCache) partialSet(userKey string, upstreamName string, loginSchema *types.LoginSchema, phase uint) { + c.Lock() + defer c.Unlock() + + // Get existing tokens or create new map + tokens, exists := c.data[userKey] + if exists && tokens.NextLoginPhase == phase { + // Update the specific upstream entry + tokens.LoginData[upstreamName] = loginSchema + } +} + +// LoginParameterWithHardForkName constructs new payload for login +type LoginParameterWithUpstreamTokens struct { + *types.LoginParameter + Tokens UpstreamTokens +} + +const upstreamConnTimeout = time.Second * 2 +const expireTolerant = 10 * time.Minute +const LoginParamCache = "login_param" +const ProverTypesKey = "prover_types" +const SignatureKey = "prover_signature" + // NewAuthController returns an LoginController instance func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier) *AuthController { loginLogic := auth.NewLoginLogicWithSimpleDEduplicator(cfg.ProxyManager.Verifier, vf) - auth := api.NewAuthControllerWithLogic(loginLogic) - return &AuthController{ - AuthController: auth, - clients: clients, + + // Create the token cache update channel + tokenCacheUpdateChan := make(chan *TokenUpdate) + + authController := &AuthController{ + apiLogin: api.NewAuthControllerWithLogic(loginLogic), + clients: clients, + userTokenCache: newUserCache(), + tokenCacheUpdate: tokenCacheUpdateChan, + } + + // Launch token cache manager in a separate goroutine + go authController.toeknCacheManager(tokenCacheUpdateChan) + + return authController +} + +func (a *AuthController) doUpdateRequest(ctx context.Context, req *TokenUpdate) (ret *types.LoginSchema) { + if req.CompleteNotify != nil { + defer func(ctx context.Context) { + select { + case <-ctx.Done(): + case req.CompleteNotify <- ret: + } + + }(ctx) } + + cli := a.clients[req.Upstream] + if cli := cli.Client(ctx); cli != nil { + var err error + if ret, err = cli.ProxyLogin(ctx, req.LoginParam); err == nil { + a.userTokenCache.partialSet(req.PublicKey, req.Upstream, ret, req.Phase) + } else { + log.Error("proxy login failed during token cache update", + "userKey", req.PublicKey, + "upstream", req.Upstream, + "phase", req.Phase, + "error", err) + } + } + return + +} + +func (a *AuthController) toeknCacheManager(request <-chan *TokenUpdate) { + + ctx := context.TODO() + var managerStatusLock sync.Mutex + managerStatus := make(map[string]map[string]uint) + + for { + req, ok := <-request + if !ok { + return + } + + // ensure the manager request is not outdated + tokens := a.userTokenCache.Get(req.PublicKey) + if tokens == nil { + // Highly not possible, if raise, the reason is unknown, just log the Error + continue + } + phase := tokens.NextLoginPhase + if req.Phase < phase { + // drop the out-dated request + continue + } + + // ensure only one login request is launched for the same phase + managerStatusLock.Lock() + stat, ok := managerStatus[req.Upstream] + if !ok { + managerStatus[req.Upstream] = make(map[string]uint) + stat = managerStatus[req.Upstream] + } + if phase, running := stat[req.PublicKey]; running && phase >= req.Phase { + managerStatusLock.Unlock() + continue + } else { + stat[req.PublicKey] = req.Phase + } + managerStatusLock.Unlock() + + go a.doUpdateRequest(ctx, req) + + } + } // Login extended the Login hander in api controller func (a *AuthController) Login(c *gin.Context) (interface{}, error) { - ret, err := a.AuthController.Login(c) + loginRes, err := a.apiLogin.Login(c) if err != nil { return nil, err } - loginParam := ret.(types.LoginParameterWithHardForkName) - // band recursive proxy now ... - if loginParam.Message.ProverProviderType == types.ProverProviderTypeProxy { - return nil, fmt.Errorf("do not allow recursive proxy for login %v", loginParam.Message) + loginParam := loginRes.(types.LoginParameterWithHardForkName) + + if loginParam.LoginParameter.Message.ProverProviderType == types.ProverProviderTypeProxy { + return nil, fmt.Errorf("proxy do not support recursive login") + } + + tokens := a.userTokenCache.updatePrepare(loginParam.PublicKey) + notifies := make([]chan *types.LoginSchema, len(a.clients)) + + for n := range a.clients { + + // Check if we have a valid cached token that hasn't expired + if knownEntry, existed := tokens.LoginData[n]; existed { + timeRemaining := time.Until(knownEntry.Time) + if timeRemaining > expireTolerant { + // Token is still valid enouth, continue to next client + continue + } + } + + notify := make(chan *types.LoginSchema) + notifies = append(notifies, notify) + request := TokenUpdate{ + PublicKey: loginParam.PublicKey, + Upstream: n, + Phase: tokens.NextLoginPhase, + LoginParam: loginParam.LoginParameter, + CompleteNotify: notify, + } + defer close(notify) + select { + case <-c.Done(): + case a.tokenCacheUpdate <- &request: + } + + } + + // collect all request's compeletions + for _, chn := range notifies { + select { + case <-c.Done(): + case <-chn: + } + } + + return LoginParameterWithUpstreamTokens{ + LoginParameter: &loginParam.LoginParameter, + Tokens: tokens, + }, nil +} + +// PayloadFunc returns jwt.MapClaims with {public key, prover name}. +func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims { + v, ok := data.(LoginParameterWithUpstreamTokens) + if !ok { + return jwt.MapClaims{} + } + + return jwt.MapClaims{ + types.PublicKey: v.PublicKey, + types.ProverName: v.Message.ProverName, + types.ProverVersion: v.Message.ProverVersion, + types.ProverProviderTypeKey: v.Message.ProverProviderType, + SignatureKey: v.Signature, + ProverTypesKey: v.Message.ProverTypes, + } +} + +// IdentityHandler replies to client for /login +func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { + claims := jwt.ExtractClaims(c) + loginParam := &types.LoginParameter{} + + if proverName, ok := claims[types.ProverName]; ok { + loginParam.Message.ProverName, _ = proverName.(string) + } + + if proverVersion, ok := claims[types.ProverVersion]; ok { + loginParam.Message.ProverVersion, _ = proverVersion.(string) + } + + if providerType, ok := claims[types.ProverProviderTypeKey]; ok { + num, _ := providerType.(float64) + loginParam.Message.ProverProviderType = types.ProverProviderType(num) + } + + if signature, ok := claims[SignatureKey]; ok { + loginParam.Signature, _ = signature.(string) + } + + if proverTypes, ok := claims[ProverTypesKey]; ok { + arr, _ := proverTypes.([]any) + for _, elm := range arr { + num, _ := elm.(float64) + loginParam.Message.ProverTypes = append(loginParam.Message.ProverTypes, types.ProverType(num)) + } + } + + if publicKey, ok := claims[types.PublicKey]; ok { + loginParam.PublicKey, _ = publicKey.(string) + } + + if loginParam.PublicKey != "" { + // ensure tokenCache + a.userTokenCache.RLock() + _, exists := a.userTokenCache.data[loginParam.PublicKey] + if !exists { + a.userTokenCache.RUnlock() + a.userTokenCache.Lock() + if _, exists := a.userTokenCache.data[loginParam.PublicKey]; !exists { + log.Info("creating token cache for user after proxy restart", + "publicKey", loginParam.PublicKey, + "proverName", loginParam.Message.ProverName, + "reason", "prover using JWT token from before proxy restart") + a.userTokenCache.data[loginParam.PublicKey] = newUserTokens() + } + a.userTokenCache.Unlock() + } else { + a.userTokenCache.RUnlock() + } + + c.Set(LoginParamCache, loginParam) + return loginParam.PublicKey } - return loginParam, nil + return nil } diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 92b0e5d3c8..29c78cf216 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -8,8 +8,6 @@ import ( "net/http" "time" - "github.com/gin-gonic/gin" - "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" ) @@ -104,7 +102,7 @@ func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *upClient) ProxyLogin(ctx *gin.Context, param types.LoginParameter) (*http.Response, error) { +func (c *upClient) ProxyLogin(ctx context.Context, param types.LoginParameter) (*types.LoginSchema, error) { url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) @@ -120,11 +118,30 @@ func (c *upClient) ProxyLogin(ctx *gin.Context, param types.LoginParameter) (*ht req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+c.loginToken) - return c.httpClient.Do(req) + proxyLoginResp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform proxy login request: %w", err) + } + defer proxyLoginResp.Body.Close() + + // Call helper's OnResp method with the response + c.helper.OnResp(c, proxyLoginResp) + + // Parse proxy login response as LoginSchema + if proxyLoginResp.StatusCode == http.StatusOK { + var loginResult types.LoginSchema + if err := json.NewDecoder(proxyLoginResp.Body).Decode(&loginResult); err == nil { + return &loginResult, nil + } + // If parsing fails, still return success but with nil result + return nil, nil + } + + return nil, fmt.Errorf("proxy login request failed with status: %d", proxyLoginResp.StatusCode) } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *upClient) GetTask(ctx *gin.Context, param types.GetTaskParameter, token string) (*http.Response, error) { +func (c *upClient) GetTask(ctx context.Context, param types.GetTaskParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) @@ -146,7 +163,7 @@ func (c *upClient) GetTask(ctx *gin.Context, param types.GetTaskParameter, token } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *upClient) SubmitProof(ctx *gin.Context, param types.SubmitProofParameter, token string) (*http.Response, error) { +func (c *upClient) SubmitProof(ctx context.Context, param types.SubmitProofParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index ec57a3b351..31e0a324cf 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -30,7 +30,6 @@ type ClientManager struct { sync.RWMutex cli *upClient completionCtx context.Context - resultChan chan *upClient } } diff --git a/coordinator/internal/middleware/login_jwt.go b/coordinator/internal/middleware/login_jwt.go index b04810b0b7..565aff9daf 100644 --- a/coordinator/internal/middleware/login_jwt.go +++ b/coordinator/internal/middleware/login_jwt.go @@ -4,6 +4,7 @@ import ( "time" jwt "github.com/appleboy/gin-jwt/v2" + "github.com/gin-gonic/gin" "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" @@ -11,6 +12,13 @@ import ( "scroll-tech/coordinator/internal/types" ) +func nonIdendityAuthorizator(data interface{}, _ *gin.Context) bool { + if data == nil { + return false + } + return true +} + // LoginMiddleware jwt auth middleware func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ @@ -20,6 +28,7 @@ func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { Key: []byte(conf.Auth.Secret), Timeout: time.Second * time.Duration(conf.Auth.LoginExpireDurationSec), Authenticator: api.Auth.Login, + Authorizator: nonIdendityAuthorizator, Unauthorized: unauthorized, TokenLookup: "header: Authorization, query: token, cookie: jwt", TokenHeadName: "Bearer", From 322766f54f8432452336454eb3fb84bea38e23c2 Mon Sep 17 00:00:00 2001 From: Ho Date: Tue, 2 Sep 2025 17:22:35 +0900 Subject: [PATCH 16/43] WIP --- coordinator/internal/controller/proxy/auth.go | 40 ++--- .../internal/controller/proxy/client.go | 24 ++- .../controller/proxy/client_manager.go | 8 + .../internal/controller/proxy/get_task.go | 142 ++++++++++++++++-- 4 files changed, 176 insertions(+), 38 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 5c293fae3a..2fc1d76aa0 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -20,10 +20,9 @@ import ( // AuthController is login API type AuthController struct { - apiLogin *api.AuthController - clients Clients - userTokenCache *UserTokenCache - tokenCacheUpdate chan<- *TokenUpdate + apiLogin *api.AuthController + clients Clients + userTokenCache *UserTokenCache } type TokenUpdate struct { @@ -42,7 +41,8 @@ type UpstreamTokens struct { type UserTokenCache struct { sync.RWMutex - data map[string]UpstreamTokens + data map[string]UpstreamTokens + tokenCacheUpdate chan<- *TokenUpdate } func newUserTokens() UpstreamTokens { @@ -51,8 +51,11 @@ func newUserTokens() UpstreamTokens { } } -func newUserCache() *UserTokenCache { - return &UserTokenCache{data: make(map[string]UpstreamTokens)} +func newUserCache(tokenCacheUpdate chan<- *TokenUpdate) *UserTokenCache { + return &UserTokenCache{ + data: make(map[string]UpstreamTokens), + tokenCacheUpdate: tokenCacheUpdate, + } } // get retrieves UpstreamTokens for a given user key, returns empty if still not exists @@ -117,10 +120,9 @@ func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Ve tokenCacheUpdateChan := make(chan *TokenUpdate) authController := &AuthController{ - apiLogin: api.NewAuthControllerWithLogic(loginLogic), - clients: clients, - userTokenCache: newUserCache(), - tokenCacheUpdate: tokenCacheUpdateChan, + apiLogin: api.NewAuthControllerWithLogic(loginLogic), + clients: clients, + userTokenCache: newUserCache(tokenCacheUpdateChan), } // Launch token cache manager in a separate goroutine @@ -129,6 +131,8 @@ func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Ve return authController } +func (a *AuthController) TokenCache() *UserTokenCache { return a.userTokenCache } + func (a *AuthController) doUpdateRequest(ctx context.Context, req *TokenUpdate) (ret *types.LoginSchema) { if req.CompleteNotify != nil { defer func(ctx context.Context) { @@ -143,13 +147,13 @@ func (a *AuthController) doUpdateRequest(ctx context.Context, req *TokenUpdate) cli := a.clients[req.Upstream] if cli := cli.Client(ctx); cli != nil { var err error - if ret, err = cli.ProxyLogin(ctx, req.LoginParam); err == nil { + if ret, err = cli.ProxyLogin(ctx, &req.LoginParam); err == nil { a.userTokenCache.partialSet(req.PublicKey, req.Upstream, ret, req.Phase) } else { - log.Error("proxy login failed during token cache update", - "userKey", req.PublicKey, - "upstream", req.Upstream, - "phase", req.Phase, + log.Error("proxy login failed during token cache update", + "userKey", req.PublicKey, + "upstream", req.Upstream, + "phase", req.Phase, "error", err) } } @@ -241,7 +245,7 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { defer close(notify) select { case <-c.Done(): - case a.tokenCacheUpdate <- &request: + case a.userTokenCache.tokenCacheUpdate <- &request: } } @@ -319,7 +323,7 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { a.userTokenCache.RUnlock() a.userTokenCache.Lock() if _, exists := a.userTokenCache.data[loginParam.PublicKey]; !exists { - log.Info("creating token cache for user after proxy restart", + log.Info("creating token cache for user after proxy restart", "publicKey", loginParam.PublicKey, "proverName", loginParam.Message.ProverName, "reason", "prover using JWT token from before proxy restart") diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 29c78cf216..549fbc3670 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -8,8 +8,11 @@ import ( "net/http" "time" + ctypes "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" + + "github.com/mitchellh/mapstructure" ) type ClientHelper interface { @@ -90,19 +93,26 @@ func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { // Parse login response as LoginSchema and store the token if loginResp.StatusCode == http.StatusOK { - var loginResult types.LoginSchema - if err := json.NewDecoder(loginResp.Body).Decode(&loginResult); err == nil { + var respWithData ctypes.Response + // Note: Body is consumed after decoding, caller should not read it again + if err := json.NewDecoder(loginResp.Body).Decode(&respWithData); err == nil { + var loginResult types.LoginSchema + err = mapstructure.Decode(respWithData.Data, &loginResult) + if err != nil { + return nil, fmt.Errorf("login parsing data fail, get %v", respWithData.Data) + } c.loginToken = loginResult.Token + return &loginResult, nil + } else { + return nil, fmt.Errorf("login parsing response failed: %v", err) } - // Note: Body is consumed after decoding, caller should not read it again - return &loginResult, nil } return nil, fmt.Errorf("login request failed with status: %d", loginResp.StatusCode) } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *upClient) ProxyLogin(ctx context.Context, param types.LoginParameter) (*types.LoginSchema, error) { +func (c *upClient) ProxyLogin(ctx context.Context, param *types.LoginParameter) (*types.LoginSchema, error) { url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) @@ -141,7 +151,7 @@ func (c *upClient) ProxyLogin(ctx context.Context, param types.LoginParameter) ( } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *upClient) GetTask(ctx context.Context, param types.GetTaskParameter, token string) (*http.Response, error) { +func (c *upClient) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) @@ -163,7 +173,7 @@ func (c *upClient) GetTask(ctx context.Context, param types.GetTaskParameter, to } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *upClient) SubmitProof(ctx context.Context, param types.SubmitProofParameter, token string) (*http.Response, error) { +func (c *upClient) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 31e0a324cf..e090c3ee73 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -18,6 +18,7 @@ import ( type Client interface { Client(context.Context) *upClient + PeekClient() *upClient } type ClientManager struct { @@ -95,6 +96,13 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) ti } } +func (cliMgr *ClientManager) PeekClient() *upClient { + cliMgr.cachedCli.RLock() + defer cliMgr.cachedCli.RUnlock() + + return cliMgr.cachedCli.cli +} + func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { cliMgr.cachedCli.RLock() if cliMgr.cachedCli.cli != nil { diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index f351389a24..261219aa2f 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -1,31 +1,62 @@ package proxy import ( + "encoding/json" + "fmt" + "net/http" + "github.com/gin-gonic/gin" + "github.com/mitchellh/mapstructure" "github.com/prometheus/client_golang/prometheus" - "github.com/scroll-tech/go-ethereum/params" - "gorm.io/gorm" + "github.com/scroll-tech/go-ethereum/log" - "scroll-tech/common/types/message" + "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" - "scroll-tech/coordinator/internal/logic/provertask" - "scroll-tech/coordinator/internal/logic/verifier" coordinatorType "scroll-tech/coordinator/internal/types" ) +func getSessionData(ctx *gin.Context) (string, *coordinatorType.LoginParameter) { + + publicKeyData, publicKeyExist := ctx.Get(coordinatorType.PublicKey) + publicKey, castOk := publicKeyData.(string) + if !publicKeyExist || !castOk { + nerr := fmt.Errorf("no public key binding: %v", publicKeyData) + log.Warn("get_task parameter fail", "error", nerr) + + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return "", nil + } + + loginParamData, publicKeyExist := ctx.Get(LoginParamCache) + loginParam, castOk := loginParamData.(*coordinatorType.LoginParameter) + if !publicKeyExist || !castOk { + nerr := fmt.Errorf("no login param binding: %v", loginParamData) + log.Warn("get_task parameter fail", "error", nerr) + + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return "", nil + } + + return publicKey, loginParam +} + // GetTaskController the get prover task api controller type GetTaskController struct { - proverTasks map[message.ProofType]provertask.ProverTask + tokenCache *UserTokenCache + clients Clients + priorityUpstream map[string]string getTaskAccessCounter *prometheus.CounterVec } // NewGetTaskController create a get prover task controller -func NewGetTaskController(cfg *config.Config, chainCfg *params.ChainConfig, db *gorm.DB, verifier *verifier.Verifier, reg prometheus.Registerer) *GetTaskController { +func NewGetTaskController(cfg *config.Config, clients Clients, tokenCache *UserTokenCache, reg prometheus.Registerer) *GetTaskController { // TODO: implement proxy get task controller initialization return &GetTaskController{ - proverTasks: make(map[message.ProofType]provertask.ProverTask), + priorityUpstream: make(map[string]string), + tokenCache: tokenCache, + clients: clients, } } @@ -36,10 +67,95 @@ func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { // GetTasks get assigned chunk/batch task func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { - // TODO: implement proxy get tasks logic + var getTaskParameter coordinatorType.GetTaskParameter + if err := ctx.ShouldBind(&getTaskParameter); err != nil { + nerr := fmt.Errorf("prover task parameter invalid, err:%w", err) + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return + } + + publicKey, loginParam := getSessionData(ctx) + if publicKey == "" || loginParam == nil { + return + } + + tokens := ptc.tokenCache.Get(publicKey) + + onClientFail := func(upstream string) { + //TODO: log re-connect request in info level + + request := TokenUpdate{ + PublicKey: publicKey, + Upstream: upstream, + Phase: tokens.LoginPhase, + LoginParam: *loginParam, + CompleteNotify: nil, + } + select { + case <-ctx.Done(): + case ptc.tokenCache.tokenCacheUpdate <- &request: + } + + } + + priorityUpstream, exist := ptc.priorityUpstream[publicKey] + if exist { + cli := ptc.clients[priorityUpstream] + loginSchema := tokens.LoginData[priorityUpstream] + if loginSchema == nil { + onClientFail(priorityUpstream) + } else { + ret, triggerUpdate := getTaskFromClient(ctx, cli, &getTaskParameter, loginSchema.Token) + if ret != nil { + + } else if triggerUpdate { + onClientFail(priorityUpstream) + } + } + types.RenderFailure(ctx, types.ErrCoordinatorEmptyProofData, fmt.Errorf("get empty prover task")) + } + + for n, cli := range ptc.clients { + + } } -func (ptc *GetTaskController) proofType(para *coordinatorType.GetTaskParameter) message.ProofType { - // TODO: implement proxy proof type logic - return message.ProofTypeChunk -} \ No newline at end of file +func getTaskFromClient(ctx *gin.Context, cli Client, param *coordinatorType.GetTaskParameter, token string) (*coordinatorType.GetTaskSchema, bool) { + + theCli := cli.PeekClient() + if theCli == nil { + return nil, true + } + + resp, err := theCli.GetTask(ctx, param, token) + if err != nil { + // log the err in error level + return nil, false + } + + // Parse response + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusUnauthorized { + unAuth := resp.StatusCode == http.StatusUnauthorized + var respWithData types.Response + // Note: Body is consumed after decoding, caller should not read it again + if err := json.NewDecoder(resp.Body).Decode(&respWithData); err == nil { + if unAuth && respWithData.ErrCode == types.ErrJWTTokenExpired { + return nil, true + } + + var getTaskResult coordinatorType.GetTaskSchema + err = mapstructure.Decode(respWithData.Data, &getTaskResult) + if err != nil { + log.Error("parse get task data fail", "respdata", respWithData.Data) + return nil, false + } + return &getTaskResult, false + } else { + log.Error("parse get task response failed", "error", err) + //fmt.Errorf("login parsing response failed: %v", err) + return nil, false + } + } + + return nil, false +} From e6be62f6333c1c1b4f061cca600c4c3ec3dd06f7 Mon Sep 17 00:00:00 2001 From: Ho Date: Fri, 5 Sep 2025 22:31:45 +0900 Subject: [PATCH 17/43] WIP --- common/types/response.go | 5 + .../internal/controller/proxy/client.go | 86 +++++------ .../controller/proxy/client_manager.go | 105 ++++++------- .../controller/proxy/prover_session.go | 140 ++++++++++++++++++ 4 files changed, 230 insertions(+), 106 deletions(-) create mode 100644 coordinator/internal/controller/proxy/prover_session.go diff --git a/common/types/response.go b/common/types/response.go index d616aa8013..abd29041e7 100644 --- a/common/types/response.go +++ b/common/types/response.go @@ -4,6 +4,7 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/mitchellh/mapstructure" ) // Response the response schema @@ -13,6 +14,10 @@ type Response struct { Data interface{} `json:"data"` } +func (resp *Response) DecodeData(out interface{}) error { + return mapstructure.Decode(resp.Data, out) +} + // RenderJSON renders response with json func RenderJSON(ctx *gin.Context, errCode int, err error, data interface{}) { var errMsg string diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 549fbc3670..e49caa83f3 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -11,36 +11,27 @@ import ( ctypes "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" - - "github.com/mitchellh/mapstructure" ) -type ClientHelper interface { - GenLoginParam(string) (*types.LoginParameter, error) - OnResp(*upClient, *http.Response) -} - // Client wraps an http client with a preset host for coordinator API calls type upClient struct { httpClient *http.Client baseURL string loginToken string - helper ClientHelper } // NewClient creates a new Client with the specified host -func newUpClient(cfg *config.UpStream, helper ClientHelper) *upClient { +func newUpClient(cfg *config.UpStream) *upClient { return &upClient{ httpClient: &http.Client{ Timeout: time.Duration(cfg.ConnectionTimeoutSec) * time.Second, }, baseURL: cfg.BaseUrl, - helper: helper, } } // FullLogin performs the complete login process: get challenge then login -func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { +func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.LoginParameter, error)) (*types.LoginSchema, error) { // Step 1: Get challenge url := fmt.Sprintf("%s/coordinator/v1/challenge", c.baseURL) @@ -68,7 +59,7 @@ func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { // Step 3: Use the token from challenge as Bearer token for login url = fmt.Sprintf("%s/coordinator/v1/login", c.baseURL) - param, err := c.helper.GenLoginParam(loginSchema.Token) + param, err := genLogin(loginSchema.Token) if err != nil { return nil, fmt.Errorf("failed to setup login parameter: %w", err) } @@ -91,28 +82,38 @@ func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { return nil, fmt.Errorf("failed to perform login request: %w", err) } - // Parse login response as LoginSchema and store the token - if loginResp.StatusCode == http.StatusOK { + parsedResp, err := handleHttpResp(loginResp) + if err != nil { + return nil, err + } + + var loginResult types.LoginSchema + err = parsedResp.DecodeData(&loginResult) + if err != nil { + return nil, fmt.Errorf("login parsing data fail: %v", err) + } + c.loginToken = loginResult.Token + return &loginResult, nil + +} + +func handleHttpResp(resp *http.Response) (*ctypes.Response, error) { + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusUnauthorized { + defer resp.Body.Close() var respWithData ctypes.Response // Note: Body is consumed after decoding, caller should not read it again - if err := json.NewDecoder(loginResp.Body).Decode(&respWithData); err == nil { - var loginResult types.LoginSchema - err = mapstructure.Decode(respWithData.Data, &loginResult) - if err != nil { - return nil, fmt.Errorf("login parsing data fail, get %v", respWithData.Data) - } - c.loginToken = loginResult.Token - return &loginResult, nil + if err := json.NewDecoder(resp.Body).Decode(&respWithData); err == nil { + return &respWithData, nil } else { - return nil, fmt.Errorf("login parsing response failed: %v", err) + return nil, fmt.Errorf("login parsing expected response failed: %v", err) } - } - return nil, fmt.Errorf("login request failed with status: %d", loginResp.StatusCode) + } + return nil, fmt.Errorf("login request failed with status: %d", resp.StatusCode) } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *upClient) ProxyLogin(ctx context.Context, param *types.LoginParameter) (*types.LoginSchema, error) { +func (c *upClient) ProxyLogin(ctx context.Context, param *types.LoginParameter) (*ctypes.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) @@ -132,26 +133,11 @@ func (c *upClient) ProxyLogin(ctx context.Context, param *types.LoginParameter) if err != nil { return nil, fmt.Errorf("failed to perform proxy login request: %w", err) } - defer proxyLoginResp.Body.Close() - - // Call helper's OnResp method with the response - c.helper.OnResp(c, proxyLoginResp) - - // Parse proxy login response as LoginSchema - if proxyLoginResp.StatusCode == http.StatusOK { - var loginResult types.LoginSchema - if err := json.NewDecoder(proxyLoginResp.Body).Decode(&loginResult); err == nil { - return &loginResult, nil - } - // If parsing fails, still return success but with nil result - return nil, nil - } - - return nil, fmt.Errorf("proxy login request failed with status: %d", proxyLoginResp.StatusCode) + return handleHttpResp(proxyLoginResp) } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *upClient) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*http.Response, error) { +func (c *upClient) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*ctypes.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) @@ -169,11 +155,15 @@ func (c *upClient) GetTask(ctx context.Context, param *types.GetTaskParameter, t req.Header.Set("Authorization", "Bearer "+token) } - return c.httpClient.Do(req) + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + return handleHttpResp(resp) } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *upClient) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*http.Response, error) { +func (c *upClient) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*ctypes.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) @@ -191,5 +181,9 @@ func (c *upClient) SubmitProof(ctx context.Context, param *types.SubmitProofPara req.Header.Set("Authorization", "Bearer "+token) } - return c.httpClient.Do(req) + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + return handleHttpResp(resp) } diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index e090c3ee73..968c66b328 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -4,7 +4,6 @@ import ( "context" "crypto/ecdsa" "fmt" - "net/http" "sync" "time" @@ -18,7 +17,7 @@ import ( type Client interface { Client(context.Context) *upClient - PeekClient() *upClient + Reset(cli *upClient) } type ClientManager struct { @@ -78,7 +77,7 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) ti for { log.Info("attempting login to upstream coordinator", "name", cliMgr.name) - loginResult, err := loginCli.Login(ctx) + loginResult, err := loginCli.Login(ctx, cliMgr.genLoginParam) if err == nil && loginResult != nil { log.Info("login to upstream coordinator successful", "name", cliMgr.name, "time", loginResult.Time) return loginResult.Time @@ -96,11 +95,13 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) ti } } -func (cliMgr *ClientManager) PeekClient() *upClient { - cliMgr.cachedCli.RLock() - defer cliMgr.cachedCli.RUnlock() - - return cliMgr.cachedCli.cli +func (cliMgr *ClientManager) Reset(cli *upClient) { + cliMgr.cachedCli.Lock() + if cliMgr.cachedCli.cli == cli { + cliMgr.cachedCli.cli = nil + } + cliMgr.cachedCli.Unlock() + log.Info("cached client cleared", "name", cliMgr.name) } func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { @@ -124,51 +125,52 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { } else { // Set new completion context and launch login goroutine ctx, completionDone := context.WithCancel(context.TODO()) - loginCli := newUpClient(cliMgr.cfg, cliMgr) + loginCli := newUpClient(cliMgr.cfg) cliMgr.cachedCli.completionCtx = context.WithValue(ctx, "cli", loginCli) - // Launch login goroutine + // Launch keep-login goroutine go func() { defer completionDone() expiredT := cliMgr.doLogin(context.Background(), loginCli) + log.Info("login compeleted", "name", cliMgr.name, "expired", expiredT) cliMgr.cachedCli.Lock() cliMgr.cachedCli.cli = loginCli cliMgr.cachedCli.completionCtx = nil // Launch waiting thread to clear cached client before expiration - go func() { - now := time.Now() - clearTime := expiredT.Add(-10 * time.Second) // 10s before expiration - - // If clear time is too soon (less than 10s from now), set it to 10s from now - if clearTime.Before(now.Add(10 * time.Second)) { - clearTime = now.Add(10 * time.Second) - log.Error("token expiration time is too close, delaying clear time", - "name", cliMgr.name, - "expiredT", expiredT, - "adjustedClearTime", clearTime) - } - - waitDuration := time.Until(clearTime) - log.Info("token expiration monitor started", - "name", cliMgr.name, - "expiredT", expiredT, - "clearTime", clearTime, - "waitDuration", waitDuration) - - timer := time.NewTimer(waitDuration) - select { - case <-ctx.Done(): - timer.Stop() - log.Info("token expiration monitor cancelled", "name", cliMgr.name) - case <-timer.C: - log.Info("clearing cached client before token expiration", - "name", cliMgr.name, - "expiredT", expiredT) - cliMgr.clearCachedCli(loginCli) - } - }() + // go func() { + // now := time.Now() + // clearTime := expiredT.Add(-10 * time.Second) // 10s before expiration + + // // If clear time is too soon (less than 10s from now), set it to 10s from now + // if clearTime.Before(now.Add(10 * time.Second)) { + // clearTime = now.Add(10 * time.Second) + // log.Error("token expiration time is too close, delaying clear time", + // "name", cliMgr.name, + // "expiredT", expiredT, + // "adjustedClearTime", clearTime) + // } + + // waitDuration := time.Until(clearTime) + // log.Info("token expiration monitor started", + // "name", cliMgr.name, + // "expiredT", expiredT, + // "clearTime", clearTime, + // "waitDuration", waitDuration) + + // timer := time.NewTimer(waitDuration) + // select { + // case <-ctx.Done(): + // timer.Stop() + // log.Info("token expiration monitor cancelled", "name", cliMgr.name) + // case <-timer.C: + // log.Info("clearing cached client before token expiration", + // "name", cliMgr.name, + // "expiredT", expiredT) + // cliMgr.clearCachedCli(loginCli) + // } + // }() cliMgr.cachedCli.Unlock() @@ -186,24 +188,7 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { } } -func (cliMgr *ClientManager) clearCachedCli(cli *upClient) { - cliMgr.cachedCli.Lock() - if cliMgr.cachedCli.cli == cli { - cliMgr.cachedCli.cli = nil - cliMgr.cachedCli.completionCtx = nil - log.Info("cached client cleared due to forbidden response", "name", cliMgr.name) - } - cliMgr.cachedCli.Unlock() -} - -func (cliMgr *ClientManager) OnResp(cli *upClient, resp *http.Response) { - if resp.StatusCode == http.StatusForbidden { - log.Info("cached client cleared due to forbidden response", "name", cliMgr.name) - cliMgr.clearCachedCli(cli) - } -} - -func (cliMgr *ClientManager) GenLoginParam(challenge string) (*types.LoginParameter, error) { +func (cliMgr *ClientManager) genLoginParam(challenge string) (*types.LoginParameter, error) { // Generate public key string publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&cliMgr.privKey.PublicKey)) diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go new file mode 100644 index 0000000000..0463f3409c --- /dev/null +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -0,0 +1,140 @@ +package proxy + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + + ctypes "scroll-tech/common/types" + "scroll-tech/coordinator/internal/types" +) + +// Client wraps an http client with a preset host for coordinator API calls +type proverSession struct { + sync.RWMutex + proverToken string +} + +func (c *proverSession) doProverLogin(ctx context.Context, cliMgr Client, param *types.LoginParameter) (*types.LoginSchema, error) { + cli := cliMgr.Client(ctx) + if cli == nil { + return nil, fmt.Errorf("get upstream cli fail") + } + + // like SDK, we would try one more time if the upstream token is expired + resp, err := cli.ProxyLogin(ctx, param) + if err != nil { + return nil, fmt.Errorf("proxylogin fail: %v", err) + } + + if resp.ErrCode == ctypes.ErrJWTTokenExpired { + cliMgr.Reset(cli) + cli = cliMgr.Client(ctx) + if cli == nil { + return nil, fmt.Errorf("get upstream cli fail (secondary try)") + } + + // like SDK, we would try one more time if the upstream token is expired + resp, err = cli.ProxyLogin(ctx, param) + if err != nil { + return nil, fmt.Errorf("proxylogin fail: %v", err) + } + } + + if resp.ErrCode != 0 { + return nil, fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) + } + + var loginResult types.LoginSchema + if err := resp.DecodeData(&loginResult); err != nil { + return nil, err + } + + return &loginResult, nil +} + +// ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter +func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) (*types.LoginSchema, error) { + url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal proxy login parameter: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create proxy login request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.loginToken) + + proxyLoginResp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform proxy login request: %w", err) + } + defer proxyLoginResp.Body.Close() + + // Call helper's OnResp method with the response + c.helper.OnResp(c, proxyLoginResp) + + // Parse proxy login response as LoginSchema + if proxyLoginResp.StatusCode == http.StatusOK { + var loginResult types.LoginSchema + if err := json.NewDecoder(proxyLoginResp.Body).Decode(&loginResult); err == nil { + return &loginResult, nil + } + // If parsing fails, still return success but with nil result + return nil, nil + } + + return nil, fmt.Errorf("proxy login request failed with status: %d", proxyLoginResp.StatusCode) +} + +// GetTask makes a POST request to /v1/get_task with GetTaskParameter +func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*http.Response, error) { + url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal get task parameter: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create get task request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + return c.httpClient.Do(req) +} + +// SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter +func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*http.Response, error) { + url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal submit proof parameter: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create submit proof request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + return c.httpClient.Do(req) +} From 9df6429d986a2bda98253fd149cc0e6dd5662dd5 Mon Sep 17 00:00:00 2001 From: Ho Date: Sat, 6 Sep 2025 21:50:55 +0900 Subject: [PATCH 18/43] wip --- .../controller/proxy/prover_session.go | 166 ++++++++++-------- 1 file changed, 96 insertions(+), 70 deletions(-) diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 0463f3409c..dbd854508b 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -1,11 +1,8 @@ package proxy import ( - "bytes" "context" - "encoding/json" "fmt" - "net/http" "sync" ctypes "scroll-tech/common/types" @@ -15,126 +12,155 @@ import ( // Client wraps an http client with a preset host for coordinator API calls type proverSession struct { sync.RWMutex - proverToken string + proverToken string + phase uint + completionCtx context.Context } -func (c *proverSession) doProverLogin(ctx context.Context, cliMgr Client, param *types.LoginParameter) (*types.LoginSchema, error) { +func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, param *types.LoginParameter, phase uint) error { + c.Lock() + curPhase := c.phase + if c.completionCtx != nil { + waitctx := c.completionCtx + c.Unlock() + select { + case <-waitctx.Done(): + return c.maintainLogin(ctx, cliMgr, param, phase) + case <-ctx.Done(): + return fmt.Errorf("ctx fail") + } + } + + if phase < curPhase { + // outdate login phase, give up + c.Unlock() + return nil + } + + // occupy the update slot + completeCtx, cf := context.WithCancel(ctx) + defer cf() + c.completionCtx = completeCtx + c.Unlock() + cli := cliMgr.Client(ctx) if cli == nil { - return nil, fmt.Errorf("get upstream cli fail") + return fmt.Errorf("get upstream cli fail") } - // like SDK, we would try one more time if the upstream token is expired resp, err := cli.ProxyLogin(ctx, param) if err != nil { - return nil, fmt.Errorf("proxylogin fail: %v", err) + return fmt.Errorf("proxylogin fail: %v", err) } if resp.ErrCode == ctypes.ErrJWTTokenExpired { cliMgr.Reset(cli) cli = cliMgr.Client(ctx) if cli == nil { - return nil, fmt.Errorf("get upstream cli fail (secondary try)") + return fmt.Errorf("get upstream cli fail (secondary try)") } // like SDK, we would try one more time if the upstream token is expired resp, err = cli.ProxyLogin(ctx, param) if err != nil { - return nil, fmt.Errorf("proxylogin fail: %v", err) + return fmt.Errorf("proxylogin fail: %v", err) } } if resp.ErrCode != 0 { - return nil, fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) + return fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) } var loginResult types.LoginSchema if err := resp.DecodeData(&loginResult); err != nil { - return nil, err + return err } - return &loginResult, nil + c.Lock() + defer c.Unlock() + c.proverToken = loginResult.Token + c.completionCtx = nil + + return nil } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) (*types.LoginSchema, error) { - url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) - - jsonData, err := json.Marshal(param) - if err != nil { - return nil, fmt.Errorf("failed to marshal proxy login parameter: %w", err) - } +func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) error { + c.RLock() + phase := c.phase + 1 + c.RUnlock() - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) - if err != nil { - return nil, fmt.Errorf("failed to create proxy login request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+c.loginToken) - - proxyLoginResp, err := c.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to perform proxy login request: %w", err) - } - defer proxyLoginResp.Body.Close() - - // Call helper's OnResp method with the response - c.helper.OnResp(c, proxyLoginResp) - - // Parse proxy login response as LoginSchema - if proxyLoginResp.StatusCode == http.StatusOK { - var loginResult types.LoginSchema - if err := json.NewDecoder(proxyLoginResp.Body).Decode(&loginResult); err == nil { - return &loginResult, nil - } - // If parsing fails, still return success but with nil result - return nil, nil - } - - return nil, fmt.Errorf("proxy login request failed with status: %d", proxyLoginResp.StatusCode) + return c.maintainLogin(ctx, cli, param, phase) } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*http.Response, error) { - url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) +func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, cliMgr Client) (*ctypes.Response, error) { + c.RLock() + phase := c.phase + token := c.proverToken + c.RUnlock() - jsonData, err := json.Marshal(param) - if err != nil { - return nil, fmt.Errorf("failed to marshal get task parameter: %w", err) + cli := cliMgr.Client(ctx) + if cli == nil { + return nil, fmt.Errorf("get upstream cli fail") } - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + resp, err := cli.GetTask(ctx, param, token) if err != nil { - return nil, fmt.Errorf("failed to create get task request: %w", err) + return nil, err } - req.Header.Set("Content-Type", "application/json") - if token != "" { - req.Header.Set("Authorization", "Bearer "+token) + if resp.ErrCode == ctypes.ErrJWTTokenExpired { + // get param from ctx + loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) + if !ok { + return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") + } + + err = c.maintainLogin(ctx, cliMgr, loginParam, phase) + if err != nil { + return nil, fmt.Errorf("update prover token fail: %V", err) + } + + // like SDK, we would try one more time if the upstream token is expired + return cli.GetTask(ctx, param, token) } - return c.httpClient.Do(req) + return resp, nil } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*http.Response, error) { - url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) +func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, cliMgr Client) (*ctypes.Response, error) { + c.RLock() + phase := c.phase + token := c.proverToken + c.RUnlock() - jsonData, err := json.Marshal(param) - if err != nil { - return nil, fmt.Errorf("failed to marshal submit proof parameter: %w", err) + cli := cliMgr.Client(ctx) + if cli == nil { + return nil, fmt.Errorf("get upstream cli fail") } - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + resp, err := cli.SubmitProof(ctx, param, token) if err != nil { - return nil, fmt.Errorf("failed to create submit proof request: %w", err) + return nil, err } - req.Header.Set("Content-Type", "application/json") - if token != "" { - req.Header.Set("Authorization", "Bearer "+token) + if resp.ErrCode == ctypes.ErrJWTTokenExpired { + // get param from ctx + loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) + if !ok { + return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") + } + + err = c.maintainLogin(ctx, cliMgr, loginParam, phase) + if err != nil { + return nil, fmt.Errorf("update prover token fail: %V", err) + } + + // like SDK, we would try one more time if the upstream token is expired + return cli.SubmitProof(ctx, param, token) } - return c.httpClient.Do(req) + return resp, nil } From 78dbe6cde14f6069e92b6e01fa3fcf5166d15ef0 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 7 Sep 2025 22:39:32 +0900 Subject: [PATCH 19/43] controller WIP --- coordinator/internal/controller/proxy/auth.go | 243 ++---------------- .../internal/controller/proxy/controller.go | 11 +- .../internal/controller/proxy/get_task.go | 111 ++------ .../controller/proxy/prover_session.go | 183 ++++++++----- .../internal/controller/proxy/submit_proof.go | 58 ++++- 5 files changed, 216 insertions(+), 390 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 2fc1d76aa0..627be4955d 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -2,9 +2,7 @@ package proxy import ( "fmt" - "sync" - "context" "time" jwt "github.com/appleboy/gin-jwt/v2" @@ -20,192 +18,30 @@ import ( // AuthController is login API type AuthController struct { - apiLogin *api.AuthController - clients Clients - userTokenCache *UserTokenCache -} - -type TokenUpdate struct { - PublicKey string - Upstream string - Phase uint - LoginParam types.LoginParameter - CompleteNotify chan<- *types.LoginSchema -} - -type UpstreamTokens struct { - LoginData map[string]*types.LoginSchema - LoginPhase uint - NextLoginPhase uint -} - -type UserTokenCache struct { - sync.RWMutex - data map[string]UpstreamTokens - tokenCacheUpdate chan<- *TokenUpdate -} - -func newUserTokens() UpstreamTokens { - return UpstreamTokens{ - LoginData: make(map[string]*types.LoginSchema), - } -} - -func newUserCache(tokenCacheUpdate chan<- *TokenUpdate) *UserTokenCache { - return &UserTokenCache{ - data: make(map[string]UpstreamTokens), - tokenCacheUpdate: tokenCacheUpdate, - } -} - -// get retrieves UpstreamTokens for a given user key, returns empty if still not exists -func (c *UserTokenCache) Get(userKey string) *UpstreamTokens { - c.RLock() - defer c.RUnlock() - - tokens, exists := c.data[userKey] - if !exists { - return nil - } - - return &tokens -} - -// prepare for a total update via Login request -func (c *UserTokenCache) updatePrepare(userKey string) UpstreamTokens { - c.Lock() - defer c.Unlock() - - if _, exists := c.data[userKey]; !exists { - log.Info("initializing user token cache", "userKey", userKey) - c.data[userKey] = newUserTokens() - } - updated := c.data[userKey] - updated.NextLoginPhase = updated.LoginPhase + 1 - c.data[userKey] = updated - return updated -} - -// partialSet updates a single entry in upstreamTokens for a given user -func (c *UserTokenCache) partialSet(userKey string, upstreamName string, loginSchema *types.LoginSchema, phase uint) { - c.Lock() - defer c.Unlock() - - // Get existing tokens or create new map - tokens, exists := c.data[userKey] - if exists && tokens.NextLoginPhase == phase { - // Update the specific upstream entry - tokens.LoginData[upstreamName] = loginSchema - } -} - -// LoginParameterWithHardForkName constructs new payload for login -type LoginParameterWithUpstreamTokens struct { - *types.LoginParameter - Tokens UpstreamTokens + apiLogin *api.AuthController + clients Clients + proverMgr *ProverManager } const upstreamConnTimeout = time.Second * 2 -const expireTolerant = 10 * time.Minute const LoginParamCache = "login_param" const ProverTypesKey = "prover_types" const SignatureKey = "prover_signature" // NewAuthController returns an LoginController instance -func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier) *AuthController { +func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier, proverMgr *ProverManager) *AuthController { loginLogic := auth.NewLoginLogicWithSimpleDEduplicator(cfg.ProxyManager.Verifier, vf) - // Create the token cache update channel - tokenCacheUpdateChan := make(chan *TokenUpdate) - authController := &AuthController{ - apiLogin: api.NewAuthControllerWithLogic(loginLogic), - clients: clients, - userTokenCache: newUserCache(tokenCacheUpdateChan), + apiLogin: api.NewAuthControllerWithLogic(loginLogic), + clients: clients, + proverMgr: proverMgr, } - // Launch token cache manager in a separate goroutine - go authController.toeknCacheManager(tokenCacheUpdateChan) - return authController } -func (a *AuthController) TokenCache() *UserTokenCache { return a.userTokenCache } - -func (a *AuthController) doUpdateRequest(ctx context.Context, req *TokenUpdate) (ret *types.LoginSchema) { - if req.CompleteNotify != nil { - defer func(ctx context.Context) { - select { - case <-ctx.Done(): - case req.CompleteNotify <- ret: - } - - }(ctx) - } - - cli := a.clients[req.Upstream] - if cli := cli.Client(ctx); cli != nil { - var err error - if ret, err = cli.ProxyLogin(ctx, &req.LoginParam); err == nil { - a.userTokenCache.partialSet(req.PublicKey, req.Upstream, ret, req.Phase) - } else { - log.Error("proxy login failed during token cache update", - "userKey", req.PublicKey, - "upstream", req.Upstream, - "phase", req.Phase, - "error", err) - } - } - return - -} - -func (a *AuthController) toeknCacheManager(request <-chan *TokenUpdate) { - - ctx := context.TODO() - var managerStatusLock sync.Mutex - managerStatus := make(map[string]map[string]uint) - - for { - req, ok := <-request - if !ok { - return - } - - // ensure the manager request is not outdated - tokens := a.userTokenCache.Get(req.PublicKey) - if tokens == nil { - // Highly not possible, if raise, the reason is unknown, just log the Error - continue - } - phase := tokens.NextLoginPhase - if req.Phase < phase { - // drop the out-dated request - continue - } - - // ensure only one login request is launched for the same phase - managerStatusLock.Lock() - stat, ok := managerStatus[req.Upstream] - if !ok { - managerStatus[req.Upstream] = make(map[string]uint) - stat = managerStatus[req.Upstream] - } - if phase, running := stat[req.PublicKey]; running && phase >= req.Phase { - managerStatusLock.Unlock() - continue - } else { - stat[req.PublicKey] = req.Phase - } - managerStatusLock.Unlock() - - go a.doUpdateRequest(ctx, req) - - } - -} - // Login extended the Login hander in api controller func (a *AuthController) Login(c *gin.Context) (interface{}, error) { @@ -219,54 +55,24 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { return nil, fmt.Errorf("proxy do not support recursive login") } - tokens := a.userTokenCache.updatePrepare(loginParam.PublicKey) - notifies := make([]chan *types.LoginSchema, len(a.clients)) - - for n := range a.clients { - - // Check if we have a valid cached token that hasn't expired - if knownEntry, existed := tokens.LoginData[n]; existed { - timeRemaining := time.Until(knownEntry.Time) - if timeRemaining > expireTolerant { - // Token is still valid enouth, continue to next client - continue - } - } - - notify := make(chan *types.LoginSchema) - notifies = append(notifies, notify) - request := TokenUpdate{ - PublicKey: loginParam.PublicKey, - Upstream: n, - Phase: tokens.NextLoginPhase, - LoginParam: loginParam.LoginParameter, - CompleteNotify: notify, - } - defer close(notify) - select { - case <-c.Done(): - case a.userTokenCache.tokenCacheUpdate <- &request: - } + session := a.proverMgr.GetOrCreate(loginParam.PublicKey) - } + for n, cli := range a.clients { - // collect all request's compeletions - for _, chn := range notifies { - select { - case <-c.Done(): - case <-chn: + if err := session.ProxyLogin(c, cli, n, &loginParam.LoginParameter); err != nil { + log.Error("proxy login failed during token cache update", + "userKey", loginParam.PublicKey, + "upstream", n, + "error", err) } } - return LoginParameterWithUpstreamTokens{ - LoginParameter: &loginParam.LoginParameter, - Tokens: tokens, - }, nil + return loginParam.LoginParameter, nil } // PayloadFunc returns jwt.MapClaims with {public key, prover name}. func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims { - v, ok := data.(LoginParameterWithUpstreamTokens) + v, ok := data.(types.LoginParameter) if !ok { return jwt.MapClaims{} } @@ -316,23 +122,6 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { } if loginParam.PublicKey != "" { - // ensure tokenCache - a.userTokenCache.RLock() - _, exists := a.userTokenCache.data[loginParam.PublicKey] - if !exists { - a.userTokenCache.RUnlock() - a.userTokenCache.Lock() - if _, exists := a.userTokenCache.data[loginParam.PublicKey]; !exists { - log.Info("creating token cache for user after proxy restart", - "publicKey", loginParam.PublicKey, - "proverName", loginParam.Message.ProverName, - "reason", "prover using JWT token from before proxy restart") - a.userTokenCache.data[loginParam.PublicKey] = newUserTokens() - } - a.userTokenCache.Unlock() - } else { - a.userTokenCache.RUnlock() - } c.Set(LoginParamCache, loginParam) return loginParam.PublicKey diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 5c42927685..122caa85f0 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -1,6 +1,7 @@ package proxy import ( + "github.com/prometheus/client_golang/prometheus" "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" @@ -21,7 +22,7 @@ var ( type Clients map[string]Client // InitController inits Controller with database -func InitController(cfg *config.ProxyConfig) { +func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { // normalize cfg cfg.ProxyManager.Normalize() @@ -42,7 +43,9 @@ func InitController(cfg *config.ProxyConfig) { clients[nm] = cli } - Auth = NewAuthController(cfg, clients, vf) - // GetTask = NewGetTaskController(cfg, chainCfg, db, vf, reg) - // SubmitProof = NewSubmitProofController(cfg, chainCfg, db, vf, reg) + proverManager := NewProverManager() + + Auth = NewAuthController(cfg, clients, vf, proverManager) + GetTask = NewGetTaskController(cfg, clients, proverManager, reg) + SubmitProof = NewSubmitProofController(cfg, clients, proverManager, reg) } diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index 261219aa2f..fad16e3d78 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -1,12 +1,9 @@ package proxy import ( - "encoding/json" "fmt" - "net/http" "github.com/gin-gonic/gin" - "github.com/mitchellh/mapstructure" "github.com/prometheus/client_golang/prometheus" "github.com/scroll-tech/go-ethereum/log" @@ -16,7 +13,7 @@ import ( coordinatorType "scroll-tech/coordinator/internal/types" ) -func getSessionData(ctx *gin.Context) (string, *coordinatorType.LoginParameter) { +func getSessionData(ctx *gin.Context) string { publicKeyData, publicKeyExist := ctx.Get(coordinatorType.PublicKey) publicKey, castOk := publicKeyData.(string) @@ -25,25 +22,15 @@ func getSessionData(ctx *gin.Context) (string, *coordinatorType.LoginParameter) log.Warn("get_task parameter fail", "error", nerr) types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) - return "", nil + return "" } - loginParamData, publicKeyExist := ctx.Get(LoginParamCache) - loginParam, castOk := loginParamData.(*coordinatorType.LoginParameter) - if !publicKeyExist || !castOk { - nerr := fmt.Errorf("no login param binding: %v", loginParamData) - log.Warn("get_task parameter fail", "error", nerr) - - types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) - return "", nil - } - - return publicKey, loginParam + return publicKey } // GetTaskController the get prover task api controller type GetTaskController struct { - tokenCache *UserTokenCache + proverMgr *ProverManager clients Clients priorityUpstream map[string]string @@ -51,11 +38,11 @@ type GetTaskController struct { } // NewGetTaskController create a get prover task controller -func NewGetTaskController(cfg *config.Config, clients Clients, tokenCache *UserTokenCache, reg prometheus.Registerer) *GetTaskController { +func NewGetTaskController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, reg prometheus.Registerer) *GetTaskController { // TODO: implement proxy get task controller initialization return &GetTaskController{ priorityUpstream: make(map[string]string), - tokenCache: tokenCache, + proverMgr: proverMgr, clients: clients, } } @@ -74,88 +61,30 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { return } - publicKey, loginParam := getSessionData(ctx) - if publicKey == "" || loginParam == nil { + publicKey := getSessionData(ctx) + if publicKey == "" { return } - tokens := ptc.tokenCache.Get(publicKey) - - onClientFail := func(upstream string) { - //TODO: log re-connect request in info level - - request := TokenUpdate{ - PublicKey: publicKey, - Upstream: upstream, - Phase: tokens.LoginPhase, - LoginParam: *loginParam, - CompleteNotify: nil, - } - select { - case <-ctx.Done(): - case ptc.tokenCache.tokenCacheUpdate <- &request: - } - - } + session := ptc.proverMgr.Get(publicKey) + // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp priorityUpstream, exist := ptc.priorityUpstream[publicKey] if exist { cli := ptc.clients[priorityUpstream] - loginSchema := tokens.LoginData[priorityUpstream] - if loginSchema == nil { - onClientFail(priorityUpstream) - } else { - ret, triggerUpdate := getTaskFromClient(ctx, cli, &getTaskParameter, loginSchema.Token) - if ret != nil { - - } else if triggerUpdate { - onClientFail(priorityUpstream) - } + resp, err := session.GetTask(ctx, &getTaskParameter, cli, priorityUpstream) + if err != nil { + types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) + return + } else if resp.ErrCode != types.ErrCoordinatorEmptyProofData { + // simply dispatch the error from upstream to prover + types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) + return } - types.RenderFailure(ctx, types.ErrCoordinatorEmptyProofData, fmt.Errorf("get empty prover task")) } for n, cli := range ptc.clients { - - } -} - -func getTaskFromClient(ctx *gin.Context, cli Client, param *coordinatorType.GetTaskParameter, token string) (*coordinatorType.GetTaskSchema, bool) { - - theCli := cli.PeekClient() - if theCli == nil { - return nil, true + // return the first task we can get + // TODO: use random array for all clients } - - resp, err := theCli.GetTask(ctx, param, token) - if err != nil { - // log the err in error level - return nil, false - } - - // Parse response - if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusUnauthorized { - unAuth := resp.StatusCode == http.StatusUnauthorized - var respWithData types.Response - // Note: Body is consumed after decoding, caller should not read it again - if err := json.NewDecoder(resp.Body).Decode(&respWithData); err == nil { - if unAuth && respWithData.ErrCode == types.ErrJWTTokenExpired { - return nil, true - } - - var getTaskResult coordinatorType.GetTaskSchema - err = mapstructure.Decode(respWithData.Data, &getTaskResult) - if err != nil { - log.Error("parse get task data fail", "respdata", respWithData.Data) - return nil, false - } - return &getTaskResult, false - } else { - log.Error("parse get task response failed", "error", err) - //fmt.Errorf("login parsing response failed: %v", err) - return nil, false - } - } - - return nil, false } diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index dbd854508b..13657c2b60 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -4,37 +4,84 @@ import ( "context" "fmt" "sync" + "time" ctypes "scroll-tech/common/types" "scroll-tech/coordinator/internal/types" ) +type ProverManager struct { + sync.RWMutex + data map[string]*proverSession +} + +func NewProverManager() *ProverManager { + return &ProverManager{ + data: make(map[string]*proverSession), + } +} + +// get retrieves ProverSession for a given user key, returns empty if still not exists +func (m *ProverManager) Get(userKey string) *proverSession { + m.RLock() + defer m.RUnlock() + + return m.data[userKey] +} + +func (m *ProverManager) GetOrCreate(userKey string) *proverSession { + m.Lock() + defer m.Unlock() + + if ret, ok := m.data[userKey]; ok { + return ret + } + + ret := &proverSession{ + proverToken: make(map[string]loginToken), + } + + m.data[userKey] = ret + return ret +} + +func (m *ProverManager) Set(userKey string, session *proverSession) { + m.Lock() + defer m.Unlock() + + m.data[userKey] = session +} + +type loginToken struct { + *types.LoginSchema + phase uint +} + // Client wraps an http client with a preset host for coordinator API calls type proverSession struct { sync.RWMutex - proverToken string - phase uint + proverToken map[string]loginToken completionCtx context.Context } -func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, param *types.LoginParameter, phase uint) error { +func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up string, param *types.LoginParameter, phase uint) (*types.LoginSchema, error) { c.Lock() - curPhase := c.phase + curPhase := c.proverToken[up].phase if c.completionCtx != nil { waitctx := c.completionCtx c.Unlock() select { case <-waitctx.Done(): - return c.maintainLogin(ctx, cliMgr, param, phase) + return c.maintainLogin(ctx, cliMgr, up, param, phase) case <-ctx.Done(): - return fmt.Errorf("ctx fail") + return nil, fmt.Errorf("ctx fail") } } if phase < curPhase { // outdate login phase, give up - c.Unlock() - return nil + defer c.Unlock() + return c.proverToken[up].LoginSchema, nil } // occupy the update slot @@ -45,59 +92,75 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, param cli := cliMgr.Client(ctx) if cli == nil { - return fmt.Errorf("get upstream cli fail") + return nil, fmt.Errorf("get upstream cli fail") } resp, err := cli.ProxyLogin(ctx, param) if err != nil { - return fmt.Errorf("proxylogin fail: %v", err) + return nil, fmt.Errorf("proxylogin fail: %v", err) } if resp.ErrCode == ctypes.ErrJWTTokenExpired { cliMgr.Reset(cli) cli = cliMgr.Client(ctx) if cli == nil { - return fmt.Errorf("get upstream cli fail (secondary try)") + return nil, fmt.Errorf("get upstream cli fail (secondary try)") } // like SDK, we would try one more time if the upstream token is expired resp, err = cli.ProxyLogin(ctx, param) if err != nil { - return fmt.Errorf("proxylogin fail: %v", err) + return nil, fmt.Errorf("proxylogin fail: %v", err) } } if resp.ErrCode != 0 { - return fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) + return nil, fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) } var loginResult types.LoginSchema if err := resp.DecodeData(&loginResult); err != nil { - return err + return nil, err } c.Lock() defer c.Unlock() - c.proverToken = loginResult.Token + + c.proverToken[up] = loginToken{ + LoginSchema: &loginResult, + phase: phase, + } c.completionCtx = nil - return nil + return &loginResult, nil } +const expireTolerant = 10 * time.Minute + // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) error { +func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, up string, param *types.LoginParameter) error { c.RLock() - phase := c.phase + 1 + existedToken := c.proverToken[up].LoginSchema + phase := c.proverToken[up].phase + 1 c.RUnlock() - return c.maintainLogin(ctx, cli, param, phase) + // Check if we have a valid cached token that hasn't expired + if existedToken != nil { + timeRemaining := time.Until(existedToken.Time) + if timeRemaining > expireTolerant { + // Token is still valid enouth, continue to next client + return nil + } + } + + _, err := c.maintainLogin(ctx, cli, up, param, phase) + return err } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, cliMgr Client) (*ctypes.Response, error) { +func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, cliMgr Client, up string) (*ctypes.Response, error) { c.RLock() - phase := c.phase - token := c.proverToken + token := c.proverToken[up] c.RUnlock() cli := cliMgr.Client(ctx) @@ -105,35 +168,36 @@ func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParamet return nil, fmt.Errorf("get upstream cli fail") } - resp, err := cli.GetTask(ctx, param, token) - if err != nil { - return nil, err - } - - if resp.ErrCode == ctypes.ErrJWTTokenExpired { - // get param from ctx - loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) - if !ok { - return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") - } - - err = c.maintainLogin(ctx, cliMgr, loginParam, phase) + if token.LoginSchema != nil { + resp, err := cli.GetTask(ctx, param, token.Token) if err != nil { - return nil, fmt.Errorf("update prover token fail: %V", err) + return nil, err + } + if resp.ErrCode != ctypes.ErrJWTTokenExpired { + return resp, nil } + } - // like SDK, we would try one more time if the upstream token is expired - return cli.GetTask(ctx, param, token) + // like SDK, we would try one more time if the upstream token is expired + // get param from ctx + loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) + if !ok { + return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") + } + + newToken, err := c.maintainLogin(ctx, cliMgr, up, loginParam, token.phase) + if err != nil { + return nil, fmt.Errorf("update prover token fail: %V", err) } - return resp, nil + return cli.GetTask(ctx, param, newToken.Token) + } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, cliMgr Client) (*ctypes.Response, error) { +func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, cliMgr Client, up string) (*ctypes.Response, error) { c.RLock() - phase := c.phase - token := c.proverToken + token := c.proverToken[up] c.RUnlock() cli := cliMgr.Client(ctx) @@ -141,26 +205,27 @@ func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProo return nil, fmt.Errorf("get upstream cli fail") } - resp, err := cli.SubmitProof(ctx, param, token) - if err != nil { - return nil, err - } - - if resp.ErrCode == ctypes.ErrJWTTokenExpired { - // get param from ctx - loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) - if !ok { - return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") - } - - err = c.maintainLogin(ctx, cliMgr, loginParam, phase) + if token.LoginSchema != nil { + resp, err := cli.SubmitProof(ctx, param, token.Token) if err != nil { - return nil, fmt.Errorf("update prover token fail: %V", err) + return nil, err + } + if resp.ErrCode != ctypes.ErrJWTTokenExpired { + return resp, nil } + } - // like SDK, we would try one more time if the upstream token is expired - return cli.SubmitProof(ctx, param, token) + // like SDK, we would try one more time if the upstream token is expired + // get param from ctx + loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) + if !ok { + return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") + } + + newToken, err := c.maintainLogin(ctx, cliMgr, up, loginParam, token.phase) + if err != nil { + return nil, fmt.Errorf("update prover token fail: %V", err) } - return resp, nil + return cli.SubmitProof(ctx, param, newToken.Token) } diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index f2afa12945..1841ad1a4a 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -1,29 +1,69 @@ package proxy import ( + "fmt" + "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" - "github.com/scroll-tech/go-ethereum/params" - "gorm.io/gorm" + "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" - "scroll-tech/coordinator/internal/logic/submitproof" - "scroll-tech/coordinator/internal/logic/verifier" + coordinatorType "scroll-tech/coordinator/internal/types" ) // SubmitProofController the submit proof api controller type SubmitProofController struct { - submitProofReceiverLogic *submitproof.ProofReceiverLogic + proverMgr *ProverManager + clients Clients } // NewSubmitProofController create the submit proof api controller instance -func NewSubmitProofController(cfg *config.Config, chainCfg *params.ChainConfig, db *gorm.DB, vf *verifier.Verifier, reg prometheus.Registerer) *SubmitProofController { +func NewSubmitProofController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, reg prometheus.Registerer) *SubmitProofController { return &SubmitProofController{ - submitProofReceiverLogic: submitproof.NewSubmitProofReceiverLogic(cfg.ProverManager, chainCfg, db, vf, reg), + proverMgr: proverMgr, + clients: clients, } } +func upstreamFromTaskName(taskID string) string { + // TODO + return "" +} + // SubmitProof prover submit the proof to coordinator func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { - // TODO: implement proxy submit proof logic -} \ No newline at end of file + var submitParameter coordinatorType.SubmitProofParameter + if err := ctx.ShouldBind(&submitParameter); err != nil { + nerr := fmt.Errorf("prover submitProof parameter invalid, err:%w", err) + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return + } + + publicKey := getSessionData(ctx) + if publicKey == "" { + return + } + + session := spc.proverMgr.Get(publicKey) + upstream := upstreamFromTaskName(submitParameter.TaskID) + cli, existed := spc.clients[upstream] + if !existed { + // TODO: log error + nerr := fmt.Errorf("Invalid upstream name (%s) from taskID %s", upstream, submitParameter.TaskID) + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return + } + + resp, err := session.SubmitProof(ctx, &submitParameter, cli, upstream) + if err != nil { + types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) + return + } else if resp.ErrCode != 0 { + // simply dispatch the error from upstream to prover + types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) + return + } else { + types.RenderSuccess(ctx, resp.Data) + return + } +} From a04b64df0362e22f7eb6340b1b3670e6ae80fa6e Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 8 Sep 2025 22:30:51 +0900 Subject: [PATCH 20/43] routes --- .../internal/controller/proxy/get_task.go | 46 +++++++++++++++---- .../internal/controller/proxy/submit_proof.go | 10 +++- .../internal/middleware/challenge_jwt.go | 6 +-- coordinator/internal/middleware/login_jwt.go | 29 ++++++++++++ coordinator/internal/route/route.go | 20 +++++++- 5 files changed, 97 insertions(+), 14 deletions(-) diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index fad16e3d78..0f0347a2c2 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -68,23 +68,51 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { session := ptc.proverMgr.Get(publicKey) - // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp - priorityUpstream, exist := ptc.priorityUpstream[publicKey] - if exist { - cli := ptc.clients[priorityUpstream] - resp, err := session.GetTask(ctx, &getTaskParameter, cli, priorityUpstream) + getTask := func(upStream string, cli Client) (tryNext bool) { + resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) if err != nil { types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return } else if resp.ErrCode != types.ErrCoordinatorEmptyProofData { - // simply dispatch the error from upstream to prover - types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) + + if resp.ErrCode != 0 { + // simply dispatch the error from upstream to prover + types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) + return + } + + var task coordinatorType.GetTaskSchema + if err = resp.DecodeData(&task); err == nil { + task.TaskID = formUpstreamWithTaskName(upStream, task.TaskID) + // TODO: log the new id in debug level + types.RenderSuccess(ctx, &task) + } else { + types.RenderFailure(ctx, types.InternalServerError, fmt.Errorf("decode task fail: %v", err)) + } + + return + } + tryNext = true + return + } + + // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp + priorityUpstream, exist := ptc.priorityUpstream[publicKey] + if exist { + cli := ptc.clients[priorityUpstream] + if cli != nil && !getTask(priorityUpstream, cli) { return + } else if cli == nil { + // TODO: log error } } for n, cli := range ptc.clients { - // return the first task we can get - // TODO: use random array for all clients + if !getTask(n, cli) { + return + } } + + // if all get task failed, throw empty proof resp + types.RenderFailure(ctx, types.ErrCoordinatorEmptyProofData, fmt.Errorf("get empty prover task")) } diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index 1841ad1a4a..647097785a 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -2,6 +2,7 @@ package proxy import ( "fmt" + "strings" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" @@ -26,10 +27,17 @@ func NewSubmitProofController(cfg *config.ProxyConfig, clients Clients, proverMg } func upstreamFromTaskName(taskID string) string { - // TODO + parts, _, found := strings.Cut(taskID, ":") + if found { + return parts + } return "" } +func formUpstreamWithTaskName(upstream string, taskID string) string { + return fmt.Sprintf("%s:%s", upstream, taskID) +} + // SubmitProof prover submit the proof to coordinator func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { var submitParameter coordinatorType.SubmitProofParameter diff --git a/coordinator/internal/middleware/challenge_jwt.go b/coordinator/internal/middleware/challenge_jwt.go index 6ee8254b07..99a58cc8db 100644 --- a/coordinator/internal/middleware/challenge_jwt.go +++ b/coordinator/internal/middleware/challenge_jwt.go @@ -14,7 +14,7 @@ import ( ) // ChallengeMiddleware jwt challenge middleware -func ChallengeMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { +func ChallengeMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ Authenticator: func(c *gin.Context) (interface{}, error) { return nil, nil @@ -30,8 +30,8 @@ func ChallengeMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { } }, Unauthorized: unauthorized, - Key: []byte(conf.Auth.Secret), - Timeout: time.Second * time.Duration(conf.Auth.ChallengeExpireDurationSec), + Key: []byte(auth.Secret), + Timeout: time.Second * time.Duration(auth.ChallengeExpireDurationSec), TokenLookup: "header: Authorization, query: token, cookie: jwt", TokenHeadName: "Bearer", TimeFunc: time.Now, diff --git a/coordinator/internal/middleware/login_jwt.go b/coordinator/internal/middleware/login_jwt.go index 565aff9daf..7421622d1b 100644 --- a/coordinator/internal/middleware/login_jwt.go +++ b/coordinator/internal/middleware/login_jwt.go @@ -9,6 +9,7 @@ import ( "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/api" + "scroll-tech/coordinator/internal/controller/proxy" "scroll-tech/coordinator/internal/types" ) @@ -46,3 +47,31 @@ func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { return jwtMiddleware } + +// ProxyLoginMiddleware jwt auth middleware for proxy login +func ProxyLoginMiddleware(conf *config.ProxyConfig) *jwt.GinJWTMiddleware { + jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ + PayloadFunc: api.Auth.PayloadFunc, + IdentityHandler: api.Auth.IdentityHandler, + IdentityKey: types.PublicKey, + Key: []byte(conf.Auth.Secret), + Timeout: time.Second * time.Duration(conf.Auth.LoginExpireDurationSec), + Authenticator: proxy.Auth.Login, + Authorizator: nonIdendityAuthorizator, + Unauthorized: unauthorized, + TokenLookup: "header: Authorization, query: token, cookie: jwt", + TokenHeadName: "Bearer", + TimeFunc: time.Now, + LoginResponse: loginResponse, + }) + + if err != nil { + log.Crit("new jwt middleware panic", "error", err) + } + + if errInit := jwtMiddleware.MiddlewareInit(); errInit != nil { + log.Crit("init jwt middleware panic", "error", errInit) + } + + return jwtMiddleware +} diff --git a/coordinator/internal/route/route.go b/coordinator/internal/route/route.go index 2a0383aa8d..6babf7053a 100644 --- a/coordinator/internal/route/route.go +++ b/coordinator/internal/route/route.go @@ -8,6 +8,7 @@ import ( "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/api" + "scroll-tech/coordinator/internal/controller/proxy" "scroll-tech/coordinator/internal/middleware" ) @@ -25,7 +26,7 @@ func Route(router *gin.Engine, cfg *config.Config, reg prometheus.Registerer) { func v1(router *gin.RouterGroup, conf *config.Config) { r := router.Group("/v1") - challengeMiddleware := middleware.ChallengeMiddleware(conf) + challengeMiddleware := middleware.ChallengeMiddleware(conf.Auth) r.GET("/challenge", challengeMiddleware.LoginHandler) loginMiddleware := middleware.LoginMiddleware(conf) @@ -39,3 +40,20 @@ func v1(router *gin.RouterGroup, conf *config.Config) { r.POST("/submit_proof", api.SubmitProof.SubmitProof) } } + +func v1_proxy(router *gin.RouterGroup, conf *config.ProxyConfig) { + r := router.Group("/v1") + + challengeMiddleware := middleware.ChallengeMiddleware(conf.Auth) + r.GET("/challenge", challengeMiddleware.LoginHandler) + + loginMiddleware := middleware.ProxyLoginMiddleware(conf) + r.POST("/login", challengeMiddleware.MiddlewareFunc(), loginMiddleware.LoginHandler) + + // need jwt token api + r.Use(loginMiddleware.MiddlewareFunc()) + { + r.POST("/get_task", proxy.GetTask.GetTasks) + r.POST("/submit_proof", proxy.SubmitProof.SubmitProof) + } +} From 272150365772e0dbc05fbf5b1a0b37e6541091a7 Mon Sep 17 00:00:00 2001 From: Ho Date: Tue, 9 Sep 2025 20:10:18 +0900 Subject: [PATCH 21/43] refining --- coordinator/internal/config/proxy_config.go | 1 - coordinator/internal/controller/proxy/auth.go | 8 +++- .../internal/controller/proxy/client.go | 40 ++++++++++++++----- .../controller/proxy/client_manager.go | 37 +---------------- .../internal/controller/proxy/controller.go | 11 +---- coordinator/internal/logic/auth/login.go | 2 +- coordinator/internal/middleware/login_jwt.go | 12 +++--- coordinator/internal/route/route.go | 17 ++++++-- 8 files changed, 59 insertions(+), 69 deletions(-) diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index e2d510a29b..1ea8bd295c 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -45,7 +45,6 @@ type UpStream struct { type ProxyConfig struct { ProxyManager *ProxyManager `json:"proxy_manager"` ProxyName string `json:"proxy_name"` - Auth *Auth `json:"auth"` Coordinators map[string]*UpStream `json:"coondiators"` } diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 627be4955d..e2d20d58f4 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -29,9 +29,13 @@ const ProverTypesKey = "prover_types" const SignatureKey = "prover_signature" // NewAuthController returns an LoginController instance -func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier, proverMgr *ProverManager) *AuthController { +func NewAuthController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager) *AuthController { - loginLogic := auth.NewLoginLogicWithSimpleDEduplicator(cfg.ProxyManager.Verifier, vf) + // use a dummy Verifier to create login logic (we do not use any information in verifier) + dummyVf := verifier.Verifier{ + OpenVMVkMap: make(map[string]struct{}), + } + loginLogic := auth.NewLoginLogicWithSimpleDeduplicator(cfg.ProxyManager.Verifier, &dummyVf) authController := &AuthController{ apiLogin: api.NewAuthControllerWithLogic(loginLogic), diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index e49caa83f3..e3a6ebb456 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -30,6 +30,16 @@ func newUpClient(cfg *config.UpStream) *upClient { } } +func (c *upClient) Token() string { + return c.loginToken +} + +// need a parsable schema defination +type loginSchema struct { + Time string `json:"time"` + Token string `json:"token"` +} + // FullLogin performs the complete login process: get challenge then login func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.LoginParameter, error)) (*types.LoginSchema, error) { // Step 1: Get challenge @@ -44,22 +54,24 @@ func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.Logi if err != nil { return nil, fmt.Errorf("failed to get challenge: %w", err) } - defer challengeResp.Body.Close() - if challengeResp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("challenge request failed with status: %d", challengeResp.StatusCode) + parsedResp, err := handleHttpResp(challengeResp) + if err != nil { + return nil, err + } else if parsedResp.ErrCode != 0 { + return nil, fmt.Errorf("challenge failed: %d (%s)", parsedResp.ErrCode, parsedResp.ErrMsg) } - // Step 2: Parse challenge response - var loginSchema types.LoginSchema - if err := json.NewDecoder(challengeResp.Body).Decode(&loginSchema); err != nil { + // Ste p2: Parse challenge response + var challengeSchema loginSchema + if err := parsedResp.DecodeData(&challengeSchema); err != nil { return nil, fmt.Errorf("failed to parse challenge response: %w", err) } // Step 3: Use the token from challenge as Bearer token for login url = fmt.Sprintf("%s/coordinator/v1/login", c.baseURL) - param, err := genLogin(loginSchema.Token) + param, err := genLogin(challengeSchema.Token) if err != nil { return nil, fmt.Errorf("failed to setup login parameter: %w", err) } @@ -75,26 +87,32 @@ func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.Logi } req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+loginSchema.Token) + req.Header.Set("Authorization", "Bearer "+challengeSchema.Token) loginResp, err := c.httpClient.Do(req) if err != nil { return nil, fmt.Errorf("failed to perform login request: %w", err) } - parsedResp, err := handleHttpResp(loginResp) + parsedResp, err = handleHttpResp(loginResp) if err != nil { return nil, err + } else if parsedResp.ErrCode != 0 { + return nil, fmt.Errorf("login failed: %d (%s)", parsedResp.ErrCode, parsedResp.ErrMsg) } - var loginResult types.LoginSchema + var loginResult loginSchema err = parsedResp.DecodeData(&loginResult) if err != nil { return nil, fmt.Errorf("login parsing data fail: %v", err) } c.loginToken = loginResult.Token - return &loginResult, nil + // TODO: we need to parse time if we start making use of it + + return &types.LoginSchema{ + Token: loginResult.Token, + }, nil } func handleHttpResp(resp *http.Response) (*ctypes.Response, error) { diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 968c66b328..bab6ef96f9 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -126,7 +126,8 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { // Set new completion context and launch login goroutine ctx, completionDone := context.WithCancel(context.TODO()) loginCli := newUpClient(cliMgr.cfg) - cliMgr.cachedCli.completionCtx = context.WithValue(ctx, "cli", loginCli) + completionCtx = context.WithValue(ctx, "cli", loginCli) + cliMgr.cachedCli.completionCtx = completionCtx // Launch keep-login goroutine go func() { @@ -138,40 +139,6 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { cliMgr.cachedCli.cli = loginCli cliMgr.cachedCli.completionCtx = nil - // Launch waiting thread to clear cached client before expiration - // go func() { - // now := time.Now() - // clearTime := expiredT.Add(-10 * time.Second) // 10s before expiration - - // // If clear time is too soon (less than 10s from now), set it to 10s from now - // if clearTime.Before(now.Add(10 * time.Second)) { - // clearTime = now.Add(10 * time.Second) - // log.Error("token expiration time is too close, delaying clear time", - // "name", cliMgr.name, - // "expiredT", expiredT, - // "adjustedClearTime", clearTime) - // } - - // waitDuration := time.Until(clearTime) - // log.Info("token expiration monitor started", - // "name", cliMgr.name, - // "expiredT", expiredT, - // "clearTime", clearTime, - // "waitDuration", waitDuration) - - // timer := time.NewTimer(waitDuration) - // select { - // case <-ctx.Done(): - // timer.Stop() - // log.Info("token expiration monitor cancelled", "name", cliMgr.name) - // case <-timer.C: - // log.Info("clearing cached client before token expiration", - // "name", cliMgr.name, - // "expiredT", expiredT) - // cliMgr.clearCachedCli(loginCli) - // } - // }() - cliMgr.cachedCli.Unlock() }() diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 122caa85f0..35ff02abf8 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -2,10 +2,8 @@ package proxy import ( "github.com/prometheus/client_golang/prometheus" - "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" - "scroll-tech/coordinator/internal/logic/verifier" ) var ( @@ -26,13 +24,6 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { // normalize cfg cfg.ProxyManager.Normalize() - vf, err := verifier.NewVerifier(cfg.ProxyManager.Verifier) - if err != nil { - panic("proof receiver new verifier failure") - } - - log.Info("verifier created", "openVmVerifier", vf.OpenVMVkMap) - clients := make(map[string]Client) for nm, upCfg := range cfg.Coordinators { @@ -45,7 +36,7 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { proverManager := NewProverManager() - Auth = NewAuthController(cfg, clients, vf, proverManager) + Auth = NewAuthController(cfg, clients, proverManager) GetTask = NewGetTaskController(cfg, clients, proverManager, reg) SubmitProof = NewSubmitProofController(cfg, clients, proverManager, reg) } diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index d64fe56bc9..4207f3b5be 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -40,7 +40,7 @@ func (s *SimpleDeduplicator) InsertChallenge(ctx context.Context, challengeStrin } // NewLoginLogicWithSimpleDEduplicator new a LoginLogic, do not use db to deduplicate challege -func NewLoginLogicWithSimpleDEduplicator(vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { +func NewLoginLogicWithSimpleDeduplicator(vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { return newLoginLogic(&SimpleDeduplicator{}, vcfg, vf) } diff --git a/coordinator/internal/middleware/login_jwt.go b/coordinator/internal/middleware/login_jwt.go index 7421622d1b..9aba202c3e 100644 --- a/coordinator/internal/middleware/login_jwt.go +++ b/coordinator/internal/middleware/login_jwt.go @@ -21,13 +21,13 @@ func nonIdendityAuthorizator(data interface{}, _ *gin.Context) bool { } // LoginMiddleware jwt auth middleware -func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { +func LoginMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ PayloadFunc: api.Auth.PayloadFunc, IdentityHandler: api.Auth.IdentityHandler, IdentityKey: types.PublicKey, - Key: []byte(conf.Auth.Secret), - Timeout: time.Second * time.Duration(conf.Auth.LoginExpireDurationSec), + Key: []byte(auth.Secret), + Timeout: time.Second * time.Duration(auth.LoginExpireDurationSec), Authenticator: api.Auth.Login, Authorizator: nonIdendityAuthorizator, Unauthorized: unauthorized, @@ -49,13 +49,13 @@ func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { } // ProxyLoginMiddleware jwt auth middleware for proxy login -func ProxyLoginMiddleware(conf *config.ProxyConfig) *jwt.GinJWTMiddleware { +func ProxyLoginMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ PayloadFunc: api.Auth.PayloadFunc, IdentityHandler: api.Auth.IdentityHandler, IdentityKey: types.PublicKey, - Key: []byte(conf.Auth.Secret), - Timeout: time.Second * time.Duration(conf.Auth.LoginExpireDurationSec), + Key: []byte(auth.Secret), + Timeout: time.Second * time.Duration(auth.LoginExpireDurationSec), Authenticator: proxy.Auth.Login, Authorizator: nonIdendityAuthorizator, Unauthorized: unauthorized, diff --git a/coordinator/internal/route/route.go b/coordinator/internal/route/route.go index 6babf7053a..5d9a7c65a4 100644 --- a/coordinator/internal/route/route.go +++ b/coordinator/internal/route/route.go @@ -29,7 +29,7 @@ func v1(router *gin.RouterGroup, conf *config.Config) { challengeMiddleware := middleware.ChallengeMiddleware(conf.Auth) r.GET("/challenge", challengeMiddleware.LoginHandler) - loginMiddleware := middleware.LoginMiddleware(conf) + loginMiddleware := middleware.LoginMiddleware(conf.Auth) r.POST("/login", challengeMiddleware.MiddlewareFunc(), loginMiddleware.LoginHandler) // need jwt token api @@ -41,13 +41,24 @@ func v1(router *gin.RouterGroup, conf *config.Config) { } } +// Route register route for coordinator +func ProxyRoute(router *gin.Engine, cfg *config.ProxyConfig, reg prometheus.Registerer) { + router.Use(gin.Recovery()) + + observability.Use(router, "coordinator", reg) + + r := router.Group("coordinator") + + v1_proxy(r, cfg) +} + func v1_proxy(router *gin.RouterGroup, conf *config.ProxyConfig) { r := router.Group("/v1") - challengeMiddleware := middleware.ChallengeMiddleware(conf.Auth) + challengeMiddleware := middleware.ChallengeMiddleware(conf.ProxyManager.Auth) r.GET("/challenge", challengeMiddleware.LoginHandler) - loginMiddleware := middleware.ProxyLoginMiddleware(conf) + loginMiddleware := middleware.ProxyLoginMiddleware(conf.ProxyManager.Auth) r.POST("/login", challengeMiddleware.MiddlewareFunc(), loginMiddleware.LoginHandler) // need jwt token api From 50f3e1a97c462af6e0d0843b1d36c80c1bcf1174 Mon Sep 17 00:00:00 2001 From: Ho Date: Tue, 9 Sep 2025 22:21:24 +0900 Subject: [PATCH 22/43] fix issues from test --- coordinator/internal/controller/proxy/auth.go | 2 + .../internal/controller/proxy/get_task.go | 2 + .../controller/proxy/prover_session.go | 10 +- coordinator/internal/logic/auth/login.go | 21 ++-- coordinator/internal/middleware/login_jwt.go | 4 +- coordinator/test/proxy_test.go | 111 +++++++++++++++++- 6 files changed, 131 insertions(+), 19 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index e2d20d58f4..ccc32e0aa0 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -128,8 +128,10 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { if loginParam.PublicKey != "" { c.Set(LoginParamCache, loginParam) + fmt.Println("identify", loginParam) return loginParam.PublicKey } + fmt.Println("identify empty") return nil } diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index 0f0347a2c2..13eef26d4e 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -54,6 +54,7 @@ func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { // GetTasks get assigned chunk/batch task func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { + fmt.Println("start get task") var getTaskParameter coordinatorType.GetTaskParameter if err := ctx.ShouldBind(&getTaskParameter); err != nil { nerr := fmt.Errorf("prover task parameter invalid, err:%w", err) @@ -70,6 +71,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { getTask := func(upStream string, cli Client) (tryNext bool) { resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) + fmt.Println("upstream get task", resp) if err != nil { types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 13657c2b60..30c69b9a63 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -118,7 +118,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str return nil, fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) } - var loginResult types.LoginSchema + var loginResult loginSchema if err := resp.DecodeData(&loginResult); err != nil { return nil, err } @@ -127,12 +127,14 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str defer c.Unlock() c.proverToken[up] = loginToken{ - LoginSchema: &loginResult, - phase: phase, + LoginSchema: &types.LoginSchema{ + Token: loginResult.Token, + }, + phase: phase, } c.completionCtx = nil - return &loginResult, nil + return c.proverToken[up].LoginSchema, nil } const expireTolerant = 10 * time.Minute diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index 4207f3b5be..f691eb0e85 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -93,16 +93,19 @@ func (l *LoginLogic) CompatiblityCheck(login *types.LoginParameter) error { vks[vk] = struct{}{} } - for _, vk := range login.Message.VKs { - if _, ok := vks[vk]; !ok { - log.Error("vk inconsistency", "prover vk", vk, "prover name", login.Message.ProverName, - "prover_version", login.Message.ProverVersion, "message", login.Message) - if !version.CheckScrollProverVersion(login.Message.ProverVersion) { - return fmt.Errorf("incompatible prover version. please upgrade your prover, expect version: %s, actual version: %s", - version.Version, login.Message.ProverVersion) + // new coordinator / proxy do not check vks while login, code only for backward compatibility + if len(vks) != 0 { + for _, vk := range login.Message.VKs { + if _, ok := vks[vk]; !ok { + log.Error("vk inconsistency", "prover vk", vk, "prover name", login.Message.ProverName, + "prover_version", login.Message.ProverVersion, "message", login.Message) + if !version.CheckScrollProverVersion(login.Message.ProverVersion) { + return fmt.Errorf("incompatible prover version. please upgrade your prover, expect version: %s, actual version: %s", + version.Version, login.Message.ProverVersion) + } + // if the prover reports a same prover version + return errors.New("incompatible vk. please check your params files or config files") } - // if the prover reports a same prover version - return errors.New("incompatible vk. please check your params files or config files") } } diff --git a/coordinator/internal/middleware/login_jwt.go b/coordinator/internal/middleware/login_jwt.go index 9aba202c3e..66d9702ac8 100644 --- a/coordinator/internal/middleware/login_jwt.go +++ b/coordinator/internal/middleware/login_jwt.go @@ -51,8 +51,8 @@ func LoginMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { // ProxyLoginMiddleware jwt auth middleware for proxy login func ProxyLoginMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ - PayloadFunc: api.Auth.PayloadFunc, - IdentityHandler: api.Auth.IdentityHandler, + PayloadFunc: proxy.Auth.PayloadFunc, + IdentityHandler: proxy.Auth.IdentityHandler, IdentityKey: types.PublicKey, Key: []byte(auth.Secret), Timeout: time.Second * time.Duration(auth.LoginExpireDurationSec), diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index 670d6548ca..3a5a234632 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -2,21 +2,30 @@ package test import ( "context" + "errors" "fmt" + "net/http" "testing" "time" + "github.com/gin-gonic/gin" + "github.com/scroll-tech/da-codec/encoding" "github.com/stretchr/testify/assert" + "scroll-tech/common/types/message" + "scroll-tech/common/version" + "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/proxy" + "scroll-tech/coordinator/internal/route" ) func testProxyClientCfg() *config.ProxyClient { return &config.ProxyClient{ - Secret: "test-secret-key", - ProxyName: "test-proxy", + Secret: "test-secret-key", + ProxyName: "test-proxy", + ProxyVersion: version.Version, } } @@ -57,13 +66,107 @@ func testProxyClient(t *testing.T) { // Client should not be nil if login succeeds // Note: This might be nil if the coordinator is not properly set up for proxy authentication // but the test validates that the Client method completes without panic - t.Logf("Client toke: %v", client) + assert.NotNil(t, client) + assert.NotEmpty(t, client.Token()) + t.Logf("Client token: %s (%v)", client.Token(), client) +} + +var ( + proxyConf *config.ProxyConfig +) + +func setupProxy(t *testing.T, proxyURL string, coordinatorURL []string) *http.Server { + var err error + assert.NoError(t, err) + + coordinators := make(map[string]*config.UpStream) + for i, n := range coordinatorURL { + coordinators[fmt.Sprintf("coordinator_%d", i)] = testProxyUpStreamCfg(n) + } + + tokenTimeout = 60 + proxyConf = &config.ProxyConfig{ + ProxyName: "test_proxy", + ProxyManager: &config.ProxyManager{ + Verifier: &config.VerifierConfig{ + MinProverVersion: "v4.4.89", + Verifiers: []config.AssetConfig{{ + AssetsPath: "", + ForkName: "euclidV2", + }}, + }, + Client: testProxyClientCfg(), + Auth: &config.Auth{ + Secret: "proxy", + ChallengeExpireDurationSec: tokenTimeout, + LoginExpireDurationSec: tokenTimeout, + }, + }, + Coordinators: coordinators, + } + + router := gin.New() + proxy.InitController(proxyConf, nil) + route.ProxyRoute(router, proxyConf, nil) + srv := &http.Server{ + Addr: proxyURL, + Handler: router, + } + go func() { + runErr := srv.ListenAndServe() + if runErr != nil && !errors.Is(runErr, http.ErrServerClosed) { + assert.NoError(t, runErr) + } + }() + time.Sleep(time.Second * 2) + + return srv +} + +func testProxyHandshake(t *testing.T) { + // Setup proxy http server. + proxyURL := randomURL() + proxyHttpHandler := setupProxy(t, proxyURL, []string{}) + defer func() { + assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) + }() + + chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) + assert.True(t, chunkProver.healthCheckSuccess(t)) +} + +func testProxyGetTask(t *testing.T) { + // Setup coordinator and http server. + coordinatorURL := randomURL() + collector, httpHandler := setupCoordinator(t, 3, coordinatorURL) + defer func() { + collector.Stop() + assert.NoError(t, httpHandler.Shutdown(context.Background())) + }() + + proxyURL := randomURL() + proxyHttpHandler := setupProxy(t, proxyURL, []string{coordinatorURL}) + defer func() { + assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) + }() + + err := l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2}) + assert.NoError(t, err) + dbChunk, err := chunkOrm.InsertChunk(context.Background(), chunk) + assert.NoError(t, err) + err = l2BlockOrm.UpdateChunkHashInRange(context.Background(), 0, 100, dbChunk.Hash) + assert.NoError(t, err) + chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) + code, _ := chunkProver.tryGetProverTask(t, message.ProofTypeChunk) + assert.Empty(t, code) } func TestProxyClient(t *testing.T) { // Set up the test environment. setEnv(t) - t.Run("TestProxyHandshake", testProxyClient) + t.Run("TestProxyClient", testProxyClient) + t.Run("TestProxyHandshake", testProxyHandshake) + t.Run("TestProxyGetTask", testProxyGetTask) } From 92ca7a6b76e4f285245e27985a7314fc4c50279e Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 10 Sep 2025 13:55:38 +0900 Subject: [PATCH 23/43] improve get_task proxy --- coordinator/internal/controller/proxy/auth.go | 16 ++--- .../internal/controller/proxy/controller.go | 3 +- .../internal/controller/proxy/get_task.go | 66 ++++++++++++++++--- .../controller/proxy/prover_session.go | 16 ++--- 4 files changed, 76 insertions(+), 25 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index ccc32e0aa0..a312384d77 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -63,12 +63,14 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { for n, cli := range a.clients { - if err := session.ProxyLogin(c, cli, n, &loginParam.LoginParameter); err != nil { - log.Error("proxy login failed during token cache update", - "userKey", loginParam.PublicKey, - "upstream", n, - "error", err) - } + go func(n string, cli Client) { + if err := session.ProxyLogin(c, cli, n, &loginParam.LoginParameter); err != nil { + log.Error("proxy login failed during token cache update", + "userKey", loginParam.PublicKey, + "upstream", n, + "error", err) + } + }(n, cli) } return loginParam.LoginParameter, nil @@ -128,10 +130,8 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { if loginParam.PublicKey != "" { c.Set(LoginParamCache, loginParam) - fmt.Println("identify", loginParam) return loginParam.PublicKey } - fmt.Println("identify empty") return nil } diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 35ff02abf8..d1b32ecaff 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -35,8 +35,9 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { } proverManager := NewProverManager() + priorityManager := NewPriorityUpstreamManager() Auth = NewAuthController(cfg, clients, proverManager) - GetTask = NewGetTaskController(cfg, clients, proverManager, reg) + GetTask = NewGetTaskController(cfg, clients, proverManager, priorityManager, reg) SubmitProof = NewSubmitProofController(cfg, clients, proverManager, reg) } diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index 13eef26d4e..b941a53c7c 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -2,6 +2,8 @@ package proxy import ( "fmt" + "math/rand" + "sync" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" @@ -28,20 +30,56 @@ func getSessionData(ctx *gin.Context) string { return publicKey } +// PriorityUpstreamManager manages priority upstream mappings with thread safety +type PriorityUpstreamManager struct { + sync.RWMutex + data map[string]string +} + +// NewPriorityUpstreamManager creates a new PriorityUpstreamManager +func NewPriorityUpstreamManager() *PriorityUpstreamManager { + return &PriorityUpstreamManager{ + data: make(map[string]string), + } +} + +// Get retrieves the priority upstream for a given key +func (p *PriorityUpstreamManager) Get(key string) (string, bool) { + p.RLock() + defer p.RUnlock() + value, exists := p.data[key] + return value, exists +} + +// Set sets the priority upstream for a given key +func (p *PriorityUpstreamManager) Set(key, value string) { + p.Lock() + defer p.Unlock() + p.data[key] = value +} + +// Delete removes the priority upstream for a given key +func (p *PriorityUpstreamManager) Delete(key string) { + p.Lock() + defer p.Unlock() + delete(p.data, key) +} + // GetTaskController the get prover task api controller type GetTaskController struct { proverMgr *ProverManager clients Clients - priorityUpstream map[string]string + priorityUpstream *PriorityUpstreamManager + workingRnd *rand.Rand getTaskAccessCounter *prometheus.CounterVec } // NewGetTaskController create a get prover task controller -func NewGetTaskController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, reg prometheus.Registerer) *GetTaskController { +func NewGetTaskController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, priorityMgr *PriorityUpstreamManager, reg prometheus.Registerer) *GetTaskController { // TODO: implement proxy get task controller initialization return &GetTaskController{ - priorityUpstream: make(map[string]string), + priorityUpstream: priorityMgr, proverMgr: proverMgr, clients: clients, } @@ -54,7 +92,7 @@ func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { // GetTasks get assigned chunk/batch task func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { - fmt.Println("start get task") + var getTaskParameter coordinatorType.GetTaskParameter if err := ctx.ShouldBind(&getTaskParameter); err != nil { nerr := fmt.Errorf("prover task parameter invalid, err:%w", err) @@ -71,7 +109,6 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { getTask := func(upStream string, cli Client) (tryNext bool) { resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) - fmt.Println("upstream get task", resp) if err != nil { types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return @@ -86,6 +123,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { var task coordinatorType.GetTaskSchema if err = resp.DecodeData(&task); err == nil { task.TaskID = formUpstreamWithTaskName(upStream, task.TaskID) + ptc.priorityUpstream.Set(publicKey, upStream) // TODO: log the new id in debug level types.RenderSuccess(ctx, &task) } else { @@ -99,7 +137,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { } // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp - priorityUpstream, exist := ptc.priorityUpstream[publicKey] + priorityUpstream, exist := ptc.priorityUpstream.Get(publicKey) if exist { cli := ptc.clients[priorityUpstream] if cli != nil && !getTask(priorityUpstream, cli) { @@ -109,8 +147,20 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { } } - for n, cli := range ptc.clients { - if !getTask(n, cli) { + // Create a slice to hold the keys + keys := make([]string, 0, len(ptc.clients)) + for k := range ptc.clients { + keys = append(keys, k) + } + + // Shuffle the keys using a local RNG (avoid deprecated rand.Seed) + rand.Shuffle(len(keys), func(i, j int) { + keys[i], keys[j] = keys[j], keys[i] + }) + + // Iterate over the shuffled keys + for _, n := range keys { + if !getTask(n, ptc.clients[n]) { return } } diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 30c69b9a63..7e2d6dd7c4 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -3,6 +3,7 @@ package proxy import ( "context" "fmt" + "math" "sync" "time" @@ -130,7 +131,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str LoginSchema: &types.LoginSchema{ Token: loginResult.Token, }, - phase: phase, + phase: curPhase + 1, } c.completionCtx = nil @@ -143,19 +144,18 @@ const expireTolerant = 10 * time.Minute func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, up string, param *types.LoginParameter) error { c.RLock() existedToken := c.proverToken[up].LoginSchema - phase := c.proverToken[up].phase + 1 c.RUnlock() // Check if we have a valid cached token that hasn't expired if existedToken != nil { - timeRemaining := time.Until(existedToken.Time) - if timeRemaining > expireTolerant { - // Token is still valid enouth, continue to next client - return nil - } + // TODO: how to reduce the unnecessary re-login? + // timeRemaining := time.Until(existedToken.Time) + // if timeRemaining > expireTolerant { + // return nil + // } } - _, err := c.maintainLogin(ctx, cli, up, param, phase) + _, err := c.maintainLogin(ctx, cli, up, param, math.MaxUint) return err } From c7b83a0784df9f2b811a65a76edfbd5f41cb4fe3 Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 10 Sep 2025 13:55:45 +0900 Subject: [PATCH 24/43] fix issue in test --- coordinator/test/api_test.go | 27 +++++++++++++++++++++++++++ coordinator/test/mock_prover.go | 2 +- coordinator/test/proxy_test.go | 18 +++++++++++++----- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/coordinator/test/api_test.go b/coordinator/test/api_test.go index 053f6b715e..f435dd16b6 100644 --- a/coordinator/test/api_test.go +++ b/coordinator/test/api_test.go @@ -51,6 +51,8 @@ var ( chunk *encoding.Chunk batch *encoding.Batch tokenTimeout int + + envSet bool ) func TestMain(m *testing.M) { @@ -67,6 +69,25 @@ func randomURL() string { return fmt.Sprintf("localhost:%d", 10000+2000+id.Int64()) } +// Generate a batch of random localhost URLs with different ports, similar to randomURL. +func randmURLBatch(n int) []string { + if n <= 0 { + return nil + } + urls := make([]string, 0, n) + used := make(map[int64]struct{}, n) + for len(urls) < n { + id, _ := rand.Int(rand.Reader, big.NewInt(2000-1)) + port := 10000 + 2000 + id.Int64() + if _, ok := used[port]; ok { + continue + } + used[port] = struct{}{} + urls = append(urls, fmt.Sprintf("localhost:%d", port)) + } + return urls +} + func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL string) (*cron.Collector, *http.Server) { var err error db, err = testApps.GetGormDBClient() @@ -130,6 +151,11 @@ func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL stri } func setEnv(t *testing.T) { + if envSet { + t.Log("SetEnv is re-entried") + return + } + var err error version.Version = "v4.4.89" @@ -169,6 +195,7 @@ func setEnv(t *testing.T) { assert.NoError(t, err) batch = &encoding.Batch{Chunks: []*encoding.Chunk{chunk}} + envSet = true } func TestApis(t *testing.T) { diff --git a/coordinator/test/mock_prover.go b/coordinator/test/mock_prover.go index 0076199b33..958c230547 100644 --- a/coordinator/test/mock_prover.go +++ b/coordinator/test/mock_prover.go @@ -191,7 +191,7 @@ func (r *mockProver) tryGetProverTask(t *testing.T, proofType message.ProofType) resp, err := client.R(). SetHeader("Content-Type", "application/json"). SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)). - SetBody(map[string]interface{}{"prover_height": 100, "task_type": int(proofType), "universal": true}). + SetBody(map[string]interface{}{"prover_height": 100, "task_types": []int{int(proofType)}, "universal": true}). SetResult(&result). Post("http://" + r.coordinatorURL + "/coordinator/v1/get_task") assert.NoError(t, err) diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index 3a5a234632..4318c96a0d 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -137,14 +137,15 @@ func testProxyHandshake(t *testing.T) { func testProxyGetTask(t *testing.T) { // Setup coordinator and http server. - coordinatorURL := randomURL() + urls := randmURLBatch(2) + coordinatorURL := urls[0] collector, httpHandler := setupCoordinator(t, 3, coordinatorURL) defer func() { collector.Stop() assert.NoError(t, httpHandler.Shutdown(context.Background())) }() - proxyURL := randomURL() + proxyURL := urls[1] proxyHttpHandler := setupProxy(t, proxyURL, []string{coordinatorURL}) defer func() { assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) @@ -157,16 +158,23 @@ func testProxyGetTask(t *testing.T) { err = l2BlockOrm.UpdateChunkHashInRange(context.Background(), 0, 100, dbChunk.Hash) assert.NoError(t, err) + time.Sleep(time.Second) + chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) - code, _ := chunkProver.tryGetProverTask(t, message.ProofTypeChunk) + task, code, msg := chunkProver.getProverTask(t, message.ProofTypeChunk) assert.Empty(t, code) + if code == 0 { + t.Log("get task id", task.TaskID) + } else { + t.Log("get task error msg", msg) + } } func TestProxyClient(t *testing.T) { // Set up the test environment. setEnv(t) - t.Run("TestProxyClient", testProxyClient) - t.Run("TestProxyHandshake", testProxyHandshake) + //t.Run("TestProxyClient", testProxyClient) + //t.Run("TestProxyHandshake", testProxyHandshake) t.Run("TestProxyGetTask", testProxyGetTask) } From 057e22072c52fee350a387412dc9b62b058efd2c Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 10 Sep 2025 20:38:21 +0900 Subject: [PATCH 25/43] fix issues --- common/types/response.go | 11 ++++- .../internal/controller/proxy/controller.go | 2 +- .../internal/controller/proxy/get_task.go | 1 + .../controller/proxy/prover_session.go | 28 ++++++----- .../internal/controller/proxy/submit_proof.go | 25 ++++++---- coordinator/internal/types/response_test.go | 48 +++++++++++++++++++ 6 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 coordinator/internal/types/response_test.go diff --git a/common/types/response.go b/common/types/response.go index abd29041e7..714c0dcb05 100644 --- a/common/types/response.go +++ b/common/types/response.go @@ -15,7 +15,16 @@ type Response struct { } func (resp *Response) DecodeData(out interface{}) error { - return mapstructure.Decode(resp.Data, out) + // Decode generically unmarshaled JSON (map[string]any, []any) into a typed struct + // honoring `json` tags and allowing weak type conversions. + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + TagName: "json", + Result: out, + }) + if err != nil { + return err + } + return dec.Decode(resp.Data) } // RenderJSON renders response with json diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index d1b32ecaff..0e1b217a3f 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -39,5 +39,5 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { Auth = NewAuthController(cfg, clients, proverManager) GetTask = NewGetTaskController(cfg, clients, proverManager, priorityManager, reg) - SubmitProof = NewSubmitProofController(cfg, clients, proverManager, reg) + SubmitProof = NewSubmitProofController(cfg, clients, proverManager, priorityManager, reg) } diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index b941a53c7c..8261ff0f16 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -146,6 +146,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { // TODO: log error } } + ptc.priorityUpstream.Delete(publicKey) // Create a slice to hold the keys keys := make([]string, 0, len(ptc.clients)) diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 7e2d6dd7c4..eb41fbe3d4 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -65,7 +65,7 @@ type proverSession struct { completionCtx context.Context } -func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up string, param *types.LoginParameter, phase uint) (*types.LoginSchema, error) { +func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up string, param *types.LoginParameter, phase uint) (result *types.LoginSchema, nerr error) { c.Lock() curPhase := c.proverToken[up].phase if c.completionCtx != nil { @@ -89,6 +89,17 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str completeCtx, cf := context.WithCancel(ctx) defer cf() c.completionCtx = completeCtx + defer func() { + c.Lock() + c.completionCtx = nil + if result != nil { + c.proverToken[up] = loginToken{ + LoginSchema: result, + phase: curPhase + 1, + } + } + c.Unlock() + }() c.Unlock() cli := cliMgr.Client(ctx) @@ -124,18 +135,9 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str return nil, err } - c.Lock() - defer c.Unlock() - - c.proverToken[up] = loginToken{ - LoginSchema: &types.LoginSchema{ - Token: loginResult.Token, - }, - phase: curPhase + 1, - } - c.completionCtx = nil - - return c.proverToken[up].LoginSchema, nil + return &types.LoginSchema{ + Token: loginResult.Token, + }, nil } const expireTolerant = 10 * time.Minute diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index 647097785a..90c582620b 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -14,24 +14,26 @@ import ( // SubmitProofController the submit proof api controller type SubmitProofController struct { - proverMgr *ProverManager - clients Clients + proverMgr *ProverManager + clients Clients + priorityUpstream *PriorityUpstreamManager } // NewSubmitProofController create the submit proof api controller instance -func NewSubmitProofController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, reg prometheus.Registerer) *SubmitProofController { +func NewSubmitProofController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, priorityMgr *PriorityUpstreamManager, reg prometheus.Registerer) *SubmitProofController { return &SubmitProofController{ - proverMgr: proverMgr, - clients: clients, + proverMgr: proverMgr, + clients: clients, + priorityUpstream: priorityMgr, } } -func upstreamFromTaskName(taskID string) string { - parts, _, found := strings.Cut(taskID, ":") +func upstreamFromTaskName(taskID string) (string, string) { + parts, rest, found := strings.Cut(taskID, ":") if found { - return parts + return parts, rest } - return "" + return "", parts } func formUpstreamWithTaskName(upstream string, taskID string) string { @@ -40,6 +42,7 @@ func formUpstreamWithTaskName(upstream string, taskID string) string { // SubmitProof prover submit the proof to coordinator func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { + var submitParameter coordinatorType.SubmitProofParameter if err := ctx.ShouldBind(&submitParameter); err != nil { nerr := fmt.Errorf("prover submitProof parameter invalid, err:%w", err) @@ -53,7 +56,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { } session := spc.proverMgr.Get(publicKey) - upstream := upstreamFromTaskName(submitParameter.TaskID) + upstream, realTaskID := upstreamFromTaskName(submitParameter.TaskID) cli, existed := spc.clients[upstream] if !existed { // TODO: log error @@ -61,6 +64,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) return } + submitParameter.TaskID = realTaskID resp, err := session.SubmitProof(ctx, &submitParameter, cli, upstream) if err != nil { @@ -71,6 +75,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) return } else { + spc.priorityUpstream.Delete(upstream) types.RenderSuccess(ctx, resp.Data) return } diff --git a/coordinator/internal/types/response_test.go b/coordinator/internal/types/response_test.go new file mode 100644 index 0000000000..6508d870a6 --- /dev/null +++ b/coordinator/internal/types/response_test.go @@ -0,0 +1,48 @@ +package types + +import ( + "encoding/json" + "reflect" + "testing" + + "scroll-tech/common/types" +) + +func TestResponseDecodeData_GetTaskSchema(t *testing.T) { + // Arrange: build a dummy payload and wrap it in Response + in := GetTaskSchema{ + UUID: "uuid-123", + TaskID: "task-abc", + TaskType: 1, + UseSnark: true, + TaskData: "dummy-data", + HardForkName: "cancun", + } + + resp := types.Response{ + ErrCode: 0, + ErrMsg: "", + Data: in, + } + + // Act: JSON round-trip the Response to simulate real HTTP encoding/decoding + b, err := json.Marshal(resp) + if err != nil { + t.Fatalf("marshal response: %v", err) + } + + var decoded types.Response + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + + var out GetTaskSchema + if err := decoded.DecodeData(&out); err != nil { + t.Fatalf("DecodeData error: %v", err) + } + + // Assert: structs match after decode + if !reflect.DeepEqual(in, out) { + t.Fatalf("decoded struct mismatch:\nwant: %+v\n got: %+v", in, out) + } +} From b7f23c6734011a9b1316bd4c43d3cb1dda6b0e68 Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 10 Sep 2025 20:48:21 +0900 Subject: [PATCH 26/43] basic tests --- .../controller/proxy/client_manager.go | 3 +- coordinator/test/proxy_test.go | 101 +++++++++++++++++- 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index bab6ef96f9..ad170384f9 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -132,8 +132,7 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { // Launch keep-login goroutine go func() { defer completionDone() - expiredT := cliMgr.doLogin(context.Background(), loginCli) - log.Info("login compeleted", "name", cliMgr.name, "expired", expiredT) + cliMgr.doLogin(context.Background(), loginCli) cliMgr.cachedCli.Lock() cliMgr.cachedCli.cli = loginCli diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index 4318c96a0d..b8a09afbac 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strings" "testing" "time" @@ -12,6 +13,7 @@ import ( "github.com/scroll-tech/da-codec/encoding" "github.com/stretchr/testify/assert" + "scroll-tech/common/types" "scroll-tech/common/types/message" "scroll-tech/common/version" @@ -151,6 +153,10 @@ func testProxyGetTask(t *testing.T) { assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) }() + chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) + code, msg := chunkProver.tryGetProverTask(t, message.ProofTypeChunk) + assert.Equal(t, int(types.ErrCoordinatorEmptyProofData), code) + err := l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2}) assert.NoError(t, err) dbChunk, err := chunkOrm.InsertChunk(context.Background(), chunk) @@ -158,23 +164,110 @@ func testProxyGetTask(t *testing.T) { err = l2BlockOrm.UpdateChunkHashInRange(context.Background(), 0, 100, dbChunk.Hash) assert.NoError(t, err) - time.Sleep(time.Second) + task, code, msg := chunkProver.getProverTask(t, message.ProofTypeChunk) + assert.Empty(t, code) + if code == 0 { + t.Log("get task id", task.TaskID) + } else { + t.Log("get task error msg", msg) + } + +} + +func testProxyProof(t *testing.T) { + urls := randmURLBatch(3) + coordinatorURL0 := urls[0] + collector0, httpHandler0 := setupCoordinator(t, 3, coordinatorURL0) + defer func() { + collector0.Stop() + httpHandler0.Shutdown(context.Background()) + }() + coordinatorURL1 := urls[1] + collector1, httpHandler1 := setupCoordinator(t, 3, coordinatorURL1) + defer func() { + collector1.Stop() + httpHandler1.Shutdown(context.Background()) + }() + coordinators := map[string]*http.Server{ + "coordinator_0": httpHandler0, + "coordinator_1": httpHandler1, + } + + proxyURL := urls[2] + proxyHttpHandler := setupProxy(t, proxyURL, []string{coordinatorURL0, coordinatorURL1}) + defer func() { + fmt.Println("px end start") + assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) + fmt.Println("px end") + }() + + err := l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2}) + assert.NoError(t, err) + dbChunk, err := chunkOrm.InsertChunk(context.Background(), chunk) + assert.NoError(t, err) + err = l2BlockOrm.UpdateChunkHashInRange(context.Background(), 0, 100, dbChunk.Hash) + assert.NoError(t, err) chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) task, code, msg := chunkProver.getProverTask(t, message.ProofTypeChunk) assert.Empty(t, code) if code == 0 { - t.Log("get task id", task.TaskID) + t.Log("get task", task) + parts, _, _ := strings.Cut(task.TaskID, ":") + // close the coordinator which do not dispatch task first, so if we submit to wrong target, + // there would be a chance the submit failed (to the closed coordinator) + for n, srv := range coordinators { + if n != parts { + t.Log("close coordinator", n) + assert.NoError(t, srv.Shutdown(context.Background())) + } + } + exceptProofStatus := verifiedSuccess + chunkProver.submitProof(t, task, exceptProofStatus, types.Success) + } else { t.Log("get task error msg", msg) } + + // verify proof status + var ( + tick = time.Tick(1500 * time.Millisecond) + tickStop = time.Tick(time.Minute) + ) + + var ( + chunkProofStatus types.ProvingStatus + chunkActiveAttempts int16 + chunkMaxAttempts int16 + ) + + for { + select { + case <-tick: + chunkProofStatus, err = chunkOrm.GetProvingStatusByHash(context.Background(), dbChunk.Hash) + assert.NoError(t, err) + if chunkProofStatus == types.ProvingTaskVerified { + return + } + + chunkActiveAttempts, chunkMaxAttempts, err = chunkOrm.GetAttemptsByHash(context.Background(), dbChunk.Hash) + assert.NoError(t, err) + assert.Equal(t, 1, int(chunkMaxAttempts)) + assert.Equal(t, 0, int(chunkActiveAttempts)) + + case <-tickStop: + t.Error("failed to check proof status", "chunkProofStatus", chunkProofStatus.String()) + return + } + } } func TestProxyClient(t *testing.T) { // Set up the test environment. setEnv(t) - //t.Run("TestProxyClient", testProxyClient) - //t.Run("TestProxyHandshake", testProxyHandshake) + t.Run("TestProxyClient", testProxyClient) + t.Run("TestProxyHandshake", testProxyHandshake) t.Run("TestProxyGetTask", testProxyGetTask) + t.Run("TestProxyValidProof", testProxyProof) } From c79ad57fb7d08cc0911b2bdb25b990c838ea26aa Mon Sep 17 00:00:00 2001 From: Ho Date: Tue, 7 Oct 2025 10:54:41 +0900 Subject: [PATCH 27/43] finish binary --- coordinator/Makefile | 4 ++ coordinator/cmd/proxy/app/app.go | 106 +++++++++++++++++++++++++++++ coordinator/cmd/proxy/app/flags.go | 30 ++++++++ coordinator/cmd/proxy/main.go | 7 ++ coordinator/conf/config_proxy.json | 2 +- 5 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 coordinator/cmd/proxy/app/app.go create mode 100644 coordinator/cmd/proxy/app/flags.go create mode 100644 coordinator/cmd/proxy/main.go diff --git a/coordinator/Makefile b/coordinator/Makefile index dce6556840..5749e54a48 100644 --- a/coordinator/Makefile +++ b/coordinator/Makefile @@ -34,6 +34,10 @@ coordinator_cron: coordinator_tool: go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_tool ./cmd/tool +coordinator_proxy: + go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_proxy ./cmd/proxy + + localsetup: coordinator_api ## Local setup: build coordinator_api, copy config, and setup releases @echo "Copying configuration files..." cp -r $(PWD)/conf $(PWD)/build/bin/ diff --git a/coordinator/cmd/proxy/app/app.go b/coordinator/cmd/proxy/app/app.go new file mode 100644 index 0000000000..97a331625e --- /dev/null +++ b/coordinator/cmd/proxy/app/app.go @@ -0,0 +1,106 @@ +package app + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "os/signal" + "time" + + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" + "github.com/scroll-tech/go-ethereum/log" + "github.com/urfave/cli/v2" + + "scroll-tech/common/utils" + "scroll-tech/common/version" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/controller/proxy" + "scroll-tech/coordinator/internal/route" +) + +var app *cli.App + +func init() { + // Set up coordinator app info. + app = cli.NewApp() + app.Action = action + app.Name = "coordinator proxy" + app.Usage = "Proxy for multiple Scroll L2 Coordinators" + app.Version = version.Version + app.Flags = append(app.Flags, utils.CommonFlags...) + app.Flags = append(app.Flags, apiFlags...) + app.Before = func(ctx *cli.Context) error { + return utils.LogSetup(ctx) + } + // Register `coordinator-test` app for integration-test. + utils.RegisterSimulation(app, utils.CoordinatorAPIApp) +} + +func action(ctx *cli.Context) error { + cfgFile := ctx.String(utils.ConfigFileFlag.Name) + cfg, err := config.NewProxyConfig(cfgFile) + if err != nil { + log.Crit("failed to load config file", "config file", cfgFile, "error", err) + } + + //observability.Server(ctx, db) + registry := prometheus.DefaultRegisterer + + apiSrv := server(ctx, cfg, registry) + + log.Info( + "Start coordinator api successfully.", + "version", version.Version, + ) + + // Catch CTRL-C to ensure a graceful shutdown. + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + + // Wait until the interrupt signal is received from an OS signal. + <-interrupt + log.Info("start shutdown coordinator proxy server ...") + + closeCtx, cancelExit := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelExit() + if err = apiSrv.Shutdown(closeCtx); err != nil { + log.Warn("shutdown coordinator proxy server failure", "error", err) + return nil + } + + <-closeCtx.Done() + log.Info("coordinator proxy server exiting success") + return nil +} + +func server(ctx *cli.Context, cfg *config.ProxyConfig, reg prometheus.Registerer) *http.Server { + router := gin.New() + proxy.InitController(cfg, reg) + route.ProxyRoute(router, cfg, reg) + port := ctx.String(httpPortFlag.Name) + srv := &http.Server{ + Addr: fmt.Sprintf(":%s", port), + Handler: router, + ReadHeaderTimeout: time.Minute, + } + + go func() { + if runServerErr := srv.ListenAndServe(); runServerErr != nil && !errors.Is(runServerErr, http.ErrServerClosed) { + log.Crit("run coordinator proxy http server failure", "error", runServerErr) + } + }() + return srv +} + +// Run coordinator. +func Run() { + // RunApp the coordinator. + if err := app.Run(os.Args); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/coordinator/cmd/proxy/app/flags.go b/coordinator/cmd/proxy/app/flags.go new file mode 100644 index 0000000000..48cdc88b27 --- /dev/null +++ b/coordinator/cmd/proxy/app/flags.go @@ -0,0 +1,30 @@ +package app + +import "github.com/urfave/cli/v2" + +var ( + apiFlags = []cli.Flag{ + // http flags + &httpEnabledFlag, + &httpListenAddrFlag, + &httpPortFlag, + } + // httpEnabledFlag enable rpc server. + httpEnabledFlag = cli.BoolFlag{ + Name: "http", + Usage: "Enable the HTTP-RPC server", + Value: false, + } + // httpListenAddrFlag set the http address. + httpListenAddrFlag = cli.StringFlag{ + Name: "http.addr", + Usage: "HTTP-RPC server listening interface", + Value: "localhost", + } + // httpPortFlag set http.port. + httpPortFlag = cli.IntFlag{ + Name: "http.port", + Usage: "HTTP-RPC server listening port", + Value: 8590, + } +) diff --git a/coordinator/cmd/proxy/main.go b/coordinator/cmd/proxy/main.go new file mode 100644 index 0000000000..10b44d62d6 --- /dev/null +++ b/coordinator/cmd/proxy/main.go @@ -0,0 +1,7 @@ +package main + +import "scroll-tech/coordinator/cmd/proxy/app" + +func main() { + app.Run() +} diff --git a/coordinator/conf/config_proxy.json b/coordinator/conf/config_proxy.json index 886c10bf51..b968a4d28e 100644 --- a/coordinator/conf/config_proxy.json +++ b/coordinator/conf/config_proxy.json @@ -22,7 +22,7 @@ } ] } - }, + }, "coordinators": { "sepolia": { "base_url": "http://localhost:8555", From 6ee026fa16ab17888358ee5e547914fa3aa6075b Mon Sep 17 00:00:00 2001 From: Ho Date: Tue, 7 Oct 2025 11:04:04 +0900 Subject: [PATCH 28/43] depress link for libzkp --- coordinator/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coordinator/Makefile b/coordinator/Makefile index 5749e54a48..8a16a04666 100644 --- a/coordinator/Makefile +++ b/coordinator/Makefile @@ -34,8 +34,8 @@ coordinator_cron: coordinator_tool: go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_tool ./cmd/tool -coordinator_proxy: - go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_proxy ./cmd/proxy +coordinator_proxy: + go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -tags="mock_prover mock_verifier" -o $(PWD)/build/bin/coordinator_proxy ./cmd/proxy localsetup: coordinator_api ## Local setup: build coordinator_api, copy config, and setup releases From 4365aafa9a288c318d6efc18469762efeff1ef27 Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 8 Oct 2025 11:32:13 +0900 Subject: [PATCH 29/43] refactor libzkp to be completely mocked out --- coordinator/internal/logic/libzkp/lib.go | 29 ++-------------- coordinator/internal/logic/libzkp/lib_mock.go | 34 +++++++++++++++++++ .../internal/logic/libzkp/message_types.go | 26 ++++++++++++++ .../logic/libzkp/mock_universal_task.go | 5 +++ .../internal/logic/libzkp/universal_task.go | 10 +++++- 5 files changed, 76 insertions(+), 28 deletions(-) create mode 100644 coordinator/internal/logic/libzkp/lib_mock.go create mode 100644 coordinator/internal/logic/libzkp/message_types.go diff --git a/coordinator/internal/logic/libzkp/lib.go b/coordinator/internal/logic/libzkp/lib.go index 63bbd82a27..f311698ced 100644 --- a/coordinator/internal/logic/libzkp/lib.go +++ b/coordinator/internal/logic/libzkp/lib.go @@ -1,3 +1,5 @@ +//go:build !mock_verifier + package libzkp /* @@ -13,8 +15,6 @@ import ( "os" "strings" "unsafe" - - "scroll-tech/common/types/message" ) func init() { @@ -72,31 +72,6 @@ func VerifyBundleProof(proofData, forkName string) bool { return result != 0 } -// TaskType enum values matching the Rust enum -const ( - TaskTypeChunk = 0 - TaskTypeBatch = 1 - TaskTypeBundle = 2 -) - -func fromMessageTaskType(taskType int) int { - switch message.ProofType(taskType) { - case message.ProofTypeChunk: - return TaskTypeChunk - case message.ProofTypeBatch: - return TaskTypeBatch - case message.ProofTypeBundle: - return TaskTypeBundle - default: - panic(fmt.Sprintf("unsupported proof type: %d", taskType)) - } -} - -// Generate a universal task -func GenerateUniversalTask(taskType int, taskJSON, forkName string, expectedVk []byte) (bool, string, string, []byte) { - return generateUniversalTask(fromMessageTaskType(taskType), taskJSON, strings.ToLower(forkName), expectedVk) -} - // Generate wrapped proof func GenerateWrappedProof(proofJSON, metadata string, vkData []byte) string { cProofJSON := goToCString(proofJSON) diff --git a/coordinator/internal/logic/libzkp/lib_mock.go b/coordinator/internal/logic/libzkp/lib_mock.go new file mode 100644 index 0000000000..b836a6afa6 --- /dev/null +++ b/coordinator/internal/logic/libzkp/lib_mock.go @@ -0,0 +1,34 @@ +//go:build mock_verifier + +package libzkp + +// // InitVerifier is a no-op in the mock. +// func InitVerifier(configJSON string) {} + +// // VerifyChunkProof returns a fixed success in the mock. +// func VerifyChunkProof(proofData, forkName string) bool { +// return true +// } + +// // VerifyBatchProof returns a fixed success in the mock. +// func VerifyBatchProof(proofData, forkName string) bool { +// return true +// } + +// // VerifyBundleProof returns a fixed success in the mock. +// func VerifyBundleProof(proofData, forkName string) bool { +// return true +// } + +// GenerateWrappedProof returns a fixed dummy proof string in the mock. +func GenerateWrappedProof(proofJSON, metadata string, vkData []byte) string { + return "mock-wrapped-proof" +} + +// DumpVk is a no-op and returns nil in the mock. +func DumpVk(forkName, filePath string) error { + return nil +} + +// SetDynamicFeature is a no-op in the mock. +func SetDynamicFeature(feats string) {} diff --git a/coordinator/internal/logic/libzkp/message_types.go b/coordinator/internal/logic/libzkp/message_types.go new file mode 100644 index 0000000000..8ac465e9a6 --- /dev/null +++ b/coordinator/internal/logic/libzkp/message_types.go @@ -0,0 +1,26 @@ +package libzkp + +import ( + "fmt" + "scroll-tech/common/types/message" +) + +// TaskType enum values matching the Rust enum +const ( + TaskTypeChunk = 0 + TaskTypeBatch = 1 + TaskTypeBundle = 2 +) + +func fromMessageTaskType(taskType int) int { + switch message.ProofType(taskType) { + case message.ProofTypeChunk: + return TaskTypeChunk + case message.ProofTypeBatch: + return TaskTypeBatch + case message.ProofTypeBundle: + return TaskTypeBundle + default: + panic(fmt.Sprintf("unsupported proof type: %d", taskType)) + } +} diff --git a/coordinator/internal/logic/libzkp/mock_universal_task.go b/coordinator/internal/logic/libzkp/mock_universal_task.go index b4cee34de6..1cc3a41a34 100644 --- a/coordinator/internal/logic/libzkp/mock_universal_task.go +++ b/coordinator/internal/logic/libzkp/mock_universal_task.go @@ -5,6 +5,7 @@ package libzkp import ( "encoding/json" "fmt" + "strings" "scroll-tech/common/types/message" @@ -14,6 +15,10 @@ import ( func InitL2geth(configJSON string) { } +func GenerateUniversalTask(taskType int, taskJSON, forkName string, expectedVk []byte) (bool, string, string, []byte) { + return generateUniversalTask(fromMessageTaskType(taskType), taskJSON, strings.ToLower(forkName), expectedVk) +} + func generateUniversalTask(taskType int, taskJSON, forkName string, expectedVk []byte) (bool, string, string, []byte) { fmt.Printf("call mocked generate universal task %d, taskJson %s\n", taskType, taskJSON) diff --git a/coordinator/internal/logic/libzkp/universal_task.go b/coordinator/internal/logic/libzkp/universal_task.go index 82292e6b68..e3df30c510 100644 --- a/coordinator/internal/logic/libzkp/universal_task.go +++ b/coordinator/internal/logic/libzkp/universal_task.go @@ -7,7 +7,10 @@ package libzkp #include "libzkp.h" */ import "C" //nolint:typecheck -import "unsafe" +import ( + "strings" + "unsafe" +) // Initialize the handler for universal task func InitL2geth(configJSON string) { @@ -17,6 +20,11 @@ func InitL2geth(configJSON string) { C.init_l2geth(cConfig) } +// Generate a universal task +func GenerateUniversalTask(taskType int, taskJSON, forkName string, expectedVk []byte) (bool, string, string, []byte) { + return generateUniversalTask(fromMessageTaskType(taskType), taskJSON, strings.ToLower(forkName), expectedVk) +} + func generateUniversalTask(taskType int, taskJSON, forkName string, expectedVk []byte) (bool, string, string, []byte) { cTask := goToCString(taskJSON) cForkName := goToCString(forkName) From 8a15836d207c28339567b30c72f9e9508191a313 Mon Sep 17 00:00:00 2001 From: Ho Date: Thu, 9 Oct 2025 14:30:43 +0900 Subject: [PATCH 30/43] add compatibile mode and more logs --- coordinator/conf/config_proxy.json | 11 +-- coordinator/internal/config/proxy_config.go | 1 + coordinator/internal/controller/proxy/auth.go | 1 + .../internal/controller/proxy/client.go | 70 ++++++++++++------- .../controller/proxy/client_manager.go | 31 ++++++-- .../internal/controller/proxy/get_task.go | 10 ++- .../controller/proxy/prover_session.go | 12 ++++ .../internal/controller/proxy/submit_proof.go | 7 +- .../logic/submitproof/proof_receiver.go | 2 +- coordinator/test/proxy_test.go | 13 ++++ 10 files changed, 112 insertions(+), 46 deletions(-) diff --git a/coordinator/conf/config_proxy.json b/coordinator/conf/config_proxy.json index b968a4d28e..dd1139ba6e 100644 --- a/coordinator/conf/config_proxy.json +++ b/coordinator/conf/config_proxy.json @@ -11,16 +11,7 @@ }, "verifier": { "min_prover_version": "v4.4.45", - "verifiers": [ - { - "assets_path": "assets", - "fork_name": "euclidV2" - }, - { - "assets_path": "assets", - "fork_name": "feynman" - } - ] + "verifiers": [] } }, "coordinators": { diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index 1ea8bd295c..640fbdcbfd 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -39,6 +39,7 @@ type UpStream struct { RetryCount uint `json:"retry_count"` RetryWaitTime uint `json:"retry_wait_time_sec"` ConnectionTimeoutSec uint `json:"connection_timeout_sec"` + CompatibileMode bool `json:"compatibile_mode,omitempty"` } // Config load configuration items. diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index a312384d77..d4f188d3ed 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -60,6 +60,7 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { } session := a.proverMgr.GetOrCreate(loginParam.PublicKey) + log.Debug("start handling login", "cli", loginParam.Message.ProverName) for n, cli := range a.clients { diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index e3a6ebb456..8ba9e12716 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -8,6 +8,9 @@ import ( "net/http" "time" + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/crypto" + ctypes "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" @@ -15,9 +18,10 @@ import ( // Client wraps an http client with a preset host for coordinator API calls type upClient struct { - httpClient *http.Client - baseURL string - loginToken string + httpClient *http.Client + baseURL string + loginToken string + compatibileMode bool } // NewClient creates a new Client with the specified host @@ -26,7 +30,8 @@ func newUpClient(cfg *config.UpStream) *upClient { httpClient: &http.Client{ Timeout: time.Duration(cfg.ConnectionTimeoutSec) * time.Second, }, - baseURL: cfg.BaseUrl, + baseURL: cfg.BaseUrl, + compatibileMode: cfg.CompatibileMode, } } @@ -40,8 +45,8 @@ type loginSchema struct { Token string `json:"token"` } -// FullLogin performs the complete login process: get challenge then login -func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.LoginParameter, error)) (*types.LoginSchema, error) { +// Login performs the complete login process: get challenge then login +func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.LoginParameter, error)) (*ctypes.Response, error) { // Step 1: Get challenge url := fmt.Sprintf("%s/coordinator/v1/challenge", c.baseURL) @@ -93,26 +98,7 @@ func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.Logi if err != nil { return nil, fmt.Errorf("failed to perform login request: %w", err) } - - parsedResp, err = handleHttpResp(loginResp) - if err != nil { - return nil, err - } else if parsedResp.ErrCode != 0 { - return nil, fmt.Errorf("login failed: %d (%s)", parsedResp.ErrCode, parsedResp.ErrMsg) - } - - var loginResult loginSchema - err = parsedResp.DecodeData(&loginResult) - if err != nil { - return nil, fmt.Errorf("login parsing data fail: %v", err) - } - c.loginToken = loginResult.Token - - // TODO: we need to parse time if we start making use of it - - return &types.LoginSchema{ - Token: loginResult.Token, - }, nil + return handleHttpResp(loginResp) } func handleHttpResp(resp *http.Response) (*ctypes.Response, error) { @@ -130,8 +116,40 @@ func handleHttpResp(resp *http.Response) (*ctypes.Response, error) { return nil, fmt.Errorf("login request failed with status: %d", resp.StatusCode) } +func (c *upClient) proxyLoginCompatibleMode(ctx context.Context, param *types.LoginParameter) (*ctypes.Response, error) { + mimePrivK, err := buildPrivateKey([]byte(param.PublicKey)) + if err != nil { + return nil, err + } + mimePkHex := common.Bytes2Hex(crypto.CompressPubkey(&mimePrivK.PublicKey)) + + genLoginParam := func(challenge string) (*types.LoginParameter, error) { + + // Create login parameter with proxy settings + loginParam := &types.LoginParameter{ + Message: param.Message, + PublicKey: mimePkHex, + } + loginParam.Message.Challenge = challenge + + // Sign the message with the private key + if err := loginParam.SignWithKey(mimePrivK); err != nil { + return nil, fmt.Errorf("failed to sign login parameter: %w", err) + } + + return loginParam, nil + } + + return c.Login(ctx, genLoginParam) +} + // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter func (c *upClient) ProxyLogin(ctx context.Context, param *types.LoginParameter) (*ctypes.Response, error) { + + if c.compatibileMode { + return c.proxyLoginCompatibleMode(ctx, param) + } + url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index ad170384f9..73f0569812 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -67,7 +67,13 @@ func NewClientManager(name string, cliCfg *config.ProxyClient, cfg *config.UpStr }, nil } -func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) time.Time { +func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) { + if cliMgr.cfg.CompatibileMode { + loginCli.loginToken = "dummy" + log.Info("Skip login process for compatibile mode") + return + } + // Calculate wait time between 2 seconds and cfg.RetryWaitTime minWait := 2 * time.Second waitDuration := time.Duration(cliMgr.cfg.RetryWaitTime) * time.Second @@ -77,18 +83,31 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) ti for { log.Info("attempting login to upstream coordinator", "name", cliMgr.name) - loginResult, err := loginCli.Login(ctx, cliMgr.genLoginParam) - if err == nil && loginResult != nil { - log.Info("login to upstream coordinator successful", "name", cliMgr.name, "time", loginResult.Time) - return loginResult.Time + loginResp, err := loginCli.Login(ctx, cliMgr.genLoginParam) + if err == nil && loginResp.ErrCode == 0 { + var loginResult loginSchema + err = loginResp.DecodeData(&loginResult) + if err != nil { + log.Error("login parsing data fail", "error", err) + } else { + loginCli.loginToken = loginResult.Token + log.Info("login to upstream coordinator successful", "name", cliMgr.name, "time", loginResult.Time) + // TODO: we need to parse time if we start making use of it + return + } + } else if err != nil { + log.Error("login process fail", "error", err) + } else { + log.Error("login get fail resp", "code", loginResp.ErrCode, "msg", loginResp.ErrMsg) } + log.Info("login to upstream coordinator failed, retrying", "name", cliMgr.name, "error", err, "waitDuration", waitDuration) timer := time.NewTimer(waitDuration) select { case <-ctx.Done(): timer.Stop() - return time.Now() + return case <-timer.C: // Continue to next retry } diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index 8261ff0f16..a7539574d0 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -108,13 +108,16 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { session := ptc.proverMgr.Get(publicKey) getTask := func(upStream string, cli Client) (tryNext bool) { + log.Debug("Start get task", "up", upStream, "cli", session.CliName) resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) if err != nil { + log.Error("Upstream error for get task", "error", err, "up", upStream, "cli", session.CliName) types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return } else if resp.ErrCode != types.ErrCoordinatorEmptyProofData { if resp.ErrCode != 0 { + log.Error("Upstream has error resp for get task", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upStream, "cli", session.CliName) // simply dispatch the error from upstream to prover types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) return @@ -124,9 +127,10 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { if err = resp.DecodeData(&task); err == nil { task.TaskID = formUpstreamWithTaskName(upStream, task.TaskID) ptc.priorityUpstream.Set(publicKey, upStream) - // TODO: log the new id in debug level + log.Debug("Upstream get task", "up", upStream, "cli", session.CliName, "taskID", task.TaskID, "taskType", task.TaskType) types.RenderSuccess(ctx, &task) } else { + log.Error("Upstream has wrong data for get task", "error", err, "up", upStream, "cli", session.CliName) types.RenderFailure(ctx, types.InternalServerError, fmt.Errorf("decode task fail: %v", err)) } @@ -140,10 +144,11 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { priorityUpstream, exist := ptc.priorityUpstream.Get(publicKey) if exist { cli := ptc.clients[priorityUpstream] + log.Debug("Try get task from priority stream", "up", priorityUpstream) if cli != nil && !getTask(priorityUpstream, cli) { return } else if cli == nil { - // TODO: log error + log.Warn("A upstream is removed or lost for some reason while running", "up", priorityUpstream) } } ptc.priorityUpstream.Delete(publicKey) @@ -166,6 +171,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { } } + log.Debug("get no task from upstream", "cli", session.CliName) // if all get task failed, throw empty proof resp types.RenderFailure(ctx, types.ErrCoordinatorEmptyProofData, fmt.Errorf("get empty prover task")) } diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index eb41fbe3d4..6651db7436 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/scroll-tech/go-ethereum/log" + ctypes "scroll-tech/common/types" "scroll-tech/coordinator/internal/types" ) @@ -40,6 +42,7 @@ func (m *ProverManager) GetOrCreate(userKey string) *proverSession { ret := &proverSession{ proverToken: make(map[string]loginToken), + CliName: "pending for login", } m.data[userKey] = ret @@ -60,6 +63,8 @@ type loginToken struct { // Client wraps an http client with a preset host for coordinator API calls type proverSession struct { + CliName string + sync.RWMutex proverToken map[string]loginToken completionCtx context.Context @@ -97,11 +102,18 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str LoginSchema: result, phase: curPhase + 1, } + log.Info("maintain login status", "upstream", up, "cli", param.Message.ProverName, "phase", curPhase+1) } c.Unlock() + if nerr != nil { + log.Error("maintain login fail", "error", nerr, "upstream", up, "cli", param.Message.ProverName, "phase", curPhase) + } }() c.Unlock() + log.Debug("start proxy login process", "upstream", up, "cli", param.Message.ProverName) + c.CliName = param.Message.ProverName + cli := cliMgr.Client(ctx) if cli == nil { return nil, fmt.Errorf("get upstream cli fail") diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index 90c582620b..b8e7f775a7 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" + "github.com/scroll-tech/go-ethereum/log" "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" @@ -59,22 +60,26 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { upstream, realTaskID := upstreamFromTaskName(submitParameter.TaskID) cli, existed := spc.clients[upstream] if !existed { - // TODO: log error + log.Warn("A upstream for submitting is removed or lost for some reason while running", "up", upstream) nerr := fmt.Errorf("Invalid upstream name (%s) from taskID %s", upstream, submitParameter.TaskID) types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) return } + log.Debug("Start submitting", "up", upstream, "cli", session.CliName, "id", realTaskID, "status", submitParameter.Status) submitParameter.TaskID = realTaskID resp, err := session.SubmitProof(ctx, &submitParameter, cli, upstream) if err != nil { + log.Error("Upstream has error resp for submit", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upstream, "cli", session.CliName) types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return } else if resp.ErrCode != 0 { + log.Error("Upstream has error resp for get task", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upstream, "cli", session.CliName) // simply dispatch the error from upstream to prover types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) return } else { + log.Debug("Submit proof to upstream", "up", upstream, "cli", session.CliName, "taskID", realTaskID) spc.priorityUpstream.Delete(upstream) types.RenderSuccess(ctx, resp.Data) return diff --git a/coordinator/internal/logic/submitproof/proof_receiver.go b/coordinator/internal/logic/submitproof/proof_receiver.go index 3d1693affa..84d062690b 100644 --- a/coordinator/internal/logic/submitproof/proof_receiver.go +++ b/coordinator/internal/logic/submitproof/proof_receiver.go @@ -216,7 +216,7 @@ func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofParameter coor switch message.ProofType(proofParameter.TaskType) { case message.ProofTypeChunk: chunkProof := &message.OpenVMChunkProof{} - if unmarshalErr := json.Unmarshal([]byte(proofParameter.Proof), &chunkProof); unmarshalErr != nil { + if unmarshalErr := json.Unmarshal([]byte(proofParameter.Proof), &chunkProof); unmarshalErr == nil { return unmarshalErr } success, verifyErr = m.verifier.VerifyChunkProof(chunkProof, hardForkName) diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index b8a09afbac..53654b9ee0 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -31,12 +31,15 @@ func testProxyClientCfg() *config.ProxyClient { } } +var testCompatibileMode bool + func testProxyUpStreamCfg(coordinatorURL string) *config.UpStream { return &config.UpStream{ BaseUrl: fmt.Sprintf("http://%s", coordinatorURL), RetryWaitTime: 3, ConnectionTimeoutSec: 30, + CompatibileMode: testCompatibileMode, } } @@ -263,7 +266,17 @@ func testProxyProof(t *testing.T) { } func TestProxyClient(t *testing.T) { + testCompatibileMode = false + // Set up the test environment. + setEnv(t) + t.Run("TestProxyClient", testProxyClient) + t.Run("TestProxyHandshake", testProxyHandshake) + t.Run("TestProxyGetTask", testProxyGetTask) + t.Run("TestProxyValidProof", testProxyProof) +} +func TestProxyClientCompatibleMode(t *testing.T) { + testCompatibileMode = true // Set up the test environment. setEnv(t) t.Run("TestProxyClient", testProxyClient) From 404c664e10c8aee0b1e25f7ad2fa261b3a1e4676 Mon Sep 17 00:00:00 2001 From: Ho Date: Fri, 10 Oct 2025 15:33:55 +0900 Subject: [PATCH 31/43] fix unittest --- .../controller/proxy/prover_session.go | 2 +- coordinator/internal/logic/libzkp/lib_mock.go | 21 ++++++++++++++++++- .../logic/submitproof/proof_receiver.go | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 6651db7436..c45c11a1ab 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -203,7 +203,7 @@ func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParamet newToken, err := c.maintainLogin(ctx, cliMgr, up, loginParam, token.phase) if err != nil { - return nil, fmt.Errorf("update prover token fail: %V", err) + return nil, fmt.Errorf("update prover token fail: %v", err) } return cli.GetTask(ctx, param, newToken.Token) diff --git a/coordinator/internal/logic/libzkp/lib_mock.go b/coordinator/internal/logic/libzkp/lib_mock.go index b836a6afa6..2f5445faa4 100644 --- a/coordinator/internal/logic/libzkp/lib_mock.go +++ b/coordinator/internal/logic/libzkp/lib_mock.go @@ -2,6 +2,10 @@ package libzkp +import ( + "encoding/json" +) + // // InitVerifier is a no-op in the mock. // func InitVerifier(configJSON string) {} @@ -22,7 +26,22 @@ package libzkp // GenerateWrappedProof returns a fixed dummy proof string in the mock. func GenerateWrappedProof(proofJSON, metadata string, vkData []byte) string { - return "mock-wrapped-proof" + + payload := struct { + Metadata json.RawMessage `json:"metadata"` + Proof json.RawMessage `json:"proof"` + GitVersion string `json:"git_version"` + }{ + Metadata: json.RawMessage(metadata), + Proof: json.RawMessage(proofJSON), + GitVersion: "mock-git-version", + } + + out, err := json.Marshal(payload) + if err != nil { + panic(err) + } + return string(out) } // DumpVk is a no-op and returns nil in the mock. diff --git a/coordinator/internal/logic/submitproof/proof_receiver.go b/coordinator/internal/logic/submitproof/proof_receiver.go index 84d062690b..3d1693affa 100644 --- a/coordinator/internal/logic/submitproof/proof_receiver.go +++ b/coordinator/internal/logic/submitproof/proof_receiver.go @@ -216,7 +216,7 @@ func (m *ProofReceiverLogic) HandleZkProof(ctx *gin.Context, proofParameter coor switch message.ProofType(proofParameter.TaskType) { case message.ProofTypeChunk: chunkProof := &message.OpenVMChunkProof{} - if unmarshalErr := json.Unmarshal([]byte(proofParameter.Proof), &chunkProof); unmarshalErr == nil { + if unmarshalErr := json.Unmarshal([]byte(proofParameter.Proof), &chunkProof); unmarshalErr != nil { return unmarshalErr } success, verifyErr = m.verifier.VerifyChunkProof(chunkProof, hardForkName) From d9a29cddcef4ae7cf53acc105302f408528fef52 Mon Sep 17 00:00:00 2001 From: Ho Date: Fri, 17 Oct 2025 22:26:29 +0900 Subject: [PATCH 32/43] fix config issue --- coordinator/internal/config/proxy_config.go | 2 +- coordinator/internal/controller/proxy/prover_session.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index 640fbdcbfd..44e6ac1e16 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -46,7 +46,7 @@ type UpStream struct { type ProxyConfig struct { ProxyManager *ProxyManager `json:"proxy_manager"` ProxyName string `json:"proxy_name"` - Coordinators map[string]*UpStream `json:"coondiators"` + Coordinators map[string]*UpStream `json:"coordinators"` } // NewConfig returns a new instance of Config. diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index c45c11a1ab..ddc1c8f568 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -86,6 +86,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str if phase < curPhase { // outdate login phase, give up + log.Debug("drop outdated proxy login attemp", "upstream", up, "cli", param.Message.ProverName, "phase", phase, "now", curPhase) defer c.Unlock() return c.proverToken[up].LoginSchema, nil } @@ -147,6 +148,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str return nil, err } + log.Debug("Proxy login done", "upstream", up, "cli", param.Message.ProverName) return &types.LoginSchema{ Token: loginResult.Token, }, nil From b1c3a4ecc000eabe4b9ab986dc74e46b571c6057 Mon Sep 17 00:00:00 2001 From: Ho Date: Fri, 17 Oct 2025 22:27:51 +0900 Subject: [PATCH 33/43] more log for init --- coordinator/internal/controller/proxy/client_manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 73f0569812..bf6530738a 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -54,6 +54,7 @@ func buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { func NewClientManager(name string, cliCfg *config.ProxyClient, cfg *config.UpStream) (*ClientManager, error) { + log.Info("init client", "name", name, "upcfg", cfg.BaseUrl, "compatible mode", cfg.CompatibileMode) privKey, err := buildPrivateKey([]byte(cliCfg.Secret)) if err != nil { return nil, err From 5d41788b07d7268bafa955d250d2696fcfa705d3 Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 20 Oct 2025 14:42:05 +0900 Subject: [PATCH 34/43] + fix get task behavior + improve the robust of tests --- .../internal/controller/proxy/get_task.go | 33 ++++++++++++------- coordinator/test/api_test.go | 17 ++++++---- coordinator/test/proxy_test.go | 1 + 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index a7539574d0..3746d3f085 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -107,20 +107,20 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { session := ptc.proverMgr.Get(publicKey) - getTask := func(upStream string, cli Client) (tryNext bool) { + getTask := func(upStream string, cli Client) (error, int) { log.Debug("Start get task", "up", upStream, "cli", session.CliName) resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) if err != nil { log.Error("Upstream error for get task", "error", err, "up", upStream, "cli", session.CliName) types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) - return + return err, types.ErrCoordinatorGetTaskFailure } else if resp.ErrCode != types.ErrCoordinatorEmptyProofData { if resp.ErrCode != 0 { log.Error("Upstream has error resp for get task", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upStream, "cli", session.CliName) // simply dispatch the error from upstream to prover types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) - return + return fmt.Errorf("upstream failure %s:", resp.ErrMsg), resp.ErrCode } var task coordinatorType.GetTaskSchema @@ -129,15 +129,15 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { ptc.priorityUpstream.Set(publicKey, upStream) log.Debug("Upstream get task", "up", upStream, "cli", session.CliName, "taskID", task.TaskID, "taskType", task.TaskType) types.RenderSuccess(ctx, &task) + return nil, 0 } else { log.Error("Upstream has wrong data for get task", "error", err, "up", upStream, "cli", session.CliName) types.RenderFailure(ctx, types.InternalServerError, fmt.Errorf("decode task fail: %v", err)) + return fmt.Errorf("decode task fail: %v", err), types.InternalServerError } - - return } - tryNext = true - return + + return nil, resp.ErrCode } // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp @@ -145,9 +145,19 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { if exist { cli := ptc.clients[priorityUpstream] log.Debug("Try get task from priority stream", "up", priorityUpstream) - if cli != nil && !getTask(priorityUpstream, cli) { - return - } else if cli == nil { + if cli != nil { + err, code := getTask(priorityUpstream, cli) + if err != nil { + types.RenderFailure(ctx, code, err) + return + } else if code == 0 { + // get task done and rendered, return + return + } + // only continue if get empty task (the task has been removed in upstream) + log.Debug("can not get priority task from upstream", "up", priorityUpstream) + + } else { log.Warn("A upstream is removed or lost for some reason while running", "up", priorityUpstream) } } @@ -166,7 +176,8 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { // Iterate over the shuffled keys for _, n := range keys { - if !getTask(n, ptc.clients[n]) { + if err, code := getTask(n, ptc.clients[n]); err == nil && code == 0 { + // get task done return } } diff --git a/coordinator/test/api_test.go b/coordinator/test/api_test.go index f435dd16b6..139ddfc9a2 100644 --- a/coordinator/test/api_test.go +++ b/coordinator/test/api_test.go @@ -52,7 +52,8 @@ var ( batch *encoding.Batch tokenTimeout int - envSet bool + envSet bool + portUsed map[int64]struct{} ) func TestMain(m *testing.M) { @@ -65,8 +66,7 @@ func TestMain(m *testing.M) { } func randomURL() string { - id, _ := rand.Int(rand.Reader, big.NewInt(2000-1)) - return fmt.Sprintf("localhost:%d", 10000+2000+id.Int64()) + return randmURLBatch(1)[0] } // Generate a batch of random localhost URLs with different ports, similar to randomURL. @@ -75,14 +75,16 @@ func randmURLBatch(n int) []string { return nil } urls := make([]string, 0, n) - used := make(map[int64]struct{}, n) + if portUsed == nil { + portUsed = make(map[int64]struct{}) + } for len(urls) < n { id, _ := rand.Int(rand.Reader, big.NewInt(2000-1)) - port := 10000 + 2000 + id.Int64() - if _, ok := used[port]; ok { + port := 20000 + 2000 + id.Int64() + if _, exist := portUsed[port]; exist { continue } - used[port] = struct{}{} + portUsed[port] = struct{}{} urls = append(urls, fmt.Sprintf("localhost:%d", port)) } return urls @@ -135,6 +137,7 @@ func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL stri EuclidV2Time: new(uint64), }, db, nil) route.Route(router, conf, nil) + t.Log("coordinator server url", coordinatorURL) srv := &http.Server{ Addr: coordinatorURL, Handler: router, diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index 53654b9ee0..21e2a768ab 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -113,6 +113,7 @@ func setupProxy(t *testing.T, proxyURL string, coordinatorURL []string) *http.Se router := gin.New() proxy.InitController(proxyConf, nil) route.ProxyRoute(router, proxyConf, nil) + t.Log("proxy server url", proxyURL) srv := &http.Server{ Addr: proxyURL, Handler: router, From 7572bf8923e5413c111df6c3c143acfe0819e1ea Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 20 Oct 2025 15:21:13 +0900 Subject: [PATCH 35/43] fix --- coordinator/internal/controller/proxy/get_task.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index 3746d3f085..b2d6941102 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -112,14 +112,12 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) if err != nil { log.Error("Upstream error for get task", "error", err, "up", upStream, "cli", session.CliName) - types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return err, types.ErrCoordinatorGetTaskFailure } else if resp.ErrCode != types.ErrCoordinatorEmptyProofData { if resp.ErrCode != 0 { - log.Error("Upstream has error resp for get task", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upStream, "cli", session.CliName) // simply dispatch the error from upstream to prover - types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) + log.Error("Upstream has error resp for get task", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upStream, "cli", session.CliName) return fmt.Errorf("upstream failure %s:", resp.ErrMsg), resp.ErrCode } @@ -132,7 +130,6 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { return nil, 0 } else { log.Error("Upstream has wrong data for get task", "error", err, "up", upStream, "cli", session.CliName) - types.RenderFailure(ctx, types.InternalServerError, fmt.Errorf("decode task fail: %v", err)) return fmt.Errorf("decode task fail: %v", err), types.InternalServerError } } From b6e33456fa2cc1f86bcb24c415f5036a581a278e Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 20 Oct 2025 22:02:48 +0900 Subject: [PATCH 36/43] fix issue --- coordinator/internal/controller/proxy/get_task.go | 6 +++--- coordinator/internal/controller/proxy/submit_proof.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index b2d6941102..ea523737f2 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -141,7 +141,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { priorityUpstream, exist := ptc.priorityUpstream.Get(publicKey) if exist { cli := ptc.clients[priorityUpstream] - log.Debug("Try get task from priority stream", "up", priorityUpstream) + log.Debug("Try get task from priority stream", "up", priorityUpstream, "cli", session.CliName) if cli != nil { err, code := getTask(priorityUpstream, cli) if err != nil { @@ -152,10 +152,10 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { return } // only continue if get empty task (the task has been removed in upstream) - log.Debug("can not get priority task from upstream", "up", priorityUpstream) + log.Debug("can not get priority task from upstream", "up", priorityUpstream, "cli", session.CliName) } else { - log.Warn("A upstream is removed or lost for some reason while running", "up", priorityUpstream) + log.Warn("A upstream is removed or lost for some reason while running", "up", priorityUpstream, "cli", session.CliName) } } ptc.priorityUpstream.Delete(publicKey) diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index b8e7f775a7..5110eb741d 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -80,7 +80,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { return } else { log.Debug("Submit proof to upstream", "up", upstream, "cli", session.CliName, "taskID", realTaskID) - spc.priorityUpstream.Delete(upstream) + spc.priorityUpstream.Delete(publicKey) types.RenderSuccess(ctx, resp.Data) return } From 17e6c5b7ac3ed9c47647bec9bb9e4101e28e196f Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 20 Oct 2025 22:24:04 +0900 Subject: [PATCH 37/43] robust prover manager --- .../internal/controller/proxy/controller.go | 2 +- .../controller/proxy/prover_session.go | 44 ++++--- .../controller/proxy/prover_session_test.go | 112 ++++++++++++++++++ 3 files changed, 142 insertions(+), 16 deletions(-) create mode 100644 coordinator/internal/controller/proxy/prover_session_test.go diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 0e1b217a3f..2c408cb743 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -34,7 +34,7 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { clients[nm] = cli } - proverManager := NewProverManager() + proverManager := NewProverManager(100) priorityManager := NewPriorityUpstreamManager() Auth = NewAuthController(cfg, clients, proverManager) diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index ddc1c8f568..ea2f8b5a6a 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -15,47 +15,61 @@ import ( type ProverManager struct { sync.RWMutex - data map[string]*proverSession + data map[string]*proverSession + willDeprecatedData map[string]*proverSession + sizeLimit int } -func NewProverManager() *ProverManager { +func NewProverManager(size int) *ProverManager { return &ProverManager{ - data: make(map[string]*proverSession), + data: make(map[string]*proverSession), + willDeprecatedData: make(map[string]*proverSession), + sizeLimit: size, } } // get retrieves ProverSession for a given user key, returns empty if still not exists func (m *ProverManager) Get(userKey string) *proverSession { m.RLock() - defer m.RUnlock() - return m.data[userKey] + if r, existed := m.data[userKey]; existed { + m.RUnlock() + return r + } else { + r, existed = m.willDeprecatedData[userKey] + m.RUnlock() + if existed { + m.Lock() + m.data[userKey] = r + m.Unlock() + } + return r + } } func (m *ProverManager) GetOrCreate(userKey string) *proverSession { - m.Lock() - defer m.Unlock() - if ret, ok := m.data[userKey]; ok { + if ret := m.Get(userKey); ret != nil { return ret } + m.Lock() + defer m.Unlock() + ret := &proverSession{ proverToken: make(map[string]loginToken), CliName: "pending for login", } + if len(m.data) >= m.sizeLimit { + m.willDeprecatedData = m.data + m.data = make(map[string]*proverSession) + } + m.data[userKey] = ret return ret } -func (m *ProverManager) Set(userKey string, session *proverSession) { - m.Lock() - defer m.Unlock() - - m.data[userKey] = session -} - type loginToken struct { *types.LoginSchema phase uint diff --git a/coordinator/internal/controller/proxy/prover_session_test.go b/coordinator/internal/controller/proxy/prover_session_test.go new file mode 100644 index 0000000000..51ce094966 --- /dev/null +++ b/coordinator/internal/controller/proxy/prover_session_test.go @@ -0,0 +1,112 @@ +package proxy + +import ( + "testing" +) + +// TestProverManagerGetAndCreate validates basic creation and retrieval semantics. +func TestProverManagerGetAndCreate(t *testing.T) { + pm := NewProverManager(2) + + if got := pm.Get("user1"); got != nil { + t.Fatalf("expected nil for non-existent key, got: %+v", got) + } + + sess1 := pm.GetOrCreate("user1") + if sess1 == nil { + t.Fatalf("expected non-nil session from GetOrCreate") + } + + // Should be stable on subsequent Get + if got := pm.Get("user1"); got != sess1 { + t.Fatalf("expected same session pointer on Get, got different instance: %p vs %p", got, sess1) + } + + // Sanity check default value set during creation + if sess1.CliName != "pending for login" { + t.Fatalf("expected default CliName 'pending for login', got %q", sess1.CliName) + } +} + +// TestProverManagerRolloverAndPromotion verifies rollover when sizeLimit is reached +// and that old entries are accessible and promoted back to active data map. +func TestProverManagerRolloverAndPromotion(t *testing.T) { + pm := NewProverManager(2) + + s1 := pm.GetOrCreate("u1") + s2 := pm.GetOrCreate("u2") + if s1 == nil || s2 == nil { + t.Fatalf("expected sessions to be created for u1/u2") + } + + // Precondition: data should contain 2 entries, no deprecated yet. + pm.RLock() + if len(pm.data) != 2 { + pm.RUnlock() + t.Fatalf("expected data len=2 before rollover, got %d", len(pm.data)) + } + if len(pm.willDeprecatedData) != 0 { + pm.RUnlock() + t.Fatalf("expected willDeprecatedData len=0 before rollover, got %d", len(pm.willDeprecatedData)) + } + pm.RUnlock() + + // Trigger rollover by creating a third key. + s3 := pm.GetOrCreate("u3") + if s3 == nil { + t.Fatalf("expected session for u3 after rollover") + } + + // After rollover: current data should only have u3, deprecated should hold u1 and u2. + pm.RLock() + if len(pm.data) != 1 { + pm.RUnlock() + t.Fatalf("expected data len=1 after rollover (only u3), got %d", len(pm.data)) + } + if _, ok := pm.data["u3"]; !ok { + pm.RUnlock() + t.Fatalf("expected 'u3' to be in active data after rollover") + } + if len(pm.willDeprecatedData) != 2 { + pm.RUnlock() + t.Fatalf("expected willDeprecatedData len=2 after rollover, got %d", len(pm.willDeprecatedData)) + } + pm.RUnlock() + + // Accessing an old key should return the same pointer and promote it to active data map. + got1 := pm.Get("u1") + if got1 != s1 { + t.Fatalf("expected same pointer for u1 after promotion, got %p want %p", got1, s1) + } + + // The promotion should add it to active data (without enforcing size limit on promotion). + pm.RLock() + if _, ok := pm.data["u1"]; !ok { + pm.RUnlock() + t.Fatalf("expected 'u1' to be present in active data after promotion") + } + if len(pm.data) != 2 { + // Now should contain u3 and u1 + pm.RUnlock() + t.Fatalf("expected data len=2 after promotion of u1, got %d", len(pm.data)) + } + pm.RUnlock() + + // Access the other deprecated key and ensure behavior is consistent. + got2 := pm.Get("u2") + if got2 != s2 { + t.Fatalf("expected same pointer for u2 after promotion, got %p want %p", got2, s2) + } + + pm.RLock() + if _, ok := pm.data["u2"]; !ok { + pm.RUnlock() + t.Fatalf("expected 'u2' to be present in active data after promotion") + } + // Note: promotion does not enforce sizeLimit, so data can grow beyond sizeLimit after promotions. + if len(pm.data) != 3 { + pm.RUnlock() + t.Fatalf("expected data len=3 after promoting both u1 and u2, got %d", len(pm.data)) + } + pm.RUnlock() +} From ac0396db3cdbf8e43dcb2a691bee6cdbae9f6c59 Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 22 Oct 2025 08:31:55 +0900 Subject: [PATCH 38/43] add persistent for running status --- coordinator/cmd/proxy/app/app.go | 23 +++- coordinator/internal/config/proxy_config.go | 8 +- coordinator/internal/controller/proxy/auth.go | 2 +- .../internal/controller/proxy/controller.go | 5 +- .../proxy/migrate/0001_running_tables.sql | 32 +++++ .../controller/proxy/migrate/config.json | 4 + .../internal/controller/proxy/persistent.go | 120 ++++++++++++++++++ .../controller/proxy/prover_session.go | 106 ++++++++++------ .../controller/proxy/prover_session_test.go | 12 +- coordinator/test/proxy_test.go | 2 +- 10 files changed, 259 insertions(+), 55 deletions(-) create mode 100644 coordinator/internal/controller/proxy/migrate/0001_running_tables.sql create mode 100644 coordinator/internal/controller/proxy/migrate/config.json create mode 100644 coordinator/internal/controller/proxy/persistent.go diff --git a/coordinator/cmd/proxy/app/app.go b/coordinator/cmd/proxy/app/app.go index 97a331625e..f4fd88d52b 100644 --- a/coordinator/cmd/proxy/app/app.go +++ b/coordinator/cmd/proxy/app/app.go @@ -13,7 +13,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/scroll-tech/go-ethereum/log" "github.com/urfave/cli/v2" + "gorm.io/gorm" + "scroll-tech/common/database" + "scroll-tech/common/observability" "scroll-tech/common/utils" "scroll-tech/common/version" @@ -47,10 +50,22 @@ func action(ctx *cli.Context) error { log.Crit("failed to load config file", "config file", cfgFile, "error", err) } - //observability.Server(ctx, db) + var db *gorm.DB + if dbCfg := cfg.ProxyManager.DB; dbCfg != nil { + db, err = database.InitDB(cfg.ProxyManager.DB) + if err != nil { + log.Crit("failed to init db connection", "err", err) + } + defer func() { + if err = database.CloseDB(db); err != nil { + log.Error("can not close db connection", "error", err) + } + }() + observability.Server(ctx, db) + } registry := prometheus.DefaultRegisterer - apiSrv := server(ctx, cfg, registry) + apiSrv := server(ctx, cfg, db, registry) log.Info( "Start coordinator api successfully.", @@ -77,9 +92,9 @@ func action(ctx *cli.Context) error { return nil } -func server(ctx *cli.Context, cfg *config.ProxyConfig, reg prometheus.Registerer) *http.Server { +func server(ctx *cli.Context, cfg *config.ProxyConfig, db *gorm.DB, reg prometheus.Registerer) *http.Server { router := gin.New() - proxy.InitController(cfg, reg) + proxy.InitController(cfg, db, reg) route.ProxyRoute(router, cfg, reg) port := ctx.String(httpPortFlag.Name) srv := &http.Server{ diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index 44e6ac1e16..2548a790a3 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -5,15 +5,17 @@ import ( "os" "path/filepath" + "scroll-tech/common/database" "scroll-tech/common/utils" ) // Proxy loads proxy configuration items. type ProxyManager struct { // Zk verifier config help to confine the connected prover. - Verifier *VerifierConfig `json:"verifier"` - Client *ProxyClient `json:"proxy_cli"` - Auth *Auth `json:"auth"` + Verifier *VerifierConfig `json:"verifier"` + Client *ProxyClient `json:"proxy_cli"` + Auth *Auth `json:"auth"` + DB *database.Config `json:"db,omitempty"` } func (m *ProxyManager) Normalize() { diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index d4f188d3ed..3a0e862c23 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -59,7 +59,7 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { return nil, fmt.Errorf("proxy do not support recursive login") } - session := a.proverMgr.GetOrCreate(loginParam.PublicKey) + session := a.proverMgr.GetOrCreate(loginParam.PublicKey, loginParam.Message.ProverName) log.Debug("start handling login", "cli", loginParam.Message.ProverName) for n, cli := range a.clients { diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 2c408cb743..67ff185ea2 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -2,6 +2,7 @@ package proxy import ( "github.com/prometheus/client_golang/prometheus" + "gorm.io/gorm" "scroll-tech/coordinator/internal/config" ) @@ -20,7 +21,7 @@ var ( type Clients map[string]Client // InitController inits Controller with database -func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { +func InitController(cfg *config.ProxyConfig, db *gorm.DB, reg prometheus.Registerer) { // normalize cfg cfg.ProxyManager.Normalize() @@ -34,7 +35,7 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { clients[nm] = cli } - proverManager := NewProverManager(100) + proverManager := NewProverManagerWithPersistent(100, db) priorityManager := NewPriorityUpstreamManager() Auth = NewAuthController(cfg, clients, proverManager) diff --git a/coordinator/internal/controller/proxy/migrate/0001_running_tables.sql b/coordinator/internal/controller/proxy/migrate/0001_running_tables.sql new file mode 100644 index 0000000000..1539d35406 --- /dev/null +++ b/coordinator/internal/controller/proxy/migrate/0001_running_tables.sql @@ -0,0 +1,32 @@ +-- +goose Up +-- +goose StatementBegin +create table prover_sessions +( + public_key TEXT NOT NULL, + upstream TEXT NOT NULL, + up_token TEXT NOT NULL, + expired TIMESTAMP(0) NOT NULL +); + +create unique index idx_prover_sessions_public_key on prover_sessions (public_key); +create index idx_prover_sessions_expired on prover_sessions (expired); + +create table priority_upstream +( + public_key TEXT NOT NULL, + upstream TEXT NOT NULL, + update_time TIMESTAMP(0) NOT NULL DEFAULT now() +); + +create unique index idx_priority_upstream_public_key on priority_upstream (public_key); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +drop index if exists idx_prover_sessions_public_key; +drop index if exists idx_prover_sessions_expired; +drop index if exists idx_priority_upstream_public_key; + +drop table if exists prover_sessions; +drop table if exists priority_upstream; +-- +goose StatementEnd \ No newline at end of file diff --git a/coordinator/internal/controller/proxy/migrate/config.json b/coordinator/internal/controller/proxy/migrate/config.json new file mode 100644 index 0000000000..e4fcf4ebe0 --- /dev/null +++ b/coordinator/internal/controller/proxy/migrate/config.json @@ -0,0 +1,4 @@ +{ + "dsn": "postgres://localhost/proxy_run?sslmode=disable", + "driver_name": "postgres" +} \ No newline at end of file diff --git a/coordinator/internal/controller/proxy/persistent.go b/coordinator/internal/controller/proxy/persistent.go new file mode 100644 index 0000000000..6d7ad62f99 --- /dev/null +++ b/coordinator/internal/controller/proxy/persistent.go @@ -0,0 +1,120 @@ +package proxy + +import ( + "time" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "scroll-tech/coordinator/internal/types" +) + +type proverDataPersist struct { + db *gorm.DB +} + +// NewProverDataPersist creates a persistence instance backed by a gorm DB. +func NewProverDataPersist(db *gorm.DB) *proverDataPersist { + return &proverDataPersist{db: db} +} + +// gorm model mapping to table `prover_sessions` +type proverSessionRecord struct { + PublicKey string `gorm:"column:public_key;not null"` + Upstream string `gorm:"column:upstream;not null"` + UpToken string `gorm:"column:up_token;not null"` + Expired time.Time `gorm:"column:expired;not null"` +} + +func (proverSessionRecord) TableName() string { return "prover_sessions" } + +// priority_upstream model +type priorityUpstreamRecord struct { + PublicKey string `gorm:"column:public_key;not null"` + Upstream string `gorm:"column:upstream;not null"` +} + +func (priorityUpstreamRecord) TableName() string { return "priority_upstream" } + +// get retrieves ProverSession for a given user key, returns empty if still not exists +func (p *proverDataPersist) Get(userKey string) (*proverSession, error) { + if p == nil || p.db == nil { + return nil, nil + } + + var rows []proverSessionRecord + if err := p.db.Where("public_key = ?", userKey).Find(&rows).Error; err != nil || len(rows) == 0 { + return nil, err + } + + ret := &proverSession{ + proverToken: make(map[string]loginToken), + } + for _, r := range rows { + ls := &types.LoginSchema{ + Token: r.UpToken, + Time: r.Expired, + } + ret.proverToken[r.Upstream] = loginToken{LoginSchema: ls} + } + return ret, nil +} + +func (p *proverDataPersist) Update(userKey, up string, login *types.LoginSchema) error { + if p == nil || p.db == nil || login == nil { + return nil + } + + rec := proverSessionRecord{ + PublicKey: userKey, + Upstream: up, + UpToken: login.Token, + Expired: login.Time, + } + + return p.db.Clauses( + clause.OnConflict{ + Columns: []clause.Column{{Name: "public_key"}, {Name: "upstream"}}, // fixed typo + DoUpdates: clause.AssignmentColumns([]string{"up_token", "expired"}), + }, + ).Create(&rec).Error +} + +type proverPriorityPersist struct { + db *gorm.DB +} + +func NewProverPriorityPersist(db *gorm.DB) *proverPriorityPersist { + return &proverPriorityPersist{db: db} +} + +func (p *proverPriorityPersist) Get(userKey string) string { + if p == nil || p.db == nil { + return "" + } + var rec priorityUpstreamRecord + if err := p.db.Where("public_key = ?", userKey).First(&rec).Error; err != nil { + return "" + } + return rec.Upstream +} + +func (p *proverPriorityPersist) Update(userKey, up string) error { + if p == nil || p.db == nil { + return nil + } + rec := priorityUpstreamRecord{PublicKey: userKey, Upstream: up} + return p.db.Clauses( + clause.OnConflict{ + Columns: []clause.Column{{Name: "public_key"}}, + DoUpdates: clause.Assignments(map[string]interface{}{"upstream": up}), + }, + ).Create(&rec).Error +} + +func (p *proverPriorityPersist) Del(userKey string) error { + if p == nil || p.db == nil { + return nil + } + return p.db.Where("public_key = ?", userKey).Delete(&priorityUpstreamRecord{}).Error +} diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index ea2f8b5a6a..81eaa531fe 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "gorm.io/gorm" + "github.com/scroll-tech/go-ethereum/log" ctypes "scroll-tech/common/types" @@ -18,6 +20,7 @@ type ProverManager struct { data map[string]*proverSession willDeprecatedData map[string]*proverSession sizeLimit int + persistent *proverDataPersist } func NewProverManager(size int) *ProverManager { @@ -28,26 +31,46 @@ func NewProverManager(size int) *ProverManager { } } +func NewProverManagerWithPersistent(size int, db *gorm.DB) *ProverManager { + return &ProverManager{ + data: make(map[string]*proverSession), + willDeprecatedData: make(map[string]*proverSession), + sizeLimit: size, + persistent: NewProverDataPersist(db), + } +} + // get retrieves ProverSession for a given user key, returns empty if still not exists -func (m *ProverManager) Get(userKey string) *proverSession { - m.RLock() +func (m *ProverManager) Get(userKey string) (ret *proverSession) { + defer func() { + r := ret + if r == nil { + var err error + r, err = m.persistent.Get(userKey) + if err != nil { + log.Error("Get persistent layer for prover tokens fail", "error", err) + } else if r != nil { + r.persistent = m.persistent + } + } - if r, existed := m.data[userKey]; existed { - m.RUnlock() - return r - } else { - r, existed = m.willDeprecatedData[userKey] - m.RUnlock() - if existed { + if r != nil { m.Lock() m.data[userKey] = r m.Unlock() } + }() + + m.RLock() + defer m.RUnlock() + if r, existed := m.data[userKey]; existed { return r + } else { + return m.willDeprecatedData[userKey] } } -func (m *ProverManager) GetOrCreate(userKey string) *proverSession { +func (m *ProverManager) GetOrCreate(userKey, cliName string) *proverSession { if ret := m.Get(userKey); ret != nil { return ret @@ -58,7 +81,8 @@ func (m *ProverManager) GetOrCreate(userKey string) *proverSession { ret := &proverSession{ proverToken: make(map[string]loginToken), - CliName: "pending for login", + CliName: cliName, + persistent: m.persistent, } if len(m.data) >= m.sizeLimit { @@ -77,14 +101,15 @@ type loginToken struct { // Client wraps an http client with a preset host for coordinator API calls type proverSession struct { - CliName string + CliName string + persistent *proverDataPersist sync.RWMutex proverToken map[string]loginToken completionCtx context.Context } -func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up string, param *types.LoginParameter, phase uint) (result *types.LoginSchema, nerr error) { +func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up string, param *types.LoginParameter, phase uint) (result loginToken, nerr error) { c.Lock() curPhase := c.proverToken[up].phase if c.completionCtx != nil { @@ -94,7 +119,8 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str case <-waitctx.Done(): return c.maintainLogin(ctx, cliMgr, up, param, phase) case <-ctx.Done(): - return nil, fmt.Errorf("ctx fail") + nerr = fmt.Errorf("ctx fail") + return } } @@ -102,7 +128,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str // outdate login phase, give up log.Debug("drop outdated proxy login attemp", "upstream", up, "cli", param.Message.ProverName, "phase", phase, "now", curPhase) defer c.Unlock() - return c.proverToken[up].LoginSchema, nil + return c.proverToken[up], nil } // occupy the update slot @@ -112,11 +138,8 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str defer func() { c.Lock() c.completionCtx = nil - if result != nil { - c.proverToken[up] = loginToken{ - LoginSchema: result, - phase: curPhase + 1, - } + if result.LoginSchema != nil { + c.proverToken[up] = result log.Info("maintain login status", "upstream", up, "cli", param.Message.ProverName, "phase", curPhase+1) } c.Unlock() @@ -131,41 +154,51 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str cli := cliMgr.Client(ctx) if cli == nil { - return nil, fmt.Errorf("get upstream cli fail") + nerr = fmt.Errorf("get upstream cli fail") + return } resp, err := cli.ProxyLogin(ctx, param) if err != nil { - return nil, fmt.Errorf("proxylogin fail: %v", err) + nerr = fmt.Errorf("proxylogin fail: %v", err) + return } if resp.ErrCode == ctypes.ErrJWTTokenExpired { cliMgr.Reset(cli) cli = cliMgr.Client(ctx) if cli == nil { - return nil, fmt.Errorf("get upstream cli fail (secondary try)") + nerr = fmt.Errorf("get upstream cli fail (secondary try)") + return } // like SDK, we would try one more time if the upstream token is expired resp, err = cli.ProxyLogin(ctx, param) if err != nil { - return nil, fmt.Errorf("proxylogin fail: %v", err) + nerr = fmt.Errorf("proxylogin fail: %v", err) + return } } if resp.ErrCode != 0 { - return nil, fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) + nerr = fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) + return } var loginResult loginSchema if err := resp.DecodeData(&loginResult); err != nil { - return nil, err + nerr = err + return } log.Debug("Proxy login done", "upstream", up, "cli", param.Message.ProverName) - return &types.LoginSchema{ - Token: loginResult.Token, - }, nil + result = loginToken{ + LoginSchema: &types.LoginSchema{ + Token: loginResult.Token, + }, + phase: curPhase + 1, + } + return } const expireTolerant = 10 * time.Minute @@ -173,19 +206,16 @@ const expireTolerant = 10 * time.Minute // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, up string, param *types.LoginParameter) error { c.RLock() - existedToken := c.proverToken[up].LoginSchema + existedToken := c.proverToken[up] c.RUnlock() - // Check if we have a valid cached token that hasn't expired - if existedToken != nil { - // TODO: how to reduce the unnecessary re-login? - // timeRemaining := time.Until(existedToken.Time) - // if timeRemaining > expireTolerant { - // return nil - // } + newtoken, err := c.maintainLogin(ctx, cli, up, param, math.MaxUint) + if newtoken.phase > existedToken.phase { + if err := c.persistent.Update(param.PublicKey, up, newtoken.LoginSchema); err != nil { + log.Error("Update persistent layer for prover tokens fail", "error", err) + } } - _, err := c.maintainLogin(ctx, cli, up, param, math.MaxUint) return err } diff --git a/coordinator/internal/controller/proxy/prover_session_test.go b/coordinator/internal/controller/proxy/prover_session_test.go index 51ce094966..b941f14b17 100644 --- a/coordinator/internal/controller/proxy/prover_session_test.go +++ b/coordinator/internal/controller/proxy/prover_session_test.go @@ -12,7 +12,7 @@ func TestProverManagerGetAndCreate(t *testing.T) { t.Fatalf("expected nil for non-existent key, got: %+v", got) } - sess1 := pm.GetOrCreate("user1") + sess1 := pm.GetOrCreate("user1", "u1_cli") if sess1 == nil { t.Fatalf("expected non-nil session from GetOrCreate") } @@ -23,8 +23,8 @@ func TestProverManagerGetAndCreate(t *testing.T) { } // Sanity check default value set during creation - if sess1.CliName != "pending for login" { - t.Fatalf("expected default CliName 'pending for login', got %q", sess1.CliName) + if sess1.CliName != "u1_cli" { + t.Fatalf("expected default CliName 'u1_cli', got %q", sess1.CliName) } } @@ -33,8 +33,8 @@ func TestProverManagerGetAndCreate(t *testing.T) { func TestProverManagerRolloverAndPromotion(t *testing.T) { pm := NewProverManager(2) - s1 := pm.GetOrCreate("u1") - s2 := pm.GetOrCreate("u2") + s1 := pm.GetOrCreate("u1", "u1_cli") + s2 := pm.GetOrCreate("u2", "u2_cli") if s1 == nil || s2 == nil { t.Fatalf("expected sessions to be created for u1/u2") } @@ -52,7 +52,7 @@ func TestProverManagerRolloverAndPromotion(t *testing.T) { pm.RUnlock() // Trigger rollover by creating a third key. - s3 := pm.GetOrCreate("u3") + s3 := pm.GetOrCreate("u3", "u3_cli") if s3 == nil { t.Fatalf("expected session for u3 after rollover") } diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index 21e2a768ab..44be33a946 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -111,7 +111,7 @@ func setupProxy(t *testing.T, proxyURL string, coordinatorURL []string) *http.Se } router := gin.New() - proxy.InitController(proxyConf, nil) + proxy.InitController(proxyConf, nil, nil) route.ProxyRoute(router, proxyConf, nil) t.Log("proxy server url", proxyURL) srv := &http.Server{ From 4b79e63c9b2e4718db1c3a386fed81454cdc4afb Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 22 Oct 2025 10:27:38 +0900 Subject: [PATCH 39/43] WIP: some refactors --- coordinator/internal/controller/proxy/auth.go | 10 ++++---- .../controller/proxy/client_manager.go | 5 ++++ .../internal/controller/proxy/controller.go | 2 +- .../internal/controller/proxy/get_task.go | 23 ++++++++++++++----- .../internal/controller/proxy/persistent.go | 13 +++++++---- .../controller/proxy/prover_session.go | 9 +++++--- .../internal/controller/proxy/submit_proof.go | 2 +- 7 files changed, 44 insertions(+), 20 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 3a0e862c23..5a8bde5769 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -62,16 +62,16 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { session := a.proverMgr.GetOrCreate(loginParam.PublicKey, loginParam.Message.ProverName) log.Debug("start handling login", "cli", loginParam.Message.ProverName) - for n, cli := range a.clients { + for _, cli := range a.clients { - go func(n string, cli Client) { - if err := session.ProxyLogin(c, cli, n, &loginParam.LoginParameter); err != nil { + go func(cli Client) { + if err := session.ProxyLogin(c, cli, &loginParam.LoginParameter); err != nil { log.Error("proxy login failed during token cache update", "userKey", loginParam.PublicKey, - "upstream", n, + "upstream", cli.Name(), "error", err) } - }(n, cli) + }(cli) } return loginParam.LoginParameter, nil diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index bf6530738a..6fd4c267d1 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -18,6 +18,7 @@ import ( type Client interface { Client(context.Context) *upClient Reset(cli *upClient) + Name() string } type ClientManager struct { @@ -124,6 +125,10 @@ func (cliMgr *ClientManager) Reset(cli *upClient) { log.Info("cached client cleared", "name", cliMgr.name) } +func (cliMgr *ClientManager) Name() string { + return cliMgr.name +} + func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { cliMgr.cachedCli.RLock() if cliMgr.cachedCli.cli != nil { diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 67ff185ea2..3044f6f5cf 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -32,7 +32,7 @@ func InitController(cfg *config.ProxyConfig, db *gorm.DB, reg prometheus.Registe if err != nil { panic("create new client fail") } - clients[nm] = cli + clients[cli.Name()] = cli } proverManager := NewProverManagerWithPersistent(100, db) diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index ea523737f2..ec73301213 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -33,6 +33,7 @@ func getSessionData(ctx *gin.Context) string { // PriorityUpstreamManager manages priority upstream mappings with thread safety type PriorityUpstreamManager struct { sync.RWMutex + *proverPriorityPersist data map[string]string } @@ -46,8 +47,17 @@ func NewPriorityUpstreamManager() *PriorityUpstreamManager { // Get retrieves the priority upstream for a given key func (p *PriorityUpstreamManager) Get(key string) (string, bool) { p.RLock() - defer p.RUnlock() value, exists := p.data[key] + p.RUnlock() + + if !exists { + if v, err := p.proverPriorityPersist.Get(key); err != nil { + log.Error("") + } else if v != "" { + return v, true + } + } + return value, exists } @@ -107,9 +117,10 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { session := ptc.proverMgr.Get(publicKey) - getTask := func(upStream string, cli Client) (error, int) { - log.Debug("Start get task", "up", upStream, "cli", session.CliName) - resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) + getTask := func(cli Client) (error, int) { + log.Debug("Start get task", "up", cli.Name(), "cli", session.CliName) + upStream := cli.Name() + resp, err := session.GetTask(ctx, &getTaskParameter, cli) if err != nil { log.Error("Upstream error for get task", "error", err, "up", upStream, "cli", session.CliName) return err, types.ErrCoordinatorGetTaskFailure @@ -143,7 +154,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { cli := ptc.clients[priorityUpstream] log.Debug("Try get task from priority stream", "up", priorityUpstream, "cli", session.CliName) if cli != nil { - err, code := getTask(priorityUpstream, cli) + err, code := getTask(cli) if err != nil { types.RenderFailure(ctx, code, err) return @@ -173,7 +184,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { // Iterate over the shuffled keys for _, n := range keys { - if err, code := getTask(n, ptc.clients[n]); err == nil && code == 0 { + if err, code := getTask(ptc.clients[n]); err == nil && code == 0 { // get task done return } diff --git a/coordinator/internal/controller/proxy/persistent.go b/coordinator/internal/controller/proxy/persistent.go index 6d7ad62f99..fdbbac57ce 100644 --- a/coordinator/internal/controller/proxy/persistent.go +++ b/coordinator/internal/controller/proxy/persistent.go @@ -88,15 +88,20 @@ func NewProverPriorityPersist(db *gorm.DB) *proverPriorityPersist { return &proverPriorityPersist{db: db} } -func (p *proverPriorityPersist) Get(userKey string) string { +func (p *proverPriorityPersist) Get(userKey string) (string, error) { if p == nil || p.db == nil { - return "" + return "", nil } var rec priorityUpstreamRecord if err := p.db.Where("public_key = ?", userKey).First(&rec).Error; err != nil { - return "" + if err != gorm.ErrRecordNotFound { + return "", err + } else { + return "", nil + } + } - return rec.Upstream + return rec.Upstream, nil } func (p *proverPriorityPersist) Update(userKey, up string) error { diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 81eaa531fe..1a6c94699c 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -204,7 +204,8 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str const expireTolerant = 10 * time.Minute // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, up string, param *types.LoginParameter) error { +func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) error { + up := cli.Name() c.RLock() existedToken := c.proverToken[up] c.RUnlock() @@ -220,7 +221,8 @@ func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, up string, p } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, cliMgr Client, up string) (*ctypes.Response, error) { +func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, cliMgr Client) (*ctypes.Response, error) { + up := cliMgr.Name() c.RLock() token := c.proverToken[up] c.RUnlock() @@ -257,7 +259,8 @@ func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParamet } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, cliMgr Client, up string) (*ctypes.Response, error) { +func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, cliMgr Client) (*ctypes.Response, error) { + up := cliMgr.Name() c.RLock() token := c.proverToken[up] c.RUnlock() diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index 5110eb741d..44ed473d7a 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -68,7 +68,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { log.Debug("Start submitting", "up", upstream, "cli", session.CliName, "id", realTaskID, "status", submitParameter.Status) submitParameter.TaskID = realTaskID - resp, err := session.SubmitProof(ctx, &submitParameter, cli, upstream) + resp, err := session.SubmitProof(ctx, &submitParameter, cli) if err != nil { log.Error("Upstream has error resp for submit", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upstream, "cli", session.CliName) types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) From 6696aac16aeb7f8ffe758d25739d6e8e7b53a2d0 Mon Sep 17 00:00:00 2001 From: Ho Date: Thu, 23 Oct 2025 15:23:59 +0900 Subject: [PATCH 40/43] WIP --- database/migrate/migrate.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/database/migrate/migrate.go b/database/migrate/migrate.go index a7ee894300..79f18350af 100644 --- a/database/migrate/migrate.go +++ b/database/migrate/migrate.go @@ -24,6 +24,15 @@ func init() { goose.SetVerbose(verbose) } +// FreeMigrate migrate db with specified goose TableName and Dir +func FreeMigrate(db *sql.DB, dir, tableName string) error { + if tableName != "" { + goose.SetTableName(tableName) + } + + return goose.Up(db, dir, goose.WithAllowMissing()) +} + // Migrate migrate db func Migrate(db *sql.DB) error { //return goose.Up(db, MIGRATIONS_DIR, goose.WithAllowMissing()) From 20fde41be8512c9ba8a8c454e60bd9dc833dee88 Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 5 Nov 2025 22:02:14 +0900 Subject: [PATCH 41/43] complete persistent layer and unit test --- coordinator/conf/config_proxy.json | 8 +- coordinator/internal/config/proxy_config.go | 2 +- coordinator/internal/controller/proxy/auth.go | 7 +- .../internal/controller/proxy/client.go | 4 +- .../controller/proxy/client_manager.go | 12 +- .../internal/controller/proxy/controller.go | 2 +- .../internal/controller/proxy/get_task.go | 70 +++++++--- .../controller/proxy/migrate/config.json | 4 - .../internal/controller/proxy/persistent.go | 2 +- .../controller/proxy/prover_session.go | 26 ++-- .../controller/proxy/prover_session_test.go | 13 +- .../internal/controller/proxy/submit_proof.go | 17 ++- coordinator/internal/logic/auth/login.go | 2 +- coordinator/internal/logic/libzkp/lib_mock.go | 4 + coordinator/internal/middleware/login_jwt.go | 5 +- .../internal/middleware/proxy_bearer.go | 1 - coordinator/test/api_test.go | 82 +++++++++++- coordinator/test/mock_prover.go | 50 ++++--- coordinator/test/proxy_test.go | 125 +++++++++--------- database/migrate/migrate.go | 39 +++++- .../migrations/proxy}/0001_running_tables.sql | 6 +- 21 files changed, 316 insertions(+), 165 deletions(-) delete mode 100644 coordinator/internal/controller/proxy/migrate/config.json delete mode 100644 coordinator/internal/middleware/proxy_bearer.go rename {coordinator/internal/controller/proxy/migrate => database/migrate/migrations/proxy}/0001_running_tables.sql (76%) diff --git a/coordinator/conf/config_proxy.json b/coordinator/conf/config_proxy.json index dd1139ba6e..15a171159c 100644 --- a/coordinator/conf/config_proxy.json +++ b/coordinator/conf/config_proxy.json @@ -21,5 +21,11 @@ "retry_wait_time_sec": 10, "connection_timeout_sec": 30 } - } + }, + "db": { + "driver_name": "postgres", + "dsn": "postgres://localhost/scroll?sslmode=disable", + "maxOpenNum": 200, + "maxIdleNum": 20 + } } diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index 2548a790a3..7aa4dca662 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -41,7 +41,7 @@ type UpStream struct { RetryCount uint `json:"retry_count"` RetryWaitTime uint `json:"retry_wait_time_sec"` ConnectionTimeoutSec uint `json:"connection_timeout_sec"` - CompatibileMode bool `json:"compatibile_mode,omitempty"` + CompatibileMode bool `json:"compatible_mode,omitempty"` } // Config load configuration items. diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 5a8bde5769..41c99bb673 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -3,8 +3,6 @@ package proxy import ( "fmt" - "time" - jwt "github.com/appleboy/gin-jwt/v2" "github.com/gin-gonic/gin" "github.com/scroll-tech/go-ethereum/log" @@ -23,7 +21,6 @@ type AuthController struct { proverMgr *ProverManager } -const upstreamConnTimeout = time.Second * 2 const LoginParamCache = "login_param" const ProverTypesKey = "prover_types" const SignatureKey = "prover_signature" @@ -59,7 +56,7 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { return nil, fmt.Errorf("proxy do not support recursive login") } - session := a.proverMgr.GetOrCreate(loginParam.PublicKey, loginParam.Message.ProverName) + session := a.proverMgr.GetOrCreate(loginParam.PublicKey) log.Debug("start handling login", "cli", loginParam.Message.ProverName) for _, cli := range a.clients { @@ -131,6 +128,8 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { if loginParam.PublicKey != "" { c.Set(LoginParamCache, loginParam) + c.Set(types.ProverName, loginParam.Message.ProverName) + // publickey will also be set since we have specified public_key as identical key return loginParam.PublicKey } diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 8ba9e12716..257a337401 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -1,3 +1,4 @@ +//nolint:errcheck,bodyclose // body is closed in the following handleHttpResp call package proxy import ( @@ -12,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/crypto" ctypes "scroll-tech/common/types" + "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" ) @@ -39,7 +41,7 @@ func (c *upClient) Token() string { return c.loginToken } -// need a parsable schema defination +// need a parsable schema definition type loginSchema struct { Time string `json:"time"` Token string `json:"token"` diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 6fd4c267d1..b2bf750603 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -69,10 +69,14 @@ func NewClientManager(name string, cliCfg *config.ProxyClient, cfg *config.UpStr }, nil } +type ctxKeyType string + +const loginCliKey ctxKeyType = "cli" + func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) { if cliMgr.cfg.CompatibileMode { loginCli.loginToken = "dummy" - log.Info("Skip login process for compatibile mode") + log.Info("Skip login process for compatible mode") return } @@ -84,7 +88,7 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) { } for { - log.Info("attempting login to upstream coordinator", "name", cliMgr.name) + log.Info("proxy attempting login to upstream coordinator", "name", cliMgr.name) loginResp, err := loginCli.Login(ctx, cliMgr.genLoginParam) if err == nil && loginResp.ErrCode == 0 { var loginResult loginSchema @@ -151,7 +155,7 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { // Set new completion context and launch login goroutine ctx, completionDone := context.WithCancel(context.TODO()) loginCli := newUpClient(cliMgr.cfg) - completionCtx = context.WithValue(ctx, "cli", loginCli) + completionCtx = context.WithValue(ctx, loginCliKey, loginCli) cliMgr.cachedCli.completionCtx = completionCtx // Launch keep-login goroutine @@ -174,7 +178,7 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { case <-ctx.Done(): return nil case <-completionCtx.Done(): - cli := completionCtx.Value("cli").(*upClient) + cli := completionCtx.Value(loginCliKey).(*upClient) return cli } } diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 3044f6f5cf..188637cb2e 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -36,7 +36,7 @@ func InitController(cfg *config.ProxyConfig, db *gorm.DB, reg prometheus.Registe } proverManager := NewProverManagerWithPersistent(100, db) - priorityManager := NewPriorityUpstreamManager() + priorityManager := NewPriorityUpstreamManagerPersistent(db) Auth = NewAuthController(cfg, clients, proverManager) GetTask = NewGetTaskController(cfg, clients, proverManager, priorityManager, reg) diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index ec73301213..5063a6814a 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -8,6 +8,7 @@ import ( "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" "github.com/scroll-tech/go-ethereum/log" + "gorm.io/gorm" "scroll-tech/common/types" @@ -15,7 +16,7 @@ import ( coordinatorType "scroll-tech/coordinator/internal/types" ) -func getSessionData(ctx *gin.Context) string { +func getSessionData(ctx *gin.Context) (string, string) { publicKeyData, publicKeyExist := ctx.Get(coordinatorType.PublicKey) publicKey, castOk := publicKeyData.(string) @@ -24,10 +25,17 @@ func getSessionData(ctx *gin.Context) string { log.Warn("get_task parameter fail", "error", nerr) types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) - return "" + return "", "" } - return publicKey + publicNameData, publicNameExist := ctx.Get(coordinatorType.ProverName) + publicName, castOk := publicNameData.(string) + if !publicNameExist || !castOk { + log.Error("no public name binding for unknown reason, but we still forward with name = 'unknown'", "data", publicNameData) + publicName = "unknown" + } + + return publicKey, publicName } // PriorityUpstreamManager manages priority upstream mappings with thread safety @@ -44,16 +52,26 @@ func NewPriorityUpstreamManager() *PriorityUpstreamManager { } } +// NewPriorityUpstreamManager creates a new PriorityUpstreamManager +func NewPriorityUpstreamManagerPersistent(db *gorm.DB) *PriorityUpstreamManager { + return &PriorityUpstreamManager{ + data: make(map[string]string), + proverPriorityPersist: NewProverPriorityPersist(db), + } +} + // Get retrieves the priority upstream for a given key func (p *PriorityUpstreamManager) Get(key string) (string, bool) { + p.RLock() value, exists := p.data[key] p.RUnlock() if !exists { if v, err := p.proverPriorityPersist.Get(key); err != nil { - log.Error("") + log.Error("persistent priority record read failure", "error", err, "key", key) } else if v != "" { + log.Debug("restore record from persistent layer", "key", key, "value", v) return v, true } } @@ -63,6 +81,11 @@ func (p *PriorityUpstreamManager) Get(key string) (string, bool) { // Set sets the priority upstream for a given key func (p *PriorityUpstreamManager) Set(key, value string) { + defer func() { + if err := p.proverPriorityPersist.Update(key, value); err != nil { + log.Error("persistent priority record failure", "error", err, "key", key, "value", value) + } + }() p.Lock() defer p.Unlock() p.data[key] = value @@ -81,8 +104,8 @@ type GetTaskController struct { clients Clients priorityUpstream *PriorityUpstreamManager - workingRnd *rand.Rand - getTaskAccessCounter *prometheus.CounterVec + //workingRnd *rand.Rand + //getTaskAccessCounter *prometheus.CounterVec } // NewGetTaskController create a get prover task controller @@ -95,10 +118,10 @@ func NewGetTaskController(cfg *config.ProxyConfig, clients Clients, proverMgr *P } } -func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { - // TODO: implement proxy get task access counter - return nil -} +// func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { +// // TODO: implement proxy get task access counter +// return nil +// } // GetTasks get assigned chunk/batch task func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { @@ -110,25 +133,30 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { return } - publicKey := getSessionData(ctx) + publicKey, proverName := getSessionData(ctx) if publicKey == "" { return } session := ptc.proverMgr.Get(publicKey) + if session == nil { + nerr := fmt.Errorf("can not get session for prover %s", proverName) + types.RenderFailure(ctx, types.InternalServerError, nerr) + return + } getTask := func(cli Client) (error, int) { - log.Debug("Start get task", "up", cli.Name(), "cli", session.CliName) + log.Debug("Start get task", "up", cli.Name(), "cli", proverName) upStream := cli.Name() resp, err := session.GetTask(ctx, &getTaskParameter, cli) if err != nil { - log.Error("Upstream error for get task", "error", err, "up", upStream, "cli", session.CliName) + log.Error("Upstream error for get task", "error", err, "up", upStream, "cli", proverName) return err, types.ErrCoordinatorGetTaskFailure } else if resp.ErrCode != types.ErrCoordinatorEmptyProofData { if resp.ErrCode != 0 { // simply dispatch the error from upstream to prover - log.Error("Upstream has error resp for get task", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upStream, "cli", session.CliName) + log.Error("Upstream has error resp for get task", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upStream, "cli", proverName) return fmt.Errorf("upstream failure %s:", resp.ErrMsg), resp.ErrCode } @@ -136,11 +164,11 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { if err = resp.DecodeData(&task); err == nil { task.TaskID = formUpstreamWithTaskName(upStream, task.TaskID) ptc.priorityUpstream.Set(publicKey, upStream) - log.Debug("Upstream get task", "up", upStream, "cli", session.CliName, "taskID", task.TaskID, "taskType", task.TaskType) + log.Debug("Upstream get task", "up", upStream, "cli", proverName, "taskID", task.TaskID, "taskType", task.TaskType) types.RenderSuccess(ctx, &task) return nil, 0 } else { - log.Error("Upstream has wrong data for get task", "error", err, "up", upStream, "cli", session.CliName) + log.Error("Upstream has wrong data for get task", "error", err, "up", upStream, "cli", proverName) return fmt.Errorf("decode task fail: %v", err), types.InternalServerError } } @@ -148,11 +176,11 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { return nil, resp.ErrCode } - // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp + // if the priority upstream is set, we try this upstream first until get the task resp or no task resp priorityUpstream, exist := ptc.priorityUpstream.Get(publicKey) if exist { cli := ptc.clients[priorityUpstream] - log.Debug("Try get task from priority stream", "up", priorityUpstream, "cli", session.CliName) + log.Debug("Try get task from priority stream", "up", priorityUpstream, "cli", proverName) if cli != nil { err, code := getTask(cli) if err != nil { @@ -163,10 +191,10 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { return } // only continue if get empty task (the task has been removed in upstream) - log.Debug("can not get priority task from upstream", "up", priorityUpstream, "cli", session.CliName) + log.Debug("can not get priority task from upstream", "up", priorityUpstream, "cli", proverName) } else { - log.Warn("A upstream is removed or lost for some reason while running", "up", priorityUpstream, "cli", session.CliName) + log.Warn("A upstream is removed or lost for some reason while running", "up", priorityUpstream, "cli", proverName) } } ptc.priorityUpstream.Delete(publicKey) @@ -190,7 +218,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { } } - log.Debug("get no task from upstream", "cli", session.CliName) + log.Debug("get no task from upstream", "cli", proverName) // if all get task failed, throw empty proof resp types.RenderFailure(ctx, types.ErrCoordinatorEmptyProofData, fmt.Errorf("get empty prover task")) } diff --git a/coordinator/internal/controller/proxy/migrate/config.json b/coordinator/internal/controller/proxy/migrate/config.json deleted file mode 100644 index e4fcf4ebe0..0000000000 --- a/coordinator/internal/controller/proxy/migrate/config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "dsn": "postgres://localhost/proxy_run?sslmode=disable", - "driver_name": "postgres" -} \ No newline at end of file diff --git a/coordinator/internal/controller/proxy/persistent.go b/coordinator/internal/controller/proxy/persistent.go index fdbbac57ce..48d8c81578 100644 --- a/coordinator/internal/controller/proxy/persistent.go +++ b/coordinator/internal/controller/proxy/persistent.go @@ -74,7 +74,7 @@ func (p *proverDataPersist) Update(userKey, up string, login *types.LoginSchema) return p.db.Clauses( clause.OnConflict{ - Columns: []clause.Column{{Name: "public_key"}, {Name: "upstream"}}, // fixed typo + Columns: []clause.Column{{Name: "public_key"}, {Name: "upstream"}}, DoUpdates: clause.AssignmentColumns([]string{"up_token", "expired"}), }, ).Create(&rec).Error diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 1a6c94699c..17c37425ce 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -5,13 +5,13 @@ import ( "fmt" "math" "sync" - "time" "gorm.io/gorm" "github.com/scroll-tech/go-ethereum/log" ctypes "scroll-tech/common/types" + "scroll-tech/coordinator/internal/types" ) @@ -43,20 +43,20 @@ func NewProverManagerWithPersistent(size int, db *gorm.DB) *ProverManager { // get retrieves ProverSession for a given user key, returns empty if still not exists func (m *ProverManager) Get(userKey string) (ret *proverSession) { defer func() { - r := ret - if r == nil { + if ret == nil { var err error - r, err = m.persistent.Get(userKey) + ret, err = m.persistent.Get(userKey) if err != nil { log.Error("Get persistent layer for prover tokens fail", "error", err) - } else if r != nil { - r.persistent = m.persistent + } else if ret != nil { + fmt.Println("restore record from persistent", userKey, ret.proverToken) + ret.persistent = m.persistent } } - if r != nil { + if ret != nil { m.Lock() - m.data[userKey] = r + m.data[userKey] = ret m.Unlock() } }() @@ -70,7 +70,7 @@ func (m *ProverManager) Get(userKey string) (ret *proverSession) { } } -func (m *ProverManager) GetOrCreate(userKey, cliName string) *proverSession { +func (m *ProverManager) GetOrCreate(userKey string) *proverSession { if ret := m.Get(userKey); ret != nil { return ret @@ -81,7 +81,6 @@ func (m *ProverManager) GetOrCreate(userKey, cliName string) *proverSession { ret := &proverSession{ proverToken: make(map[string]loginToken), - CliName: cliName, persistent: m.persistent, } @@ -101,7 +100,6 @@ type loginToken struct { // Client wraps an http client with a preset host for coordinator API calls type proverSession struct { - CliName string persistent *proverDataPersist sync.RWMutex @@ -126,7 +124,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str if phase < curPhase { // outdate login phase, give up - log.Debug("drop outdated proxy login attemp", "upstream", up, "cli", param.Message.ProverName, "phase", phase, "now", curPhase) + log.Debug("drop outdated proxy login attempt", "upstream", up, "cli", param.Message.ProverName, "phase", phase, "now", curPhase) defer c.Unlock() return c.proverToken[up], nil } @@ -150,7 +148,6 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str c.Unlock() log.Debug("start proxy login process", "upstream", up, "cli", param.Message.ProverName) - c.CliName = param.Message.ProverName cli := cliMgr.Client(ctx) if cli == nil { @@ -201,7 +198,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str return } -const expireTolerant = 10 * time.Minute +// const expireTolerant = 10 * time.Minute // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) error { @@ -224,6 +221,7 @@ func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, cliMgr Client) (*ctypes.Response, error) { up := cliMgr.Name() c.RLock() + log.Debug("call get task", "up", up, "tokens", c.proverToken) token := c.proverToken[up] c.RUnlock() diff --git a/coordinator/internal/controller/proxy/prover_session_test.go b/coordinator/internal/controller/proxy/prover_session_test.go index b941f14b17..7563908579 100644 --- a/coordinator/internal/controller/proxy/prover_session_test.go +++ b/coordinator/internal/controller/proxy/prover_session_test.go @@ -12,7 +12,7 @@ func TestProverManagerGetAndCreate(t *testing.T) { t.Fatalf("expected nil for non-existent key, got: %+v", got) } - sess1 := pm.GetOrCreate("user1", "u1_cli") + sess1 := pm.GetOrCreate("user1") if sess1 == nil { t.Fatalf("expected non-nil session from GetOrCreate") } @@ -21,11 +21,6 @@ func TestProverManagerGetAndCreate(t *testing.T) { if got := pm.Get("user1"); got != sess1 { t.Fatalf("expected same session pointer on Get, got different instance: %p vs %p", got, sess1) } - - // Sanity check default value set during creation - if sess1.CliName != "u1_cli" { - t.Fatalf("expected default CliName 'u1_cli', got %q", sess1.CliName) - } } // TestProverManagerRolloverAndPromotion verifies rollover when sizeLimit is reached @@ -33,8 +28,8 @@ func TestProverManagerGetAndCreate(t *testing.T) { func TestProverManagerRolloverAndPromotion(t *testing.T) { pm := NewProverManager(2) - s1 := pm.GetOrCreate("u1", "u1_cli") - s2 := pm.GetOrCreate("u2", "u2_cli") + s1 := pm.GetOrCreate("u1") + s2 := pm.GetOrCreate("u2") if s1 == nil || s2 == nil { t.Fatalf("expected sessions to be created for u1/u2") } @@ -52,7 +47,7 @@ func TestProverManagerRolloverAndPromotion(t *testing.T) { pm.RUnlock() // Trigger rollover by creating a third key. - s3 := pm.GetOrCreate("u3", "u3_cli") + s3 := pm.GetOrCreate("u3") if s3 == nil { t.Fatalf("expected session for u3 after rollover") } diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index 44ed473d7a..4c9db29038 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -9,6 +9,7 @@ import ( "github.com/scroll-tech/go-ethereum/log" "scroll-tech/common/types" + "scroll-tech/coordinator/internal/config" coordinatorType "scroll-tech/coordinator/internal/types" ) @@ -51,12 +52,18 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { return } - publicKey := getSessionData(ctx) + publicKey, proverName := getSessionData(ctx) if publicKey == "" { return } session := spc.proverMgr.Get(publicKey) + if session == nil { + nerr := fmt.Errorf("can not get session for prover %s", proverName) + types.RenderFailure(ctx, types.InternalServerError, nerr) + return + } + upstream, realTaskID := upstreamFromTaskName(submitParameter.TaskID) cli, existed := spc.clients[upstream] if !existed { @@ -65,21 +72,21 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) return } - log.Debug("Start submitting", "up", upstream, "cli", session.CliName, "id", realTaskID, "status", submitParameter.Status) + log.Debug("Start submitting", "up", upstream, "cli", proverName, "id", realTaskID, "status", submitParameter.Status) submitParameter.TaskID = realTaskID resp, err := session.SubmitProof(ctx, &submitParameter, cli) if err != nil { - log.Error("Upstream has error resp for submit", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upstream, "cli", session.CliName) + log.Error("Upstream has error resp for submit", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upstream, "cli", proverName) types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return } else if resp.ErrCode != 0 { - log.Error("Upstream has error resp for get task", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upstream, "cli", session.CliName) + log.Error("Upstream has error resp for get task", "code", resp.ErrCode, "msg", resp.ErrMsg, "up", upstream, "cli", proverName) // simply dispatch the error from upstream to prover types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) return } else { - log.Debug("Submit proof to upstream", "up", upstream, "cli", session.CliName, "taskID", realTaskID) + log.Debug("Submit proof to upstream", "up", upstream, "cli", proverName, "taskID", realTaskID) spc.priorityUpstream.Delete(publicKey) types.RenderSuccess(ctx, resp.Data) return diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index f691eb0e85..fb4cd39bdb 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -39,7 +39,7 @@ func (s *SimpleDeduplicator) InsertChallenge(ctx context.Context, challengeStrin return nil } -// NewLoginLogicWithSimpleDEduplicator new a LoginLogic, do not use db to deduplicate challege +// NewLoginLogicWithSimpleDEduplicator new a LoginLogic, do not use db to deduplicate challenge func NewLoginLogicWithSimpleDeduplicator(vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { return newLoginLogic(&SimpleDeduplicator{}, vcfg, vf) } diff --git a/coordinator/internal/logic/libzkp/lib_mock.go b/coordinator/internal/logic/libzkp/lib_mock.go index 2f5445faa4..a56d175b25 100644 --- a/coordinator/internal/logic/libzkp/lib_mock.go +++ b/coordinator/internal/logic/libzkp/lib_mock.go @@ -24,6 +24,10 @@ import ( // return true // } +func UniversalTaskCompatibilityFix(taskJSON string) (string, error) { + panic("should not run here") +} + // GenerateWrappedProof returns a fixed dummy proof string in the mock. func GenerateWrappedProof(proofJSON, metadata string, vkData []byte) string { diff --git a/coordinator/internal/middleware/login_jwt.go b/coordinator/internal/middleware/login_jwt.go index 66d9702ac8..ed80613daf 100644 --- a/coordinator/internal/middleware/login_jwt.go +++ b/coordinator/internal/middleware/login_jwt.go @@ -14,10 +14,7 @@ import ( ) func nonIdendityAuthorizator(data interface{}, _ *gin.Context) bool { - if data == nil { - return false - } - return true + return data != nil } // LoginMiddleware jwt auth middleware diff --git a/coordinator/internal/middleware/proxy_bearer.go b/coordinator/internal/middleware/proxy_bearer.go deleted file mode 100644 index c870d7c164..0000000000 --- a/coordinator/internal/middleware/proxy_bearer.go +++ /dev/null @@ -1 +0,0 @@ -package middleware diff --git a/coordinator/test/api_test.go b/coordinator/test/api_test.go index 9bbe9e95ce..9fc303d760 100644 --- a/coordinator/test/api_test.go +++ b/coordinator/test/api_test.go @@ -30,12 +30,14 @@ import ( "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/api" "scroll-tech/coordinator/internal/controller/cron" + "scroll-tech/coordinator/internal/controller/proxy" "scroll-tech/coordinator/internal/orm" "scroll-tech/coordinator/internal/route" ) var ( - conf *config.Config + conf *config.Config + proxyConf *config.ProxyConfig testApps *testcontainers.TestcontainerApps @@ -90,14 +92,20 @@ func randmURLBatch(n int) []string { return urls } -func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL string) (*cron.Collector, *http.Server) { +func setupCoordinatorDb(t *testing.T) { var err error - db, err = testApps.GetGormDBClient() + assert.NotNil(t, db, "setEnv must be called before") + // db, err = testApps.GetGormDBClient() - assert.NoError(t, err) + // assert.NoError(t, err) sqlDB, err := db.DB() assert.NoError(t, err) assert.NoError(t, migrate.ResetDB(sqlDB)) +} + +func launchCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL string) (*cron.Collector, *http.Server) { + + assert.NotNil(t, db, "db must be set") tokenTimeout = 60 conf = &config.Config{ @@ -153,6 +161,71 @@ func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL stri return proofCollector, srv } +func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL string) (*cron.Collector, *http.Server) { + setupCoordinatorDb(t) + return launchCoordinator(t, proversPerSession, coordinatorURL) +} + +func setupProxyDb(t *testing.T) { + assert.NotNil(t, db, "setEnv must be called before") + sqlDB, err := db.DB() + assert.NoError(t, err) + assert.NoError(t, migrate.ResetModuleDB(sqlDB, "proxy")) +} + +func launchProxy(t *testing.T, proxyURL string, coordinatorURL []string, usePersistent bool) *http.Server { + var err error + assert.NoError(t, err) + + coordinators := make(map[string]*config.UpStream) + for i, n := range coordinatorURL { + coordinators[fmt.Sprintf("coordinator_%d", i)] = testProxyUpStreamCfg(n) + } + + tokenTimeout = 60 + proxyConf = &config.ProxyConfig{ + ProxyName: "test_proxy", + ProxyManager: &config.ProxyManager{ + Verifier: &config.VerifierConfig{ + MinProverVersion: "v4.4.89", + Verifiers: []config.AssetConfig{{ + AssetsPath: "", + ForkName: "euclidV2", + }}, + }, + Client: testProxyClientCfg(), + Auth: &config.Auth{ + Secret: "proxy", + ChallengeExpireDurationSec: tokenTimeout, + LoginExpireDurationSec: tokenTimeout, + }, + }, + Coordinators: coordinators, + } + + router := gin.New() + if usePersistent { + proxy.InitController(proxyConf, db, nil) + } else { + proxy.InitController(proxyConf, nil, nil) + } + route.ProxyRoute(router, proxyConf, nil) + t.Log("proxy server url", proxyURL) + srv := &http.Server{ + Addr: proxyURL, + Handler: router, + } + go func() { + runErr := srv.ListenAndServe() + if runErr != nil && !errors.Is(runErr, http.ErrServerClosed) { + assert.NoError(t, runErr) + } + }() + time.Sleep(time.Second * 2) + + return srv +} + func setEnv(t *testing.T) { if envSet { t.Log("SetEnv is re-entried") @@ -175,6 +248,7 @@ func setEnv(t *testing.T) { sqlDB, err := db.DB() assert.NoError(t, err) assert.NoError(t, migrate.ResetDB(sqlDB)) + assert.NoError(t, migrate.MigrateModule(sqlDB, "proxy")) batchOrm = orm.NewBatch(db) chunkOrm = orm.NewChunk(db) diff --git a/coordinator/test/mock_prover.go b/coordinator/test/mock_prover.go index 958c230547..913344e856 100644 --- a/coordinator/test/mock_prover.go +++ b/coordinator/test/mock_prover.go @@ -34,6 +34,8 @@ type mockProver struct { privKey *ecdsa.PrivateKey proofType message.ProofType coordinatorURL string + token string + useCacheToken bool } func newMockProver(t *testing.T, proverName string, coordinatorURL string, proofType message.ProofType, version string) *mockProver { @@ -50,6 +52,14 @@ func newMockProver(t *testing.T, proverName string, coordinatorURL string, proof return prover } +func (r *mockProver) resetConnection(coordinatorURL string) { + r.coordinatorURL = coordinatorURL +} + +func (r *mockProver) setUseCacheToken(enable bool) { + r.useCacheToken = enable +} + // connectToCoordinator sets up a websocket client to connect to the prover manager. func (r *mockProver) connectToCoordinator(t *testing.T, proverTypes []types.ProverType) (string, int, string) { challengeString := r.challenge(t) @@ -115,6 +125,7 @@ func (r *mockProver) login(t *testing.T, challengeString string, proverTypes []t assert.NoError(t, err) assert.Equal(t, http.StatusOK, resp.StatusCode()) assert.Empty(t, result.ErrMsg) + r.token = loginData.Token return loginData.Token, 0, "" } @@ -144,11 +155,14 @@ func (r *mockProver) healthCheckFailure(t *testing.T) bool { func (r *mockProver) getProverTask(t *testing.T, proofType message.ProofType) (*types.GetTaskSchema, int, string) { // get task from coordinator - token, errCode, errMsg := r.connectToCoordinator(t, []types.ProverType{types.MakeProverType(proofType)}) - if errCode != 0 { - return nil, errCode, errMsg + if !r.useCacheToken || r.token == "" { + token, errCode, errMsg := r.connectToCoordinator(t, []types.ProverType{types.MakeProverType(proofType)}) + if errCode != 0 { + return nil, errCode, errMsg + } + assert.NotEmpty(t, token) + assert.Equal(t, token, r.token) } - assert.NotEmpty(t, token) type response struct { ErrCode int `json:"errcode"` @@ -160,7 +174,7 @@ func (r *mockProver) getProverTask(t *testing.T, proofType message.ProofType) (* client := resty.New() resp, err := client.R(). SetHeader("Content-Type", "application/json"). - SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)). + SetHeader("Authorization", fmt.Sprintf("Bearer %s", r.token)). SetBody(map[string]interface{}{"universal": true, "prover_height": 100, "task_types": []int{int(proofType)}}). SetResult(&result). Post("http://" + r.coordinatorURL + "/coordinator/v1/get_task") @@ -174,11 +188,14 @@ func (r *mockProver) getProverTask(t *testing.T, proofType message.ProofType) (* //nolint:unparam func (r *mockProver) tryGetProverTask(t *testing.T, proofType message.ProofType) (int, string) { // get task from coordinator - token, errCode, errMsg := r.connectToCoordinator(t, []types.ProverType{types.MakeProverType(proofType)}) - if errCode != 0 { - return errCode, errMsg + if !r.useCacheToken || r.token == "" { + token, errCode, errMsg := r.connectToCoordinator(t, []types.ProverType{types.MakeProverType(proofType)}) + if errCode != 0 { + return errCode, errMsg + } + assert.NotEmpty(t, token) + assert.Equal(t, token, r.token) } - assert.NotEmpty(t, token) type response struct { ErrCode int `json:"errcode"` @@ -190,7 +207,7 @@ func (r *mockProver) tryGetProverTask(t *testing.T, proofType message.ProofType) client := resty.New() resp, err := client.R(). SetHeader("Content-Type", "application/json"). - SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)). + SetHeader("Authorization", fmt.Sprintf("Bearer %s", r.token)). SetBody(map[string]interface{}{"prover_height": 100, "task_types": []int{int(proofType)}, "universal": true}). SetResult(&result). Post("http://" + r.coordinatorURL + "/coordinator/v1/get_task") @@ -249,10 +266,13 @@ func (r *mockProver) submitProof(t *testing.T, proverTaskSchema *types.GetTaskSc Universal: true, } - token, authErrCode, errMsg := r.connectToCoordinator(t, []types.ProverType{types.MakeProverType(message.ProofType(proverTaskSchema.TaskType))}) - assert.Equal(t, authErrCode, 0) - assert.Equal(t, errMsg, "") - assert.NotEmpty(t, token) + if !r.useCacheToken || r.token == "" { + token, authErrCode, errMsg := r.connectToCoordinator(t, []types.ProverType{types.MakeProverType(message.ProofType(proverTaskSchema.TaskType))}) + assert.Equal(t, authErrCode, 0) + assert.Equal(t, errMsg, "") + assert.NotEmpty(t, token) + assert.Equal(t, token, r.token) + } submitProofData, err := json.Marshal(submitProof) assert.NoError(t, err) @@ -262,7 +282,7 @@ func (r *mockProver) submitProof(t *testing.T, proverTaskSchema *types.GetTaskSc client := resty.New() resp, err := client.R(). SetHeader("Content-Type", "application/json"). - SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)). + SetHeader("Authorization", fmt.Sprintf("Bearer %s", r.token)). SetBody(string(submitProofData)). SetResult(&result). Post("http://" + r.coordinatorURL + "/coordinator/v1/submit_proof") diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index 44be33a946..013bad7660 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -2,14 +2,12 @@ package test import ( "context" - "errors" "fmt" "net/http" "strings" "testing" "time" - "github.com/gin-gonic/gin" "github.com/scroll-tech/da-codec/encoding" "github.com/stretchr/testify/assert" @@ -19,7 +17,6 @@ import ( "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/proxy" - "scroll-tech/coordinator/internal/route" ) func testProxyClientCfg() *config.ProxyClient { @@ -76,63 +73,10 @@ func testProxyClient(t *testing.T) { t.Logf("Client token: %s (%v)", client.Token(), client) } -var ( - proxyConf *config.ProxyConfig -) - -func setupProxy(t *testing.T, proxyURL string, coordinatorURL []string) *http.Server { - var err error - assert.NoError(t, err) - - coordinators := make(map[string]*config.UpStream) - for i, n := range coordinatorURL { - coordinators[fmt.Sprintf("coordinator_%d", i)] = testProxyUpStreamCfg(n) - } - - tokenTimeout = 60 - proxyConf = &config.ProxyConfig{ - ProxyName: "test_proxy", - ProxyManager: &config.ProxyManager{ - Verifier: &config.VerifierConfig{ - MinProverVersion: "v4.4.89", - Verifiers: []config.AssetConfig{{ - AssetsPath: "", - ForkName: "euclidV2", - }}, - }, - Client: testProxyClientCfg(), - Auth: &config.Auth{ - Secret: "proxy", - ChallengeExpireDurationSec: tokenTimeout, - LoginExpireDurationSec: tokenTimeout, - }, - }, - Coordinators: coordinators, - } - - router := gin.New() - proxy.InitController(proxyConf, nil, nil) - route.ProxyRoute(router, proxyConf, nil) - t.Log("proxy server url", proxyURL) - srv := &http.Server{ - Addr: proxyURL, - Handler: router, - } - go func() { - runErr := srv.ListenAndServe() - if runErr != nil && !errors.Is(runErr, http.ErrServerClosed) { - assert.NoError(t, runErr) - } - }() - time.Sleep(time.Second * 2) - - return srv -} - func testProxyHandshake(t *testing.T) { // Setup proxy http server. proxyURL := randomURL() - proxyHttpHandler := setupProxy(t, proxyURL, []string{}) + proxyHttpHandler := launchProxy(t, proxyURL, []string{}, false) defer func() { assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) }() @@ -152,13 +96,14 @@ func testProxyGetTask(t *testing.T) { }() proxyURL := urls[1] - proxyHttpHandler := setupProxy(t, proxyURL, []string{coordinatorURL}) + proxyHttpHandler := launchProxy(t, proxyURL, []string{coordinatorURL}, false) defer func() { assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) }() chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) - code, msg := chunkProver.tryGetProverTask(t, message.ProofTypeChunk) + chunkProver.setUseCacheToken(true) + code, _ := chunkProver.tryGetProverTask(t, message.ProofTypeChunk) assert.Equal(t, int(types.ErrCoordinatorEmptyProofData), code) err := l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2}) @@ -181,13 +126,14 @@ func testProxyGetTask(t *testing.T) { func testProxyProof(t *testing.T) { urls := randmURLBatch(3) coordinatorURL0 := urls[0] - collector0, httpHandler0 := setupCoordinator(t, 3, coordinatorURL0) + setupCoordinatorDb(t) + collector0, httpHandler0 := launchCoordinator(t, 3, coordinatorURL0) defer func() { collector0.Stop() httpHandler0.Shutdown(context.Background()) }() coordinatorURL1 := urls[1] - collector1, httpHandler1 := setupCoordinator(t, 3, coordinatorURL1) + collector1, httpHandler1 := launchCoordinator(t, 3, coordinatorURL1) defer func() { collector1.Stop() httpHandler1.Shutdown(context.Background()) @@ -198,11 +144,9 @@ func testProxyProof(t *testing.T) { } proxyURL := urls[2] - proxyHttpHandler := setupProxy(t, proxyURL, []string{coordinatorURL0, coordinatorURL1}) + proxyHttpHandler := launchProxy(t, proxyURL, []string{coordinatorURL0, coordinatorURL1}, false) defer func() { - fmt.Println("px end start") assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) - fmt.Println("px end") }() err := l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2}) @@ -213,6 +157,7 @@ func testProxyProof(t *testing.T) { assert.NoError(t, err) chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) + chunkProver.setUseCacheToken(true) task, code, msg := chunkProver.getProverTask(t, message.ProofTypeChunk) assert.Empty(t, code) if code == 0 { @@ -266,6 +211,56 @@ func testProxyProof(t *testing.T) { } } +func testProxyPersistent(t *testing.T) { + urls := randmURLBatch(4) + coordinatorURL0 := urls[0] + setupCoordinatorDb(t) + collector0, httpHandler0 := launchCoordinator(t, 3, coordinatorURL0) + defer func() { + collector0.Stop() + httpHandler0.Shutdown(context.Background()) + }() + coordinatorURL1 := urls[1] + collector1, httpHandler1 := launchCoordinator(t, 3, coordinatorURL1) + defer func() { + collector1.Stop() + httpHandler1.Shutdown(context.Background()) + }() + + setupProxyDb(t) + proxyURL1 := urls[2] + proxyHttpHandler := launchProxy(t, proxyURL1, []string{coordinatorURL0, coordinatorURL1}, true) + defer func() { + assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) + }() + + proxyURL2 := urls[3] + proxyHttpHandler2 := launchProxy(t, proxyURL2, []string{coordinatorURL0, coordinatorURL1}, true) + defer func() { + assert.NoError(t, proxyHttpHandler2.Shutdown(context.Background())) + }() + + err := l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2}) + assert.NoError(t, err) + dbChunk, err := chunkOrm.InsertChunk(context.Background(), chunk) + assert.NoError(t, err) + err = l2BlockOrm.UpdateChunkHashInRange(context.Background(), 0, 100, dbChunk.Hash) + assert.NoError(t, err) + + chunkProver := newMockProver(t, "prover_chunk_test", proxyURL1, message.ProofTypeChunk, version.Version) + chunkProver.setUseCacheToken(true) + task, _, _ := chunkProver.getProverTask(t, message.ProofTypeChunk) + assert.NotNil(t, task) + taskFrom, _, _ := strings.Cut(task.TaskID, ":") + t.Log("get task from coordinator:", taskFrom) + + chunkProver.resetConnection(proxyURL2) + task, _, _ = chunkProver.getProverTask(t, message.ProofTypeChunk) + assert.NotNil(t, task) + taskFrom2, _, _ := strings.Cut(task.TaskID, ":") + assert.Equal(t, taskFrom, taskFrom2) +} + func TestProxyClient(t *testing.T) { testCompatibileMode = false // Set up the test environment. @@ -274,6 +269,7 @@ func TestProxyClient(t *testing.T) { t.Run("TestProxyHandshake", testProxyHandshake) t.Run("TestProxyGetTask", testProxyGetTask) t.Run("TestProxyValidProof", testProxyProof) + t.Run("testProxyPersistent", testProxyPersistent) } func TestProxyClientCompatibleMode(t *testing.T) { @@ -284,4 +280,5 @@ func TestProxyClientCompatibleMode(t *testing.T) { t.Run("TestProxyHandshake", testProxyHandshake) t.Run("TestProxyGetTask", testProxyGetTask) t.Run("TestProxyValidProof", testProxyProof) + t.Run("testProxyPersistent", testProxyPersistent) } diff --git a/database/migrate/migrate.go b/database/migrate/migrate.go index 79f18350af..009159da3d 100644 --- a/database/migrate/migrate.go +++ b/database/migrate/migrate.go @@ -9,13 +9,14 @@ import ( "github.com/pressly/goose/v3" ) -//go:embed migrations/*.sql +//go:embed migrations var embedMigrations embed.FS // MigrationsDir migration dir const MigrationsDir string = "migrations" func init() { + // note goose ignore ono-sql files by default so we do not need to specify *.sql goose.SetBaseFS(embedMigrations) goose.SetSequential(true) goose.SetTableName("scroll_migrations") @@ -24,13 +25,39 @@ func init() { goose.SetVerbose(verbose) } -// FreeMigrate migrate db with specified goose TableName and Dir -func FreeMigrate(db *sql.DB, dir, tableName string) error { - if tableName != "" { - goose.SetTableName(tableName) +// MigrateModule migrate db used by other module with specified goose TableName +// sql file for that module must be put as a sub-directory under `MigrationsDir` +func MigrateModule(db *sql.DB, moduleName string) error { + + goose.SetTableName(moduleName + "_migrations") + defer func() { + goose.SetTableName("scroll_migrations") + }() + + return goose.Up(db, MigrationsDir+"/"+moduleName, goose.WithAllowMissing()) +} + +// RollbackModule rollback the specified module to the given version +func RollbackModule(db *sql.DB, moduleName string, version *int64) error { + + goose.SetTableName(moduleName + "_migrations") + defer func() { + goose.SetTableName("scroll_migrations") + }() + moduleDir := MigrationsDir + "/" + moduleName + + if version != nil { + return goose.DownTo(db, moduleDir, *version) } + return goose.Down(db, moduleDir) +} - return goose.Up(db, dir, goose.WithAllowMissing()) +// ResetModuleDB clean and migrate db for a module. +func ResetModuleDB(db *sql.DB, moduleName string) error { + if err := RollbackModule(db, moduleName, new(int64)); err != nil { + return err + } + return MigrateModule(db, moduleName) } // Migrate migrate db diff --git a/coordinator/internal/controller/proxy/migrate/0001_running_tables.sql b/database/migrate/migrations/proxy/0001_running_tables.sql similarity index 76% rename from coordinator/internal/controller/proxy/migrate/0001_running_tables.sql rename to database/migrate/migrations/proxy/0001_running_tables.sql index 1539d35406..bc2ea1f779 100644 --- a/coordinator/internal/controller/proxy/migrate/0001_running_tables.sql +++ b/database/migrate/migrations/proxy/0001_running_tables.sql @@ -5,10 +5,10 @@ create table prover_sessions public_key TEXT NOT NULL, upstream TEXT NOT NULL, up_token TEXT NOT NULL, - expired TIMESTAMP(0) NOT NULL + expired TIMESTAMP(0) NOT NULL, + constraint uk_prover_sessions_public_key_upstream unique (public_key, upstream) ); -create unique index idx_prover_sessions_public_key on prover_sessions (public_key); create index idx_prover_sessions_expired on prover_sessions (expired); create table priority_upstream @@ -23,9 +23,7 @@ create unique index idx_priority_upstream_public_key on priority_upstream (publi -- +goose Down -- +goose StatementBegin -drop index if exists idx_prover_sessions_public_key; drop index if exists idx_prover_sessions_expired; -drop index if exists idx_priority_upstream_public_key; drop table if exists prover_sessions; drop table if exists priority_upstream; From e7551650b28bf38adab288618875cca23aa0fcd7 Mon Sep 17 00:00:00 2001 From: Ho Date: Thu, 6 Nov 2025 16:08:39 +0900 Subject: [PATCH 42/43] fix concurrent issue --- coordinator/internal/controller/proxy/auth.go | 16 ++++++++++++++-- .../internal/controller/proxy/client_manager.go | 2 +- .../internal/controller/proxy/prover_session.go | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 41c99bb673..3c754846a7 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -1,7 +1,10 @@ package proxy import ( + "context" "fmt" + "sync" + "time" jwt "github.com/appleboy/gin-jwt/v2" "github.com/gin-gonic/gin" @@ -21,6 +24,7 @@ type AuthController struct { proverMgr *ProverManager } +const upstreamConnTimeout = time.Second * 5 const LoginParamCache = "login_param" const ProverTypesKey = "prover_types" const SignatureKey = "prover_signature" @@ -59,10 +63,13 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { session := a.proverMgr.GetOrCreate(loginParam.PublicKey) log.Debug("start handling login", "cli", loginParam.Message.ProverName) + loginCtx, cf := context.WithTimeout(context.Background(), upstreamConnTimeout) + var wg sync.WaitGroup for _, cli := range a.clients { - + wg.Add(1) go func(cli Client) { - if err := session.ProxyLogin(c, cli, &loginParam.LoginParameter); err != nil { + defer wg.Done() + if err := session.ProxyLogin(loginCtx, cli, &loginParam.LoginParameter); err != nil { log.Error("proxy login failed during token cache update", "userKey", loginParam.PublicKey, "upstream", cli.Name(), @@ -70,6 +77,11 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { } }(cli) } + go func(cliName string) { + wg.Wait() + cf() + log.Debug("first login attempt has completed", "cli", cliName) + }(loginParam.Message.ProverName) return loginParam.LoginParameter, nil } diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index b2bf750603..f533c8af08 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -123,10 +123,10 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) { func (cliMgr *ClientManager) Reset(cli *upClient) { cliMgr.cachedCli.Lock() if cliMgr.cachedCli.cli == cli { + log.Info("cached client cleared", "name", cliMgr.name) cliMgr.cachedCli.cli = nil } cliMgr.cachedCli.Unlock() - log.Info("cached client cleared", "name", cliMgr.name) } func (cliMgr *ClientManager) Name() string { diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 17c37425ce..ad0c2c5633 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -162,6 +162,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str } if resp.ErrCode == ctypes.ErrJWTTokenExpired { + log.Info("up stream has expired, renew upstream connection", "up", up) cliMgr.Reset(cli) cli = cliMgr.Client(ctx) if cli == nil { From c22d9ecad14c9943aeb6e12d199addde21d81e4a Mon Sep 17 00:00:00 2001 From: Ho Date: Thu, 6 Nov 2025 16:11:59 +0900 Subject: [PATCH 43/43] fix goimport issue --- common/go.mod | 2 +- coordinator/internal/logic/libzkp/message_types.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/common/go.mod b/common/go.mod index c21dfe2b81..331d258a3f 100644 --- a/common/go.mod +++ b/common/go.mod @@ -12,6 +12,7 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 + github.com/mitchellh/mapstructure v1.5.0 github.com/modern-go/reflect2 v1.0.2 github.com/orcaman/concurrent-map v1.0.0 github.com/prometheus/client_golang v1.19.0 @@ -147,7 +148,6 @@ require ( github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/miekg/pkcs11 v1.1.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect diff --git a/coordinator/internal/logic/libzkp/message_types.go b/coordinator/internal/logic/libzkp/message_types.go index 8ac465e9a6..12012b6334 100644 --- a/coordinator/internal/logic/libzkp/message_types.go +++ b/coordinator/internal/logic/libzkp/message_types.go @@ -2,6 +2,7 @@ package libzkp import ( "fmt" + "scroll-tech/common/types/message" )