From 821253b7d8c4fc9c598c6e8a8d515465c2974488 Mon Sep 17 00:00:00 2001 From: huangzhiran <30522704+huangzhiran@users.noreply.github.com> Date: Thu, 28 Nov 2024 20:56:13 +0800 Subject: [PATCH] store hex format for binary data in sqlite (#771) --- service/apinode/api/http.go | 4 ++-- service/apinode/db/sqlite.go | 28 +++++++++++------------ service/prover/db/db.go | 43 ++++++++++++++++++++---------------- service/sequencer/db/db.go | 22 ++++++++++-------- 4 files changed, 53 insertions(+), 44 deletions(-) diff --git a/service/apinode/api/http.go b/service/apinode/api/http.go index bd322223..22b3e96d 100644 --- a/service/apinode/api/http.go +++ b/service/apinode/api/http.go @@ -279,7 +279,7 @@ func (s *httpServer) queryTask(c *gin.Context) { resp.States = append(resp.States, &StateLog{ State: "assigned", Time: assignedTask.CreatedAt, - ProverID: "did:io:" + strings.TrimPrefix(assignedTask.Prover.String(), "0x"), + ProverID: "did:io:" + strings.TrimPrefix(assignedTask.Prover, "0x"), }) // Get settled state @@ -300,7 +300,7 @@ func (s *httpServer) queryTask(c *gin.Context) { State: "settled", Time: settledTask.CreatedAt, Comment: "The task has been completed. Please check the generated proof in the corresponding DApp contract.", - Tx: settledTask.Tx.String(), + Tx: settledTask.Tx, }) c.JSON(http.StatusOK, resp) return diff --git a/service/apinode/db/sqlite.go b/service/apinode/db/sqlite.go index 9c3b51dd..bc9a6ab9 100644 --- a/service/apinode/db/sqlite.go +++ b/service/apinode/db/sqlite.go @@ -18,26 +18,26 @@ type scannedBlockNumber struct { type AssignedTask struct { gorm.Model - TaskID common.Hash `gorm:"uniqueIndex:assigned_task_uniq,not null"` - Prover common.Address + TaskID string `gorm:"uniqueIndex:assigned_task_uniq,not null"` + Prover string } type SettledTask struct { gorm.Model - TaskID common.Hash `gorm:"uniqueIndex:settled_task_uniq,not null"` - Tx common.Hash `gorm:"not null"` + TaskID string `gorm:"uniqueIndex:settled_task_uniq,not null"` + Tx string `gorm:"not null"` } type ProjectDevice struct { gorm.Model - ProjectID string `gorm:"uniqueIndex:project_device_uniq,not null"` - DeviceAddress common.Address `gorm:"uniqueIndex:project_device_uniq,not null"` + ProjectID string `gorm:"uniqueIndex:project_device_uniq,not null"` + DeviceAddress string `gorm:"uniqueIndex:project_device_uniq,not null"` } func (p *DB) UpsertAssignedTask(taskID common.Hash, prover common.Address) error { t := AssignedTask{ - TaskID: taskID, - Prover: prover, + TaskID: taskID.Hex(), + Prover: prover.Hex(), } err := p.sqlite.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "task_id"}}, @@ -48,8 +48,8 @@ func (p *DB) UpsertAssignedTask(taskID common.Hash, prover common.Address) error func (p *DB) UpsertSettledTask(taskID, tx common.Hash) error { t := SettledTask{ - TaskID: taskID, - Tx: tx, + TaskID: taskID.Hex(), + Tx: tx.Hex(), } err := p.sqlite.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "task_id"}}, @@ -60,7 +60,7 @@ func (p *DB) UpsertSettledTask(taskID, tx common.Hash) error { func (p *DB) FetchAssignedTask(taskID common.Hash) (*AssignedTask, error) { t := AssignedTask{} - if err := p.sqlite.Where("task_id = ?", taskID).First(&t).Error; err != nil { + if err := p.sqlite.Where("task_id = ?", taskID.Hex()).First(&t).Error; err != nil { if err == gorm.ErrRecordNotFound { return nil, nil } @@ -71,7 +71,7 @@ func (p *DB) FetchAssignedTask(taskID common.Hash) (*AssignedTask, error) { func (p *DB) FetchSettledTask(taskID common.Hash) (*SettledTask, error) { t := SettledTask{} - if err := p.sqlite.Where("task_id = ?", taskID).First(&t).Error; err != nil { + if err := p.sqlite.Where("task_id = ?", taskID.Hex()).First(&t).Error; err != nil { if err == gorm.ErrRecordNotFound { return nil, nil } @@ -83,7 +83,7 @@ func (p *DB) FetchSettledTask(taskID common.Hash) (*SettledTask, error) { func (p *DB) UpsertProjectDevice(projectID *big.Int, address common.Address) error { t := ProjectDevice{ ProjectID: projectID.String(), - DeviceAddress: address, + DeviceAddress: address.Hex(), } err := p.sqlite.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "project_id"}, {Name: "device_address"}}, @@ -94,7 +94,7 @@ func (p *DB) UpsertProjectDevice(projectID *big.Int, address common.Address) err func (p *DB) IsDeviceApproved(projectID *big.Int, address common.Address) (bool, error) { t := ProjectDevice{} - if err := p.sqlite.Where("project_id = ?", projectID.String()).Where("device_address = ?", address).First(&t).Error; err != nil { + if err := p.sqlite.Where("project_id = ?", projectID.String()).Where("device_address = ?", address.Hex()).First(&t).Error; err != nil { if err == gorm.ErrRecordNotFound { return false, nil } diff --git a/service/prover/db/db.go b/service/prover/db/db.go index a90a0c7f..676f1902 100644 --- a/service/prover/db/db.go +++ b/service/prover/db/db.go @@ -6,6 +6,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/pkg/errors" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -20,23 +21,23 @@ type scannedBlockNumber struct { type project struct { gorm.Model - ProjectID string `gorm:"uniqueIndex:project_id_project,not null"` - URI string `gorm:"not null"` - Hash common.Hash `gorm:"not null"` + ProjectID string `gorm:"uniqueIndex:project_id_project,not null"` + URI string `gorm:"not null"` + Hash string `gorm:"not null"` } type projectFile struct { gorm.Model - ProjectID string `gorm:"uniqueIndex:project_id_project_file,not null"` - File []byte `gorm:"not null"` - Hash common.Hash `gorm:"not null"` + ProjectID string `gorm:"uniqueIndex:project_id_project_file,not null"` + File string `gorm:"not null"` + Hash string `gorm:"not null"` } type task struct { gorm.Model - TaskID common.Hash `gorm:"uniqueIndex:task_uniq,not null"` - Processed bool `gorm:"index:unprocessed_task,not null,default:false"` - Error string `gorm:"not null,default:''"` + TaskID string `gorm:"uniqueIndex:task_uniq,not null"` + Processed bool `gorm:"index:unprocessed_task,not null,default:false"` + Error string `gorm:"not null,default:''"` } type DB struct { @@ -74,14 +75,14 @@ func (p *DB) Project(projectID *big.Int) (string, common.Hash, error) { if err := p.db.Where("project_id = ?", projectID.String()).First(&t).Error; err != nil { return "", common.Hash{}, errors.Wrap(err, "failed to query project") } - return t.URI, t.Hash, nil + return t.URI, common.HexToHash(t.Hash), nil } func (p *DB) UpsertProject(projectID *big.Int, uri string, hash common.Hash) error { t := project{ ProjectID: projectID.String(), URI: uri, - Hash: hash, + Hash: hash.Hex(), } err := p.db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "project_id"}}, @@ -98,14 +99,18 @@ func (p *DB) ProjectFile(projectID *big.Int) ([]byte, common.Hash, error) { } return nil, common.Hash{}, errors.Wrap(err, "failed to query project file") } - return t.File, t.Hash, nil + f, err := hexutil.Decode(t.File) + if err != nil { + return nil, common.Hash{}, errors.Wrap(err, "failed to decode file from hex format") + } + return f, common.HexToHash(t.Hash), nil } func (p *DB) UpsertProjectFile(projectID *big.Int, file []byte, hash common.Hash) error { t := projectFile{ ProjectID: projectID.String(), - File: file, - Hash: hash, + File: hexutil.Encode(file), + Hash: hash.Hex(), } err := p.db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "project_id"}}, @@ -119,7 +124,7 @@ func (p *DB) CreateTask(taskID common.Hash, prover common.Address) error { return nil } t := &task{ - TaskID: taskID, + TaskID: taskID.Hex(), Processed: false, } err := p.db.Clauses(clause.OnConflict{ @@ -136,18 +141,18 @@ func (p *DB) ProcessTask(taskID common.Hash, err error) error { if err != nil { t.Error = err.Error() } - err = p.db.Model(t).Where("task_id = ?", taskID).Updates(t).Error + err = p.db.Model(t).Where("task_id = ?", taskID.Hex()).Updates(t).Error return errors.Wrap(err, "failed to update task") } func (p *DB) DeleteTask(taskID, tx common.Hash) error { - err := p.db.Where("task_id = ?", taskID).Delete(&task{}).Error + err := p.db.Where("task_id = ?", taskID.Hex()).Delete(&task{}).Error return errors.Wrap(err, "failed to delete task") } func (p *DB) ProcessedTask(taskID common.Hash) (bool, string, time.Time, error) { t := task{} - if err := p.db.Where("task_id = ?", taskID).First(&t).Error; err != nil { + if err := p.db.Where("task_id = ?", taskID.Hex()).First(&t).Error; err != nil { if err == gorm.ErrRecordNotFound { return false, "", time.Now(), nil } @@ -164,7 +169,7 @@ func (p *DB) UnprocessedTask() (common.Hash, error) { } return common.Hash{}, errors.Wrap(err, "failed to query unprocessed task") } - return t.TaskID, nil + return common.HexToHash(t.TaskID), nil } func New(localDBDir string, prover common.Address) (*DB, error) { diff --git a/service/sequencer/db/db.go b/service/sequencer/db/db.go index 42eca0d8..f28c3633 100644 --- a/service/sequencer/db/db.go +++ b/service/sequencer/db/db.go @@ -18,13 +18,13 @@ type scannedBlockNumber struct { type prover struct { gorm.Model - Prover common.Address `gorm:"uniqueIndex:prover,not null"` + Prover string `gorm:"uniqueIndex:prover,not null"` } type task struct { gorm.Model - TaskID common.Hash `gorm:"uniqueIndex:task_uniq,not null"` - Assigned bool `gorm:"index:unassigned_task,not null,default:false"` + TaskID string `gorm:"uniqueIndex:task_uniq,not null"` + Assigned bool `gorm:"index:unassigned_task,not null,default:false"` } type DB struct { @@ -63,7 +63,7 @@ func (p *DB) Provers() ([]common.Address, error) { } res := make([]common.Address, 0, len(ts)) for _, t := range ts { - res = append(res, t.Prover) + res = append(res, common.HexToAddress(t.Prover)) } metrics.ProverMtc.Set(float64(len(ts))) return res, nil @@ -71,7 +71,7 @@ func (p *DB) Provers() ([]common.Address, error) { func (p *DB) UpsertProver(addr common.Address) error { t := prover{ - Prover: addr, + Prover: addr.Hex(), } err := p.db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "prover"}}, @@ -82,7 +82,7 @@ func (p *DB) UpsertProver(addr common.Address) error { func (p *DB) CreateTask(taskID common.Hash) error { t := &task{ - TaskID: taskID, + TaskID: taskID.Hex(), Assigned: false, } if err := p.db.Create(t).Error; err != nil { @@ -95,15 +95,19 @@ func (p *DB) AssignTasks(ids []common.Hash) error { if len(ids) == 0 { return nil } + sids := make([]string, 0, len(ids)) + for _, id := range ids { + sids = append(sids, id.Hex()) + } t := &task{ Assigned: true, } - err := p.db.Model(t).Where("task_id IN ?", ids).Updates(t).Error + err := p.db.Model(t).Where("task_id IN ?", sids).Updates(t).Error return errors.Wrap(err, "failed to assign tasks") } func (p *DB) DeleteTask(taskID, tx common.Hash) error { - err := p.db.Where("task_id = ?", taskID).Delete(&task{}).Error + err := p.db.Where("task_id = ?", taskID.Hex()).Delete(&task{}).Error return errors.Wrap(err, "failed to delete task") } @@ -114,7 +118,7 @@ func (p *DB) UnassignedTasks(limit int) ([]common.Hash, error) { } ids := []common.Hash{} for _, t := range ts { - ids = append(ids, t.TaskID) + ids = append(ids, common.HexToHash(t.TaskID)) } return ids, nil }