From 28b41334e4757a8037a4d541e2dc3def18517f9f Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 2 Oct 2024 14:20:35 +0000 Subject: [PATCH 1/2] Use uint64 in delete msg --- cmd/client/main.go | 8 ++++---- pkg/backups/backups.go | 4 ++-- pkg/database/database.go | 8 ++++---- pkg/message/delete_message.go | 10 +++++----- pkg/message/message_test.go | 4 ++-- pkg/mock/backups.go | 10 +++++----- pkg/mock/database.go | 12 ++++++------ 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/cmd/client/main.go b/cmd/client/main.go index 9463b67..3e0a9ed 100644 --- a/cmd/client/main.go +++ b/cmd/client/main.go @@ -30,8 +30,8 @@ var ( offset uint64 - segmentPort int - segmentNum int + segmentPort uint64 + segmentNum uint64 confirm bool garbage bool ) @@ -367,8 +367,8 @@ func init() { rootCmd.AddCommand(listCmd) rootCmd.AddCommand(goolCmd) - deleteCmd.PersistentFlags().IntVarP(&segmentPort, "port", "p", 6000, "port that segment is listening on") - deleteCmd.PersistentFlags().IntVarP(&segmentNum, "segnum", "s", 0, "logical number of a segment") + deleteCmd.PersistentFlags().Uint64VarP(&segmentPort, "port", "p", 6000, "port that segment is listening on") + deleteCmd.PersistentFlags().Uint64VarP(&segmentNum, "segnum", "s", 0, "logical number of a segment") deleteCmd.PersistentFlags().BoolVarP(&confirm, "confirm", "", false, "confirm deletion") deleteCmd.PersistentFlags().BoolVarP(&garbage, "garbage", "g", false, "delete garbage") rootCmd.AddCommand(deleteCmd) diff --git a/pkg/backups/backups.go b/pkg/backups/backups.go index 6245ad1..35fc5c5 100644 --- a/pkg/backups/backups.go +++ b/pkg/backups/backups.go @@ -16,14 +16,14 @@ type BackupLSN struct { //go:generate mockgen -destination=pkg/mock/backups.go -package=mock type BackupInterractor interface { - GetFirstLSN(int) (uint64, error) + GetFirstLSN(seg uint64) (uint64, error) } type WalgBackupInterractor struct { //TODO: rewrite to using s3 instead of wal-g cmd } // get lsn of the oldest backup -func (b *WalgBackupInterractor) GetFirstLSN(seg int) (uint64, error) { +func (b *WalgBackupInterractor) GetFirstLSN(seg uint64) (uint64, error) { cmd := exec.Command("/usr/bin/wal-g", "st", "ls", fmt.Sprintf("segments_005/seg%d/basebackups_005/", seg), "--config=/etc/wal-g/wal-g.yaml") ylogger.Zero.Debug().Any("flags", cmd.Args).Msg("Command args") var out bytes.Buffer diff --git a/pkg/database/database.go b/pkg/database/database.go index a57a031..6b9ebe2 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -12,7 +12,7 @@ import ( //go:generate mockgen -destination=../mock/mock_database_interractor.go -package mock type DatabaseInterractor interface { - GetVirtualExpireIndexes(int) (map[string]bool, map[string]uint64, error) + GetVirtualExpireIndexes(port uint64) (map[string]bool, map[string]uint64, error) } type DatabaseHandler struct { @@ -31,7 +31,7 @@ type Ei struct { fqnmd5 string } -func (database *DatabaseHandler) GetVirtualExpireIndexes(port int) (map[string]bool, map[string]uint64, error) { //TODO несколько баз +func (database *DatabaseHandler) GetVirtualExpireIndexes(port uint64) (map[string]bool, map[string]uint64, error) { //TODO несколько баз db, err := getDatabase(port) if err != nil { return nil, nil, fmt.Errorf("unable to get ao/aocs tables %v", err) //fix @@ -95,7 +95,7 @@ func (database *DatabaseHandler) GetVirtualExpireIndexes(port int) (map[string]b return c2, c, err } -func getDatabase(port int) (DB, error) { +func getDatabase(port uint64) (DB, error) { conn, err := connectToDatabase(port, "postgres") if err != nil { return DB{}, err @@ -154,7 +154,7 @@ func getDatabase(port int) (DB, error) { return DB{}, fmt.Errorf("no yezzey schema across databases") } -func connectToDatabase(port int, database string) (*pgx.Conn, error) { +func connectToDatabase(port uint64, database string) (*pgx.Conn, error) { config, err := pgx.ParseEnvLibpq() if err != nil { return nil, errors.Wrap(err, "Connect: unable to read environment variables") diff --git a/pkg/message/delete_message.go b/pkg/message/delete_message.go index 599ba1c..596e9c7 100644 --- a/pkg/message/delete_message.go +++ b/pkg/message/delete_message.go @@ -7,15 +7,15 @@ import ( type DeleteMessage struct { //seg port Name string - Port int - Segnum int + Port uint64 + Segnum uint64 Confirm bool Garbage bool } var _ ProtoMessage = &DeleteMessage{} -func NewDeleteMessage(name string, port int, seg int, confirm bool, garbage bool) *DeleteMessage { +func NewDeleteMessage(name string, port uint64, seg uint64, confirm bool, garbage bool) *DeleteMessage { return &DeleteMessage{ Name: name, Port: port, @@ -65,8 +65,8 @@ func (c *DeleteMessage) Decode(body []byte) { c.Garbage = true } c.Name = c.GetDeleteName(body[4:]) - c.Port = int(binary.BigEndian.Uint64(body[len(body)-16 : len(body)-8])) - c.Segnum = int(binary.BigEndian.Uint64(body[len(body)-8:])) + c.Port = binary.BigEndian.Uint64(body[len(body)-16 : len(body)-8]) + c.Segnum = binary.BigEndian.Uint64(body[len(body)-8:]) } func (c *DeleteMessage) GetDeleteName(b []byte) string { diff --git a/pkg/message/message_test.go b/pkg/message/message_test.go index 096b88a..1461bfd 100644 --- a/pkg/message/message_test.go +++ b/pkg/message/message_test.go @@ -297,8 +297,8 @@ func TestDeleteMsg(t *testing.T) { msg2.Decode(body[8:]) assert.Equal("myname/mynextname", msg2.Name) - assert.Equal(5432, msg2.Port) - assert.Equal(42, msg2.Segnum) + assert.Equal(uint64(5432), msg2.Port) + assert.Equal(uint64(42), msg2.Segnum) assert.True(msg2.Confirm) assert.True(msg2.Garbage) } diff --git a/pkg/mock/backups.go b/pkg/mock/backups.go index 8ec8647..6172b3c 100644 --- a/pkg/mock/backups.go +++ b/pkg/mock/backups.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -source=pkg/backups/backups.go -destination=pkg/mock/backups.go -package=mock +// mockgen -destination=./pkg/mock/backups.go -source=pkg/backups/backups.go -package mock // // Package mock is a generated GoMock package. @@ -39,16 +39,16 @@ func (m *MockBackupInterractor) EXPECT() *MockBackupInterractorMockRecorder { } // GetFirstLSN mocks base method. -func (m *MockBackupInterractor) GetFirstLSN(arg0 int) (uint64, error) { +func (m *MockBackupInterractor) GetFirstLSN(seg uint64) (uint64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetFirstLSN", arg0) + ret := m.ctrl.Call(m, "GetFirstLSN", seg) ret0, _ := ret[0].(uint64) ret1, _ := ret[1].(error) return ret0, ret1 } // GetFirstLSN indicates an expected call of GetFirstLSN. -func (mr *MockBackupInterractorMockRecorder) GetFirstLSN(arg0 any) *gomock.Call { +func (mr *MockBackupInterractorMockRecorder) GetFirstLSN(seg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFirstLSN", reflect.TypeOf((*MockBackupInterractor)(nil).GetFirstLSN), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFirstLSN", reflect.TypeOf((*MockBackupInterractor)(nil).GetFirstLSN), seg) } diff --git a/pkg/mock/database.go b/pkg/mock/database.go index 04408c2..93e768a 100644 --- a/pkg/mock/database.go +++ b/pkg/mock/database.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: pkg/database/database.go +// Source: ./pkg/database/database.go // // Generated by this command: // -// mockgen -source=pkg/database/database.go -destination=pkg/mock/database.go -package=mock +// mockgen -destination=./pkg/mock/database.go -source=./pkg/database/database.go -package mock // // Package mock is a generated GoMock package. @@ -39,9 +39,9 @@ func (m *MockDatabaseInterractor) EXPECT() *MockDatabaseInterractorMockRecorder } // GetVirtualExpireIndexes mocks base method. -func (m *MockDatabaseInterractor) GetVirtualExpireIndexes(arg0 int) (map[string]bool, map[string]uint64, error) { +func (m *MockDatabaseInterractor) GetVirtualExpireIndexes(port uint64) (map[string]bool, map[string]uint64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetVirtualExpireIndexes", arg0) + ret := m.ctrl.Call(m, "GetVirtualExpireIndexes", port) ret0, _ := ret[0].(map[string]bool) ret1, _ := ret[1].(map[string]uint64) ret2, _ := ret[2].(error) @@ -49,7 +49,7 @@ func (m *MockDatabaseInterractor) GetVirtualExpireIndexes(arg0 int) (map[string] } // GetVirtualExpireIndexes indicates an expected call of GetVirtualExpireIndexes. -func (mr *MockDatabaseInterractorMockRecorder) GetVirtualExpireIndexes(arg0 any) *gomock.Call { +func (mr *MockDatabaseInterractorMockRecorder) GetVirtualExpireIndexes(port any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVirtualExpireIndexes", reflect.TypeOf((*MockDatabaseInterractor)(nil).GetVirtualExpireIndexes), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVirtualExpireIndexes", reflect.TypeOf((*MockDatabaseInterractor)(nil).GetVirtualExpireIndexes), port) } From bbc46df70cadae70a9cdea08c49eee8c68492781 Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 2 Oct 2024 17:05:25 +0000 Subject: [PATCH 2/2] Several fixes for DELETE API --- pkg/proc/interaction.go | 15 ++++++++++++--- pkg/storage/s3storage.go | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pkg/proc/interaction.go b/pkg/proc/interaction.go index a360d25..ee9c5de 100644 --- a/pkg/proc/interaction.go +++ b/pkg/proc/interaction.go @@ -398,6 +398,8 @@ func ProcConn(s storage.StorageInteractor, cr crypt.Crypter, ycl client.YproxyCl BackupInterractor: backupHandler, } + ylogger.Zero.Debug().Str("Name", msg.Name).Bool("garb", msg.Garbage).Bool("confirm", msg.Confirm).Msg("requested to remove external chunk") + if msg.Garbage { err = dh.HandleDeleteGarbage(msg) if err != nil { @@ -416,10 +418,17 @@ func ProcConn(s storage.StorageInteractor, cr crypt.Crypter, ycl client.YproxyCl _ = ycl.ReplyError(err, "failed to upload") return err } - ylogger.Zero.Info().Msg("Deleted garbage successfully") - if !msg.Confirm { - ylogger.Zero.Warn().Msg("It was a dry-run, nothing was deleted") + + if msg.Garbage { + if !msg.Confirm { + ylogger.Zero.Warn().Msg("It was a dry-run, nothing was deleted") + } else { + ylogger.Zero.Info().Msg("Deleted garbage successfully") + } + } else { + ylogger.Zero.Info().Msg("Deleted chunk successfully") } + case message.MessageTypeGool: return ProcMotion(s, cr, ycl) diff --git a/pkg/storage/s3storage.go b/pkg/storage/s3storage.go index 6c5c14c..5ab15cf 100644 --- a/pkg/storage/s3storage.go +++ b/pkg/storage/s3storage.go @@ -181,6 +181,6 @@ func (s *S3StorageInteractor) DeleteObject(key string) error { ylogger.Zero.Err(err).Msg("failed to delete old object") return err } - ylogger.Zero.Debug().Msg("deleted object") + ylogger.Zero.Debug().Str("path", key).Msg("deleted object") return nil }