-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Yezzey data files vacuum. WAL-G backup integration (#44)
* some changes * working delete garbadge * fixed some cases * added tests * fixes after refactoring
- Loading branch information
1 parent
d73f0a7
commit 403e564
Showing
16 changed files
with
1,070 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| package backups | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "fmt" | ||
| "os/exec" | ||
| "strings" | ||
|
|
||
| "github.com/yezzey-gp/yproxy/pkg/ylogger" | ||
| ) | ||
|
|
||
| type BackupLSN struct { | ||
| Lsn uint64 `json:"LSN"` | ||
| } | ||
|
|
||
| //go:generate mockgen -destination=pkg/mock/backups.go -package=mock | ||
| type BackupInterractor interface { | ||
| GetFirstLSN(int) (uint64, error) | ||
| } | ||
|
|
||
| type WalgBackupInterractor struct { //TODO: rewrite to using s3 instead of wal-g cmd | ||
| } | ||
|
|
||
| // get lsn of the oldest backup | ||
| func (b *WalgBackupInterractor) GetFirstLSN(seg int) (uint64, error) { | ||
| cmd := exec.Command("/usr/bin/wal-g", "st", "ls", fmt.Sprintf("segments_005/seg%d/basebackups_005/", seg), "--config=/etc/wal-g/wal-g.yaml") | ||
| ylogger.Zero.Debug().Any("flags", cmd.Args).Msg("Command args") | ||
| var out bytes.Buffer | ||
| cmd.Stdout = &out | ||
|
|
||
| err := cmd.Run() | ||
| if err != nil { | ||
| ylogger.Zero.Debug().AnErr("error", err).Msg("Failed to run st ls") | ||
| return 0, err | ||
| } | ||
| lines := strings.Split(out.String(), "\n") | ||
|
|
||
| minLSN := BackupLSN{Lsn: ^uint64(0)} | ||
| for _, line := range lines { | ||
| if !strings.Contains(line, ".json") { | ||
| continue | ||
| } | ||
| parts := strings.Split(line, " ") | ||
| fileName := parts[len(parts)-1] | ||
|
|
||
| ylogger.Zero.Debug().Str("file: %s", fmt.Sprintf("segments_005/seg%d/basebackups_005/%s", seg, fileName)).Msg("check lsn in file") | ||
| catCmd := exec.Command("/usr/bin/wal-g", "st", "cat", fmt.Sprintf("segments_005/seg%d/basebackups_005/%s", seg, fileName), "--config=/etc/wal-g/wal-g.yaml") | ||
|
|
||
| var catOut bytes.Buffer | ||
| catCmd.Stdout = &catOut | ||
|
|
||
| err = catCmd.Run() | ||
| if err != nil { | ||
| ylogger.Zero.Debug().AnErr("error", err).Msg("Failed to run st cat") | ||
| return 0, err | ||
| } | ||
| lsn := BackupLSN{} | ||
| err = json.Unmarshal(catOut.Bytes(), &lsn) | ||
|
|
||
| if lsn.Lsn < minLSN.Lsn { | ||
| minLSN.Lsn = lsn.Lsn | ||
| } | ||
| } | ||
|
|
||
| return minLSN.Lsn, err | ||
| } |
Oops, something went wrong.