diff --git a/pkg/diagnostics/db_stat.go b/pkg/diagnostics/db_stat.go new file mode 100644 index 000000000..66b7255d9 --- /dev/null +++ b/pkg/diagnostics/db_stat.go @@ -0,0 +1,189 @@ +package diagnostics + +import ( + "fmt" + "os" + "os/exec" + "time" + + "github.com/noobaa/noobaa-operator/v5/pkg/options" + "github.com/noobaa/noobaa-operator/v5/pkg/util" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +// CollectorDbStatData configuration for diagnostics +type CollectorDbStatData struct { + folderName string // Local folder to store the raw db stat + dbStatDataFolderPath string // Local path of the folder containing the raw stat file + remoteTarPath string // Remote path (at pod) of tar'd db stat + kubeconfig string + kubeCommand string + log *logrus.Entry +} + +func newCollectorDbStatData(kubeconfig string) *CollectorDbStatData { + c := new(CollectorDbStatData) + c.folderName = fmt.Sprintf("%s_%d", "noobaa_db_Statdata", time.Now().Unix()) + c.remoteTarPath = fmt.Sprintf("%s/%s%s", "/tmp", c.folderName, ".tar.gz") + c.log = util.Logger() + c.kubeconfig = kubeconfig + c.kubeCommand = util.GetAvailabeKubeCli() + return c +} + +// RunDBStat generates a tar file of a db Statdata +func RunDBStat(cmd *cobra.Command, args []string) { + kubeconfig, _ := cmd.Flags().GetString("kubeconfig") + destDir, _ := cmd.Flags().GetString("dir") + + CollectStatData(kubeconfig, destDir) +} + +// RunDBStatPrepare generates a tar file of a db Statdata +func RunDBStatPrepare(cmd *cobra.Command, args []string) { + kubeconfig, _ := cmd.Flags().GetString("kubeconfig") + + PrepareStatData(kubeconfig) +} + +func PrepareStatData(kubeconfig string) { + c := newCollectorDbStatData(kubeconfig) + + err := c.prepareDBStatData() + if err == nil { + + } +} + +// CollectStatData exposes the functionality to the diagnostics collect mechanism +func CollectStatData(kubeconfig string, destDir string) { + c := newCollectorDbStatData(kubeconfig) + + c.log.Printf("Collecting db Statdate destDir : %s", destDir) + err := c.generateDBStatData(destDir) + if err == nil { + err = c.exportDBStatdata(destDir) + if err == nil { + tarPath := fmt.Sprintf("%s.tar.gz", c.dbStatDataFolderPath) + c.log.Printf("✅ Generated db stat was saved in %s\n", tarPath) + c.deleteDbRawResources() + } + } +} + +// prepareDBStatData create the stat resources +func (c *CollectorDbStatData) prepareDBStatData() error { + + c.log.Println("Prepare db Statdata at pod") + + // create pg_stat_statements extension and reset + cmd := exec.Command(c.kubeCommand, "exec", "-n", options.Namespace, "pod/noobaa-db-pg-0", "--", "psql", "-c", + "create extension pg_stat_statements", "-c", "select pg_stat_reset()", "-c", "select pg_stat_statements_reset()") + // handle custom path for kubeconfig file, + // see --kubeconfig cli options + if len(c.kubeconfig) > 0 { + cmd.Env = append(cmd.Env, "KUBECONFIG="+c.kubeconfig) + } + + if err := cmd.Run(); err != nil { + c.log.Printf(`❌ cannot generate db Statdata: %v`, err) + return err + } + + return nil +} + +// generateDBStatData create the stat resources +func (c *CollectorDbStatData) generateDBStatData(destDir string) error { + + c.log.Println("Generating db Stat data at pod, destDir : ", destDir) + + // In case of a postgres db, the stat is a single file so in this case we can + // redirect the output of the stat straight to a local file + + // Compose the path of the folder containing the stat file + if destDir != "" { + c.dbStatDataFolderPath = fmt.Sprintf("%s/%s", destDir, c.folderName) + } else { + c.dbStatDataFolderPath = c.folderName + } + + // Create the folder containing the stat file + err := os.Mkdir(c.dbStatDataFolderPath, os.ModePerm) + if err != nil { + c.log.Fatalf(`❌ Could not create directory %s, reason: %s`, c.folderName, err) + return err + } + + statFilePath := fmt.Sprintf("%s/%s.sql", c.dbStatDataFolderPath, c.folderName) + + // Create the stat file + outfile, err := os.Create(statFilePath) + if err != nil { + c.log.Printf(`❌ can not create db stat at path %v: %v`, c.dbStatDataFolderPath, err) + return err + } + + // Redirect the command's output to the local stat file + defer outfile.Close() + + cmd := exec.Command(c.kubeCommand, "exec", "-n", options.Namespace, "pod/noobaa-db-pg-0", "--", "psql", "-c", "SELECT total_plan_time*'1ms'::interval AS total, mean_plan_time*'1ms'::interval AS avg, calls, query from pg_stat_statements ORDER BY total_plan_time DESC LIMIT 10") + if len(c.kubeconfig) > 0 { + cmd.Env = append(cmd.Env, "KUBECONFIG="+c.kubeconfig) + } + cmd.Stdout = outfile + // Execute the command, generating the stat file + if err := cmd.Run(); err != nil { + c.log.Printf(`❌ cannot generate db Statdata: %v`, err) + return err + } + + return nil +} + +// Tar local stat resources +func (c *CollectorDbStatData) exportDBStatdata(destDir string) error { + // In case of a postgres, the stat was created as a local file, so all is left to + // do is tar it + var cmd *exec.Cmd + + // Compose the path of the tar output + tarPath := fmt.Sprintf("%s.tar.gz", c.dbStatDataFolderPath) + + // Genrate the tar command + if destDir != "" { + cmd = exec.Command("tar", "-C", destDir, "-cvzf", tarPath, c.folderName) + } else { + cmd = exec.Command("tar", "-cvzf", tarPath, c.folderName) + } + c.log.Printf("tarPath %s, cmd : %v", tarPath, cmd) + // Execute the tar command + if err := cmd.Run(); err != nil { + c.log.Printf(`❌ failed to tar db stat`) + return err + } + return nil +} + +// deleteDbRawResources stat on Postgres remote machine +func (c *CollectorDbStatData) deleteDbRawResources() { + + // Compose the deletion command + cmd := exec.Command("rm", "-rf", c.dbStatDataFolderPath) + + // Execute the deletion of raw resources + if err := cmd.Run(); err != nil { + c.log.Printf(`❌ failed to tar remove stat folder`) + } + //time.Sleep(2 * time.Second) + + cmd_select_drop := exec.Command(c.kubeCommand, "exec", "-n", options.Namespace, "pod/noobaa-db-pg-0", "--", "psql", "-c", "drop extension pg_stat_statements") + if len(c.kubeconfig) > 0 { + cmd_select_drop.Env = append(cmd_select_drop.Env, "KUBECONFIG="+c.kubeconfig) + } + // Execute the command, delete the extension + if err := cmd_select_drop.Run(); err != nil { + c.log.Printf(`❌ cannot generate db Statdata: %v`, err) + } +} diff --git a/pkg/diagnostics/diagnostics.go b/pkg/diagnostics/diagnostics.go index 215d4deef..79ef0293e 100644 --- a/pkg/diagnostics/diagnostics.go +++ b/pkg/diagnostics/diagnostics.go @@ -25,6 +25,9 @@ func Cmd() *cobra.Command { CmdDbDump(), CmdAnalyze(), CmdReport(), + CmdDbStatPrepare(), + CmdDbStat(), + CmdIntCheck(), ) return cmd } @@ -54,6 +57,29 @@ func CmdDbDump() *cobra.Command { return cmd } +// CmdDbStatPrepare returns a CLI command +func CmdDbStatPrepare() *cobra.Command { + cmd := &cobra.Command{ + Use: "db-stat-prepare", + Short: "Prepare db stat data", + Run: RunDBStatPrepare, + Args: cobra.NoArgs, + } + return cmd +} + +// CmdDbStat returns a CLI command +func CmdDbStat() *cobra.Command { + cmd := &cobra.Command{ + Use: "db-stat", + Short: "Collect db statdata and delete the extension", + Run: RunDBStat, + Args: cobra.NoArgs, + } + cmd.Flags().String("dir", "", "collect db dump file into destination directory") + return cmd +} + // CmdAnalyze returns a CLI command func CmdAnalyze() *cobra.Command { cmd := &cobra.Command{ @@ -151,3 +177,17 @@ func CmdDiagnoseDeprecated() *cobra.Command { cmd.Flags().Bool("db-dump", false, "collect db dump in addition to diagnostics") return cmd } + +func CmdIntCheck() *cobra.Command { + cmd := &cobra.Command{ + Use: "int-check", + Short: "Run db integrity check", + Run: RunIntCheck, + Args: cobra.NoArgs, + } + + cmd.Flags().String("kubeconfig", "", "kubeconfig path") + cmd.Flags().Bool("dump-data-map", false, "if set to true will dump a json object with a full map of object metadata") + cmd.Flags().String("dir", "", "dump dir path") + return cmd +} diff --git a/pkg/diagnostics/intcheck.go b/pkg/diagnostics/intcheck.go new file mode 100644 index 000000000..aeb8c9307 --- /dev/null +++ b/pkg/diagnostics/intcheck.go @@ -0,0 +1,204 @@ +package diagnostics + +import ( + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + "time" + + "github.com/spf13/cobra" + + "github.com/noobaa/noobaa-operator/v5/pkg/options" + "github.com/noobaa/noobaa-operator/v5/pkg/util" +) + +const defaultDBName = "nbcore" + +var dbStatDataFolderPath = "" + +func RunIntCheck(cmd *cobra.Command, args []string) { + kubeconfig, _ := cmd.Flags().GetString("kubeconfig") + dataDump, _ := cmd.Flags().GetBool("dump-data-map") + destDir, _ := cmd.Flags().GetString("dir") + + dbname := os.Getenv("NOOBAA_DB") + if dbname == "" { + dbname = defaultDBName + } + + runIntCheck(kubeconfig, dbname, dataDump, destDir) +} + +func runIntCheck(kubeconfig, dbname string, dataDump bool, destDir string) { + kubecommand := util.GetAvailabeKubeCli() + + var folderName = fmt.Sprintf("%s_%d", "noobaa_db_int_check", time.Now().Unix()) + if destDir != "" { + dbStatDataFolderPath = fmt.Sprintf("%s/%s", destDir, folderName) + } else { + dbStatDataFolderPath = folderName + } + + // Create the folder containing the stat file + err := os.Mkdir(dbStatDataFolderPath, os.ModePerm) + if err != nil { + log.Fatalf(`❌ Could not create directory %s, reason: %s`, folderName, err) + } + + statFilePath := fmt.Sprintf("%s/%s.sql", dbStatDataFolderPath, folderName) + + // Create the stat file + outfile, err := os.Create(statFilePath) + if err != nil { + log.Printf(`❌ can not create db stat at path %v: %v`, dbStatDataFolderPath, err) + } + + // Redirect the command's output to the local stat file + defer outfile.Close() + + objectmds, err := getObjectMds(kubeconfig, kubecommand, dbname, 0) + if err != nil { + log.Fatalln(err) + } + + partsIntegrityIssues := 0 + for _, objectmd := range objectmds { + parts_data, err := getObjectParts(kubeconfig, kubecommand, dbname, objectmd["_id"].(string)) + if err != nil { + log.Fatalln("failed to get object part data for object:", objectmd) + } + + if len(parts_data) != int(objectmd["num_parts"].(float64)) { + partsIntegrityIssues += 1 + log.Printf( + "invalid object parts found for object: %s - expected: %d, got: %d\n", + objectmd["_id"].(string), + int(objectmd["num_parts"].(float64)), + len(parts_data), + ) + } + + objectmd["parts_data"] = parts_data + } + + log.Println("Object Parts integrity issues - ", partsIntegrityIssues) + + chunkIntegrityIssues := 0 + for _, objectmd := range objectmds { + for _, objpart := range objectmd["parts_data"].([]map[string]any) { + chunksdata, err := getChunkData(kubeconfig, kubecommand, dbname, objpart["chunk"].(string)) + if err != nil { + log.Fatalf( + "failed to get chunk data for part %s of object %s\n", + objpart["_id"].(string), + objectmd["_id"].(string), + ) + } + + if len(chunksdata) == 0 { + chunkIntegrityIssues += 1 + log.Println("unexpected chunks 0 for part id -", objpart["_id"].(string)) + } + + fragsIntegrityIssues := 0 + objpart["chunks_data"] = chunksdata + + for _, chunkdata := range chunksdata { + frags, ok := chunkdata["frags"].([]any) + if !ok { + fragsIntegrityIssues += 1 + log.Println("invalid fragment data found for chunk -", chunkdata["_id"]) + } + + fragIDs := []string{} + for _, frag := range frags { + frag := frag.(map[string]any) + fragIDs = append(fragIDs, frag["_id"].(string)) + } + + realFragData, err := getFragsData(kubeconfig, kubecommand, dbname, fragIDs) + if err != nil { + fragsIntegrityIssues += 1 + log.Println("failed to fragment data for chunk - ", chunkdata["_id"]) + continue + } + + if len(realFragData) != len(frags) { + chunkIntegrityIssues += 1 + log.Printf("unexpected frags %v for part id - %s", len(realFragData), objpart["_id"].(string)) + continue + } + } + + log.Println("Object Fragments integrity issues - ", fragsIntegrityIssues) + } + } + + log.Println("Object Chunks integrity issues - ", chunkIntegrityIssues) + + if dataDump { + json.NewEncoder(outfile).Encode(objectmds) + } +} + +func getObjectMds(kubeconfig, kubecommand, dbname string, page int) ([]map[string]any, error) { + sql := fmt.Sprintf("SELECT json_agg(data) FROM objectmds LIMIT 1000 OFFSET %d;", page*1000) + return getJSONDataFromDB(kubeconfig, kubecommand, dbname, sql) +} + +func getObjectParts(kubeconfig, kubecommand, dbname, objectID string) ([]map[string]any, error) { + sql := fmt.Sprintf("SELECT json_agg(data) FROM objectparts WHERE data->>'obj' = '%s';", objectID) + return getJSONDataFromDB(kubeconfig, kubecommand, dbname, sql) +} + +func getChunkData(kubeconfig, kubecommand, dbname, chunkID string) ([]map[string]any, error) { + sql := fmt.Sprintf("SELECT json_agg(data) FROM datachunks WHERE data->>'_id' = '%s';", chunkID) + return getJSONDataFromDB(kubeconfig, kubecommand, dbname, sql) +} + +func getFragsData(kubeconfig, kubecommand, dbname string, frags []string) ([]map[string]any, error) { + ids := "" + for idx, frag := range frags { + ids += ("'" + frag + "'") + if idx != len(frags)-1 { + ids += "," + } + } + + sql := fmt.Sprintf("SELECT json_agg(data) FROM datablocks WHERE data->>'frag' IN (%s);", ids) + return getJSONDataFromDB(kubeconfig, kubecommand, dbname, sql) +} + +func getJSONDataFromDB(kubeconfig, kubecommand, dbname, sql string) ([]map[string]any, error) { + cmd := exec.Command( + kubecommand, + "exec", + "-n", options.Namespace, + "pod/noobaa-db-pg-0", + "--", + "psql", + "-d", + dbname, + "-c", + sql, + "-q", + "-t", + ) + if len(kubeconfig) > 0 { + cmd.Env = append(cmd.Env, "KUBECONFIG="+kubeconfig) + } + + output, err := cmd.Output() + if err != nil { + return nil, err + } + + parsed := []map[string]any{} + if err := json.Unmarshal(output, &parsed); err != nil { + return nil, err + } + + return parsed, err +}