From 112a644cc4f7349ea11938b0b12d679664e44d17 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Mon, 4 Mar 2024 21:22:39 -0500 Subject: [PATCH] Attempting to work around RAM limits I wonder if the probelem in cgov is RAM limits in a container? --- cmd/bucket.go | 81 ++++++++++++++++++++++++++------------- cmd/clone.go | 18 ++++++++- cmd/root.go | 1 + config.json | 27 ++++++++++++- internal/pipes/mc.go | 15 ++++++-- internal/pipes/pg_dump.go | 36 ++++++++++++++++- internal/pipes/psql.go | 5 ++- internal/pipes/s3.go | 11 +++++- internal/util/db.go | 40 +++++++++++++++++++ internal/vcap/vcap.go | 1 + 10 files changed, 198 insertions(+), 37 deletions(-) create mode 100644 internal/util/db.go diff --git a/cmd/bucket.go b/cmd/bucket.go index 55198dd..fa97bd3 100644 --- a/cmd/bucket.go +++ b/cmd/bucket.go @@ -11,34 +11,60 @@ import ( "github.com/spf13/cobra" "gov.gsa.fac.cgov-util/internal/logging" "gov.gsa.fac.cgov-util/internal/pipes" + "gov.gsa.fac.cgov-util/internal/util" + vcap "gov.gsa.fac.cgov-util/internal/vcap" ) -func bucket_local(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) { - mc_pipe := pipes.Mc( - pipes.PG_Dump(source_creds), - up, - "LOCAL", - "local_db", - ) - mc_pipe.Wait() - if err := mc_pipe.Error(); err != nil { - logging.Logger.Println("BACKUPS `dump | mc` pipe failed") - os.Exit(-1) +var backup_tag string + +// func bucket_local(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) { +// mc_pipe := pipes.Mc( +// pipes.PG_Dump(source_creds), +// up, +// "LOCAL", +// "local_db", +// ) +// mc_pipe.Wait() +// if err := mc_pipe.Error(); err != nil { +// logging.Logger.Println("BACKUPS `dump | mc` pipe failed") +// os.Exit(-1) +// } +// } + +func bucket_local_tables(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) { + table_to_schema := util.Get_table_and_schema_names(source_creds) + for table, schema := range table_to_schema { + mc_pipe := pipes.Mc( + pipes.PG_Dump_Table(source_creds, schema, table, Debug), + up, + backup_tag, + source_creds.DB_Name, + schema, table, Debug, + ) + mc_pipe.Wait() + if err := mc_pipe.Error(); err != nil { + logging.Logger.Println("BACKUPS `dump | mc` pipe failed") + os.Exit(-1) + } } } -func bucket_cgov(source_creds *vcap.CredentialsRDS, up *vcap.CredentialsS3) { - s3_pipe := pipes.S3( - pipes.PG_Dump(source_creds), - up, - "LOCAL", - "local_db", - ) - s3_pipe.Wait() - if err := s3_pipe.Error(); err != nil { - logging.Logger.Println("BACKUPS `dump | s3` pipe failed") - os.Exit(-1) +func bucket_cgov_tables(source_creds *vcap.CredentialsRDS, up *vcap.CredentialsS3) { + table_to_schema := util.Get_table_and_schema_names(source_creds) + for table, schema := range table_to_schema { + s3_pipe := pipes.S3( + pipes.PG_Dump_Table(source_creds, schema, table, Debug), + up, + backup_tag, + source_creds.DB_Name, + schema, table, Debug, + ) + s3_pipe.Wait() + if err := s3_pipe.Error(); err != nil { + logging.Logger.Println("BACKUPS `dump | s3` pipe failed") + os.Exit(-1) + } } } @@ -56,10 +82,10 @@ to quickly create a Cobra application.`, source_creds, _ := vcap.GetRDSCreds(SourceDB, "") if slices.Contains([]string{"LOCAL", "TESTING"}, os.Getenv("ENV")) { up, _ := vcap.GetUserProvidedCredentials("mc") - bucket_local(source_creds, up) + bucket_local_tables(source_creds, up) } else { up, _ := vcap.GetS3Credentials(DestinationBucket) - bucket_cgov(source_creds, up) + bucket_cgov_tables(source_creds, up) } }, @@ -69,7 +95,10 @@ func init() { rootCmd.AddCommand(bucketCmd) bucketCmd.Flags().StringVarP(&SourceDB, "source-db", "", "", "source database (req)") bucketCmd.Flags().StringVarP(&DestinationBucket, "destination-bucket", "", "", "destination database (req)") - cloneCmd.MarkFlagRequired("source-db") - cloneCmd.MarkFlagRequired("destination-bucket") + bucketCmd.Flags().StringVarP(&backup_tag, "backup-tag", "", "", "SNAPSHOT, HOURLY-03, etc. (req)") + bucketCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "Log debug statements") + bucketCmd.MarkFlagRequired("source-db") + bucketCmd.MarkFlagRequired("destination-bucket") + bucketCmd.MarkFlagRequired("backup_tag") } diff --git a/cmd/clone.go b/cmd/clone.go index 5ef8c98..d5b5a98 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -9,13 +9,14 @@ import ( "github.com/spf13/cobra" "gov.gsa.fac.cgov-util/internal/logging" "gov.gsa.fac.cgov-util/internal/pipes" + "gov.gsa.fac.cgov-util/internal/util" vcap "gov.gsa.fac.cgov-util/internal/vcap" _ "github.com/lib/pq" ) func clone(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS) { - psql_pipe := pipes.Psql(pipes.PG_Dump(source), dest) + psql_pipe := pipes.Psql(pipes.PG_Dump(source, Debug), dest, Debug) psql_pipe.Wait() if err := psql_pipe.Error(); err != nil { logging.Logger.Println("BACKUPS Pipe failed") @@ -23,6 +24,18 @@ func clone(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS) { } } +func clone_tables(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS) { + table_to_schema := util.Get_table_and_schema_names(source) + for table, schema := range table_to_schema { + psql_pipe := pipes.Psql(pipes.PG_Dump_Table(source, schema, table, Debug), dest, Debug) + psql_pipe.Wait() + if err := psql_pipe.Error(); err != nil { + logging.Logger.Printf("BACKUPS Pipe failed for %s, %s\n", schema, table) + os.Exit(-1) + } + } +} + // snapshotDbToDbCmd represents the snapshotDbToDb command var cloneCmd = &cobra.Command{ Use: "clone", @@ -37,7 +50,7 @@ writes to a snapshot clone DB. `, Run: func(cmd *cobra.Command, args []string) { source_creds, dest_creds := vcap.GetRDSCreds(SourceDB, DestinationDB) - clone(source_creds, dest_creds) + clone_tables(source_creds, dest_creds) }, } @@ -45,6 +58,7 @@ func init() { rootCmd.AddCommand(cloneCmd) cloneCmd.Flags().StringVarP(&SourceDB, "source-db", "", "", "source database (req)") cloneCmd.Flags().StringVarP(&DestinationDB, "destination-db", "", "", "destination database (req)") + cloneCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "Log debug statements") cloneCmd.MarkFlagRequired("source-db") cloneCmd.MarkFlagRequired("destination-db") diff --git a/cmd/root.go b/cmd/root.go index 228578f..c95d769 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,6 +15,7 @@ var ( DestinationDB string DestinationBucket string SHA1 string + Debug bool // rootCmd represents the base command when called without any subcommands rootCmd = &cobra.Command{ diff --git a/config.json b/config.json index 7664302..ff16c47 100644 --- a/config.json +++ b/config.json @@ -38,7 +38,32 @@ "name": "postgres", "password": "", "port": "5432", - "uri": "postgres://USERNAMEINDIA:PASSWORDINDIA@host.us-gov-india-1.rds.amazonaws.com:5432/DBNAMEINDIA", + "uri": "postgres://postgres@127.0.0.10:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "fac-snapshot-db", + "provider": null, + "plan": null, + "name": "fac-snapshot-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDJULIET1", + "instance_name": "db", + "binding_guid": "UUIDJULIET2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "127.0.0.1", + "name": "postgres", + "password": "", + "port": "6543", + "uri": "postgres://postgres@127.0.0.10:6543/postgres?sslmode=disable", "username": "postgres" }, "syslog_drain_url": null, diff --git a/internal/pipes/mc.go b/internal/pipes/mc.go index 6a027c1..9919e08 100644 --- a/internal/pipes/mc.go +++ b/internal/pipes/mc.go @@ -12,7 +12,12 @@ import ( ) // https://bitfieldconsulting.com/golang/scripting -func Mc(in_pipe *script.Pipe, upc vcap.UserProvidedCredentials, prefix string, source_db string) *script.Pipe { +func Mc(in_pipe *script.Pipe, + upc vcap.UserProvidedCredentials, + prefix string, + source_db string, + schema string, + table string, debug bool) *script.Pipe { // // mc pipe myminio/gsa-fac-private-s3/backups/${PREFIX}-${FROM_DATABASE}.dump // Always set the alias first. os.Setenv("AWS_PRIVATE_ACCESS_KEY_ID", upc["access_key_id"]) @@ -33,14 +38,18 @@ func Mc(in_pipe *script.Pipe, upc vcap.UserProvidedCredentials, prefix string, s cmd := []string{ "mc", "pipe", - fmt.Sprintf("%s/%s/backups/%s-%s.dump", + fmt.Sprintf("%s/%s/backups/%s-%s-%s_%s.dump", minio_alias, upc["bucket"], prefix, - source_db), + source_db, + schema, table), } // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") + if debug { + fmt.Printf("command: %s\n", combined) + } logging.Logger.Printf("BACKUPS mc targeting %s", prefix) return in_pipe.Exec(combined) } diff --git a/internal/pipes/pg_dump.go b/internal/pipes/pg_dump.go index 4d5386a..8fc070b 100644 --- a/internal/pipes/pg_dump.go +++ b/internal/pipes/pg_dump.go @@ -9,8 +9,38 @@ import ( "gov.gsa.fac.cgov-util/internal/vcap" ) +func PG_Dump_Table(creds *vcap.CredentialsRDS, schema string, table string, debug bool) *script.Pipe { + // Compose the command as a slice + cmd := []string{ + "pg_dump", + "--clean", + "--no-password", + "--if-exists", + "--no-privileges", + "--no-owner", + "--format plain", + "--table", + fmt.Sprintf("%s.%s", schema, table), + "--dbname", + fmt.Sprintf("postgres://%s:%s@%s:%s/%s", + creds.Username, + creds.Password, + creds.Host, + creds.Port, + creds.DB_Name, + ), + } + // Combine the slice for printing and execution. + combined := strings.Join(cmd[:], " ") + if debug { + fmt.Printf("command: %s\n", combined) + } + logging.Logger.Printf("BACKUPS pg_dump targeting %s\n", creds.DB_Name) + return script.Exec(combined) +} + // https://bitfieldconsulting.com/golang/scripting -func PG_Dump(creds *vcap.CredentialsRDS) *script.Pipe { +func PG_Dump(creds *vcap.CredentialsRDS, debug bool) *script.Pipe { // Compose the command as a slice cmd := []string{ "pg_dump", @@ -31,7 +61,9 @@ func PG_Dump(creds *vcap.CredentialsRDS) *script.Pipe { } // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") - fmt.Printf("command: %s\n", combined) + if debug { + fmt.Printf("command: %s\n", combined) + } logging.Logger.Printf("BACKUPS pg_dump targeting %s\n", creds.DB_Name) return script.Exec(combined) } diff --git a/internal/pipes/psql.go b/internal/pipes/psql.go index dedad73..c6a7463 100644 --- a/internal/pipes/psql.go +++ b/internal/pipes/psql.go @@ -9,7 +9,7 @@ import ( "gov.gsa.fac.cgov-util/internal/vcap" ) -func Psql(in_pipe *script.Pipe, creds *vcap.CredentialsRDS) *script.Pipe { +func Psql(in_pipe *script.Pipe, creds *vcap.CredentialsRDS, debug bool) *script.Pipe { cmd := []string{ "psql", "--no-password", @@ -23,6 +23,9 @@ func Psql(in_pipe *script.Pipe, creds *vcap.CredentialsRDS) *script.Pipe { ), } combined := strings.Join(cmd[:], " ") + if debug { + logging.Logger.Printf("command: %s\n", combined) + } logging.Logger.Printf("BACKUPS psql targeting %s\n", creds.DB_Name) return in_pipe.Exec(combined) } diff --git a/internal/pipes/s3.go b/internal/pipes/s3.go index 2816796..e7d93ec 100644 --- a/internal/pipes/s3.go +++ b/internal/pipes/s3.go @@ -11,7 +11,11 @@ import ( ) // https://bitfieldconsulting.com/golang/scripting -func S3(in_pipe *script.Pipe, up *vcap.CredentialsS3, prefix string, source_db string) *script.Pipe { +func S3(in_pipe *script.Pipe, + up *vcap.CredentialsS3, + prefix string, + source_db string, + schema string, table string, debug bool) *script.Pipe { os.Setenv("AWS_ACCESS_KEY_ID", up.AccessKeyId) os.Setenv("AWS_SECRET_ACCESS_KEY", up.SecretAccessKey) os.Setenv("AWS_DEFAULT_REGION", up.Region) @@ -30,7 +34,10 @@ func S3(in_pipe *script.Pipe, up *vcap.CredentialsS3, prefix string, source_db s // Combine the slice for printing and execution. combined := strings.Join(cmd[:], " ") - fmt.Printf("command: %s\n", combined) + if debug { + fmt.Printf("command: %s\n", combined) + + } logging.Logger.Printf("BACKUPS s3 targeting %s\n", prefix) return in_pipe.Exec(combined) } diff --git a/internal/util/db.go b/internal/util/db.go new file mode 100644 index 0000000..7d7d617 --- /dev/null +++ b/internal/util/db.go @@ -0,0 +1,40 @@ +package util + +import ( + "database/sql" + "os" + + "gov.gsa.fac.cgov-util/internal/logging" + "gov.gsa.fac.cgov-util/internal/vcap" +) + +func Get_table_and_schema_names(source_creds *vcap.CredentialsRDS) map[string]string { + // Do this table-by-table for RAM reasons. + db, err := sql.Open("postgres", source_creds.Uri) + if err != nil { + logging.Logger.Println("BACKUPS could not connect to DB for table-by-table dump") + logging.Logger.Printf("BACKUPS %s\n", err) + os.Exit(-1) + } + + tables, err := db.Query("SELECT schemaname, tablename FROM pg_tables WHERE schemaname = 'public'") + if err != nil { + logging.Logger.Println("BACKUPS could not get table names for table-by-table dump") + logging.Logger.Printf("BACKUPS %s\n", err) + os.Exit(-1) + } + + table_names := make(map[string]string, 0) + + for tables.Next() { + var table string + var schema string + if err := tables.Scan(&schema, &table); err != nil { + logging.Logger.Println("BACKUPS could not scan table names in SELECT") + os.Exit(-1) + } + table_names[table] = schema + } + + return table_names +} diff --git a/internal/vcap/vcap.go b/internal/vcap/vcap.go index 65027bb..dcb063f 100644 --- a/internal/vcap/vcap.go +++ b/internal/vcap/vcap.go @@ -141,6 +141,7 @@ func GetRDSCreds(source_db string, dest_db string) (*CredentialsRDS, *Credential source, err = GetLocalRDSCredentials(source_db) if err != nil { logging.Logger.Println("BACKUPS Cannot get local source credentials") + logging.Logger.Println(err) os.Exit(-1) } } else {