From d5b75946963d0dd451d629c79bdd35c9cd897790 Mon Sep 17 00:00:00 2001 From: "Valentyn Kamyshenko (at Zeus)" Date: Fri, 3 Feb 2023 22:52:35 -0800 Subject: [PATCH] issues #145 and #149 fixed --- DbAnalyzer/go/oracle.go | 5 +- .../mysql_fdw/mysql_fdw.c | 40 +++++++++++++++- .../postgres_fdw/postgres_fdw.c | 47 +++++++++++++++---- DbGuardian/go/DbSetup.go | 20 ++++---- DbSync/go/DbSetup.go | 20 ++++---- 5 files changed, 100 insertions(+), 32 deletions(-) diff --git a/DbAnalyzer/go/oracle.go b/DbAnalyzer/go/oracle.go index 0b8251f6..17ef9fe2 100644 --- a/DbAnalyzer/go/oracle.go +++ b/DbAnalyzer/go/oracle.go @@ -14,7 +14,6 @@ import ( func getOraInfo(c types.Connection) (interface{}, error) { query := url.Values{} - query.Add("database", c.Database) if c.Tls { query.Add("SSL", "true") } else { @@ -24,7 +23,7 @@ func getOraInfo(c types.Connection) (interface{}, error) { Scheme: "oracle", User: url.UserPassword(c.User, c.Password), Host: fmt.Sprintf("%s:%d", c.Address, c.Port), - Path: "XEPDB1", + Path: strings.ToUpper(c.Database), RawQuery: query.Encode(), } oracleconn := u.String() @@ -143,7 +142,7 @@ func getOraInfo(c types.Connection) (interface{}, error) { } if curSchema == -1 || schema != database.Schemas[curSchema].Name { switch schema { - case "information_schema", "mysql", "sys", "performance_schema": + case "RDSADMIN": isSystem = true default: isSystem = false diff --git a/DbGuardian/foreign_data_wrappers/mysql_fdw/mysql_fdw.c b/DbGuardian/foreign_data_wrappers/mysql_fdw/mysql_fdw.c index 2e184ba5..cc8c8e29 100644 --- a/DbGuardian/foreign_data_wrappers/mysql_fdw/mysql_fdw.c +++ b/DbGuardian/foreign_data_wrappers/mysql_fdw/mysql_fdw.c @@ -664,9 +664,9 @@ mysqlBeginForeignScan(ForeignScanState *node, int eflags) options = mysql_get_options(rte->relid, true); /* Dymium */ + festate->how_to_redact = (int*)palloc0(sizeof(int) * tupleDescriptor->natts); if (fsplan->scan.scanrelid > 0) { - festate->how_to_redact = (int*)palloc0(sizeof(int) * tupleDescriptor->natts); /* !Dymium! */ for(int k = 0; k != tupleDescriptor->natts; ++k) { List *options = GetForeignColumnOptions(rte->relid, TupleDescAttr(tupleDescriptor, k)->attnum); @@ -684,6 +684,44 @@ mysqlBeginForeignScan(ForeignScanState *node, int eflags) } } } + else + { + for(int k = 0; k != tupleDescriptor->natts; ++k) { + Form_pg_attribute att = TupleDescAttr(tupleDescriptor, k); + Var *var; + RangeTblEntry *rte; + Oid reltype; + List *options; + ListCell *lc; + + /* + * If we can't identify the referenced table, do nothing. This'll + * likely lead to failure later, but perhaps we can muddle through. + */ + var = (Var *) list_nth_node(TargetEntry, fsplan->fdw_scan_tlist, + k)->expr; + if (!IsA(var, Var) /* !Dymium: move this condition down: || var->varattno != 0 */) + continue; + rte = list_nth(estate->es_range_table, var->varno - 1); + if (rte->rtekind != RTE_RELATION) + continue; + reltype = get_rel_type_id(rte->relid); + if (!OidIsValid(reltype)) + continue; + + options = GetForeignColumnOptions(rte->relid, att->attnum); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + if (strcmp(def->defname, "redact") == 0) + { + char *redact = defGetString(def); + festate->how_to_redact[k] = atoi(redact); + break; + } + } + } + } /* * Get the already connected connection, otherwise connect and get the diff --git a/DbGuardian/foreign_data_wrappers/postgres_fdw/postgres_fdw.c b/DbGuardian/foreign_data_wrappers/postgres_fdw/postgres_fdw.c index e886e5c5..c9312d1e 100644 --- a/DbGuardian/foreign_data_wrappers/postgres_fdw/postgres_fdw.c +++ b/DbGuardian/foreign_data_wrappers/postgres_fdw/postgres_fdw.c @@ -1437,7 +1437,7 @@ postgresGetForeignPlan(PlannerInfo *root, * Construct a tuple descriptor for the scan tuples handled by a foreign join. */ static TupleDesc -get_tupdesc_for_join_scan_tuples(ForeignScanState *node) +get_tupdesc_for_join_scan_tuples(ForeignScanState *node, int **how_to_redact /* !Dymium */) { ForeignScan *fsplan = (ForeignScan *) node->ss.ps.plan; EState *estate = node->ss.ps.state; @@ -1453,6 +1453,9 @@ get_tupdesc_for_join_scan_tuples(ForeignScanState *node) * we can convert them to a composite type the local server knows. */ tupdesc = CreateTupleDescCopy(node->ss.ss_ScanTupleSlot->tts_tupleDescriptor); + + *how_to_redact = (int*)palloc0(sizeof(int) * tupdesc->natts); + for (int i = 0; i < tupdesc->natts; i++) { Form_pg_attribute att = TupleDescAttr(tupdesc, i); @@ -1460,9 +1463,8 @@ get_tupdesc_for_join_scan_tuples(ForeignScanState *node) RangeTblEntry *rte; Oid reltype; - /* Nothing to do if it's not a generic RECORD attribute */ - if (att->atttypid != RECORDOID || att->atttypmod >= 0) - continue; + List *options; + ListCell *lc; /* * If we can't identify the referenced table, do nothing. This'll @@ -1470,7 +1472,7 @@ get_tupdesc_for_join_scan_tuples(ForeignScanState *node) */ var = (Var *) list_nth_node(TargetEntry, fsplan->fdw_scan_tlist, i)->expr; - if (!IsA(var, Var) || var->varattno != 0) + if (!IsA(var, Var) /* !Dymium: move this condition down: || var->varattno != 0 */) continue; rte = list_nth(estate->es_range_table, var->varno - 1); if (rte->rtekind != RTE_RELATION) @@ -1478,6 +1480,29 @@ get_tupdesc_for_join_scan_tuples(ForeignScanState *node) reltype = get_rel_type_id(rte->relid); if (!OidIsValid(reltype)) continue; + + + /* !Dymium: find how_to_redact */ + options = GetForeignColumnOptions(rte->relid, att->attnum); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + if (strcmp(def->defname, "redact") == 0) + { + char *redact = defGetString(def); + (*how_to_redact)[i] = atoi(redact); + break; + } + } + + /* !Dymium: move these conditions here */ + /* Nothing to do if it's not a generic RECORD attribute */ + if (att->atttypid != RECORDOID || att->atttypmod >= 0) + continue; + + if (var->varattno != 0) + continue; + att->atttypid = reltype; /* shouldn't need to change anything else */ } @@ -1508,7 +1533,9 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags) return; // Dymium + // puts("===== FSPLAN START ====="); // pprint(fsplan); + // puts("===== FSPLAN END ====="); /* * We'll save private state in node->fdw_state. @@ -1546,6 +1573,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags) /* Get private info created by planner functions. */ fsstate->query = strVal(list_nth(fsplan->fdw_private, FdwScanPrivateSelectSql)); + // puts(fsstate->query); fsstate->retrieved_attrs = (List *) list_nth(fsplan->fdw_private, FdwScanPrivateRetrievedAttrs); fsstate->fetch_size = intVal(list_nth(fsplan->fdw_private, @@ -1592,7 +1620,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags) else { fsstate->rel = NULL; - fsstate->tupdesc = get_tupdesc_for_join_scan_tuples(node); + fsstate->tupdesc = get_tupdesc_for_join_scan_tuples(node, &fsstate->how_to_redact); } fsstate->attinmeta = TupleDescGetAttInMetadata(fsstate->tupdesc); @@ -2730,8 +2758,11 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags) { TupleDesc tupdesc; - if (fsplan->scan.scanrelid == 0) - tupdesc = get_tupdesc_for_join_scan_tuples(node); + if (fsplan->scan.scanrelid == 0) { + /* !Dymium: not used */ + int *how_to_redact; + tupdesc = get_tupdesc_for_join_scan_tuples(node, &how_to_redact); + } else tupdesc = RelationGetDescr(dmstate->rel); diff --git a/DbGuardian/go/DbSetup.go b/DbGuardian/go/DbSetup.go index ae450abf..f69753cc 100644 --- a/DbGuardian/go/DbSetup.go +++ b/DbGuardian/go/DbSetup.go @@ -87,12 +87,12 @@ func options(connectionType types.ConnectionType) iOptions { host, port, strings.ToUpper(dbname)) }, userMapping: func(user, password string) string { - return fmt.Sprintf("username '%s', password '%s'", + return fmt.Sprintf("user '%s', password '%s'", esc(user), esc(password)) }, table: func(remoteSchema, remoteTable string) string { return fmt.Sprintf("schema '%s', table '%s'", - esc(remoteSchema), esc(strings.ToUpper(remoteTable))) + esc(remoteSchema), esc(remoteTable)) }, } } @@ -218,21 +218,21 @@ func configureDatabase(db *sql.DB, } for k := range shortSchemas { - if err := exec("CREATE SCHEMA IF NOT EXISTS " + k); err != nil { + if err := exec("CREATE SCHEMA IF NOT EXISTS " + strings.ToLower(k)); err != nil { return err } - if err := exec("GRANT USAGE ON SCHEMA " + k + " TO " + localUser); err != nil { + if err := exec("GRANT USAGE ON SCHEMA " + strings.ToLower(k) + " TO " + localUser); err != nil { return err } - if err := exec("ALTER DEFAULT PRIVILEGES IN SCHEMA " + k + " GRANT SELECT ON TABLES TO " + localUser); err != nil { + if err := exec("ALTER DEFAULT PRIVILEGES IN SCHEMA " + strings.ToLower(k) + " GRANT SELECT ON TABLES TO " + localUser); err != nil { return err } - if _, err := tx.ExecContext(ctx, "INSERT INTO _dymium.schemas (\"schema\") VALUES ( $1 )", k); err != nil { - return rollback(err, "Registering schema "+k+"_server failed") + if _, err := tx.ExecContext(ctx, "INSERT INTO _dymium.schemas (\"schema\") VALUES ( $1 )", strings.ToLower(k)); err != nil { + return rollback(err, "Registering schema "+strings.ToLower(k)+"_server failed") } } for k := range longSchemas { - kk := k.k1 + "_" + k.k2 + kk := strings.ToLower(k.k1 + "_" + k.k2) if err := exec(fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %q", kk)); err != nil { return err } @@ -341,7 +341,7 @@ func configureDatabase(db *sql.DB, rtyp = uuid } defs = append(defs, fmt.Sprintf(" %q %s OPTIONS( redact '%d' )%s", - c.Name, c.Typ, int(ract)|(rnul<<2)|(int(rtyp)<<3), notNull)) + strings.ToLower(c.Name), c.Typ, int(ract)|(rnul<<2)|(int(rtyp)<<3), notNull)) } } e := "CREATE FOREIGN TABLE %q.%q (\n" + strings.Join(defs, ",\n") + "\n)\n" + @@ -358,7 +358,7 @@ func configureDatabase(db *sql.DB, } } for _, sch := range schs { - if err := exec(fmt.Sprintf(e, sch, t.Name, con.Name, opts.table(s.Name, t.Name))); err != nil { + if err := exec(fmt.Sprintf(e, strings.ToLower(sch), strings.ToLower(t.Name), con.Name, opts.table(s.Name, t.Name))); err != nil { return err } } diff --git a/DbSync/go/DbSetup.go b/DbSync/go/DbSetup.go index d237b14a..90b15b37 100644 --- a/DbSync/go/DbSetup.go +++ b/DbSync/go/DbSetup.go @@ -87,12 +87,12 @@ func options(connectionType types.ConnectionType) iOptions { host, port, strings.ToUpper(dbname)) }, userMapping: func(user, password string) string { - return fmt.Sprintf("username '%s', password '%s'", + return fmt.Sprintf("user '%s', password '%s'", esc(user), esc(password)) }, table: func(remoteSchema, remoteTable string) string { return fmt.Sprintf("schema '%s', table '%s'", - esc(remoteSchema), esc(strings.ToUpper(remoteTable))) + esc(remoteSchema), esc(remoteTable)) }, } } @@ -218,21 +218,21 @@ func configureDatabase(db *sql.DB, } for k := range shortSchemas { - if err := exec("CREATE SCHEMA IF NOT EXISTS " + k); err != nil { + if err := exec("CREATE SCHEMA IF NOT EXISTS " + strings.ToLower(k)); err != nil { return err } - if err := exec("GRANT USAGE ON SCHEMA " + k + " TO " + localUser); err != nil { + if err := exec("GRANT USAGE ON SCHEMA " + strings.ToLower(k) + " TO " + localUser); err != nil { return err } - if err := exec("ALTER DEFAULT PRIVILEGES IN SCHEMA " + k + " GRANT SELECT ON TABLES TO " + localUser); err != nil { + if err := exec("ALTER DEFAULT PRIVILEGES IN SCHEMA " + strings.ToLower(k) + " GRANT SELECT ON TABLES TO " + localUser); err != nil { return err } - if _, err := tx.ExecContext(ctx, "INSERT INTO _dymium.schemas (\"schema\") VALUES ( $1 )", k); err != nil { - return rollback(err, "Registering schema "+k+"_server failed") + if _, err := tx.ExecContext(ctx, "INSERT INTO _dymium.schemas (\"schema\") VALUES ( $1 )", strings.ToLower(k)); err != nil { + return rollback(err, "Registering schema "+strings.ToLower(k)+"_server failed") } } for k := range longSchemas { - kk := k.k1 + "_" + k.k2 + kk := strings.ToLower(k.k1 + "_" + k.k2) if err := exec(fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %q", kk)); err != nil { return err } @@ -341,7 +341,7 @@ func configureDatabase(db *sql.DB, rtyp = uuid } defs = append(defs, fmt.Sprintf(" %q %s OPTIONS( redact '%d' )%s", - c.Name, c.Typ, int(ract)|(rnul<<2)|(int(rtyp)<<3), notNull)) + strings.ToLower(c.Name), c.Typ, int(ract)|(rnul<<2)|(int(rtyp)<<3), notNull)) } } e := "CREATE FOREIGN TABLE %q.%q (\n" + strings.Join(defs, ",\n") + "\n)\n" + @@ -358,7 +358,7 @@ func configureDatabase(db *sql.DB, } } for _, sch := range schs { - if err := exec(fmt.Sprintf(e, sch, t.Name, con.Name, opts.table(s.Name, t.Name))); err != nil { + if err := exec(fmt.Sprintf(e, strings.ToLower(sch), strings.ToLower(t.Name), con.Name, opts.table(s.Name, t.Name))); err != nil { return err } }