From b9c06b64d739e1f9a501736c6d1f6c50fe1f6a7f Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 17 Apr 2024 14:18:10 -0500 Subject: [PATCH] 1 --- curiosrc/gc/storage_endpoint_gc.go | 10 +++--- go.mod | 7 +++-- go.sum | 8 +++++ lib/harmony/harmonydb/harmonydb.go | 6 ++-- lib/harmony/harmonydb/userfuncs.go | 49 ++++++++++++++++++++++-------- 5 files changed, 56 insertions(+), 24 deletions(-) diff --git a/curiosrc/gc/storage_endpoint_gc.go b/curiosrc/gc/storage_endpoint_gc.go index 8c6e6a0bf30..1dd26ee037c 100644 --- a/curiosrc/gc/storage_endpoint_gc.go +++ b/curiosrc/gc/storage_endpoint_gc.go @@ -197,16 +197,14 @@ func (s *StorageEndpointGC) Do(taskID harmonytask.TaskID, stillOwned func() bool // Remove dead URLs from storage_path entries and handle path cleanup for _, du := range deadURLs { // Fetch the current URLs for the storage path - var currentPath struct { - URLs string - } - err = tx.Get(¤tPath, "SELECT urls FROM storage_path WHERE storage_id = $1", du.StorageID) + var URLs string + err = tx.QueryRow("SELECT urls FROM storage_path WHERE storage_id = $1", du.StorageID).Scan(&URLs) if err != nil { return false, xerrors.Errorf("fetching storage paths: %w", err) } // Filter out the dead URL using lo.Reject and prepare the updated list - urls := strings.Split(currentPath.URLs, paths.URLSeparator) + urls := strings.Split(URLs, paths.URLSeparator) urls = lo.Reject(urls, func(u string, _ int) bool { return u == du.URL }) @@ -214,7 +212,7 @@ func (s *StorageEndpointGC) Do(taskID harmonytask.TaskID, stillOwned func() bool log.Debugw("filtered urls", "urls", urls, "dead_url", du.URL, "storage_id", du.StorageID) if os.Getenv("CURIO_STORAGE_META_GC_DRYRUN") != "no" { // todo drop this after testing - log.Debugw("dryrun: not updating storage path", "storage_id", du.StorageID, "urls", urls, "dead_url", du.URL, "current_urls", currentPath.URLs, "dead_urls", deadURLs) + log.Debugw("dryrun: not updating storage path", "storage_id", du.StorageID, "urls", urls, "dead_url", du.URL, "current_urls", URLs, "dead_urls", deadURLs) continue } diff --git a/go.mod b/go.mod index 054a20f9ed2..cd92fcac01a 100644 --- a/go.mod +++ b/go.mod @@ -159,7 +159,7 @@ require ( go.uber.org/fx v1.20.1 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.19.0 + golang.org/x/crypto v0.20.0 golang.org/x/exp v0.0.0-20240213143201-ec583247a57a golang.org/x/net v0.21.0 golang.org/x/sync v0.6.0 @@ -249,8 +249,8 @@ require ( github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/puddle/v2 v2.2.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect @@ -318,6 +318,7 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/yugabyte/pgx/v5 v5.5.3-yb-2 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect go.opentelemetry.io/otel/metric v1.21.0 // indirect diff --git a/go.sum b/go.sum index 4fe1f13df94..6c2bf17cfb5 100644 --- a/go.sum +++ b/go.sum @@ -884,10 +884,14 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE= github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk= github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -1711,6 +1715,8 @@ github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZM github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yugabyte/pgx/v5 v5.5.3-yb-2 h1:SDk2waZb2o6dSLYqk+vq0Ur2jnIv+X2A+P+QPR1UThU= +github.com/yugabyte/pgx/v5 v5.5.3-yb-2/go.mod h1:2SxizGfDY7UDCRTtbI/xd98C/oGN7S/3YoGF8l9gx/c= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1840,6 +1846,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/lib/harmony/harmonydb/harmonydb.go b/lib/harmony/harmonydb/harmonydb.go index 3d05f04fb4f..56b5acdfee2 100644 --- a/lib/harmony/harmonydb/harmonydb.go +++ b/lib/harmony/harmonydb/harmonydb.go @@ -15,9 +15,9 @@ import ( "time" logging "github.com/ipfs/go-log/v2" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" - "github.com/jackc/pgx/v5/pgxpool" + "github.com/yugabyte/pgx/v5" + "github.com/yugabyte/pgx/v5/pgconn" + "github.com/yugabyte/pgx/v5/pgxpool" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/node/config" diff --git a/lib/harmony/harmonydb/userfuncs.go b/lib/harmony/harmonydb/userfuncs.go index 5cca8de57e7..1f39504b81e 100644 --- a/lib/harmony/harmonydb/userfuncs.go +++ b/lib/harmony/harmonydb/userfuncs.go @@ -3,17 +3,18 @@ package harmonydb import ( "context" "errors" + "fmt" "runtime" "time" - "github.com/georgysavva/scany/v2/pgxscan" + "github.com/georgysavva/scany/v2/dbscan" "github.com/jackc/pgerrcode" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" "github.com/samber/lo" + "github.com/yugabyte/pgx/v5" + "github.com/yugabyte/pgx/v5/pgconn" ) -var errTx = errors.New("Cannot use a non-transaction func in a transaction") +var errTx = errors.New("cannot use a non-transaction func in a transaction") // rawStringOnly is _intentionally_private_ to force only basic strings in SQL queries. // In any package, raw strings will satisfy compilation. Ex: @@ -42,7 +43,7 @@ type Qry interface { Values() ([]any, error) } -// Query offers Next/Err/Close/Scan/Values/StructScan +// Query offers Next/Err/Close/Scan/Values type Query struct { Qry } @@ -69,8 +70,12 @@ func (db *DB) Query(ctx context.Context, sql rawStringOnly, arguments ...any) (* q, err := db.pgx.Query(ctx, string(sql), arguments...) return &Query{q}, err } + +// StructScan allows scanning a single row into a struct. +// This improves efficiency of processing large result sets +// by avoiding the need to allocate a slice of structs. func (q *Query) StructScan(s any) error { - return pgxscan.ScanRow(s, q.Qry.(pgx.Rows)) + return dbscan.ScanRow(s, dbscanRows{q.Qry.(pgx.Rows)}) } type Row interface { @@ -95,6 +100,20 @@ func (db *DB) QueryRow(ctx context.Context, sql rawStringOnly, arguments ...any) return db.pgx.QueryRow(ctx, string(sql), arguments...) } +type dbscanRows struct { + pgx.Rows +} + +func (d dbscanRows) Close() error { + d.Rows.Close() + return nil +} +func (d dbscanRows) Columns() ([]string, error) { + return lo.Map(d.Rows.FieldDescriptions(), func(fd pgconn.FieldDescription, _ int) string { + return fd.Name + }), nil +} + /* Select multiple rows into a slice using name matching Ex: @@ -113,7 +132,12 @@ func (db *DB) Select(ctx context.Context, sliceOfStructPtr any, sql rawStringOnl if db.usedInTransaction() { return errTx } - return pgxscan.Select(ctx, db.pgx, sliceOfStructPtr, string(sql), arguments...) + rows, err := db.pgx.Query(ctx, string(sql), arguments...) + if err != nil { + return err + } + defer rows.Close() + return dbscan.ScanAll(sliceOfStructPtr, dbscanRows{rows}) } type Tx struct { @@ -233,11 +257,12 @@ func (t *Tx) QueryRow(sql rawStringOnly, arguments ...any) Row { // Select in a transaction. func (t *Tx) Select(sliceOfStructPtr any, sql rawStringOnly, arguments ...any) error { - return pgxscan.Select(t.ctx, t.Tx, sliceOfStructPtr, string(sql), arguments...) -} - -func (t *Tx) Get(s any, sql rawStringOnly, arguments ...any) error { - return pgxscan.Get(t.ctx, t.Tx, s, string(sql), arguments...) + rows, err := t.Query(sql, arguments...) + if err != nil { + return fmt.Errorf("scany: query multiple result rows: %w", err) + } + defer rows.Close() + return dbscan.ScanAll(sliceOfStructPtr, dbscanRows{rows.Qry.(pgx.Rows)}) } func IsErrUniqueContraint(err error) bool {