Update core dependencies (#1652)

Updating the core dependencies on `master` to make sure we get all the
fixes into `v1.1.0`.

api/wallet.go

@@ -12,8 +12,8 @@ type (
 	// A SiacoinElement is a SiacoinOutput along with its ID.
 	SiacoinElement struct {
 		types.SiacoinOutput
-		ID             types.Hash256 `json:"id"`
-		MaturityHeight uint64        `json:"maturityHeight"`
+		ID             types.SiacoinOutputID `json:"id"`
+		MaturityHeight uint64                `json:"maturityHeight"`
 	}
 
 	// A Transaction is an on-chain transaction relevant to a particular wallet,

autopilot/contractor/contractor.go

@@ -963,6 +963,15 @@ func performContractChecks(ctx *mCtx, alerter alerts.Alerter, bus Bus, w Worker,
 			continue
 		}
 
+		// NOTE: if we have a contract with a host that is not scanned, we either
+		// added the host and contract manually or reset the host scans. In that case,
+		// we ignore the fact that the host is not scanned for now to avoid churn.
+		if inSet && check.Usability.NotCompletingScan {
+			keepContract(c.ContractMetadata, host)
+			logger.Info("ignoring contract with unscanned host")
+			continue // no more checks until host is scanned
+		}
+
 		// check usability
 		if !check.Usability.IsUsable() {
 			reasons := strings.Join(check.Usability.UnusableReasons(), ",")
@@ -1097,9 +1106,10 @@ func performContractFormations(ctx *mCtx, bus Bus, w Worker, cr contractReviser,
 		usedHosts[c.HostKey] = struct{}{}
 	}
 	allHosts, err := bus.SearchHosts(ctx, api.SearchHostOptions{
+		AutopilotID:   ctx.ApID(),
 		Limit:         -1,
 		FilterMode:    api.HostFilterModeAllowed,
-		UsabilityMode: api.UsabilityFilterModeAll,
+		UsabilityMode: api.UsabilityFilterModeUsable,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch usable hosts: %w", err)

bus/routes.go

@@ -376,7 +376,7 @@ func (b *Bus) walletOutputsHandler(jc jape.Context) {
 		elements := make([]api.SiacoinElement, len(utxos))
 		for i, sce := range utxos {
 			elements[i] = api.SiacoinElement{
-				ID: sce.StateElement.ID,
+				ID: sce.ID,
 				SiacoinOutput: types.SiacoinOutput{
 					Value:   sce.SiacoinOutput.Value,
 					Address: sce.SiacoinOutput.Address,

go.mod

@@ -14,10 +14,10 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.24
 	github.com/montanaflynn/stats v0.7.1
 	github.com/shopspring/decimal v1.4.0
-	go.sia.tech/core v0.4.8-0.20240926222149-2c8b541119dc
-	go.sia.tech/coreutils v0.4.1
+	go.sia.tech/core v0.6.1
+	go.sia.tech/coreutils v0.6.0
 	go.sia.tech/gofakes3 v0.0.5
-	go.sia.tech/hostd v1.1.3-0.20241022083902-7d61fc09be7c
+	go.sia.tech/hostd v1.1.3-0.20241101150257-8319b75af8b2
 	go.sia.tech/jape v0.12.1
 	go.sia.tech/mux v1.3.0
 	go.sia.tech/web/renterd v0.61.0

go.sum

@@ -55,14 +55,14 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
 go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
-go.sia.tech/core v0.4.8-0.20240926222149-2c8b541119dc h1:+hCcYky+23HtiAnirXsq0U/NaCt1WuIu308lmfTtJNM=
-go.sia.tech/core v0.4.8-0.20240926222149-2c8b541119dc/go.mod h1:j2Ke8ihV8or7d2VDrFZWcCkwSVHO0DNMQJAGs9Qop2M=
-go.sia.tech/coreutils v0.4.1 h1:ExQ9g6EtnFe70ptNBG+OtZyFU3aBoEzE/06rtbN6f4c=
-go.sia.tech/coreutils v0.4.1/go.mod h1:v60kPqZERsb1ZS0PVe4S8hr2ArNEwTdp7XTzErXnV2U=
+go.sia.tech/core v0.6.1 h1:eaExM2E2eNr43su2XDkY5J24E3F54YGS7hcC3WtVjVk=
+go.sia.tech/core v0.6.1/go.mod h1:P3C1BWa/7J4XgdzWuaYHBvLo2RzZ0UBaJM4TG1GWB2g=
+go.sia.tech/coreutils v0.6.0 h1:r0IZt+aVdGG2uIHl7OtaWRYdVx4NQ7ezRoSGa0Ej8GY=
+go.sia.tech/coreutils v0.6.0/go.mod h1:XlsnogeYU/Tdjzp/HUNAj5T7tZCdmeBHIBjymbPC+uQ=
 go.sia.tech/gofakes3 v0.0.5 h1:vFhVBUFbKE9ZplvLE2w4TQxFMQyF8qvgxV4TaTph+Vw=
 go.sia.tech/gofakes3 v0.0.5/go.mod h1:LXEzwGw+OHysWLmagleCttX93cJZlT9rBu/icOZjQ54=
-go.sia.tech/hostd v1.1.3-0.20241022083902-7d61fc09be7c h1:oGRzqksT2MFoS4SvkrGbUG1LXNbnmv2KBzzv8uf0R4E=
-go.sia.tech/hostd v1.1.3-0.20241022083902-7d61fc09be7c/go.mod h1:EIuirLN/+714h+XaIpnJkU2BFdX/JF6Q1VXhX4N5sbc=
+go.sia.tech/hostd v1.1.3-0.20241101150257-8319b75af8b2 h1:6fogOsOr7efjeDHCO0mqIibEa5cOagSSo3JXe1fuifM=
+go.sia.tech/hostd v1.1.3-0.20241101150257-8319b75af8b2/go.mod h1:096sIYupHRUGkVyAe6i511la7wQo5LfRHGNUSMIKcT4=
 go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU=
 go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4=
 go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c=

internal/sql/migrations.go

@@ -288,6 +288,12 @@ var (
 					return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00018_directory_buckets_2", log)
 				},
 			},
+			{
+				ID: "00019_scan_reset",
+				Migrate: func(tx Tx) error {
+					return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00019_scan_reset", log)
+				},
+			},
 		}
 	}
 	MetricsMigrations = func(ctx context.Context, migrationsFs embed.FS, log *zap.SugaredLogger) []Migration {

internal/worker/accounts.go

@@ -351,8 +351,6 @@ func (a *AccountMgr) refillAccounts() {
 
 				if err != nil && shouldLog {
 					a.logger.Error("failed to refill account for host", zap.Stringer("hostKey", contract.HostKey), zap.Error(err))
-				} else {
-					a.logger.Infow("successfully refilled account for host", zap.Stringer("hostKey", contract.HostKey), zap.Error(err))
 				}
 			}(c)
 		}
@@ -409,6 +407,7 @@ func (a *AccountMgr) refillAccount(ctx context.Context, contract api.ContractMet
 	if err != nil {
 		return fmt.Errorf("failed to fund account: %w", err)
 	}
+	a.logger.Infow("successfully refilled account for host", zap.Stringer("hostKey", contract.HostKey), zap.Error(err))
 	return nil
 }
 

stores/sql/chain.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"go.sia.tech/core/types"
+	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/renterd/api"
 	"go.sia.tech/renterd/internal/sql"
 	"go.uber.org/zap"
@@ -138,12 +139,16 @@ func UpdateFailedContracts(ctx context.Context, tx sql.Tx, blockHeight uint64, l
 	return nil
 }
 
-func UpdateWalletStateElements(ctx context.Context, tx sql.Tx, elements []types.StateElement) error {
-	if len(elements) == 0 {
-		return nil
+func UpdateWalletSiacoinElementProofs(ctx context.Context, tx sql.Tx, updater wallet.ProofUpdater) error {
+	elements, err := walletStateElements(ctx, tx)
+	if err != nil {
+		return fmt.Errorf("failed to get wallet state elements: %w", err)
+	}
+	for i := range elements {
+		updater.UpdateElementProof(&elements[i].StateElement)
 	}
 
-	updateStmt, err := tx.Prepare(ctx, "UPDATE wallet_outputs SET leaf_index = ?, merkle_proof= ?  WHERE output_id = ?")
+	updateStmt, err := tx.Prepare(ctx, "UPDATE wallet_outputs SET leaf_index = ?, merkle_proof = ? WHERE output_id = ?")
 	if err != nil {
 		return fmt.Errorf("failed to prepare statement to update state elements: %w", err)
 	}
@@ -154,28 +159,9 @@ func UpdateWalletStateElements(ctx context.Context, tx sql.Tx, elements []types.
 			return fmt.Errorf("failed to update state element '%v': %w", el.ID, err)
 		}
 	}
-
 	return nil
 }
 
-func WalletStateElements(ctx context.Context, tx sql.Tx) ([]types.StateElement, error) {
-	rows, err := tx.Query(ctx, "SELECT output_id, leaf_index, merkle_proof FROM wallet_outputs")
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch state elements: %w", err)
-	}
-	defer rows.Close()
-
-	var elements []types.StateElement
-	for rows.Next() {
-		if el, err := scanStateElement(rows); err != nil {
-			return nil, fmt.Errorf("failed to scan state element: %w", err)
-		} else {
-			elements = append(elements, el)
-		}
-	}
-	return elements, nil
-}
-
 func contractNotFoundErr(fcid types.FileContractID) error {
 	return fmt.Errorf("%w: %v", api.ErrContractNotFound, fcid)
 }
@@ -224,3 +210,26 @@ func updateContractState(ctx context.Context, tx sql.Tx, table string, fcid type
 	}
 	return n == 1, nil
 }
+
+type stateElement struct {
+	ID types.Hash256
+	types.StateElement
+}
+
+func walletStateElements(ctx context.Context, tx sql.Tx) ([]stateElement, error) {
+	rows, err := tx.Query(ctx, "SELECT output_id, leaf_index, merkle_proof FROM wallet_outputs")
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch state elements: %w", err)
+	}
+	defer rows.Close()
+
+	var elements []stateElement
+	for rows.Next() {
+		if el, err := scanStateElement(rows); err != nil {
+			return nil, fmt.Errorf("failed to scan state element: %w", err)
+		} else {
+			elements = append(elements, el)
+		}
+	}
+	return elements, nil
+}

stores/sql/main.go

@@ -2698,8 +2698,8 @@ func scanSiacoinElement(s Scanner) (el types.SiacoinElement, err error) {
 		return types.SiacoinElement{}, err
 	}
 	return types.SiacoinElement{
+		ID: types.SiacoinOutputID(id),
 		StateElement: types.StateElement{
-			ID:          types.Hash256(id),
 			LeafIndex:   leafIndex,
 			MerkleProof: merkleProof.Hashes,
 		},
@@ -2711,17 +2711,19 @@ func scanSiacoinElement(s Scanner) (el types.SiacoinElement, err error) {
 	}, nil
 }
 
-func scanStateElement(s Scanner) (types.StateElement, error) {
+func scanStateElement(s Scanner) (stateElement, error) {
 	var id Hash256
 	var leafIndex uint64
 	var merkleProof MerkleProof
 	if err := s.Scan(&id, &leafIndex, &merkleProof); err != nil {
-		return types.StateElement{}, err
+		return stateElement{}, err
 	}
-	return types.StateElement{
-		ID:          types.Hash256(id),
-		LeafIndex:   leafIndex,
-		MerkleProof: merkleProof.Hashes,
+	return stateElement{
+		ID: types.Hash256(id),
+		StateElement: types.StateElement{
+			LeafIndex:   leafIndex,
+			MerkleProof: merkleProof.Hashes,
+		},
 	}, nil
 }
 

stores/sql/mysql/chain.go

@@ -330,10 +330,6 @@ func (c chainUpdateTx) UpdateHost(hk types.PublicKey, ha chain.HostAnnouncement,
 	return nil
 }
 
-func (c chainUpdateTx) UpdateWalletStateElements(elements []types.StateElement) error {
-	return ssql.UpdateWalletStateElements(c.ctx, c.tx, elements)
-}
-
-func (c chainUpdateTx) WalletStateElements() ([]types.StateElement, error) {
-	return ssql.WalletStateElements(c.ctx, c.tx)
+func (c chainUpdateTx) UpdateWalletSiacoinElementProofs(pu wallet.ProofUpdater) error {
+	return ssql.UpdateWalletSiacoinElementProofs(c.ctx, c.tx, pu)
 }

stores/sql/mysql/migrations/main/migration_00019_scan_reset.sql

@@ -0,0 +1 @@
+UPDATE hosts SET settings = '{}', price_table = '{}', last_scan = 0, last_scan_success = 0, second_to_last_scan_success = 0, scanned = 0 WHERE 1=1;

stores/sql/sqlite/chain.go

@@ -342,10 +342,6 @@ func (c chainUpdateTx) UpdateHost(hk types.PublicKey, ha chain.HostAnnouncement,
 	return nil
 }
 
-func (c chainUpdateTx) UpdateWalletStateElements(elements []types.StateElement) error {
-	return ssql.UpdateWalletStateElements(c.ctx, c.tx, elements)
-}
-
-func (c chainUpdateTx) WalletStateElements() ([]types.StateElement, error) {
-	return ssql.WalletStateElements(c.ctx, c.tx)
+func (c chainUpdateTx) UpdateWalletSiacoinElementProofs(pu wallet.ProofUpdater) error {
+	return ssql.UpdateWalletSiacoinElementProofs(c.ctx, c.tx, pu)
 }

stores/sql/sqlite/migrations/main/migration_00019_scan_reset.sql

@@ -0,0 +1 @@
+UPDATE hosts SET settings = '{}', price_table = '{}', last_scan = 0, last_scan_success = 0, second_to_last_scan_success = 0, scanned = 0 WHERE 1=1;

stores/sql/types.go

@@ -195,8 +195,13 @@ func (h Hash256) Value() (driver.Value, error) {
 
 // Scan scan value into HostSettings, implements sql.Scanner interface.
 func (hs *HostSettings) Scan(value interface{}) error {
-	bytes, ok := value.([]byte)
-	if !ok {
+	var bytes []byte
+	switch value := value.(type) {
+	case string:
+		bytes = []byte(value)
+	case []byte:
+		bytes = value
+	default:
 		return errors.New(fmt.Sprint("failed to unmarshal Settings value:", value))
 	}
 	return json.Unmarshal(bytes, hs)
@@ -212,8 +217,13 @@ func (hs HostSettings) Value() (driver.Value, error) {
 
 // Scan scan value into PriceTable, implements sql.Scanner interface.
 func (pt *PriceTable) Scan(value interface{}) error {
-	bytes, ok := value.([]byte)
-	if !ok {
+	var bytes []byte
+	switch value := value.(type) {
+	case string:
+		bytes = []byte(value)
+	case []byte:
+		bytes = value
+	default:
 		return errors.New(fmt.Sprint("failed to unmarshal PriceTable value:", value))
 	}
 	return json.Unmarshal(bytes, pt)