eth: refactor tx montoring, handle tx resubmissions better, general r…

…efactoring (decred#2752)

* fix fee and tip caps

* redo nonces and pending txs

* safer fee rates

* use NonceAt along with PendingNonceAt to characterize discrepancies

* better waitgroup and db connect/run pattern

client/asset/btc/btc.go

@@ -1944,7 +1944,9 @@ func (btc *baseWallet) maxOrder(lotSize, feeSuggestion, maxFeeRate uint64) (utxo
 		return utxos, est, err
 	}
 
-	return utxos, &asset.SwapEstimate{}, nil
+	return utxos, &asset.SwapEstimate{
+		FeeReservesPerLot: basicFee,
+	}, nil
 }
 
 // sizeUnit returns the short form of the unit used to measure size, either
@@ -2170,6 +2172,8 @@ func (btc *baseWallet) estimateSwap(lots, lotSize, feeSuggestion, maxFeeRate uin
 		bumpedNetRate = uint64(math.Ceil(float64(bumpedNetRate) * feeBump))
 	}
 
+	feeReservesPerLot := bumpedMaxRate * btc.initTxSize
+
 	val := lots * lotSize
 	// The orderEnough func does not account for a split transaction at the start,
 	// so it is possible that funding for trySplit would actually choose more
@@ -2211,6 +2215,7 @@ func (btc *baseWallet) estimateSwap(lots, lotSize, feeSuggestion, maxFeeRate uin
 				MaxFees:            maxFees + splitMaxFees,
 				RealisticBestCase:  estLowFees + splitFees,
 				RealisticWorstCase: estHighFees + splitFees,
+				FeeReservesPerLot:  feeReservesPerLot,
 			}, true, reqFunds, nil // requires reqTotal, but locks reqFunds in the split output
 		}
 	}
@@ -2234,6 +2239,7 @@ func (btc *baseWallet) estimateSwap(lots, lotSize, feeSuggestion, maxFeeRate uin
 		MaxFees:            maxFees,
 		RealisticBestCase:  estLowFees,
 		RealisticWorstCase: estHighFees,
+		FeeReservesPerLot:  feeReservesPerLot,
 	}, false, sum, nil
 }
 

client/asset/dcr/dcr.go

@@ -1454,7 +1454,9 @@ func (dcr *ExchangeWallet) maxOrder(lotSize, feeSuggestion, maxFeeRate uint64) (
 		return utxos, est, err
 	}
 
-	return nil, &asset.SwapEstimate{}, nil
+	return nil, &asset.SwapEstimate{
+		FeeReservesPerLot: basicFee,
+	}, nil
 }
 
 // estimateSwap prepares an *asset.SwapEstimate.
@@ -1469,6 +1471,8 @@ func (dcr *ExchangeWallet) estimateSwap(lots, lotSize, feeSuggestion, maxFeeRate
 		bumpedNetRate = uint64(math.Ceil(float64(bumpedNetRate) * feeBump))
 	}
 
+	feeReservesPerLot := bumpedMaxRate * dexdcr.InitTxSize
+
 	val := lots * lotSize
 	// The orderEnough func does not account for a split transaction at the
 	// start, so it is possible that funding for trySplit would actually choose
@@ -1515,6 +1519,7 @@ func (dcr *ExchangeWallet) estimateSwap(lots, lotSize, feeSuggestion, maxFeeRate
 				MaxFees:            maxFees + splitMaxFees,
 				RealisticBestCase:  estLowFees + splitFees,
 				RealisticWorstCase: estHighFees + splitFees,
+				FeeReservesPerLot:  feeReservesPerLot,
 			}, true, reqFunds, nil // requires reqTotal, but locks reqFunds in the split output
 		}
 	}
@@ -1541,6 +1546,7 @@ func (dcr *ExchangeWallet) estimateSwap(lots, lotSize, feeSuggestion, maxFeeRate
 		MaxFees:            maxFees,
 		RealisticBestCase:  estLowFees,
 		RealisticWorstCase: estHighFees,
+		FeeReservesPerLot:  feeReservesPerLot,
 	}, false, sum, nil
 }
 

client/asset/dcr/externaltx.go

@@ -100,7 +100,7 @@ func (dcr *ExchangeWallet) externalTxOutput(ctx context.Context, op outPoint, pk
 
 	// Scan block filters to find the tx block if it is yet unknown.
 	if txBlock == nil {
-		dcr.log.Infof("Output %s:%d NOT yet found; now searching with block filters.", op.txHash, op.vout)
+		dcr.log.Tracef("Output %s:%d NOT yet found; now searching with block filters.", op.txHash, op.vout)
 		txBlock, err = dcr.scanFiltersForTxBlock(ctx, tx, [][]byte{pkScript}, earliestTxTime)
 		if err != nil {
 			return nil, nil, fmt.Errorf("error checking if tx %s is mined: %w", tx.hash, err)

client/asset/estimation.go

@@ -20,6 +20,9 @@ type SwapEstimate struct {
 	// RealisticBestCase is an estimation of the fees that might be assessed in
 	// a best-case scenario of 1 tx and 1 output for the entire order.
 	RealisticBestCase uint64 `json:"realisticBestCase"`
+	// FeeReservesPerLot is the amount that must be reserved per lot to cover
+	// fees for swap transactions.
+	FeeReservesPerLot uint64 `json:"feeReservesPerLot"`
 }
 
 // RedeemEstimate is an estimate of the range of fees that might realistically

client/asset/eth/contractor.go

@@ -41,7 +41,6 @@ type contractor interface {
 	// case will always be zero.
 	value(context.Context, *types.Transaction) (incoming, outgoing uint64, err error)
 	isRefundable(secretHash [32]byte) (bool, error)
-	voidUnusedNonce()
 }
 
 // tokenContractor interacts with an ERC20 token contract and a token swap
@@ -307,15 +306,6 @@ func (c *contractorV0) outgoingValue(tx *types.Transaction) (swapped uint64) {
 	return
 }
 
-// voidUnusedNonce allows the next nonce received from a provider to be the same
-// as a recent nonce. Use when we fetch a nonce but error before or while
-// sending a transaction.
-func (c *contractorV0) voidUnusedNonce() {
-	if mRPC, is := c.cb.(*multiRPCClient); is {
-		mRPC.voidUnusedNonce()
-	}
-}
-
 // tokenContractorV0 is a contractor that implements the tokenContractor
 // methods, providing access to the methods of the token's ERC20 contract.
 type tokenContractorV0 struct {

client/asset/eth/deploy.go

@@ -72,7 +72,7 @@ func (contractDeployer) estimateDeployFunding(
 	}
 	defer os.RemoveAll(walletDir)
 
-	cl, feeRate, err := ContractDeployer.nodeAndRate(ctx, chain, walletDir, credentialsPath, chainCfg, log, net)
+	cl, maxFeeRate, _, err := ContractDeployer.nodeAndRate(ctx, chain, walletDir, credentialsPath, chainCfg, log, net)
 	if err != nil {
 		return err
 	}
@@ -102,6 +102,7 @@ func (contractDeployer) estimateDeployFunding(
 		}
 	}
 
+	feeRate := dexeth.WeiToGweiCeil(maxFeeRate)
 	if gas == 0 {
 		gas = deploymentGas
 	}
@@ -237,7 +238,7 @@ func (contractDeployer) deployContract(
 	}
 	defer os.RemoveAll(walletDir)
 
-	cl, feeRate, err := ContractDeployer.nodeAndRate(ctx, chain, walletDir, credentialsPath, chainCfg, log, net)
+	cl, maxFeeRate, tipRate, err := ContractDeployer.nodeAndRate(ctx, chain, walletDir, credentialsPath, chainCfg, log, net)
 	if err != nil {
 		return err
 	}
@@ -262,7 +263,8 @@ func (contractDeployer) deployContract(
 		return fmt.Errorf("EstimateGas error: %v", err)
 	}
 
-	log.Infof("Estimated fees: %s", ui.ConventionalString(feeRate*gas))
+	feeRate := dexeth.WeiToGweiCeil(maxFeeRate)
+	log.Infof("Estimated fees: %s gwei / gas", ui.ConventionalString(feeRate*gas))
 
 	gas *= 5 / 4 // Add 20% buffer
 	feesWithBuffer := feeRate * gas
@@ -275,7 +277,7 @@ func (contractDeployer) deployContract(
 			ui.ConventionalString(shortage), cl.address())
 	}
 
-	txOpts, err := cl.txOpts(ctx, 0, gas, dexeth.GweiToWei(feeRate), nil)
+	txOpts, err := cl.txOpts(ctx, 0, gas, dexeth.GweiToWei(feeRate), tipRate, nil)
 	if err != nil {
 		return err
 	}
@@ -309,13 +311,13 @@ func (contractDeployer) ReturnETH(
 	}
 	defer os.RemoveAll(walletDir)
 
-	cl, feeRate, err := ContractDeployer.nodeAndRate(ctx, chain, walletDir, credentialsPath, chainCfg, log, net)
+	cl, maxFeeRate, tipRate, err := ContractDeployer.nodeAndRate(ctx, chain, walletDir, credentialsPath, chainCfg, log, net)
 	if err != nil {
 		return err
 	}
 	defer cl.shutdown()
 
-	return GetGas.returnFunds(ctx, cl, dexeth.GweiToWei(feeRate), returnAddr, nil, ui, log, net)
+	return GetGas.returnFunds(ctx, cl, maxFeeRate, tipRate, returnAddr, nil, ui, log, net)
 }
 
 func (contractDeployer) nodeAndRate(
@@ -327,11 +329,11 @@ func (contractDeployer) nodeAndRate(
 	chainCfg *params.ChainConfig,
 	log dex.Logger,
 	net dex.Network,
-) (*multiRPCClient, uint64, error) {
+) (*multiRPCClient, *big.Int, *big.Int, error) {
 
 	seed, providers, err := getFileCredentials(chain, credentialsPath, net)
 	if err != nil {
-		return nil, 0, err
+		return nil, nil, nil, err
 	}
 
 	pw := []byte("abc")
@@ -346,30 +348,30 @@ func (contractDeployer) nodeAndRate(
 		Net:      net,
 		Logger:   log,
 	}, nil /* we don't need the full api, skipConnect = true allows for nil compat */, true); err != nil {
-		return nil, 0, fmt.Errorf("error creating wallet: %w", err)
+		return nil, nil, nil, fmt.Errorf("error creating wallet: %w", err)
 	}
 
-	cl, err := newMultiRPCClient(walletDir, providers, log, chainCfg, net)
+	cl, err := newMultiRPCClient(walletDir, providers, log, chainCfg, 3, net)
 	if err != nil {
-		return nil, 0, fmt.Errorf("error creating rpc client: %w", err)
+		return nil, nil, nil, fmt.Errorf("error creating rpc client: %w", err)
 	}
 
 	if err := cl.unlock(string(pw)); err != nil {
-		return nil, 0, fmt.Errorf("error unlocking rpc client: %w", err)
+		return nil, nil, nil, fmt.Errorf("error unlocking rpc client: %w", err)
 	}
 
 	if err = cl.connect(ctx); err != nil {
-		return nil, 0, fmt.Errorf("error connecting: %w", err)
+		return nil, nil, nil, fmt.Errorf("error connecting: %w", err)
 	}
 
-	base, tip, err := cl.currentFees(ctx)
+	baseRate, tipRate, err := cl.currentFees(ctx)
 	if err != nil {
 		cl.shutdown()
-		return nil, 0, fmt.Errorf("Error estimating fee rate: %v", err)
+		return nil, nil, nil, fmt.Errorf("Error estimating fee rate: %v", err)
 	}
 
-	feeRate := dexeth.WeiToGwei(new(big.Int).Add(tip, new(big.Int).Mul(base, big.NewInt(2))))
-	return cl, feeRate, nil
+	maxFeeRate := new(big.Int).Add(tipRate, new(big.Int).Mul(baseRate, big.NewInt(2)))
+	return cl, maxFeeRate, tipRate, nil
 }
 
 // DeployMultiBalance deployes a contract with a function for reading all