Merge pull request PelicanPlatform#1513 from bbockelm/conn_reset

Rewrite TCP connection reset / broken pipe messages

client/errorAccum.go

@@ -146,6 +146,12 @@ func IsRetryable(err error) bool {
 	if errors.Is(err, config.MetadataTimeoutErr) {
 		return true
 	}
+	// There's little a user can do about a TCP connection reset besides retry; if it
+	// was due to the server crashing, then on a subsequent retry they should get a different
+	// error message (connection refused).
+	if errors.Is(err, &NetworkResetError{}) {
+		return true
+	}
 	if errors.Is(err, &StoppedTransferError{}) {
 		return true
 	}

client/handle_http.go

@@ -124,6 +124,8 @@ type (
 
 	HeaderTimeoutError struct{}
 
+	NetworkResetError struct{}
+
 	allocateMemoryError struct {
 		Err error
 	}
@@ -338,6 +340,10 @@ func (e *HeaderTimeoutError) Error() string {
 	return "timeout waiting for HTTP response (TCP connection successful)"
 }
 
+func (e *NetworkResetError) Error() string {
+	return "the existing TCP connection was broken (potentially caused by server restart or NAT/firewall issue)"
+}
+
 func (e *StoppedTransferError) Error() (errMsg string) {
 	if e.StoppedTime > 0 {
 		errMsg = "no progress for more than " + e.StoppedTime.Truncate(time.Millisecond).String()
@@ -1754,6 +1760,8 @@ func downloadObject(transfer *transferFile) (transferResults TransferResults, er
 					proxyStr += "(" + ope.Addr.String() + ")"
 				}
 				attempt.Error = newTransferAttemptError(serviceStr, proxyStr, true, false, err)
+			} else if errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) {
+				attempt.Error = newTransferAttemptError(serviceStr, proxyStr, false, false, &NetworkResetError{})
 			} else if errors.As(err, &cse) {
 				if sce, ok := cse.Unwrap().(*StatusCodeError); ok {
 					attempt.Error = newTransferAttemptError(serviceStr, proxyStr, false, false, sce)
@@ -2343,6 +2351,9 @@ Loop:
 					err = &HeaderTimeoutError{}
 				}
 			}
+			if errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) {
+				err = &NetworkResetError{}
+			}
 			lastError = err
 			break Loop