Add timeout for stage writings to avoid EIO when it hangs for more th…

…an 5 minutes (#5230)

Signed-off-by: Changxin Miao <[email protected]>

pkg/chunk/cached_store.go

@@ -447,7 +447,11 @@ func (s *wSlice) upload(indx int) {
 			panic(fmt.Sprintf("block length does not match: %v != %v", off, blen))
 		}
 		if s.store.conf.Writeback {
-			stagingPath, err := s.store.bcache.stage(key, block.Data, s.store.shouldCache(blen))
+			stagingPath := "unknown"
+			err := utils.WithTimeout(func() (err error) { // In case it hangs for more than 5 minutes(see fileWriter.flush), fallback to uploading directly to avoid `EIO`
+				stagingPath, err = s.store.bcache.stage(key, block.Data, s.store.shouldCache(blen))
+				return err
+			}, s.store.conf.PutTimeout)
 			if err != nil {
 				if !errors.Is(err, errStageConcurrency) {
 					s.store.stageBlockErrors.Add(1)