[Bug]: Flakey Test TestLakectlFsUpload

[nadavsteindler]

What happened?

What actually happened, including error codes if applies.

Test fails randomly in PR https://github.com/treeverse/lakeFS-Enterprise/actions/runs/13617318919/job/38062161221?pr=136

esti-1 | panic: test timed out after 10m0s
esti-1 | running tests:
esti-1 | TestLakectlFsUpload (7m46s)
esti-1 | TestLakectlFsUpload/exist_dir (7m45s)
esti-1 |
esti-1 | goroutine 754 [running]:
esti-1 | testing.(*M).startAlarm.func1()
esti-1 | /usr/local/go/src/testing/testing.go:2373 +0x385
esti-1 | created by time.goFunc
esti-1 | /usr/local/go/src/time/sleep.go:215 +0x2d
esti-1 |
esti-1 | goroutine 1 [chan receive, 7 minutes]:
esti-1 | testing.(*T).Run(0xc00014a4e0, {0x1b34976?, 0x0?}, 0x1bce850)
esti-1 | /usr/local/go/src/testing/testing.go:1751 +0x3ab
esti-1 | testing.runTests.func1(0xc00014a4e0)
esti-1 | /usr/local/go/src/testing/testing.go:2168 +0x37
esti-1 | testing.tRunner(0xc00014a4e0, 0xc0000d1b38)
esti-1 | /usr/local/go/src/testing/testing.go:1690 +0xf4
esti-1 | testing.runTests(0xc000010ca8, {0x2ac0560, 0x6a, 0x6a}, {0x0?, 0xa3?, 0x2acbca0?})
esti-1 | /usr/local/go/src/testing/testing.go:2166 +0x43d
esti-1 | testing.(*M).Run(0xc00027b540)
esti-1 | /usr/local/go/src/testing/testing.go:2034 +0x64a
esti-1 | github.com/treeverse/lakefs/esti.TestMain(0xc00027b540)
esti-1 | /lakefs/esti/main_test.go:65 +0x8f7
esti-1 | main.main()
esti-1 | _testmain.go:257 +0xa8
esti-1 |
esti-1 | goroutine 18 [select]:
esti-1 | github.com/golang/glog.(*fileSink).flushDaemon(0x2acbdb8)
esti-1 | /go/pkg/mod/github.com/golang/[email protected]/glog_file.go:351 +0xb4
esti-1 | created by github.com/golang/glog.init.1 in goroutine 1
esti-1 | /usr/local/go/src/os/exec/exec.go:906 +0x45
esti-1 | os/exec.(*Cmd).Run(0xc000311e00)
esti-1 | /usr/local/go/src/os/exec/exec.go:610 +0x2d
esti-1 | os/exec.(*Cmd).CombinedOutput(0xc000311e00)
esti-1 | /usr/local/go/src/os/exec/exec.go:1021 +0x85
esti-1 | github.com/treeverse/lakefs/esti.runShellCommand(0xc0006dcb60, {0xc00054b340, 0x14d}, 0x0)
esti-1 | /lakefs/esti/lakectl_util.go:75 +0x205
esti-1 | github.com/treeverse/lakefs/esti.runCmd(0xc0006dcb60, {0xc00054b340, 0x14d}, 0x0, 0x0, 0xc0000db6b0)
esti-1 | /lakefs/esti/lakectl_util.go:222 +0x76
esti-1 | github.com/treeverse/lakefs/esti.TestLakectlFsUpload.func5(0xc0006dcb60)
esti-1 | /lakefs/esti/lakectl_test.go:730 +0x1a5
esti-1 | testing.tRunner(0xc0006dcb60, 0xc000499cc0)
esti-1 | /usr/local/go/src/testing/testing.go:1690 +0xf4
esti-1 | created by testing.(*T).Run in goroutine 710
esti-1 | /usr/local/go/src/testing/testing.go:1743 +0x390
esti-1 |
esti-1 | goroutine 753 [IO wait, 7 minutes]:
esti-1 | internal/poll.runtime_pollWait(0x7fc84398cef0, 0x72)
esti-1 | /usr/local/go/src/runtime/netpoll.go:351 +0x85
esti-1 | internal/poll.(*pollDesc).wait(0xc000136de0?, 0xc000595052?, 0x1)
esti-1 | /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27
esti-1 | internal/poll.(*pollDesc).waitRead(...)
esti-1 | /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
esti-1 | internal/poll.(*FD).Read(0xc000136de0, {0xc000595052, 0x3ae, 0x3ae})
esti-1 | /usr/local/go/src/internal/poll/fd_unix.go:165 +0x27a
esti-1 | os.(*File).read(...)
esti-1 | /usr/local/go/src/os/file_posix.go:29
esti-1 | os.(*File).Read(0xc00007a900, {0xc000595052?, 0x9?, 0x52?})
esti-1 | /usr/local/go/src/os/file.go:124 +0x52
esti-1 | bytes.(*Buffer).ReadFrom(0xc0000dbdd0, {0x1dc86c0, 0xc0001255d0})
esti-1 | /usr/local/go/src/bytes/buffer.go:211 +0x98
esti-1 | io.copyBuffer({0x1dc8860, 0xc0000dbdd0}, {0x1dc86c0, 0xc0001255d0}, {0x0, 0x0, 0x0})
esti-1 | /usr/local/go/src/io/io.go:415 +0x151
esti-1 | io.Copy(...)
esti-1 | /usr/local/go/src/io/io.go:388
esti-1 | os.genericWriteTo(0xc00007a900?, {0x1dc8860, 0xc0000dbdd0})
esti-1 | /usr/local/go/src/os/file.go:275 +0x4f
esti-1 | os.(*File).WriteTo(0xc00007a900, {0x1dc8860, 0xc0000dbdd0})
esti-1 | /usr/local/go/src/os/file.go:253 +0x9c
esti-1 | io.copyBuffer({0x1dc8860, 0xc0000dbdd0}, {0x1dc8740, 0xc00007a900}, {0x0, 0x0, 0x0})
esti-1 | /usr/local/go/src/io/io.go:411 +0x9d
esti-1 | io.Copy(...)
esti-1 | /usr/local/go/src/io/io.go:388
esti-1 | os/exec.(*Cmd).writerDescriptor.func1()
esti-1 | /usr/local/go/src/os/exec/exec.go:580 +0x34
esti-1 | os/exec.(*Cmd).Start.func2(0xc000499cc0?)
esti-1 | /usr/local/go/src/os/exec/exec.go:733 +0x2c
esti-1 | created by os/exec.(*Cmd).Start in goroutine 720
esti-1 | /usr/local/go/src/os/exec/exec.go:732 +0x98b
esti-1 | FAIL github.com/treeverse/lakefs/esti 600.203s
esti-1 | FAIL
Aborting on container exit...

Expected behavior

No response

lakeFS version

No response

How lakeFS is installed

No response

Affected clients

No response

Relevant log output

Contact details

No response

[itaiad200]

@nadavsteindler how often does this happen?

[nadavsteindler]

@nadavsteindler how often does this happen?

This one just happened to me once. Not sure if others have encountered it...