From 3591bc13356d5eeb919cd7fd8720e9ef6b166749 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 12 Jun 2024 20:06:44 +0300 Subject: [PATCH 1/2] shard: improve logging on object putting Masked error-less shard skipping makes debugging awful, relates #2860. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/engine/evacuate.go | 2 ++ pkg/local_object_storage/engine/put.go | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 3f127b75d9..aaade013fd 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -164,6 +164,8 @@ mainLoop: } continue loop } + + e.log.Debug("could not put to shard, trying another", zap.String("shard", shards[j].ID().String())) } if prm.handler == nil { diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index a8791f37d2..e85a3c9a7d 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -114,6 +114,11 @@ func (e *StorageEngine) putToShard(sh hashedShard, ind int, pool util.WorkerPool exists, err := sh.Exists(existPrm) if err != nil { + e.log.Warn("object put: check object existence", + zap.Stringer("addr", addr), + zap.Stringer("shard", sh.ID()), + zap.Error(err)) + if shard.IsErrObjectExpired(err) { // object is already found but // expired => do nothing with it @@ -138,6 +143,10 @@ func (e *StorageEngine) putToShard(sh hashedShard, ind int, pool util.WorkerPool } } + e.log.Debug("object put: object already exists", + zap.Stringer("shard", sh.ID()), + zap.Stringer("addr", addr)) + return } @@ -163,6 +172,7 @@ func (e *StorageEngine) putToShard(sh hashedShard, ind int, pool util.WorkerPool putSuccess = true }); err != nil { + e.log.Warn("object put: pool task submitting", zap.Error(err)) close(exitCh) } From 9680cd57b2635996816ea1bf472bc035cd19f436 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 12 Jun 2024 20:11:47 +0300 Subject: [PATCH 2/2] engine: fix `TestEvacuateShard` It looks like on a slow (few-core?) machine put operation can be failed because the internal pool has not been freed from the previous iteration (another shard is tried then and additional "fake" relocation is detected in the test). Closes #2860. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/engine/evacuate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 29563217f5..8bd45879a0 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -28,7 +28,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng e := New( WithLogger(zaptest.NewLogger(t)), - WithShardPoolSize(1)) + WithShardPoolSize(uint32(objPerShard))) ids := make([]*shard.ID, shardNum)