From c3c8d4927808846e193def17bd535bd29894bec2 Mon Sep 17 00:00:00 2001 From: Urmas Rist Date: Tue, 5 Nov 2024 11:19:47 +0200 Subject: [PATCH] chore: Improve logging for the NO_CHUNK error * Print system error messages instead of ignoring them --- src/chunkserver/bgjobs.cc | 3 +++ src/chunkserver/chunkserver-common/cmr_disk.cc | 13 ++++++++++++- src/chunkserver/hddspacemgr.cc | 14 ++++++++++++-- src/master/chunks.cc | 14 ++++++++++++++ src/master/filesystem_operations.cc | 3 +++ src/master/matoclserv.cc | 3 ++- 6 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/chunkserver/bgjobs.cc b/src/chunkserver/bgjobs.cc index d9da1af1d..ae3bb5c54 100644 --- a/src/chunkserver/bgjobs.cc +++ b/src/chunkserver/bgjobs.cc @@ -306,6 +306,9 @@ void* job_worker(void *th_arg) { wrargs->chunkType, wrargs->blocknum, wrargs->offset, wrargs->size, wrargs->crc, wrargs->buffer); } + if (status != SAUNAFS_STATUS_OK) { + safs::log_err("Failed to write chunk id {}: {}", wrargs->chunkId, saunafs_error_string(status)); + } break; } case OP_GET_BLOCKS: diff --git a/src/chunkserver/chunkserver-common/cmr_disk.cc b/src/chunkserver/chunkserver-common/cmr_disk.cc index 4cdddab1f..16a079acb 100644 --- a/src/chunkserver/chunkserver-common/cmr_disk.cc +++ b/src/chunkserver/chunkserver-common/cmr_disk.cc @@ -1,5 +1,6 @@ #include "cmr_disk.h" +#include #include #include "chunkserver-common/chunk_interface.h" @@ -93,20 +94,30 @@ int CmrDisk::updateChunkAttributes(IChunk *chunk, bool isFromScan) { struct stat metaStat {}; if (stat(chunk->metaFilename().c_str(), &metaStat) < 0) { + safs::log_err("CmrDisk::updateChunkAttributes: could not access chunk metadata: chunk->metaFilename ({}), strerror ({})", + chunk->metaFilename(), strerror(errno)); return SAUNAFS_ERROR_NOCHUNK; } - if ((metaStat.st_mode & S_IFMT) != S_IFREG) { + if (!S_ISREG(metaStat.st_mode)) { + safs::log_critical("CmrDisk::updateChunkAttributes: chunk metadata file not a regular file: metaFilename ({})", + chunk->metaFilename()); return SAUNAFS_ERROR_NOCHUNK; } struct stat dataStat {}; if (stat(chunk->dataFilename().c_str(), &dataStat) < 0) { + safs::log_err("CmrDisk::updateChunkAttributes: could not access chunk data {}: {}", + chunk->dataFilename(), strerror(errno)); return SAUNAFS_ERROR_NOCHUNK; } if ((dataStat.st_mode & S_IFMT) != S_IFREG) { + safs::log_critical("CmrDisk::updateChunkAttributes: chunk data file not a regular file: dataFilename ({})", + chunk->dataFilename()); return SAUNAFS_ERROR_NOCHUNK; } if (!chunk->isDataFileSizeValid(dataStat.st_size)) { + safs::log_critical("CmrDisk::updateChunkAttributes: chunk data file size is invalid: data file ({}), file size ({}), block size ({}), max blocks in chunk ({})", + chunk->dataFilename(), dataStat.st_size, SFSBLOCKSIZE, chunk->maxBlocksInFile()); return SAUNAFS_ERROR_NOCHUNK; } diff --git a/src/chunkserver/hddspacemgr.cc b/src/chunkserver/hddspacemgr.cc index ec177694e..6e361da55 100644 --- a/src/chunkserver/hddspacemgr.cc +++ b/src/chunkserver/hddspacemgr.cc @@ -524,6 +524,7 @@ int hddOpen(IChunk *chunk) { int hddOpen(uint64_t chunkId, ChunkPartType chunkType) { auto *chunk = hddChunkFindAndLock(chunkId, chunkType); if (chunk == ChunkNotFound) { + safs::log_err("hddOpen: could not find chunkid {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } @@ -548,6 +549,7 @@ int hddClose(IChunk *chunk) { int hddClose(uint64_t chunkId, ChunkPartType chunkType) { auto *chunk = hddChunkFindAndLock(chunkId, chunkType); if (chunk == NULL) { + safs::log_err("hddClose: could not find chunkid {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } int status = hddClose(chunk); @@ -604,8 +606,7 @@ int hddPrefetchBlocks(uint64_t chunkId, ChunkPartType chunkType, auto *chunk = hddChunkFindAndLock(chunkId, chunkType); if (chunk == ChunkNotFound) { - safs_pretty_syslog(LOG_WARNING, "error finding chunk for prefetching: %" - PRIu64, chunkId); + safs::log_err("Couldn't find chunkID {} for prefetching", chunkId); return SAUNAFS_ERROR_NOCHUNK; } @@ -713,6 +714,7 @@ int hddRead(uint64_t chunkId, uint32_t version, ChunkPartType chunkType, auto* chunk = hddChunkFindAndLock(chunkId, chunkType); if (chunk == ChunkNotFound) { + safs::log_err("hddRead: Couldn't find chunkID {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } @@ -786,6 +788,7 @@ int hddChunkWriteBlock(uint64_t chunkId, uint32_t version, auto *chunk = hddChunkFindAndLock(chunkId, chunkType); if (chunk == ChunkNotFound) { + safs::log_err("hddChunkWriteBlock: ChunkNotFound; chunkId {}, version {}, type {}", chunkId, version, chunkType.toString()); return SAUNAFS_ERROR_NOCHUNK; } @@ -806,6 +809,7 @@ int hddChunkGetNumberOfBlocks(uint64_t chunkId, ChunkPartType chunkType, auto *chunk = hddChunkFindAndLock(chunkId, chunkType); *blocks = 0; if (chunk == ChunkNotFound) { + safs::log_err("hddChunkGetNumberOfBlocks: Couldn't find chunkID {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } @@ -920,6 +924,7 @@ static int hddInternalTestChunk(uint64_t chunkId, uint32_t version, auto *chunk = hddChunkFindAndLock(chunkId, chunkType); if (chunk == ChunkNotFound) { + safs::log_err("hddInternalTestChunk: Couldn't find chunkID {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } @@ -1006,6 +1011,7 @@ static int hddInternalDuplicate(uint64_t chunkId, uint32_t chunkVersion, originalChunk = hddChunkFindAndLock(chunkId, chunkType); if (originalChunk == ChunkNotFound) { + safs::log_err("hddInternalDuplicate: Couldn't find original chunk, ID {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } if (originalChunk->version() != chunkVersion && chunkVersion > 0) { @@ -1265,6 +1271,7 @@ int hddInternalUpdateVersion(uint64_t chunkId, uint32_t version, auto *chunk = hddChunkFindAndLock(chunkId, chunkType); if (chunk == ChunkNotFound) { + safs::log_err("hddInternalUpdateVersion: Couldn't find original chunk, ID {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } @@ -1293,6 +1300,7 @@ static int hddInternalTruncate(uint64_t chunkId, ChunkPartType chunkType, // step 1 - change version if (chunk == ChunkNotFound) { + safs::log_err("hddInternalTruncate: Couldn't find original chunk, ID {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } if (chunk->version() != oldVersion && oldVersion > 0) { @@ -1506,6 +1514,7 @@ static int hddInternalDuplicateTruncate(uint64_t chunkId, uint32_t chunkVersion, originalChunk = hddChunkFindAndLock(chunkId, chunkType); if (originalChunk == nullptr) { + safs::log_err("hddInternalDuplicateTruncate: Couldn't find original chunk, ID {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } if (originalChunk->version() != chunkVersion && chunkVersion > 0) { @@ -1998,6 +2007,7 @@ int hddInternalDelete(uint64_t chunkId, uint32_t version, auto *chunk = hddChunkFindAndLock(chunkId, chunkType); if (chunk == ChunkNotFound) { + safs::log_err("hddInternalDelete: could not find chunkid {}", chunkId); return SAUNAFS_ERROR_NOCHUNK; } diff --git a/src/master/chunks.cc b/src/master/chunks.cc index fb1ea0d82..d2b761e1a 100644 --- a/src/master/chunks.cc +++ b/src/master/chunks.cc @@ -896,6 +896,7 @@ int chunk_change_file(uint64_t chunkid,uint8_t prevgoal,uint8_t newgoal) { } c = chunk_find(chunkid); if (c==NULL) { + safs::log_err("chunk_change_file: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } try { @@ -936,6 +937,7 @@ int chunk_delete_file(uint64_t chunkid,uint8_t goal) { Chunk *c; c = chunk_find(chunkid); if (c==NULL) { + safs::log_err("chunk_delete_file: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } return chunk_delete_file_int(c,goal); @@ -945,6 +947,7 @@ int chunk_add_file(uint64_t chunkid,uint8_t goal) { Chunk *c; c = chunk_find(chunkid); if (c==NULL) { + safs::log_err("chunk_add_file: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } return chunk_add_file_int(c,goal); @@ -954,6 +957,7 @@ int chunk_can_unlock(uint64_t chunkid, uint32_t lockid) { Chunk *c; c = chunk_find(chunkid); if (c==NULL) { + safs::log_err("chunk_can_unlock: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } if (lockid == 0) { @@ -977,6 +981,7 @@ int chunk_unlock(uint64_t chunkid) { Chunk *c; c = chunk_find(chunkid); if (c==NULL) { + safs::log_err("chunk_unlock: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } // Don't remove lockid to safely accept retransmission of FUSE_CHUNK_UNLOCK message @@ -1012,6 +1017,7 @@ int chunk_get_fullcopies(uint64_t chunkid,uint8_t *vcopies) { *vcopies = 0; c = chunk_find(chunkid); if (c==NULL) { + safs::log_err("chunk_get_fullcopies: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } @@ -1026,6 +1032,7 @@ int chunk_get_partstomodify(uint64_t chunkid, int &recover, int &remove) { remove = 0; c = chunk_find(chunkid); if (c==NULL) { + safs::log_err("chunk_get_partstomodify: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } recover = c->countMissingParts(); @@ -1076,6 +1083,7 @@ uint8_t chunk_multi_modify(uint64_t ochunkid, uint32_t *lockid, uint8_t goal, } else { Chunk *oc = chunk_find(ochunkid); if (oc==NULL) { + safs::log_err("chunk_multi_modify: could not find chunkid {}", ochunkid); return SAUNAFS_ERROR_NOCHUNK; } if (*lockid != 0 && *lockid != oc->lockid) { @@ -1173,6 +1181,7 @@ uint8_t chunk_multi_truncate(uint64_t ochunkid, uint32_t lockid, uint32_t length c=NULL; oc = chunk_find(ochunkid); if (oc==NULL) { + safs::log_err("chunk_multi_truncate: could not find chunkid {}", ochunkid); return SAUNAFS_ERROR_NOCHUNK; } if (!oc->isWritable()) { @@ -1253,6 +1262,7 @@ uint8_t chunk_apply_modification(uint32_t ts, uint64_t oldChunkId, uint32_t lock } else { Chunk *oc = chunk_find(oldChunkId); if (oc == NULL) { + safs::log_err("chunk_apply_modification: could not find old chunkid {}", oldChunkId); return SAUNAFS_ERROR_NOCHUNK; } if (oc->fileCount() == 0) { // refcount == 0 @@ -1353,6 +1363,7 @@ int chunk_set_version(uint64_t chunkid,uint32_t version) { Chunk *c; c = chunk_find(chunkid); if (c==NULL) { + safs::log_err("chunk_set_version: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } c->version = version; @@ -1364,6 +1375,7 @@ int chunk_increase_version(uint64_t chunkid) { Chunk *c; c = chunk_find(chunkid); if (c==NULL) { + safs::log_err("chunk_increase_version: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } c->version++; @@ -1424,6 +1436,7 @@ int chunk_getversionandlocations(uint64_t chunkid, uint32_t currentIp, uint32_t& c = chunk_find(chunkid); if (c == NULL) { + safs::log_err("chunk_getversionandlocations: could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } version = c->version; @@ -1464,6 +1477,7 @@ int chunk_getversionandlocations(uint64_t chunkid, uint32_t currentIp, uint32_t& c = chunk_find(chunkid); if (c == NULL) { + safs::log_err("chunk_getversionandlocations (2?): could not find chunkid {}", chunkid); return SAUNAFS_ERROR_NOCHUNK; } version = c->version; diff --git a/src/master/filesystem_operations.cc b/src/master/filesystem_operations.cc index f2bb9b49e..f661da549 100644 --- a/src/master/filesystem_operations.cc +++ b/src/master/filesystem_operations.cc @@ -598,6 +598,7 @@ uint8_t fs_apply_trunc(uint32_t ts, uint32_t inode, uint32_t indx, uint64_t chun } ochunkid = p->chunks[indx]; if (ochunkid == 0) { + safs::log_err("fs_apply_trunc: node does not have a chunk at index {} chunks, inode {}", indx, inode); return SAUNAFS_ERROR_NOCHUNK; } status = chunk_apply_modification(ts, ochunkid, lockid, p->goal, true, &nchunkid); @@ -2135,8 +2136,10 @@ uint8_t fs_apply_repair(uint32_t ts, uint32_t inode, uint32_t indx, uint32_t nve } if (indx >= p->chunks.size()) { return SAUNAFS_ERROR_NOCHUNK; + safs::log_err("fs_apply_repair: indx {} is greater than number of chunks ({}), inode {}", indx, p->chunks.size(), inode); } if (p->chunks[indx] == 0) { + safs::log_err("fs_apply_repair: node chunks at index {} has no chunks, inode {}", indx, inode); return SAUNAFS_ERROR_NOCHUNK; } fsnodes_get_stats(p, &psr); diff --git a/src/master/matoclserv.cc b/src/master/matoclserv.cc index 9fa48e0bb..5cccbba37 100644 --- a/src/master/matoclserv.cc +++ b/src/master/matoclserv.cc @@ -1024,6 +1024,7 @@ uint8_t matoclserv_fuse_write_chunk_respond(matoclserventry *eptr, if (status == SAUNAFS_STATUS_OK && !serializer->isSaunaFsPacketSerializer()) { for (const ChunkTypeWithAddress& chunkCopy : allChunkCopies) { if (!slice_traits::isStandard(chunkCopy.chunk_type)) { + safs::log_err("matoclserv_fuse_write_chunk_respond: client tried to modify standard copy of a xor chunk, chunkID {}", chunkId); status = SAUNAFS_ERROR_NOCHUNK; break; } @@ -1922,7 +1923,7 @@ void matoclserv_fuse_reserved_inodes(matoclserventry *eptr,const uint8_t *data,u auto it = eptr->sesdata->openedfiles.begin(); while (it != eptr->sesdata->openedfiles.end()) { uint32_t openFileIno = *it; - if (!inodes_to_reserve.contains(openFileIno)) { + if (!inodes_to_reserve.contains(openFileIno)) { // erase files not belonging to the reserve inodes list provided fs_release(context, openFileIno, eptr->sesdata->sessionid); it = eptr->sesdata->openedfiles.erase(it);