add check before put/get/del according to eBay/HomeStore#610

eBay · Jan 3, 2025 · b66acf5 · b66acf5
1 parent 067612f
commit b66acf5
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 3 deletions.
diff --git a/conanfile.py b/conanfile.py
@@ -9,7 +9,7 @@
 
 class HomeObjectConan(ConanFile):
     name = "homeobject"
-    version = "2.1.19"
+    version = "2.1.20"
 
     homepage = "https://github.com/eBay/HomeObject"
     description = "Blob Store built on HomeReplication"

diff --git a/src/include/homeobject/blob_manager.hpp b/src/include/homeobject/blob_manager.hpp
@@ -10,7 +10,7 @@
 namespace homeobject {
 
 ENUM(BlobErrorCode, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, UNSUPPORTED_OP, NOT_LEADER, REPLICATION_ERROR,
-     UNKNOWN_SHARD, UNKNOWN_BLOB, CHECKSUM_MISMATCH, READ_FAILED, INDEX_ERROR, SEALED_SHARD);
+     UNKNOWN_SHARD, UNKNOWN_BLOB, CHECKSUM_MISMATCH, READ_FAILED, INDEX_ERROR, SEALED_SHARD, RETRY_REQUEST);
 struct BlobError {
     BlobErrorCode code;
     // set when we are not the current leader of the PG.

diff --git a/src/include/homeobject/shard_manager.hpp b/src/include/homeobject/shard_manager.hpp
@@ -10,7 +10,7 @@
 namespace homeobject {
 
 ENUM(ShardError, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, NOT_LEADER, UNSUPPORTED_OP, UNKNOWN_PG, UNKNOWN_SHARD,
-     PG_NOT_READY, CRC_MISMATCH, NO_SPACE_LEFT);
+     PG_NOT_READY, CRC_MISMATCH, NO_SPACE_LEFT, RETRY_REQUEST);
 
 struct ShardInfo {
     enum class State : uint8_t {

diff --git a/src/lib/homestore_backend/hs_blob_manager.cpp b/src/lib/homestore_backend/hs_blob_manager.cpp
@@ -104,6 +104,11 @@ BlobManager::AsyncResult< blob_id_t > HSHomeObject::_put_blob(ShardInfo const& s
         return folly::makeUnexpected(BlobError(BlobErrorCode::NOT_LEADER, repl_dev->get_leader_id()));
     }
 
+    if (!repl_dev->is_ready_for_traffic()) {
+        LOGW("failed to put blob for pg [{}], shard [{}], not ready for traffic", pg_id, shard.id);
+        return folly::makeUnexpected(BlobError(BlobErrorCode::RETRY_REQUEST));
+    }
+
     // Create a put_blob request which allocates for header, key and blob_header, user_key. Data sgs are added later
     auto req = put_blob_req_ctx::make(sizeof(BlobHeader) + blob.user_key.size());
     req->header()->msg_type = ReplicationMessageType::PUT_BLOB_MSG;
@@ -260,6 +265,11 @@ BlobManager::AsyncResult< Blob > HSHomeObject::_get_blob(ShardInfo const& shard,
     RELEASE_ASSERT(repl_dev != nullptr, "Repl dev instance null");
     RELEASE_ASSERT(index_table != nullptr, "Index table instance null");
 
+    if (!repl_dev->is_ready_for_traffic()) {
+        LOGW("failed to get blob for pg [{}], shard [{}], not ready for traffic", pg_id, shard.id);
+        return folly::makeUnexpected(BlobError(BlobErrorCode::RETRY_REQUEST));
+    }
+
     auto r = get_blob_from_index_table(index_table, shard.id, blob_id);
     if (!r) {
         BLOGE(shard.id, blob_id, "Blob not found in index during get blob");
@@ -381,6 +391,11 @@ BlobManager::NullAsyncResult HSHomeObject::_del_blob(ShardInfo const& shard, blo
         return folly::makeUnexpected(BlobError(BlobErrorCode::NOT_LEADER, repl_dev->get_leader_id()));
     }
 
+    if (!repl_dev->is_ready_for_traffic()) {
+        LOGW("failed to del blob for pg [{}], shard [{}], not ready for traffic", pg_id, shard.id);
+        return folly::makeUnexpected(BlobError(BlobErrorCode::RETRY_REQUEST));
+    }
+
     // Create an unaligned header request unaligned
     auto req = repl_result_ctx< BlobManager::Result< BlobInfo > >::make(0u /* header_extn */,
                                                                         sizeof(blob_id_t) /* key_size */);

diff --git a/src/lib/homestore_backend/hs_shard_manager.cpp b/src/lib/homestore_backend/hs_shard_manager.cpp
@@ -114,6 +114,11 @@ ShardManager::AsyncResult< ShardInfo > HSHomeObject::_create_shard(pg_id_t pg_ow
         return folly::makeUnexpected(ShardError::NOT_LEADER);
     }
 
+    if (!repl_dev->is_ready_for_traffic()) {
+        LOGW("failed to create shard for pg [{}], not ready for traffic", pg_owner);
+        return folly::makeUnexpected(ShardError::RETRY_REQUEST);
+    }
+
     auto new_shard_id = generate_new_shard_id(pg_owner);
     auto create_time = get_current_timestamp();
 
@@ -183,6 +188,11 @@ ShardManager::AsyncResult< ShardInfo > HSHomeObject::_seal_shard(ShardInfo const
         return folly::makeUnexpected(ShardError::NOT_LEADER);
     }
 
+    if (!repl_dev->is_ready_for_traffic()) {
+        LOGW("failed to seal shard for shard [{}], not ready for traffic", shard_id);
+        return folly::makeUnexpected(ShardError::RETRY_REQUEST);
+    }
+
     ShardInfo tmp_info = info;
     tmp_info.state = ShardInfo::State::SEALED;