From 062c1985973ab6e0aadae3775f36ed70989f728b Mon Sep 17 00:00:00 2001 From: Samarah Date: Tue, 29 Oct 2024 22:07:19 +0000 Subject: [PATCH] rgw/d4n: Remove dirty prefix and make xattr instead Signed-off-by: Samarah --- src/rgw/driver/d4n/d4n_policy.cc | 41 ++-- src/rgw/driver/d4n/rgw_sal_d4n.cc | 132 +++++------- src/rgw/driver/d4n/rgw_sal_d4n.h | 2 +- src/rgw/rgw_cache_driver.h | 2 +- src/rgw/rgw_ssd_driver.cc | 336 ++++++++++++++---------------- 5 files changed, 220 insertions(+), 293 deletions(-) diff --git a/src/rgw/driver/d4n/d4n_policy.cc b/src/rgw/driver/d4n/d4n_policy.cc index 96455752e249c7..97b74867d99a8a 100644 --- a/src/rgw/driver/d4n/d4n_policy.cc +++ b/src/rgw/driver/d4n/d4n_policy.cc @@ -263,7 +263,7 @@ bool LFUDAPolicy::invalidate_dirty_object(const DoutPrefixProvider* dpp, std::st if (p->second.second == State::INIT) { ldpp_dout(dpp, 10) << "LFUDAPolicy::" << __func__ << "(): Setting State::INVALID for key=" << key << dendl; p->second.second = State::INVALID; - int ret = cacheDriver->set_attr(dpp, DIRTY_BLOCK_PREFIX + key, RGW_CACHE_ATTR_INVALID, "1", y); + int ret = cacheDriver->set_attr(dpp, key, RGW_CACHE_ATTR_INVALID, "1", y); if (ret < 0) { ldpp_dout(dpp, 0) << "LFUDAPolicy::" << __func__ << "(): Failed to set xattr, ret=" << ret << dendl; return false; @@ -433,11 +433,6 @@ void LFUDAPolicy::update(const DoutPrefixProvider* dpp, std::string& key, uint64 bool updateLocalWeight = true; uint64_t refcount = 0; - std::string oid_in_cache = key; - if (dirty == true) { - oid_in_cache = DIRTY_BLOCK_PREFIX + key; - } - if (!restore_val.empty()) { updateLocalWeight = false; localWeight = std::stoull(restore_val); @@ -473,7 +468,7 @@ void LFUDAPolicy::update(const DoutPrefixProvider* dpp, std::string& key, uint64 if (updateLocalWeight) { int ret = -1; - if ((ret = cacheDriver->set_attr(dpp, oid_in_cache, RGW_CACHE_ATTR_LOCAL_WEIGHT, std::to_string(localWeight), y)) < 0) + if ((ret = cacheDriver->set_attr(dpp, key, RGW_CACHE_ATTR_LOCAL_WEIGHT, std::to_string(localWeight), y)) < 0) ldpp_dout(dpp, 0) << "LFUDAPolicy::" << __func__ << "(): CacheDriver set_attr method failed, ret=" << ret << dendl; } @@ -550,12 +545,11 @@ int LFUDAPolicy::delete_data_blocks(const DoutPrefixProvider* dpp, LFUDAObjEntry } off_t cur_size = std::min(fst + dpp->get_cct()->_conf->rgw_max_chunk_size, lst); off_t cur_len = cur_size - fst; - std::string prefix = e->key + CACHE_DELIM + std::to_string(fst) + CACHE_DELIM + std::to_string(cur_len); - std::string oid_in_cache = DIRTY_BLOCK_PREFIX + prefix; + std::string oid_in_cache = e->key + CACHE_DELIM + std::to_string(fst) + CACHE_DELIM + std::to_string(cur_len); int ret = -1; std::unique_lock ll(lfuda_lock); - auto it = entries_map.find(prefix); + auto it = entries_map.find(oid_in_cache); if (it != entries_map.end()) { if (it->second->refcount > 0) { return -EBUSY;//better error code? @@ -563,7 +557,7 @@ int LFUDAPolicy::delete_data_blocks(const DoutPrefixProvider* dpp, LFUDAObjEntry } ll.unlock(); if ((ret = cacheDriver->delete_data(dpp, oid_in_cache, y)) == 0) { // Sam: do we want del or delete_data here? - if (!(ret = erase(dpp, prefix, y))) { + if (!(ret = erase(dpp, oid_in_cache, y))) { ldpp_dout(dpp, 0) << "Failed to delete policy entry for: " << oid_in_cache << ", ret=" << ret << dendl; return -EINVAL; } @@ -638,7 +632,7 @@ void LFUDAPolicy::cleaning(const DoutPrefixProvider* dpp) continue; } ll.unlock(); - if ((ret = cacheDriver->delete_data(dpp, DIRTY_BLOCK_PREFIX + e->key, y)) == 0) { // Sam: do we want del or delete_data here? + if ((ret = cacheDriver->delete_data(dpp, e->key, y)) == 0) { // Sam: do we want del or delete_data here? if (!(ret = erase(dpp, e->key, y))) { ldpp_dout(dpp, 0) << "Failed to delete head policy entry for: " << e->key << ", ret=" << ret << dendl; // TODO: what must occur during failure? } @@ -696,11 +690,8 @@ void LFUDAPolicy::cleaning(const DoutPrefixProvider* dpp) ACLOwner owner{c_user->get_id(), c_user->get_display_name()}; - std::string prefix = url_encode(e->bucket_id) + CACHE_DELIM + url_encode(e->version) + CACHE_DELIM + url_encode(c_obj->get_name()); - std::string head_oid_in_cache = DIRTY_BLOCK_PREFIX + prefix; - std::string new_head_oid_in_cache = prefix; + std::string head_oid_in_cache = url_encode(e->bucket_id) + CACHE_DELIM + url_encode(e->version) + CACHE_DELIM + url_encode(c_obj->get_name()); ldpp_dout(dpp, 10) << __func__ << "(): head_oid_in_cache=" << head_oid_in_cache << dendl; - ldpp_dout(dpp, 10) << __func__ << "(): new_head_oid_in_cache=" << new_head_oid_in_cache << dendl; int op_ret; if (e->delete_marker) { bool null_delete_marker = (c_obj->get_instance() == "null"); @@ -774,7 +765,7 @@ void LFUDAPolicy::cleaning(const DoutPrefixProvider* dpp) } off_t cur_size = std::min(fst + dpp->get_cct()->_conf->rgw_max_chunk_size, lst); off_t cur_len = cur_size - fst; - std::string oid_in_cache = DIRTY_BLOCK_PREFIX + prefix + CACHE_DELIM + std::to_string(fst) + CACHE_DELIM + std::to_string(cur_len); + std::string oid_in_cache = head_oid_in_cache + CACHE_DELIM + std::to_string(fst) + CACHE_DELIM + std::to_string(cur_len); ldpp_dout(dpp, 10) << __func__ << "(): oid_in_cache=" << oid_in_cache << dendl; rgw::sal::Attrs attrs; cacheDriver->get(dpp, oid_in_cache, 0, cur_len, data, attrs, null_yield); @@ -827,13 +818,10 @@ void LFUDAPolicy::cleaning(const DoutPrefixProvider* dpp) off_t cur_size = std::min(fst + dpp->get_cct()->_conf->rgw_max_chunk_size, lst); off_t cur_len = cur_size - fst; - std::string oid_in_cache = DIRTY_BLOCK_PREFIX + prefix + CACHE_DELIM + std::to_string(fst) + CACHE_DELIM + std::to_string(cur_len); + std::string oid_in_cache = head_oid_in_cache + CACHE_DELIM + std::to_string(fst) + CACHE_DELIM + std::to_string(cur_len); ldpp_dout(dpp, 20) << __func__ << "(): oid_in_cache =" << oid_in_cache << dendl; - std::string new_oid_in_cache = prefix + CACHE_DELIM + std::to_string(fst) + CACHE_DELIM + std::to_string(cur_len); - //Rename block to remove "D" prefix - cacheDriver->rename(dpp, oid_in_cache, new_oid_in_cache, null_yield); //Update in-memory data structure for each block - this->update(dpp, new_oid_in_cache, 0, 0, e->version, false, 0, y); + this->update(dpp, oid_in_cache, 0, 0, e->version, false, 0, y); rgw::d4n::CacheBlock block; block.cacheObj.bucketName = c_obj->get_bucket()->get_bucket_id(); @@ -848,10 +836,8 @@ void LFUDAPolicy::cleaning(const DoutPrefixProvider* dpp) } while(fst < lst); } //end-else if delete_marker - cacheDriver->rename(dpp, head_oid_in_cache, new_head_oid_in_cache, null_yield); - //invoke update() with dirty flag set to false, to update in-memory metadata for head - this->update(dpp, new_head_oid_in_cache, 0, 0, e->version, false, 0, y); + this->update(dpp, head_oid_in_cache, 0, 0, e->version, false, 0, y); if (null_instance) { //restore instance for directory data processing in later steps @@ -981,6 +967,11 @@ void LFUDAPolicy::cleaning(const DoutPrefixProvider* dpp) break; }//end-while (retry) } + + if ((ret = cacheDriver->set_attr(dpp, head_oid_in_cache, RGW_CACHE_ATTR_DIRTY, "0", y)) < 0) { + ldpp_dout(dpp, 0) << __func__ << "(): Failed to update dirty attr in cache, ret=" << op_ret << dendl; + // TODO: continue here? + } //remove entry from map and queue, erase_dirty_object locks correctly erase_dirty_object(dpp, e->key, null_yield); } diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc index 4d79258227d24e..1ab4fcc3a55bc4 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.cc +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -34,12 +34,8 @@ static inline Object* nextObject(Object* t) return dynamic_cast(t)->get_next(); } -static std::string get_cache_block_prefix(rgw::sal::Object* object, std::string version, bool is_dirty) { - if (is_dirty) { - return DIRTY_BLOCK_PREFIX + url_encode(object->get_bucket()->get_bucket_id(), true) + CACHE_DELIM + url_encode(version, true) + CACHE_DELIM + url_encode(object->get_name(), true); - } else { - return url_encode(object->get_bucket()->get_bucket_id(), true) + CACHE_DELIM + url_encode(version, true) + CACHE_DELIM + url_encode(object->get_name(), true); - } +static std::string get_cache_block_prefix(rgw::sal::Object* object, std::string version) { + return url_encode(object->get_bucket()->get_bucket_id(), true) + CACHE_DELIM + url_encode(version, true) + CACHE_DELIM + url_encode(object->get_name(), true); } D4NFilterDriver::D4NFilterDriver(Driver* _next, boost::asio::io_context& io_context) : FilterDriver(_next), @@ -277,11 +273,11 @@ int D4NFilterObject::copy_object(const ACLOwner& owner, dest_object->set_obj_size(this->get_size()); dest_object->set_accounted_size(this->get_accounted_size()); ldpp_dout(dpp, 20) << "D4NFilterObject::" << __func__ << " size is: " << dest_object->get_size() << dendl; - d4n_dest_object->set_attrs_from_obj_state(dpp, y, baseAttrs); + d4n_dest_object->set_attrs_from_obj_state(dpp, y, baseAttrs, dirty); } else { dest_object->load_obj_state(dpp, y); baseAttrs = dest_object->get_attrs(); - d4n_dest_object->set_attrs_from_obj_state(dpp, y, baseAttrs); + d4n_dest_object->set_attrs_from_obj_state(dpp, y, baseAttrs, dirty); auto ret = d4n_dest_object->calculate_version(dpp, y, dest_version); if (ret < 0 || dest_version.empty()) { ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): version could not be calculated." << dendl; @@ -294,13 +290,9 @@ int D4NFilterObject::copy_object(const ACLOwner& owner, bufferlist bl_data; dest_version = d4n_dest_object->get_object_version(); - std::string key = get_cache_block_prefix(dest_object, dest_version, false); + std::string key = get_cache_block_prefix(dest_object, dest_version); std::string head_oid_in_cache; - if (dirty) { - head_oid_in_cache = DIRTY_BLOCK_PREFIX + key; //same as key, as there is no len or offset attached to head oid in cache - } else { - head_oid_in_cache = key; - } + head_oid_in_cache = key; //same as key, as there is no len or offset attached to head oid in cache auto ret = driver->get_policy_driver()->get_cache_policy()->eviction(dpp, baseAttrs.size(), y); if (ret == 0) { ret = driver->get_cache_driver()->put(dpp, head_oid_in_cache, bl_data, 0, baseAttrs, y); @@ -311,6 +303,7 @@ int D4NFilterObject::copy_object(const ACLOwner& owner, baseAttrs.erase(RGW_CACHE_ATTR_MULTIPART); baseAttrs.erase(RGW_CACHE_ATTR_OBJECT_NS); baseAttrs.erase(RGW_CACHE_ATTR_BUCKET_NAME); + baseAttrs.erase(RGW_CACHE_ATTR_DIRTY); if (ret == 0) { ldpp_dout(dpp, 20) << "D4NFilterObject::" << __func__ << " version stored in update method is: " << dest_version << dendl; bufferlist bl; @@ -493,7 +486,7 @@ int D4NFilterObject::set_attr_crypt_parts(const DoutPrefixProvider* dpp, optiona return 0; } -void D4NFilterObject::set_attrs_from_obj_state(const DoutPrefixProvider* dpp, optional_yield y, rgw::sal::Attrs& attrs) +void D4NFilterObject::set_attrs_from_obj_state(const DoutPrefixProvider* dpp, optional_yield y, rgw::sal::Attrs& attrs, bool dirty) { bufferlist bl_val; bl_val.append(std::to_string(this->get_size())); @@ -521,6 +514,13 @@ void D4NFilterObject::set_attrs_from_obj_state(const DoutPrefixProvider* dpp, op bl_val.append(this->get_bucket()->get_name()); attrs[RGW_CACHE_ATTR_BUCKET_NAME] = std::move(bl_val); + + if (dirty) { + bl_val.append("1"); + } else { + bl_val.append("0"); + } + attrs[RGW_CACHE_ATTR_DIRTY] = std::move(bl_val); return; } @@ -573,18 +573,17 @@ int D4NFilterObject::create_delete_marker(const DoutPrefixProvider* dpp, optiona this->set_obj_size(0); // setting 0 as this is a delete marker ldpp_dout(dpp, 20) << "D4NFilterObject::" << __func__ << " size is: " << this->get_size() << dendl; rgw::sal::Attrs attrs; - this->set_attrs_from_obj_state(dpp, y, attrs); + this->set_attrs_from_obj_state(dpp, y, attrs, true); bufferlist bl_val; bl_val.append(std::to_string(this->delete_marker)); attrs[RGW_CACHE_ATTR_DELETE_MARKER] = std::move(bl_val); - std::string key = get_cache_block_prefix(this, this->version, false); - std::string oid_in_cache = DIRTY_BLOCK_PREFIX + key; + std::string key = get_cache_block_prefix(this, this->version); bufferlist bl; ldpp_dout(dpp, 20) << "D4NFilterObject::" << __func__ << "(): key is: " << key << dendl; auto ret = driver->get_policy_driver()->get_cache_policy()->eviction(dpp, attrs.size(), y); if (ret == 0) { - ret = driver->get_cache_driver()->put(dpp, oid_in_cache, bl, 0, attrs, y); + ret = driver->get_cache_driver()->put(dpp, key, bl, 0, attrs, y); if (ret == 0) { ldpp_dout(dpp, 20) << "D4NFilterObject::" << __func__ << "(): version stored in update method is: " << version << dendl; driver->get_policy_driver()->get_cache_policy()->update(dpp, key, 0, bl.length(), version, true, rgw::d4n::REFCOUNT_NOOP, y); @@ -593,7 +592,7 @@ int D4NFilterObject::create_delete_marker(const DoutPrefixProvider* dpp, optiona if (exec_responses.empty()) { ldpp_dout(dpp, 0) << "D4NFilterObject::" << __func__ << "(): Exec respones are empty, error occured!" << dendl; driver->get_policy_driver()->get_cache_policy()->erase(dpp, key, y); - driver->get_cache_driver()->delete_data(dpp, oid_in_cache, y); + driver->get_cache_driver()->delete_data(dpp, key, y); return -ERR_INTERNAL_ERROR; } if (ret < 0) { @@ -605,11 +604,11 @@ int D4NFilterObject::create_delete_marker(const DoutPrefixProvider* dpp, optiona std::string objEtag = ""; driver->get_policy_driver()->get_cache_policy()->update_dirty_object(dpp, key, version, true, this->get_accounted_size(), creationTime, std::get(this->get_bucket()->get_owner()), objEtag, this->get_bucket()->get_name(), this->get_bucket()->get_bucket_id(), this->get_key(), rgw::d4n::REFCOUNT_NOOP, y); } else { //if get_cache_driver()->put() - ldpp_dout(dpp, 0) << "D4NFilterObject::" << __func__ << "(): put failed for oid_in_cache, ret=" << ret << " oid_in_cache: " << oid_in_cache << dendl; + ldpp_dout(dpp, 0) << "D4NFilterObject::" << __func__ << "(): put failed for key, ret=" << ret << " key: " << key << dendl; return ret; } } else { - ldpp_dout(dpp, 0) << "D4NFilterObject::" << __func__ << "(): eviction failed for oid_in_cache, ret=" << ret << dendl; + ldpp_dout(dpp, 0) << "D4NFilterObject::" << __func__ << "(): eviction failed for key, ret=" << ret << dendl; return ret; } @@ -884,13 +883,9 @@ int D4NFilterObject::delete_data_block_cache_entries(const DoutPrefixProvider* d off_t cur_size = std::min(fst + dpp->get_cct()->_conf->rgw_max_chunk_size, lst); off_t cur_len = cur_size - fst; - std::string key = get_cache_block_prefix(this, version, false) + CACHE_DELIM + std::to_string(fst) + CACHE_DELIM + std::to_string(cur_len); + std::string key = get_cache_block_prefix(this, version) + CACHE_DELIM + std::to_string(fst) + CACHE_DELIM + std::to_string(cur_len); std::string key_in_cache; - if (dirty) { - key_in_cache = DIRTY_BLOCK_PREFIX + key; - } else { - key_in_cache = key; - } + key_in_cache = key; int ret; if ((ret = driver->get_cache_driver()->delete_data(dpp, key_in_cache, y)) == 0) { if (!(ret = driver->get_policy_driver()->get_cache_policy()->erase(dpp, key, y))) { @@ -942,11 +937,11 @@ bool D4NFilterObject::check_head_exists_in_cache_get_oid(const DoutPrefixProvide //for distributed cache-the blockHostsList can be used to determine if the head block resides on the localhost, then get the block from localhost, whether or not the block is dirty //can be determined using the block entry. - std::string key = get_cache_block_prefix(this, version, false); + std::string key = get_cache_block_prefix(this, version); if (this->driver->get_policy_driver()->get_cache_policy()->update_refcount_if_key_exists(dpp, key, rgw::d4n::REFCOUNT_INCR, y)) { ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): Is block dirty: " << block.cacheObj.dirty << dendl; ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): version: " << block.version << dendl; - head_oid_in_cache = get_cache_block_prefix(this, version, block.cacheObj.dirty); + head_oid_in_cache = get_cache_block_prefix(this, version); ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): Fetching attrs from cache for head obj id: " << head_oid_in_cache << dendl; auto ret = this->driver->get_cache_driver()->get_attrs(dpp, head_oid_in_cache, attrs, y); if (ret < 0) { @@ -954,9 +949,6 @@ bool D4NFilterObject::check_head_exists_in_cache_get_oid(const DoutPrefixProvide ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): CacheDriver get_attrs method failed." << dendl; } std::string key = head_oid_in_cache; - if (block.cacheObj.dirty) { - key = key.erase(0, 2); // Remove dirty prefix - } this->driver->get_policy_driver()->get_cache_policy()->update(dpp, key, 0, 0, version, block.cacheObj.dirty, rgw::d4n::REFCOUNT_DECR, y); } else { found_in_cache = false; @@ -1004,14 +996,14 @@ int D4NFilterObject::get_obj_attrs(optional_yield y, const DoutPrefixProvider* d } ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): this->obj oid is: " << this->obj.key.name << "instance is: " << this->obj.key.instance << dendl; attrs = this->get_attrs(); - this->set_attrs_from_obj_state(dpp, y, attrs); + this->set_attrs_from_obj_state(dpp, y, attrs, false); ret = calculate_version(dpp, y, version); if (ret < 0 || version.empty()) { ldpp_dout(dpp, 0) << "D4NFilterObject::" << __func__ << "(): version could not be calculated." << dendl; } std::string objName = this->get_name(); - head_oid_in_cache = get_cache_block_prefix(this, version, false); + head_oid_in_cache = get_cache_block_prefix(this, version); if (this->driver->get_policy_driver()->get_cache_policy()->update_refcount_if_key_exists(dpp, head_oid_in_cache, rgw::d4n::REFCOUNT_INCR, y)) { ret = this->driver->get_cache_driver()->set_attrs(dpp, head_oid_in_cache, attrs, y); } else { @@ -1182,7 +1174,7 @@ int D4NFilterObject::D4NFilterReadOp::prepare(optional_yield y, const DoutPrefix } this->source->load_obj_state(dpp, y); attrs = source->get_attrs(); - source->set_attrs_from_obj_state(dpp, y, attrs); + source->set_attrs_from_obj_state(dpp, y, attrs, false); ret = source->calculate_version(dpp, y, version); if (ret < 0 || version.empty()) { ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): version could not be calculated." << dendl; @@ -1191,7 +1183,7 @@ int D4NFilterObject::D4NFilterReadOp::prepare(optional_yield y, const DoutPrefix this->source->set_attr_crypt_parts(dpp, y, attrs); bufferlist bl; - head_oid_in_cache = get_cache_block_prefix(source, version, false); + head_oid_in_cache = get_cache_block_prefix(source, version); ret = source->driver->get_policy_driver()->get_cache_policy()->eviction(dpp, attrs.size(), y); if (ret == 0) { ret = source->driver->get_cache_driver()->put(dpp, head_oid_in_cache, bl, 0, attrs, y); @@ -1366,14 +1358,14 @@ int D4NFilterObject::D4NFilterReadOp::flush(const DoutPrefixProvider* dpp, rgw:: dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address); dest_block.version = dest_version; dest_block.cacheObj.dirty = true; - std::string key = get_cache_block_prefix(source->dest_object, dest_version, false) + + std::string key = get_cache_block_prefix(source->dest_object, dest_version) + CACHE_DELIM + std::to_string(ofs) + CACHE_DELIM + std::to_string(len); - std::string dest_oid_in_cache = DIRTY_BLOCK_PREFIX + key; auto ret = source->driver->get_policy_driver()->get_cache_policy()->eviction(dpp, dest_block.size, y); if (ret == 0) { rgw::sal::Attrs attrs; ldpp_dout(dpp, 20) << "D4NFilterObject::" << __func__ << " destination object version in update method is: " << dest_version << dendl; - ret = source->driver->get_cache_driver()->put(dpp, dest_oid_in_cache, bl, bl.length(), attrs, y); + // destination key is the same as key + ret = source->driver->get_cache_driver()->put(dpp, key, bl, bl.length(), attrs, y); if (ret == 0) { source->driver->get_policy_driver()->get_cache_policy()->update(dpp, key, ofs, bl.length(), dest_version, true, rgw::d4n::REFCOUNT_NOOP, y); } @@ -1404,7 +1396,7 @@ int D4NFilterObject::D4NFilterReadOp::iterate(const DoutPrefixProvider* dpp, int { const uint64_t window_size = g_conf()->rgw_get_obj_window_size; std::string version = source->get_object_version(); - std::string prefix = get_cache_block_prefix(source, version, false); + std::string prefix = get_cache_block_prefix(source, version); ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << "prefix: " << prefix << dendl; ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << "oid: " << source->get_key().get_oid() << " ofs: " << ofs << " end: " << end << dendl; @@ -1473,21 +1465,14 @@ int D4NFilterObject::D4NFilterReadOp::iterate(const DoutPrefixProvider* dpp, int if (it != block.cacheObj.hostsList.end()) { /* Local copy */ ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): Block found in directory. " << oid_in_cache << dendl; - std::string key = oid_in_cache; + // we keep track of dirty data in the cache for the metadata failure case ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): READ FROM CACHE: block is dirty = " << block.cacheObj.dirty << dendl; - - if (block.cacheObj.dirty == true) { - key = DIRTY_BLOCK_PREFIX + oid_in_cache; // we keep track of dirty data in the cache for the metadata failure case - ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): READ FROM CACHE: key=" << key << " data is Dirty." << dendl; - } - - ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): " << __LINE__ << ": READ FROM CACHE: block dirty =" << block.cacheObj.dirty << dendl; - ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): " << __LINE__ << ": READ FROM CACHE: key=" << key << dendl; + ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): " << __LINE__ << ": READ FROM CACHE: oid_in_cache=" << oid_in_cache << dendl; if (block.version == version) { if (source->driver->get_policy_driver()->get_cache_policy()->update_refcount_if_key_exists(dpp, oid_in_cache, rgw::d4n::REFCOUNT_INCR, y) > 0) { // Read From Cache - auto completed = source->driver->get_cache_driver()->get_async(dpp, y, aio.get(), key, read_ofs, len_to_read, cost, id); + auto completed = source->driver->get_cache_driver()->get_async(dpp, y, aio.get(), oid_in_cache, read_ofs, len_to_read, cost, id); this->blocks_info.insert(std::make_pair(id, std::make_pair(adjusted_start_ofs, part_len))); @@ -1561,7 +1546,6 @@ int D4NFilterObject::D4NFilterReadOp::iterate(const DoutPrefixProvider* dpp, int if (block.version == version) { oid_in_cache = prefix + CACHE_DELIM + std::to_string(adjusted_start_ofs) + CACHE_DELIM + std::to_string(chunk_size); - std::string key = oid_in_cache; //for range requests, for last part, the whole part might exist in the cache ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): READ FROM CACHE: oid=" << oid_in_cache << @@ -1569,14 +1553,10 @@ int D4NFilterObject::D4NFilterReadOp::iterate(const DoutPrefixProvider* dpp, int if ((part_len != chunk_size) && source->driver->get_policy_driver()->get_cache_policy()->update_refcount_if_key_exists(dpp, oid_in_cache, rgw::d4n::REFCOUNT_INCR, y) > 0) { // Read From Cache - if (block.cacheObj.dirty == true){ - key = DIRTY_BLOCK_PREFIX + oid_in_cache; - } - ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): " << __LINE__ << ": READ FROM CACHE: block dirty =" << block.cacheObj.dirty << dendl; - ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): " << __LINE__ << ": READ FROM CACHE: key=" << key << dendl; + ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): " << __LINE__ << ": READ FROM CACHE: oid_in_cache=" << oid_in_cache << dendl; - auto completed = source->driver->get_cache_driver()->get_async(dpp, y, aio.get(), key, read_ofs, len_to_read, cost, id); + auto completed = source->driver->get_cache_driver()->get_async(dpp, y, aio.get(), oid_in_cache, read_ofs, len_to_read, cost, id); this->blocks_info.insert(std::make_pair(id, std::make_pair(adjusted_start_ofs, chunk_size))); @@ -1779,7 +1759,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl if (source->dest_object && source->dest_bucket) { D4NFilterObject* d4n_dest_object = dynamic_cast(source->dest_object); std::string dest_version = d4n_dest_object->get_object_version(); - dest_prefix = get_cache_block_prefix(source->dest_object, dest_version, false); + dest_prefix = get_cache_block_prefix(source->dest_object, dest_version); dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address); dest_block.cacheObj.objName = source->dest_object->get_key().get_oid(); dest_block.cacheObj.bucketName = source->dest_object->get_bucket()->get_bucket_id(); @@ -2001,15 +1981,13 @@ int D4NFilterObject::D4NFilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp bool objDirty = block.cacheObj.dirty; auto blockDir = source->driver->get_block_dir(); auto objDir = source->driver->get_obj_dir(); - std::string policy_prefix = head_oid_in_cache; std::string version = source->get_object_version(); if (objDirty) { // head object dirty flag represents object dirty flag //for versioned buckets, for a simple delete we need to create a delete marker (and not invalidate/delete any object) if (!source->get_bucket()->versioned() || (block.cacheObj.objName != source->get_name())) { - policy_prefix.erase(0, 2); // remove "D_" prefix from policy key since the policy keys do not hold this information ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): calling invalidate_dirty_object for: " << head_oid_in_cache << dendl; - if (!source->driver->get_policy_driver()->get_cache_policy()->invalidate_dirty_object(dpp, policy_prefix)) { + if (!source->driver->get_policy_driver()->get_cache_policy()->invalidate_dirty_object(dpp, head_oid_in_cache)) { objDirty = false; } } @@ -2256,7 +2234,7 @@ int D4NFilterObject::D4NFilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp off_t fst = 0; do { // loop through the data blocks - std::string prefix = get_cache_block_prefix(source, version, false); + std::string prefix = get_cache_block_prefix(source, version); if (fst >= lst) { break; } @@ -2355,7 +2333,7 @@ int D4NFilterWriter::process(bufferlist&& data, uint64_t offset) bool dirty = true; std::string version = object->get_object_version(); - std::string prefix = get_cache_block_prefix(obj, version, false); + std::string prefix = get_cache_block_prefix(obj, version); int ret = 0; @@ -2365,16 +2343,14 @@ int D4NFilterWriter::process(bufferlist&& data, uint64_t offset) } else { rgw::sal::Attrs attrs; std::string oid = prefix + CACHE_DELIM + std::to_string(ofs); - std::string key = DIRTY_BLOCK_PREFIX + oid + CACHE_DELIM + std::to_string(bl_len); std::string oid_in_cache = oid + CACHE_DELIM + std::to_string(bl_len); dirty = true; ret = driver->get_policy_driver()->get_cache_policy()->eviction(dpp, bl.length(), y); if (ret == 0) { if (bl.length() > 0) { - ldpp_dout(dpp, 10) << "D4NFilterWriter::" << __func__ << "(): key is: " << key << dendl; - ret = driver->get_cache_driver()->put(dpp, key, bl, bl.length(), attrs, y); + ldpp_dout(dpp, 10) << "D4NFilterWriter::" << __func__ << "(): oid_in_cache is: " << oid_in_cache << dendl; + ret = driver->get_cache_driver()->put(dpp, oid_in_cache, bl, bl.length(), attrs, y); if (ret == 0) { - ldpp_dout(dpp, 10) << "D4NFilterWriter::" << __func__ << "(): oid_in_cache is: " << oid_in_cache << dendl; driver->get_policy_driver()->get_cache_policy()->update(dpp, oid_in_cache, ofs, bl.length(), version, dirty, rgw::d4n::REFCOUNT_NOOP, y); } else { ldpp_dout(dpp, 0) << "D4NFilterWriter::" << __func__ << "(): ERROR: writting data to the cache failed, ret=" << ret << dendl; @@ -2482,7 +2458,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag, ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << " size is: " << object->get_size() << dendl; object->set_attr_crypt_parts(dpp, y, attrs); object->set_attrs(attrs); - object->set_attrs_from_obj_state(dpp, y, attrs); + object->set_attrs_from_obj_state(dpp, y, attrs, dirty); } else { // we need to call next->complete here so that we are able to correctly get the object state needed for caching head ret = next->complete(accounted_size, etag, mtime, set_mtime, attrs, cksum, @@ -2494,7 +2470,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag, } object->load_obj_state(dpp, y); attrs = object->get_attrs(); - object->set_attrs_from_obj_state(dpp, y, attrs); + object->set_attrs_from_obj_state(dpp, y, attrs, dirty); std::string version; ret = object->calculate_version(dpp, y, version); @@ -2504,20 +2480,14 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag, } std::string version = object->get_object_version(); - std::string key = get_cache_block_prefix(obj, version, false); + std::string key = get_cache_block_prefix(obj, version); bufferlist bl; - std::string head_oid_in_cache; //same as key, as there is no len or offset attached to head oid in cache - if (dirty) { - head_oid_in_cache = DIRTY_BLOCK_PREFIX + key; - } else { - head_oid_in_cache = key; - } - ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << "(): head_oid_in_cache is: " << head_oid_in_cache << dendl; + ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << "(): head_oid_in_cache is: " << key << dendl; ret = driver->get_policy_driver()->get_cache_policy()->eviction(dpp, attrs.size(), y); if (ret == 0) { - ret = driver->get_cache_driver()->put(dpp, head_oid_in_cache, bl, 0, attrs, y); + ret = driver->get_cache_driver()->put(dpp, key, bl, 0, attrs, y); attrs.erase(RGW_CACHE_ATTR_MTIME); attrs.erase(RGW_CACHE_ATTR_OBJECT_SIZE); attrs.erase(RGW_CACHE_ATTR_ACCOUNTED_SIZE); @@ -2576,7 +2546,7 @@ int D4NFilterMultipartUpload::complete(const DoutPrefixProvider *dpp, rgw::sal::Attrs attrs; d4n_target_obj->load_obj_state(dpp, y); attrs = d4n_target_obj->get_attrs(); - d4n_target_obj->set_attrs_from_obj_state(dpp, y, attrs); + d4n_target_obj->set_attrs_from_obj_state(dpp, y, attrs, false); // TODO: is this value correct? bufferlist bl_val; bool is_multipart = true; bl_val.append(std::to_string(is_multipart)); @@ -2589,7 +2559,7 @@ int D4NFilterMultipartUpload::complete(const DoutPrefixProvider *dpp, } bufferlist bl; - head_oid_in_cache = get_cache_block_prefix(d4n_target_obj, version, false); + head_oid_in_cache = get_cache_block_prefix(d4n_target_obj, version); // we are evicting data if needed, since the head object will be a part of read cache, as the whole multipart object is written to the backend store ret = driver->get_policy_driver()->get_cache_policy()->eviction(dpp, attrs.size(), y); if (ret == 0) { diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h index d1b2f81487411f..d522abb6348872 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.h +++ b/src/rgw/driver/d4n/rgw_sal_d4n.h @@ -243,7 +243,7 @@ class D4NFilterObject : public FilterObject { void set_prefix(const std::string& prefix) { this->prefix = prefix; } const std::string get_prefix() { return this->prefix; } int get_obj_attrs_from_cache(const DoutPrefixProvider* dpp, optional_yield y); - void set_attrs_from_obj_state(const DoutPrefixProvider* dpp, optional_yield y, rgw::sal::Attrs& attrs); + void set_attrs_from_obj_state(const DoutPrefixProvider* dpp, optional_yield y, rgw::sal::Attrs& attrs, bool dirty); int calculate_version(const DoutPrefixProvider* dpp, optional_yield y, std::string& version); int set_head_obj_dir_entry(const DoutPrefixProvider* dpp, std::vector* exec_responses, optional_yield y, bool is_latest_version = true, bool dirty = false); int set_data_block_dir_entries(const DoutPrefixProvider* dpp, optional_yield y, std::string& version, bool dirty = false); diff --git a/src/rgw/rgw_cache_driver.h b/src/rgw/rgw_cache_driver.h index 2ad492c05c5a16..98c5b08c66b353 100644 --- a/src/rgw/rgw_cache_driver.h +++ b/src/rgw/rgw_cache_driver.h @@ -15,8 +15,8 @@ constexpr char RGW_CACHE_ATTR_SOURC_ZONE[] = "user.rgw.source_zone"; constexpr char RGW_CACHE_ATTR_LOCAL_WEIGHT[] = "user.rgw.localWeight"; constexpr char RGW_CACHE_ATTR_DELETE_MARKER[] = "user.rgw.deleteMarker"; constexpr char RGW_CACHE_ATTR_INVALID[] = "user.rgw.invalid"; +constexpr char RGW_CACHE_ATTR_DIRTY[] = "user.rgw.dirty"; -constexpr char DIRTY_BLOCK_PREFIX[] = "D#"; constexpr char CACHE_DELIM = '#'; namespace rgw { namespace cache { diff --git a/src/rgw/rgw_ssd_driver.cc b/src/rgw/rgw_ssd_driver.cc index a0de0455ea19c1..439573049225b0 100644 --- a/src/rgw/rgw_ssd_driver.cc +++ b/src/rgw/rgw_ssd_driver.cc @@ -17,7 +17,7 @@ static std::atomic index{0}; /* * Parses key to return directory path and file name */ -static void parse_key(const DoutPrefixProvider* dpp, const std::string& location, const std::string& key, std::string& dir_path, std::string& file_name, bool& is_dirty, bool temp = false) { +static void parse_key(const DoutPrefixProvider* dpp, const std::string& location, const std::string& key, std::string& dir_path, std::string& file_name, bool temp = false) { ldpp_dout(dpp, 10) << __func__ << "() key is: " << key << dendl; std::stringstream ss(key); std::vector parts; @@ -25,48 +25,24 @@ static void parse_key(const DoutPrefixProvider* dpp, const std::string& location while (std::getline(ss, part, CACHE_DELIM)) { parts.push_back(part); } - is_dirty = false; - + std::string dirtyStr; ldpp_dout(dpp, 10) << __func__ << "() parts.size() is " << parts.size() << dendl; - //dirty blocks - if (parts.size() == 4 || parts.size() == 6) { - if (parts[0] == "D") { - is_dirty = true; - bucket_id = parts[1]; - ldpp_dout(dpp, 10) << __func__ << "() bucket_id is " << bucket_id << dendl; - object = parts[3]; - ldpp_dout(dpp, 10) << __func__ << "() object is " << object << dendl; - version = DIRTY_BLOCK_PREFIX + parts[2]; - if (parts.size() == 6) { //has offset and length - version += CACHE_DELIM + parts[4] + CACHE_DELIM + parts[5]; - } - if (temp) { - version += "_" + std::to_string(index++); - } - ldpp_dout(dpp, 10) << __func__ << "() version is " << version << dendl; - dir_path = location + "/" + bucket_id + "/" + object; - file_name = version; - ldpp_dout(dpp, 10) << __func__ << "() dir_path is " << dir_path << dendl; - } - } - - //clean blocks if (parts.size() == 3 || parts.size() == 5) { - bucket_id = parts[0]; - ldpp_dout(dpp, 10) << __func__ << "() bucket_id is " << bucket_id << dendl; - object = parts[2]; - ldpp_dout(dpp, 10) << __func__ << "() object is " << object << dendl; - version = parts[1]; - if (parts.size() == 5) { //has offset and length - version += CACHE_DELIM + parts[3] + CACHE_DELIM + parts[4]; - } - if (temp) { - version += "_" + std::to_string(index++); - } - ldpp_dout(dpp, 10) << __func__ << "() version is " << version << dendl; - dir_path = location + "/" + bucket_id + "/" + object; - file_name = version; - ldpp_dout(dpp, 10) << __func__ << "() dir_path is " << dir_path << dendl; + bucket_id = parts[0]; + ldpp_dout(dpp, 10) << __func__ << "() bucket_id is " << bucket_id << dendl; + object = parts[2]; + ldpp_dout(dpp, 10) << __func__ << "() object is " << object << dendl; + version = parts[1]; + if (parts.size() == 5) { //has offset and length + version += CACHE_DELIM + parts[3] + CACHE_DELIM + parts[4]; + } + if (temp) { + version += "_" + std::to_string(index++); + } + ldpp_dout(dpp, 10) << __func__ << "() version is " << version << dendl; + dir_path = location + "/" + bucket_id + "/" + object; + file_name = version; + ldpp_dout(dpp, 10) << __func__ << "() dir_path is " << dir_path << dendl; } return; } @@ -106,8 +82,7 @@ static std::string get_file_path(const DoutPrefixProvider* dpp, const std::strin static std::string create_dirs_get_filepath_from_key(const DoutPrefixProvider* dpp, const std::string& location, const std::string& key, bool temp=false) { std::string dir_path, file_name; - bool is_dirty; - parse_key(dpp, location, key, dir_path, file_name, is_dirty, temp); + parse_key(dpp, location, key, dir_path, file_name, temp); create_directories(dpp, dir_path); return get_file_path(dpp, dir_path, file_name); @@ -224,147 +199,139 @@ int SSDDriver::restore_blocks_objects(const DoutPrefixProvider* dpp, ObjectDataC parts.push_back(part); } ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): parts.size(): " << parts.size() << dendl; - //non-dirty or clean blocks - version in head block and offset, len in data blocks + + std::string dirtyStr; + auto ret = get_attr(dpp, file_entry.path(), RGW_CACHE_ATTR_DIRTY, dirtyStr, null_yield); + if (ret < 0) { + ldpp_dout(dpp, 0) << "SSDCache: " << __func__ << "(): Failed to get attr: " << RGW_CACHE_ATTR_DIRTY << dendl; + dirtyStr = "0"; // TODO: assume false? + } + bool dirty; + if (dirtyStr == "1") { + dirty = true; + } else { + dirty = false; + } + if (parts.size() == 1 || parts.size() == 3) { - std::string version = url_decode(parts[0]); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): version: " << version << dendl; - - std::string key = url_encode(bucket_id, true) + CACHE_DELIM + url_encode(version, true) + CACHE_DELIM + url_encode(object_name, true); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): key: " << key << dendl; - - uint64_t offset = 0, len = 0; - if (parts.size() == 3) { - offset = std::stoull(parts[1]); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): offset: " << offset << dendl; - - len = std::stoull(parts[2]); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): len: " << len << dendl; - - key = key + CACHE_DELIM + std::to_string(offset) + CACHE_DELIM + std::to_string(len); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): key: " << key << dendl; - } - std::string localWeightStr; - auto ret = get_attr(dpp, file_entry.path(), RGW_CACHE_ATTR_LOCAL_WEIGHT, localWeightStr, null_yield); - if (ret < 0) { - ldpp_dout(dpp, 0) << "SSDCache: " << __func__ << "(): Failed to get attr: " << RGW_CACHE_ATTR_LOCAL_WEIGHT << dendl; - } else { - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): localWeightStr: " << localWeightStr << dendl; - } - block_func(dpp, key, offset, len, version, false, null_yield, localWeightStr); - parsed = true; - } - //dirty blocks - "D", version in head block and offset, len in data blocks - if ((parts.size() == 2 || parts.size() == 4) && parts[0] == "D") { - std::string prefix = DIRTY_BLOCK_PREFIX; - if (file_name.starts_with(prefix)) { - bool dirty = true; - - std::string version = url_decode(parts[1]); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): version: " << version << dendl; - - std::string key = url_encode(bucket_id, true) + CACHE_DELIM + url_encode(version, true) + CACHE_DELIM + url_encode(object_name, true); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): key: " << key << dendl; - - uint64_t len = 0, offset = 0; - std::string localWeightStr; - std::string invalidStr; - if (parts.size() == 2) { - rgw::sal::Attrs attrs; - get_attrs(dpp, file_entry.path(), attrs, null_yield); - std::string etag, bucket_name; - uint64_t size = 0; - time_t creationTime = time_t(nullptr); - rgw_user user; - rgw_obj_key obj_key; - bool deleteMarker = false; - if (attrs.find(RGW_ATTR_ETAG) != attrs.end()) { - etag = attrs[RGW_ATTR_ETAG].to_str(); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): etag: " << etag << dendl; - } - if (attrs.find(RGW_CACHE_ATTR_OBJECT_SIZE) != attrs.end()) { - size = std::stoull(attrs[RGW_CACHE_ATTR_OBJECT_SIZE].to_str()); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): size: " << size << dendl; - } - if (attrs.find(RGW_CACHE_ATTR_MTIME) != attrs.end()) { - creationTime = ceph::real_clock::to_time_t(ceph::real_clock::from_double(std::stod(attrs[RGW_CACHE_ATTR_MTIME].to_str()))); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): creationTime: " << creationTime << dendl; - } - if (attrs.find(RGW_ATTR_ACL) != attrs.end()) { - bufferlist bl_acl = attrs[RGW_ATTR_ACL]; - RGWAccessControlPolicy policy; - auto iter = bl_acl.cbegin(); - try { - policy.decode(iter); - } catch (buffer::error& err) { - ldpp_dout(dpp, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl; - continue; - } - user = std::get(policy.get_owner().id); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): rgw_user: " << user.to_str() << dendl; - } - obj_key.name = object_name; - if (attrs.find(RGW_CACHE_ATTR_VERSION_ID) != attrs.end()) { - std::string instance = attrs[RGW_CACHE_ATTR_VERSION_ID].to_str(); - if (instance != "null") { - obj_key.instance = instance; - } - } - if (attrs.find(RGW_CACHE_ATTR_OBJECT_NS) != attrs.end()) { - obj_key.ns = attrs[RGW_CACHE_ATTR_OBJECT_NS].to_str(); - } - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): rgw_obj_key: " << obj_key.get_oid() << dendl; - if (attrs.find(RGW_CACHE_ATTR_BUCKET_NAME) != attrs.end()) { - bucket_name = attrs[RGW_CACHE_ATTR_BUCKET_NAME].to_str(); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): bucket_name: " << bucket_name << dendl; - } - - if (attrs.find(RGW_CACHE_ATTR_LOCAL_WEIGHT) != attrs.end()) { - localWeightStr = attrs[RGW_CACHE_ATTR_LOCAL_WEIGHT].to_str(); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): localWeightStr: " << localWeightStr << dendl; - } - - if (attrs.find(RGW_CACHE_ATTR_DELETE_MARKER) != attrs.end()) { - std::string deleteMarkerStr = attrs[RGW_CACHE_ATTR_LOCAL_WEIGHT].to_str(); - deleteMarker = (deleteMarkerStr == "1") ? true : false; - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): deleteMarker: " << deleteMarker << dendl; - } - - if (attrs.find(RGW_CACHE_ATTR_INVALID) != attrs.end()) { - invalidStr = attrs[RGW_CACHE_ATTR_INVALID].to_str(); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): invalidStr: " << invalidStr << dendl; - } - - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): calling func for: " << key << dendl; - obj_func(dpp, key, version, deleteMarker, size, creationTime, user, etag, bucket_name, bucket_id, obj_key, null_yield, invalidStr); - block_func(dpp, key, offset, len, version, dirty, null_yield, localWeightStr); - parsed = true; - } //end-if part.size() == 2 - if (parts.size() == 4) { - offset = std::stoull(parts[2]); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): offset: " << offset << dendl; - - len = std::stoull(parts[3]); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): len: " << len << dendl; - - key = key + CACHE_DELIM + std::to_string(offset) + CACHE_DELIM + std::to_string(len); - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): key: " << key << dendl; - - std::string localWeightStr; - auto ret = get_attr(dpp, file_entry.path(), RGW_CACHE_ATTR_LOCAL_WEIGHT, localWeightStr, null_yield); - if (ret < 0) { - ldpp_dout(dpp, 0) << "SSDCache: " << __func__ << "(): Failed to get attr: " << RGW_CACHE_ATTR_LOCAL_WEIGHT << dendl; - } else { - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): localWeightStr: " << localWeightStr << dendl; - } - block_func(dpp, key, offset, len, version, dirty, null_yield, localWeightStr); - parsed = true; - } - } //end-if file_name.starts_with - } //end-if parts.size() == 2 || parts.size() == 4 - if (!parsed) { - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): Unable to parse file_name: " << file_name << dendl; - continue; - } + std::string version = url_decode(parts[0]); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): version: " << version << dendl; + + std::string key = url_encode(bucket_id, true) + CACHE_DELIM + url_encode(version, true) + CACHE_DELIM + url_encode(object_name, true); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): key: " << key << dendl; + + uint64_t len = 0, offset = 0; + if (parts.size() == 1) { + if (dirtyStr == "0") { + //non-dirty or clean blocks - version in head block and offset, len in data blocks + std::string localWeightStr; + ret = get_attr(dpp, file_entry.path(), RGW_CACHE_ATTR_LOCAL_WEIGHT, localWeightStr, null_yield); + if (ret < 0) { + ldpp_dout(dpp, 0) << "SSDCache: " << __func__ << "(): Failed to get attr: " << RGW_CACHE_ATTR_LOCAL_WEIGHT << dendl; + } else { + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): localWeightStr: " << localWeightStr << dendl; + } + block_func(dpp, key, offset, len, version, false, null_yield, localWeightStr); + parsed = true; + } else if (dirtyStr == "1") { + //dirty blocks - version in head block and offset, len in data blocks + std::string localWeightStr; + std::string invalidStr; + rgw::sal::Attrs attrs; + get_attrs(dpp, file_entry.path(), attrs, null_yield); + std::string etag, bucket_name; + uint64_t size = 0; + time_t creationTime = time_t(nullptr); + rgw_user user; + rgw_obj_key obj_key; + bool deleteMarker = false; + if (attrs.find(RGW_ATTR_ETAG) != attrs.end()) { + etag = attrs[RGW_ATTR_ETAG].to_str(); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): etag: " << etag << dendl; + } + if (attrs.find(RGW_CACHE_ATTR_OBJECT_SIZE) != attrs.end()) { + size = std::stoull(attrs[RGW_CACHE_ATTR_OBJECT_SIZE].to_str()); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): size: " << size << dendl; + } + if (attrs.find(RGW_CACHE_ATTR_MTIME) != attrs.end()) { + creationTime = ceph::real_clock::to_time_t(ceph::real_clock::from_double(std::stod(attrs[RGW_CACHE_ATTR_MTIME].to_str()))); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): creationTime: " << creationTime << dendl; + } + if (attrs.find(RGW_ATTR_ACL) != attrs.end()) { + bufferlist bl_acl = attrs[RGW_ATTR_ACL]; + RGWAccessControlPolicy policy; + auto iter = bl_acl.cbegin(); + try { + policy.decode(iter); + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl; + continue; + } + user = std::get(policy.get_owner().id); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): rgw_user: " << user.to_str() << dendl; + } + obj_key.name = object_name; + if (attrs.find(RGW_CACHE_ATTR_VERSION_ID) != attrs.end()) { + std::string instance = attrs[RGW_CACHE_ATTR_VERSION_ID].to_str(); + if (instance != "null") { + obj_key.instance = instance; + } + } + if (attrs.find(RGW_CACHE_ATTR_OBJECT_NS) != attrs.end()) { + obj_key.ns = attrs[RGW_CACHE_ATTR_OBJECT_NS].to_str(); + } + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): rgw_obj_key: " << obj_key.get_oid() << dendl; + if (attrs.find(RGW_CACHE_ATTR_BUCKET_NAME) != attrs.end()) { + bucket_name = attrs[RGW_CACHE_ATTR_BUCKET_NAME].to_str(); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): bucket_name: " << bucket_name << dendl; + } + + if (attrs.find(RGW_CACHE_ATTR_LOCAL_WEIGHT) != attrs.end()) { + localWeightStr = attrs[RGW_CACHE_ATTR_LOCAL_WEIGHT].to_str(); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): localWeightStr: " << localWeightStr << dendl; + } + + if (attrs.find(RGW_CACHE_ATTR_DELETE_MARKER) != attrs.end()) { + std::string deleteMarkerStr = attrs[RGW_CACHE_ATTR_LOCAL_WEIGHT].to_str(); + deleteMarker = (deleteMarkerStr == "1") ? true : false; + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): deleteMarker: " << deleteMarker << dendl; + } + + if (attrs.find(RGW_CACHE_ATTR_INVALID) != attrs.end()) { + invalidStr = attrs[RGW_CACHE_ATTR_INVALID].to_str(); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): invalidStr: " << invalidStr << dendl; + } + + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): calling func for: " << key << dendl; + obj_func(dpp, key, version, deleteMarker, size, creationTime, user, etag, bucket_name, bucket_id, obj_key, null_yield, invalidStr); + block_func(dpp, key, offset, len, version, dirty, null_yield, localWeightStr); + parsed = true; + } // end-if dirtyStr == "1" + } else if (parts.size() == 3) { //end-if parts.size() == 1 + offset = std::stoull(parts[1]); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): offset: " << offset << dendl; + + len = std::stoull(parts[2]); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): len: " << len << dendl; + + key = key + CACHE_DELIM + std::to_string(offset) + CACHE_DELIM + std::to_string(len); + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): key: " << key << dendl; + + std::string localWeightStr; + auto ret = get_attr(dpp, file_entry.path(), RGW_CACHE_ATTR_LOCAL_WEIGHT, localWeightStr, null_yield); + if (ret < 0) { + ldpp_dout(dpp, 0) << "SSDCache: " << __func__ << "(): Failed to get attr: " << RGW_CACHE_ATTR_LOCAL_WEIGHT << dendl; + } else { + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): localWeightStr: " << localWeightStr << dendl; + } + block_func(dpp, key, offset, len, version, dirty, null_yield, localWeightStr); + parsed = true; + } + if (!parsed) { + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): Unable to parse file_name: " << file_name << dendl; + continue; + } + } } }//end - try catch(...) { @@ -622,8 +589,7 @@ rgw::AioResultList SSDDriver::put_async(const DoutPrefixProvider* dpp, optional_ int SSDDriver::delete_data(const DoutPrefixProvider* dpp, const::std::string& key, optional_yield y) { std::string dir_path, file_name; - bool is_dirty; - parse_key(dpp, partition_info.location, key, dir_path, file_name, is_dirty); + parse_key(dpp, partition_info.location, key, dir_path, file_name); std::string location = get_file_path(dpp, dir_path, file_name); ldpp_dout(dpp, 20) << "INFO: delete_data::file to remove: " << location << dendl;