From 617ffccbca0169ac0f1cd713962d44e8cc74a8af Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Mon, 31 Oct 2022 12:40:50 -0400 Subject: [PATCH] rgwlc: prevent lc for one bucket from exceeding time budget Fixes: https://tracker.ceph.com/issues/57951 Signed-off-by: Matt Benjamin --- src/rgw/rgw_lc.cc | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 4ec0fae5c412..b5bae39eca04 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -880,6 +880,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target, } params.prefix = prefix_iter->first; do { + auto offset = 0; results.objs.clear(); ret = target->list(this, params, 1000, results, null_yield); if (ret < 0) { @@ -889,7 +890,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target, return ret; } - for (auto obj_iter = results.objs.begin(); obj_iter != results.objs.end(); ++obj_iter) { + for (auto obj_iter = results.objs.begin(); obj_iter != results.objs.end(); ++obj_iter, ++offset) { std::tuple t1 = {prefix_iter->second, *obj_iter}; worker->workpool->enqueue(WorkItem{t1}); @@ -898,6 +899,15 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target, } } /* for objs */ + if ((offset % 100) == 0) { + if (worker_should_stop(stop_at, once)) { + ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker " + << worker->ix + << dendl; + return 0; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); } while(results.is_truncated); } /* for prefix_map */ @@ -1641,10 +1651,18 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker, LCOpRule orule(oenv); orule.build(); // why can't ctor do it? rgw_bucket_dir_entry* o{nullptr}; - for (; ol.get_obj(this, &o /* , fetch_barrier */); ol.next()) { + for (auto offset = 0; ol.get_obj(this, &o /* , fetch_barrier */); ++offset, ol.next()) { orule.update(); std::tuple t1 = {orule, *o}; worker->workpool->enqueue(WorkItem{t1}); + if ((offset % 100) == 0) { + if (worker_should_stop(stop_at, once)) { + ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker " + << worker->ix + << dendl; + return 0; + } + } } worker->workpool->drain(); }