diff --git a/ceph.spec.in b/ceph.spec.in index 46d9aae02ae8..5e101ed119b6 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -2132,6 +2132,7 @@ fi %{_bindir}/rgw-gap-list-comparator %{_bindir}/rgw-orphan-list %{_bindir}/rgw-policy-check +%{_bindir}/rgw-restore-bucket-index %{_mandir}/man8/radosgw.8* %{_mandir}/man8/rgw-policy-check.8* %dir %{_localstatedir}/lib/ceph/radosgw diff --git a/debian/radosgw.install b/debian/radosgw.install index 40940c0e1ac2..df522926619f 100644 --- a/debian/radosgw.install +++ b/debian/radosgw.install @@ -7,6 +7,7 @@ usr/bin/radosgw-token usr/bin/rgw-gap-list usr/bin/rgw-gap-list-comparator usr/bin/rgw-orphan-list +usr/bin/rgw-restore-bucket-index usr/share/man/man8/ceph-diff-sorted.8 usr/share/man/man8/radosgw.8 usr/share/man/man8/rgw-orphan-list.8 diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 9d155a1728fc..34a346281850 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -578,4 +578,5 @@ install(PROGRAMS rgw-gap-list rgw-gap-list-comparator rgw-orphan-list + rgw-restore-bucket-index DESTINATION bin) diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index db546a3ca68f..e9e4f8dff5c5 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -3600,6 +3600,34 @@ int RGWRados::rewrite_obj(RGWBucketInfo& dest_bucket_info, const rgw_obj& obj, c attrset, 0, real_time(), NULL, dpp, y); } +int RGWRados::reindex_obj(const RGWBucketInfo& bucket_info, + const rgw_obj& obj, + const DoutPrefixProvider* dpp, + optional_yield y) +{ + if (bucket_info.versioned()) { + ldpp_dout(dpp, 10) << "WARNING: " << __func__ << + ": cannot process versioned bucket \"" << + bucket_info.bucket.get_key() << "\"" << + dendl; + return -ENOTSUP; + } + + Bucket target(this, bucket_info); + RGWRados::Bucket::UpdateIndex update_idx(&target, obj); + const std::string* no_write_tag = nullptr; + + int ret = update_idx.prepare(dpp, RGWModifyOp::CLS_RGW_OP_ADD, no_write_tag, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: " << __func__ << + ": update index prepare for \"" << obj << "\" returned: " << + cpp_strerror(-ret) << dendl; + return ret; + } + + return 0; +} + struct obj_time_weight { real_time mtime; uint32_t zone_short_id; diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h index 043e9569e816..96244b15a4ab 100644 --- a/src/rgw/driver/rados/rgw_rados.h +++ b/src/rgw/driver/rados/rgw_rados.h @@ -1085,6 +1085,10 @@ class RGWRados D3nDataCache* d3n_data_cache{nullptr}; int rewrite_obj(RGWBucketInfo& dest_bucket_info, const rgw_obj& obj, const DoutPrefixProvider *dpp, optional_yield y); + int reindex_obj(const RGWBucketInfo& dest_bucket_info, + const rgw_obj& obj, + const DoutPrefixProvider* dpp, + optional_yield y); int stat_remote_obj(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, diff --git a/src/rgw/rgw-restore-bucket-index b/src/rgw/rgw-restore-bucket-index new file mode 100755 index 000000000000..e8503ca47562 --- /dev/null +++ b/src/rgw/rgw-restore-bucket-index @@ -0,0 +1,193 @@ +#!/usr/bin/env bash + +# version 2023-03-07 + +# rgw-restore-bucket-index is an EXPERIMENTAL tool to use in case +# bucket index entries for objects in the bucket are somehow lost. It +# is expected to be needed and used rarely. A bucket name is provided +# and the data pool for that bucket is scanned for all head objects +# matching the bucket's marker. The rgw object name is then extracted +# from the rados object name, and `radosgw-admin bucket reindex ...` +# is used to add the bucket index entry. +# +# Because this script must process json objects, the `jq` tool must be +# installed on the system. +# +# Usage: $0 [--proceed] [data-pool-name] +# +# This tool is designed to be interactive, allowing the user to +# examine the list of objects to be reindexed before +# proceeding. However, if the "--proceed" option is provided, the +# script will not prompt the user and simply proceed. + +trap "clean ; exit 1" TERM +export TOP_PID=$$ + +# IMPORTANT: affects order produced by 'sort' and 'ceph-diff-sorted' +# relies on this ordering +export LC_ALL=C + +export bkt_entry=/tmp/rgwrbi-bkt-entry.$$ +export bkt_inst=/tmp/rgwrbi-bkt-inst.$$ +export bkt_inst_new=/tmp/rgwrbi-bkt-inst-new.$$ +export obj_list=/tmp/rgwrbi-object-list.$$ +export zone_info=/tmp/rgwrbi-zone-info.$$ +export clean_temps=1 + +# make sure jq is available +if which jq > /dev/null ;then + : +else + echo 'Error: must have command `jq` installed and on $PATH for json parsing.' + exit 1 +fi + +clean() { + if [ -n "$clean_temps" ] ;then + rm -f $bkt_entry $bkt_inst $bkt_inst_new $obj_list $zone_info + fi +} + +super_exit() { + kill -s TERM $TOP_PID +} + +usage() { + >&2 cat << EOF + +Usage: $0 [--proceed] [data-pool-name] + NOTE: This tool is currently considered EXPERIMENTAL. + NOTE: If a data-pool-name is not supplied then it will be inferred from bucket and zone information. + NOTE: If --proceed is provided then user will not be prompted to proceed. Use with caution. +EOF + super_exit +} + +# strips the starting and ending double quotes from a string, so: +# "dog" -> dog +# "dog -> "dog +# d"o"g -> d"o"g +# "do"g" -> do"g +strip_quotes() { + echo "$1" | sed 's/^"\(.*\)"$/\1/' +} + +# Determines the name of the data pool. Expects the optional +# command-line argument to appear as $1 if there is one. The +# command-line has the highest priority, then the "explicit_placement" +# in the bucket instance data, and finally the "placement_rule" in the +# bucket instance data. +get_pool() { + # command-line + if [ -n "$1" ] ;then + echo "$1" + exit 0 + fi + + # explicit_placement + expl_pool=$(strip_quotes $(jq '.data.bucket_info.bucket.explicit_placement.data_pool' $bkt_inst)) + if [ -n "$expl_pool" ] ;then + echo "$expl_pool" + exit 0 + fi + + # placement_rule + plmt_rule=$(strip_quotes $(jq '.data.bucket_info.placement_rule' $bkt_inst)) + plmt_pool=$(echo "$plmt_rule" | awk -F / '{print $1}') + plmt_class=$(echo "$plmt_rule" | awk -F / '{print $2}') + if [ -z "$plmt_class" ] ;then + plmt_class=STANDARD + fi + + radosgw-admin zone get >$zone_info 2>/dev/null + pool=$(strip_quotes $(jq ".placement_pools [] | select(.key | contains(\"${plmt_pool}\")) .val .storage_classes.${plmt_class}.data_pool" $zone_info)) + + if [ -z "$pool" ] ;then + echo ERROR: unable to determine pool. + super_exit + fi + echo "$pool" +} + +if [ $1 == "--proceed" ] ;then + echo "NOTICE: This tool is currently considered EXPERIMENTAL." + proceed=1 + shift +fi + +# expect 1 or 2 arguments +if [ $# -eq 0 -o $# -gt 2 ] ;then + usage +fi + +bucket=$1 + +# read bucket entry metadata +radosgw-admin metadata get bucket:$bucket >$bkt_entry 2>/dev/null +marker=$(strip_quotes $(jq ".data.bucket.marker" $bkt_entry)) +bucket_id=$(strip_quotes $(jq ".data.bucket.bucket_id" $bkt_entry)) +echo marker is $marker +echo bucket_id is $bucket_id + +# read bucket instance metadata +radosgw-admin metadata get bucket.instance:${bucket}:$bucket_id >$bkt_inst 2>/dev/null + +# handle versioned buckets +bkt_flags=$(jq ".data.bucket_info.flags" $bkt_inst) +is_versioned=$(( $bkt_flags & 2)) # mask bit indicating it's a versioned bucket +if [ "$is_versioned" -ne 0 ] ;then + echo "Error: this bucket appears to be versioned, and this tool cannot work with versioned buckets." + clean + exit 1 +fi + +# examine number of bucket index shards +num_shards=$(jq ".data.bucket_info.num_shards" $bkt_inst) +echo number of bucket index shards is $num_shards + +# determine data pool +pool=$(get_pool $2) +echo data pool is $pool + +# search the data pool for all of the head objects that begin with the +# marker that are not in namespaces (indicated by an extra underscore) +# and then strip away all but the rgw object name +( rados -p $pool ls | grep "^${marker}_[^_]" | sed "s/^${marker}_\(.*\)/\1/" >$obj_list ) 2>/dev/null + +# handle the case where the resulting object list file is empty +if [ -s $obj_list ] ;then + : +else + echo "NOTICE: No head objects for bucket \"$bucket\" were found in pool \"$pool\", so nothing was recovered." + clean + exit 0 +fi + +if [ -z "$proceed" ] ;then + # warn user and get permission to proceed + echo "NOTICE: This tool is currently considered EXPERIMENTAL." + echo "The list of objects that we will attempt to restore can be found in \"$obj_list\"." + echo "Please review the object names in that file (either below or in another window/terminal) before proceeding." + while true ; do + read -p "Type \"proceed!\" to proceed, \"view\" to view object list, or \"q\" to quit: " action + if [ "$action" == "q" ] ;then + echo "Exiting..." + clean + exit 0 + elif [ "$action" == "view" ] ;then + echo "Viewing..." + less $obj_list + elif [ "$action" == "proceed!" ] ;then + echo "Proceeding..." + break + else + echo "Error: response \"$action\" is not understood." + fi + done +fi + +# execute object rewrite on all of the head objects +radosgw-admin object reindex --bucket=$bucket --objects-file=$obj_list 2>/dev/null + +clean +echo Done diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 32eb80b19d1b..1c0304d81411 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -72,6 +72,7 @@ extern "C" { #include "services/svc_zone.h" #include "driver/rados/rgw_bucket.h" +#include "driver/rados/rgw_sal_rados.h" #define dout_context g_ceph_context @@ -171,6 +172,7 @@ void usage() cout << " object stat stat an object for its metadata\n"; cout << " object unlink unlink object from bucket index\n"; cout << " object rewrite rewrite the specified object\n"; + cout << " object reindex reindex the object(s) indicated by --bucket and either --object or --objects-file\n"; cout << " objects expire run expired objects cleanup\n"; cout << " objects expire-stale list list stale expired objects (caused by reshard)\n"; cout << " objects expire-stale rm remove stale expired objects\n"; @@ -340,6 +342,7 @@ void usage() cout << " --bucket= Specify the bucket name. Also used by the quota command.\n"; cout << " --pool= Specify the pool name. Also used to scan for leaked rados objects.\n"; cout << " --object= object name\n"; + cout << " --objects-file= file containing a list of object names to process\n"; cout << " --object-version= object version\n"; cout << " --date= date in the format yyyy-mm-dd\n"; cout << " --start-date= start date in the format yyyy-mm-dd\n"; @@ -425,7 +428,7 @@ void usage() cout << " --show-log-sum= enable/disable dump of log summation on log show\n"; cout << " --skip-zero-entries log show only dumps entries that don't have zero value\n"; cout << " in one of the numeric field\n"; - cout << " --infile= specify a file to read in when setting data\n"; + cout << " --infile= file to read in when setting data\n"; cout << " --categories= comma separated list of categories, used in usage show\n"; cout << " --caps= list of caps (e.g., \"usage=read, write; user=read\")\n"; cout << " --op-mask= permission of user's operations (e.g., \"read, write, delete, *\")\n"; @@ -682,6 +685,7 @@ enum class OPT { OBJECT_UNLINK, OBJECT_STAT, OBJECT_REWRITE, + OBJECT_REINDEX, OBJECTS_EXPIRE, OBJECTS_EXPIRE_STALE_LIST, OBJECTS_EXPIRE_STALE_RM, @@ -898,6 +902,7 @@ static SimpleCmd::Commands all_cmds = { { "object unlink", OPT::OBJECT_UNLINK }, { "object stat", OPT::OBJECT_STAT }, { "object rewrite", OPT::OBJECT_REWRITE }, + { "object reindex", OPT::OBJECT_REINDEX }, { "objects expire", OPT::OBJECTS_EXPIRE }, { "objects expire-stale list", OPT::OBJECTS_EXPIRE_STALE_LIST }, { "objects expire-stale rm", OPT::OBJECTS_EXPIRE_STALE_RM }, @@ -3402,6 +3407,7 @@ int main(int argc, const char **argv) string op_mask_str; string quota_scope; string ratelimit_scope; + std::string objects_file; string object_version; string placement_id; std::optional opt_storage_class; @@ -3579,6 +3585,8 @@ int main(int argc, const char **argv) pool = rgw_pool(pool_name); } else if (ceph_argparse_witharg(args, i, &val, "-o", "--object", (char*)NULL)) { object = val; + } else if (ceph_argparse_witharg(args, i, &val, "--objects-file", (char*)NULL)) { + objects_file = val; } else if (ceph_argparse_witharg(args, i, &val, "--object-version", (char*)NULL)) { object_version = val; } else if (ceph_argparse_witharg(args, i, &val, "--client-id", (char*)NULL)) { @@ -7774,7 +7782,81 @@ int main(int argc, const char **argv) } else { ldpp_dout(dpp(), 20) << "skipped object" << dendl; } - } + } // OPT::OBJECT_REWRITE + + if (opt_cmd == OPT::OBJECT_REINDEX) { + if (bucket_name.empty()) { + cerr << "ERROR: --bucket not specified." << std::endl; + return EINVAL; + } + if (object.empty() && objects_file.empty()) { + cerr << "ERROR: neither --object nor --objects-file specified." << std::endl; + return EINVAL; + } else if (!object.empty() && !objects_file.empty()) { + cerr << "ERROR: both --object and --objects-file specified and only one is allowed." << std::endl; + return EINVAL; + } else if (!objects_file.empty() && !object_version.empty()) { + cerr << "ERROR: cannot specify --object_version when --objects-file specified." << std::endl; + return EINVAL; + } + + int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket); + if (ret < 0) { + cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << + "." << std::endl; + return -ret; + } + + rgw::sal::RadosStore* rados_store = dynamic_cast(driver); + if (!rados_store) { + cerr << + "ERROR: this command can only work when the cluster has a RADOS backing store." << + std::endl; + return EPERM; + } + RGWRados* store = rados_store->getRados(); + + auto process = [&](const std::string& p_object, const std::string& p_object_version) -> int { + std::unique_ptr obj = bucket->get_object(p_object); + obj->set_instance(p_object_version); + ret = store->reindex_obj(bucket->get_info(), obj->get_obj(), dpp(), null_yield); + if (ret < 0) { + return ret; + } + return 0; + }; + + if (!object.empty()) { + ret = process(object, object_version); + if (ret < 0) { + return -ret; + } + } else { + std::ifstream file; + file.open(objects_file); + if (!file.is_open()) { + std::cerr << "ERROR: unable to open objects-file \"" << + objects_file << "\"." << std::endl; + return ENOENT; + } + + std::string obj_name; + const std::string empty_version; + while (std::getline(file, obj_name)) { + ret = process(obj_name, empty_version); + if (ret < 0) { + std::cerr << "ERROR: while processing \"" << obj_name << + "\", received " << cpp_strerror(-ret) << "." << std::endl; + if (!yes_i_really_mean_it) { + std::cerr << + "NOTE: with *caution* you can use --yes-i-really-mean-it to push through errors and continue processing." << + std::endl; + return -ret; + } + } + } // while + } + } // OPT::OBJECT_REINDEX if (opt_cmd == OPT::OBJECTS_EXPIRE) { if (!static_cast(driver)->getRados()->process_expire_objects(dpp())) { diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 6c987c413ecb..72b14c09c835 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -42,6 +42,7 @@ object stat stat an object for its metadata object unlink unlink object from bucket index object rewrite rewrite the specified object + object reindex reindex the object(s) indicated by --bucket and either --object or --objects-file objects expire run expired objects cleanup objects expire-stale list list stale expired objects (caused by reshard) objects expire-stale rm remove stale expired objects @@ -211,6 +212,7 @@ --bucket= Specify the bucket name. Also used by the quota command. --pool= Specify the pool name. Also used to scan for leaked rados objects. --object= object name + --objects-file= file containing a list of object names to process --object-version= object version --date= date in the format yyyy-mm-dd --start-date= start date in the format yyyy-mm-dd @@ -296,7 +298,7 @@ --show-log-sum= enable/disable dump of log summation on log show --skip-zero-entries log show only dumps entries that don't have zero value in one of the numeric field - --infile= specify a file to read in when setting data + --infile= file to read in when setting data --categories= comma separated list of categories, used in usage show --caps= list of caps (e.g., "usage=read, write; user=read") --op-mask= permission of user's operations (e.g., "read, write, delete, *")