From 910fad0a40c2c5e4b912652a55b0df7b4a81d557 Mon Sep 17 00:00:00 2001 From: Nick Santana Date: Wed, 23 Oct 2024 13:42:23 -0700 Subject: [PATCH 1/3] Treat 404 as bucket doesn't exist Previously the fog distribution resync logic would only look for missing keys. Now it handles a missing bucket or 404. --- ledger/distribution/src/main.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ledger/distribution/src/main.rs b/ledger/distribution/src/main.rs index 70545764bc..b7edce3c1a 100644 --- a/ledger/distribution/src/main.rs +++ b/ledger/distribution/src/main.rs @@ -15,7 +15,7 @@ use mc_ledger_db::{Ledger, LedgerDB}; use mc_util_telemetry::{mark_span_as_active, start_block_span, tracer, Tracer}; use protobuf::Message; use retry::{delay, retry, OperationResult}; -use rusoto_core::{Region, RusotoError}; +use rusoto_core::{request::BufferedHttpResponse, Region, RusotoError}; use rusoto_s3::{HeadObjectError, HeadObjectRequest, PutObjectRequest, S3Client, S3}; use serde::{Deserialize, Serialize}; use std::{fs, path::PathBuf}; @@ -212,6 +212,13 @@ impl BlockHandler for S3BlockWriter { Err(RusotoError::Service(HeadObjectError::NoSuchKey(_))) => { OperationResult::Ok(false) } + // Happens when the bucket itself doesn't exist + // Separate from the above condition for the `status.as_u16()` logic + Err(RusotoError::Unknown(BufferedHttpResponse { status, .. })) + if status.as_u16() == 404 => + { + OperationResult::Ok(false) + } Err(e) => { log::warn!(self.logger, "Failed to talk to S3 {e:?}, retrying..."); OperationResult::Retry(e) From 55b319c6ca0cd7b47224b110676030c47cbf5b78 Mon Sep 17 00:00:00 2001 From: Nick Santana Date: Thu, 24 Oct 2024 08:49:52 -0700 Subject: [PATCH 2/3] Remove NoSuchKey path, it can't be hit for head-objects --- ledger/distribution/src/main.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ledger/distribution/src/main.rs b/ledger/distribution/src/main.rs index b7edce3c1a..d22246acf9 100644 --- a/ledger/distribution/src/main.rs +++ b/ledger/distribution/src/main.rs @@ -209,11 +209,8 @@ impl BlockHandler for S3BlockWriter { let result = runtime.block_on(self.s3_client.head_object(req)); match result { Ok(_) => OperationResult::Ok(true), - Err(RusotoError::Service(HeadObjectError::NoSuchKey(_))) => { - OperationResult::Ok(false) - } - // Happens when the bucket itself doesn't exist - // Separate from the above condition for the `status.as_u16()` logic + // Happens when the object doesn't exist, see + // https://docs.aws.amazon.com/cli/latest/reference/s3api/head-object.html Err(RusotoError::Unknown(BufferedHttpResponse { status, .. })) if status.as_u16() == 404 => { From e366ae76438f8c0f8fc6ae1aa13d0d7ffe59d209 Mon Sep 17 00:00:00 2001 From: Nick Santana Date: Thu, 24 Oct 2024 09:08:13 -0700 Subject: [PATCH 3/3] Fix clippy warnings --- ledger/distribution/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/distribution/src/main.rs b/ledger/distribution/src/main.rs index d22246acf9..b14cb4bc49 100644 --- a/ledger/distribution/src/main.rs +++ b/ledger/distribution/src/main.rs @@ -16,7 +16,7 @@ use mc_util_telemetry::{mark_span_as_active, start_block_span, tracer, Tracer}; use protobuf::Message; use retry::{delay, retry, OperationResult}; use rusoto_core::{request::BufferedHttpResponse, Region, RusotoError}; -use rusoto_s3::{HeadObjectError, HeadObjectRequest, PutObjectRequest, S3Client, S3}; +use rusoto_s3::{HeadObjectRequest, PutObjectRequest, S3Client, S3}; use serde::{Deserialize, Serialize}; use std::{fs, path::PathBuf}; use tokio::runtime::Handle;