From 90b774127d6933000a127fe62c77b68fa4dd03b5 Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Tue, 7 Nov 2023 08:35:26 -0800 Subject: [PATCH] chore: upgrade to the latest dynamodb-lock crate The new version of this crate properly sets a lease duration such that the locks can actually expire --- crates/deltalake-core/Cargo.toml | 2 +- crates/deltalake-core/src/storage/s3.rs | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/deltalake-core/Cargo.toml b/crates/deltalake-core/Cargo.toml index ce1c7490ad..b3a6178203 100644 --- a/crates/deltalake-core/Cargo.toml +++ b/crates/deltalake-core/Cargo.toml @@ -117,7 +117,7 @@ sqlparser = { version = "0.38", optional = true } fs_extra = { version = "1.3.0", optional = true } tempdir = { version = "0", optional = true } -dynamodb_lock = { version = "0", default-features = false, optional = true } +dynamodb_lock = { version = "0.6.0", default-features = false, optional = true } [dev-dependencies] dotenvy = "0" diff --git a/crates/deltalake-core/src/storage/s3.rs b/crates/deltalake-core/src/storage/s3.rs index 23e091dea5..7594d1b06c 100644 --- a/crates/deltalake-core/src/storage/s3.rs +++ b/crates/deltalake-core/src/storage/s3.rs @@ -3,7 +3,7 @@ use super::utils::str_is_truthy; use crate::table::builder::{s3_storage_options, str_option}; use bytes::Bytes; -use dynamodb_lock::{DynamoError, LockClient, LockItem, DEFAULT_MAX_RETRY_ACQUIRE_LOCK_ATTEMPTS}; +use dynamodb_lock::{DynamoError, LockClient, LockItem}; use futures::stream::BoxStream; use object_store::path::Path; use object_store::{ @@ -23,6 +23,7 @@ use std::time::Duration; use tokio::io::AsyncWrite; const STORE_NAME: &str = "DeltaS3ObjectStore"; +const DEFAULT_MAX_RETRY_ACQUIRE_LOCK_ATTEMPTS: u32 = 1_000; /// Error raised by storage lock client #[derive(thiserror::Error, Debug)] @@ -535,10 +536,11 @@ fn try_create_lock_client(options: &S3StorageOptions) -> Result rusoto_dynamodb::DynamoDbClient::new(options.region.clone()), }; - let lock_client = dynamodb_lock::DynamoDbLockClient::new( - dynamodb_client, - dynamodb_lock::DynamoDbOptions::from_map(options.extra_opts.clone()), - ); + let lock_client = dynamodb_lock::DynamoDbLockClient::for_region(options.region.clone()) + .with_client(dynamodb_client) + .with_options(dynamodb_lock::DynamoDbOptions::from_map( + options.extra_opts.clone(), + )); Ok(Some(S3LockClient { lock_client: Box::new(lock_client), }))