From f3a3eefd26284776ab3179116374009ec537ab11 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Thu, 17 Oct 2024 10:29:53 -0400 Subject: [PATCH 01/21] feat(pageserver): do space check before gc-compaction (#9250) part of https://github.com/neondatabase/neon/issues/9114 ## Summary of changes gc-compaction may take a lot of disk space, and if it does, the caller should do a partial gc-compaction. This patch adds space check for the compaction job. --------- Signed-off-by: Alex Chi Z --- pageserver/src/disk_usage_eviction_task.rs | 11 +---- pageserver/src/statvfs.rs | 16 ++++++++ pageserver/src/tenant/storage_layer/layer.rs | 4 ++ pageserver/src/tenant/timeline/compaction.rs | 42 ++++++++++++++++++++ 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index a58fa2c0b1e1..7ab2ba87420f 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -1218,16 +1218,7 @@ mod filesystem_level_usage { let stat = Statvfs::get(tenants_dir, mock_config) .context("statvfs failed, presumably directory got unlinked")?; - // https://unix.stackexchange.com/a/703650 - let blocksize = if stat.fragment_size() > 0 { - stat.fragment_size() - } else { - stat.block_size() - }; - - // use blocks_available (b_avail) since, pageserver runs as unprivileged user - let avail_bytes = stat.blocks_available() * blocksize; - let total_bytes = stat.blocks() * blocksize; + let (avail_bytes, total_bytes) = stat.get_avail_total_bytes(); Ok(Usage { config, diff --git a/pageserver/src/statvfs.rs b/pageserver/src/statvfs.rs index 5a6f6e5176ac..205605bc86a8 100644 --- a/pageserver/src/statvfs.rs +++ b/pageserver/src/statvfs.rs @@ -53,6 +53,22 @@ impl Statvfs { Statvfs::Mock(stat) => stat.block_size, } } + + /// Get the available and total bytes on the filesystem. + pub fn get_avail_total_bytes(&self) -> (u64, u64) { + // https://unix.stackexchange.com/a/703650 + let blocksize = if self.fragment_size() > 0 { + self.fragment_size() + } else { + self.block_size() + }; + + // use blocks_available (b_avail) since, pageserver runs as unprivileged user + let avail_bytes = self.blocks_available() * blocksize; + let total_bytes = self.blocks() * blocksize; + + (avail_bytes, total_bytes) + } } pub mod mock { diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index bbb21b180ed5..f29a33bae6de 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -341,6 +341,10 @@ impl Layer { Ok(()) } + pub(crate) async fn needs_download(&self) -> Result, std::io::Error> { + self.0.needs_download().await + } + /// Assuming the layer is already downloaded, returns a guard which will prohibit eviction /// while the guard exists. /// diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index 8b9ace1e5bbf..5588363330d0 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -29,6 +29,7 @@ use utils::id::TimelineId; use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder}; use crate::page_cache; +use crate::statvfs::Statvfs; use crate::tenant::checks::check_valid_layermap; use crate::tenant::remote_timeline_client::WaitCompletionError; use crate::tenant::storage_layer::filter_iterator::FilterIterator; @@ -1691,6 +1692,45 @@ impl Timeline { unreachable!("key retention is empty") } + /// Check how much space is left on the disk + async fn check_available_space(self: &Arc) -> anyhow::Result { + let tenants_dir = self.conf.tenants_path(); + + let stat = Statvfs::get(&tenants_dir, None) + .context("statvfs failed, presumably directory got unlinked")?; + + let (avail_bytes, _) = stat.get_avail_total_bytes(); + + Ok(avail_bytes) + } + + /// Check if the compaction can proceed safely without running out of space. We assume the size + /// upper bound of the produced files of a compaction job is the same as all layers involved in + /// the compaction. Therefore, we need `2 * layers_to_be_compacted_size` at least to do a + /// compaction. + async fn check_compaction_space( + self: &Arc, + layer_selection: &[Layer], + ) -> anyhow::Result<()> { + let available_space = self.check_available_space().await?; + let mut remote_layer_size = 0; + let mut all_layer_size = 0; + for layer in layer_selection { + let needs_download = layer.needs_download().await?; + if needs_download.is_some() { + remote_layer_size += layer.layer_desc().file_size; + } + all_layer_size += layer.layer_desc().file_size; + } + let allocated_space = (available_space as f64 * 0.8) as u64; /* reserve 20% space for other tasks */ + if all_layer_size /* space needed for newly-generated file */ + remote_layer_size /* space for downloading layers */ > allocated_space + { + return Err(anyhow!("not enough space for compaction: available_space={}, allocated_space={}, all_layer_size={}, remote_layer_size={}, required_space={}", + available_space, allocated_space, all_layer_size, remote_layer_size, all_layer_size + remote_layer_size)); + } + Ok(()) + } + /// An experimental compaction building block that combines compaction with garbage collection. /// /// The current implementation picks all delta + image layers that are below or intersecting with @@ -1806,6 +1846,8 @@ impl Timeline { lowest_retain_lsn ); + self.check_compaction_space(&layer_selection).await?; + // Step 1: (In the future) construct a k-merge iterator over all layers. For now, simply collect all keys + LSNs. // Also, verify if the layer map can be split by drawing a horizontal line at every LSN start/end split point. let mut lsn_split_point = BTreeSet::new(); // TODO: use a better data structure (range tree / range set?) From 4c9835f4a3065648c2d6ecd721664b88557aca0f Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 17 Oct 2024 16:34:51 +0200 Subject: [PATCH 02/21] storage_controller: delete stale shards when deleting tenant (#9333) ## Problem Tenant deletion only removes the current shards from remote storage. Any stale parent shards (before splits) will be left behind. These shards are kept since child shards may reference data from the parent until new image layers are generated. ## Summary of changes * Document a special case for pageserver tenant deletion that deletes all shards in remote storage when given an unsharded tenant ID, as well as any unsharded tenant data. * Pass an unsharded tenant ID to delete all remote storage under the tenant ID prefix. * Split out `RemoteStorage::delete_prefix()` to delete a bucket prefix, with additional test coverage. * Add a `delimiter` argument to `asset_prefix_empty()` to support partial prefix matches (i.e. all shards starting with a given tenant ID). --- libs/remote_storage/src/lib.rs | 53 +++++- libs/remote_storage/tests/common/tests.rs | 206 ++++++++++++++++++++++ pageserver/src/tenant/mgr.rs | 71 +++----- storage_controller/src/service.rs | 73 ++++---- test_runner/fixtures/pageserver/utils.py | 15 +- test_runner/regress/test_tenant_delete.py | 55 ++++++ 6 files changed, 376 insertions(+), 97 deletions(-) diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index c6466237bf97..719608dd5f1b 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -19,7 +19,12 @@ mod simulate_failures; mod support; use std::{ - collections::HashMap, fmt::Debug, num::NonZeroU32, ops::Bound, pin::Pin, sync::Arc, + collections::HashMap, + fmt::Debug, + num::NonZeroU32, + ops::Bound, + pin::{pin, Pin}, + sync::Arc, time::SystemTime, }; @@ -28,6 +33,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use bytes::Bytes; use futures::{stream::Stream, StreamExt}; +use itertools::Itertools as _; use serde::{Deserialize, Serialize}; use tokio::sync::Semaphore; use tokio_util::sync::CancellationToken; @@ -261,7 +267,7 @@ pub trait RemoteStorage: Send + Sync + 'static { max_keys: Option, cancel: &CancellationToken, ) -> Result { - let mut stream = std::pin::pin!(self.list_streaming(prefix, mode, max_keys, cancel)); + let mut stream = pin!(self.list_streaming(prefix, mode, max_keys, cancel)); let mut combined = stream.next().await.expect("At least one item required")?; while let Some(list) = stream.next().await { let list = list?; @@ -324,6 +330,35 @@ pub trait RemoteStorage: Send + Sync + 'static { cancel: &CancellationToken, ) -> anyhow::Result<()>; + /// Deletes all objects matching the given prefix. + /// + /// NB: this uses NoDelimiter and will match partial prefixes. For example, the prefix /a/b will + /// delete /a/b, /a/b/*, /a/bc, /a/bc/*, etc. + /// + /// If the operation fails because of timeout or cancellation, the root cause of the error will + /// be set to `TimeoutOrCancel`. In such situation it is unknown which deletions, if any, went + /// through. + async fn delete_prefix( + &self, + prefix: &RemotePath, + cancel: &CancellationToken, + ) -> anyhow::Result<()> { + let mut stream = + pin!(self.list_streaming(Some(prefix), ListingMode::NoDelimiter, None, cancel)); + while let Some(result) = stream.next().await { + let keys = match result { + Ok(listing) if listing.keys.is_empty() => continue, + Ok(listing) => listing.keys.into_iter().map(|o| o.key).collect_vec(), + Err(DownloadError::Cancelled) => return Err(TimeoutOrCancel::Cancel.into()), + Err(DownloadError::Timeout) => return Err(TimeoutOrCancel::Timeout.into()), + Err(err) => return Err(err.into()), + }; + tracing::info!("Deleting {} keys from remote storage", keys.len()); + self.delete_objects(&keys, cancel).await?; + } + Ok(()) + } + /// Copy a remote object inside a bucket from one path to another. async fn copy( &self, @@ -488,6 +523,20 @@ impl GenericRemoteStorage> { } } + /// See [`RemoteStorage::delete_prefix`] + pub async fn delete_prefix( + &self, + prefix: &RemotePath, + cancel: &CancellationToken, + ) -> anyhow::Result<()> { + match self { + Self::LocalFs(s) => s.delete_prefix(prefix, cancel).await, + Self::AwsS3(s) => s.delete_prefix(prefix, cancel).await, + Self::AzureBlob(s) => s.delete_prefix(prefix, cancel).await, + Self::Unreliable(s) => s.delete_prefix(prefix, cancel).await, + } + } + /// See [`RemoteStorage::copy`] pub async fn copy_object( &self, diff --git a/libs/remote_storage/tests/common/tests.rs b/libs/remote_storage/tests/common/tests.rs index e6f33fc3f83c..d5da1d48e9c6 100644 --- a/libs/remote_storage/tests/common/tests.rs +++ b/libs/remote_storage/tests/common/tests.rs @@ -199,6 +199,138 @@ async fn list_no_delimiter_works( Ok(()) } +/// Tests that giving a partial prefix returns all matches (e.g. "/foo" yields "/foobar/baz"), +/// but only with NoDelimiter. +#[test_context(MaybeEnabledStorageWithSimpleTestBlobs)] +#[tokio::test] +async fn list_partial_prefix( + ctx: &mut MaybeEnabledStorageWithSimpleTestBlobs, +) -> anyhow::Result<()> { + let ctx = match ctx { + MaybeEnabledStorageWithSimpleTestBlobs::Enabled(ctx) => ctx, + MaybeEnabledStorageWithSimpleTestBlobs::Disabled => return Ok(()), + MaybeEnabledStorageWithSimpleTestBlobs::UploadsFailed(e, _) => { + anyhow::bail!("S3 init failed: {e:?}") + } + }; + + let cancel = CancellationToken::new(); + let test_client = Arc::clone(&ctx.enabled.client); + + // Prefix "fold" should match all "folder{i}" directories with NoDelimiter. + let objects: HashSet<_> = test_client + .list( + Some(&RemotePath::from_string("fold")?), + ListingMode::NoDelimiter, + None, + &cancel, + ) + .await? + .keys + .into_iter() + .map(|o| o.key) + .collect(); + assert_eq!(&objects, &ctx.remote_blobs); + + // Prefix "fold" matches nothing with WithDelimiter. + let objects: HashSet<_> = test_client + .list( + Some(&RemotePath::from_string("fold")?), + ListingMode::WithDelimiter, + None, + &cancel, + ) + .await? + .keys + .into_iter() + .map(|o| o.key) + .collect(); + assert!(objects.is_empty()); + + // Prefix "" matches everything. + let objects: HashSet<_> = test_client + .list( + Some(&RemotePath::from_string("")?), + ListingMode::NoDelimiter, + None, + &cancel, + ) + .await? + .keys + .into_iter() + .map(|o| o.key) + .collect(); + assert_eq!(&objects, &ctx.remote_blobs); + + // Prefix "" matches nothing with WithDelimiter. + let objects: HashSet<_> = test_client + .list( + Some(&RemotePath::from_string("")?), + ListingMode::WithDelimiter, + None, + &cancel, + ) + .await? + .keys + .into_iter() + .map(|o| o.key) + .collect(); + assert!(objects.is_empty()); + + // Prefix "foo" matches nothing. + let objects: HashSet<_> = test_client + .list( + Some(&RemotePath::from_string("foo")?), + ListingMode::NoDelimiter, + None, + &cancel, + ) + .await? + .keys + .into_iter() + .map(|o| o.key) + .collect(); + assert!(objects.is_empty()); + + // Prefix "folder2/blob" matches. + let objects: HashSet<_> = test_client + .list( + Some(&RemotePath::from_string("folder2/blob")?), + ListingMode::NoDelimiter, + None, + &cancel, + ) + .await? + .keys + .into_iter() + .map(|o| o.key) + .collect(); + let expect: HashSet<_> = ctx + .remote_blobs + .iter() + .filter(|o| o.get_path().starts_with("folder2")) + .cloned() + .collect(); + assert_eq!(&objects, &expect); + + // Prefix "folder2/foo" matches nothing. + let objects: HashSet<_> = test_client + .list( + Some(&RemotePath::from_string("folder2/foo")?), + ListingMode::NoDelimiter, + None, + &cancel, + ) + .await? + .keys + .into_iter() + .map(|o| o.key) + .collect(); + assert!(objects.is_empty()); + + Ok(()) +} + #[test_context(MaybeEnabledStorage)] #[tokio::test] async fn delete_non_exising_works(ctx: &mut MaybeEnabledStorage) -> anyhow::Result<()> { @@ -265,6 +397,80 @@ async fn delete_objects_works(ctx: &mut MaybeEnabledStorage) -> anyhow::Result<( Ok(()) } +/// Tests that delete_prefix() will delete all objects matching a prefix, including +/// partial prefixes (i.e. "/foo" matches "/foobar"). +#[test_context(MaybeEnabledStorageWithSimpleTestBlobs)] +#[tokio::test] +async fn delete_prefix(ctx: &mut MaybeEnabledStorageWithSimpleTestBlobs) -> anyhow::Result<()> { + let ctx = match ctx { + MaybeEnabledStorageWithSimpleTestBlobs::Enabled(ctx) => ctx, + MaybeEnabledStorageWithSimpleTestBlobs::Disabled => return Ok(()), + MaybeEnabledStorageWithSimpleTestBlobs::UploadsFailed(e, _) => { + anyhow::bail!("S3 init failed: {e:?}") + } + }; + + let cancel = CancellationToken::new(); + let test_client = Arc::clone(&ctx.enabled.client); + + /// Asserts that the S3 listing matches the given paths. + macro_rules! assert_list { + ($expect:expr) => {{ + let listing = test_client + .list(None, ListingMode::NoDelimiter, None, &cancel) + .await? + .keys + .into_iter() + .map(|o| o.key) + .collect(); + assert_eq!($expect, listing); + }}; + } + + // We start with the full set of uploaded files. + let mut expect = ctx.remote_blobs.clone(); + + // Deleting a non-existing prefix should do nothing. + test_client + .delete_prefix(&RemotePath::from_string("xyz")?, &cancel) + .await?; + assert_list!(expect); + + // Prefixes are case-sensitive. + test_client + .delete_prefix(&RemotePath::from_string("Folder")?, &cancel) + .await?; + assert_list!(expect); + + // Deleting a path which overlaps with an existing object should do nothing. We pick the first + // path in the set as our common prefix. + let path = expect.iter().next().expect("empty set").clone().join("xyz"); + test_client.delete_prefix(&path, &cancel).await?; + assert_list!(expect); + + // Deleting an exact path should work. We pick the first path in the set. + let path = expect.iter().next().expect("empty set").clone(); + test_client.delete_prefix(&path, &cancel).await?; + expect.remove(&path); + assert_list!(expect); + + // Deleting a prefix should delete all matching objects. + test_client + .delete_prefix(&RemotePath::from_string("folder0/blob_")?, &cancel) + .await?; + expect.retain(|p| !p.get_path().as_str().starts_with("folder0/")); + assert_list!(expect); + + // Deleting a common prefix should delete all objects. + test_client + .delete_prefix(&RemotePath::from_string("fold")?, &cancel) + .await?; + expect.clear(); + assert_list!(expect); + + Ok(()) +} + #[test_context(MaybeEnabledStorage)] #[tokio::test] async fn upload_download_works(ctx: &mut MaybeEnabledStorage) -> anyhow::Result<()> { diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 9d9852c525b9..0567f8f3a7b9 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -11,6 +11,7 @@ use pageserver_api::shard::{ }; use pageserver_api::upcall_api::ReAttachResponseTenant; use rand::{distributions::Alphanumeric, Rng}; +use remote_storage::TimeoutOrCancel; use std::borrow::Cow; use std::cmp::Ordering; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -1350,47 +1351,17 @@ impl TenantManager { } } - async fn delete_tenant_remote( - &self, - tenant_shard_id: TenantShardId, - ) -> Result<(), DeleteTenantError> { - let remote_path = remote_tenant_path(&tenant_shard_id); - let mut keys_stream = self.resources.remote_storage.list_streaming( - Some(&remote_path), - remote_storage::ListingMode::NoDelimiter, - None, - &self.cancel, - ); - while let Some(chunk) = keys_stream.next().await { - let keys = match chunk { - Ok(listing) => listing.keys, - Err(remote_storage::DownloadError::Cancelled) => { - return Err(DeleteTenantError::Cancelled) - } - Err(remote_storage::DownloadError::NotFound) => return Ok(()), - Err(other) => return Err(DeleteTenantError::Other(anyhow::anyhow!(other))), - }; - - if keys.is_empty() { - tracing::info!("Remote storage already deleted"); - } else { - tracing::info!("Deleting {} keys from remote storage", keys.len()); - let keys = keys.into_iter().map(|o| o.key).collect::>(); - self.resources - .remote_storage - .delete_objects(&keys, &self.cancel) - .await?; - } - } - - Ok(()) - } - /// If a tenant is attached, detach it. Then remove its data from remote storage. /// /// A tenant is considered deleted once it is gone from remote storage. It is the caller's /// responsibility to avoid trying to attach the tenant again or use it any way once deletion /// has started: this operation is not atomic, and must be retried until it succeeds. + /// + /// As a special case, if an unsharded tenant ID is given for a sharded tenant, it will remove + /// all tenant shards in remote storage (removing all paths with the tenant prefix). The storage + /// controller uses this to purge all remote tenant data, including any stale parent shards that + /// may remain after splits. Ideally, this special case would be handled elsewhere. See: + /// . pub(crate) async fn delete_tenant( &self, tenant_shard_id: TenantShardId, @@ -1442,25 +1413,29 @@ impl TenantManager { // in 500 responses to delete requests. // - We keep the `SlotGuard` during this I/O, so that if a concurrent delete request comes in, it will // 503/retry, rather than kicking off a wasteful concurrent deletion. - match backoff::retry( - || async move { self.delete_tenant_remote(tenant_shard_id).await }, - |e| match e { - DeleteTenantError::Cancelled => true, - DeleteTenantError::SlotError(_) => { - unreachable!("Remote deletion doesn't touch slots") - } - _ => false, + // NB: this also deletes partial prefixes, i.e. a path will delete all + // _/* objects. See method comment for why. + backoff::retry( + || async move { + self.resources + .remote_storage + .delete_prefix(&remote_tenant_path(&tenant_shard_id), &self.cancel) + .await }, + |_| false, // backoff::retry handles cancellation 1, 3, &format!("delete_tenant[tenant_shard_id={tenant_shard_id}]"), &self.cancel, ) .await - { - Some(r) => r, - None => Err(DeleteTenantError::Cancelled), - } + .unwrap_or(Err(TimeoutOrCancel::Cancel.into())) + .map_err(|err| { + if TimeoutOrCancel::caused_by_cancel(&err) { + return DeleteTenantError::Cancelled; + } + DeleteTenantError::Other(err) + }) } #[instrument(skip_all, fields(tenant_id=%tenant.get_tenant_shard_id().tenant_id, shard_id=%tenant.get_tenant_shard_id().shard_slug(), new_shard_count=%new_shard_count.literal()))] diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 25e1fb5e1f80..ab2c3b5e480f 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -2862,17 +2862,12 @@ impl Service { let _tenant_lock = trace_exclusive_lock(&self.tenant_op_locks, tenant_id, TenantOperations::Delete).await; - // Detach all shards - let (detach_waiters, shard_ids, node) = { - let mut shard_ids = Vec::new(); + // Detach all shards. This also deletes local pageserver shard data. + let (detach_waiters, node) = { let mut detach_waiters = Vec::new(); let mut locked = self.inner.write().unwrap(); let (nodes, tenants, scheduler) = locked.parts_mut(); - for (tenant_shard_id, shard) in - tenants.range_mut(TenantShardId::tenant_range(tenant_id)) - { - shard_ids.push(*tenant_shard_id); - + for (_, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) { // Update the tenant's intent to remove all attachments shard.policy = PlacementPolicy::Detached; shard @@ -2892,7 +2887,7 @@ impl Service { let node = nodes .get(&node_id) .expect("Pageservers may not be deleted while lock is active"); - (detach_waiters, shard_ids, node.clone()) + (detach_waiters, node.clone()) }; // This reconcile wait can fail in a few ways: @@ -2907,38 +2902,34 @@ impl Service { self.await_waiters(detach_waiters, RECONCILE_TIMEOUT) .await?; - let locations = shard_ids - .into_iter() - .map(|s| (s, node.clone())) - .collect::>(); - let results = self.tenant_for_shards_api( - locations, - |tenant_shard_id, client| async move { client.tenant_delete(tenant_shard_id).await }, - 1, - 3, - RECONCILE_TIMEOUT, - &self.cancel, - ) - .await; - for result in results { - match result { - Ok(StatusCode::ACCEPTED) => { - // This should never happen: we waited for detaches to finish above - return Err(ApiError::InternalServerError(anyhow::anyhow!( - "Unexpectedly still attached on {}", - node - ))); - } - Ok(_) => {} - Err(mgmt_api::Error::Cancelled) => { - return Err(ApiError::ShuttingDown); - } - Err(e) => { - // This is unexpected: remote deletion should be infallible, unless the object store - // at large is unavailable. - tracing::error!("Error deleting via node {}: {e}", node); - return Err(ApiError::InternalServerError(anyhow::anyhow!(e))); - } + // Delete the entire tenant (all shards) from remote storage via a random pageserver. + // Passing an unsharded tenant ID will cause the pageserver to remove all remote paths with + // the tenant ID prefix, including all shards (even possibly stale ones). + match node + .with_client_retries( + |client| async move { + client + .tenant_delete(TenantShardId::unsharded(tenant_id)) + .await + }, + &self.config.jwt_token, + 1, + 3, + RECONCILE_TIMEOUT, + &self.cancel, + ) + .await + .unwrap_or(Err(mgmt_api::Error::Cancelled)) + { + Ok(_) => {} + Err(mgmt_api::Error::Cancelled) => { + return Err(ApiError::ShuttingDown); + } + Err(e) => { + // This is unexpected: remote deletion should be infallible, unless the object store + // at large is unavailable. + tracing::error!("Error deleting via node {node}: {e}"); + return Err(ApiError::InternalServerError(anyhow::anyhow!(e))); } } diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index 377a95fbebc6..4c4306be9ef5 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -303,9 +303,10 @@ def assert_prefix_empty( remote_storage: Optional[RemoteStorage], prefix: Optional[str] = None, allowed_postfix: Optional[str] = None, + delimiter: str = "/", ) -> None: assert remote_storage is not None - response = list_prefix(remote_storage, prefix) + response = list_prefix(remote_storage, prefix, delimiter) keys = response["KeyCount"] objects: list[ObjectTypeDef] = response.get("Contents", []) common_prefixes = response.get("CommonPrefixes", []) @@ -338,16 +339,18 @@ def assert_prefix_empty( if not (allowed_postfix.endswith(key)): filtered_count += 1 - assert ( - filtered_count == 0 - ), f"remote dir with prefix {prefix} is not empty after deletion: {objects}" + assert filtered_count == 0, f"remote prefix {prefix} is not empty: {objects}" # remote_storage must not be None, but that's easier for callers to make mypy happy -def assert_prefix_not_empty(remote_storage: Optional[RemoteStorage], prefix: Optional[str] = None): +def assert_prefix_not_empty( + remote_storage: Optional[RemoteStorage], + prefix: Optional[str] = None, + delimiter: str = "/", +): assert remote_storage is not None response = list_prefix(remote_storage, prefix) - assert response["KeyCount"] != 0, f"remote dir with prefix {prefix} is empty: {response}" + assert response["KeyCount"] != 0, f"remote prefix {prefix} is empty: {response}" def list_prefix( diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index 294c1248c525..f4863274457d 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -20,6 +20,7 @@ ) from fixtures.remote_storage import RemoteStorageKind, s3_storage from fixtures.utils import run_pg_bench_small, wait_until +from fixtures.workload import Workload from requests.exceptions import ReadTimeout from werkzeug.wrappers.request import Request from werkzeug.wrappers.response import Response @@ -404,3 +405,57 @@ def get_branches(request: Request): cloud_admin_api_token=cloud_admin_token, ) assert healthy + + +def test_tenant_delete_stale_shards(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): + """ + Deleting a tenant should also delete any stale (pre-split) shards from remote storage. + """ + remote_storage_kind = s3_storage() + neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + + env = neon_env_builder.init_start() + + # Create an unsharded tenant. + tenant_id, timeline_id = env.create_tenant() + + # Write some data. + workload = Workload(env, tenant_id, timeline_id, branch_name="main") + workload.init() + workload.write_rows(256) + workload.validate() + + assert_prefix_not_empty( + neon_env_builder.pageserver_remote_storage, + prefix="/".join(("tenants", str(tenant_id))), + ) + + # Upload a heatmap as well. + env.pageserver.http_client().tenant_heatmap_upload(tenant_id) + + # Split off a few shards, in two rounds. + env.storage_controller.tenant_shard_split(tenant_id, shard_count=4) + env.storage_controller.tenant_shard_split(tenant_id, shard_count=16) + + # Delete the tenant. This should also delete data for the unsharded and count=4 parents. + env.storage_controller.pageserver_api().tenant_delete(tenant_id=tenant_id) + + assert_prefix_empty( + neon_env_builder.pageserver_remote_storage, + prefix="/".join(("tenants", str(tenant_id))), + delimiter="", # match partial prefixes, i.e. all shards + ) + + dirs = list(env.pageserver.tenant_dir(None).glob(f"{tenant_id}*")) + assert dirs == [], f"found tenant directories: {dirs}" + + # The initial tenant created by the test harness should still be there. + # Only the tenant we deleted should be removed. + assert_prefix_not_empty( + neon_env_builder.pageserver_remote_storage, + prefix="/".join(("tenants", str(env.initial_tenant))), + ) + dirs = list(env.pageserver.tenant_dir(None).glob(f"{env.initial_tenant}*")) + assert dirs != [], "missing initial tenant directory" + + env.stop() From 299cde899b7b9a31723508afdf7b9e0f0be13912 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 17 Oct 2024 17:19:18 +0200 Subject: [PATCH 03/21] safekeeper: flush WAL on compute disconnect (#9436) ## Problem In #9259, we found that the `check_safekeepers_synced` fast path could result in a lower basebackup LSN than the `flush_lsn` reported by Safekeepers in `VoteResponse`, causing the compute to panic once on startup. This would happen if the Safekeeper had unflushed WAL records due to a compute disconnect. The `TIMELINE_STATUS` query would report a `flush_lsn` below these unflushed records, while `VoteResponse` would flush the WAL and report the advanced `flush_lsn`. See https://github.com/neondatabase/neon/issues/9259#issuecomment-2410849032. ## Summary of changes Flush the WAL if the compute disconnects during WAL processing. --- safekeeper/src/receive_wal.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index e35f806e9011..2a9ca8529925 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -498,21 +498,18 @@ impl WalAcceptor { // we will send keepalives by replying to these requests once per second. let mut next_keepalive = Instant::now(); - loop { - let opt_msg = self.msg_rx.recv().await; - if opt_msg.is_none() { - return Ok(()); // chan closed, streaming terminated - } - let mut next_msg = opt_msg.unwrap(); - + while let Some(mut next_msg) = self.msg_rx.recv().await { // Update walreceiver state in shmem for reporting. if let ProposerAcceptorMessage::Elected(_) = &next_msg { walreceiver_guard.get().status = WalReceiverStatus::Streaming; } let reply_msg = if matches!(next_msg, ProposerAcceptorMessage::AppendRequest(_)) { - // loop through AppendRequest's while it's readily available to - // write as many WAL as possible without fsyncing + // Loop through AppendRequests while available to write as many WAL records as + // possible without fsyncing. + // + // Make sure the WAL is flushed before returning, see: + // https://github.com/neondatabase/neon/issues/9259 // // Note: this will need to be rewritten if we want to read non-AppendRequest messages here. // Otherwise, we might end up in a situation where we read a message, but don't @@ -522,7 +519,7 @@ impl WalAcceptor { if let Some(reply) = self.tli.process_msg(&noflush_msg).await? { if self.reply_tx.send(reply).await.is_err() { - return Ok(()); // chan closed, streaming terminated + break; // disconnected, flush WAL and return on next send/recv } } @@ -531,11 +528,13 @@ impl WalAcceptor { break; } + // continue pulling AppendRequests if available match self.msg_rx.try_recv() { Ok(msg) => next_msg = msg, Err(TryRecvError::Empty) => break, - Err(TryRecvError::Disconnected) => return Ok(()), // chan closed, streaming terminated - } + // on disconnect, flush WAL and return on next send/recv + Err(TryRecvError::Disconnected) => break, + }; } // flush all written WAL to the disk @@ -555,5 +554,6 @@ impl WalAcceptor { next_keepalive = Instant::now() + KEEPALIVE_INTERVAL; } } + Ok(()) } } From 858867c62771e7f24c3d33820a8ca87c5f4f146f Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Thu, 17 Oct 2024 16:35:19 +0100 Subject: [PATCH 04/21] Add logging of installed_extensions (#9438) Simple PR to log installed_extensions statistics. in the following format: ``` 2024-10-17T13:53:02.860595Z INFO [NEON_EXT_STAT] {"extensions":[{"extname":"plpgsql","versions":["1.0"],"n_databases":2},{"extname":"neon","versions":["1.5"],"n_databases":1}]} ``` --- compute_tools/src/compute.rs | 28 +++++------------------ compute_tools/src/installed_extensions.rs | 21 +++++++++++++++++ 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 285be56264a2..6aec008f3a36 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -34,6 +34,7 @@ use nix::sys::signal::{kill, Signal}; use remote_storage::{DownloadError, RemotePath}; use crate::checker::create_availability_check_data; +use crate::installed_extensions::get_installed_extensions_sync; use crate::local_proxy; use crate::logger::inlinify; use crate::pg_helpers::*; @@ -1121,6 +1122,11 @@ impl ComputeNode { self.pg_reload_conf()?; } self.post_apply_config()?; + + let connstr = self.connstr.clone(); + thread::spawn(move || { + get_installed_extensions_sync(connstr).context("get_installed_extensions") + }); } let startup_end_time = Utc::now(); @@ -1484,28 +1490,6 @@ LIMIT 100", info!("Pageserver config changed"); } } - - // Gather info about installed extensions - pub fn get_installed_extensions(&self) -> Result<()> { - let connstr = self.connstr.clone(); - - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("failed to create runtime"); - let result = rt - .block_on(crate::installed_extensions::get_installed_extensions( - connstr, - )) - .expect("failed to get installed extensions"); - - info!( - "{}", - serde_json::to_string(&result).expect("failed to serialize extensions list") - ); - - Ok(()) - } } pub fn forward_termination_signal() { diff --git a/compute_tools/src/installed_extensions.rs b/compute_tools/src/installed_extensions.rs index 72578b1f342a..877f99bff715 100644 --- a/compute_tools/src/installed_extensions.rs +++ b/compute_tools/src/installed_extensions.rs @@ -1,6 +1,7 @@ use compute_api::responses::{InstalledExtension, InstalledExtensions}; use std::collections::HashMap; use std::collections::HashSet; +use tracing::info; use url::Url; use anyhow::Result; @@ -79,3 +80,23 @@ pub async fn get_installed_extensions(connstr: Url) -> Result Result<()> { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to create runtime"); + let result = rt + .block_on(crate::installed_extensions::get_installed_extensions( + connstr, + )) + .expect("failed to get installed extensions"); + + info!( + "[NEON_EXT_STAT] {}", + serde_json::to_string(&result).expect("failed to serialize extensions list") + ); + + Ok(()) +} From 63b3491c1b489487e9d94b8499f401cd57e12290 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Thu, 17 Oct 2024 12:22:44 -0400 Subject: [PATCH 05/21] refactor(pageserver): remove aux v1 code path (#9424) Part of the aux v1 retirement https://github.com/neondatabase/neon/issues/8623 ## Summary of changes Remove write/read path for aux v1, but keeping the config item and the index part field for now. --------- Signed-off-by: Alex Chi Z --- libs/pageserver_api/src/models.rs | 2 - pageserver/src/http/routes.rs | 32 -- pageserver/src/pgdatadir_mapping.rs | 323 +++------------ pageserver/src/tenant.rs | 380 +----------------- .../src/tenant/remote_timeline_client.rs | 14 +- .../tenant/remote_timeline_client/index.rs | 4 - pageserver/src/tenant/timeline.rs | 51 +-- pageserver/src/tenant/timeline/delete.rs | 2 - pageserver/src/walredo/apply_neon.rs | 71 +--- test_runner/regress/test_aux_files.py | 78 ---- 10 files changed, 60 insertions(+), 897 deletions(-) delete mode 100644 test_runner/regress/test_aux_files.py diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 3ec9cac2c321..5b0b6bebe38a 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -743,8 +743,6 @@ pub struct TimelineInfo { // Forward compatibility: a previous version of the pageserver will receive a JSON. serde::Deserialize does // not deny unknown fields by default so it's safe to set the field to some value, though it won't be // read. - /// The last aux file policy being used on this timeline - pub last_aux_file_policy: Option, pub is_archived: Option, } diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 36a6ed427b9b..e6663ef56fa4 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -18,7 +18,6 @@ use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; use metrics::launch_timestamp::LaunchTimestamp; use pageserver_api::models::virtual_file::IoMode; -use pageserver_api::models::AuxFilePolicy; use pageserver_api::models::DownloadRemoteLayersTaskSpawnRequest; use pageserver_api::models::IngestAuxFilesRequest; use pageserver_api::models::ListAuxFilesRequest; @@ -474,8 +473,6 @@ async fn build_timeline_info_common( is_archived: Some(is_archived), walreceiver_status, - - last_aux_file_policy: timeline.last_aux_file_policy.load(), }; Ok(info) } @@ -2399,31 +2396,6 @@ async fn post_tracing_event_handler( json_response(StatusCode::OK, ()) } -async fn force_aux_policy_switch_handler( - mut r: Request, - _cancel: CancellationToken, -) -> Result, ApiError> { - check_permission(&r, None)?; - let tenant_shard_id: TenantShardId = parse_request_param(&r, "tenant_shard_id")?; - let timeline_id: TimelineId = parse_request_param(&r, "timeline_id")?; - let policy: AuxFilePolicy = json_request(&mut r).await?; - - let state = get_state(&r); - - let tenant = state - .tenant_manager - .get_attached_tenant_shard(tenant_shard_id)?; - tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?; - let timeline = - active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id) - .await?; - timeline - .do_switch_aux_policy(policy) - .map_err(ApiError::InternalServerError)?; - - json_response(StatusCode::OK, ()) -} - async fn put_io_engine_handler( mut r: Request, _cancel: CancellationToken, @@ -3136,10 +3108,6 @@ pub fn make_router( ) .put("/v1/io_engine", |r| api_handler(r, put_io_engine_handler)) .put("/v1/io_mode", |r| api_handler(r, put_io_mode_handler)) - .put( - "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/force_aux_policy_switch", - |r| api_handler(r, force_aux_policy_switch_handler), - ) .get("/v1/utilization", |r| api_handler(r, get_utilization)) .post( "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/ingest_aux_files", diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 900da5beabe5..f2a11e65c171 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -22,7 +22,6 @@ use pageserver_api::key::{ CompactKey, AUX_FILES_KEY, CHECKPOINT_KEY, CONTROLFILE_KEY, DBDIR_KEY, TWOPHASEDIR_KEY, }; use pageserver_api::keyspace::SparseKeySpace; -use pageserver_api::models::AuxFilePolicy; use pageserver_api::reltag::{BlockNumber, RelTag, SlruKind}; use postgres_ffi::relfile_utils::{FSM_FORKNUM, VISIBILITYMAP_FORKNUM}; use postgres_ffi::BLCKSZ; @@ -33,7 +32,7 @@ use std::ops::ControlFlow; use std::ops::Range; use strum::IntoEnumIterator; use tokio_util::sync::CancellationToken; -use tracing::{debug, info, trace, warn}; +use tracing::{debug, trace, warn}; use utils::bin_ser::DeserializeError; use utils::pausable_failpoint; use utils::{bin_ser::BeSer, lsn::Lsn}; @@ -677,21 +676,6 @@ impl Timeline { self.get(CHECKPOINT_KEY, lsn, ctx).await } - async fn list_aux_files_v1( - &self, - lsn: Lsn, - ctx: &RequestContext, - ) -> Result, PageReconstructError> { - match self.get(AUX_FILES_KEY, lsn, ctx).await { - Ok(buf) => Ok(AuxFilesDirectory::des(&buf)?.files), - Err(e) => { - // This is expected: historical databases do not have the key. - debug!("Failed to get info about AUX files: {}", e); - Ok(HashMap::new()) - } - } - } - async fn list_aux_files_v2( &self, lsn: Lsn, @@ -722,10 +706,7 @@ impl Timeline { lsn: Lsn, ctx: &RequestContext, ) -> Result<(), PageReconstructError> { - let current_policy = self.last_aux_file_policy.load(); - if let Some(AuxFilePolicy::V2) | Some(AuxFilePolicy::CrossValidation) = current_policy { - self.list_aux_files_v2(lsn, ctx).await?; - } + self.list_aux_files_v2(lsn, ctx).await?; Ok(()) } @@ -734,51 +715,7 @@ impl Timeline { lsn: Lsn, ctx: &RequestContext, ) -> Result, PageReconstructError> { - let current_policy = self.last_aux_file_policy.load(); - match current_policy { - Some(AuxFilePolicy::V1) => { - let res = self.list_aux_files_v1(lsn, ctx).await?; - let empty_str = if res.is_empty() { ", empty" } else { "" }; - warn!( - "this timeline is using deprecated aux file policy V1 (policy=v1{empty_str})" - ); - Ok(res) - } - None => { - let res = self.list_aux_files_v1(lsn, ctx).await?; - if !res.is_empty() { - warn!("this timeline is using deprecated aux file policy V1 (policy=None)"); - } - Ok(res) - } - Some(AuxFilePolicy::V2) => self.list_aux_files_v2(lsn, ctx).await, - Some(AuxFilePolicy::CrossValidation) => { - let v1_result = self.list_aux_files_v1(lsn, ctx).await; - let v2_result = self.list_aux_files_v2(lsn, ctx).await; - match (v1_result, v2_result) { - (Ok(v1), Ok(v2)) => { - if v1 != v2 { - tracing::error!( - "unmatched aux file v1 v2 result:\nv1 {v1:?}\nv2 {v2:?}" - ); - return Err(PageReconstructError::Other(anyhow::anyhow!( - "unmatched aux file v1 v2 result" - ))); - } - Ok(v1) - } - (Ok(_), Err(v2)) => { - tracing::error!("aux file v1 returns Ok while aux file v2 returns an err"); - Err(v2) - } - (Err(v1), Ok(_)) => { - tracing::error!("aux file v2 returns Ok while aux file v1 returns an err"); - Err(v1) - } - (Err(_), Err(v2)) => Err(v2), - } - } - } + self.list_aux_files_v2(lsn, ctx).await } pub(crate) async fn get_replorigins( @@ -954,9 +891,6 @@ impl Timeline { result.add_key(CONTROLFILE_KEY); result.add_key(CHECKPOINT_KEY); - if self.get(AUX_FILES_KEY, lsn, ctx).await.is_ok() { - result.add_key(AUX_FILES_KEY); - } // Add extra keyspaces in the test cases. Some test cases write keys into the storage without // creating directory keys. These test cases will add such keyspaces into `extra_test_dense_keyspace` @@ -1166,9 +1100,6 @@ impl<'a> DatadirModification<'a> { self.pending_directory_entries.push((DirectoryKind::Db, 0)); self.put(DBDIR_KEY, Value::Image(buf.into())); - // Create AuxFilesDirectory - self.init_aux_dir()?; - let buf = if self.tline.pg_version >= 17 { TwoPhaseDirectoryV17::ser(&TwoPhaseDirectoryV17 { xids: HashSet::new(), @@ -1347,9 +1278,6 @@ impl<'a> DatadirModification<'a> { // 'true', now write the updated 'dbdirs' map back. let buf = DbDirectory::ser(&dbdir)?; self.put(DBDIR_KEY, Value::Image(buf.into())); - - // Create AuxFilesDirectory as well - self.init_aux_dir()?; } if r.is_none() { // Create RelDirectory @@ -1726,200 +1654,60 @@ impl<'a> DatadirModification<'a> { Ok(()) } - pub fn init_aux_dir(&mut self) -> anyhow::Result<()> { - if let AuxFilePolicy::V2 = self.tline.get_switch_aux_file_policy() { - return Ok(()); - } - let buf = AuxFilesDirectory::ser(&AuxFilesDirectory { - files: HashMap::new(), - })?; - self.pending_directory_entries - .push((DirectoryKind::AuxFiles, 0)); - self.put(AUX_FILES_KEY, Value::Image(Bytes::from(buf))); - Ok(()) - } - pub async fn put_file( &mut self, path: &str, content: &[u8], ctx: &RequestContext, ) -> anyhow::Result<()> { - let switch_policy = self.tline.get_switch_aux_file_policy(); - - let policy = { - let current_policy = self.tline.last_aux_file_policy.load(); - // Allowed switch path: - // * no aux files -> v1/v2/cross-validation - // * cross-validation->v2 - - let current_policy = if current_policy.is_none() { - // This path will only be hit once per tenant: we will decide the final policy in this code block. - // The next call to `put_file` will always have `last_aux_file_policy != None`. - let lsn = Lsn::max(self.tline.get_last_record_lsn(), self.lsn); - let aux_files_key_v1 = self.tline.list_aux_files_v1(lsn, ctx).await?; - if aux_files_key_v1.is_empty() { - None - } else { - warn!("this timeline is using deprecated aux file policy V1 (detected existing v1 files)"); - self.tline.do_switch_aux_policy(AuxFilePolicy::V1)?; - Some(AuxFilePolicy::V1) - } - } else { - current_policy - }; - - if AuxFilePolicy::is_valid_migration_path(current_policy, switch_policy) { - self.tline.do_switch_aux_policy(switch_policy)?; - info!(current=?current_policy, next=?switch_policy, "switching aux file policy"); - switch_policy - } else { - // This branch handles non-valid migration path, and the case that switch_policy == current_policy. - // And actually, because the migration path always allow unspecified -> *, this unwrap_or will never be hit. - current_policy.unwrap_or(AuxFilePolicy::default_tenant_config()) - } + let key = aux_file::encode_aux_file_key(path); + // retrieve the key from the engine + let old_val = match self.get(key, ctx).await { + Ok(val) => Some(val), + Err(PageReconstructError::MissingKey(_)) => None, + Err(e) => return Err(e.into()), }; - - if let AuxFilePolicy::V2 | AuxFilePolicy::CrossValidation = policy { - let key = aux_file::encode_aux_file_key(path); - // retrieve the key from the engine - let old_val = match self.get(key, ctx).await { - Ok(val) => Some(val), - Err(PageReconstructError::MissingKey(_)) => None, - Err(e) => return Err(e.into()), - }; - let files: Vec<(&str, &[u8])> = if let Some(ref old_val) = old_val { - aux_file::decode_file_value(old_val)? + let files: Vec<(&str, &[u8])> = if let Some(ref old_val) = old_val { + aux_file::decode_file_value(old_val)? + } else { + Vec::new() + }; + let mut other_files = Vec::with_capacity(files.len()); + let mut modifying_file = None; + for file @ (p, content) in files { + if path == p { + assert!( + modifying_file.is_none(), + "duplicated entries found for {}", + path + ); + modifying_file = Some(content); } else { - Vec::new() - }; - let mut other_files = Vec::with_capacity(files.len()); - let mut modifying_file = None; - for file @ (p, content) in files { - if path == p { - assert!( - modifying_file.is_none(), - "duplicated entries found for {}", - path - ); - modifying_file = Some(content); - } else { - other_files.push(file); - } - } - let mut new_files = other_files; - match (modifying_file, content.is_empty()) { - (Some(old_content), false) => { - self.tline - .aux_file_size_estimator - .on_update(old_content.len(), content.len()); - new_files.push((path, content)); - } - (Some(old_content), true) => { - self.tline - .aux_file_size_estimator - .on_remove(old_content.len()); - // not adding the file key to the final `new_files` vec. - } - (None, false) => { - self.tline.aux_file_size_estimator.on_add(content.len()); - new_files.push((path, content)); - } - (None, true) => warn!("removing non-existing aux file: {}", path), + other_files.push(file); } - let new_val = aux_file::encode_file_value(&new_files)?; - self.put(key, Value::Image(new_val.into())); } - - if let AuxFilePolicy::V1 | AuxFilePolicy::CrossValidation = policy { - let file_path = path.to_string(); - let content = if content.is_empty() { - None - } else { - Some(Bytes::copy_from_slice(content)) - }; - - let n_files; - let mut aux_files = self.tline.aux_files.lock().await; - if let Some(mut dir) = aux_files.dir.take() { - // We already updated aux files in `self`: emit a delta and update our latest value. - dir.upsert(file_path.clone(), content.clone()); - n_files = dir.files.len(); - if aux_files.n_deltas == MAX_AUX_FILE_DELTAS { - self.put( - AUX_FILES_KEY, - Value::Image(Bytes::from( - AuxFilesDirectory::ser(&dir).context("serialize")?, - )), - ); - aux_files.n_deltas = 0; - } else { - self.put( - AUX_FILES_KEY, - Value::WalRecord(NeonWalRecord::AuxFile { file_path, content }), - ); - aux_files.n_deltas += 1; - } - aux_files.dir = Some(dir); - } else { - // Check if the AUX_FILES_KEY is initialized - match self.get(AUX_FILES_KEY, ctx).await { - Ok(dir_bytes) => { - let mut dir = AuxFilesDirectory::des(&dir_bytes)?; - // Key is already set, we may append a delta - self.put( - AUX_FILES_KEY, - Value::WalRecord(NeonWalRecord::AuxFile { - file_path: file_path.clone(), - content: content.clone(), - }), - ); - dir.upsert(file_path, content); - n_files = dir.files.len(); - aux_files.dir = Some(dir); - } - Err( - e @ (PageReconstructError::Cancelled - | PageReconstructError::AncestorLsnTimeout(_)), - ) => { - // Important that we do not interpret a shutdown error as "not found" and thereby - // reset the map. - return Err(e.into()); - } - // Note: we added missing key error variant in https://github.com/neondatabase/neon/pull/7393 but - // the original code assumes all other errors are missing keys. Therefore, we keep the code path - // the same for now, though in theory, we should only match the `MissingKey` variant. - Err( - e @ (PageReconstructError::Other(_) - | PageReconstructError::WalRedo(_) - | PageReconstructError::MissingKey(_)), - ) => { - // Key is missing, we must insert an image as the basis for subsequent deltas. - - if !matches!(e, PageReconstructError::MissingKey(_)) { - let e = utils::error::report_compact_sources(&e); - tracing::warn!("treating error as if it was a missing key: {}", e); - } - - let mut dir = AuxFilesDirectory { - files: HashMap::new(), - }; - dir.upsert(file_path, content); - self.put( - AUX_FILES_KEY, - Value::Image(Bytes::from( - AuxFilesDirectory::ser(&dir).context("serialize")?, - )), - ); - n_files = 1; - aux_files.dir = Some(dir); - } - } + let mut new_files = other_files; + match (modifying_file, content.is_empty()) { + (Some(old_content), false) => { + self.tline + .aux_file_size_estimator + .on_update(old_content.len(), content.len()); + new_files.push((path, content)); } - - self.pending_directory_entries - .push((DirectoryKind::AuxFiles, n_files)); + (Some(old_content), true) => { + self.tline + .aux_file_size_estimator + .on_remove(old_content.len()); + // not adding the file key to the final `new_files` vec. + } + (None, false) => { + self.tline.aux_file_size_estimator.on_add(content.len()); + new_files.push((path, content)); + } + (None, true) => warn!("removing non-existing aux file: {}", path), } + let new_val = aux_file::encode_file_value(&new_files)?; + self.put(key, Value::Image(new_val.into())); Ok(()) } @@ -2089,12 +1877,6 @@ impl<'a> DatadirModification<'a> { self.tline.get(key, lsn, ctx).await } - /// Only used during unit tests, force putting a key into the modification. - #[cfg(test)] - pub(crate) fn put_for_test(&mut self, key: Key, val: Value) { - self.put(key, val); - } - fn put(&mut self, key: Key, val: Value) { if Self::is_data_key(&key) { self.put_data(key.to_compact(), val) @@ -2212,21 +1994,6 @@ struct RelDirectory { rels: HashSet<(Oid, u8)>, } -#[derive(Debug, Serialize, Deserialize, Default, PartialEq)] -pub(crate) struct AuxFilesDirectory { - pub(crate) files: HashMap, -} - -impl AuxFilesDirectory { - pub(crate) fn upsert(&mut self, key: String, value: Option) { - if let Some(value) = value { - self.files.insert(key, value); - } else { - self.files.remove(&key); - } - } -} - #[derive(Debug, Serialize, Deserialize)] struct RelSizeEntry { nblocks: u32, diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index baa236565810..1066d165cd38 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -20,7 +20,6 @@ use enumset::EnumSet; use futures::stream::FuturesUnordered; use futures::StreamExt; use pageserver_api::models; -use pageserver_api::models::AuxFilePolicy; use pageserver_api::models::LsnLease; use pageserver_api::models::TimelineArchivalState; use pageserver_api::models::TimelineState; @@ -800,7 +799,6 @@ impl Tenant { index_part: Option, metadata: TimelineMetadata, ancestor: Option>, - last_aux_file_policy: Option, _ctx: &RequestContext, ) -> anyhow::Result<()> { let tenant_id = self.tenant_shard_id; @@ -811,10 +809,6 @@ impl Tenant { ancestor.clone(), resources, CreateTimelineCause::Load, - // This could be derived from ancestor branch + index part. Though the only caller of `timeline_init_and_sync` is `load_remote_timeline`, - // there will potentially be other caller of this function in the future, and we don't know whether `index_part` or `ancestor` takes precedence. - // Therefore, we pass this field explicitly for now, and remove it once we fully migrate to aux file v2. - last_aux_file_policy, )?; let disk_consistent_lsn = timeline.get_disk_consistent_lsn(); anyhow::ensure!( @@ -829,10 +823,6 @@ impl Tenant { if let Some(index_part) = index_part.as_ref() { timeline.remote_client.init_upload_queue(index_part)?; - - timeline - .last_aux_file_policy - .store(index_part.last_aux_file_policy()); } else { // No data on the remote storage, but we have local metadata file. We can end up // here with timeline_create being interrupted before finishing index part upload. @@ -1403,15 +1393,12 @@ impl Tenant { None }; - let last_aux_file_policy = index_part.last_aux_file_policy(); - self.timeline_init_and_sync( timeline_id, resources, Some(index_part), remote_metadata, ancestor, - last_aux_file_policy, ctx, ) .await @@ -1824,7 +1811,6 @@ impl Tenant { create_guard, initdb_lsn, None, - None, ) .await } @@ -3032,7 +3018,6 @@ impl Tenant { ancestor: Option>, resources: TimelineResources, cause: CreateTimelineCause, - last_aux_file_policy: Option, ) -> anyhow::Result> { let state = match cause { CreateTimelineCause::Load => { @@ -3061,7 +3046,6 @@ impl Tenant { resources, pg_version, state, - last_aux_file_policy, self.attach_wal_lag_cooldown.clone(), self.cancel.child_token(), ); @@ -3720,7 +3704,6 @@ impl Tenant { timeline_create_guard, start_lsn + 1, Some(Arc::clone(src_timeline)), - src_timeline.last_aux_file_policy.load(), ) .await?; @@ -3914,7 +3897,6 @@ impl Tenant { timeline_create_guard, pgdata_lsn, None, - None, ) .await?; @@ -3986,7 +3968,6 @@ impl Tenant { create_guard: TimelineCreateGuard<'a>, start_lsn: Lsn, ancestor: Option>, - last_aux_file_policy: Option, ) -> anyhow::Result> { let tenant_shard_id = self.tenant_shard_id; @@ -4002,7 +3983,6 @@ impl Tenant { ancestor, resources, CreateTimelineCause::Load, - last_aux_file_policy, ) .context("Failed to create timeline data structure")?; @@ -4600,7 +4580,6 @@ mod tests { use super::*; use crate::keyspace::KeySpaceAccum; - use crate::pgdatadir_mapping::AuxFilesDirectory; use crate::repository::{Key, Value}; use crate::tenant::harness::*; use crate::tenant::timeline::CompactFlags; @@ -4609,7 +4588,7 @@ mod tests { use bytes::{Bytes, BytesMut}; use hex_literal::hex; use itertools::Itertools; - use pageserver_api::key::{AUX_FILES_KEY, AUX_KEY_PREFIX, NON_INHERITED_RANGE}; + use pageserver_api::key::{AUX_KEY_PREFIX, NON_INHERITED_RANGE}; use pageserver_api::keyspace::KeySpace; use pageserver_api::models::{CompactionAlgorithm, CompactionAlgorithmSettings}; use rand::{thread_rng, Rng}; @@ -4618,7 +4597,6 @@ mod tests { use tests::timeline::{GetVectoredError, ShutdownMode}; use timeline::compaction::{KeyHistoryRetention, KeyLogAtLsn}; use timeline::{DeltaLayerTestDesc, GcInfo}; - use utils::bin_ser::BeSer; use utils::id::TenantId; static TEST_KEY: Lazy = @@ -6422,16 +6400,9 @@ mod tests { } #[tokio::test] - async fn test_branch_copies_dirty_aux_file_flag() { - let harness = TenantHarness::create("test_branch_copies_dirty_aux_file_flag") - .await - .unwrap(); + async fn test_aux_file_e2e() { + let harness = TenantHarness::create("test_aux_file_e2e").await.unwrap(); - // the default aux file policy to switch is v2 if not set by the admins - assert_eq!( - harness.tenant_conf.switch_aux_file_policy, - AuxFilePolicy::default_tenant_config() - ); let (tenant, ctx) = harness.load().await; let mut lsn = Lsn(0x08); @@ -6441,9 +6412,6 @@ mod tests { .await .unwrap(); - // no aux file is written at this point, so the persistent flag should be unset - assert_eq!(tline.last_aux_file_policy.load(), None); - { lsn += 8; let mut modification = tline.begin_modification(lsn); @@ -6454,30 +6422,6 @@ mod tests { modification.commit(&ctx).await.unwrap(); } - // there is no tenant manager to pass the configuration through, so lets mimic it - tenant.set_new_location_config( - AttachedTenantConf::try_from(LocationConf::attached_single( - TenantConfOpt { - switch_aux_file_policy: Some(AuxFilePolicy::V2), - ..Default::default() - }, - tenant.generation, - &pageserver_api::models::ShardParameters::default(), - )) - .unwrap(), - ); - - assert_eq!( - tline.get_switch_aux_file_policy(), - AuxFilePolicy::V2, - "wanted state has been updated" - ); - assert_eq!( - tline.last_aux_file_policy.load(), - Some(AuxFilePolicy::V2), - "aux file is written with switch_aux_file_policy unset (which is v2), so we should use v2 there" - ); - // we can read everything from the storage let files = tline.list_aux_files(lsn, &ctx).await.unwrap(); assert_eq!( @@ -6495,12 +6439,6 @@ mod tests { modification.commit(&ctx).await.unwrap(); } - assert_eq!( - tline.last_aux_file_policy.load(), - Some(AuxFilePolicy::V2), - "keep v2 storage format when new files are written" - ); - let files = tline.list_aux_files(lsn, &ctx).await.unwrap(); assert_eq!( files.get("pg_logical/mappings/test2"), @@ -6512,321 +6450,9 @@ mod tests { .await .unwrap(); - // child copies the last flag even if that is not on remote storage yet - assert_eq!(child.get_switch_aux_file_policy(), AuxFilePolicy::V2); - assert_eq!(child.last_aux_file_policy.load(), Some(AuxFilePolicy::V2)); - let files = child.list_aux_files(lsn, &ctx).await.unwrap(); assert_eq!(files.get("pg_logical/mappings/test1"), None); assert_eq!(files.get("pg_logical/mappings/test2"), None); - - // even if we crash here without flushing parent timeline with it's new - // last_aux_file_policy we are safe, because child was never meant to access ancestor's - // files. the ancestor can even switch back to V1 because of a migration safely. - } - - #[tokio::test] - async fn aux_file_policy_switch() { - let mut harness = TenantHarness::create("aux_file_policy_switch") - .await - .unwrap(); - harness.tenant_conf.switch_aux_file_policy = AuxFilePolicy::CrossValidation; // set to cross-validation mode - let (tenant, ctx) = harness.load().await; - - let mut lsn = Lsn(0x08); - - let tline: Arc = tenant - .create_test_timeline(TIMELINE_ID, lsn, DEFAULT_PG_VERSION, &ctx) - .await - .unwrap(); - - assert_eq!( - tline.last_aux_file_policy.load(), - None, - "no aux file is written so it should be unset" - ); - - { - lsn += 8; - let mut modification = tline.begin_modification(lsn); - modification - .put_file("pg_logical/mappings/test1", b"first", &ctx) - .await - .unwrap(); - modification.commit(&ctx).await.unwrap(); - } - - // there is no tenant manager to pass the configuration through, so lets mimic it - tenant.set_new_location_config( - AttachedTenantConf::try_from(LocationConf::attached_single( - TenantConfOpt { - switch_aux_file_policy: Some(AuxFilePolicy::V2), - ..Default::default() - }, - tenant.generation, - &pageserver_api::models::ShardParameters::default(), - )) - .unwrap(), - ); - - assert_eq!( - tline.get_switch_aux_file_policy(), - AuxFilePolicy::V2, - "wanted state has been updated" - ); - assert_eq!( - tline.last_aux_file_policy.load(), - Some(AuxFilePolicy::CrossValidation), - "dirty index_part.json reflected state is yet to be updated" - ); - - // we can still read the auxfile v1 before we ingest anything new - let files = tline.list_aux_files(lsn, &ctx).await.unwrap(); - assert_eq!( - files.get("pg_logical/mappings/test1"), - Some(&bytes::Bytes::from_static(b"first")) - ); - - { - lsn += 8; - let mut modification = tline.begin_modification(lsn); - modification - .put_file("pg_logical/mappings/test2", b"second", &ctx) - .await - .unwrap(); - modification.commit(&ctx).await.unwrap(); - } - - assert_eq!( - tline.last_aux_file_policy.load(), - Some(AuxFilePolicy::V2), - "ingesting a file should apply the wanted switch state when applicable" - ); - - let files = tline.list_aux_files(lsn, &ctx).await.unwrap(); - assert_eq!( - files.get("pg_logical/mappings/test1"), - Some(&bytes::Bytes::from_static(b"first")), - "cross validation writes to both v1 and v2 so this should be available in v2" - ); - assert_eq!( - files.get("pg_logical/mappings/test2"), - Some(&bytes::Bytes::from_static(b"second")) - ); - - // mimic again by trying to flip it from V2 to V1 (not switched to while ingesting a file) - tenant.set_new_location_config( - AttachedTenantConf::try_from(LocationConf::attached_single( - TenantConfOpt { - switch_aux_file_policy: Some(AuxFilePolicy::V1), - ..Default::default() - }, - tenant.generation, - &pageserver_api::models::ShardParameters::default(), - )) - .unwrap(), - ); - - { - lsn += 8; - let mut modification = tline.begin_modification(lsn); - modification - .put_file("pg_logical/mappings/test2", b"third", &ctx) - .await - .unwrap(); - modification.commit(&ctx).await.unwrap(); - } - - assert_eq!( - tline.get_switch_aux_file_policy(), - AuxFilePolicy::V1, - "wanted state has been updated again, even if invalid request" - ); - - assert_eq!( - tline.last_aux_file_policy.load(), - Some(AuxFilePolicy::V2), - "ingesting a file should apply the wanted switch state when applicable" - ); - - let files = tline.list_aux_files(lsn, &ctx).await.unwrap(); - assert_eq!( - files.get("pg_logical/mappings/test1"), - Some(&bytes::Bytes::from_static(b"first")) - ); - assert_eq!( - files.get("pg_logical/mappings/test2"), - Some(&bytes::Bytes::from_static(b"third")) - ); - - // mimic again by trying to flip it from from V1 to V2 (not switched to while ingesting a file) - tenant.set_new_location_config( - AttachedTenantConf::try_from(LocationConf::attached_single( - TenantConfOpt { - switch_aux_file_policy: Some(AuxFilePolicy::V2), - ..Default::default() - }, - tenant.generation, - &pageserver_api::models::ShardParameters::default(), - )) - .unwrap(), - ); - - { - lsn += 8; - let mut modification = tline.begin_modification(lsn); - modification - .put_file("pg_logical/mappings/test3", b"last", &ctx) - .await - .unwrap(); - modification.commit(&ctx).await.unwrap(); - } - - assert_eq!(tline.get_switch_aux_file_policy(), AuxFilePolicy::V2); - - assert_eq!(tline.last_aux_file_policy.load(), Some(AuxFilePolicy::V2)); - - let files = tline.list_aux_files(lsn, &ctx).await.unwrap(); - assert_eq!( - files.get("pg_logical/mappings/test1"), - Some(&bytes::Bytes::from_static(b"first")) - ); - assert_eq!( - files.get("pg_logical/mappings/test2"), - Some(&bytes::Bytes::from_static(b"third")) - ); - assert_eq!( - files.get("pg_logical/mappings/test3"), - Some(&bytes::Bytes::from_static(b"last")) - ); - } - - #[tokio::test] - async fn aux_file_policy_force_switch() { - let mut harness = TenantHarness::create("aux_file_policy_force_switch") - .await - .unwrap(); - harness.tenant_conf.switch_aux_file_policy = AuxFilePolicy::V1; - let (tenant, ctx) = harness.load().await; - - let mut lsn = Lsn(0x08); - - let tline: Arc = tenant - .create_test_timeline(TIMELINE_ID, lsn, DEFAULT_PG_VERSION, &ctx) - .await - .unwrap(); - - assert_eq!( - tline.last_aux_file_policy.load(), - None, - "no aux file is written so it should be unset" - ); - - { - lsn += 8; - let mut modification = tline.begin_modification(lsn); - modification - .put_file("pg_logical/mappings/test1", b"first", &ctx) - .await - .unwrap(); - modification.commit(&ctx).await.unwrap(); - } - - tline.do_switch_aux_policy(AuxFilePolicy::V2).unwrap(); - - assert_eq!( - tline.last_aux_file_policy.load(), - Some(AuxFilePolicy::V2), - "dirty index_part.json reflected state is yet to be updated" - ); - - // lose all data from v1 - let files = tline.list_aux_files(lsn, &ctx).await.unwrap(); - assert_eq!(files.get("pg_logical/mappings/test1"), None); - - { - lsn += 8; - let mut modification = tline.begin_modification(lsn); - modification - .put_file("pg_logical/mappings/test2", b"second", &ctx) - .await - .unwrap(); - modification.commit(&ctx).await.unwrap(); - } - - // read data ingested in v2 - let files = tline.list_aux_files(lsn, &ctx).await.unwrap(); - assert_eq!( - files.get("pg_logical/mappings/test2"), - Some(&bytes::Bytes::from_static(b"second")) - ); - // lose all data from v1 - assert_eq!(files.get("pg_logical/mappings/test1"), None); - } - - #[tokio::test] - async fn aux_file_policy_auto_detect() { - let mut harness = TenantHarness::create("aux_file_policy_auto_detect") - .await - .unwrap(); - harness.tenant_conf.switch_aux_file_policy = AuxFilePolicy::V2; // set to cross-validation mode - let (tenant, ctx) = harness.load().await; - - let mut lsn = Lsn(0x08); - - let tline: Arc = tenant - .create_test_timeline(TIMELINE_ID, lsn, DEFAULT_PG_VERSION, &ctx) - .await - .unwrap(); - - assert_eq!( - tline.last_aux_file_policy.load(), - None, - "no aux file is written so it should be unset" - ); - - { - lsn += 8; - let mut modification = tline.begin_modification(lsn); - let buf = AuxFilesDirectory::ser(&AuxFilesDirectory { - files: vec![( - "test_file".to_string(), - Bytes::copy_from_slice(b"test_file"), - )] - .into_iter() - .collect(), - }) - .unwrap(); - modification.put_for_test(AUX_FILES_KEY, Value::Image(Bytes::from(buf))); - modification.commit(&ctx).await.unwrap(); - } - - { - lsn += 8; - let mut modification = tline.begin_modification(lsn); - modification - .put_file("pg_logical/mappings/test1", b"first", &ctx) - .await - .unwrap(); - modification.commit(&ctx).await.unwrap(); - } - - assert_eq!( - tline.last_aux_file_policy.load(), - Some(AuxFilePolicy::V1), - "keep using v1 because there are aux files writting with v1" - ); - - // we can still read the auxfile v1 - let files = tline.list_aux_files(lsn, &ctx).await.unwrap(); - assert_eq!( - files.get("pg_logical/mappings/test1"), - Some(&bytes::Bytes::from_static(b"first")) - ); - assert_eq!( - files.get("test_file"), - Some(&bytes::Bytes::from_static(b"test_file")) - ); } #[tokio::test] diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 1f9ae40af534..5e9702bd3dba 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -187,7 +187,7 @@ use camino::Utf8Path; use chrono::{NaiveDateTime, Utc}; pub(crate) use download::download_initdb_tar_zst; -use pageserver_api::models::{AuxFilePolicy, TimelineArchivalState}; +use pageserver_api::models::TimelineArchivalState; use pageserver_api::shard::{ShardIndex, TenantShardId}; use scopeguard::ScopeGuard; use tokio_util::sync::CancellationToken; @@ -628,18 +628,6 @@ impl RemoteTimelineClient { Ok(()) } - /// Launch an index-file upload operation in the background, with only the `aux_file_policy` flag updated. - pub(crate) fn schedule_index_upload_for_aux_file_policy_update( - self: &Arc, - last_aux_file_policy: Option, - ) -> anyhow::Result<()> { - let mut guard = self.upload_queue.lock().unwrap(); - let upload_queue = guard.initialized_mut()?; - upload_queue.dirty.last_aux_file_policy = last_aux_file_policy; - self.schedule_index_upload(upload_queue)?; - Ok(()) - } - /// Launch an index-file upload operation in the background, with only the `archived_at` field updated. /// /// Returns whether it is required to wait for the queue to be empty to ensure that the change is uploaded, diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index c51ff549195a..3a74a4ed1186 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -133,10 +133,6 @@ impl IndexPart { pub(crate) fn example() -> Self { Self::empty(TimelineMetadata::example()) } - - pub(crate) fn last_aux_file_policy(&self) -> Option { - self.last_aux_file_policy - } } /// Metadata gathered for each of the layer files. diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 2b4f949c76da..d67a139dfa25 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -28,9 +28,9 @@ use pageserver_api::{ }, keyspace::{KeySpaceAccum, KeySpaceRandomAccum, SparseKeyPartitioning}, models::{ - AtomicAuxFilePolicy, AuxFilePolicy, CompactionAlgorithm, CompactionAlgorithmSettings, - DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskSpawnRequest, EvictionPolicy, - InMemoryLayerInfo, LayerMapInfo, LsnLease, TimelineState, + CompactionAlgorithm, CompactionAlgorithmSettings, DownloadRemoteLayersTaskInfo, + DownloadRemoteLayersTaskSpawnRequest, EvictionPolicy, InMemoryLayerInfo, LayerMapInfo, + LsnLease, TimelineState, }, reltag::BlockNumber, shard::{ShardIdentity, ShardNumber, TenantShardId}, @@ -98,12 +98,12 @@ use crate::{ use crate::{ metrics::ScanLatencyOngoingRecording, tenant::timeline::logical_size::CurrentLogicalSize, }; -use crate::{pgdatadir_mapping::LsnForTimestamp, tenant::tasks::BackgroundLoopKind}; -use crate::{pgdatadir_mapping::MAX_AUX_FILE_V2_DELTAS, tenant::storage_layer::PersistentLayerKey}; use crate::{ - pgdatadir_mapping::{AuxFilesDirectory, DirectoryKind}, + pgdatadir_mapping::DirectoryKind, virtual_file::{MaybeFatalIo, VirtualFile}, }; +use crate::{pgdatadir_mapping::LsnForTimestamp, tenant::tasks::BackgroundLoopKind}; +use crate::{pgdatadir_mapping::MAX_AUX_FILE_V2_DELTAS, tenant::storage_layer::PersistentLayerKey}; use pageserver_api::config::tenant_conf_defaults::DEFAULT_PITR_INTERVAL; use crate::config::PageServerConf; @@ -206,11 +206,6 @@ pub struct TimelineResources { pub l0_flush_global_state: l0_flush::L0FlushGlobalState, } -pub(crate) struct AuxFilesState { - pub(crate) dir: Option, - pub(crate) n_deltas: usize, -} - /// The relation size cache caches relation sizes at the end of the timeline. It speeds up WAL /// ingestion considerably, because WAL ingestion needs to check on most records if the record /// implicitly extends the relation. At startup, `complete_as_of` is initialized to the current end @@ -413,15 +408,9 @@ pub struct Timeline { timeline_get_throttle: Arc>, - /// Keep aux directory cache to avoid it's reconstruction on each update - pub(crate) aux_files: tokio::sync::Mutex, - /// Size estimator for aux file v2 pub(crate) aux_file_size_estimator: AuxFileSizeEstimator, - /// Indicate whether aux file v2 storage is enabled. - pub(crate) last_aux_file_policy: AtomicAuxFilePolicy, - /// Some test cases directly place keys into the timeline without actually modifying the directory /// keys (i.e., DB_DIR). The test cases creating such keys will put the keyspaces here, so that /// these keys won't get garbage-collected during compaction/GC. This field only modifies the dense @@ -2012,14 +2001,6 @@ impl Timeline { .unwrap_or(self.conf.default_tenant_conf.lsn_lease_length_for_ts) } - pub(crate) fn get_switch_aux_file_policy(&self) -> AuxFilePolicy { - let tenant_conf = self.tenant_conf.load(); - tenant_conf - .tenant_conf - .switch_aux_file_policy - .unwrap_or(self.conf.default_tenant_conf.switch_aux_file_policy) - } - pub(crate) fn get_lazy_slru_download(&self) -> bool { let tenant_conf = self.tenant_conf.load(); tenant_conf @@ -2152,7 +2133,6 @@ impl Timeline { resources: TimelineResources, pg_version: u32, state: TimelineState, - aux_file_policy: Option, attach_wal_lag_cooldown: Arc>, cancel: CancellationToken, ) -> Arc { @@ -2282,15 +2262,8 @@ impl Timeline { timeline_get_throttle: resources.timeline_get_throttle, - aux_files: tokio::sync::Mutex::new(AuxFilesState { - dir: None, - n_deltas: 0, - }), - aux_file_size_estimator: AuxFileSizeEstimator::new(aux_file_metrics), - last_aux_file_policy: AtomicAuxFilePolicy::new(aux_file_policy), - #[cfg(test)] extra_test_dense_keyspace: ArcSwap::new(Arc::new(KeySpace::default())), @@ -2301,10 +2274,6 @@ impl Timeline { attach_wal_lag_cooldown, }; - if aux_file_policy == Some(AuxFilePolicy::V1) { - warn!("this timeline is using deprecated aux file policy V1 (when loading the timeline)"); - } - result.repartition_threshold = result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE; @@ -4479,14 +4448,6 @@ impl Timeline { ) -> Result<(), detach_ancestor::Error> { detach_ancestor::complete(self, tenant, attempt, ctx).await } - - /// Switch aux file policy and schedule upload to the index part. - pub(crate) fn do_switch_aux_policy(&self, policy: AuxFilePolicy) -> anyhow::Result<()> { - self.last_aux_file_policy.store(Some(policy)); - self.remote_client - .schedule_index_upload_for_aux_file_policy_update(Some(policy))?; - Ok(()) - } } impl Drop for Timeline { diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 305c5758ccf6..71b9e4e288e5 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -283,8 +283,6 @@ impl DeleteTimelineFlow { // Important. We dont pass ancestor above because it can be missing. // Thus we need to skip the validation here. CreateTimelineCause::Delete, - // Aux file policy is not needed for deletion, assuming deletion does not read aux keyspace - None, ) .context("create_timeline_struct")?; diff --git a/pageserver/src/walredo/apply_neon.rs b/pageserver/src/walredo/apply_neon.rs index facf01004c81..c067787f9764 100644 --- a/pageserver/src/walredo/apply_neon.rs +++ b/pageserver/src/walredo/apply_neon.rs @@ -1,8 +1,7 @@ -use crate::pgdatadir_mapping::AuxFilesDirectory; use crate::walrecord::NeonWalRecord; use anyhow::Context; use byteorder::{ByteOrder, LittleEndian}; -use bytes::{BufMut, BytesMut}; +use bytes::BytesMut; use pageserver_api::key::Key; use pageserver_api::reltag::SlruKind; use postgres_ffi::pg_constants; @@ -13,7 +12,6 @@ use postgres_ffi::v14::nonrelfile_utils::{ }; use postgres_ffi::BLCKSZ; use tracing::*; -use utils::bin_ser::BeSer; use utils::lsn::Lsn; /// Can this request be served by neon redo functions @@ -236,13 +234,9 @@ pub(crate) fn apply_in_neon( LittleEndian::write_u32(&mut page[memberoff..memberoff + 4], member.xid); } } - NeonWalRecord::AuxFile { file_path, content } => { - let mut dir = AuxFilesDirectory::des(page)?; - dir.upsert(file_path.clone(), content.clone()); - - page.clear(); - let mut writer = page.writer(); - dir.ser_into(&mut writer)?; + NeonWalRecord::AuxFile { .. } => { + // No-op: this record will never be created in aux v2. + warn!("AuxFile record should not be created in aux v2"); } #[cfg(test)] NeonWalRecord::Test { @@ -250,6 +244,7 @@ pub(crate) fn apply_in_neon( clear, will_init, } => { + use bytes::BufMut; if *will_init { assert!(*clear, "init record must be clear to ensure correctness"); } @@ -261,59 +256,3 @@ pub(crate) fn apply_in_neon( } Ok(()) } - -#[cfg(test)] -mod test { - use bytes::Bytes; - use pageserver_api::key::AUX_FILES_KEY; - - use super::*; - use std::collections::HashMap; - - /// Test [`apply_in_neon`]'s handling of NeonWalRecord::AuxFile - #[test] - fn apply_aux_file_deltas() -> anyhow::Result<()> { - let base_dir = AuxFilesDirectory { - files: HashMap::from([ - ("two".to_string(), Bytes::from_static(b"content0")), - ("three".to_string(), Bytes::from_static(b"contentX")), - ]), - }; - let base_image = AuxFilesDirectory::ser(&base_dir)?; - - let deltas = vec![ - // Insert - NeonWalRecord::AuxFile { - file_path: "one".to_string(), - content: Some(Bytes::from_static(b"content1")), - }, - // Update - NeonWalRecord::AuxFile { - file_path: "two".to_string(), - content: Some(Bytes::from_static(b"content99")), - }, - // Delete - NeonWalRecord::AuxFile { - file_path: "three".to_string(), - content: None, - }, - ]; - - let file_path = AUX_FILES_KEY; - let mut page = BytesMut::from_iter(base_image); - - for record in deltas { - apply_in_neon(&record, Lsn(8), file_path, &mut page)?; - } - - let reconstructed = AuxFilesDirectory::des(&page)?; - let expect = HashMap::from([ - ("one".to_string(), Bytes::from_static(b"content1")), - ("two".to_string(), Bytes::from_static(b"content99")), - ]); - - assert_eq!(reconstructed.files, expect); - - Ok(()) - } -} diff --git a/test_runner/regress/test_aux_files.py b/test_runner/regress/test_aux_files.py deleted file mode 100644 index 91d674d0db4d..000000000000 --- a/test_runner/regress/test_aux_files.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -from fixtures.log_helper import log -from fixtures.neon_fixtures import ( - AuxFileStore, - NeonEnvBuilder, - logical_replication_sync, -) - - -def test_aux_v2_config_switch(neon_env_builder: NeonEnvBuilder, vanilla_pg): - env = neon_env_builder.init_start() - endpoint = env.endpoints.create_start("main") - client = env.pageserver.http_client() - - tenant_id = env.initial_tenant - timeline_id = env.initial_timeline - - tenant_config = client.tenant_config(tenant_id).effective_config - tenant_config["switch_aux_file_policy"] = AuxFileStore.V2 - client.set_tenant_config(tenant_id, tenant_config) - # aux file v2 is enabled on the write path, so for now, it should be unset (or null) - assert ( - client.timeline_detail(tenant_id=tenant_id, timeline_id=timeline_id)["last_aux_file_policy"] - is None - ) - - pg_conn = endpoint.connect() - cur = pg_conn.cursor() - - cur.execute("create table t(pk integer primary key, payload integer)") - cur.execute( - "CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120));" - ) - cur.execute("create publication pub1 for table t, replication_example") - - # now start subscriber, aux files will be created at this point. TODO: find better ways of testing aux files (i.e., neon_test_utils) - # instead of going through the full logical replication process. - vanilla_pg.start() - vanilla_pg.safe_psql("create table t(pk integer primary key, payload integer)") - vanilla_pg.safe_psql( - "CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120), testcolumn1 int, testcolumn2 int, testcolumn3 int);" - ) - connstr = endpoint.connstr().replace("'", "''") - log.info(f"ep connstr is {endpoint.connstr()}, subscriber connstr {vanilla_pg.connstr()}") - vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") - - # Wait logical replication channel to be established - logical_replication_sync(vanilla_pg, endpoint) - vanilla_pg.stop() - endpoint.stop() - - with env.pageserver.http_client() as client: - # aux file v2 flag should be enabled at this point - assert ( - client.timeline_detail(tenant_id, timeline_id)["last_aux_file_policy"] - == AuxFileStore.V2 - ) - with env.pageserver.http_client() as client: - tenant_config = client.tenant_config(tenant_id).effective_config - tenant_config["switch_aux_file_policy"] = "V1" - client.set_tenant_config(tenant_id, tenant_config) - # the flag should still be enabled - assert ( - client.timeline_detail(tenant_id=tenant_id, timeline_id=timeline_id)[ - "last_aux_file_policy" - ] - == AuxFileStore.V2 - ) - env.pageserver.restart() - with env.pageserver.http_client() as client: - # aux file v2 flag should be persisted - assert ( - client.timeline_detail(tenant_id=tenant_id, timeline_id=timeline_id)[ - "last_aux_file_policy" - ] - == AuxFileStore.V2 - ) From 24398bf0600223fb74fb3aa33ca4e4374209f84d Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 17 Oct 2024 19:02:24 +0100 Subject: [PATCH 06/21] pageserver: detect & warn on loading an old index which is probably the result of a bad generation (#9383) ## Problem The pageserver generally trusts the storage controller/control plane to give it valid generations. However, sometimes it should be obvious that a generation is bad, and for defense in depth we should detect that on the pageserver. This PR is part 1 of 2: 1. in this PR we detect and warn on such situations, but do not block starting up the tenant. Once we have confidence that the check is not firing unexpectedly in the field 2. part 2 of 2 will introduce a condition that refuses to start a tenant in this situtation, and a test for that (maybe, if we can figure out how to spoof an ancient mtime) Related: #6951 ## Summary of changes - When loading an index older than 2 weeks, log an INFO message noting that we will check for other indices - When loading an index older than 2 weeks _and_ a newer-generation index exists, log a warning. --- pageserver/src/http/routes.rs | 2 +- .../src/tenant/remote_timeline_client.rs | 45 ++++++++++++++++++- .../tenant/remote_timeline_client/download.rs | 11 ++--- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index e6663ef56fa4..8f928fd81bba 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -2251,7 +2251,7 @@ async fn tenant_scan_remote_handler( %timeline_id)) .await { - Ok((index_part, index_generation)) => { + Ok((index_part, index_generation, _index_mtime)) => { tracing::info!("Found timeline {tenant_shard_id}/{timeline_id} metadata (gen {index_generation:?}, {} layers, {} consistent LSN)", index_part.layer_metadata.len(), index_part.metadata.disk_consistent_lsn()); generation = std::cmp::max(generation, index_generation); diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 5e9702bd3dba..450084aca249 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -505,7 +505,7 @@ impl RemoteTimelineClient { }, ); - let (index_part, _index_generation) = download::download_index_part( + let (index_part, index_generation, index_last_modified) = download::download_index_part( &self.storage_impl, &self.tenant_shard_id, &self.timeline_id, @@ -519,6 +519,49 @@ impl RemoteTimelineClient { ) .await?; + // Defense in depth: monotonicity of generation numbers is an important correctness guarantee, so when we see a very + // old index, we do extra checks in case this is the result of backward time-travel of the generation number (e.g. + // in case of a bug in the service that issues generation numbers). Indices are allowed to be old, but we expect that + // when we load an old index we are loading the _latest_ index: if we are asked to load an old index and there is + // also a newer index available, that is surprising. + const INDEX_AGE_CHECKS_THRESHOLD: Duration = Duration::from_secs(14 * 24 * 3600); + let index_age = index_last_modified.elapsed().unwrap_or_else(|e| { + if e.duration() > Duration::from_secs(5) { + // We only warn if the S3 clock and our local clock are >5s out: because this is a low resolution + // timestamp, it is common to be out by at least 1 second. + tracing::warn!("Index has modification time in the future: {e}"); + } + Duration::ZERO + }); + if index_age > INDEX_AGE_CHECKS_THRESHOLD { + tracing::info!( + ?index_generation, + age = index_age.as_secs_f64(), + "Loaded an old index, checking for other indices..." + ); + + // Find the highest-generation index + let (_latest_index_part, latest_index_generation, latest_index_mtime) = + download::download_index_part( + &self.storage_impl, + &self.tenant_shard_id, + &self.timeline_id, + Generation::MAX, + cancel, + ) + .await?; + + if latest_index_generation > index_generation { + // Unexpected! Why are we loading such an old index if a more recent one exists? + tracing::warn!( + ?index_generation, + ?latest_index_generation, + ?latest_index_mtime, + "Found a newer index while loading an old one" + ); + } + } + if index_part.deleted_at.is_some() { Ok(MaybeDeletedIndexPart::Deleted(index_part)) } else { diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index 692e4d3096de..b5d4b0f0bb7d 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -6,6 +6,7 @@ use std::collections::HashSet; use std::future::Future; use std::str::FromStr; +use std::time::SystemTime; use anyhow::{anyhow, Context}; use camino::{Utf8Path, Utf8PathBuf}; @@ -343,10 +344,10 @@ async fn do_download_index_part( timeline_id: &TimelineId, index_generation: Generation, cancel: &CancellationToken, -) -> Result<(IndexPart, Generation), DownloadError> { +) -> Result<(IndexPart, Generation, SystemTime), DownloadError> { let remote_path = remote_index_path(tenant_shard_id, timeline_id, index_generation); - let index_part_bytes = download_retry_forever( + let (index_part_bytes, index_part_mtime) = download_retry_forever( || async { let download = storage .download(&remote_path, &DownloadOpts::default(), cancel) @@ -359,7 +360,7 @@ async fn do_download_index_part( tokio::io::copy_buf(&mut stream, &mut bytes).await?; - Ok(bytes) + Ok((bytes, download.last_modified)) }, &format!("download {remote_path:?}"), cancel, @@ -370,7 +371,7 @@ async fn do_download_index_part( .with_context(|| format!("deserialize index part file at {remote_path:?}")) .map_err(DownloadError::Other)?; - Ok((index_part, index_generation)) + Ok((index_part, index_generation, index_part_mtime)) } /// index_part.json objects are suffixed with a generation number, so we cannot @@ -385,7 +386,7 @@ pub(crate) async fn download_index_part( timeline_id: &TimelineId, my_generation: Generation, cancel: &CancellationToken, -) -> Result<(IndexPart, Generation), DownloadError> { +) -> Result<(IndexPart, Generation, SystemTime), DownloadError> { debug_assert_current_span_has_tenant_and_timeline_id(); if my_generation.is_none() { From 928d98b6dcb57ae22a3da18fc6786b90c8dcae0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 17 Oct 2024 21:25:51 +0200 Subject: [PATCH 07/21] Update Rust to 1.82.0 and mold to 2.34.0 (#9445) We keep the practice of keeping the compiler up to date, pointing to the latest release. This is done by many other projects in the Rust ecosystem as well. [Release notes](https://github.com/rust-lang/rust/blob/master/RELEASES.md#version-1820-2024-10-17). Also update mold. [release notes for 2.34.0](https://github.com/rui314/mold/releases/tag/v2.34.0), [release notes for 2.34.1](https://github.com/rui314/mold/releases/tag/v2.34.1). Prior update was in #8939. --- Dockerfile.build-tools | 6 +++--- rust-toolchain.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.build-tools b/Dockerfile.build-tools index 7cba1c863599..f05c60661c4c 100644 --- a/Dockerfile.build-tools +++ b/Dockerfile.build-tools @@ -72,7 +72,7 @@ RUN curl -sL "https://github.com/peak/s5cmd/releases/download/v${S5CMD_VERSION}/ && mv s5cmd /usr/local/bin/s5cmd # LLVM -ENV LLVM_VERSION=18 +ENV LLVM_VERSION=19 RUN curl -fsSL 'https://apt.llvm.org/llvm-snapshot.gpg.key' | apt-key add - \ && echo "deb http://apt.llvm.org/${DEBIAN_VERSION}/ llvm-toolchain-${DEBIAN_VERSION}-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.stable.list \ && apt update \ @@ -99,7 +99,7 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "aws && rm awscliv2.zip # Mold: A Modern Linker -ENV MOLD_VERSION=v2.33.0 +ENV MOLD_VERSION=v2.34.1 RUN set -e \ && git clone https://github.com/rui314/mold.git \ && mkdir mold/build \ @@ -192,7 +192,7 @@ WORKDIR /home/nonroot # Rust # Please keep the version of llvm (installed above) in sync with rust llvm (`rustc --version --verbose | grep LLVM`) -ENV RUSTC_VERSION=1.81.0 +ENV RUSTC_VERSION=1.82.0 ENV RUSTUP_HOME="/home/nonroot/.rustup" ENV PATH="/home/nonroot/.cargo/bin:${PATH}" ARG RUSTFILT_VERSION=0.2.1 diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 3c5d0b12a669..92b7929c7f0e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.81.0" +channel = "1.82.0" profile = "default" # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy. # https://rust-lang.github.io/rustup/concepts/profiles.html From d762ad0883f204dee1b15729db8a6a3d6d5497e5 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 17 Oct 2024 20:45:37 +0100 Subject: [PATCH 08/21] update rustls (#9396) The forever ongoing effort of juggling multiple versions of rustls :3 now with new crypto library aws-lc. Because of dependencies, it is currently impossible to not have both ring and aws-lc in the dep tree, therefore our only options are not updating rustls or having both crypto backends enabled... According to benchmarks run by the rustls maintainer, aws-lc is faster than ring in some cases too , so it's not without its upsides, --- Cargo.lock | 220 +++++++++++++----- Cargo.toml | 12 +- libs/postgres_backend/tests/simple_select.rs | 29 ++- proxy/src/bin/pg_sni_router.rs | 10 +- proxy/src/compute.rs | 30 ++- proxy/src/config.rs | 14 +- proxy/src/proxy/tests/mod.rs | 51 ++-- .../src/scan_safekeeper_metadata.rs | 22 +- workspace_hack/Cargo.toml | 11 +- 9 files changed, 276 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b212bac2eea..ad29fa4634d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "asn1-rs" -version = "0.5.2" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -164,25 +164,25 @@ dependencies = [ [[package]] name = "asn1-rs-derive" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.52", "synstructure", ] [[package]] name = "asn1-rs-impl" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.52", ] [[package]] @@ -310,6 +310,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-lc-rs" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ddc4a5b231dd6958b140ff3151b6412b3f4321fab354f399eec8f14b06df62" +dependencies = [ + "bindgen 0.69.5", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + [[package]] name = "aws-runtime" version = "1.4.3" @@ -595,7 +622,7 @@ dependencies = [ "once_cell", "pin-project-lite", "pin-utils", - "rustls 0.21.11", + "rustls 0.21.12", "tokio", "tracing", ] @@ -915,6 +942,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.4.1", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.52", + "which", +] + [[package]] name = "bindgen" version = "0.70.1" @@ -924,7 +974,7 @@ dependencies = [ "bitflags 2.4.1", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.10.5", "log", "prettyplease", "proc-macro2", @@ -1038,12 +1088,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.83" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -1169,6 +1220,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +[[package]] +name = "cmake" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -1624,9 +1684,9 @@ dependencies = [ [[package]] name = "der-parser" -version = "8.2.0" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ "asn1-rs", "displaydoc", @@ -1755,6 +1815,12 @@ dependencies = [ "syn 2.0.52", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.14" @@ -2059,6 +2125,12 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -2412,6 +2484,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "hostname" version = "0.4.0" @@ -2581,7 +2662,7 @@ dependencies = [ "http 0.2.9", "hyper 0.14.30", "log", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-native-certs 0.6.2", "tokio", "tokio-rustls 0.24.0", @@ -2801,9 +2882,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -2907,6 +2988,12 @@ dependencies = [ "spin", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.150" @@ -3137,6 +3224,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + [[package]] name = "multimap" version = "0.8.3" @@ -3356,9 +3449,9 @@ dependencies = [ [[package]] name = "oid-registry" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" dependencies = [ "asn1-rs", ] @@ -4053,14 +4146,14 @@ dependencies = [ "bytes", "once_cell", "pq_proto", - "rustls 0.22.4", + "rustls 0.23.7", "rustls-pemfile 2.1.1", "serde", "thiserror", "tokio", "tokio-postgres", "tokio-postgres-rustls", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tokio-util", "tracing", ] @@ -4082,7 +4175,7 @@ name = "postgres_ffi" version = "0.1.0" dependencies = [ "anyhow", - "bindgen", + "bindgen 0.70.1", "bytes", "crc32c", "env_logger", @@ -4219,7 +4312,7 @@ checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", "heck 0.5.0", - "itertools 0.12.1", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -4239,7 +4332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.52", @@ -4327,8 +4420,8 @@ dependencies = [ "rsa", "rstest", "rustc-hash", - "rustls 0.22.4", - "rustls-native-certs 0.7.0", + "rustls 0.23.7", + "rustls-native-certs 0.8.0", "rustls-pemfile 2.1.1", "scopeguard", "serde", @@ -4345,7 +4438,7 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-postgres-rustls", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tokio-tungstenite", "tokio-util", "tracing", @@ -4509,12 +4602,13 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1" +checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779" dependencies = [ "pem", "ring", + "rustls-pki-types", "time", "yasna", ] @@ -4693,7 +4787,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-pemfile 1.0.2", "serde", "serde_json", @@ -4991,9 +5085,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", @@ -5021,6 +5115,7 @@ version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebbbdb961df0ad3f2652da8f3fdc4b36122f568f968f45ad3316f26c025c677b" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -5089,9 +5184,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.3.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" @@ -5109,6 +5204,7 @@ version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5312,7 +5408,7 @@ checksum = "00421ed8fa0c995f07cde48ba6c89e80f2b312f74ff637326f392fbfd23abe02" dependencies = [ "httpdate", "reqwest 0.12.4", - "rustls 0.21.11", + "rustls 0.21.12", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -5807,8 +5903,8 @@ dependencies = [ "postgres_ffi", "remote_storage", "reqwest 0.12.4", - "rustls 0.22.4", - "rustls-native-certs 0.7.0", + "rustls 0.23.7", + "rustls-native-certs 0.8.0", "serde", "serde_json", "storage_controller_client", @@ -5930,14 +6026,13 @@ checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" [[package]] name = "synstructure" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", - "unicode-xid", + "syn 2.0.52", ] [[package]] @@ -6236,16 +6331,15 @@ dependencies = [ [[package]] name = "tokio-postgres-rustls" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea13f22eda7127c827983bdaf0d7fff9df21c8817bab02815ac277a21143677" +checksum = "04fb792ccd6bbcd4bba408eb8a292f70fc4a3589e5d793626f45190e6454b6ab" dependencies = [ - "futures", "ring", - "rustls 0.22.4", + "rustls 0.23.7", "tokio", "tokio-postgres", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "x509-certificate", ] @@ -6255,7 +6349,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" dependencies = [ - "rustls 0.21.11", + "rustls 0.21.12", "tokio", ] @@ -6678,16 +6772,15 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.9.7" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11a831e3c0b56e438a28308e7c810799e3c118417f342d30ecec080105395cd" +checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.22.4", + "rustls 0.23.7", "rustls-pki-types", - "rustls-webpki 0.102.2", "url", "webpki-roots 0.26.1", ] @@ -6876,7 +6969,7 @@ name = "walproposer" version = "0.1.0" dependencies = [ "anyhow", - "bindgen", + "bindgen 0.70.1", "postgres_ffi", "utils", ] @@ -7051,6 +7144,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "whoami" version = "1.5.1" @@ -7295,7 +7400,6 @@ dependencies = [ "digest", "either", "fail", - "futures", "futures-channel", "futures-executor", "futures-io", @@ -7311,7 +7415,7 @@ dependencies = [ "hyper-util", "indexmap 1.9.3", "indexmap 2.0.1", - "itertools 0.12.1", + "itertools 0.10.5", "lazy_static", "libc", "log", @@ -7332,6 +7436,8 @@ dependencies = [ "regex-automata 0.4.3", "regex-syntax 0.8.2", "reqwest 0.12.4", + "rustls 0.23.7", + "rustls-webpki 0.102.2", "scopeguard", "serde", "serde_json", @@ -7340,7 +7446,6 @@ dependencies = [ "smallvec", "spki 0.7.3", "subtle", - "syn 1.0.109", "syn 2.0.52", "sync_wrapper 0.1.2", "tikv-jemalloc-sys", @@ -7348,6 +7453,7 @@ dependencies = [ "time-macros", "tokio", "tokio-postgres", + "tokio-rustls 0.26.0", "tokio-stream", "tokio-util", "toml_edit", @@ -7383,9 +7489,9 @@ dependencies = [ [[package]] name = "x509-parser" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab0c2f54ae1d92f4fcb99c0b7ccf0b1e3451cbd395e5f115ccbdbcb18d4f634" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" dependencies = [ "asn1-rs", "data-encoding", diff --git a/Cargo.toml b/Cargo.toml index a1a974b33b48..4c6a24ecdefd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,7 +142,7 @@ reqwest-retry = "0.5" routerify = "3" rpds = "0.13" rustc-hash = "1.1.0" -rustls = "0.22" +rustls = "0.23" rustls-pemfile = "2" scopeguard = "1.1" sysinfo = "0.29.2" @@ -172,8 +172,8 @@ tikv-jemalloc-ctl = "0.5" tokio = { version = "1.17", features = ["macros"] } tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" } tokio-io-timeout = "1.2.0" -tokio-postgres-rustls = "0.11.0" -tokio-rustls = "0.25" +tokio-postgres-rustls = "0.12.0" +tokio-rustls = "0.26" tokio-stream = "0.1" tokio-tar = "0.3" tokio-util = { version = "0.7.10", features = ["io", "rt"] } @@ -192,8 +192,8 @@ url = "2.2" urlencoding = "2.1" uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] } walkdir = "2.3.2" -rustls-native-certs = "0.7" -x509-parser = "0.15" +rustls-native-certs = "0.8" +x509-parser = "0.16" whoami = "1.5.1" ## TODO replace this with tracing @@ -244,7 +244,7 @@ workspace_hack = { version = "0.1", path = "./workspace_hack/" } ## Build dependencies criterion = "0.5.1" -rcgen = "0.12" +rcgen = "0.13" rstest = "0.18" camino-tempfile = "1.0.2" tonic-build = "0.12" diff --git a/libs/postgres_backend/tests/simple_select.rs b/libs/postgres_backend/tests/simple_select.rs index 900083ea7fc8..9d3031d6998b 100644 --- a/libs/postgres_backend/tests/simple_select.rs +++ b/libs/postgres_backend/tests/simple_select.rs @@ -2,6 +2,7 @@ use once_cell::sync::Lazy; use postgres_backend::{AuthType, Handler, PostgresBackend, QueryError}; use pq_proto::{BeMessage, RowDescriptor}; +use rustls::crypto::aws_lc_rs; use std::io::Cursor; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; @@ -92,10 +93,13 @@ static CERT: Lazy> = Lazy::new(|| { async fn simple_select_ssl() { let (client_sock, server_sock) = make_tcp_pair().await; - let server_cfg = rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(vec![CERT.clone()], KEY.clone_key()) - .unwrap(); + let server_cfg = + rustls::ServerConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .expect("aws_lc_rs should support the default protocol versions") + .with_no_client_auth() + .with_single_cert(vec![CERT.clone()], KEY.clone_key()) + .unwrap(); let tls_config = Some(Arc::new(server_cfg)); let pgbackend = PostgresBackend::new(server_sock, AuthType::Trust, tls_config).expect("pgbackend creation"); @@ -105,13 +109,16 @@ async fn simple_select_ssl() { pgbackend.run(&mut handler, &CancellationToken::new()).await }); - let client_cfg = rustls::ClientConfig::builder() - .with_root_certificates({ - let mut store = rustls::RootCertStore::empty(); - store.add(CERT.clone()).unwrap(); - store - }) - .with_no_client_auth(); + let client_cfg = + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .expect("aws_lc_rs should support the default protocol versions") + .with_root_certificates({ + let mut store = rustls::RootCertStore::empty(); + store.add(CERT.clone()).unwrap(); + store + }) + .with_no_client_auth(); let mut make_tls_connect = tokio_postgres_rustls::MakeRustlsConnect::new(client_cfg); let tls_connect = >::make_tls_connect( &mut make_tls_connect, diff --git a/proxy/src/bin/pg_sni_router.rs b/proxy/src/bin/pg_sni_router.rs index 00eb830d98a6..13b7fdd40a98 100644 --- a/proxy/src/bin/pg_sni_router.rs +++ b/proxy/src/bin/pg_sni_router.rs @@ -15,6 +15,7 @@ use proxy::context::RequestMonitoring; use proxy::metrics::{Metrics, ThreadPoolMetrics}; use proxy::proxy::{copy_bidirectional_client_compute, run_until_cancelled, ErrorSource}; use proxy::stream::{PqStream, Stream}; +use rustls::crypto::aws_lc_rs; use rustls::pki_types::PrivateKeyDer; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::net::TcpListener; @@ -104,10 +105,11 @@ async fn main() -> anyhow::Result<()> { let first_cert = cert_chain.first().context("missing certificate")?; let tls_server_end_point = TlsServerEndPoint::new(first_cert)?; - let tls_config = rustls::ServerConfig::builder_with_protocol_versions(&[ - &rustls::version::TLS13, - &rustls::version::TLS12, - ]) + let tls_config = rustls::ServerConfig::builder_with_provider(Arc::new( + aws_lc_rs::default_provider(), + )) + .with_protocol_versions(&[&rustls::version::TLS13, &rustls::version::TLS12]) + .context("aws_lc_rs should support TLS1.2 and TLS1.3")? .with_no_client_auth() .with_single_cert(cert_chain, key)? .into(); diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index 212e82497f48..a7c2cab4a1d8 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -8,6 +8,7 @@ use itertools::Itertools; use once_cell::sync::OnceCell; use pq_proto::StartupMessageParams; use rustls::client::danger::ServerCertVerifier; +use rustls::crypto::aws_lc_rs; use rustls::pki_types::InvalidDnsNameError; use thiserror::Error; use tokio::net::TcpStream; @@ -38,6 +39,9 @@ pub(crate) enum ConnectionError { #[error("{COULD_NOT_CONNECT}: {0}")] CouldNotConnect(#[from] io::Error), + #[error("Couldn't load native TLS certificates: {0:?}")] + TlsCertificateError(Vec), + #[error("{COULD_NOT_CONNECT}: {0}")] TlsError(#[from] InvalidDnsNameError), @@ -84,6 +88,7 @@ impl ReportableError for ConnectionError { } ConnectionError::Postgres(_) => crate::error::ErrorKind::Compute, ConnectionError::CouldNotConnect(_) => crate::error::ErrorKind::Compute, + ConnectionError::TlsCertificateError(_) => crate::error::ErrorKind::Service, ConnectionError::TlsError(_) => crate::error::ErrorKind::Compute, ConnectionError::WakeComputeError(e) => e.get_error_kind(), ConnectionError::TooManyConnectionAttempts(e) => e.get_error_kind(), @@ -293,12 +298,20 @@ impl ConnCfg { let client_config = if allow_self_signed_compute { // Allow all certificates for creating the connection let verifier = Arc::new(AcceptEverythingVerifier); - rustls::ClientConfig::builder() + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .expect("aws_lc_rs should support the default protocol versions") .dangerous() .with_custom_certificate_verifier(verifier) } else { - let root_store = TLS_ROOTS.get_or_try_init(load_certs)?.clone(); - rustls::ClientConfig::builder().with_root_certificates(root_store) + let root_store = TLS_ROOTS + .get_or_try_init(load_certs) + .map_err(ConnectionError::TlsCertificateError)? + .clone(); + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .expect("aws_lc_rs should support the default protocol versions") + .with_root_certificates(root_store) }; let client_config = client_config.with_no_client_auth(); @@ -359,10 +372,15 @@ fn filtered_options(params: &StartupMessageParams) -> Option { Some(options) } -fn load_certs() -> Result, io::Error> { - let der_certs = rustls_native_certs::load_native_certs()?; +fn load_certs() -> Result, Vec> { + let der_certs = rustls_native_certs::load_native_certs(); + + if !der_certs.errors.is_empty() { + return Err(der_certs.errors); + } + let mut store = rustls::RootCertStore::empty(); - store.add_parsable_certificates(der_certs); + store.add_parsable_certificates(der_certs.certs); Ok(Arc::new(store)) } static TLS_ROOTS: OnceCell> = OnceCell::new(); diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 2ec8c7adda9a..0d5ebd88f99e 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -7,7 +7,7 @@ use anyhow::{bail, ensure, Context, Ok}; use clap::ValueEnum; use itertools::Itertools; use remote_storage::RemoteStorageConfig; -use rustls::crypto::ring::sign; +use rustls::crypto::aws_lc_rs::{self, sign}; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use sha2::{Digest, Sha256}; use tracing::{error, info}; @@ -126,12 +126,12 @@ pub fn configure_tls( let cert_resolver = Arc::new(cert_resolver); // allow TLS 1.2 to be compatible with older client libraries - let mut config = rustls::ServerConfig::builder_with_protocol_versions(&[ - &rustls::version::TLS13, - &rustls::version::TLS12, - ]) - .with_no_client_auth() - .with_cert_resolver(cert_resolver.clone()); + let mut config = + rustls::ServerConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_protocol_versions(&[&rustls::version::TLS13, &rustls::version::TLS12]) + .context("aws_lc_rs should support TLS1.2 and TLS1.3")? + .with_no_client_auth() + .with_cert_resolver(cert_resolver.clone()); config.alpn_protocols = vec![PG_ALPN_PROTOCOL.to_vec()]; diff --git a/proxy/src/proxy/tests/mod.rs b/proxy/src/proxy/tests/mod.rs index e50ae4bc93c5..88175d73b163 100644 --- a/proxy/src/proxy/tests/mod.rs +++ b/proxy/src/proxy/tests/mod.rs @@ -9,6 +9,7 @@ use async_trait::async_trait; use http::StatusCode; use retry::{retry_after, ShouldRetryWakeCompute}; use rstest::rstest; +use rustls::crypto::aws_lc_rs; use rustls::pki_types; use tokio_postgres::config::SslMode; use tokio_postgres::tls::{MakeTlsConnect, NoTls}; @@ -38,25 +39,27 @@ fn generate_certs( pki_types::CertificateDer<'static>, pki_types::PrivateKeyDer<'static>, )> { - let ca = rcgen::Certificate::from_params({ + let ca_key = rcgen::KeyPair::generate()?; + let ca = { let mut params = rcgen::CertificateParams::default(); params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); - params - })?; + params.self_signed(&ca_key)? + }; - let cert = rcgen::Certificate::from_params({ - let mut params = rcgen::CertificateParams::new(vec![hostname.into()]); + let cert_key = rcgen::KeyPair::generate()?; + let cert = { + let mut params = rcgen::CertificateParams::new(vec![hostname.into()])?; params.distinguished_name = rcgen::DistinguishedName::new(); params .distinguished_name .push(rcgen::DnType::CommonName, common_name); - params - })?; + params.signed_by(&cert_key, &ca, &ca_key)? + }; Ok(( - pki_types::CertificateDer::from(ca.serialize_der()?), - pki_types::CertificateDer::from(cert.serialize_der_with_signer(&ca)?), - pki_types::PrivateKeyDer::Pkcs8(cert.serialize_private_key_der().into()), + ca.der().clone(), + cert.der().clone(), + pki_types::PrivateKeyDer::Pkcs8(cert_key.serialize_der().into()), )) } @@ -90,10 +93,13 @@ fn generate_tls_config<'a>( let (ca, cert, key) = generate_certs(hostname, common_name)?; let tls_config = { - let config = rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(vec![cert.clone()], key.clone_key())? - .into(); + let config = + rustls::ServerConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .context("aws_lc_rs should support the default protocol versions")? + .with_no_client_auth() + .with_single_cert(vec![cert.clone()], key.clone_key())? + .into(); let mut cert_resolver = CertResolver::new(); cert_resolver.add_cert(key, vec![cert], true)?; @@ -108,13 +114,16 @@ fn generate_tls_config<'a>( }; let client_config = { - let config = rustls::ClientConfig::builder() - .with_root_certificates({ - let mut store = rustls::RootCertStore::empty(); - store.add(ca)?; - store - }) - .with_no_client_auth(); + let config = + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .context("aws_lc_rs should support the default protocol versions")? + .with_root_certificates({ + let mut store = rustls::RootCertStore::empty(); + store.add(ca)?; + store + }) + .with_no_client_auth(); ClientConfig { config, hostname } }; diff --git a/storage_scrubber/src/scan_safekeeper_metadata.rs b/storage_scrubber/src/scan_safekeeper_metadata.rs index 15f3665fac37..6c312d003687 100644 --- a/storage_scrubber/src/scan_safekeeper_metadata.rs +++ b/storage_scrubber/src/scan_safekeeper_metadata.rs @@ -1,10 +1,12 @@ use std::{collections::HashSet, str::FromStr, sync::Arc}; +use anyhow::{bail, Context}; use futures::stream::{StreamExt, TryStreamExt}; use once_cell::sync::OnceCell; use pageserver_api::shard::TenantShardId; use postgres_ffi::{XLogFileName, PG_TLI}; use remote_storage::GenericRemoteStorage; +use rustls::crypto::aws_lc_rs; use serde::Serialize; use tokio_postgres::types::PgLsn; use tracing::{debug, error, info}; @@ -231,10 +233,15 @@ async fn check_timeline( }) } -fn load_certs() -> Result, std::io::Error> { - let der_certs = rustls_native_certs::load_native_certs()?; +fn load_certs() -> anyhow::Result> { + let der_certs = rustls_native_certs::load_native_certs(); + + if !der_certs.errors.is_empty() { + bail!("could not load native tls certs: {:?}", der_certs.errors); + } + let mut store = rustls::RootCertStore::empty(); - store.add_parsable_certificates(der_certs); + store.add_parsable_certificates(der_certs.certs); Ok(Arc::new(store)) } static TLS_ROOTS: OnceCell> = OnceCell::new(); @@ -248,9 +255,12 @@ async fn load_timelines_from_db( // Use rustls (Neon requires TLS) let root_store = TLS_ROOTS.get_or_try_init(load_certs)?.clone(); - let client_config = rustls::ClientConfig::builder() - .with_root_certificates(root_store) - .with_no_client_auth(); + let client_config = + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .context("aws_lc_rs should support the default protocol versions")? + .with_root_certificates(root_store) + .with_no_client_auth(); let tls_connector = tokio_postgres_rustls::MakeRustlsConnect::new(client_config); let (client, connection) = tokio_postgres::connect(&dump_db_connstr, tls_connector).await?; // The connection object performs the actual communication with the database, diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 1347d6ddff64..28c51b8ac120 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -32,7 +32,6 @@ deranged = { version = "0.3", default-features = false, features = ["powerfmt", digest = { version = "0.10", features = ["mac", "oid", "std"] } either = { version = "1" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } -futures = { version = "0.3" } futures-channel = { version = "0.3", features = ["sink"] } futures-executor = { version = "0.3" } futures-io = { version = "0.3" } @@ -48,7 +47,7 @@ hyper-dff4ba8e3ae991db = { package = "hyper", version = "1", features = ["full"] hyper-util = { version = "0.1", features = ["client-legacy", "server-auto", "service"] } indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["std"] } indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] } -itertools = { version = "0.12" } +itertools = { version = "0.10" } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } log = { version = "0.4", default-features = false, features = ["std"] } @@ -66,6 +65,8 @@ regex = { version = "1" } regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8" } reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "stream"] } +rustls = { version = "0.23", features = ["ring"] } +rustls-webpki = { version = "0.102", default-features = false, features = ["aws_lc_rs", "ring", "std"] } scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["alloc", "raw_value"] } @@ -79,6 +80,7 @@ tikv-jemalloc-sys = { version = "0.5" } time = { version = "0.3", features = ["macros", "serde-well-known"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "20031d7a9ee1addeae6e0968e3899ae6bf01cee2", features = ["with-serde_json-1"] } +tokio-rustls = { version = "0.26", features = ["ring"] } tokio-stream = { version = "0.1", features = ["net"] } tokio-util = { version = "0.7", features = ["codec", "compat", "io", "rt"] } toml_edit = { version = "0.22", features = ["serde"] } @@ -104,7 +106,7 @@ half = { version = "2", default-features = false, features = ["num-traits"] } hashbrown = { version = "0.14", features = ["raw"] } indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["std"] } indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] } -itertools = { version = "0.12" } +itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits", "use_std"] } log = { version = "0.4", default-features = false, features = ["std"] } memchr = { version = "2" } @@ -122,8 +124,7 @@ regex = { version = "1" } regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8" } serde = { version = "1", features = ["alloc", "derive"] } -syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit"] } -syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +syn = { version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] } toml_edit = { version = "0.22", features = ["serde"] } zstd = { version = "0.13" } From b8304f90d6ad9a5f118a59ac392b3330495827d3 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Fri, 18 Oct 2024 10:27:50 +0100 Subject: [PATCH 09/21] 2024 oct new clippy lints (#9448) Fixes new lints from `cargo +nightly clippy` (`clippy 0.1.83 (798fb83f 2024-10-16)`) --- compute_tools/src/extension_server.rs | 2 +- .../pageserver_api/src/models/partitioning.rs | 6 ++-- libs/postgres_backend/src/lib.rs | 3 +- libs/pq_proto/src/lib.rs | 2 +- libs/tenant_size_model/src/svg.rs | 2 +- libs/tracing-utils/src/http.rs | 2 +- libs/utils/src/lsn.rs | 2 +- libs/utils/src/poison.rs | 4 +-- libs/utils/src/shard.rs | 2 +- libs/utils/src/simple_rcu.rs | 4 +-- libs/utils/src/sync/heavier_once_cell.rs | 4 +-- libs/utils/src/tracing_span_assert.rs | 10 +++---- pageserver/compaction/src/helpers.rs | 10 +++---- pageserver/src/consumption_metrics/upload.rs | 2 +- pageserver/src/disk_usage_eviction_task.rs | 2 +- pageserver/src/metrics.rs | 4 +-- pageserver/src/statvfs.rs | 2 +- pageserver/src/tenant/block_io.rs | 4 +-- pageserver/src/tenant/disk_btree.rs | 2 +- .../src/tenant/remote_timeline_client.rs | 2 +- .../src/tenant/secondary/heatmap_uploader.rs | 1 - pageserver/src/tenant/storage_layer.rs | 2 +- .../src/tenant/storage_layer/delta_layer.rs | 3 +- .../src/tenant/storage_layer/image_layer.rs | 3 +- pageserver/src/tenant/storage_layer/layer.rs | 2 +- .../src/tenant/storage_layer/layer_name.rs | 2 +- .../tenant/storage_layer/merge_iterator.rs | 8 +++--- pageserver/src/tenant/vectored_blob_io.rs | 21 +++----------- pageserver/src/virtual_file.rs | 4 +-- proxy/src/auth/credentials.rs | 2 +- proxy/src/config.rs | 2 +- proxy/src/context/parquet.rs | 2 +- proxy/src/intern.rs | 2 +- proxy/src/lib.rs | 6 +--- proxy/src/proxy/tests/mod.rs | 10 +++---- proxy/src/scram/exchange.rs | 4 --- proxy/src/serverless/conn_pool.rs | 12 ++++---- proxy/src/serverless/conn_pool_lib.rs | 28 +++++++++---------- proxy/src/serverless/http_conn_pool.rs | 3 +- proxy/src/serverless/json.rs | 6 ++-- proxy/src/serverless/local_conn_pool.rs | 3 +- proxy/src/serverless/sql_over_http.rs | 1 - proxy/src/usage_metrics.rs | 10 +++---- proxy/src/waiters.rs | 2 +- safekeeper/src/timeline.rs | 6 ++-- 45 files changed, 92 insertions(+), 124 deletions(-) diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index 6ef7e0837f81..da2d107b5488 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -107,7 +107,7 @@ pub fn get_pg_version(pgbin: &str) -> String { // pg_config --version returns a (platform specific) human readable string // such as "PostgreSQL 15.4". We parse this to v14/v15/v16 etc. let human_version = get_pg_config("--version", pgbin); - return parse_pg_version(&human_version).to_string(); + parse_pg_version(&human_version).to_string() } fn parse_pg_version(human_version: &str) -> &str { diff --git a/libs/pageserver_api/src/models/partitioning.rs b/libs/pageserver_api/src/models/partitioning.rs index f6644be6359c..69832b9a0dee 100644 --- a/libs/pageserver_api/src/models/partitioning.rs +++ b/libs/pageserver_api/src/models/partitioning.rs @@ -16,7 +16,7 @@ impl serde::Serialize for Partitioning { { pub struct KeySpace<'a>(&'a crate::keyspace::KeySpace); - impl<'a> serde::Serialize for KeySpace<'a> { + impl serde::Serialize for KeySpace<'_> { fn serialize(&self, serializer: S) -> std::result::Result where S: serde::Serializer, @@ -44,7 +44,7 @@ impl serde::Serialize for Partitioning { pub struct WithDisplay<'a, T>(&'a T); -impl<'a, T: std::fmt::Display> serde::Serialize for WithDisplay<'a, T> { +impl serde::Serialize for WithDisplay<'_, T> { fn serialize(&self, serializer: S) -> std::result::Result where S: serde::Serializer, @@ -55,7 +55,7 @@ impl<'a, T: std::fmt::Display> serde::Serialize for WithDisplay<'a, T> { pub struct KeyRange<'a>(&'a std::ops::Range); -impl<'a> serde::Serialize for KeyRange<'a> { +impl serde::Serialize for KeyRange<'_> { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 085540e7b97b..9d274b25e6c3 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -921,12 +921,11 @@ impl PostgresBackendReader { /// A futures::AsyncWrite implementation that wraps all data written to it in CopyData /// messages. /// - pub struct CopyDataWriter<'a, IO> { pgb: &'a mut PostgresBackend, } -impl<'a, IO: AsyncRead + AsyncWrite + Unpin> AsyncWrite for CopyDataWriter<'a, IO> { +impl AsyncWrite for CopyDataWriter<'_, IO> { fn poll_write( self: Pin<&mut Self>, cx: &mut std::task::Context<'_>, diff --git a/libs/pq_proto/src/lib.rs b/libs/pq_proto/src/lib.rs index a01191bd5de3..9ffaaba584c2 100644 --- a/libs/pq_proto/src/lib.rs +++ b/libs/pq_proto/src/lib.rs @@ -727,7 +727,7 @@ pub const SQLSTATE_INTERNAL_ERROR: &[u8; 5] = b"XX000"; pub const SQLSTATE_ADMIN_SHUTDOWN: &[u8; 5] = b"57P01"; pub const SQLSTATE_SUCCESSFUL_COMPLETION: &[u8; 5] = b"00000"; -impl<'a> BeMessage<'a> { +impl BeMessage<'_> { /// Serialize `message` to the given `buf`. /// Apart from smart memory managemet, BytesMut is good here as msg len /// precedes its body and it is handy to write it down first and then fill diff --git a/libs/tenant_size_model/src/svg.rs b/libs/tenant_size_model/src/svg.rs index 0de2890bb414..25ebb1c3d8c9 100644 --- a/libs/tenant_size_model/src/svg.rs +++ b/libs/tenant_size_model/src/svg.rs @@ -97,7 +97,7 @@ pub fn draw_svg( Ok(result) } -impl<'a> SvgDraw<'a> { +impl SvgDraw<'_> { fn calculate_svg_layout(&mut self) { // Find x scale let segments = &self.storage.segments; diff --git a/libs/tracing-utils/src/http.rs b/libs/tracing-utils/src/http.rs index e6fdf9be45ff..2168beee8809 100644 --- a/libs/tracing-utils/src/http.rs +++ b/libs/tracing-utils/src/http.rs @@ -82,7 +82,7 @@ where fn extract_remote_context(headers: &HeaderMap) -> opentelemetry::Context { struct HeaderExtractor<'a>(&'a HeaderMap); - impl<'a> opentelemetry::propagation::Extractor for HeaderExtractor<'a> { + impl opentelemetry::propagation::Extractor for HeaderExtractor<'_> { fn get(&self, key: &str) -> Option<&str> { self.0.get(key).and_then(|value| value.to_str().ok()) } diff --git a/libs/utils/src/lsn.rs b/libs/utils/src/lsn.rs index 06d5c27ebf50..3ec2c130bdb4 100644 --- a/libs/utils/src/lsn.rs +++ b/libs/utils/src/lsn.rs @@ -37,7 +37,7 @@ impl<'de> Deserialize<'de> for Lsn { is_human_readable_deserializer: bool, } - impl<'de> Visitor<'de> for LsnVisitor { + impl Visitor<'_> for LsnVisitor { type Value = Lsn; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { diff --git a/libs/utils/src/poison.rs b/libs/utils/src/poison.rs index c3e2fba20c3c..ab9ebb3c5a8f 100644 --- a/libs/utils/src/poison.rs +++ b/libs/utils/src/poison.rs @@ -73,7 +73,7 @@ impl Poison { /// and subsequent calls to [`Poison::check_and_arm`] will fail with an error. pub struct Guard<'a, T>(&'a mut Poison); -impl<'a, T> Guard<'a, T> { +impl Guard<'_, T> { pub fn data(&self) -> &T { &self.0.data } @@ -94,7 +94,7 @@ impl<'a, T> Guard<'a, T> { } } -impl<'a, T> Drop for Guard<'a, T> { +impl Drop for Guard<'_, T> { fn drop(&mut self) { match self.0.state { State::Clean => { diff --git a/libs/utils/src/shard.rs b/libs/utils/src/shard.rs index d146010b417a..782cddc599b0 100644 --- a/libs/utils/src/shard.rs +++ b/libs/utils/src/shard.rs @@ -164,7 +164,7 @@ impl TenantShardId { } } -impl<'a> std::fmt::Display for ShardSlug<'a> { +impl std::fmt::Display for ShardSlug<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, diff --git a/libs/utils/src/simple_rcu.rs b/libs/utils/src/simple_rcu.rs index 01750b2aef12..6700f86e4aea 100644 --- a/libs/utils/src/simple_rcu.rs +++ b/libs/utils/src/simple_rcu.rs @@ -152,7 +152,7 @@ pub struct RcuWriteGuard<'a, V> { inner: RwLockWriteGuard<'a, RcuInner>, } -impl<'a, V> Deref for RcuWriteGuard<'a, V> { +impl Deref for RcuWriteGuard<'_, V> { type Target = V; fn deref(&self) -> &V { @@ -160,7 +160,7 @@ impl<'a, V> Deref for RcuWriteGuard<'a, V> { } } -impl<'a, V> RcuWriteGuard<'a, V> { +impl RcuWriteGuard<'_, V> { /// /// Store a new value. The new value will be written to the Rcu immediately, /// and will be immediately seen by any `read` calls that start afterwards. diff --git a/libs/utils/src/sync/heavier_once_cell.rs b/libs/utils/src/sync/heavier_once_cell.rs index dc711fb0284e..66c20655547b 100644 --- a/libs/utils/src/sync/heavier_once_cell.rs +++ b/libs/utils/src/sync/heavier_once_cell.rs @@ -219,7 +219,7 @@ impl<'a, T> CountWaitingInitializers<'a, T> { } } -impl<'a, T> Drop for CountWaitingInitializers<'a, T> { +impl Drop for CountWaitingInitializers<'_, T> { fn drop(&mut self) { self.0.initializers.fetch_sub(1, Ordering::Relaxed); } @@ -250,7 +250,7 @@ impl std::ops::DerefMut for Guard<'_, T> { } } -impl<'a, T> Guard<'a, T> { +impl Guard<'_, T> { /// Take the current value, and a new permit for it's deinitialization. /// /// The permit will be on a semaphore part of the new internal value, and any following diff --git a/libs/utils/src/tracing_span_assert.rs b/libs/utils/src/tracing_span_assert.rs index d24c81ad0bcc..add2fa79202c 100644 --- a/libs/utils/src/tracing_span_assert.rs +++ b/libs/utils/src/tracing_span_assert.rs @@ -184,23 +184,23 @@ mod tests { struct MemoryIdentity<'a>(&'a dyn Extractor); - impl<'a> MemoryIdentity<'a> { + impl MemoryIdentity<'_> { fn as_ptr(&self) -> *const () { self.0 as *const _ as *const () } } - impl<'a> PartialEq for MemoryIdentity<'a> { + impl PartialEq for MemoryIdentity<'_> { fn eq(&self, other: &Self) -> bool { self.as_ptr() == other.as_ptr() } } - impl<'a> Eq for MemoryIdentity<'a> {} - impl<'a> Hash for MemoryIdentity<'a> { + impl Eq for MemoryIdentity<'_> {} + impl Hash for MemoryIdentity<'_> { fn hash(&self, state: &mut H) { self.as_ptr().hash(state); } } - impl<'a> fmt::Debug for MemoryIdentity<'a> { + impl fmt::Debug for MemoryIdentity<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:p}: {}", self.as_ptr(), self.0.id()) } diff --git a/pageserver/compaction/src/helpers.rs b/pageserver/compaction/src/helpers.rs index 8ed1d1608201..9dbb6ecedf23 100644 --- a/pageserver/compaction/src/helpers.rs +++ b/pageserver/compaction/src/helpers.rs @@ -133,7 +133,7 @@ enum LazyLoadLayer<'a, E: CompactionJobExecutor> { Loaded(VecDeque<>::DeltaEntry<'a>>), Unloaded(&'a E::DeltaLayer), } -impl<'a, E: CompactionJobExecutor> LazyLoadLayer<'a, E> { +impl LazyLoadLayer<'_, E> { fn min_key(&self) -> E::Key { match self { Self::Loaded(entries) => entries.front().unwrap().key(), @@ -147,23 +147,23 @@ impl<'a, E: CompactionJobExecutor> LazyLoadLayer<'a, E> { } } } -impl<'a, E: CompactionJobExecutor> PartialOrd for LazyLoadLayer<'a, E> { +impl PartialOrd for LazyLoadLayer<'_, E> { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl<'a, E: CompactionJobExecutor> Ord for LazyLoadLayer<'a, E> { +impl Ord for LazyLoadLayer<'_, E> { fn cmp(&self, other: &Self) -> std::cmp::Ordering { // reverse order so that we get a min-heap (other.min_key(), other.min_lsn()).cmp(&(self.min_key(), self.min_lsn())) } } -impl<'a, E: CompactionJobExecutor> PartialEq for LazyLoadLayer<'a, E> { +impl PartialEq for LazyLoadLayer<'_, E> { fn eq(&self, other: &Self) -> bool { self.cmp(other) == std::cmp::Ordering::Equal } } -impl<'a, E: CompactionJobExecutor> Eq for LazyLoadLayer<'a, E> {} +impl Eq for LazyLoadLayer<'_, E> {} type LoadFuture<'a, E> = BoxFuture<'a, anyhow::Result>>; diff --git a/pageserver/src/consumption_metrics/upload.rs b/pageserver/src/consumption_metrics/upload.rs index 0325ee403a4c..1eb25d337b48 100644 --- a/pageserver/src/consumption_metrics/upload.rs +++ b/pageserver/src/consumption_metrics/upload.rs @@ -198,7 +198,7 @@ fn serialize_in_chunks<'a>( } } - impl<'a> ExactSizeIterator for Iter<'a> {} + impl ExactSizeIterator for Iter<'_> {} let buffer = bytes::BytesMut::new(); let inner = input.chunks(chunk_size); diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index 7ab2ba87420f..ca44fbe6ae6e 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -654,7 +654,7 @@ impl std::fmt::Debug for EvictionCandidate { let ts = chrono::DateTime::::from(self.last_activity_ts); let ts = ts.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); struct DisplayIsDebug<'a, T>(&'a T); - impl<'a, T: std::fmt::Display> std::fmt::Debug for DisplayIsDebug<'a, T> { + impl std::fmt::Debug for DisplayIsDebug<'_, T> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.0) } diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index b76efa5b487b..3e824b59fb86 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1189,7 +1189,7 @@ struct GlobalAndPerTimelineHistogramTimer<'a, 'c> { op: SmgrQueryType, } -impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> { +impl Drop for GlobalAndPerTimelineHistogramTimer<'_, '_> { fn drop(&mut self) { let elapsed = self.start.elapsed(); let ex_throttled = self @@ -1560,7 +1560,7 @@ impl BasebackupQueryTime { } } -impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> { +impl BasebackupQueryTimeOngoingRecording<'_, '_> { pub(crate) fn observe(self, res: &Result) { let elapsed = self.start.elapsed(); let ex_throttled = self diff --git a/pageserver/src/statvfs.rs b/pageserver/src/statvfs.rs index 205605bc86a8..4e8be58d583b 100644 --- a/pageserver/src/statvfs.rs +++ b/pageserver/src/statvfs.rs @@ -90,7 +90,7 @@ pub mod mock { let used_bytes = walk_dir_disk_usage(tenants_dir, name_filter.as_deref()).unwrap(); // round it up to the nearest block multiple - let used_blocks = (used_bytes + (blocksize - 1)) / blocksize; + let used_blocks = used_bytes.div_ceil(*blocksize); if used_blocks > *total_blocks { panic!( diff --git a/pageserver/src/tenant/block_io.rs b/pageserver/src/tenant/block_io.rs index 3afa3a86b948..1c82e5454de9 100644 --- a/pageserver/src/tenant/block_io.rs +++ b/pageserver/src/tenant/block_io.rs @@ -50,13 +50,13 @@ impl From> for BlockLease<'static> { } #[cfg(test)] -impl<'a> From> for BlockLease<'a> { +impl From> for BlockLease<'_> { fn from(value: std::sync::Arc<[u8; PAGE_SZ]>) -> Self { BlockLease::Arc(value) } } -impl<'a> Deref for BlockLease<'a> { +impl Deref for BlockLease<'_> { type Target = [u8; PAGE_SZ]; fn deref(&self) -> &Self::Target { diff --git a/pageserver/src/tenant/disk_btree.rs b/pageserver/src/tenant/disk_btree.rs index 0107b0ac7e9c..b302cbc97559 100644 --- a/pageserver/src/tenant/disk_btree.rs +++ b/pageserver/src/tenant/disk_btree.rs @@ -131,7 +131,7 @@ struct OnDiskNode<'a, const L: usize> { values: &'a [u8], } -impl<'a, const L: usize> OnDiskNode<'a, L> { +impl OnDiskNode<'_, L> { /// /// Interpret a PAGE_SZ page as a node. /// diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 450084aca249..14b894d17c01 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -2182,7 +2182,7 @@ pub(crate) struct UploadQueueAccessor<'a> { inner: std::sync::MutexGuard<'a, UploadQueue>, } -impl<'a> UploadQueueAccessor<'a> { +impl UploadQueueAccessor<'_> { pub(crate) fn latest_uploaded_index_part(&self) -> &IndexPart { match &*self.inner { UploadQueue::Initialized(x) => &x.clean.0, diff --git a/pageserver/src/tenant/secondary/heatmap_uploader.rs b/pageserver/src/tenant/secondary/heatmap_uploader.rs index 0aad5bf392d5..e680fd705b42 100644 --- a/pageserver/src/tenant/secondary/heatmap_uploader.rs +++ b/pageserver/src/tenant/secondary/heatmap_uploader.rs @@ -108,7 +108,6 @@ impl scheduler::Completion for WriteComplete { /// when we last did a write. We only populate this after doing at least one /// write for a tenant -- this avoids holding state for tenants that have /// uploads disabled. - struct UploaderTenantState { // This Weak only exists to enable culling idle instances of this type // when the Tenant has been deallocated. diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 99bd0ece5763..a229b5956059 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -705,7 +705,7 @@ pub mod tests { /// Useful with `Key`, which has too verbose `{:?}` for printing multiple layers. struct RangeDisplayDebug<'a, T: std::fmt::Display>(&'a Range); -impl<'a, T: std::fmt::Display> std::fmt::Debug for RangeDisplayDebug<'a, T> { +impl std::fmt::Debug for RangeDisplayDebug<'_, T> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}..{}", self.0.start, self.0.end) } diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 8be7d7876f82..d1079876f8fe 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -529,8 +529,7 @@ impl DeltaLayerWriterInner { key_end: Key, ctx: &RequestContext, ) -> anyhow::Result<(PersistentLayerDesc, Utf8PathBuf)> { - let index_start_blk = - ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32; + let index_start_blk = self.blob_writer.size().div_ceil(PAGE_SZ as u64) as u32; let mut file = self.blob_writer.into_inner(ctx).await?; diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index de8155f455d1..6c1a943470c2 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -828,8 +828,7 @@ impl ImageLayerWriterInner { ctx: &RequestContext, end_key: Option, ) -> anyhow::Result<(PersistentLayerDesc, Utf8PathBuf)> { - let index_start_blk = - ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32; + let index_start_blk = self.blob_writer.size().div_ceil(PAGE_SZ as u64) as u32; // Calculate compression ratio let compressed_size = self.blob_writer.size() - PAGE_SZ as u64; // Subtract PAGE_SZ for header diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index f29a33bae6de..38a7cd09aff3 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -978,7 +978,7 @@ impl LayerInner { let timeline = self .timeline .upgrade() - .ok_or_else(|| DownloadError::TimelineShutdown)?; + .ok_or(DownloadError::TimelineShutdown)?; // count cancellations, which currently remain largely unexpected let init_cancelled = scopeguard::guard((), |_| LAYER_IMPL_METRICS.inc_init_cancelled()); diff --git a/pageserver/src/tenant/storage_layer/layer_name.rs b/pageserver/src/tenant/storage_layer/layer_name.rs index ffe7ca5f3e40..8e750e1187a3 100644 --- a/pageserver/src/tenant/storage_layer/layer_name.rs +++ b/pageserver/src/tenant/storage_layer/layer_name.rs @@ -339,7 +339,7 @@ impl<'de> serde::Deserialize<'de> for LayerName { struct LayerNameVisitor; -impl<'de> serde::de::Visitor<'de> for LayerNameVisitor { +impl serde::de::Visitor<'_> for LayerNameVisitor { type Value = LayerName; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 0831fd9530da..f91e27241ddc 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -99,21 +99,21 @@ impl<'a> PeekableLayerIterRef<'a> { } } -impl<'a> std::cmp::PartialEq for IteratorWrapper<'a> { +impl std::cmp::PartialEq for IteratorWrapper<'_> { fn eq(&self, other: &Self) -> bool { self.cmp(other) == Ordering::Equal } } -impl<'a> std::cmp::Eq for IteratorWrapper<'a> {} +impl std::cmp::Eq for IteratorWrapper<'_> {} -impl<'a> std::cmp::PartialOrd for IteratorWrapper<'a> { +impl std::cmp::PartialOrd for IteratorWrapper<'_> { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl<'a> std::cmp::Ord for IteratorWrapper<'a> { +impl std::cmp::Ord for IteratorWrapper<'_> { fn cmp(&self, other: &Self) -> std::cmp::Ordering { use std::cmp::Ordering; let a = self.peek_next_key_lsn_value(); diff --git a/pageserver/src/tenant/vectored_blob_io.rs b/pageserver/src/tenant/vectored_blob_io.rs index 792c769b4fc0..0c037910344c 100644 --- a/pageserver/src/tenant/vectored_blob_io.rs +++ b/pageserver/src/tenant/vectored_blob_io.rs @@ -73,7 +73,7 @@ impl<'a> BufView<'a> { } } -impl<'a> Deref for BufView<'a> { +impl Deref for BufView<'_> { type Target = [u8]; fn deref(&self) -> &Self::Target { @@ -84,7 +84,7 @@ impl<'a> Deref for BufView<'a> { } } -impl<'a> AsRef<[u8]> for BufView<'a> { +impl AsRef<[u8]> for BufView<'_> { fn as_ref(&self) -> &[u8] { match self { BufView::Slice(slice) => slice, @@ -196,11 +196,6 @@ pub(crate) struct ChunkedVectoredReadBuilder { max_read_size: Option, } -/// Computes x / d rounded up. -fn div_round_up(x: usize, d: usize) -> usize { - (x + (d - 1)) / d -} - impl ChunkedVectoredReadBuilder { const CHUNK_SIZE: usize = virtual_file::get_io_buffer_alignment(); /// Start building a new vectored read. @@ -220,7 +215,7 @@ impl ChunkedVectoredReadBuilder { .expect("First insertion always succeeds"); let start_blk_no = start_offset as usize / Self::CHUNK_SIZE; - let end_blk_no = div_round_up(end_offset as usize, Self::CHUNK_SIZE); + let end_blk_no = (end_offset as usize).div_ceil(Self::CHUNK_SIZE); Self { start_blk_no, end_blk_no, @@ -248,7 +243,7 @@ impl ChunkedVectoredReadBuilder { pub(crate) fn extend(&mut self, start: u64, end: u64, meta: BlobMeta) -> VectoredReadExtended { tracing::trace!(start, end, "trying to extend"); let start_blk_no = start as usize / Self::CHUNK_SIZE; - let end_blk_no = div_round_up(end as usize, Self::CHUNK_SIZE); + let end_blk_no = (end as usize).div_ceil(Self::CHUNK_SIZE); let not_limited_by_max_read_size = { if let Some(max_read_size) = self.max_read_size { @@ -975,12 +970,4 @@ mod tests { round_trip_test_compressed(&blobs, true).await?; Ok(()) } - - #[test] - fn test_div_round_up() { - const CHUNK_SIZE: usize = 512; - assert_eq!(1, div_round_up(200, CHUNK_SIZE)); - assert_eq!(1, div_round_up(CHUNK_SIZE, CHUNK_SIZE)); - assert_eq!(2, div_round_up(CHUNK_SIZE + 1, CHUNK_SIZE)); - } } diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index d260116b3892..5a364b7aaf03 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -724,9 +724,9 @@ impl VirtualFileInner { *handle_guard = handle; - return Ok(FileGuard { + Ok(FileGuard { slot_guard: slot_guard.downgrade(), - }); + }) } pub fn remove(self) { diff --git a/proxy/src/auth/credentials.rs b/proxy/src/auth/credentials.rs index fa6bc4c6f5b7..465e427f7c69 100644 --- a/proxy/src/auth/credentials.rs +++ b/proxy/src/auth/credentials.rs @@ -193,7 +193,7 @@ impl<'de> serde::de::Deserialize<'de> for IpPattern { D: serde::Deserializer<'de>, { struct StrVisitor; - impl<'de> serde::de::Visitor<'de> for StrVisitor { + impl serde::de::Visitor<'_> for StrVisitor { type Value = IpPattern; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 0d5ebd88f99e..3baa7ec751eb 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -558,7 +558,7 @@ pub struct RetryConfig { } impl RetryConfig { - /// Default options for RetryConfig. + // Default options for RetryConfig. /// Total delay for 5 retries with 200ms base delay and 2 backoff factor is about 6s. pub const CONNECT_TO_COMPUTE_DEFAULT_VALUES: &'static str = diff --git a/proxy/src/context/parquet.rs b/proxy/src/context/parquet.rs index b0ad0e45662b..3432ac5ff660 100644 --- a/proxy/src/context/parquet.rs +++ b/proxy/src/context/parquet.rs @@ -104,7 +104,7 @@ struct Options<'a> { options: &'a StartupMessageParams, } -impl<'a> serde::Serialize for Options<'a> { +impl serde::Serialize for Options<'_> { fn serialize(&self, s: S) -> Result where S: serde::Serializer, diff --git a/proxy/src/intern.rs b/proxy/src/intern.rs index 09fd9657d076..49aab917e4ee 100644 --- a/proxy/src/intern.rs +++ b/proxy/src/intern.rs @@ -55,7 +55,7 @@ impl std::ops::Deref for InternedString { impl<'de, Id: InternId> serde::de::Deserialize<'de> for InternedString { fn deserialize>(d: D) -> Result { struct Visitor(PhantomData); - impl<'de, Id: InternId> serde::de::Visitor<'de> for Visitor { + impl serde::de::Visitor<'_> for Visitor { type Value = InternedString; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/proxy/src/lib.rs b/proxy/src/lib.rs index 74bc778a36c4..a7b3d45c95e6 100644 --- a/proxy/src/lib.rs +++ b/proxy/src/lib.rs @@ -76,11 +76,7 @@ ) )] // List of temporarily allowed lints to unblock beta/nightly. -#![allow( - unknown_lints, - // TODO: 1.82: Add `use` where necessary and remove from this list. - impl_trait_overcaptures, -)] +#![allow(unknown_lints)] use std::convert::Infallible; diff --git a/proxy/src/proxy/tests/mod.rs b/proxy/src/proxy/tests/mod.rs index 88175d73b163..3f54b0661b4e 100644 --- a/proxy/src/proxy/tests/mod.rs +++ b/proxy/src/proxy/tests/mod.rs @@ -73,11 +73,11 @@ impl ClientConfig<'_> { self, ) -> anyhow::Result< impl tokio_postgres::tls::TlsConnect< - S, - Error = impl std::fmt::Debug, - Future = impl Send, - Stream = RustlsStream, - >, + S, + Error = impl std::fmt::Debug + use, + Future = impl Send + use, + Stream = RustlsStream, + > + use, > { let mut mk = MakeRustlsConnect::new(self.config); let tls = MakeTlsConnect::::make_tls_connect(&mut mk, self.hostname)?; diff --git a/proxy/src/scram/exchange.rs b/proxy/src/scram/exchange.rs index 493295c938f6..6a13f645a5ab 100644 --- a/proxy/src/scram/exchange.rs +++ b/proxy/src/scram/exchange.rs @@ -218,16 +218,12 @@ impl sasl::Mechanism for Exchange<'_> { self.state = ExchangeState::SaltSent(sent); Ok(Step::Continue(self, msg)) } - #[allow(unreachable_patterns)] // TODO: 1.82: simply drop this match - Step::Success(x, _) => match x {}, Step::Failure(msg) => Ok(Step::Failure(msg)), } } ExchangeState::SaltSent(sent) => { match sent.transition(self.secret, &self.tls_server_end_point, input)? { Step::Success(keys, msg) => Ok(Step::Success(keys, msg)), - #[allow(unreachable_patterns)] // TODO: 1.82: simply drop this match - Step::Continue(x, _) => match x {}, Step::Failure(msg) => Ok(Step::Failure(msg)), } } diff --git a/proxy/src/serverless/conn_pool.rs b/proxy/src/serverless/conn_pool.rs index b97c6565101e..8401e3a1c9a5 100644 --- a/proxy/src/serverless/conn_pool.rs +++ b/proxy/src/serverless/conn_pool.rs @@ -11,13 +11,6 @@ use tokio_postgres::tls::NoTlsStream; use tokio_postgres::{AsyncMessage, Socket}; use tokio_util::sync::CancellationToken; use tracing::{error, info, info_span, warn, Instrument}; - -use crate::context::RequestMonitoring; -use crate::control_plane::messages::MetricsAuxInfo; -use crate::metrics::Metrics; - -use super::conn_pool_lib::{Client, ClientInnerExt, ConnInfo, GlobalConnPool}; - #[cfg(test)] use { super::conn_pool_lib::GlobalConnPoolOptions, @@ -25,6 +18,11 @@ use { std::{sync::atomic, time::Duration}, }; +use super::conn_pool_lib::{Client, ClientInnerExt, ConnInfo, GlobalConnPool}; +use crate::context::RequestMonitoring; +use crate::control_plane::messages::MetricsAuxInfo; +use crate::metrics::Metrics; + #[derive(Debug, Clone)] pub(crate) struct ConnInfoWithAuth { pub(crate) conn_info: ConnInfo, diff --git a/proxy/src/serverless/conn_pool_lib.rs b/proxy/src/serverless/conn_pool_lib.rs index 6e964ce8789f..844730194d76 100644 --- a/proxy/src/serverless/conn_pool_lib.rs +++ b/proxy/src/serverless/conn_pool_lib.rs @@ -1,25 +1,23 @@ +use std::collections::HashMap; +use std::ops::Deref; +use std::sync::atomic::{self, AtomicUsize}; +use std::sync::{Arc, Weak}; +use std::time::Duration; + use dashmap::DashMap; use parking_lot::RwLock; use rand::Rng; -use std::{collections::HashMap, sync::Arc, sync::Weak, time::Duration}; -use std::{ - ops::Deref, - sync::atomic::{self, AtomicUsize}, -}; use tokio_postgres::ReadyForQueryStatus; +use tracing::{debug, info, Span}; +use super::backend::HttpConnError; +use super::conn_pool::ClientInnerRemote; +use crate::auth::backend::ComputeUserInfo; +use crate::context::RequestMonitoring; use crate::control_plane::messages::ColdStartInfo; use crate::metrics::{HttpEndpointPoolsGuard, Metrics}; use crate::usage_metrics::{Ids, MetricCounter, USAGE_METRICS}; -use crate::{ - auth::backend::ComputeUserInfo, context::RequestMonitoring, DbName, EndpointCacheKey, RoleName, -}; - -use super::conn_pool::ClientInnerRemote; -use tracing::info; -use tracing::{debug, Span}; - -use super::backend::HttpConnError; +use crate::{DbName, EndpointCacheKey, RoleName}; #[derive(Debug, Clone)] pub(crate) struct ConnInfo { @@ -482,7 +480,7 @@ impl Client { }) } - pub(crate) fn do_drop(&mut self) -> Option { + pub(crate) fn do_drop(&mut self) -> Option> { let conn_info = self.conn_info.clone(); let client = self .inner diff --git a/proxy/src/serverless/http_conn_pool.rs b/proxy/src/serverless/http_conn_pool.rs index 79bb19328ffb..363e3979761c 100644 --- a/proxy/src/serverless/http_conn_pool.rs +++ b/proxy/src/serverless/http_conn_pool.rs @@ -10,12 +10,11 @@ use rand::Rng; use tokio::net::TcpStream; use tracing::{debug, error, info, info_span, Instrument}; +use super::conn_pool_lib::{ClientInnerExt, ConnInfo}; use crate::context::RequestMonitoring; use crate::control_plane::messages::{ColdStartInfo, MetricsAuxInfo}; use crate::metrics::{HttpEndpointPoolsGuard, Metrics}; use crate::usage_metrics::{Ids, MetricCounter, USAGE_METRICS}; - -use super::conn_pool_lib::{ClientInnerExt, ConnInfo}; use crate::EndpointCacheKey; pub(crate) type Send = http2::SendRequest; diff --git a/proxy/src/serverless/json.rs b/proxy/src/serverless/json.rs index 8c56d317ccc4..569e2da5715a 100644 --- a/proxy/src/serverless/json.rs +++ b/proxy/src/serverless/json.rs @@ -155,10 +155,10 @@ fn pg_text_to_json(pg_value: Option<&str>, pg_type: &Type) -> Result Result { - _pg_array_parse(pg_array, elem_type, false).map(|(v, _)| v) + pg_array_parse_inner(pg_array, elem_type, false).map(|(v, _)| v) } -fn _pg_array_parse( +fn pg_array_parse_inner( pg_array: &str, elem_type: &Type, nested: bool, @@ -211,7 +211,7 @@ fn _pg_array_parse( '{' if !quote => { level += 1; if level > 1 { - let (res, off) = _pg_array_parse(&pg_array[i..], elem_type, true)?; + let (res, off) = pg_array_parse_inner(&pg_array[i..], elem_type, true)?; entries.push(res); for _ in 0..off - 1 { pg_array_chr.next(); diff --git a/proxy/src/serverless/local_conn_pool.rs b/proxy/src/serverless/local_conn_pool.rs index c4fdd00f7859..a01afd28208f 100644 --- a/proxy/src/serverless/local_conn_pool.rs +++ b/proxy/src/serverless/local_conn_pool.rs @@ -25,7 +25,6 @@ use crate::context::RequestMonitoring; use crate::control_plane::messages::{ColdStartInfo, MetricsAuxInfo}; use crate::metrics::Metrics; use crate::usage_metrics::{Ids, MetricCounter, USAGE_METRICS}; - use crate::{DbName, RoleName}; struct ConnPoolEntry { @@ -530,7 +529,7 @@ impl LocalClient { }) } - fn do_drop(&mut self) -> Option { + fn do_drop(&mut self) -> Option> { let conn_info = self.conn_info.clone(); let client = self .inner diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index bb5eb390a6bc..6fbb044669ab 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -38,7 +38,6 @@ use crate::error::{ErrorKind, ReportableError, UserFacingError}; use crate::metrics::{HttpDirection, Metrics}; use crate::proxy::{run_until_cancelled, NeonOptions}; use crate::serverless::backend::HttpConnError; - use crate::usage_metrics::{MetricCounter, MetricCounterRecorder}; use crate::{DbName, RoleName}; diff --git a/proxy/src/usage_metrics.rs b/proxy/src/usage_metrics.rs index c5384c0b0ec0..f944d5aec34b 100644 --- a/proxy/src/usage_metrics.rs +++ b/proxy/src/usage_metrics.rs @@ -375,7 +375,7 @@ pub async fn task_backup( let now = Utc::now(); collect_metrics_backup_iteration( &USAGE_METRICS.backup_endpoints, - &storage, + storage.as_ref(), &hostname, prev, now, @@ -395,7 +395,7 @@ pub async fn task_backup( #[instrument(skip_all)] async fn collect_metrics_backup_iteration( endpoints: &DashMap, FastHasher>, - storage: &Option, + storage: Option<&GenericRemoteStorage>, hostname: &str, prev: DateTime, now: DateTime, @@ -446,7 +446,7 @@ async fn collect_metrics_backup_iteration( } async fn upload_events_chunk( - storage: &Option, + storage: Option<&GenericRemoteStorage>, chunk: EventChunk<'_, Event>, remote_path: &RemotePath, cancel: &CancellationToken, @@ -577,10 +577,10 @@ mod tests { // counter is unregistered assert!(metrics.endpoints.is_empty()); - collect_metrics_backup_iteration(&metrics.backup_endpoints, &None, "foo", now, now, 1000) + collect_metrics_backup_iteration(&metrics.backup_endpoints, None, "foo", now, now, 1000) .await; assert!(!metrics.backup_endpoints.is_empty()); - collect_metrics_backup_iteration(&metrics.backup_endpoints, &None, "foo", now, now, 1000) + collect_metrics_backup_iteration(&metrics.backup_endpoints, None, "foo", now, now, 1000) .await; // backup counter is unregistered after the second iteration assert!(metrics.backup_endpoints.is_empty()); diff --git a/proxy/src/waiters.rs b/proxy/src/waiters.rs index 7e07f6a2affe..330e73f02f92 100644 --- a/proxy/src/waiters.rs +++ b/proxy/src/waiters.rs @@ -73,7 +73,7 @@ struct DropKey<'a, T> { registry: &'a Waiters, } -impl<'a, T> Drop for DropKey<'a, T> { +impl Drop for DropKey<'_, T> { fn drop(&mut self) { self.registry.0.lock().remove(&self.key); } diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 3494b0b764f7..41b9490088fe 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -122,7 +122,7 @@ impl<'a> WriteGuardSharedState<'a> { } } -impl<'a> Deref for WriteGuardSharedState<'a> { +impl Deref for WriteGuardSharedState<'_> { type Target = SharedState; fn deref(&self) -> &Self::Target { @@ -130,13 +130,13 @@ impl<'a> Deref for WriteGuardSharedState<'a> { } } -impl<'a> DerefMut for WriteGuardSharedState<'a> { +impl DerefMut for WriteGuardSharedState<'_> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.guard } } -impl<'a> Drop for WriteGuardSharedState<'a> { +impl Drop for WriteGuardSharedState<'_> { fn drop(&mut self) { let term_flush_lsn = TermLsn::from((self.guard.sk.last_log_term(), self.guard.sk.flush_lsn())); From 24654b8eee8706e8ae98948733a28b56df83536b Mon Sep 17 00:00:00 2001 From: Jere Vaara Date: Fri, 18 Oct 2024 13:25:45 +0300 Subject: [PATCH 10/21] compute_ctl: Add endpoint that allows setting role grants (#9395) This PR introduces a `/grants` endpoint which allows setting specific `privileges` to certain `role` for a certain `schema`. Related to #9344 Together these endpoints will be used to configure JWT extension and set correct usage to its schema to specific roles that will need them. --------- Co-authored-by: Conrad Ludgate --- compute_tools/src/compute.rs | 43 ++++++++++++ compute_tools/src/http/api.rs | 48 ++++++++++++- compute_tools/src/http/openapi_spec.yaml | 89 ++++++++++++++++++++++++ libs/compute_api/src/lib.rs | 1 + libs/compute_api/src/privilege.rs | 35 ++++++++++ libs/compute_api/src/requests.rs | 13 +++- libs/compute_api/src/responses.rs | 13 +++- test_runner/fixtures/endpoint/http.py | 8 +++ test_runner/regress/test_role_grants.py | 41 +++++++++++ 9 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 libs/compute_api/src/privilege.rs create mode 100644 test_runner/regress/test_role_grants.py diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 6aec008f3a36..11fee73f0346 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -15,6 +15,7 @@ use std::time::Instant; use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; +use compute_api::spec::PgIdent; use futures::future::join_all; use futures::stream::FuturesUnordered; use futures::StreamExt; @@ -25,6 +26,7 @@ use tracing::{debug, error, info, instrument, warn}; use utils::id::{TenantId, TimelineId}; use utils::lsn::Lsn; +use compute_api::privilege::Privilege; use compute_api::responses::{ComputeMetrics, ComputeStatus}; use compute_api::spec::{ComputeFeature, ComputeMode, ComputeSpec}; use utils::measured_stream::MeasuredReader; @@ -1373,6 +1375,47 @@ LIMIT 100", download_size } + pub async fn set_role_grants( + &self, + db_name: &PgIdent, + schema_name: &PgIdent, + privileges: &[Privilege], + role_name: &PgIdent, + ) -> Result<()> { + use tokio_postgres::config::Config; + use tokio_postgres::NoTls; + + let mut conf = Config::from_str(self.connstr.as_str()).unwrap(); + conf.dbname(db_name); + + let (db_client, conn) = conf + .connect(NoTls) + .await + .context("Failed to connect to the database")?; + tokio::spawn(conn); + + // TODO: support other types of grants apart from schemas? + let query = format!( + "GRANT {} ON SCHEMA {} TO {}", + privileges + .iter() + // should not be quoted as it's part of the command. + // is already sanitized so it's ok + .map(|p| p.as_str()) + .collect::>() + .join(", "), + // quote the schema and role name as identifiers to sanitize them. + schema_name.pg_quote(), + role_name.pg_quote(), + ); + db_client + .simple_query(&query) + .await + .with_context(|| format!("Failed to execute query: {}", query))?; + + Ok(()) + } + #[tokio::main] pub async fn prepare_preload_libraries( &self, diff --git a/compute_tools/src/http/api.rs b/compute_tools/src/http/api.rs index 79e6158081fe..133ab9f5af49 100644 --- a/compute_tools/src/http/api.rs +++ b/compute_tools/src/http/api.rs @@ -9,8 +9,10 @@ use crate::catalog::SchemaDumpError; use crate::catalog::{get_database_schema, get_dbs_and_roles}; use crate::compute::forward_termination_signal; use crate::compute::{ComputeNode, ComputeState, ParsedSpec}; -use compute_api::requests::ConfigurationRequest; -use compute_api::responses::{ComputeStatus, ComputeStatusResponse, GenericAPIError}; +use compute_api::requests::{ConfigurationRequest, SetRoleGrantsRequest}; +use compute_api::responses::{ + ComputeStatus, ComputeStatusResponse, GenericAPIError, SetRoleGrantsResponse, +}; use anyhow::Result; use hyper::header::CONTENT_TYPE; @@ -165,6 +167,48 @@ async fn routes(req: Request, compute: &Arc) -> Response { + info!("serving /grants POST request"); + let status = compute.get_status(); + if status != ComputeStatus::Running { + let msg = format!( + "invalid compute status for set_role_grants request: {:?}", + status + ); + error!(msg); + return render_json_error(&msg, StatusCode::PRECONDITION_FAILED); + } + + let request = hyper::body::to_bytes(req.into_body()).await.unwrap(); + let request = serde_json::from_slice::(&request).unwrap(); + + let res = compute + .set_role_grants( + &request.database, + &request.schema, + &request.privileges, + &request.role, + ) + .await; + match res { + Ok(()) => render_json(Body::from( + serde_json::to_string(&SetRoleGrantsResponse { + database: request.database, + schema: request.schema, + role: request.role, + privileges: request.privileges, + }) + .unwrap(), + )), + Err(e) => render_json_error( + &format!("could not grant role privileges to the schema: {e}"), + // TODO: can we filter on role/schema not found errors + // and return appropriate error code? + StatusCode::INTERNAL_SERVER_ERROR, + ), + } + } + // get the list of installed extensions // currently only used in python tests // TODO: call it from cplane diff --git a/compute_tools/src/http/openapi_spec.yaml b/compute_tools/src/http/openapi_spec.yaml index e9fa66b323dc..73dbdc3ee9fe 100644 --- a/compute_tools/src/http/openapi_spec.yaml +++ b/compute_tools/src/http/openapi_spec.yaml @@ -127,6 +127,41 @@ paths: schema: $ref: "#/components/schemas/GenericError" + /grants: + post: + tags: + - Grants + summary: Apply grants to the database. + description: "" + operationId: setRoleGrants + requestBody: + description: Grants request. + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/SetRoleGrantsRequest" + responses: + 200: + description: Grants applied. + content: + application/json: + schema: + $ref: "#/components/schemas/SetRoleGrantsResponse" + 412: + description: | + Compute is not in the right state for processing the request. + content: + application/json: + schema: + $ref: "#/components/schemas/GenericError" + 500: + description: Error occurred during grants application. + content: + application/json: + schema: + $ref: "#/components/schemas/GenericError" + /check_writability: post: tags: @@ -427,6 +462,60 @@ components: n_databases: type: integer + SetRoleGrantsRequest: + type: object + required: + - database + - schema + - privileges + - role + properties: + database: + type: string + description: Database name. + example: "neondb" + schema: + type: string + description: Schema name. + example: "public" + privileges: + type: array + items: + type: string + description: List of privileges to set. + example: ["SELECT", "INSERT"] + role: + type: string + description: Role name. + example: "neon" + + SetRoleGrantsResponse: + type: object + required: + - database + - schema + - privileges + - role + properties: + database: + type: string + description: Database name. + example: "neondb" + schema: + type: string + description: Schema name. + example: "public" + privileges: + type: array + items: + type: string + description: List of privileges set. + example: ["SELECT", "INSERT"] + role: + type: string + description: Role name. + example: "neon" + # # Errors # diff --git a/libs/compute_api/src/lib.rs b/libs/compute_api/src/lib.rs index 210a52d089be..f4f3d92fc662 100644 --- a/libs/compute_api/src/lib.rs +++ b/libs/compute_api/src/lib.rs @@ -1,5 +1,6 @@ #![deny(unsafe_code)] #![deny(clippy::undocumented_unsafe_blocks)] +pub mod privilege; pub mod requests; pub mod responses; pub mod spec; diff --git a/libs/compute_api/src/privilege.rs b/libs/compute_api/src/privilege.rs new file mode 100644 index 000000000000..dc0d870946f0 --- /dev/null +++ b/libs/compute_api/src/privilege.rs @@ -0,0 +1,35 @@ +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum Privilege { + Select, + Insert, + Update, + Delete, + Truncate, + References, + Trigger, + Usage, + Create, + Connect, + Temporary, + Execute, +} + +impl Privilege { + pub fn as_str(&self) -> &'static str { + match self { + Privilege::Select => "SELECT", + Privilege::Insert => "INSERT", + Privilege::Update => "UPDATE", + Privilege::Delete => "DELETE", + Privilege::Truncate => "TRUNCATE", + Privilege::References => "REFERENCES", + Privilege::Trigger => "TRIGGER", + Privilege::Usage => "USAGE", + Privilege::Create => "CREATE", + Privilege::Connect => "CONNECT", + Privilege::Temporary => "TEMPORARY", + Privilege::Execute => "EXECUTE", + } + } +} diff --git a/libs/compute_api/src/requests.rs b/libs/compute_api/src/requests.rs index 5896c7dc6593..fbc7577dd9e3 100644 --- a/libs/compute_api/src/requests.rs +++ b/libs/compute_api/src/requests.rs @@ -1,6 +1,9 @@ //! Structs representing the JSON formats used in the compute_ctl's HTTP API. -use crate::spec::ComputeSpec; +use crate::{ + privilege::Privilege, + spec::{ComputeSpec, PgIdent}, +}; use serde::Deserialize; /// Request of the /configure API @@ -12,3 +15,11 @@ use serde::Deserialize; pub struct ConfigurationRequest { pub spec: ComputeSpec, } + +#[derive(Deserialize, Debug)] +pub struct SetRoleGrantsRequest { + pub database: PgIdent, + pub schema: PgIdent, + pub privileges: Vec, + pub role: PgIdent, +} diff --git a/libs/compute_api/src/responses.rs b/libs/compute_api/src/responses.rs index 5023fce0039b..fadf524273e6 100644 --- a/libs/compute_api/src/responses.rs +++ b/libs/compute_api/src/responses.rs @@ -6,7 +6,10 @@ use std::fmt::Display; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize, Serializer}; -use crate::spec::{ComputeSpec, Database, Role}; +use crate::{ + privilege::Privilege, + spec::{ComputeSpec, Database, PgIdent, Role}, +}; #[derive(Serialize, Debug, Deserialize)] pub struct GenericAPIError { @@ -168,3 +171,11 @@ pub struct InstalledExtension { pub struct InstalledExtensions { pub extensions: Vec, } + +#[derive(Clone, Debug, Default, Serialize)] +pub struct SetRoleGrantsResponse { + pub database: PgIdent, + pub schema: PgIdent, + pub privileges: Vec, + pub role: PgIdent, +} diff --git a/test_runner/fixtures/endpoint/http.py b/test_runner/fixtures/endpoint/http.py index 26895df8a6c4..e7b014b4a9dd 100644 --- a/test_runner/fixtures/endpoint/http.py +++ b/test_runner/fixtures/endpoint/http.py @@ -28,3 +28,11 @@ def installed_extensions(self): res = self.get(f"http://localhost:{self.port}/installed_extensions") res.raise_for_status() return res.json() + + def set_role_grants(self, database: str, role: str, schema: str, privileges: list[str]): + res = self.post( + f"http://localhost:{self.port}/grants", + json={"database": database, "schema": schema, "role": role, "privileges": privileges}, + ) + res.raise_for_status() + return res.json() diff --git a/test_runner/regress/test_role_grants.py b/test_runner/regress/test_role_grants.py new file mode 100644 index 000000000000..b2251875f01e --- /dev/null +++ b/test_runner/regress/test_role_grants.py @@ -0,0 +1,41 @@ +import psycopg2 +from fixtures.neon_fixtures import NeonEnv + + +def test_role_grants(neon_simple_env: NeonEnv): + """basic test for the endpoint that grants permissions for a role against a schema""" + + env = neon_simple_env + + env.create_branch("test_role_grants") + + endpoint = env.endpoints.create_start("test_role_grants") + + endpoint.safe_psql("CREATE DATABASE test_role_grants") + endpoint.safe_psql("CREATE SCHEMA IF NOT EXISTS test_schema", dbname="test_role_grants") + endpoint.safe_psql("CREATE ROLE test_role WITH LOGIN", dbname="test_role_grants") + + # confirm we do not yet have access + pg_conn = endpoint.connect(dbname="test_role_grants", user="test_role") + with pg_conn.cursor() as cur: + try: + cur.execute('CREATE TABLE "test_schema"."test_table" (id integer primary key)') + raise ValueError("create table should not succeed") + except psycopg2.errors.InsufficientPrivilege: + pass + except BaseException as e: + raise e + + client = endpoint.http_client() + res = client.set_role_grants( + "test_role_grants", "test_role", "test_schema", ["CREATE", "USAGE"] + ) + + # confirm we have access + with pg_conn.cursor() as cur: + cur.execute('CREATE TABLE "test_schema"."test_table" (id integer primary key)') + cur.execute('INSERT INTO "test_schema"."test_table" (id) VALUES (1)') + cur.execute('SELECT id from "test_schema"."test_table"') + res = cur.fetchall() + + assert res == [(1,)], "select should not succeed" From b7173b1ef05f694f3fa7968dadc4a298ea6d66e8 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 18 Oct 2024 11:29:23 +0100 Subject: [PATCH 11/21] storcon: fix case where we might fail to send compute notifications after two opposite migrations (#9435) ## Problem If we migrate A->B, then B->A, and the notification of A->B fails, then we might have retained state that makes us think "A" is the last state we sent to the compute hook, whereas when we migrate B->A we should really be sending a fresh notification in case our earlier failed notification has actually mutated the remote compute config. Closes: #9417 ## Summary of changes - Add a reproducer for the bug (`test_storage_controller_compute_hook_revert`) - Refactor compute hook code to represent remote state with `ComputeRemoteState` which stores a boolean for whether the compute has fully applied the change as well as the request that the compute accepted. - The actual bug fix: after sending a compute notification, if we got a 423 response then update our ComputeRemoteState to reflect that we have mutated the remote state. This way, when we later try and notify for our historic location, we will properly see that as a change and send the notification. Co-authored-by: Vlad Lazar --- storage_controller/src/compute_hook.rs | 80 ++++++++--- .../regress/test_storage_controller.py | 127 ++++++++++++++++-- 2 files changed, 183 insertions(+), 24 deletions(-) diff --git a/storage_controller/src/compute_hook.rs b/storage_controller/src/compute_hook.rs index bafae1f5515b..b63a322b879f 100644 --- a/storage_controller/src/compute_hook.rs +++ b/storage_controller/src/compute_hook.rs @@ -28,7 +28,7 @@ struct UnshardedComputeHookTenant { node_id: NodeId, // Must hold this lock to send a notification. - send_lock: Arc>>, + send_lock: Arc>>, } struct ShardedComputeHookTenant { stripe_size: ShardStripeSize, @@ -38,7 +38,22 @@ struct ShardedComputeHookTenant { // Must hold this lock to send a notification. The contents represent // the last successfully sent notification, and are used to coalesce multiple // updates by only sending when there is a chance since our last successful send. - send_lock: Arc>>, + send_lock: Arc>>, +} + +/// Represents our knowledge of the compute's state: we can update this when we get a +/// response from a notify API call, which tells us what has been applied. +/// +/// Should be wrapped in an Option<>, as we cannot always know the remote state. +#[derive(PartialEq, Eq, Debug)] +struct ComputeRemoteState { + // The request body which was acked by the compute + request: ComputeHookNotifyRequest, + + // Whether the cplane indicated that the state was applied to running computes, or just + // persisted. In the Neon control plane, this is the difference between a 423 response (meaning + // persisted but not applied), and a 2xx response (both persisted and applied) + applied: bool, } enum ComputeHookTenant { @@ -64,7 +79,7 @@ impl ComputeHookTenant { } } - fn get_send_lock(&self) -> &Arc>> { + fn get_send_lock(&self) -> &Arc>> { match self { Self::Unsharded(unsharded_tenant) => &unsharded_tenant.send_lock, Self::Sharded(sharded_tenant) => &sharded_tenant.send_lock, @@ -188,11 +203,11 @@ enum MaybeSendResult { Transmit( ( ComputeHookNotifyRequest, - tokio::sync::OwnedMutexGuard>, + tokio::sync::OwnedMutexGuard>, ), ), // Something requires sending, but you must wait for a current sender then call again - AwaitLock(Arc>>), + AwaitLock(Arc>>), // Nothing requires sending Noop, } @@ -201,7 +216,7 @@ impl ComputeHookTenant { fn maybe_send( &self, tenant_id: TenantId, - lock: Option>>, + lock: Option>>, ) -> MaybeSendResult { let locked = match lock { Some(already_locked) => already_locked, @@ -257,11 +272,22 @@ impl ComputeHookTenant { tracing::info!("Tenant isn't yet ready to emit a notification"); MaybeSendResult::Noop } - Some(request) if Some(&request) == locked.as_ref() => { - // No change from the last value successfully sent + Some(request) + if Some(&request) == locked.as_ref().map(|s| &s.request) + && locked.as_ref().map(|s| s.applied).unwrap_or(false) => + { + tracing::info!( + "Skipping notification because remote state already matches ({:?})", + &request + ); + // No change from the last value successfully sent, and our state indicates that the last + // value sent was fully applied on the control plane side. MaybeSendResult::Noop } - Some(request) => MaybeSendResult::Transmit((request, locked)), + Some(request) => { + // Our request differs from the last one sent, or the last one sent was not fully applied on the compute side + MaybeSendResult::Transmit((request, locked)) + } } } } @@ -550,10 +576,28 @@ impl ComputeHook { }) }; - if result.is_ok() { - // Before dropping the send lock, stash the request we just sent so that - // subsequent callers can avoid redundantly re-sending the same thing. - *send_lock_guard = Some(request); + match result { + Ok(_) => { + // Before dropping the send lock, stash the request we just sent so that + // subsequent callers can avoid redundantly re-sending the same thing. + *send_lock_guard = Some(ComputeRemoteState { + request, + applied: true, + }); + } + Err(NotifyError::Busy) => { + // Busy result means that the server responded and has stored the new configuration, + // but was not able to fully apply it to the compute + *send_lock_guard = Some(ComputeRemoteState { + request, + applied: false, + }); + } + Err(_) => { + // General error case: we can no longer know the remote state, so clear it. This will result in + // the logic in maybe_send recognizing that we should call the hook again. + *send_lock_guard = None; + } } result } @@ -707,7 +751,10 @@ pub(crate) mod tests { assert!(request.stripe_size.is_none()); // Simulate successful send - *guard = Some(request); + *guard = Some(ComputeRemoteState { + request, + applied: true, + }); drop(guard); // Try asking again: this should be a no-op @@ -750,7 +797,10 @@ pub(crate) mod tests { assert_eq!(request.stripe_size, Some(ShardStripeSize(32768))); // Simulate successful send - *guard = Some(request); + *guard = Some(ComputeRemoteState { + request, + applied: true, + }); drop(guard); Ok(()) diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 1dcc37c4076a..a4e293da9ee2 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -576,6 +576,14 @@ def received_split_notification(): env.storage_controller.consistency_check() +NOTIFY_BLOCKED_LOG = ".*Live migration blocked.*" +NOTIFY_FAILURE_LOGS = [ + ".*Failed to notify compute.*", + ".*Reconcile error.*Cancelled", + ".*Reconcile error.*Control plane tenant busy", +] + + def test_storage_controller_stuck_compute_hook( httpserver: HTTPServer, neon_env_builder: NeonEnvBuilder, @@ -620,15 +628,8 @@ def handler(request: Request): dest_pageserver = env.get_pageserver(dest_ps_id) shard_0_id = TenantShardId(tenant_id, 0, 0) - NOTIFY_BLOCKED_LOG = ".*Live migration blocked.*" - env.storage_controller.allowed_errors.extend( - [ - NOTIFY_BLOCKED_LOG, - ".*Failed to notify compute.*", - ".*Reconcile error.*Cancelled", - ".*Reconcile error.*Control plane tenant busy", - ] - ) + env.storage_controller.allowed_errors.append(NOTIFY_BLOCKED_LOG) + env.storage_controller.allowed_errors.extend(NOTIFY_FAILURE_LOGS) with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: # We expect the controller to hit the 423 (locked) and retry. Migration shouldn't complete until that @@ -719,6 +720,114 @@ def logged_stuck_again(): env.storage_controller.consistency_check() +@run_only_on_default_postgres("this test doesn't start an endpoint") +def test_storage_controller_compute_hook_revert( + httpserver: HTTPServer, + neon_env_builder: NeonEnvBuilder, + httpserver_listen_address, +): + """ + 'revert' in the sense of a migration which gets reversed shortly after, as may happen during + a rolling upgrade. + + This is a reproducer for https://github.com/neondatabase/neon/issues/9417 + + The buggy behavior was that when the compute hook gave us errors, we assumed our last successfully + sent state was still in effect, so when migrating back to the original pageserver we didn't bother + notifying of that. This is wrong because even a failed request might mutate the state on the server. + """ + + # We will run two pageserver to migrate and check that the storage controller sends notifications + # when migrating. + neon_env_builder.num_pageservers = 2 + (host, port) = httpserver_listen_address + neon_env_builder.control_plane_compute_hook_api = f"http://{host}:{port}/notify" + + # Set up fake HTTP notify endpoint + notifications = [] + + handle_params = {"status": 200} + + def handler(request: Request): + status = handle_params["status"] + log.info(f"Notify request[{status}]: {request}") + notifications.append(request.json) + return Response(status=status) + + httpserver.expect_request("/notify", method="PUT").respond_with_handler(handler) + + # Start running + env = neon_env_builder.init_start(initial_tenant_conf={"lsn_lease_length": "0s"}) + tenant_id = env.initial_tenant + tenant_shard_id = TenantShardId(tenant_id, 0, 0) + + pageserver_a = env.get_tenant_pageserver(tenant_id) + pageserver_b = [p for p in env.pageservers if p.id != pageserver_a.id][0] + + def notified_ps(ps_id: int) -> None: + latest = notifications[-1] + log.info(f"Waiting for {ps_id}, have {latest}") + assert latest is not None + assert latest["shards"] is not None + assert latest["shards"][0]["node_id"] == ps_id + + wait_until(30, 1, lambda: notified_ps(pageserver_a.id)) + + env.storage_controller.allowed_errors.append(NOTIFY_BLOCKED_LOG) + env.storage_controller.allowed_errors.extend(NOTIFY_FAILURE_LOGS) + + # Migrate A -> B, and make notifications fail while this is happening + handle_params["status"] = 423 + + with pytest.raises(StorageControllerApiException, match="Timeout waiting for shard"): + # We expect the controller to give us an error because its reconciliation timed out + # waiting for the compute hook. + env.storage_controller.tenant_shard_migrate(tenant_shard_id, pageserver_b.id) + + # Although the migration API failed, the hook should still see pageserver B (it remembers what + # was posted even when returning an error code) + wait_until(30, 1, lambda: notified_ps(pageserver_b.id)) + + # Although the migration API failed, the tenant should still have moved to the right pageserver + assert len(pageserver_b.http_client().tenant_list()) == 1 + + # Before we clear the failure on the migration hook, we need the controller to give up + # trying to notify about B -- the bug case we're reproducing is when the controller + # _never_ successfully notified for B, then tries to notify for A. + # + # The controller will give up notifying if the origin of a migration becomes unavailable. + pageserver_a.stop() + + # Preempt heartbeats for a faster test + env.storage_controller.node_configure(pageserver_a.id, {"availability": "Offline"}) + + def logged_giving_up(): + env.storage_controller.assert_log_contains(".*Giving up on compute notification.*") + + wait_until(30, 1, logged_giving_up) + + pageserver_a.start() + + # Preempt heartbeats for determinism + env.storage_controller.node_configure(pageserver_a.id, {"availability": "Active"}) + # Starting node will prompt a reconcile to clean up old AttachedStale location, for a deterministic test + # we want that complete before we start our migration. Tolerate failure because our compute hook is + # still configured to fail + try: + env.storage_controller.reconcile_all() + except StorageControllerApiException as e: + # This exception _might_ be raised: it depends if our reconcile_all hit the on-node-activation + # Reconciler lifetime or ran after it already completed. + log.info(f"Expected error from reconcile_all: {e}") + + # Migrate B -> A, with a working compute hook: the controller should notify the hook because the + # last update it made that was acked (423) by the compute was for node B. + handle_params["status"] = 200 + env.storage_controller.tenant_shard_migrate(tenant_shard_id, pageserver_a.id) + + wait_until(30, 1, lambda: notified_ps(pageserver_a.id)) + + def test_storage_controller_debug_apis(neon_env_builder: NeonEnvBuilder): """ Verify that occasional-use debug APIs work as expected. This is a lightweight test From 98fee7a97d68db55049583d403dcb21755bc4db5 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 18 Oct 2024 13:31:14 +0300 Subject: [PATCH 12/21] Increase shared_buffers in test_subscriber_synchronous_commit. (#9427) Might make the test less flaky. --- test_runner/regress/test_logical_replication.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py index 87991eadf188..c26bf058e258 100644 --- a/test_runner/regress/test_logical_replication.py +++ b/test_runner/regress/test_logical_replication.py @@ -558,10 +558,10 @@ def check_caughtup(): return publisher_flush_lsn -# Test that subscriber takes into account quorum committed flush_lsn in -# flush_lsn reporting to publisher. Without this, it may ack too far, losing -# data on restart because publisher advances START_REPLICATION position to the -# confirmed_flush_lsn of the slot. +# Test that neon subscriber takes into account quorum committed flush_lsn in +# flush_lsn reporting to publisher. Without this, subscriber may ack too far, +# losing data on restart because publisher implicitly advances positition given +# in START_REPLICATION to the confirmed_flush_lsn of the slot. def test_subscriber_synchronous_commit(neon_simple_env: NeonEnv, vanilla_pg): env = neon_simple_env # use vanilla as publisher to allow writes on it when safekeeper is down @@ -578,7 +578,10 @@ def test_subscriber_synchronous_commit(neon_simple_env: NeonEnv, vanilla_pg): vanilla_pg.safe_psql("create extension neon;") env.create_branch("subscriber") - sub = env.endpoints.create("subscriber") + # We want all data to fit into shared_buffers because later we stop + # safekeeper and insert more; this shouldn't cause page requests as they + # will be stuck. + sub = env.endpoints.create("subscriber", config_lines=["shared_buffers=128MB"]) sub.start() with vanilla_pg.cursor() as pcur: From 15fecffe6ba400693619c6a022ed6205769a61ae Mon Sep 17 00:00:00 2001 From: Folke Behrens Date: Fri, 18 Oct 2024 12:42:41 +0200 Subject: [PATCH 13/21] Update ruff to much newer version (#9433) Includes a multidict patch release to fix build with newer cpython. --- poetry.lock | 207 ++++++++++-------- pyproject.toml | 2 +- test_runner/fixtures/neon_cli.py | 4 +- test_runner/fixtures/neon_fixtures.py | 18 +- test_runner/fixtures/utils.py | 2 +- .../performance/test_logical_replication.py | 14 +- .../performance/test_physical_replication.py | 12 +- .../regress/test_download_extensions.py | 2 +- test_runner/regress/test_next_xid.py | 4 +- test_runner/regress/test_timeline_delete.py | 2 +- 10 files changed, 145 insertions(+), 122 deletions(-) diff --git a/poetry.lock b/poetry.lock index 00fe2505c9a1..e307b873f364 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -1758,85 +1758,101 @@ tests = ["pytest (>=4.6)"] [[package]] name = "multidict" -version = "6.0.4" +version = "6.0.5" description = "multidict implementation" optional = false python-versions = ">=3.7" files = [ - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, - {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, - {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, - {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, - {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, - {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, - {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, - {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, - {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, - {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, - {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, - {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, - {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, ] [[package]] @@ -2766,28 +2782,29 @@ six = "*" [[package]] name = "ruff" -version = "0.2.2" +version = "0.7.0" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.2.2-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:0a9efb032855ffb3c21f6405751d5e147b0c6b631e3ca3f6b20f917572b97eb6"}, - {file = "ruff-0.2.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d450b7fbff85913f866a5384d8912710936e2b96da74541c82c1b458472ddb39"}, - {file = "ruff-0.2.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecd46e3106850a5c26aee114e562c329f9a1fbe9e4821b008c4404f64ff9ce73"}, - {file = "ruff-0.2.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e22676a5b875bd72acd3d11d5fa9075d3a5f53b877fe7b4793e4673499318ba"}, - {file = "ruff-0.2.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1695700d1e25a99d28f7a1636d85bafcc5030bba9d0578c0781ba1790dbcf51c"}, - {file = "ruff-0.2.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:b0c232af3d0bd8f521806223723456ffebf8e323bd1e4e82b0befb20ba18388e"}, - {file = "ruff-0.2.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f63d96494eeec2fc70d909393bcd76c69f35334cdbd9e20d089fb3f0640216ca"}, - {file = "ruff-0.2.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a61ea0ff048e06de273b2e45bd72629f470f5da8f71daf09fe481278b175001"}, - {file = "ruff-0.2.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1439c8f407e4f356470e54cdecdca1bd5439a0673792dbe34a2b0a551a2fe3"}, - {file = "ruff-0.2.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:940de32dc8853eba0f67f7198b3e79bc6ba95c2edbfdfac2144c8235114d6726"}, - {file = "ruff-0.2.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0c126da55c38dd917621552ab430213bdb3273bb10ddb67bc4b761989210eb6e"}, - {file = "ruff-0.2.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3b65494f7e4bed2e74110dac1f0d17dc8e1f42faaa784e7c58a98e335ec83d7e"}, - {file = "ruff-0.2.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1ec49be4fe6ddac0503833f3ed8930528e26d1e60ad35c2446da372d16651ce9"}, - {file = "ruff-0.2.2-py3-none-win32.whl", hash = "sha256:d920499b576f6c68295bc04e7b17b6544d9d05f196bb3aac4358792ef6f34325"}, - {file = "ruff-0.2.2-py3-none-win_amd64.whl", hash = "sha256:cc9a91ae137d687f43a44c900e5d95e9617cb37d4c989e462980ba27039d239d"}, - {file = "ruff-0.2.2-py3-none-win_arm64.whl", hash = "sha256:c9d15fc41e6054bfc7200478720570078f0b41c9ae4f010bcc16bd6f4d1aacdd"}, - {file = "ruff-0.2.2.tar.gz", hash = "sha256:e62ed7f36b3068a30ba39193a14274cd706bc486fad521276458022f7bccb31d"}, + {file = "ruff-0.7.0-py3-none-linux_armv6l.whl", hash = "sha256:0cdf20c2b6ff98e37df47b2b0bd3a34aaa155f59a11182c1303cce79be715628"}, + {file = "ruff-0.7.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:496494d350c7fdeb36ca4ef1c9f21d80d182423718782222c29b3e72b3512737"}, + {file = "ruff-0.7.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:214b88498684e20b6b2b8852c01d50f0651f3cc6118dfa113b4def9f14faaf06"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:630fce3fefe9844e91ea5bbf7ceadab4f9981f42b704fae011bb8efcaf5d84be"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:211d877674e9373d4bb0f1c80f97a0201c61bcd1e9d045b6e9726adc42c156aa"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:194d6c46c98c73949a106425ed40a576f52291c12bc21399eb8f13a0f7073495"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:82c2579b82b9973a110fab281860403b397c08c403de92de19568f32f7178598"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9af971fe85dcd5eaed8f585ddbc6bdbe8c217fb8fcf510ea6bca5bdfff56040e"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b641c7f16939b7d24b7bfc0be4102c56562a18281f84f635604e8a6989948914"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d71672336e46b34e0c90a790afeac8a31954fd42872c1f6adaea1dff76fd44f9"}, + {file = "ruff-0.7.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ab7d98c7eed355166f367597e513a6c82408df4181a937628dbec79abb2a1fe4"}, + {file = "ruff-0.7.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1eb54986f770f49edb14f71d33312d79e00e629a57387382200b1ef12d6a4ef9"}, + {file = "ruff-0.7.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:dc452ba6f2bb9cf8726a84aa877061a2462afe9ae0ea1d411c53d226661c601d"}, + {file = "ruff-0.7.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:4b406c2dce5be9bad59f2de26139a86017a517e6bcd2688da515481c05a2cb11"}, + {file = "ruff-0.7.0-py3-none-win32.whl", hash = "sha256:f6c968509f767776f524a8430426539587d5ec5c662f6addb6aa25bc2e8195ec"}, + {file = "ruff-0.7.0-py3-none-win_amd64.whl", hash = "sha256:ff4aabfbaaba880e85d394603b9e75d32b0693152e16fa659a3064a85df7fce2"}, + {file = "ruff-0.7.0-py3-none-win_arm64.whl", hash = "sha256:10842f69c245e78d6adec7e1db0a7d9ddc2fff0621d730e61657b64fa36f207e"}, + {file = "ruff-0.7.0.tar.gz", hash = "sha256:47a86360cf62d9cd53ebfb0b5eb0e882193fc191c6d717e8bef4462bc3b9ea2b"}, ] [[package]] @@ -3389,4 +3406,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "9055b73352f1534f664cd8af6ebf8d93cf3bf857f115756f312ff2e3ae1bbbc1" +content-hash = "f52632571e34b0e51b059c280c35d6ff6f69f6a8c9586caca78282baf635be91" diff --git a/pyproject.toml b/pyproject.toml index 9cd315bb9629..862ed4963832 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ kafka-python = "^2.0.2" [tool.poetry.group.dev.dependencies] mypy = "==1.3.0" -ruff = "^0.2.2" +ruff = "^0.7.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/test_runner/fixtures/neon_cli.py b/test_runner/fixtures/neon_cli.py index 0d3dcd16718a..1b2767e296c2 100644 --- a/test_runner/fixtures/neon_cli.py +++ b/test_runner/fixtures/neon_cli.py @@ -1,6 +1,5 @@ from __future__ import annotations -import abc import json import os import re @@ -30,7 +29,8 @@ T = TypeVar("T") -class AbstractNeonCli(abc.ABC): +# Used to be an ABC. abc.ABC removed due to linter without name change. +class AbstractNeonCli: """ A typed wrapper around an arbitrary Neon CLI tool. Supports a way to run arbitrary command directly via CLI. diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index a313ac2ed360..3cd8019e3278 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -386,9 +386,9 @@ def __init__( self.pageserver_virtual_file_io_engine: Optional[str] = pageserver_virtual_file_io_engine - self.pageserver_default_tenant_config_compaction_algorithm: Optional[ - dict[str, Any] - ] = pageserver_default_tenant_config_compaction_algorithm + self.pageserver_default_tenant_config_compaction_algorithm: Optional[dict[str, Any]] = ( + pageserver_default_tenant_config_compaction_algorithm + ) if self.pageserver_default_tenant_config_compaction_algorithm is not None: log.debug( f"Overriding pageserver default compaction algorithm to {self.pageserver_default_tenant_config_compaction_algorithm}" @@ -1062,9 +1062,9 @@ def __init__(self, config: NeonEnvBuilder): ps_cfg["virtual_file_io_engine"] = self.pageserver_virtual_file_io_engine if config.pageserver_default_tenant_config_compaction_algorithm is not None: tenant_config = ps_cfg.setdefault("tenant_config", {}) - tenant_config[ - "compaction_algorithm" - ] = config.pageserver_default_tenant_config_compaction_algorithm + tenant_config["compaction_algorithm"] = ( + config.pageserver_default_tenant_config_compaction_algorithm + ) if self.pageserver_remote_storage is not None: ps_cfg["remote_storage"] = remote_storage_to_toml_dict( @@ -1108,9 +1108,9 @@ def __init__(self, config: NeonEnvBuilder): if config.auth_enabled: sk_cfg["auth_enabled"] = True if self.safekeepers_remote_storage is not None: - sk_cfg[ - "remote_storage" - ] = self.safekeepers_remote_storage.to_toml_inline_table().strip() + sk_cfg["remote_storage"] = ( + self.safekeepers_remote_storage.to_toml_inline_table().strip() + ) self.safekeepers.append( Safekeeper(env=self, id=id, port=port, extra_opts=config.safekeeper_extra_opts) ) diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index 76575d330c5f..7ca6b3dd1c5f 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -417,7 +417,7 @@ def wait_until( time.sleep(interval) continue return res - raise Exception("timed out while waiting for %s" % func) from last_exception + raise Exception(f"timed out while waiting for {func}") from last_exception def assert_eq(a, b) -> None: diff --git a/test_runner/performance/test_logical_replication.py b/test_runner/performance/test_logical_replication.py index dbf94a2cf52c..815d186ab939 100644 --- a/test_runner/performance/test_logical_replication.py +++ b/test_runner/performance/test_logical_replication.py @@ -144,9 +144,10 @@ def test_subscriber_lag( check_pgbench_still_running(pub_workload, "pub") check_pgbench_still_running(sub_workload, "sub") - with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( - sub_connstr - ) as sub_conn: + with ( + psycopg2.connect(pub_connstr) as pub_conn, + psycopg2.connect(sub_connstr) as sub_conn, + ): with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: lag = measure_logical_replication_lag(sub_cur, pub_cur) @@ -242,9 +243,10 @@ def test_publisher_restart( ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env, ) - with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( - sub_connstr - ) as sub_conn: + with ( + psycopg2.connect(pub_connstr) as pub_conn, + psycopg2.connect(sub_connstr) as sub_conn, + ): with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: lag = measure_logical_replication_lag(sub_cur, pub_cur) diff --git a/test_runner/performance/test_physical_replication.py b/test_runner/performance/test_physical_replication.py index 14b527accacf..8b368977df04 100644 --- a/test_runner/performance/test_physical_replication.py +++ b/test_runner/performance/test_physical_replication.py @@ -102,10 +102,14 @@ def test_ro_replica_lag( check_pgbench_still_running(master_workload) check_pgbench_still_running(replica_workload) time.sleep(sync_interval_min * 60) - with psycopg2.connect(master_connstr) as conn_master, psycopg2.connect( - replica_connstr - ) as conn_replica: - with conn_master.cursor() as cur_master, conn_replica.cursor() as cur_replica: + with ( + psycopg2.connect(master_connstr) as conn_master, + psycopg2.connect(replica_connstr) as conn_replica, + ): + with ( + conn_master.cursor() as cur_master, + conn_replica.cursor() as cur_replica, + ): lag = measure_replication_lag(cur_master, cur_replica) log.info(f"Replica lagged behind master by {lag} seconds") zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) diff --git a/test_runner/regress/test_download_extensions.py b/test_runner/regress/test_download_extensions.py index 04916a6b6f10..0134f80769ae 100644 --- a/test_runner/regress/test_download_extensions.py +++ b/test_runner/regress/test_download_extensions.py @@ -74,7 +74,7 @@ def endpoint_handler_build_tag(request: Request) -> Response: mimetype="application/octet-stream", headers=[ ("Content-Length", str(file_size)), - ("Content-Disposition", 'attachment; filename="%s"' % file_name), + ("Content-Disposition", f'attachment; filename="{file_name}"'), ], direct_passthrough=True, ) diff --git a/test_runner/regress/test_next_xid.py b/test_runner/regress/test_next_xid.py index 980f6b5694c5..db8da51125e8 100644 --- a/test_runner/regress/test_next_xid.py +++ b/test_runner/regress/test_next_xid.py @@ -254,13 +254,13 @@ def advance_multixid_to( # missing. That's OK for our purposes. Autovacuum will print some warnings about the # missing segments, but will clean it up by truncating the SLRUs up to the new value, # closing the gap. - segname = "%04X" % MultiXactIdToOffsetSegment(next_multi_xid) + segname = f"{MultiXactIdToOffsetSegment(next_multi_xid):04X}" log.info(f"Creating dummy segment pg_multixact/offsets/{segname}") with open(vanilla_pg.pgdatadir / "pg_multixact" / "offsets" / segname, "w") as of: of.write("\0" * SLRU_PAGES_PER_SEGMENT * BLCKSZ) of.flush() - segname = "%04X" % MXOffsetToMemberSegment(next_multi_offset) + segname = f"{MXOffsetToMemberSegment(next_multi_offset):04X}" log.info(f"Creating dummy segment pg_multixact/members/{segname}") with open(vanilla_pg.pgdatadir / "pg_multixact" / "members" / segname, "w") as of: of.write("\0" * SLRU_PAGES_PER_SEGMENT * BLCKSZ) diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py index 306f22acf91f..155709e1066d 100644 --- a/test_runner/regress/test_timeline_delete.py +++ b/test_runner/regress/test_timeline_delete.py @@ -649,7 +649,7 @@ def test_timeline_delete_works_for_remote_smoke( env = neon_env_builder.init_start() ps_http = env.pageserver.http_client() - pg = env.endpoints.create_start("main") + env.endpoints.create_start("main") tenant_id = env.initial_tenant timeline_id = env.initial_timeline From 3532ae76ef3a91131aee1f203a133c4d5e32b57a Mon Sep 17 00:00:00 2001 From: Jere Vaara Date: Fri, 18 Oct 2024 15:07:36 +0300 Subject: [PATCH 14/21] compute_ctl: Add endpoint that allows extensions to be installed (#9344) Adds endpoint to install extensions: **POST** `/extensions` ``` {"extension":"pg_sessions_jwt","database":"neondb","version":"1.0.0"} ``` Will be used by `local-proxy`. Example, for the JWT authentication to work the database needs to have the pg_session_jwt extension and also to enable JWT to work in RLS policies. --------- Co-authored-by: Conrad Ludgate --- compute_tools/src/compute.rs | 52 +++++++++++++++++- compute_tools/src/http/api.rs | 37 ++++++++++++- compute_tools/src/http/openapi_spec.yaml | 69 +++++++++++++++++++++++- libs/compute_api/src/requests.rs | 10 +++- libs/compute_api/src/responses.rs | 7 ++- libs/compute_api/src/spec.rs | 3 ++ test_runner/fixtures/endpoint/http.py | 10 ++++ test_runner/regress/test_extensions.py | 50 +++++++++++++++++ 8 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 test_runner/regress/test_extensions.py diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 11fee73f0346..c9dd4dcfc59e 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -28,7 +28,7 @@ use utils::lsn::Lsn; use compute_api::privilege::Privilege; use compute_api::responses::{ComputeMetrics, ComputeStatus}; -use compute_api::spec::{ComputeFeature, ComputeMode, ComputeSpec}; +use compute_api::spec::{ComputeFeature, ComputeMode, ComputeSpec, ExtVersion}; use utils::measured_stream::MeasuredReader; use nix::sys::signal::{kill, Signal}; @@ -1416,6 +1416,56 @@ LIMIT 100", Ok(()) } + pub async fn install_extension( + &self, + ext_name: &PgIdent, + db_name: &PgIdent, + ext_version: ExtVersion, + ) -> Result { + use tokio_postgres::config::Config; + use tokio_postgres::NoTls; + + let mut conf = Config::from_str(self.connstr.as_str()).unwrap(); + conf.dbname(db_name); + + let (db_client, conn) = conf + .connect(NoTls) + .await + .context("Failed to connect to the database")?; + tokio::spawn(conn); + + let version_query = "SELECT extversion FROM pg_extension WHERE extname = $1"; + let version: Option = db_client + .query_opt(version_query, &[&ext_name]) + .await + .with_context(|| format!("Failed to execute query: {}", version_query))? + .map(|row| row.get(0)); + + // sanitize the inputs as postgres idents. + let ext_name: String = ext_name.pg_quote(); + let quoted_version: String = ext_version.pg_quote(); + + if let Some(installed_version) = version { + if installed_version == ext_version { + return Ok(installed_version); + } + let query = format!("ALTER EXTENSION {ext_name} UPDATE TO {quoted_version}"); + db_client + .simple_query(&query) + .await + .with_context(|| format!("Failed to execute query: {}", query))?; + } else { + let query = + format!("CREATE EXTENSION IF NOT EXISTS {ext_name} WITH VERSION {quoted_version}"); + db_client + .simple_query(&query) + .await + .with_context(|| format!("Failed to execute query: {}", query))?; + } + + Ok(ext_version) + } + #[tokio::main] pub async fn prepare_preload_libraries( &self, diff --git a/compute_tools/src/http/api.rs b/compute_tools/src/http/api.rs index 133ab9f5af49..af35f71bf2d9 100644 --- a/compute_tools/src/http/api.rs +++ b/compute_tools/src/http/api.rs @@ -9,9 +9,10 @@ use crate::catalog::SchemaDumpError; use crate::catalog::{get_database_schema, get_dbs_and_roles}; use crate::compute::forward_termination_signal; use crate::compute::{ComputeNode, ComputeState, ParsedSpec}; -use compute_api::requests::{ConfigurationRequest, SetRoleGrantsRequest}; +use compute_api::requests::{ConfigurationRequest, ExtensionInstallRequest, SetRoleGrantsRequest}; use compute_api::responses::{ - ComputeStatus, ComputeStatusResponse, GenericAPIError, SetRoleGrantsResponse, + ComputeStatus, ComputeStatusResponse, ExtensionInstallResult, GenericAPIError, + SetRoleGrantsResponse, }; use anyhow::Result; @@ -100,6 +101,38 @@ async fn routes(req: Request, compute: &Arc) -> Response { + info!("serving /extensions POST request"); + let status = compute.get_status(); + if status != ComputeStatus::Running { + let msg = format!( + "invalid compute status for extensions request: {:?}", + status + ); + error!(msg); + return render_json_error(&msg, StatusCode::PRECONDITION_FAILED); + } + + let request = hyper::body::to_bytes(req.into_body()).await.unwrap(); + let request = serde_json::from_slice::(&request).unwrap(); + let res = compute + .install_extension(&request.extension, &request.database, request.version) + .await; + match res { + Ok(version) => render_json(Body::from( + serde_json::to_string(&ExtensionInstallResult { + extension: request.extension, + version, + }) + .unwrap(), + )), + Err(e) => { + error!("install_extension failed: {}", e); + render_json_error(&e.to_string(), StatusCode::INTERNAL_SERVER_ERROR) + } + } + } + (&Method::GET, "/info") => { let num_cpus = num_cpus::get_physical(); info!("serving /info GET request. num_cpus: {}", num_cpus); diff --git a/compute_tools/src/http/openapi_spec.yaml b/compute_tools/src/http/openapi_spec.yaml index 73dbdc3ee9fe..11eee6ccfd44 100644 --- a/compute_tools/src/http/openapi_spec.yaml +++ b/compute_tools/src/http/openapi_spec.yaml @@ -179,6 +179,41 @@ paths: description: Error text or 'true' if check passed. example: "true" + /extensions: + post: + tags: + - Extensions + summary: Install extension if possible. + description: "" + operationId: installExtension + requestBody: + description: Extension name and database to install it to. + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ExtensionInstallRequest" + responses: + 200: + description: Result from extension installation + content: + application/json: + schema: + $ref: "#/components/schemas/ExtensionInstallResult" + 412: + description: | + Compute is in the wrong state for processing the request. + content: + application/json: + schema: + $ref: "#/components/schemas/GenericError" + 500: + description: Error during extension installation. + content: + application/json: + schema: + $ref: "#/components/schemas/GenericError" + /configure: post: tags: @@ -404,7 +439,7 @@ components: moment, when spec was received. example: "2022-10-12T07:20:50.52Z" status: - $ref: '#/components/schemas/ComputeStatus' + $ref: "#/components/schemas/ComputeStatus" last_active: type: string description: | @@ -444,6 +479,38 @@ components: - configuration example: running + ExtensionInstallRequest: + type: object + required: + - extension + - database + - version + properties: + extension: + type: string + description: Extension name. + example: "pg_session_jwt" + version: + type: string + description: Version of the extension. + example: "1.0.0" + database: + type: string + description: Database name. + example: "neondb" + + ExtensionInstallResult: + type: object + properties: + extension: + description: Name of the extension. + type: string + example: "pg_session_jwt" + version: + description: Version of the extension. + type: string + example: "1.0.0" + InstalledExtensions: type: object properties: diff --git a/libs/compute_api/src/requests.rs b/libs/compute_api/src/requests.rs index fbc7577dd9e3..fc3757d9814f 100644 --- a/libs/compute_api/src/requests.rs +++ b/libs/compute_api/src/requests.rs @@ -1,8 +1,7 @@ //! Structs representing the JSON formats used in the compute_ctl's HTTP API. - use crate::{ privilege::Privilege, - spec::{ComputeSpec, PgIdent}, + spec::{ComputeSpec, ExtVersion, PgIdent}, }; use serde::Deserialize; @@ -16,6 +15,13 @@ pub struct ConfigurationRequest { pub spec: ComputeSpec, } +#[derive(Deserialize, Debug)] +pub struct ExtensionInstallRequest { + pub extension: PgIdent, + pub database: PgIdent, + pub version: ExtVersion, +} + #[derive(Deserialize, Debug)] pub struct SetRoleGrantsRequest { pub database: PgIdent, diff --git a/libs/compute_api/src/responses.rs b/libs/compute_api/src/responses.rs index fadf524273e6..79234be72094 100644 --- a/libs/compute_api/src/responses.rs +++ b/libs/compute_api/src/responses.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize, Serializer}; use crate::{ privilege::Privilege, - spec::{ComputeSpec, Database, PgIdent, Role}, + spec::{ComputeSpec, Database, ExtVersion, PgIdent, Role}, }; #[derive(Serialize, Debug, Deserialize)] @@ -172,6 +172,11 @@ pub struct InstalledExtensions { pub extensions: Vec, } +#[derive(Clone, Debug, Default, Serialize)] +pub struct ExtensionInstallResult { + pub extension: PgIdent, + pub version: ExtVersion, +} #[derive(Clone, Debug, Default, Serialize)] pub struct SetRoleGrantsResponse { pub database: PgIdent, diff --git a/libs/compute_api/src/spec.rs b/libs/compute_api/src/spec.rs index 5903db70550d..8a447563dcf2 100644 --- a/libs/compute_api/src/spec.rs +++ b/libs/compute_api/src/spec.rs @@ -16,6 +16,9 @@ use remote_storage::RemotePath; /// intended to be used for DB / role names. pub type PgIdent = String; +/// String type alias representing Postgres extension version +pub type ExtVersion = String; + /// Cluster spec or configuration represented as an optional number of /// delta operations + final cluster state description. #[derive(Clone, Debug, Default, Deserialize, Serialize)] diff --git a/test_runner/fixtures/endpoint/http.py b/test_runner/fixtures/endpoint/http.py index e7b014b4a9dd..ea8291c1e07f 100644 --- a/test_runner/fixtures/endpoint/http.py +++ b/test_runner/fixtures/endpoint/http.py @@ -29,6 +29,16 @@ def installed_extensions(self): res.raise_for_status() return res.json() + def extensions(self, extension: str, version: str, database: str): + body = { + "extension": extension, + "version": version, + "database": database, + } + res = self.post(f"http://localhost:{self.port}/extensions", json=body) + res.raise_for_status() + return res.json() + def set_role_grants(self, database: str, role: str, schema: str, privileges: list[str]): res = self.post( f"http://localhost:{self.port}/grants", diff --git a/test_runner/regress/test_extensions.py b/test_runner/regress/test_extensions.py new file mode 100644 index 000000000000..100fd4b04887 --- /dev/null +++ b/test_runner/regress/test_extensions.py @@ -0,0 +1,50 @@ +from logging import info + +from fixtures.neon_fixtures import NeonEnv + + +def test_extensions(neon_simple_env: NeonEnv): + """basic test for the extensions endpoint testing installing extensions""" + + env = neon_simple_env + + env.create_branch("test_extensions") + + endpoint = env.endpoints.create_start("test_extensions") + extension = "neon_test_utils" + database = "test_extensions" + + endpoint.safe_psql("CREATE DATABASE test_extensions") + + with endpoint.connect(dbname=database) as pg_conn: + with pg_conn.cursor() as cur: + cur.execute( + "SELECT default_version FROM pg_available_extensions WHERE name = 'neon_test_utils'" + ) + res = cur.fetchone() + assert res is not None + version = res[0] + + with pg_conn.cursor() as cur: + cur.execute( + "SELECT extname, extversion FROM pg_extension WHERE extname = 'neon_test_utils'", + ) + res = cur.fetchone() + assert not res, "The 'neon_test_utils' extension is installed" + + client = endpoint.http_client() + install_res = client.extensions(extension, version, database) + + info("Extension install result: %s", res) + assert install_res["extension"] == extension and install_res["version"] == version + + with endpoint.connect(dbname=database) as pg_conn: + with pg_conn.cursor() as cur: + cur.execute( + "SELECT extname, extversion FROM pg_extension WHERE extname = 'neon_test_utils'", + ) + res = cur.fetchone() + assert res is not None + (db_extension_name, db_extension_version) = res + + assert db_extension_name == extension and db_extension_version == version From fecff15f18f00a692ff234106b064d1693cc5441 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 18 Oct 2024 15:31:50 +0300 Subject: [PATCH 15/21] walproposer: immediately exit if sync-safekeepers collected 0/0. (#9442) Otherwise term history starting with 0/0 is streamed to safekeepers. ref https://github.com/neondatabase/neon/issues/9434 --- pgxn/neon/walproposer.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index a3f33cb261ff..d2a6104c74ca 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -841,6 +841,23 @@ HandleElectedProposer(WalProposer *wp) wp_log(FATAL, "failed to download WAL for logical replicaiton"); } + /* + * Zero propEpochStartLsn means majority of safekeepers doesn't have any + * WAL, timeline was just created. Compute bumps it to basebackup LSN, + * otherwise we must be sync-safekeepers and we have nothing to do then. + * + * Proceeding is not only pointless but harmful, because we'd give + * safekeepers term history starting with 0/0. These hacks will go away once + * we disable implicit timeline creation on safekeepers and create it with + * non zero LSN from the start. + */ + if (wp->propEpochStartLsn == InvalidXLogRecPtr) + { + Assert(wp->config->syncSafekeepers); + wp_log(LOG, "elected with zero propEpochStartLsn in sync-safekeepers, exiting"); + wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); + } + if (wp->truncateLsn == wp->propEpochStartLsn && wp->config->syncSafekeepers) { /* Sync is not needed: just exit */ From ec6d3422a5a7b6f537b029d7c3e70b7a60f99e0c Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Fri, 18 Oct 2024 13:38:59 +0100 Subject: [PATCH 16/21] pageserver: disconnect when asking client to reconnect (#9390) ## Problem Consider the following sequence of events: 1. Shard location gets downgraded to secondary while there's a libpq connection in pagestream mode from the compute 2. There's no active tenant, so we return `QueryError::Reconnect` from `PageServerHandler::handle_get_page_at_lsn_request`. 3. Error bubbles up to `PostgresBackendIO::process_message`, bailing us out of pagestream mode. 4. We instruct the client to reconnnect, but continue serving the libpq connection. The client isn't yet aware of the request to reconnect and believes it is still in pagestream mode. Pageserver fails to deserialize get page requests wrapped in `CopyData` since it's not in pagestream mode. ## Summary of Changes When we wish to instruct the client to reconnect, also disconnect from the server side after flushing the error. Closes https://github.com/neondatabase/cloud/issues/17336 --- libs/postgres_backend/src/lib.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 9d274b25e6c3..7419798a60a3 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -738,6 +738,20 @@ impl PostgresBackend { QueryError::SimulatedConnectionError => { return Err(QueryError::SimulatedConnectionError) } + err @ QueryError::Reconnect => { + // Instruct the client to reconnect, stop processing messages + // from this libpq connection and, finally, disconnect from the + // server side (returning an Err achieves the later). + // + // Note the flushing is done by the caller. + let reconnect_error = short_error(&err); + self.write_message_noflush(&BeMessage::ErrorResponse( + &reconnect_error, + Some(err.pg_error_code()), + ))?; + + return Err(err); + } e => { log_query_error(query_string, &e); let short_error = short_error(&e); From 5cbdec9c794ef414e7511d644450b1a9a944d4ff Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Fri, 18 Oct 2024 14:41:21 +0100 Subject: [PATCH 17/21] [local_proxy]: install pg_session_jwt extension on demand (#9370) Follow up on #9344. We want to install the extension automatically. We didn't want to couple the extension into compute_ctl so instead local_proxy is the one to issue requests specific to the extension. depends on #9344 and #9395 --- compute/Dockerfile.compute-node | 4 +- proxy/src/auth/backend/local.rs | 13 ++- proxy/src/bin/local_proxy.rs | 8 +- proxy/src/compute_ctl/mod.rs | 101 ++++++++++++++++++++++++ proxy/src/http/mod.rs | 13 ++- proxy/src/lib.rs | 1 + proxy/src/serverless/backend.rs | 54 +++++++++++-- proxy/src/serverless/local_conn_pool.rs | 57 +++++++++---- 8 files changed, 222 insertions(+), 29 deletions(-) create mode 100644 proxy/src/compute_ctl/mod.rs diff --git a/compute/Dockerfile.compute-node b/compute/Dockerfile.compute-node index 45c1fd9f3871..74970696b5af 100644 --- a/compute/Dockerfile.compute-node +++ b/compute/Dockerfile.compute-node @@ -975,8 +975,8 @@ ARG PG_VERSION RUN case "${PG_VERSION}" in "v17") \ echo "pg_session_jwt does not yet have a release that supports pg17" && exit 0;; \ esac && \ - wget https://github.com/neondatabase/pg_session_jwt/archive/5aee2625af38213650e1a07ae038fdc427250ee4.tar.gz -O pg_session_jwt.tar.gz && \ - echo "5d91b10bc1347d36cffc456cb87bec25047935d6503dc652ca046f04760828e7 pg_session_jwt.tar.gz" | sha256sum --check && \ + wget https://github.com/neondatabase/pg_session_jwt/archive/e642528f429dd3f5403845a50191b78d434b84a6.tar.gz -O pg_session_jwt.tar.gz && \ + echo "1a69210703cc91224785e59a0a67562dd9eed9a0914ac84b11447582ca0d5b93 pg_session_jwt.tar.gz" | sha256sum --check && \ mkdir pg_session_jwt-src && cd pg_session_jwt-src && tar xzf ../pg_session_jwt.tar.gz --strip-components=1 -C . && \ sed -i 's/pgrx = "=0.11.3"/pgrx = { version = "=0.11.3", features = [ "unsafe-postgres" ] }/g' Cargo.toml && \ cargo pgrx install --release diff --git a/proxy/src/auth/backend/local.rs b/proxy/src/auth/backend/local.rs index e3995ac6c0cc..1e029ff609d3 100644 --- a/proxy/src/auth/backend/local.rs +++ b/proxy/src/auth/backend/local.rs @@ -1,23 +1,32 @@ use std::net::SocketAddr; use arc_swap::ArcSwapOption; +use tokio::sync::Semaphore; use super::jwt::{AuthRule, FetchAuthRules}; use crate::auth::backend::jwt::FetchAuthRulesError; use crate::compute::ConnCfg; +use crate::compute_ctl::ComputeCtlApi; use crate::context::RequestMonitoring; use crate::control_plane::messages::{ColdStartInfo, EndpointJwksResponse, MetricsAuxInfo}; use crate::control_plane::NodeInfo; use crate::intern::{BranchIdTag, EndpointIdTag, InternId, ProjectIdTag}; -use crate::EndpointId; +use crate::url::ApiUrl; +use crate::{http, EndpointId}; pub struct LocalBackend { + pub(crate) initialize: Semaphore, + pub(crate) compute_ctl: ComputeCtlApi, pub(crate) node_info: NodeInfo, } impl LocalBackend { - pub fn new(postgres_addr: SocketAddr) -> Self { + pub fn new(postgres_addr: SocketAddr, compute_ctl: ApiUrl) -> Self { LocalBackend { + initialize: Semaphore::new(1), + compute_ctl: ComputeCtlApi { + api: http::Endpoint::new(compute_ctl, http::new_client()), + }, node_info: NodeInfo { config: { let mut cfg = ConnCfg::new(); diff --git a/proxy/src/bin/local_proxy.rs b/proxy/src/bin/local_proxy.rs index e6bc369d9a7c..a16c288e5dab 100644 --- a/proxy/src/bin/local_proxy.rs +++ b/proxy/src/bin/local_proxy.rs @@ -25,6 +25,7 @@ use proxy::rate_limiter::{ use proxy::scram::threadpool::ThreadPool; use proxy::serverless::cancel_set::CancelSet; use proxy::serverless::{self, GlobalConnPoolOptions}; +use proxy::url::ApiUrl; use proxy::RoleName; project_git_version!(GIT_VERSION); @@ -80,7 +81,10 @@ struct LocalProxyCliArgs { connect_to_compute_retry: String, /// Address of the postgres server #[clap(long, default_value = "127.0.0.1:5432")] - compute: SocketAddr, + postgres: SocketAddr, + /// Address of the compute-ctl api service + #[clap(long, default_value = "http://127.0.0.1:3080/")] + compute_ctl: ApiUrl, /// Path of the local proxy config file #[clap(long, default_value = "./local_proxy.json")] config_path: Utf8PathBuf, @@ -295,7 +299,7 @@ fn build_auth_backend( args: &LocalProxyCliArgs, ) -> anyhow::Result<&'static auth::Backend<'static, ()>> { let auth_backend = proxy::auth::Backend::Local(proxy::auth::backend::MaybeOwned::Owned( - LocalBackend::new(args.compute), + LocalBackend::new(args.postgres, args.compute_ctl.clone()), )); Ok(Box::leak(Box::new(auth_backend))) diff --git a/proxy/src/compute_ctl/mod.rs b/proxy/src/compute_ctl/mod.rs new file mode 100644 index 000000000000..2b5789722345 --- /dev/null +++ b/proxy/src/compute_ctl/mod.rs @@ -0,0 +1,101 @@ +use compute_api::responses::GenericAPIError; +use hyper::{Method, StatusCode}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::url::ApiUrl; +use crate::{http, DbName, RoleName}; + +pub struct ComputeCtlApi { + pub(crate) api: http::Endpoint, +} + +#[derive(Serialize, Debug)] +pub struct ExtensionInstallRequest { + pub extension: &'static str, + pub database: DbName, + pub version: &'static str, +} + +#[derive(Serialize, Debug)] +pub struct SetRoleGrantsRequest { + pub database: DbName, + pub schema: &'static str, + pub privileges: Vec, + pub role: RoleName, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ExtensionInstallResponse {} + +#[derive(Clone, Debug, Deserialize)] +pub struct SetRoleGrantsResponse {} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[serde(rename_all = "UPPERCASE")] +pub enum Privilege { + Usage, +} + +#[derive(Error, Debug)] +pub enum ComputeCtlError { + #[error("connection error: {0}")] + ConnectionError(#[source] reqwest_middleware::Error), + #[error("request error [{status}]: {body:?}")] + RequestError { + status: StatusCode, + body: Option, + }, + #[error("response parsing error: {0}")] + ResponseError(#[source] reqwest::Error), +} + +impl ComputeCtlApi { + pub async fn install_extension( + &self, + req: &ExtensionInstallRequest, + ) -> Result { + self.generic_request(req, Method::POST, |url| { + url.path_segments_mut().push("extensions"); + }) + .await + } + + pub async fn grant_role( + &self, + req: &SetRoleGrantsRequest, + ) -> Result { + self.generic_request(req, Method::POST, |url| { + url.path_segments_mut().push("grants"); + }) + .await + } + + async fn generic_request( + &self, + req: &Req, + method: Method, + url: impl for<'a> FnOnce(&'a mut ApiUrl), + ) -> Result + where + Req: Serialize, + Resp: DeserializeOwned, + { + let resp = self + .api + .request_with_url(method, url) + .json(req) + .send() + .await + .map_err(ComputeCtlError::ConnectionError)?; + + let status = resp.status(); + if status.is_client_error() || status.is_server_error() { + let body = resp.json().await.ok(); + return Err(ComputeCtlError::RequestError { status, body }); + } + + resp.json().await.map_err(ComputeCtlError::ResponseError) + } +} diff --git a/proxy/src/http/mod.rs b/proxy/src/http/mod.rs index fd587e8f01f5..f1b632e70406 100644 --- a/proxy/src/http/mod.rs +++ b/proxy/src/http/mod.rs @@ -8,6 +8,7 @@ use std::time::Duration; use anyhow::bail; use bytes::Bytes; +use http::Method; use http_body_util::BodyExt; use hyper::body::Body; pub(crate) use reqwest::{Request, Response}; @@ -93,9 +94,19 @@ impl Endpoint { /// Return a [builder](RequestBuilder) for a `GET` request, /// accepting a closure to modify the url path segments for more complex paths queries. pub(crate) fn get_with_url(&self, f: impl for<'a> FnOnce(&'a mut ApiUrl)) -> RequestBuilder { + self.request_with_url(Method::GET, f) + } + + /// Return a [builder](RequestBuilder) for a request, + /// accepting a closure to modify the url path segments for more complex paths queries. + pub(crate) fn request_with_url( + &self, + method: Method, + f: impl for<'a> FnOnce(&'a mut ApiUrl), + ) -> RequestBuilder { let mut url = self.endpoint.clone(); f(&mut url); - self.client.get(url.into_inner()) + self.client.request(method, url.into_inner()) } /// Execute a [request](reqwest::Request). diff --git a/proxy/src/lib.rs b/proxy/src/lib.rs index a7b3d45c95e6..ea17a8806756 100644 --- a/proxy/src/lib.rs +++ b/proxy/src/lib.rs @@ -90,6 +90,7 @@ pub mod auth; pub mod cache; pub mod cancellation; pub mod compute; +pub mod compute_ctl; pub mod config; pub mod console_redirect_proxy; pub mod context; diff --git a/proxy/src/serverless/backend.rs b/proxy/src/serverless/backend.rs index 82e81dbcfef6..5d59b4d252ad 100644 --- a/proxy/src/serverless/backend.rs +++ b/proxy/src/serverless/backend.rs @@ -14,10 +14,13 @@ use tracing::{debug, info}; use super::conn_pool::poll_client; use super::conn_pool_lib::{Client, ConnInfo, GlobalConnPool}; use super::http_conn_pool::{self, poll_http2_client, Send}; -use super::local_conn_pool::{self, LocalClient, LocalConnPool}; +use super::local_conn_pool::{self, LocalClient, LocalConnPool, EXT_NAME, EXT_SCHEMA, EXT_VERSION}; use crate::auth::backend::local::StaticAuthRules; use crate::auth::backend::{ComputeCredentials, ComputeUserInfo}; use crate::auth::{self, check_peer_addr_is_in_list, AuthError}; +use crate::compute_ctl::{ + ComputeCtlError, ExtensionInstallRequest, Privilege, SetRoleGrantsRequest, +}; use crate::config::ProxyConfig; use crate::context::RequestMonitoring; use crate::control_plane::errors::{GetAuthInfoError, WakeComputeError}; @@ -35,6 +38,7 @@ pub(crate) struct PoolingBackend { pub(crate) http_conn_pool: Arc>, pub(crate) local_pool: Arc>, pub(crate) pool: Arc>, + pub(crate) config: &'static ProxyConfig, pub(crate) auth_backend: &'static crate::auth::Backend<'static, ()>, pub(crate) endpoint_rate_limiter: Arc, @@ -250,17 +254,48 @@ impl PoolingBackend { return Ok(client); } - let conn_id = uuid::Uuid::new_v4(); - tracing::Span::current().record("conn_id", display(conn_id)); - info!(%conn_id, "local_pool: opening a new connection '{conn_info}'"); - - let mut node_info = match &self.auth_backend { + let local_backend = match &self.auth_backend { auth::Backend::ControlPlane(_, ()) => { unreachable!("only local_proxy can connect to local postgres") } - auth::Backend::Local(local) => local.node_info.clone(), + auth::Backend::Local(local) => local, }; + if !self.local_pool.initialized(&conn_info) { + // only install and grant usage one at a time. + let _permit = local_backend.initialize.acquire().await.unwrap(); + + // check again for race + if !self.local_pool.initialized(&conn_info) { + local_backend + .compute_ctl + .install_extension(&ExtensionInstallRequest { + extension: EXT_NAME, + database: conn_info.dbname.clone(), + version: EXT_VERSION, + }) + .await?; + + local_backend + .compute_ctl + .grant_role(&SetRoleGrantsRequest { + schema: EXT_SCHEMA, + privileges: vec![Privilege::Usage], + database: conn_info.dbname.clone(), + role: conn_info.user_info.user.clone(), + }) + .await?; + + self.local_pool.set_initialized(&conn_info); + } + } + + let conn_id = uuid::Uuid::new_v4(); + tracing::Span::current().record("conn_id", display(conn_id)); + info!(%conn_id, "local_pool: opening a new connection '{conn_info}'"); + + let mut node_info = local_backend.node_info.clone(); + let (key, jwk) = create_random_jwk(); let config = node_info @@ -324,6 +359,8 @@ pub(crate) enum HttpConnError { #[error("could not parse JWT payload")] JwtPayloadError(serde_json::Error), + #[error("could not install extension: {0}")] + ComputeCtl(#[from] ComputeCtlError), #[error("could not get auth info")] GetAuthInfo(#[from] GetAuthInfoError), #[error("user not authenticated")] @@ -348,6 +385,7 @@ impl ReportableError for HttpConnError { HttpConnError::ConnectionClosedAbruptly(_) => ErrorKind::Compute, HttpConnError::PostgresConnectionError(p) => p.get_error_kind(), HttpConnError::LocalProxyConnectionError(_) => ErrorKind::Compute, + HttpConnError::ComputeCtl(_) => ErrorKind::Service, HttpConnError::JwtPayloadError(_) => ErrorKind::User, HttpConnError::GetAuthInfo(a) => a.get_error_kind(), HttpConnError::AuthError(a) => a.get_error_kind(), @@ -363,6 +401,7 @@ impl UserFacingError for HttpConnError { HttpConnError::ConnectionClosedAbruptly(_) => self.to_string(), HttpConnError::PostgresConnectionError(p) => p.to_string(), HttpConnError::LocalProxyConnectionError(p) => p.to_string(), + HttpConnError::ComputeCtl(_) => "could not set up the JWT authorization database extension".to_string(), HttpConnError::JwtPayloadError(p) => p.to_string(), HttpConnError::GetAuthInfo(c) => c.to_string_client(), HttpConnError::AuthError(c) => c.to_string_client(), @@ -379,6 +418,7 @@ impl CouldRetry for HttpConnError { match self { HttpConnError::PostgresConnectionError(e) => e.could_retry(), HttpConnError::LocalProxyConnectionError(e) => e.could_retry(), + HttpConnError::ComputeCtl(_) => false, HttpConnError::ConnectionClosedAbruptly(_) => false, HttpConnError::JwtPayloadError(_) => false, HttpConnError::GetAuthInfo(_) => false, diff --git a/proxy/src/serverless/local_conn_pool.rs b/proxy/src/serverless/local_conn_pool.rs index a01afd28208f..beb2ad4e8fbd 100644 --- a/proxy/src/serverless/local_conn_pool.rs +++ b/proxy/src/serverless/local_conn_pool.rs @@ -1,3 +1,14 @@ +//! Manages the pool of connections between local_proxy and postgres. +//! +//! The pool is keyed by database and role_name, and can contain multiple connections +//! shared between users. +//! +//! The pool manages the pg_session_jwt extension used for authorizing +//! requests in the db. +//! +//! The first time a db/role pair is seen, local_proxy attempts to install the extension +//! and grant usage to the role on the given schema. + use std::collections::HashMap; use std::pin::pin; use std::sync::{Arc, Weak}; @@ -27,14 +38,15 @@ use crate::metrics::Metrics; use crate::usage_metrics::{Ids, MetricCounter, USAGE_METRICS}; use crate::{DbName, RoleName}; +pub(crate) const EXT_NAME: &str = "pg_session_jwt"; +pub(crate) const EXT_VERSION: &str = "0.1.1"; +pub(crate) const EXT_SCHEMA: &str = "auth"; + struct ConnPoolEntry { conn: ClientInner, _last_access: std::time::Instant, } -// /// key id for the pg_session_jwt state -// static PG_SESSION_JWT_KID: AtomicU64 = AtomicU64::new(1); - // Per-endpoint connection pool, (dbname, username) -> DbUserConnPool // Number of open connections is limited by the `max_conns_per_endpoint`. pub(crate) struct EndpointConnPool { @@ -140,11 +152,18 @@ impl Drop for EndpointConnPool { pub(crate) struct DbUserConnPool { conns: Vec>, + + // true if we have definitely installed the extension and + // granted the role access to the auth schema. + initialized: bool, } impl Default for DbUserConnPool { fn default() -> Self { - Self { conns: Vec::new() } + Self { + conns: Vec::new(), + initialized: false, + } } } @@ -199,25 +218,16 @@ impl LocalConnPool { self.config.pool_options.idle_timeout } - // pub(crate) fn shutdown(&self) { - // let mut pool = self.global_pool.write(); - // pool.pools.clear(); - // pool.total_conns = 0; - // } - pub(crate) fn get( self: &Arc, ctx: &RequestMonitoring, conn_info: &ConnInfo, ) -> Result>, HttpConnError> { - let mut client: Option> = None; - if let Some(entry) = self + let client = self .global_pool .write() .get_conn_entry(conn_info.db_and_user()) - { - client = Some(entry.conn); - } + .map(|entry| entry.conn); // ok return cached connection if found and establish a new one otherwise if let Some(client) = client { @@ -245,6 +255,23 @@ impl LocalConnPool { } Ok(None) } + + pub(crate) fn initialized(self: &Arc, conn_info: &ConnInfo) -> bool { + self.global_pool + .read() + .pools + .get(&conn_info.db_and_user()) + .map_or(false, |pool| pool.initialized) + } + + pub(crate) fn set_initialized(self: &Arc, conn_info: &ConnInfo) { + self.global_pool + .write() + .pools + .entry(conn_info.db_and_user()) + .or_default() + .initialized = true; + } } #[allow(clippy::too_many_arguments)] From e162ab8b536e8b1d2277b4e2c00abd574c394d75 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Fri, 18 Oct 2024 15:33:04 +0100 Subject: [PATCH 18/21] storcon: handle ongoing deletions gracefully (#9449) ## Problem Pageserver returns 409 (Conflict) if any of the shards are already deleting the timeline. This resulted in an error being propagated out of the HTTP handler and to the client. It's an expected scenario so we should handle it nicely. This caused failures in `test_storage_controller_smoke` [here](https://neon-github-public-dev.s3.amazonaws.com/reports/pr-9435/11390431900/index.html#suites/8fc5d1648d2225380766afde7c428d81/86eee4b002d6572d). ## Summary of Changes Instead of returning an error on 409s, we now bubble the status code up and let the HTTP handler code retry until it gets a 404 or times out. --- storage_controller/src/http.rs | 18 ++++++++++++------ storage_controller/src/service.rs | 29 +++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 46b6f4f2bf24..afefe8598c00 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -381,14 +381,16 @@ async fn handle_tenant_timeline_delete( R: std::future::Future> + Send + 'static, F: Fn(Arc) -> R + Send + Sync + 'static, { + // On subsequent retries, wait longer. + // Enable callers with a 25 second request timeout to reliably get a response + const MAX_WAIT: Duration = Duration::from_secs(25); + const MAX_RETRY_PERIOD: Duration = Duration::from_secs(5); + let started_at = Instant::now(); + // To keep deletion reasonably snappy for small tenants, initially check after 1 second if deletion // completed. let mut retry_period = Duration::from_secs(1); - // On subsequent retries, wait longer. - let max_retry_period = Duration::from_secs(5); - // Enable callers with a 30 second request timeout to reliably get a response - let max_wait = Duration::from_secs(25); loop { let status = f(service.clone()).await?; @@ -396,7 +398,11 @@ async fn handle_tenant_timeline_delete( StatusCode::ACCEPTED => { tracing::info!("Deletion accepted, waiting to try again..."); tokio::time::sleep(retry_period).await; - retry_period = max_retry_period; + retry_period = MAX_RETRY_PERIOD; + } + StatusCode::CONFLICT => { + tracing::info!("Deletion already in progress, waiting to try again..."); + tokio::time::sleep(retry_period).await; } StatusCode::NOT_FOUND => { tracing::info!("Deletion complete"); @@ -409,7 +415,7 @@ async fn handle_tenant_timeline_delete( } let now = Instant::now(); - if now + retry_period > started_at + max_wait { + if now + retry_period > started_at + MAX_WAIT { tracing::info!("Deletion timed out waiting for 404"); // REQUEST_TIMEOUT would be more appropriate, but CONFLICT is already part of // the pageserver's swagger definition for this endpoint, and has the same desired diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index ab2c3b5e480f..01aa8f1dabc6 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -3630,14 +3630,21 @@ impl Service { ); let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref()); - client + let res = client .timeline_delete(tenant_shard_id, timeline_id) - .await - .map_err(|e| { - ApiError::InternalServerError(anyhow::anyhow!( - "Error deleting timeline {timeline_id} on {tenant_shard_id} on node {node}: {e}", - )) - }) + .await; + + match res { + Ok(ok) => Ok(ok), + Err(mgmt_api::Error::ApiError(StatusCode::CONFLICT, _)) => Ok(StatusCode::CONFLICT), + Err(e) => { + Err( + ApiError::InternalServerError(anyhow::anyhow!( + "Error deleting timeline {timeline_id} on {tenant_shard_id} on node {node}: {e}", + )) + ) + } + } } let locations = targets.0.iter().map(|t| (*t.0, t.1.latest.node.clone())).collect(); @@ -3652,7 +3659,13 @@ impl Service { }) .await?; - // If any shards >0 haven't finished deletion yet, don't start deletion on shard zero + // If any shards >0 haven't finished deletion yet, don't start deletion on shard zero. + // We return 409 (Conflict) if deletion was already in progress on any of the shards + // and 202 (Accepted) if deletion was not already in progress on any of the shards. + if statuses.iter().any(|s| s == &StatusCode::CONFLICT) { + return Ok(StatusCode::CONFLICT); + } + if statuses.iter().any(|s| s != &StatusCode::NOT_FOUND) { return Ok(StatusCode::ACCEPTED); } From 62a334871fef32b754ab98a772ebbbbed8d1aa1c Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Fri, 18 Oct 2024 09:36:29 -0500 Subject: [PATCH 19/21] Take the collector name as argument when generating sql_exporter configs In neon_collector_autoscaling.jsonnet, the collector name is hardcoded to neon_collector_autoscaling. This issue manifests itself such that sql_exporter would not find the collector configuration. Signed-off-by: Tristan Partin --- compute/Makefile | 2 ++ compute/etc/sql_exporter.jsonnet | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compute/Makefile b/compute/Makefile index e4f08a223c24..e2896fe39096 100644 --- a/compute/Makefile +++ b/compute/Makefile @@ -20,12 +20,14 @@ neon_collector_autoscaling.yml: $(jsonnet_files) sql_exporter.yml: $(jsonnet_files) JSONNET_PATH=etc jsonnet \ --output-file etc/$@ \ + --tla-str collector_name=neon_collector \ --tla-str collector_file=neon_collector.yml \ etc/sql_exporter.jsonnet sql_exporter_autoscaling.yml: $(jsonnet_files) JSONNET_PATH=etc jsonnet \ --output-file etc/$@ \ + --tla-str collector_name=neon_collector_autoscaling \ --tla-str collector_file=neon_collector_autoscaling.yml \ --tla-str application_name=sql_exporter_autoscaling \ etc/sql_exporter.jsonnet diff --git a/compute/etc/sql_exporter.jsonnet b/compute/etc/sql_exporter.jsonnet index 640e2ac38df9..3c36fd4f68b9 100644 --- a/compute/etc/sql_exporter.jsonnet +++ b/compute/etc/sql_exporter.jsonnet @@ -1,4 +1,4 @@ -function(collector_file, application_name='sql_exporter') { +function(collector_name, collector_file, application_name='sql_exporter') { // Configuration for sql_exporter for autoscaling-agent // Global defaults. global: { @@ -28,7 +28,7 @@ function(collector_file, application_name='sql_exporter') { // Collectors (referenced by name) to execute on the target. // Glob patterns are supported (see for syntax). collectors: [ - 'neon_collector', + collector_name, ], }, From 71d09c78d4ffd159cfcd83c4c1b919a4c7eef7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Sat, 19 Oct 2024 00:23:49 +0200 Subject: [PATCH 20/21] Accept basebackup --gzip requests (#9456) In #9453, we want to remove the non-gzipped basebackup code in the computes, and always request gzipped basebackups. However, right now the pageserver's page service only accepts basebackup requests in the following formats: * `basebackup `, lsn is determined by the pageserver as the most recent one (`timeline.get_last_record_rlsn()`) * `basebackup ` * `basebackup --gzip` We add a fourth case, `basebackup --gzip` to allow gzipping the request for the latest lsn as well. --- pageserver/src/page_service.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index afb2f92ff80d..62b14cb83e55 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -1326,22 +1326,22 @@ where .for_command(ComputeCommandKind::Basebackup) .inc(); - let lsn = if let Some(lsn_str) = params.get(2) { - Some( - Lsn::from_str(lsn_str) - .with_context(|| format!("Failed to parse Lsn from {lsn_str}"))?, - ) - } else { - None - }; - - let gzip = match params.get(3) { - Some(&"--gzip") => true, - None => false, - Some(third_param) => { - return Err(QueryError::Other(anyhow::anyhow!( - "Parameter in position 3 unknown {third_param}", - ))) + let (lsn, gzip) = match (params.get(2), params.get(3)) { + (None, _) => (None, false), + (Some(&"--gzip"), _) => (None, true), + (Some(lsn_str), gzip_str_opt) => { + let lsn = Lsn::from_str(lsn_str) + .with_context(|| format!("Failed to parse Lsn from {lsn_str}"))?; + let gzip = match gzip_str_opt { + Some(&"--gzip") => true, + None => false, + Some(third_param) => { + return Err(QueryError::Other(anyhow::anyhow!( + "Parameter in position 3 unknown {third_param}", + ))) + } + }; + (Some(lsn), gzip) } }; From cc25ef73423ea0108986436501481b0154443932 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Sun, 20 Oct 2024 13:42:50 +0100 Subject: [PATCH 21/21] bump pg-session-jwt version (#9455) forgot to bump this before --- proxy/src/serverless/local_conn_pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/src/serverless/local_conn_pool.rs b/proxy/src/serverless/local_conn_pool.rs index beb2ad4e8fbd..e1ad46c751fc 100644 --- a/proxy/src/serverless/local_conn_pool.rs +++ b/proxy/src/serverless/local_conn_pool.rs @@ -39,7 +39,7 @@ use crate::usage_metrics::{Ids, MetricCounter, USAGE_METRICS}; use crate::{DbName, RoleName}; pub(crate) const EXT_NAME: &str = "pg_session_jwt"; -pub(crate) const EXT_VERSION: &str = "0.1.1"; +pub(crate) const EXT_VERSION: &str = "0.1.2"; pub(crate) const EXT_SCHEMA: &str = "auth"; struct ConnPoolEntry {