From 3ec7f023723a189617ea1e5ab63a34e6310d2ed2 Mon Sep 17 00:00:00 2001 From: James Bornholt Date: Thu, 27 Jun 2024 21:59:41 -0500 Subject: [PATCH] Update to latest Mountpoint client crates (#212) * Update to latest Mountpoint client crates Signed-off-by: James Bornholt * Pin numpy version Signed-off-by: James Bornholt * Appease clippy Signed-off-by: James Bornholt --------- Signed-off-by: James Bornholt --- s3torchconnector/pyproject.toml | 5 ++--- s3torchconnectorclient/Cargo.lock | 12 ++++++------ s3torchconnectorclient/Cargo.toml | 4 ++-- s3torchconnectorclient/rust/src/mock_client.rs | 2 +- .../rust/src/mountpoint_s3_client.rs | 6 +++--- .../rust/src/mountpoint_s3_client_inner.rs | 2 +- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/s3torchconnector/pyproject.toml b/s3torchconnector/pyproject.toml index b08b8c14..506997b9 100644 --- a/s3torchconnector/pyproject.toml +++ b/s3torchconnector/pyproject.toml @@ -22,8 +22,7 @@ classifiers = [ ] dependencies = [ - #TODO: Remove torch != 2.3.0 restriction when https://github.com/pytorch/data/issues/1244 is fixed - "torch >= 2.0.1, != 2.3.0", + "torch >= 2.0.1", "s3torchconnectorclient >= 1.2.3", ] @@ -42,7 +41,7 @@ e2e = [ "torchvision", "Pillow", "boto3", - "numpy", + "numpy < 2", "pytest-xdist" ] diff --git a/s3torchconnectorclient/Cargo.lock b/s3torchconnectorclient/Cargo.lock index e4948472..c128c993 100644 --- a/s3torchconnectorclient/Cargo.lock +++ b/s3torchconnectorclient/Cargo.lock @@ -658,9 +658,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mountpoint-s3-client" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322418903003c4d58baf36a049d6281f967d9e23b2daf51d51e2a68ea817db48" +checksum = "e9a8f51eac69f2d4ebb69f89327798605c48922d92a320588356d475b2d4b783" dependencies = [ "async-io", "async-lock", @@ -693,9 +693,9 @@ dependencies = [ [[package]] name = "mountpoint-s3-crt" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1226c7cec848a8ccdfc4ad1014c0e39b54f27b0b44029579b5a6a5943f483e" +checksum = "b85431534bac19718eb53447d708d233b31e502c02174b13191a2560e0685926" dependencies = [ "async-channel", "futures", @@ -709,9 +709,9 @@ dependencies = [ [[package]] name = "mountpoint-s3-crt-sys" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce60f488399cdf2d62af4c90f38042cfa58116fc2574965e44c25eabe06e55f9" +checksum = "a44b7d483f0a9096502881b50a099c48f0e0e2da5615d1fdc76db999dfd69e78" dependencies = [ "bindgen", "cc", diff --git a/s3torchconnectorclient/Cargo.toml b/s3torchconnectorclient/Cargo.toml index 9fd044f1..e9a05e62 100644 --- a/s3torchconnectorclient/Cargo.toml +++ b/s3torchconnectorclient/Cargo.toml @@ -19,8 +19,8 @@ built = "0.7" pyo3 = { version = "0.19.2" } pyo3-log = "0.8.3" futures = "0.3.28" -mountpoint-s3-client = { version = "0.8.1", features = ["mock"] } -mountpoint-s3-crt = "0.7.0" +mountpoint-s3-client = { version = "0.9.0", features = ["mock"] } +mountpoint-s3-crt = "0.8.0" log = "0.4.20" tracing = { version = "0.1.40", default-features = false, features = ["std", "log"] } tracing-subscriber = { version = "0.3.18", features = ["fmt", "env-filter"]} diff --git a/s3torchconnectorclient/rust/src/mock_client.rs b/s3torchconnectorclient/rust/src/mock_client.rs index 0a13c252..8bf8f612 100644 --- a/s3torchconnectorclient/rust/src/mock_client.rs +++ b/s3torchconnectorclient/rust/src/mock_client.rs @@ -43,7 +43,7 @@ impl PyMockClient { unsigned: bool, ) -> PyMockClient { let unordered_list_seed: Option = None; - let config = MockClientConfig { bucket, part_size, unordered_list_seed }; + let config = MockClientConfig { bucket, part_size, unordered_list_seed, ..Default::default() }; let mock_client = Arc::new(MockClient::new(config)); PyMockClient { diff --git a/s3torchconnectorclient/rust/src/mountpoint_s3_client.rs b/s3torchconnectorclient/rust/src/mountpoint_s3_client.rs index 094b7bdc..4c496c5d 100644 --- a/s3torchconnectorclient/rust/src/mountpoint_s3_client.rs +++ b/s3torchconnectorclient/rust/src/mountpoint_s3_client.rs @@ -163,7 +163,7 @@ impl MountpointS3Client { #[allow(clippy::too_many_arguments)] impl MountpointS3Client { - pub(crate) fn new( + pub(crate) fn new( region: String, user_agent_prefix: String, throughput_target_gbps: f64, @@ -175,8 +175,8 @@ impl MountpointS3Client { endpoint: Option, ) -> Self where - Client: Sync + Send + 'static, - ::GetObjectResult: Unpin + Sync, + Client: ObjectClient + Sync + Send + 'static, + ::GetObjectRequest: Unpin + Sync, ::PutObjectRequest: Sync, { Self { diff --git a/s3torchconnectorclient/rust/src/mountpoint_s3_client_inner.rs b/s3torchconnectorclient/rust/src/mountpoint_s3_client_inner.rs index aa6aa0f4..b61ca14f 100644 --- a/s3torchconnectorclient/rust/src/mountpoint_s3_client_inner.rs +++ b/s3torchconnectorclient/rust/src/mountpoint_s3_client_inner.rs @@ -57,7 +57,7 @@ impl MountpointS3ClientInnerImpl { impl MountpointS3ClientInner for MountpointS3ClientInnerImpl where Client: ObjectClient, - ::GetObjectResult: Sync + Send + Unpin + 'static, + ::GetObjectRequest: Sync + Unpin + 'static, ::PutObjectRequest: Sync + 'static, { fn get_object(&self, py: Python, bucket: String, key: String) -> PyResult {