WIP: azure

misc/python/materialize/cli/run.py

@@ -48,6 +48,7 @@
     else f"{Arch.host()}-apple-darwin"
 )
 DEFAULT_POSTGRES = "postgres://root@localhost:26257/materialize"
+DEFAULT_BLOB = "file://mzdata/persist/blob"
 
 # sets entitlements on the built binary, e.g. environmentd, so you can inspect it with Instruments
 MACOS_ENTITLEMENTS_DATA = """
@@ -88,6 +89,11 @@ def main() -> int:
         help="Postgres/CockroachDB connection string",
         default=os.getenv("MZDEV_POSTGRES", DEFAULT_POSTGRES),
     )
+    parser.add_argument(
+        "--blob",
+        help="Blob storage connection string",
+        default=os.getenv("MZDEV_BLOB", DEFAULT_BLOB),
+    )
     parser.add_argument(
         "--release",
         help="Build artifacts in release mode, with optimizations",
@@ -276,7 +282,7 @@ def main() -> int:
                 f"--orchestrator-process-scratch-directory={scratch}",
                 "--secrets-controller=local-file",
                 f"--persist-consensus-url={args.postgres}?options=--search_path=consensus",
-                f"--persist-blob-url=file://{mzdata}/persist/blob",
+                f"--persist-blob-url={args.blob}",
                 f"--timestamp-oracle-url={args.postgres}?options=--search_path=tsoracle",
                 f"--environment-id={environment_id}",
                 "--bootstrap-role=materialize",

src/mz/Cargo.toml

@@ -25,7 +25,7 @@ mz-ore = { path = "../ore", features = ["async", "cli", "test"] }
 open = "3.2.0"
 openssl-probe = "0.1.2"
 hyper = "1.4.1"
-reqwest = { version = "0.11", features = ["blocking", "json"] }
+reqwest = { version = "0.12", features = ["blocking", "json", "default-tls", "charset", "http2"], default-features = false }
 rpassword = "7.2.0"
 semver = "1.0.16"
 serde = { version = "1.0.152", features = ["derive"] }

src/ore/src/lgbytes.rs

@@ -179,6 +179,8 @@ impl Buf for LgBytes {
 pub struct LgBytesMetrics {
     /// Metrics for the "persist_s3" usage of [LgBytes].
     pub persist_s3: LgBytesOpMetrics,
+    /// Metrics for the "persist_abs" usage of [LgBytes].
+    pub persist_abs: LgBytesOpMetrics,
     /// Metrics for the "persist_arrow" usage of [LgBytes].
     pub persist_arrow: LgBytesOpMetrics,
 }
@@ -272,6 +274,7 @@ impl LgBytesMetrics {
         };
         LgBytesMetrics {
             persist_s3: op("persist_s3"),
+            persist_abs: op("persist_abs"),
             persist_arrow: op("persist_arrow"),
         }
     }

src/persist-client/src/internal/metrics.rs

@@ -32,7 +32,7 @@ use mz_ore::stats::histogram_seconds_buckets;
 use mz_persist::location::{
     Blob, BlobMetadata, CaSResult, Consensus, ExternalError, ResultStream, SeqNo, VersionedData,
 };
-use mz_persist::metrics::{ColumnarMetrics, S3BlobMetrics};
+use mz_persist::metrics::{ABSBlobMetrics, ColumnarMetrics, S3BlobMetrics};
 use mz_persist::retry::RetryStream;
 use mz_persist_types::Codec64;
 use mz_postgres_client::metrics::PostgresClientMetrics;
@@ -110,6 +110,8 @@ pub struct Metrics {
 
     /// Metrics for S3-backed blob implementation
     pub s3_blob: S3BlobMetrics,
+    /// Metrics for Azure-backed blob implementation
+    // pub azure_blob: ABSBlobMetrics,
     /// Metrics for Postgres-backed consensus implementation
     pub postgres_consensus: PostgresClientMetrics,
 
@@ -140,6 +142,7 @@ impl Metrics {
             move || start.elapsed().as_secs_f64(),
         );
         let s3_blob = S3BlobMetrics::new(registry);
+        // let azure_blob = ABSBlobMetrics::new(registry);
         let columnar = ColumnarMetrics::new(
             registry,
             &s3_blob.lgbytes,

src/persist/Cargo.toml

@@ -30,6 +30,10 @@ aws-config = { version = "1.2.0", default-features = false }
 aws-credential-types = { version = "1.1.1", features = ["hardcoded-credentials"] }
 aws-sdk-s3 = { version = "1.23.0", default-features = false, features = ["rt-tokio"] }
 aws-types = "1.1.1"
+azure_identity = { version = "0.21.0" }
+azure_storage = { version = "0.21.0" }
+azure_storage_blobs = { version = "0.21.0" }
+azure_core = "0.21.0"
 base64 = "0.13.1"
 bytes = "1.3.0"
 deadpool-postgres = "0.10.3"