Skip to content

Commit

Permalink
use temp dir instead of cache
Browse files Browse the repository at this point in the history
  • Loading branch information
jordanrfrazier committed Oct 3, 2023
1 parent ff8699e commit 584d31e
Show file tree
Hide file tree
Showing 5 changed files with 14 additions and 72 deletions.
28 changes: 0 additions & 28 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ dashmap = "5.4.0"
data-encoding = "2.3.3"
decorum = "0.3.1"
derive_more = "0.99.17"
dirs = "5.0.1"
edit-distance = "2.1.0"
egg = "0.9.3"
enum-as-inner = "0.6.0"
Expand Down
1 change: 0 additions & 1 deletion crates/sparrow-runtime/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ clap.workspace = true
dashmap.workspace = true
data-encoding.workspace = true
derive_more.workspace = true
dirs.workspace = true
enum-map.workspace = true
erased-serde.workspace = true
error-stack.workspace = true
Expand Down
28 changes: 14 additions & 14 deletions crates/sparrow-runtime/src/prepare/preparer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@ use crate::PreparedMetadata;

use super::{prepared_batches, write_parquet};

const KASKADA_PATH: &str = ".cache/kaskada";
/// For now, this is a temporary location for the prepared files.
/// In the future, we'll want to move this path to a managed cache
/// so we can reuse state.
const KASKADA_PATH: &str = "kaskada";
const PREPARED_FILE_PREFIX: &str = "part";

#[derive(derive_more::Display, Debug)]
Expand Down Expand Up @@ -96,7 +99,7 @@ impl Preparer {
) -> error_stack::Result<Vec<PreparedFile>, Error> {
// TODO: Support Slicing

let output_path_prefix = self.prepared_output_prefix()?;
let output_path_prefix = self.prepared_output_prefix();
let output_url = ObjectStoreUrl::from_str(&output_path_prefix)
.change_context_lazy(|| Error::InvalidUrl(output_path_prefix))?;

Expand Down Expand Up @@ -178,19 +181,16 @@ impl Preparer {
)
}
// Prepared files are stored in the following format:
// file:///<home_dir>/<KASKADA_PATH>/tables/<table_uuid>/prepared/<uuid>/part-<n>.parquet
pub fn prepared_output_prefix(&self) -> error_stack::Result<String, Error> {
// file:///<temp_dir>/<KASKADA_PATH>/tables/<table_uuid>/prepared/<uuid>/part-<n>.parquet
pub fn prepared_output_prefix(&self) -> String {
let uuid = Uuid::new_v4();
let home_dir = dirs::home_dir();
if let Some(home_dir) = home_dir.map(|p| p.display().to_string()) {
Ok(format!(
"file:///{}/{}/tables/{}/prepare/{uuid}/",
home_dir, KASKADA_PATH, self.table_config.uuid
))
} else {
tracing::error!("Failed to get home directory");
error_stack::bail!(Error::Internal)
}
let temp_dir = tempfile::tempdir().expect("failed to create temp dir");
format!(
"file:///{}/{}/tables/{}/prepare/{uuid}/",
temp_dir.path().display(),
KASKADA_PATH,
self.table_config.uuid
)
}
}

Expand Down
28 changes: 0 additions & 28 deletions python/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 584d31e

Please sign in to comment.