diff --git a/Cargo.lock b/Cargo.lock index 02c9fca..1e58bfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -967,9 +967,9 @@ checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" [[package]] name = "h2" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes", "fnv", @@ -1107,9 +1107,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -1242,9 +1242,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libsqlite3-sys" diff --git a/README.md b/README.md index 0c35d64..063ef5a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is an experimental binding using the [s3s](https://crates.io/crates/s3s) cr ## Why -This crate was built to test the feasiblity of using SQLite as an alternative to formats like [TFRecord](https://www.tensorflow.org/tutorials/load_data/tfrecord) for storing a large number of machine learning training data with an accessible API. Although `tfrecord` and SQLite share a similar few-large-storage-files architecture - making backups and data movement efficient/easy - having both a SQLite interface and [Amazon S3](https://aws.amazon.com/s3/)/`http` API makes access to the contained data easier with tooling that likely already exists on your machine. +This crate was built to test the feasiblity of using SQLite as an alternative to formats like [TFRecord](https://www.tensorflow.org/tutorials/load_data/tfrecord) for storing a large number of machine learning training data with an accessible API. Although `TFRecord` and SQLite share a similar few-large-storage-files architecture - making backups and data movement efficient/easy - having both a SQLite interface and [Amazon S3](https://aws.amazon.com/s3/)/`http` API makes access to the contained data easier with tooling that likely already exists on your machine. This concept is backed by benchmarks from SQLite showing that it can be [faster than filesytems](https://www.sqlite.org/fasterthanfs.html) for certain data access patterns. @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS data ( metadata TEXT, last_modified TEXT NOT NULL, md5 TEXT -) STRICT, WITHOUT ROWID; +) STRICT; ``` ### Multipart Uploads @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS multipart_upload ( key TEXT NOT NULL, last_modified TEXT NOT NULL, access_key TEXT -) STRICT, WITHOUT ROWID; +) STRICT; CREATE TABLE IF NOT EXISTS multipart_upload_part ( upload_id BLOB NOT NULL, @@ -71,7 +71,7 @@ cargo install --path . ## Run ```bash -s3ite . --access-key AKIAIOSFODNN7EXAMPLE --secret-key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +s3ite . --host 0.0.0.0 --port 8014 --access-key AKIAIOSFODNN7EXAMPLE --secret-key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY ``` ## Docker diff --git a/src/sqlite.rs b/src/sqlite.rs index 50c6bcb..bc8c654 100644 --- a/src/sqlite.rs +++ b/src/sqlite.rs @@ -89,6 +89,7 @@ impl Sqlite { .interact(|connection| { connection.execute_batch( " + PRAGMA journal_mode=WAL; PRAGMA foreign_keys=true; PRAGMA analysis_limit=1000; PRAGMA optimize; @@ -141,7 +142,12 @@ impl Sqlite { let connection = pool.get().await.unwrap(); connection .interact(|connection| { - connection.execute_batch("PRAGMA foreign_keys=true;")?; + connection.execute_batch( + " + PRAGMA journal_mode=WAL; + PRAGMA foreign_keys=true; + ", + )?; let transaction = connection.transaction()?; Self::try_create_tables(&transaction)?; @@ -165,7 +171,7 @@ impl Sqlite { metadata TEXT, last_modified TEXT NOT NULL, md5 TEXT - ) STRICT, WITHOUT ROWID;", + ) STRICT;", (), )?; transaction.execute( @@ -175,7 +181,7 @@ impl Sqlite { key TEXT NOT NULL, last_modified TEXT NOT NULL, access_key TEXT - ) STRICT, WITHOUT ROWID;", + ) STRICT;", (), )?; transaction.execute(