Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: secondary rocksdb storage #281

Draft
wants to merge 7 commits into
base: develop
Choose a base branch
from
Draft

feat: secondary rocksdb storage #281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ec24038
feat: secondary rocksdb storage
shumkov Feb 8, 2024
b20de39
style: formatting
shumkov Feb 8, 2024
09480fd
chore: set max open files -1 for secondary storage
shumkov Feb 10, 2024
1dc5d5d
chore: ignore flush for secondary storage
shumkov Feb 10, 2024
7409dcd
refactor: remove duplicate structure
shumkov Feb 12, 2024
b9f08c8
refactor: remove duplicate structure
shumkov Feb 12, 2024
3e0851d
refactor: rename test
shumkov Feb 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion grovedb/src/estimated_costs/average_case_costs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,7 @@ mod test {
fn test_get_merk_node_average_case() {
// Open a merk and insert 10 elements.
let tmp_dir = TempDir::new().expect("cannot open tempdir");
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let batch = StorageBatch::new();

Expand Down
21 changes: 19 additions & 2 deletions grovedb/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,12 +246,29 @@

#[cfg(feature = "full")]
impl GroveDb {
/// Opens a given path
/// Opens a primary storage with a given path
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
let db = RocksDbStorage::default_rocksdb_with_path(path)?;
let db = RocksDbStorage::default_primary_rocksdb(path)?;
Ok(GroveDb { db })
}

/// Open a secondary storage with given paths
pub fn open_secondary<P: AsRef<Path>>(
fominok marked this conversation as resolved.
Show resolved Hide resolved
primary_path: P,
secondary_storage: P,
) -> Result<Self, Error> {
let db = RocksDbStorage::default_secondary_rocksdb(primary_path, secondary_storage)?;
Ok(GroveDb { db })
}

/// Replicate recent changes from primary database
/// Available only for a secondary storage
pub fn try_to_catch_up_from_primary(&self) -> Result<(), Error> {
self.db.try_to_catch_up_from_primary()?;

Ok(())
}

/// Uses raw iter to delete GroveDB key values pairs from rocksdb
pub fn wipe(&self) -> Result<(), Error> {
self.db.wipe()?;
Expand Down Expand Up @@ -844,7 +861,7 @@
pub fn verify_grovedb(
&self,
transaction: TransactionArg,
) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {

Check warning on line 864 in grovedb/src/lib.rs

View workflow job for this annotation

GitHub Actions / clippy

very complex type used. Consider factoring parts into `type` definitions 

warning: very complex type used. Consider factoring parts into `type` definitions
   --> grovedb/src/lib.rs:864:10
    |
864 |     ) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
    |          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#type_complexity
    = note: `#[warn(clippy::type_complexity)]` on by default
if let Some(transaction) = transaction {
let root_merk = self
.open_transactional_merk_at_path(SubtreePath::empty(), transaction, None)
Expand All @@ -870,7 +887,7 @@
merk: Merk<S>,
path: &SubtreePath<B>,
batch: Option<&'db StorageBatch>,
) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {

Check warning on line 890 in grovedb/src/lib.rs

View workflow job for this annotation

GitHub Actions / clippy

very complex type used. Consider factoring parts into `type` definitions 

warning: very complex type used. Consider factoring parts into `type` definitions
   --> grovedb/src/lib.rs:890:10
    |
890 |     ) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
    |          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#type_complexity
let mut all_query = Query::new();
all_query.insert_all();

Expand Down Expand Up @@ -940,7 +957,7 @@
path: &SubtreePath<B>,
batch: Option<&'db StorageBatch>,
transaction: &Transaction,
) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {

Check warning on line 960 in grovedb/src/lib.rs

View workflow job for this annotation

GitHub Actions / clippy

very complex type used. Consider factoring parts into `type` definitions 

warning: very complex type used. Consider factoring parts into `type` definitions
   --> grovedb/src/lib.rs:960:10
    |
960 |     ) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
    |          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#type_complexity
let mut all_query = Query::new();
all_query.insert_all();

Expand Down
71 changes: 71 additions & 0 deletions grovedb/tests/secondary_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
use grovedb::GroveDb;
use tempfile::TempDir;

#[test]
fn test_replicating_aux_data_to_secondary_database() {
let primary_dir = TempDir::new().expect("should create temp dir");

let primary_grovedb = GroveDb::open(primary_dir.path()).expect("should open grovedb");

// Store value in primary

let key = b"key";
let value = vec![1, 2, 3];

primary_grovedb
.put_aux(key, &value, None, None)
.unwrap()
.expect("should put value to primary");

// Read value from primary

let primary_value = primary_grovedb
.get_aux(key, None)
.unwrap()
.expect("should get value from primary")
.expect("value should exist on primary");

assert_eq!(value, primary_value);

// Open secondary

let secondary_dir = TempDir::new().expect("should create temp dir");

let secondary_grovedb = GroveDb::open_secondary(primary_dir.path(), secondary_dir.path())
.expect("should open secondary");

// Read value on secondary

let secondary_value = secondary_grovedb
.get_aux(key, None)
.unwrap()
.expect("should get value from secondary")
.expect("value from primary should exist on secondary");

assert_eq!(primary_value, secondary_value);

// Update value on primary

let primary_value2 = vec![4, 5, 6];

primary_grovedb
.put_aux(key, &primary_value2, None, None)
.unwrap()
.expect("should put value to primary");

// Catch up secondary

secondary_grovedb
.try_to_catch_up_from_primary()
.expect("should catch up");

// Read updated value on secondary

let secondary_value2 = secondary_grovedb
.get_aux(key, None)
.unwrap()
.expect("should get value from secondary")
.expect("value from primary should exist on secondary");

assert_eq!(primary_value2, secondary_value2);
}
4 changes: 2 additions & 2 deletions merk/src/merk/chunks.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ mod tests {
fn chunks_from_reopen() {
let tmp_dir = TempDir::new().expect("cannot create tempdir");
let original_chunks = {
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let batch = StorageBatch::new();
let mut merk = Merk::open_base(
Expand Down Expand Up @@ -307,7 +307,7 @@ mod tests {
.collect::<Vec<_>>()
.into_iter()
};
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let merk = Merk::open_base(
storage
Expand Down
14 changes: 7 additions & 7 deletions merk/src/merk/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -739,7 +739,7 @@ mod test {
#[test]
fn reopen_check_root_hash() {
let tmp_dir = TempDir::new().expect("cannot open tempdir");
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let mut merk = Merk::open_base(
storage
Expand All @@ -763,7 +763,7 @@ mod test {
#[test]
fn test_get_node_cost() {
let tmp_dir = TempDir::new().expect("cannot open tempdir");
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let mut merk = Merk::open_base(
storage
Expand Down Expand Up @@ -807,7 +807,7 @@ mod test {
let tmp_dir = TempDir::new().expect("cannot open tempdir");

let original_nodes = {
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let batch = StorageBatch::new();
let mut merk = Merk::open_base(
Expand Down Expand Up @@ -846,7 +846,7 @@ mod test {
nodes
};

let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let merk = Merk::open_base(
storage
Expand Down Expand Up @@ -883,7 +883,7 @@ mod test {
let tmp_dir = TempDir::new().expect("cannot open tempdir");

let original_nodes = {
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let batch = StorageBatch::new();
let mut merk = Merk::open_base(
Expand Down Expand Up @@ -918,7 +918,7 @@ mod test {
collect(&mut merk.storage.raw_iter(), &mut nodes);
nodes
};
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let merk = Merk::open_base(
storage
Expand All @@ -939,7 +939,7 @@ mod test {
#[test]
fn update_node() {
let tmp_dir = TempDir::new().expect("cannot open tempdir");
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let batch = StorageBatch::new();
let mut merk = Merk::open_base(
Expand Down
2 changes: 1 addition & 1 deletion merk/src/merk/open.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ mod test {
#[test]
fn test_reopen_root_hash() {
let tmp_dir = TempDir::new().expect("cannot open tempdir");
let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path())
let storage = RocksDbStorage::default_primary_rocksdb(tmp_dir.path())
.expect("cannot open rocksdb storage");
let test_prefix = [b"ayy"];

Expand Down
2 changes: 1 addition & 1 deletion storage/src/error.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ pub enum Error {
#[error("cost error: {0}")]
CostError(grovedb_costs::error::Error),
/// Rocks DB error
#[error("rocksDB error: {0}")]
#[error("RocksDB error: {0}")]
#[cfg(feature = "rocksdb_storage")]
RocksDBError(#[from] rocksdb::Error),
}
Loading
Loading