diff --git a/datastore/datastore.nim b/datastore/datastore.nim index dd04725f..a15a305b 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -13,33 +13,37 @@ push: {.upraises: [].} type Datastore* = ref object of RootObj + QueryIterator* = iterator ( + datastore: Datastore, + query: Query): Future[QueryResponse] {.closure, gcsafe.} + +method close*(self: Datastore): Future[void] {.base, locks: "unknown".} = + raiseAssert("Not implemented!") + method contains*( self: Datastore, - key: Key): Future[?!bool] {.async, base, locks: "unknown".} = + key: Key): Future[?!bool] {.base, locks: "unknown".} = raiseAssert("Not implemented!") method delete*( self: Datastore, - key: Key): Future[?!void] {.async, base, locks: "unknown".} = + key: Key): Future[?!void] {.base, locks: "unknown".} = raiseAssert("Not implemented!") method get*( self: Datastore, - key: Key): Future[?!(?seq[byte])] {.async, base, locks: "unknown".} = + key: Key): Future[?!(?seq[byte])] {.base, locks: "unknown".} = raiseAssert("Not implemented!") method put*( self: Datastore, key: Key, - data: seq[byte]): Future[?!void] {.async, base, locks: "unknown".} = + data: seq[byte]): Future[?!void] {.base, locks: "unknown".} = raiseAssert("Not implemented!") -iterator query*( - self: Datastore, - query: Query): Future[QueryResponse] = - +method query*(self: Datastore): QueryIterator {.base, locks: "unknown".} = raiseAssert("Not implemented!") diff --git a/datastore/filesystem_datastore.nim b/datastore/filesystem_datastore.nim index 51c50a0a..27bc2dcc 100644 --- a/datastore/filesystem_datastore.nim +++ b/datastore/filesystem_datastore.nim @@ -59,6 +59,9 @@ proc path*( self.root / joinPath(segments) & objExt +method close*(self: FileSystemDatastore) {.async, locks: "unknown".} = + discard + method contains*( self: FileSystemDatastore, key: Key): Future[?!bool] {.async, locks: "unknown".} = @@ -154,9 +157,3 @@ method put*( except OSError as e: return failure e - -# method query*( -# self: FileSystemDatastore, -# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = -# -# return success ....some diff --git a/datastore/null_datastore.nim b/datastore/null_datastore.nim index cdb7444a..b5e6ad5c 100644 --- a/datastore/null_datastore.nim +++ b/datastore/null_datastore.nim @@ -15,6 +15,9 @@ type proc new*(T: type NullDatastore): T = T() +method close*(self: NullDatastore) {.async, locks: "unknown".} = + discard + method contains*( self: NullDatastore, key: Key): Future[?!bool] {.async, locks: "unknown".} = @@ -40,8 +43,11 @@ method put*( return success() -iterator query*( - self: NullDatastore, - query: Query): Future[QueryResponse] = +iterator queryImpl( + datastore: Datastore, + query: Query): Future[QueryResponse] {.closure.} = discard + +method query*(self: NullDatastore): QueryIterator {.locks: "unknown".} = + queryImpl diff --git a/datastore/sqlite_datastore.nim b/datastore/sqlite_datastore.nim index a22eceda..3e5dafc4 100644 --- a/datastore/sqlite_datastore.nim +++ b/datastore/sqlite_datastore.nim @@ -284,7 +284,10 @@ proc dbPath*(self: SQLiteDatastore): string = proc env*(self: SQLiteDatastore): SQLite = self.env -proc close*(self: SQLiteDatastore) = +proc timestamp*(t = epochTime()): int64 = + (t * 1_000_000).int64 + +method close*(self: SQLiteDatastore) {.async, locks: "unknown".} = self.containsStmt.dispose self.getStmt.dispose @@ -295,9 +298,6 @@ proc close*(self: SQLiteDatastore) = self.env.dispose self[] = SQLiteDatastore()[] -proc timestamp*(t = epochTime()): int64 = - (t * 1_000_000).int64 - method contains*( self: SQLiteDatastore, key: Key): Future[?!bool] {.async, locks: "unknown".} = @@ -367,13 +367,14 @@ method put*( return await self.put(key, data, timestamp()) -iterator query*( - self: SQLiteDatastore, - query: Query): Future[QueryResponse] = +iterator queryImpl( + datastore: Datastore, + query: Query): Future[QueryResponse] {.closure.} = let + datastore = SQLiteDatastore(datastore) queryStmt = QueryStmt.prepare( - self.env, queryStmtStr).expect("should not fail") + datastore.env, queryStmtStr).expect("should not fail") s = RawStmtPtr(queryStmt) @@ -428,3 +429,6 @@ iterator query*( break else: raise (ref Defect)(msg: $sqlite3_errstr(v)) + +method query*(self: SQLiteDatastore): QueryIterator {.locks: "unknown".} = + queryImpl diff --git a/datastore/tiered_datastore.nim b/datastore/tiered_datastore.nim index 6b2b50a9..cca814fb 100644 --- a/datastore/tiered_datastore.nim +++ b/datastore/tiered_datastore.nim @@ -28,6 +28,10 @@ proc new*( proc stores*(self: TieredDatastore): seq[Datastore] = self.stores +method close*(self: TieredDatastore) {.async, locks: "unknown".} = + for store in self.stores: + await store.close + method contains*( self: TieredDatastore, key: Key): Future[?!bool] {.async, locks: "unknown".} = @@ -49,7 +53,10 @@ method delete*( pending = await allFinished(self.stores.mapIt(it.delete(key))) for fut in pending: - if fut.read().isErr: return fut.read() + let + delRes = await fut + + if delRes.isErr: return delRes return success() @@ -90,12 +97,26 @@ method put*( pending = await allFinished(self.stores.mapIt(it.put(key, data))) for fut in pending: - if fut.read().isErr: return fut.read() + let + putRes = await fut + + if putRes.isErr: return putRes return success() -# method query*( -# self: TieredDatastore, -# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = -# -# return success ....some +iterator queryImpl( + datastore: Datastore, + query: Query): Future[QueryResponse] {.closure.} = + + let + datastore = TieredDatastore(datastore) + # https://github.com/datastore/datastore/blob/7ccf0cd4748001d3dbf5e6dda369b0f63e0269d3/datastore/core/basic.py#L1027-L1035 + bottom = datastore.stores[^1] + + try: + let q = bottom.query(); for kv in q(bottom, query): yield kv + except Exception as e: + raise (ref Defect)(msg: e.msg) + +method query*(self: TieredDatastore): QueryIterator {.locks: "unknown".} = + queryImpl diff --git a/tests/datastore/test_datastore.nim b/tests/datastore/test_datastore.nim index ccf4d75f..ffda20dc 100644 --- a/tests/datastore/test_datastore.nim +++ b/tests/datastore/test_datastore.nim @@ -25,5 +25,4 @@ suite "Datastore (base)": expect Defect: discard ds.get(key) asyncTest "query": - expect Defect: - for n in ds.query(Query.init(key)): discard + expect Defect: discard ds.query diff --git a/tests/datastore/test_filesystem_datastore.nim b/tests/datastore/test_filesystem_datastore.nim index 3c93244e..ab90ae4f 100644 --- a/tests/datastore/test_filesystem_datastore.nim +++ b/tests/datastore/test_filesystem_datastore.nim @@ -190,7 +190,3 @@ suite "FileSystemDatastore": getOpt = getRes.get check: getOpt.isNone - - # asyncTest "query": - # check: - # true diff --git a/tests/datastore/test_null_datastore.nim b/tests/datastore/test_null_datastore.nim index c37d4f3c..af4551af 100644 --- a/tests/datastore/test_null_datastore.nim +++ b/tests/datastore/test_null_datastore.nim @@ -35,7 +35,7 @@ suite "NullDatastore": var x = true - for n in ds.query(Query.init(key)): + let q = ds.query; for n in q(ds, Query.init(key)): # `iterator query` for NullDatastore never yields so the following lines # are not run (else the test would hang) x = false diff --git a/tests/datastore/test_sqlite_datastore.nim b/tests/datastore/test_sqlite_datastore.nim index 143d2fe5..272ee9d8 100644 --- a/tests/datastore/test_sqlite_datastore.nim +++ b/tests/datastore/test_sqlite_datastore.nim @@ -26,7 +26,7 @@ suite "SQLiteDatastore": createDir(basePathAbs) teardown: - if not ds.isNil: ds.close + if not ds.isNil: await ds.close ds = nil removeDir(basePathAbs) require(not dirExists(basePathAbs)) @@ -49,7 +49,7 @@ suite "SQLiteDatastore": dsRes.isOk fileExists(dbPathAbs) - dsRes.get.close + await dsRes.get.close removeDir(basePathAbs) assert not dirExists(basePathAbs) createDir(basePathAbs) @@ -60,7 +60,7 @@ suite "SQLiteDatastore": dsRes.isOk fileExists(dbPathAbs) - dsRes.get.close + await dsRes.get.close # for `readOnly = true` to succeed the database file must already exist, so # the existing file (per previous step) is not deleted prior to the next @@ -70,7 +70,7 @@ suite "SQLiteDatastore": check: dsRes.isOk - dsRes.get.close + await dsRes.get.close removeDir(basePathAbs) assert not dirExists(basePathAbs) createDir(basePathAbs) @@ -79,7 +79,7 @@ suite "SQLiteDatastore": check: dsRes.isOk - dsRes.get.close + await dsRes.get.close dsRes = SQLiteDatastore.new(memory, readOnly = true) @@ -95,7 +95,7 @@ suite "SQLiteDatastore": asyncTest "helpers": ds = SQLiteDatastore.new(basePath).get - ds.close + await ds.close check: ds.env.isNil @@ -107,7 +107,7 @@ suite "SQLiteDatastore": # for `readOnly = true` to succeed the database file must already exist ds = SQLiteDatastore.new(basePathAbs, filename).get - ds.close + await ds.close ds = SQLiteDatastore.new(basePathAbs, filename, readOnly = true).get var @@ -117,7 +117,7 @@ suite "SQLiteDatastore": check: putRes.isErr - ds.close + await ds.close removeDir(basePathAbs) assert not dirExists(basePathAbs) createDir(basePathAbs) @@ -211,7 +211,7 @@ suite "SQLiteDatastore": # for `readOnly = true` to succeed the database file must already exist ds = SQLiteDatastore.new(basePathAbs, filename).get - ds.close + await ds.close ds = SQLiteDatastore.new(basePathAbs, filename, readOnly = true).get var @@ -219,7 +219,7 @@ suite "SQLiteDatastore": check: delRes.isErr - ds.close + await ds.close removeDir(basePathAbs) assert not dirExists(basePathAbs) createDir(basePathAbs) @@ -387,19 +387,19 @@ suite "SQLiteDatastore": assert putRes.isOk var - kds: seq[QueryResponse] + kvs: seq[QueryResponse] - for kd in ds.query(Query.init(queryKey)): + var q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) # see https://sqlite.org/lang_select.html#the_order_by_clause # If a SELECT statement that returns more than one row does not have an # ORDER BY clause, the order in which the rows are returned is undefined. - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key1, data: bytes1), (key: key2, data: bytes2), (key: key3, data: bytes3), @@ -414,17 +414,17 @@ suite "SQLiteDatastore": (key: key12, data: bytes12) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("a*").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key1, data: bytes1), (key: key2, data: bytes2), (key: key3, data: bytes3), @@ -433,17 +433,17 @@ suite "SQLiteDatastore": (key: key6, data: bytes6) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("A*").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key7, data: bytes7), (key: key8, data: bytes8), (key: key9, data: bytes9), @@ -452,67 +452,67 @@ suite "SQLiteDatastore": (key: key12, data: bytes12) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("a/?").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key2, data: bytes2) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("A/?").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key8, data: bytes8) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("*/?").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key2, data: bytes2), (key: key5, data: bytes5), (key: key8, data: bytes8), (key: key11, data: bytes11) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("[Aa]/?").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (key, data) = await kv - kds.add (key, data) + kvs.add (key, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key2, data: bytes2), (key: key8, data: bytes8) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] # SQLite's GLOB operator, akin to Unix file globbing syntax, is greedy re: # wildcard "*". So a pattern such as "a:*[^/]" will not restrict results to @@ -520,33 +520,33 @@ suite "SQLiteDatastore": queryKey = Key.init("a:*[^/]").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (key, data) = await kv - kds.add (key, data) + kvs.add (key, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key4, data: bytes4), (key: key5, data: bytes5), (key: key6, data: bytes6) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("a:*[Bb]").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (key, data) = await kv - kds.add (key, data) + kvs.add (key, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key4, data: bytes4) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] var deleteRes = await ds.delete(key1) @@ -576,12 +576,12 @@ suite "SQLiteDatastore": assert deleteRes.isOk let - emptyKds: seq[QueryResponse] = @[] + emptyKvs: seq[QueryResponse] = @[] - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (key, data) = await kv - kds.add (key, data) + kvs.add (key, data) - check: kds == emptyKds + check: kvs == emptyKvs diff --git a/tests/datastore/test_tiered_datastore.nim b/tests/datastore/test_tiered_datastore.nim index 5dfa68ca..91b50fe7 100644 --- a/tests/datastore/test_tiered_datastore.nim +++ b/tests/datastore/test_tiered_datastore.nim @@ -1,3 +1,4 @@ +import std/algorithm import std/options import std/os @@ -19,21 +20,21 @@ suite "TieredDatastore": rootAbs = getCurrentDir() / root var - ds1: SQLiteDatastore - ds2: FileSystemDatastore + ds1: FileSystemDatastore + ds2: SQLiteDatastore setup: removeDir(rootAbs) require(not dirExists(rootAbs)) createDir(rootAbs) - ds1 = SQLiteDatastore.new(memory).get - ds2 = FileSystemDatastore.new(rootAbs).get + ds1 = FileSystemDatastore.new(rootAbs).get + ds2 = SQLiteDatastore.new(memory).get teardown: - if not ds1.isNil: ds1.close - ds1 = nil removeDir(rootAbs) require(not dirExists(rootAbs)) + if not ds2.isNil: await ds2.close + ds2 = nil asyncTest "new": check: @@ -132,8 +133,10 @@ suite "TieredDatastore": (await ds1.get(key)).get.get == bytes (await ds2.get(key)).get.get == bytes - ds1.close - ds1 = SQLiteDatastore.new(memory).get + removeDir(rootAbs) + assert (not dirExists(rootAbs)) + createDir(rootAbs) + ds1 = FileSystemDatastore.new(rootAbs).get ds = TieredDatastore.new(ds1, ds2).get assert (await ds1.get(key)).get.isNone @@ -149,6 +152,55 @@ suite "TieredDatastore": (await ds1.get(key)).get.isSome (await ds1.get(key)).get.get == bytes - # asyncTest "query": - # check: - # true + asyncTest "query": + let + ds = TieredDatastore.new(ds1, ds2).get + + key1 = Key.init("a/b").get + key2 = Key.init("a/b:c").get + key3 = Key.init("a/b:c/d").get + + bytes1 = @[1.byte, 2.byte, 3.byte] + bytes2 = @[4.byte, 5.byte, 6.byte] + bytes3: seq[byte] = @[] + + queryKey1 = Key.init("a/*").get + queryKey2 = Key.init("b/*").get + + var + putRes = await ds.put(key1, bytes1) + + assert putRes.isOk + putRes = await ds.put(key2, bytes2) + assert putRes.isOk + putRes = await ds.put(key3, bytes3) + assert putRes.isOk + + var + kvs: seq[QueryResponse] + + var q = ds.query(); for kv in q(ds, Query.init(queryKey1)): + let + (key, data) = await kv + + kvs.add (key, data) + + check: kvs.sortedByIt(it.key.id) == @[ + (key: key1, data: bytes1), + (key: key2, data: bytes2), + (key: key3, data: bytes3) + ].sortedByIt(it.key.id) + + kvs = @[] + q = ds.query() + + let + emptyKvs: seq[QueryResponse] = @[] + + for kv in q(ds, Query.init(queryKey2)): + let + (key, data) = await kv + + kvs.add (key, data) + + check: kvs == emptyKvs