diff --git a/datastore/datastore.nim b/datastore/datastore.nim index 13c1a0f8..dd04725f 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -4,8 +4,9 @@ import pkg/questionable/results import pkg/upraises import ./key +import ./query -export key +export key, query push: {.upraises: [].} @@ -37,8 +38,8 @@ method put*( raiseAssert("Not implemented!") -# method query*( -# self: Datastore, -# query: ...): Future[?!(?...)] {.async, base, locks: "unknown".} = -# -# raiseAssert("Not implemented!") +iterator query*( + self: Datastore, + query: Query): Future[QueryResponse] = + + raiseAssert("Not implemented!") diff --git a/datastore/key.nim b/datastore/key.nim index 70b317c7..1cd32e01 100644 --- a/datastore/key.nim +++ b/datastore/key.nim @@ -211,7 +211,7 @@ template `[]`*( proc len*(self: Key): int = self.namespaces.len -iterator items*(key: Key): Namespace {.inline.} = +iterator items*(key: Key): Namespace = var i = 0 diff --git a/datastore/null_datastore.nim b/datastore/null_datastore.nim index 66d4082f..cdb7444a 100644 --- a/datastore/null_datastore.nim +++ b/datastore/null_datastore.nim @@ -40,8 +40,8 @@ method put*( return success() -# method query*( -# self: NullDatastore, -# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = -# -# return success ....none +iterator query*( + self: NullDatastore, + query: Query): Future[QueryResponse] = + + discard diff --git a/datastore/query.nim b/datastore/query.nim new file mode 100644 index 00000000..b4aa8f1d --- /dev/null +++ b/datastore/query.nim @@ -0,0 +1,18 @@ +import ./key + +type + Query* = object + key: QueryKey + + QueryKey* = Key + + QueryResponse* = tuple[key: Key, data: seq[byte]] + +proc init*( + T: type Query, + key: QueryKey): T = + + T(key: key) + +proc key*(self: Query): QueryKey = + self.key diff --git a/datastore/sqlite.nim b/datastore/sqlite.nim index a36e4d74..7a4d66ac 100644 --- a/datastore/sqlite.nim +++ b/datastore/sqlite.nim @@ -30,6 +30,13 @@ type SQLiteStmt*[Params, Res] = distinct RawStmtPtr + # see https://github.com/arnetheduck/nim-sqlite3-abi/issues/4 + sqlite3_destructor_type_gcsafe = + proc (a1: pointer) {.cdecl, gcsafe, upraises: [].} + +const + SQLITE_TRANSIENT_GCSAFE* = cast[sqlite3_destructor_type_gcsafe](-1) + proc bindParam( s: RawStmtPtr, n: int, @@ -248,8 +255,8 @@ proc sqlite3_column_text_not_null*( # https://www.sqlite.org/c3ref/column_blob.html # a null pointer here implies an out-of-memory error let - code = sqlite3_errcode(sqlite3_db_handle(s)) + v = sqlite3_errcode(sqlite3_db_handle(s)) - raise (ref Defect)(msg: $sqlite3_errstr(code)) + raise (ref Defect)(msg: $sqlite3_errstr(v)) text diff --git a/datastore/sqlite_datastore.nim b/datastore/sqlite_datastore.nim index 9406fd1d..c188ee14 100644 --- a/datastore/sqlite_datastore.nim +++ b/datastore/sqlite_datastore.nim @@ -6,7 +6,7 @@ import pkg/questionable import pkg/questionable/results import pkg/sqlite3_abi import pkg/stew/byteutils -from pkg/stew/results as stewResults import get, isErr +from pkg/stew/results as stewResults import isErr import pkg/upraises import ./datastore @@ -34,6 +34,8 @@ type PutStmt = SQLiteStmt[(string, seq[byte], int64), void] + QueryStmt = SQLiteStmt[(string), void] + SQLiteDatastore* = ref object of Datastore dbPath: string containsStmt: ContainsStmt @@ -97,6 +99,14 @@ const ) VALUES (?, ?, ?); """ + queryStmtStr = """ + SELECT """ & idColName & """, """ & dataColName & """ FROM """ & tableName & + """ WHERE """ & idColName & """ GLOB ?; + """ + + queryStmtIdCol = 0 + queryStmtDataCol = 1 + proc checkColMetadata(s: RawStmtPtr, i: int, expectedName: string) = let colName = sqlite3_column_origin_name(s, i.cint) @@ -140,10 +150,10 @@ proc dataCol*( # result code is an error code if blob.isNil: let - code = sqlite3_errcode(sqlite3_db_handle(s)) + v = sqlite3_errcode(sqlite3_db_handle(s)) - if not (code in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): - raise (ref Defect)(msg: $sqlite3_errstr(code)) + if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): + raise (ref Defect)(msg: $sqlite3_errstr(v)) let dataLen = sqlite3_column_bytes(s, i) @@ -339,8 +349,64 @@ method put*( return await self.put(key, data, timestamp()) -# method query*( -# self: SQLiteDatastore, -# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = -# -# return success ....some +iterator query*( + self: SQLiteDatastore, + query: Query): Future[QueryResponse] = + + let + queryStmt = QueryStmt.prepare( + self.env, queryStmtStr).expect("should not fail") + + s = RawStmtPtr(queryStmt) + + defer: + discard sqlite3_reset(s) + discard sqlite3_clear_bindings(s) + s.dispose + + let + v = sqlite3_bind_text(s, 1.cint, query.key.id.cstring, -1.cint, + SQLITE_TRANSIENT_GCSAFE) + + if not (v == SQLITE_OK): + raise (ref Defect)(msg: $sqlite3_errstr(v)) + + while true: + let + v = sqlite3_step(s) + + case v + of SQLITE_ROW: + let + key = Key.init($sqlite3_column_text_not_null( + s, queryStmtIdCol)).expect("should not fail") + + blob = sqlite3_column_blob(s, queryStmtDataCol) + + # detect out-of-memory error + # see the conversion table and final paragraph of: + # https://www.sqlite.org/c3ref/column_blob.html + # see also https://www.sqlite.org/rescode.html + + # the "data" column can be NULL so in order to detect an out-of-memory + # error it is necessary to check that the result is a null pointer and + # that the result code is an error code + if blob.isNil: + let + v = sqlite3_errcode(sqlite3_db_handle(s)) + + if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]): + raise (ref Defect)(msg: $sqlite3_errstr(v)) + + let + dataLen = sqlite3_column_bytes(s, queryStmtDataCol) + dataBytes = cast[ptr UncheckedArray[byte]](blob) + data = @(toOpenArray(dataBytes, 0, dataLen - 1)) + fut = newFuture[QueryResponse]() + + fut.complete((key, data)) + yield fut + of SQLITE_DONE: + break + else: + raise (ref Defect)(msg: $sqlite3_errstr(v)) diff --git a/tests/datastore/test_datastore.nim b/tests/datastore/test_datastore.nim index 1ca3ef5c..ccf4d75f 100644 --- a/tests/datastore/test_datastore.nim +++ b/tests/datastore/test_datastore.nim @@ -11,10 +11,9 @@ suite "Datastore (base)": let key = Key.init("a").get ds = Datastore() - oneByte = @[1.byte] asyncTest "put": - expect Defect: discard ds.put(key, oneByte) + expect Defect: discard ds.put(key, @[1.byte]) asyncTest "delete": expect Defect: discard ds.delete(key) @@ -25,5 +24,6 @@ suite "Datastore (base)": asyncTest "get": expect Defect: discard ds.get(key) - # asyncTest "query": - # expect Defect: discard ds.query(...) + asyncTest "query": + expect Defect: + for n in ds.query(Query.init(key)): discard diff --git a/tests/datastore/test_null_datastore.nim b/tests/datastore/test_null_datastore.nim index 2b658786..c37d4f3c 100644 --- a/tests/datastore/test_null_datastore.nim +++ b/tests/datastore/test_null_datastore.nim @@ -31,7 +31,14 @@ suite "NullDatastore": (await ds.get(key)).isOk (await ds.get(key)).get.isNone - # asyncTest "query": - # check: - # (await ds.query(...)).isOk - # (await ds.query(...)).get.isNone + asyncTest "query": + var + x = true + + for n in ds.query(Query.init(key)): + # `iterator query` for NullDatastore never yields so the following lines + # are not run (else the test would hang) + x = false + discard (await n) + + check: x diff --git a/tests/datastore/test_sqlite_datastore.nim b/tests/datastore/test_sqlite_datastore.nim index 23302f8f..2a902a76 100644 --- a/tests/datastore/test_sqlite_datastore.nim +++ b/tests/datastore/test_sqlite_datastore.nim @@ -1,3 +1,4 @@ +import std/algorithm import std/options import std/os @@ -331,6 +332,261 @@ suite "SQLiteDatastore": check: getOpt.isNone - # asyncTest "query": - # check: - # true + asyncTest "query": + ds = SQLiteDatastore.new(basePathAbs, filename).get + + var + key1 = Key.init("a").get + key2 = Key.init("a/b").get + key3 = Key.init("a/b:c").get + key4 = Key.init("a:b").get + key5 = Key.init("a:b/c").get + key6 = Key.init("a:b/c:d").get + key7 = Key.init("A").get + key8 = Key.init("A/B").get + key9 = Key.init("A/B:C").get + key10 = Key.init("A:B").get + key11 = Key.init("A:B/C").get + key12 = Key.init("A:B/C:D").get + + bytes1 = @[1.byte, 2.byte, 3.byte] + bytes2 = @[4.byte, 5.byte, 6.byte] + bytes3: seq[byte] = @[] + bytes4 = bytes1 + bytes5 = bytes2 + bytes6 = bytes3 + bytes7 = bytes1 + bytes8 = bytes2 + bytes9 = bytes3 + bytes10 = bytes1 + bytes11 = bytes2 + bytes12 = bytes3 + + queryKey = Key.init("*").get + + var + putRes = await ds.put(key1, bytes1) + + assert putRes.isOk + putRes = await ds.put(key2, bytes2) + assert putRes.isOk + putRes = await ds.put(key3, bytes3) + assert putRes.isOk + putRes = await ds.put(key4, bytes4) + assert putRes.isOk + putRes = await ds.put(key5, bytes5) + assert putRes.isOk + putRes = await ds.put(key6, bytes6) + assert putRes.isOk + putRes = await ds.put(key7, bytes7) + assert putRes.isOk + putRes = await ds.put(key8, bytes8) + assert putRes.isOk + putRes = await ds.put(key9, bytes9) + assert putRes.isOk + putRes = await ds.put(key10, bytes10) + assert putRes.isOk + putRes = await ds.put(key11, bytes11) + assert putRes.isOk + putRes = await ds.put(key12, bytes12) + assert putRes.isOk + + var + kds: seq[QueryResponse] + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + # see https://sqlite.org/lang_select.html#the_order_by_clause + # If a SELECT statement that returns more than one row does not have an + # ORDER BY clause, the order in which the rows are returned is undefined. + + check: kds.sortedByIt(it.key.id) == @[ + (key: key1, data: bytes1), + (key: key2, data: bytes2), + (key: key3, data: bytes3), + (key: key4, data: bytes4), + (key: key5, data: bytes5), + (key: key6, data: bytes6), + (key: key7, data: bytes7), + (key: key8, data: bytes8), + (key: key9, data: bytes9), + (key: key10, data: bytes10), + (key: key11, data: bytes11), + (key: key12, data: bytes12) + ].sortedByIt(it.key.id) + + kds = @[] + + queryKey = Key.init("a*").get + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + check: kds.sortedByIt(it.key.id) == @[ + (key: key1, data: bytes1), + (key: key2, data: bytes2), + (key: key3, data: bytes3), + (key: key4, data: bytes4), + (key: key5, data: bytes5), + (key: key6, data: bytes6) + ].sortedByIt(it.key.id) + + kds = @[] + + queryKey = Key.init("A*").get + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + check: kds.sortedByIt(it.key.id) == @[ + (key: key7, data: bytes7), + (key: key8, data: bytes8), + (key: key9, data: bytes9), + (key: key10, data: bytes10), + (key: key11, data: bytes11), + (key: key12, data: bytes12) + ].sortedByIt(it.key.id) + + kds = @[] + + queryKey = Key.init("a/?").get + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + check: kds.sortedByIt(it.key.id) == @[ + (key: key2, data: bytes2) + ].sortedByIt(it.key.id) + + kds = @[] + + queryKey = Key.init("A/?").get + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + check: kds.sortedByIt(it.key.id) == @[ + (key: key8, data: bytes8) + ].sortedByIt(it.key.id) + + kds = @[] + + queryKey = Key.init("*/?").get + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + check: kds.sortedByIt(it.key.id) == @[ + (key: key2, data: bytes2), + (key: key5, data: bytes5), + (key: key8, data: bytes8), + (key: key11, data: bytes11) + ].sortedByIt(it.key.id) + + kds = @[] + + queryKey = Key.init("[Aa]/?").get + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + check: kds.sortedByIt(it.key.id) == @[ + (key: key2, data: bytes2), + (key: key8, data: bytes8) + ].sortedByIt(it.key.id) + + kds = @[] + + # SQLite's GLOB operator, akin to Unix file globbing syntax, is greedy re: + # wildcard "*". So a pattern such as "a:*[^/]" will not restrict results to + # "/a:b", i.e. it will match on "/a:b", "/a:b/c" and "/a:b/c:d". + + queryKey = Key.init("a:*[^/]").get + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + check: kds.sortedByIt(it.key.id) == @[ + (key: key4, data: bytes4), + (key: key5, data: bytes5), + (key: key6, data: bytes6) + ].sortedByIt(it.key.id) + + kds = @[] + + queryKey = Key.init("a:*[Bb]").get + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + check: kds.sortedByIt(it.key.id) == @[ + (key: key4, data: bytes4) + ].sortedByIt(it.key.id) + + kds = @[] + + var + deleteRes = await ds.delete(key1) + + assert deleteRes.isOk + deleteRes = await ds.delete(key2) + assert deleteRes.isOk + deleteRes = await ds.delete(key3) + assert deleteRes.isOk + deleteRes = await ds.delete(key4) + assert deleteRes.isOk + deleteRes = await ds.delete(key5) + assert deleteRes.isOk + deleteRes = await ds.delete(key6) + assert deleteRes.isOk + deleteRes = await ds.delete(key7) + assert deleteRes.isOk + deleteRes = await ds.delete(key8) + assert deleteRes.isOk + deleteRes = await ds.delete(key9) + assert deleteRes.isOk + deleteRes = await ds.delete(key10) + assert deleteRes.isOk + deleteRes = await ds.delete(key11) + assert deleteRes.isOk + deleteRes = await ds.delete(key12) + assert deleteRes.isOk + + let + emptyKds: seq[QueryResponse] = @[] + + for kd in ds.query(Query.init(queryKey)): + let + (key, data) = await kd + + kds.add (key, data) + + check: kds == emptyKds