Add getKeys function, handle reverse traversal with dup sort, add typ…

…e definitions and more testing, avoid using exceptions for del, #22

README.md

@@ -103,7 +103,7 @@ Again, if this is performed inside a transation, the removal will be included in
 This will set the provided value at the specified key, but will do so synchronously. If this is called inside of a synchronous transaction, this put will be added to the current transaction. If not, a transaction will be started, the put will be executed, and the transaction will be committed, and then the function will return. We do not recommend this be used for any high-frequency operations as it can be vastly slower (for the main JS thread) than the `put` operation (often taking multiple milliseconds).
 
 ### `store.removeSync(key, valueOrIfVersion?: number): boolean`
-This will delete the entry at the specified key. This functions like `putSync`, providing synchronous entry deletion, and uses the same arguments as `remove`.
+This will delete the entry at the specified key. This functions like `putSync`, providing synchronous entry deletion, and uses the same arguments as `remove`. This returns `true` if there was an existing entry deleted, `false` if there was no matching entry.
 
 ### `store.ifVersion(key, ifVersion: number, callback): Promise<boolean>`
 This executes a block of conditional writes, and conditionally execute any puts or removes that are called in the callback, using the provided condition that requires the provided key's entry to have the provided version.
@@ -114,7 +114,7 @@ This executes a block of conditional writes, and conditionally execute any puts
 ### `store.transaction(execute: Function)`
 This will begin synchronous transaction, execute the provided function, and then commit the transaction. The provided function can perform `get`s, `put`s, and `remove`s within the transaction, and the result will be committed. The execute function can return a promise to indicate an ongoing asynchronous transaction, but generally you want to minimize how long a transaction is open on the main thread, at least if you are potentially operating with multiple processes.
 
-### `store.getRange(options: { start?, end?, reverse?: boolean, limit?: number, values?: boolean, versions?: boolean}): Iterable<{ key, value: Buffer }>`
+### `store.getRange(options: { start?, end?, reverse?: boolean, limit?: number, versions?: boolean}): Iterable<{ key, value: Buffer }>`
 This starts a cursor-based query of a range of data in the database, returning an iterable that also has `map`, `filter`, and `forEach` methods. The `start` and `end` indicate the starting and ending key for the range. The `reverse` flag can be used to indicate reverse traversal. The `limit` can limit the number of entries returned. The returned cursor/query is lazy, and retrieves data _as_ iteration takes place, so a large range could specified without forcing all the entries to be read and loaded in memory upfront, and one can exit out of the loop without traversing the whole range in the database. The query is iterable, we can use it directly in a for-of:
 ```
 for (let { key, value } of db.getRange({ start, end })) {
@@ -133,7 +133,7 @@ Note that `map` and `filter` are also lazy, they will only be executed once thei
 
 If you want to get a true array from the range results, the `asArray` property will return the results as an array.
 
-### `store.getValues(key): Iterable<any>`
+### `store.getValues(key, options?): Iterable<any>`
 When using a store with duplicate entries per key (with `dupSort` flag), you can use this to retrieve all the values for a given key. This will return an iterator just like `getRange`, except each entry will be the value from the database:
 ```
 let db = store.openDB('my-index', {
@@ -149,6 +149,10 @@ for (let value of db.getValues('key1')) {
 	// just iterate value 'value1'
 }
 ```
+You can optionally provide a second argument with the same `options` that `getRange` handles.
+
+### `store.getKeys(options: { start?, end?, reverse?: boolean, limit?: number, versions?: boolean }): Iterable<any>`
+This behaves like `getRange`, but only returns the keys. If this is duplicate key database, each key is only returned once (even if it has multiple values/entries).
 
 ### `store.openDB(database: string|{name:string,...})`
 LMDB supports multiple databases per environment (an environment is a single memory-mapped file). When you initialize an LMDB store with `open`, the store uses the default root database. However, you can use multiple databases per environment/file and instantiate a store for each one. If you are going to be opening many databases, make sure you set the `maxDbs` (it defaults to 12). For example, we can open multiple stores for a single environment:

index.d.ts

@@ -14,10 +14,14 @@ declare namespace lmdb {
 		put(id: K, value: V, version: number, ifVersion?: number): Promise<boolean>
 		remove(id: K): Promise<boolean>
 		remove(id: K, ifVersion: number): Promise<boolean>
+		remove(id: K, valueToRemove: V): Promise<boolean>
 		putSync(id: K, value: V): void
 		putSync(id: K, value: V, version: number): void
-		removeSync(id: K): void
-		getRange(options: RangeOptions): ArrayLikeIterable<{ key: K, value: V, version: number }>
+		removeSync(id: K): boolean
+		removeSync(id: K, valueToRemove: V): boolean
+		getValues(key: K, options?: RangeOptions): ArrayLikeIterable<V>
+		getKeys(options: RangeOptions): ArrayLikeIterable<K>
+		getRange(options: RangeOptions): ArrayLikeIterable<{ key: K, value: V, version?: number }>
 		transaction<T>(action: () => T, abort?: boolean): T
 		ifVersion(id: K, ifVersion: number, action: () => any): Promise<boolean>
 		ifNoExists(id: K, action: () => any): Promise<boolean>
@@ -66,7 +70,6 @@ declare namespace lmdb {
 		start?: Key
 		end?: Key
 		reverse?: boolean
-		values?: boolean
 		versions?: boolean
 		limit?: number
 	}

index.js

@@ -407,21 +407,17 @@ function open(path, options) {
 					}
 				}
 				this.writes++
+				let result
 				if (deleteValue)
-					txn.del(this.db, id, deleteValue)
+					result = txn.del(this.db, id, deleteValue)
 				else
-					txn.del(this.db, id)
+					result = txn.del(this.db, id)
 				if (!writeTxn) {
 					txn.commit()
 					resetReadTxn()
 				}
-				return true // object found and deleted
+				return result // object found and deleted
 			} catch(error) {
-				if (error.message.startsWith('MDB_NOTFOUND')) {
-					if (!writeTxn)
-						txn.abort()
-					return false // calling remove on non-existent property is fine, but we will indicate its lack of existence with the return value
-				}
 				if (writeTxn)
 					throw error // if we are in a transaction, the whole transaction probably needs to restart
 				return handleError(error, this, txn, () => this.removeSync(id))
@@ -464,11 +460,18 @@ function open(path, options) {
 					return false
 				})
 		}
-		getValues(key) {
-			return this.getRange({
+		getValues(key, options) {
+			let defaultOptions = {
 				start: key,
 				valuesForKey: true
-			})
+			}
+			return this.getRange(options ? Object.assign(defaultOptions, options) : defaultOptions)
+		}
+		getKeys(options) {
+			if (!options)
+				options = {}
+			options.values = false
+			return this.getRange(options)
 		}
 		getRange(options) {
 			let iterable = new ArrayLikeIterable()
@@ -495,21 +498,31 @@ function open(path, options) {
 						cursor = new Cursor(txn, db)
 						txn.cursorCount = (txn.cursorCount || 0) + 1
 						if (reverse) {
-							// for reverse retrieval, goToRange is backwards because it positions at the key equal or *greater than* the provided key
-							let nextKey = cursor.goToRange(currentKey)
-							if (nextKey) {
-								if (compareKey(nextKey, currentKey)) {
-									// goToRange positioned us at a key after the provided key, so we need to go the previous key to be less than the provided key
-									currentKey = cursor.goToPrev()
-								} else
-									currentKey = nextKey // they match, we are good, and currentKey is already correct
+							if (valuesForKey) {
+								// position at key
+								currentKey = cursor.goToKey(currentKey)
+								// now move to next key and then previous entry to get to last value
+								if (currentKey) {
+									cursor.goToNextNoDup()
+									cursor.goToPrev()
+								}
 							} else {
-								// likewise, we have been position beyond the end of the index, need to go to last
-								currentKey = cursor.goToLast()
+								// for reverse retrieval, goToRange is backwards because it positions at the key equal or *greater than* the provided key
+								let nextKey = cursor.goToRange(currentKey)
+								if (nextKey) {
+									if (compareKey(nextKey, currentKey)) {
+										// goToRange positioned us at a key after the provided key, so we need to go the previous key to be less than the provided key
+										currentKey = cursor.goToPrev()
+									} else
+										currentKey = nextKey // they match, we are good, and currentKey is already correct
+								} else {
+									// likewise, we have been position beyond the end of the index, need to go to last
+									currentKey = cursor.goToLast()
+								}
 							}
 						} else {
 							// for forward retrieval, goToRange does what we want
-							currentKey = cursor.goToRange(currentKey)
+							currentKey = valuesForKey ? cursor.goToKey(currentKey) : cursor.goToRange(currentKey)
 						}
 						// TODO: Make a makeCompare(endKey)
 					} catch(error) {
@@ -538,7 +551,11 @@ function open(path, options) {
 						if (txn.isAborted)
 							resetCursor()
 						if (count > 0)
-							currentKey = reverse ? cursor.goToPrev() : valuesForKey ? cursor.goToNextDup() : cursor.goToNext()
+							currentKey = reverse ?
+								valuesForKey ? cursor.goToPrevDup() :
+									includeValues ? cursor.goToPrev() : cursor.goToPrevNoDup() :
+								valuesForKey ? cursor.goToNextDup() :
+									includeValues ? cursor.goToNext() : cursor.goToNextNoDup()
 						if (currentKey === undefined ||
 								(reverse ? compareKey(currentKey, endKey) <= 0 : compareKey(currentKey, endKey) >= 0) ||
 								(count++ >= options.limit)) {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "lmdb-store",
   "author": "Kris Zyp",
-  "version": "0.9.3",
+  "version": "0.9.4",
   "description": "Simple, efficient, scalable data store wrapper for LMDB",
   "license": "MIT",
   "repository": {
@@ -38,7 +38,7 @@
     "fs-extra": "^9.0.1",
     "lmdb-store-0.9": "0.7.3",
     "msgpackr": "^0.6.0",
-    "nan": "^2.14.1",
+    "nan": "^2.14.2",
     "node-gyp-build": "^4.2.3",
     "weak-lru-cache": "^0.4.0"
   },

src/cursor.cpp

@@ -286,6 +286,10 @@ MAKE_GET_FUNC(goToNextDup, MDB_NEXT_DUP);
 
 MAKE_GET_FUNC(goToPrevDup, MDB_PREV_DUP);
 
+MAKE_GET_FUNC(goToNextNoDup, MDB_NEXT_NODUP);
+
+MAKE_GET_FUNC(goToPrevNoDup, MDB_PREV_NODUP);
+
 static void fillDataFromArg1(CursorWrap* cw, Nan::NAN_METHOD_ARGS_TYPE info, MDB_val &data) {
     if (info[1]->IsString()) {
         CustomExternalStringResource::writeTo(Local<String>::Cast(info[1]), &data);
@@ -388,6 +392,8 @@ void CursorWrap::setupExports(Local<Object> exports) {
     cursorTpl->PrototypeTemplate()->Set(Nan::New<String>("goToLastDup").ToLocalChecked(), Nan::New<FunctionTemplate>(CursorWrap::goToLastDup));
     cursorTpl->PrototypeTemplate()->Set(Nan::New<String>("goToNextDup").ToLocalChecked(), Nan::New<FunctionTemplate>(CursorWrap::goToNextDup));
     cursorTpl->PrototypeTemplate()->Set(Nan::New<String>("goToPrevDup").ToLocalChecked(), Nan::New<FunctionTemplate>(CursorWrap::goToPrevDup));
+    cursorTpl->PrototypeTemplate()->Set(Nan::New<String>("goToNextNoDup").ToLocalChecked(), Nan::New<FunctionTemplate>(CursorWrap::goToNextNoDup));
+    cursorTpl->PrototypeTemplate()->Set(Nan::New<String>("goToPrevNoDup").ToLocalChecked(), Nan::New<FunctionTemplate>(CursorWrap::goToPrevNoDup));
     cursorTpl->PrototypeTemplate()->Set(Nan::New<String>("goToDup").ToLocalChecked(), Nan::New<FunctionTemplate>(CursorWrap::goToDup));
     cursorTpl->PrototypeTemplate()->Set(Nan::New<String>("goToDupRange").ToLocalChecked(), Nan::New<FunctionTemplate>(CursorWrap::goToDupRange));
     cursorTpl->PrototypeTemplate()->Set(Nan::New<String>("del").ToLocalChecked(), Nan::New<FunctionTemplate>(CursorWrap::del));

src/node-lmdb.h

@@ -827,6 +827,18 @@ class CursorWrap : public Nan::ObjectWrap {
     */
     static NAN_METHOD(goToPrevDup);
 
+    /*
+        Go to the entry for next key.
+        (Wrapper for `mdb_cursor_get`)
+    */
+    static NAN_METHOD(goToNextNoDup);
+
+    /*
+        Go to the entry for previous key.
+        (Wrapper for `mdb_cursor_get`)
+    */
+    static NAN_METHOD(goToPrevNoDup);
+
     /*
         For databases with the dupSort option. Asks the cursor to go to the specified key/data pair.
         (Wrapper for `mdb_cursor_get`)

src/txn.cpp

@@ -560,6 +560,10 @@ NAN_METHOD(TxnWrap::del) {
     }
 
     if (rc != 0) {
+        if (rc == MDB_NOTFOUND) {
+            return info.GetReturnValue().Set(Nan::False());
+        }
         return throwLmdbError(rc);
     }
+    return info.GetReturnValue().Set(Nan::True());
 }

test/index.test.js

@@ -157,6 +157,7 @@ describe('lmdb-store', function() {
       await db.put('key1',  dataIn);
       let dataOut = db.get('key1');
       dataOut.should.deep.equal(dataIn);
+      db.removeSync('not-there').should.equal(false);
     });
     it.skip('trigger sync commit', async function() {
       let dataIn = {foo: 4, bar: false}
@@ -185,15 +186,15 @@ describe('lmdb-store', function() {
       if (count != 2)
         throw new Error('Not enough entries')
     });
-    it('should iterate over dupsort query', async function() {
+    it('should iterate over dupsort query, with removal', async function() {
       let data1 = {foo: 1, bar: true}
       let data2 = {foo: 2, bar: false}
       let data3 = {foo: 3, bar: true}
       db2.put('key1',  data1);
       db2.put('key1',  data2);
       db2.put('key1',  data3);
       await db2.put('key2',  data3);
-      let count = 0
+      let count = 0;
       for (let value of db2.getValues('key1')) {
         count++
         switch(count) {
@@ -202,25 +203,41 @@ describe('lmdb-store', function() {
           case 3: data3.should.deep.equal(value); break;
         }
       }
-      if (count != 3)
-        throw new Error('Not enough entries')
-    });
-    it('should remove dupsort value and query', async function() {
-      let data1 = {foo: 1, bar: true}
-      let data2 = {foo: 2, bar: false}
-      let data3 = {foo: 3, bar: true}
+      count.should.equal(3);
       await db2.remove('key1',  data2);
-      let count = 0
+      count = 0;
       for (let value of db2.getValues('key1')) {
-        count++
+        count++;
         switch(count) {
           case 1: data1.should.deep.equal(value); break;
           case 2: data3.should.deep.equal(value); break;
         }
       }
-      if (count != 2)
-        throw new Error('Not enough entries')
+      count.should.equal(2)
+      count = 0;
+      for (let value of db2.getValues('key1', { reverse: true })) {
+        count++;
+        switch(count) {
+          case 1: data3.should.deep.equal(value); break;
+          case 2: data1.should.deep.equal(value); break;
+        }
+      }
+      count.should.equal(2);
+
+      count = 0;
+      for (let value of db2.getValues('key0')) {
+        count++;
+      }
+      count.should.equal(0);
     });
+    it('should iterate over keys without duplicates', async function() {
+      let lastKey
+      for (let key of db2.getKeys({ start: 'k' })) {
+        if (key == lastKey)
+          throw new Error('duplicate key returned')
+        lastKey = key
+      }
+    })
     it('invalid key', async function() {
       expect(() => db.get({ foo: 'bar' })).to.throw();
       //expect(() => db.put({ foo: 'bar' }, 'hello')).to.throw();