diff --git a/dependencies/lmdb/libraries/liblmdb/mdb.c b/dependencies/lmdb/libraries/liblmdb/mdb.c index 43929a510..a47fcaec3 100644 --- a/dependencies/lmdb/libraries/liblmdb/mdb.c +++ b/dependencies/lmdb/libraries/liblmdb/mdb.c @@ -1656,7 +1656,6 @@ struct MDB_env { # define me_freelist_written_end me_pgstate.mf_written_end unsigned int me_maxfreepgs_to_load; /**< max freelist entries to load into memory */ unsigned int me_maxfreepgs_to_retain; /**< max freelist entries to load into memory */ - int me_freelist_state; /**< state of writing freelist (if it is being deleted) */ MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ /** IDL of pages that became unused in a write txn */ MDB_IDL me_free_pgs; @@ -2856,7 +2855,6 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) load_more: if (op == MDB_SET_RANGE) { /* 1st iteration */ /* Prepare to fetch more and coalesce */ - if (env->me_freelist_state & MDB_FREELIST_DELETING) break; oldest = env->me_pgoldest; mdb_cursor_init(&m2, txn, FREE_DBI, NULL); #if (MDB_DEVEL) & 2 /* "& 2" so MDB_DEVEL=1 won't hide bugs breaking freeDB */ @@ -2883,20 +2881,6 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) key.mv_size = sizeof(last); rc = mdb_cursor_get(&m2, &key, NULL, op); op = MDB_NEXT; // now iterate forwards through the txns of free list - if (rc) { - if (rc == MDB_NOTFOUND) { // if not found, go to the beginning of the range and look for older txns - if (env->me_freelist_state & MDB_FREELIST_DELETING) { - //fprintf(stderr, "Skipping unsafe backwards iteration 1\n"); - break; // but can't iterate backwards in deletion mode - } - mdb_cursor_last(&m2, &key, NULL); - env->me_freelist_end = oldest; - op = MDB_PREV; - rc = mdb_cursor_get(&m2, &key, NULL, op); - } - if (rc && rc != MDB_NOTFOUND) goto fail; - } - mdb_cassert(&m2, key.mv_size > 0); last = *(txnid_t *) key.mv_data; } else { // no more transactions to read going forward through newest, we are now going @@ -2911,11 +2895,6 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) } } else { // we are now iterating through the free list entries - // first, we need to check if we are in free-list deletion mode, in which case it is not safe to iterate backwards and we have to bail out - if (op == MDB_PREV && (env->me_freelist_state & MDB_FREELIST_DELETING)) { - //fprintf(stderr, "Skipping unsafe backwards iteration 1\n"); - break; - } // now iterate rc = mdb_cursor_get(&m2, &key, NULL, op); if (rc && rc != MDB_NOTFOUND) @@ -2928,13 +2907,12 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) if (op == MDB_NEXT) { // iterating forward from the freelist range to find newer transactions if (last >= oldest || rc == MDB_NOTFOUND) { - if (!rc) env->me_freelist_end = oldest; + env->me_freelist_end = oldest; // no more newer transactions, go to the beginning of the range and look for older txns - if (env->me_freelist_state & MDB_FREELIST_DELETING) break; op = MDB_SET_RANGE; if (env->me_freelist_start <= 1) break; // should be no zero entry, break out last = env->me_freelist_start - 1; - key.mv_data = &last; // start at the end of the freelist and read newer transactions free pages + key.mv_data = &last; // start at the start of the freelist and read older transactions free pages key.mv_size = sizeof(last); mdb_cursor_init(&m2, txn, FREE_DBI, NULL); rc = mdb_cursor_get(&m2, &key, NULL, op); @@ -2946,8 +2924,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) if (rc == MDB_NOTFOUND) break; else goto fail; } - } - goto fail; + } else goto fail; } mdb_cassert(&m2, key.mv_size > 0); last = *(txnid_t*)key.mv_data; @@ -2956,21 +2933,6 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) rc = MDB_NOTFOUND; goto fail; } - if (last >= env->me_freelist_start) { - // go to previous entry, through next iteration - rc = mdb_cursor_get(&m2, &key, NULL, op); - if (rc) { - if (rc == MDB_NOTFOUND) { - // reached the very start, mark it as 1 - env->me_freelist_start = 1; - break; - } - goto fail; - } else - mdb_cassert(&m2, key.mv_size > 0); - last = *(txnid_t*)key.mv_data; - } - env->me_freelist_start = last; } else env->me_freelist_end = last + 1; } else { @@ -2979,6 +2941,26 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) env->me_freelist_start = 1; break; } + } + if (op == MDB_PREV) { + // move to previous entry until we are before the last start time. + // note that occasionally this might take multiple iterations if we are in the middle of a + // rebalance and a node being moved + while (last >= env->me_freelist_start) { + // go to previous entry, through prev iteration + rc = mdb_cursor_get(&m2, &key, NULL, MDB_PREV); + if (rc) { + if (rc == MDB_NOTFOUND) { + // reached the very start, mark it as 1 + env->me_freelist_start = 1; + break; + } + goto fail; + } else + mdb_cassert(&m2, key.mv_size > 0); + last = *(txnid_t *) key.mv_data; + } + if (rc == MDB_NOTFOUND) break; env->me_freelist_start = last; } if (!last) { @@ -4288,9 +4270,7 @@ mdb_freelist_save(MDB_txn *txn) total_room = head_room = 0; mdb_tassert(txn, head_id >= env->me_freelist_start); //fprintf(stderr, "Deleting free list record %u\n", head_id); - env->me_freelist_state = MDB_FREELIST_DELETING; // signal that we are deleting from the freelist, which means we can't iterate backwards rc = mdb_cursor_del(&mc, 0); - env->me_freelist_state = 0; if (rc) { last_error = "Attempting to delete free-space record"; return rc; @@ -10509,7 +10489,6 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) if ((rc = mdb_page_touch(csrc)) || (rc = mdb_page_touch(cdst))) return rc; - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { key.mv_size = csrc->mc_db->md_pad; key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); diff --git a/package.json b/package.json index dd0330b9d..d0a6781d8 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "dependencies": { "msgpackr": "^1.9.9", "node-addon-api": "^6.1.0", - "node-gyp-build-optional-packages": "5.1.1", + "node-gyp-build-optional-packages": "5.2.0", "ordered-binary": "^1.4.1", "weak-lru-cache": "^1.2.2" }, diff --git a/test/index.test.js b/test/index.test.js index a141f31e4..01bd71279 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -414,11 +414,18 @@ describe('lmdb-js', function () { let promise; let additive = 'this is more text'; for (let i = 0; i < 7; i++) additive += additive; + let read_txn = db.useReadTransaction(); for (let i = 0; i < 5000; i++) { + if (Math.random() < 0.3) { + read_txn.done(); + read_txn = db.useReadTransaction(); + } let text = 'this is a test'; while (random() < 0.95) text += additive; - promise = db.put(i % 10, text); - if (i % 16 == 0) { + if (random() < 0.4) promise = db.remove(i % 40); + else promise = db.put(i % 40, text); + + if (i % 2 == 0) { await promise; } }