Start to vendor lmdb-js-lite

crates/lmdb-js-lite/.cargo/config.toml

@@ -0,0 +1,5 @@
+[target.aarch64-unknown-linux-musl]
+linker = "aarch64-linux-musl-gcc"
+rustflags = ["-C", "target-feature=-crt-static"]
+[target.x86_64-pc-windows-msvc]
+rustflags = ["-C", "target-feature=+crt-static"]

crates/lmdb-js-lite/.gitignore

@@ -0,0 +1,200 @@
+# Created by https://www.toptal.com/developers/gitignore/api/node
+# Edit at https://www.toptal.com/developers/gitignore?templates=node
+
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# Next.js build output
+.next
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# End of https://www.toptal.com/developers/gitignore/api/node
+
+# Created by https://www.toptal.com/developers/gitignore/api/macos
+# Edit at https://www.toptal.com/developers/gitignore?templates=macos
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two 
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
+# End of https://www.toptal.com/developers/gitignore/api/macos
+
+# Created by https://www.toptal.com/developers/gitignore/api/windows
+# Edit at https://www.toptal.com/developers/gitignore?templates=windows
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.toptal.com/developers/gitignore/api/windows
+
+#Added by cargo
+
+/target
+Cargo.lock
+
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+*.node
+
+*.db
+databases/

crates/lmdb-js-lite/.npmignore

@@ -0,0 +1,13 @@
+target
+Cargo.lock
+.cargo
+.github
+npm
+.eslintrc
+.prettierignore
+rustfmt.toml
+yarn.lock
+*.node
+.yarn
+__test__
+renovate.json

crates/lmdb-js-lite/.nvmrc

@@ -0,0 +1 @@
+v20

crates/lmdb-js-lite/.yarnrc.yml

@@ -0,0 +1,4 @@
+nodeLinker: node-modules
+
+yarnPath: .yarn/releases/yarn-4.4.0.cjs
+npmRegistryServer: https://registry.yarnpkg.com

crates/lmdb-js-lite/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+edition = "2021"
+name = "lmdb-js-lite"
+version = "0.1.5"
+repository = "https://github.com/yamadapc/lmdb-js-lite"
+
+[[bench]]
+name = "lmdb_js_safe_benchmark"
+harness = false
+
+[lib]
+crate-type = ["cdylib", "lib"]
+
+[dependencies]
+anyhow = "1.0.86"
+crossbeam = "0.8.4"
+heed = "0.20.3"
+lazy_static = "1.5.0"
+napi = { version = "3.0.0-alpha.8", default-features = false, features = ["napi4", "tokio"] }
+napi-derive = "3.0.0-alpha.7"
+rayon = "1.10.0"
+thiserror = "1.0.63"
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
+lz4_flex = "0.11.3"
+
+[dev-dependencies]
+criterion = "0.5.1"
+rand = "0.9.0-alpha.2"
+
+[build-dependencies]
+napi-build = "2.1.3"
+
+[profile.release]
+lto = true
+strip = "symbols"

crates/lmdb-js-lite/__benchmarks__/README.md

@@ -0,0 +1,3 @@
+```
+for i in __benchmarks__/*.ts ; do printf "$i\n=========================\n"; yarn ts-node $i; done 
+```

crates/lmdb-js-lite/__benchmarks__/prepare-read-benchmark-unsafe.ts

@@ -0,0 +1,55 @@
+import { randomBytes } from "node:crypto";
+import { mkdirSync, rmSync } from "node:fs";
+import * as v8 from "node:v8";
+import { open as openLMDBUnsafe } from "lmdb";
+
+const ENTRY_SIZE = 64 * 1024; // 64KB
+const NUM_ENTRIES = Math.floor((1024 * 1024 * 1024 * 5) / ENTRY_SIZE); // Total memory used 1GB
+
+let key = 0;
+
+function generateEntry() {
+  return {
+    key: String(key++),
+    value: randomBytes(ENTRY_SIZE),
+  };
+}
+
+async function main() {
+  rmSync("./databases/unsafe", {
+    recursive: true,
+    force: true,
+  });
+  mkdirSync("./databases/unsafe", {
+    recursive: true,
+  });
+
+  const unsafeDB = openLMDBUnsafe({
+    path: "./databases/unsafe/read",
+    encoding: "binary",
+    compression: true,
+    eventTurnBatching: true,
+  });
+
+  console.log("Generating entries for testing");
+  const entries = [...Array(NUM_ENTRIES)].map(() => {
+    return generateEntry();
+  });
+  await unsafeDB.transaction(() => {
+    console.log("Writing entries");
+    for (let entry of entries) {
+      unsafeDB.put(entry.key, entry.value);
+    }
+    unsafeDB.put(
+      "benchmarkInfo",
+      v8.serialize({
+        NUM_ENTRIES,
+      }),
+    );
+  });
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exitCode = 1;
+});

crates/lmdb-js-lite/__benchmarks__/prepare-read-benchmark.ts

@@ -0,0 +1,57 @@
+import { randomBytes } from "node:crypto";
+import { mkdirSync, rmSync } from "node:fs";
+import * as v8 from "node:v8";
+import { Lmdb } from "../index";
+
+const ENTRY_SIZE = 64 * 1024; // 64KB
+const ASYNC_WRITES = true;
+const NUM_ENTRIES = Math.floor((1024 * 1024 * 1024 * 5) / ENTRY_SIZE); // Total memory used 1GB
+const MAP_SIZE = 1024 * 1024 * 1024 * 10;
+
+let key = 0;
+
+function generateEntry() {
+  return {
+    key: String(key++),
+    value: randomBytes(ENTRY_SIZE),
+  };
+}
+
+async function main() {
+  rmSync("./databases/safe", {
+    recursive: true,
+    force: true,
+  });
+  mkdirSync("./databases/safe", {
+    recursive: true,
+  });
+
+  const safeDB = new Lmdb({
+    path: "./databases/safe/read",
+    asyncWrites: ASYNC_WRITES,
+    mapSize: MAP_SIZE,
+  });
+
+  console.log("Generating entries for testing");
+  const entries = [...Array(NUM_ENTRIES)].map(() => {
+    return generateEntry();
+  });
+  console.log("Writing entries");
+  await safeDB.startWriteTransaction();
+  for (let entry of entries) {
+    await safeDB.put(entry.key, entry.value);
+  }
+  await safeDB.put(
+    "benchmarkInfo",
+    v8.serialize({
+      NUM_ENTRIES,
+    }),
+  );
+  await safeDB.commitWriteTransaction();
+  safeDB.close();
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exitCode = 1;
+});