Merge pull request #7 from powersync-ja/chore/update-wa-sqlite

Feat: Full text search and update wa-sqlite
powersync-ja · Apr 17, 2024 · f087ed7 · f087ed7
2 parents 1d1d391 + 0077436
commit f087ed7
Show file tree

Hide file tree

Showing 13 changed files with 1,848 additions and 1,507 deletions.
diff --git a/.changeset/stupid-jokes-add.md b/.changeset/stupid-jokes-add.md
@@ -0,0 +1,6 @@
+---
+"@journeyapps/wa-sqlite": minor
+---
+
+Added full text search (FTS5) support out of the box by default. There should be no configuration required.
+Updated from upstream changes
diff --git a/Makefile b/Makefile
@@ -17,6 +17,13 @@ CFILES = \
 	libvfs.c \
 	$(CFILES_EXTRA)
 
+JSFILES = \
+	src/libauthorizer.js \
+	src/libfunction.js \
+	src/libmodule.js \
+	src/libprogress.js \
+	src/libvfs.js
+
 vpath %.c src
 vpath %.c deps
 vpath %.c deps/$(SQLITE_VERSION)
@@ -56,7 +63,7 @@ EMFLAGS_COMMON = \
 
 EMFLAGS_DEBUG = \
 	-s ASSERTIONS=1 \
-	-g -Oz \
+	-g -Oz -Oz \
 	$(EMFLAGS_COMMON)
 
 EMFLAGS_DIST = \
@@ -104,6 +111,7 @@ WASQLITE_DEFINES = \
 	-DSQLITE_THREADSAFE=0 \
 	-DSQLITE_USE_ALLOCA \
 	-DSQLITE_ENABLE_BATCH_ATOMIC_WRITE \
+	-DSQLITE_ENABLE_FTS5 \
 	$(WASQLITE_EXTRA_DEFINES)
 
 # directories
@@ -112,11 +120,11 @@ all: dist
 
 .PHONY: clean
 clean:
-	rm -rf dist dist-xl debug tmp
+	rm -rf dist debug tmp
 
 .PHONY: spotless
 spotless:
-	rm -rf dist dist-xl debug tmp deps cache
+	rm -rf dist debug tmp deps cache
 
 ## cache
 .PHONY: clean-cache
@@ -194,6 +202,27 @@ debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(RS_DEBUG_BC) $(EXPORTED_FUNCTION
 		$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
 	  $(OBJ_FILES_DEBUG) *.o -o $@
 
+## Debug FTS builds
+# .PHONY: debug
+# debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs
+
+# debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG_FTS) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+# 	mkdir -p debug
+# 	$(EMCC) $(EMFLAGS_DEBUG) \
+# 	  $(EMFLAGS_INTERFACES) \
+# 	  $(EMFLAGS_LIBRARIES) \
+# 		$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
+# 	  $(OBJ_FILES_DEBUG_FTS) *.o -o $@
+
+# debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG_FTS) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+# 	mkdir -p debug
+# 	$(EMCC) $(EMFLAGS_DEBUG) \
+# 	  $(EMFLAGS_INTERFACES) \
+# 	  $(EMFLAGS_LIBRARIES) \
+# 	  $(EMFLAGS_ASYNCIFY_DEBUG) \
+# 		$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
+# 	  $(OBJ_FILES_DEBUG_FTS) *.o -o $@
+
 ## dist
 .PHONY: clean-dist
 clean-dist:
@@ -220,4 +249,32 @@ dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(RS_RELEASE_BC) $(EXPORTED_FUNCTION
 		$(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
 	  $(OBJ_FILES_DIST)  -o $@
 
-FORCE: ;
+FORCE: ;
+
+# FTS builds
+# .PHONY: clean-dist
+# clean-dist:
+# 	rm -rf dist
+
+# .PHONY: dist
+# dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs
+
+# dist/wa-sqlite.mjs: $(OBJ_FILES_DIST_FTS) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+# 	mkdir -p dist
+# 	$(EMCC) $(EMFLAGS_DIST) \
+# 	  $(EMFLAGS_INTERFACES) \
+# 	  $(EMFLAGS_LIBRARIES) \
+# 		$(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
+# 	  $(OBJ_FILES_DIST_FTS)  -o $@
+
+# dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST_FTS) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+# 	mkdir -p dist
+# 	$(EMCC) $(EMFLAGS_DIST) \
+# 	  $(EMFLAGS_INTERFACES) \
+# 	  $(EMFLAGS_LIBRARIES) \
+# 	  $(EMFLAGS_ASYNCIFY_DIST) \
+# 		$(CFLAGS_DIST) \
+# 		$(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
+# 	  $(OBJ_FILES_DIST_FTS)  -o $@
+
+# FORCE: ;
diff --git a/README.md b/README.md
@@ -29,7 +29,7 @@ Here are the build steps:
 * `git clone [email protected]:rhashimoto/wa-sqlite.git`
 * `cd wa-sqlite`
 * `yarn install`
-* `make` (or `yarn prepack`)
+* `make`
 
 The default build produces ES6 modules + WASM, [synchronous and asynchronous](https://github.com/rhashimoto/wa-sqlite/issues/7) (using Asyncify) in `dist/`.
 
@@ -51,7 +51,7 @@ git submodule init
 ```
 
 ```bash
-git submodule update --recursive
+git submodule update --init --recursive
 ```
 
 ```bash

diff --git a/dist/wa-sqlite-async.mjs b/dist/wa-sqlite-async.mjs
diff --git a/dist/wa-sqlite-async.wasm b/dist/wa-sqlite-async.wasm
diff --git a/dist/wa-sqlite.mjs b/dist/wa-sqlite.mjs
diff --git a/dist/wa-sqlite.wasm b/dist/wa-sqlite.wasm
diff --git a/package.json b/package.json
@@ -37,7 +37,7 @@
     "karma-jasmine": "^5.1.0",
     "monaco-editor": "^0.34.1",
     "puppeteer": "^19.6.3",
-    "sinon": "^15.0.1",
+    "sinon": "15.0.1",
     "typedoc": "^0.22.11",
     "typescript": "^4.2.4"
   },

diff --git a/scripts/docker-setup.sh b/scripts/docker-setup.sh
@@ -1,6 +1,7 @@
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-source ~/.bashrc
-rustup component add rust-src --toolchain nightly-2023-08-28-aarch64-unknown-linux-gnu -y
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+. $HOME/.cargo/env && \
+rustup toolchain install nightly-2023-08-28-aarch64-unknown-linux-gnu && \
+rustup component add rust-src --toolchain nightly-2023-08-28-aarch64-unknown-linux-gnu
 
 sudo apt-get update -y
 sudo apt-get install -y tclsh

diff --git a/src/libvfs.c b/src/libvfs.c
@@ -6,71 +6,75 @@
 
 // sqlite3_io_methods javascript handlers
 // 64-bit integer parameters are passed by pointer.
-extern int vfsClose(sqlite3_file* file);
-extern int vfsRead(sqlite3_file* file, void* pData, int iAmt, sqlite3_int64 iOffset);
-extern int vfsWrite(sqlite3_file* file, const void* pData, int iAmt, sqlite3_int64 iOffset);
-extern int vfsTruncate(sqlite3_file* file, sqlite3_int64 size);
-extern int vfsSync(sqlite3_file* file, int flags);
-extern int vfsFileSize(sqlite3_file* file, sqlite3_int64* pSize);
-extern int vfsLock(sqlite3_file* file, int flags);
-extern int vfsUnlock(sqlite3_file* file, int flags);
-extern int vfsCheckReservedLock(sqlite3_file* file, int* pResOut);
-extern int vfsFileControl(sqlite3_file* file, int flags, void* pOut);
-extern int vfsSectorSize(sqlite3_file* file);
-extern int vfsDeviceCharacteristics(sqlite3_file* file);
+extern int vfsClose(sqlite3_file *file);
+extern int vfsRead(sqlite3_file *file, void *pData, int iAmt, sqlite3_int64 iOffset);
+extern int vfsWrite(sqlite3_file *file, const void *pData, int iAmt, sqlite3_int64 iOffset);
+extern int vfsTruncate(sqlite3_file *file, sqlite3_int64 size);
+extern int vfsSync(sqlite3_file *file, int flags);
+extern int vfsFileSize(sqlite3_file *file, sqlite3_int64 *pSize);
+extern int vfsLock(sqlite3_file *file, int flags);
+extern int vfsUnlock(sqlite3_file *file, int flags);
+extern int vfsCheckReservedLock(sqlite3_file *file, int *pResOut);
+extern int vfsFileControl(sqlite3_file *file, int flags, void *pOut);
+extern int vfsSectorSize(sqlite3_file *file);
+extern int vfsDeviceCharacteristics(sqlite3_file *file);
 
-extern int vfsOpen(sqlite3_vfs* vfs, const char *zName, sqlite3_file* file, int flags, int *pOutFlags);
-extern int vfsDelete(sqlite3_vfs* vfs, const char *zName, int syncDir);
-extern int vfsAccess(sqlite3_vfs* vfs, const char *zName, int flags, int *pResOut);
+extern int vfsOpen(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags);
+extern int vfsDelete(sqlite3_vfs *vfs, const char *zName, int syncDir);
+extern int vfsAccess(sqlite3_vfs *vfs, const char *zName, int flags, int *pResOut);
 
 // This is undefined in the WASM linker step if not specified
 extern int __rust_no_alloc_shim_is_unstable = 0;
 extern int sqlite3_powersync_init(sqlite3 *db, char **pzErrMsg,
-                           const sqlite3_api_routines *pApi);
+                                  const sqlite3_api_routines *pApi);
 
-static int xOpen(sqlite3_vfs* vfs, const char* zName, sqlite3_file* file, int flags, int* pOutFlags) {
+static int xOpen(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags)
+{
   static sqlite3_io_methods io_methods = {
-    1,
-    vfsClose,
-    vfsRead,
-    vfsWrite,
-    vfsTruncate,
-    vfsSync,
-    vfsFileSize,
-    vfsLock,
-    vfsUnlock,
-    vfsCheckReservedLock,
-    vfsFileControl,
-    vfsSectorSize,
-    vfsDeviceCharacteristics
-  };
+      1,
+      vfsClose,
+      vfsRead,
+      vfsWrite,
+      vfsTruncate,
+      vfsSync,
+      vfsFileSize,
+      vfsLock,
+      vfsUnlock,
+      vfsCheckReservedLock,
+      vfsFileControl,
+      vfsSectorSize,
+      vfsDeviceCharacteristics};
   file->pMethods = &io_methods;
 
   return vfsOpen(vfs, zName, file, flags, pOutFlags);
 }
 
-static int xFullPathname(sqlite3_vfs* vfs, const char* zName, int nOut, char* zOut) {
+static int xFullPathname(sqlite3_vfs *vfs, const char *zName, int nOut, char *zOut)
+{
   strncpy(zOut, zName, nOut);
   return SQLITE_OK;
 }
 
-static int xCurrentTime(sqlite3_vfs* vfs, double* pJulianDay) {
+static int xCurrentTime(sqlite3_vfs *vfs, double *pJulianDay)
+{
   // UNIX epoch 1/1/1970 is Julian day 2440587.5
-  static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000;
+  static const sqlite3_int64 unixEpoch = 24405875 * (sqlite3_int64)8640000;
   struct timeval sNow;
   gettimeofday(&sNow, 0);
-  sqlite3_int64 julianMillis = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000;
+  sqlite3_int64 julianMillis = unixEpoch + 1000 * (sqlite3_int64)sNow.tv_sec + sNow.tv_usec / 1000;
   *pJulianDay = julianMillis / 86400000.0;
   return SQLITE_OK;
 }
 
 const int EMSCRIPTEN_KEEPALIVE register_vfs(
-  const char* zName,
-  int mxPathName,
-  int makeDefault,
-  sqlite3_vfs** ppVFS) {
-  sqlite3_vfs* vfs = *ppVFS = (sqlite3_vfs*)sqlite3_malloc(sizeof(sqlite3_vfs));
-  if (!vfs) {
+    const char *zName,
+    int mxPathName,
+    int makeDefault,
+    sqlite3_vfs **ppVFS)
+{
+  sqlite3_vfs *vfs = *ppVFS = (sqlite3_vfs *)sqlite3_malloc(sizeof(sqlite3_vfs));
+  if (!vfs)
+  {
     return SQLITE_NOMEM;
   }
 
@@ -85,9 +89,9 @@ const int EMSCRIPTEN_KEEPALIVE register_vfs(
   vfs->xAccess = vfsAccess;
   vfs->xFullPathname = xFullPathname;
   vfs->xCurrentTime = xCurrentTime;
-  
+
   // Get remaining functionality from the default VFS.
-  sqlite3_vfs* defer = sqlite3_vfs_find(0);
+  sqlite3_vfs *defer = sqlite3_vfs_find(0);
 #define COPY_FIELD(NAME) vfs->NAME = defer->NAME
   COPY_FIELD(xDlOpen);
   COPY_FIELD(xDlError);
@@ -99,22 +103,26 @@ const int EMSCRIPTEN_KEEPALIVE register_vfs(
 #undef COPY_FIELD
 
   const int result = sqlite3_vfs_register(vfs, makeDefault);
-  if (result != SQLITE_OK) {
+  if (result != SQLITE_OK)
+  {
     *ppVFS = 0;
     sqlite3_free(vfs);
   }
   return result;
 }
 
-void* EMSCRIPTEN_KEEPALIVE getSqliteFree() {
+void *EMSCRIPTEN_KEEPALIVE getSqliteFree()
+{
   return sqlite3_free;
 }
 
-int main() {
+int main()
+{
   sqlite3_initialize();
   return 0;
 }
 
-int setup_powersync() {
-   return sqlite3_auto_extension((void (*)(void)) &sqlite3_powersync_init);
+int setup_powersync()
+{
+  return sqlite3_auto_extension((void (*)(void)) & sqlite3_powersync_init);
 }
diff --git a/src/libvfs.js b/src/libvfs.js
@@ -22,7 +22,7 @@ const vfs_methods = {
         vfs['handleAsync'] = Asyncify.handleAsync;
       }
 
-      const mxPathName = vfs.mxPathName ?? 64;
+      const mxPathName = vfs['mxPathName'] ?? 64;
       const out = Module['_malloc'](4);
       const result = ccall('register_vfs', 'number', ['string', 'number', 'number', 'number'],
         [vfs.name, mxPathName, makeDefault ? 1 : 0, out]);

diff --git a/src/types/index.d.ts b/src/types/index.d.ts
@@ -1692,4 +1692,4 @@ declare module 'wa-sqlite/src/examples/tag.js' {
    * @returns {function(TemplateStringsArray, ...any): Promise<object[]>}
    */
   export function tag(sqlite3: any, db: number): (arg0: TemplateStringsArray, ...args: any[]) => Promise<object[]>;
-}
+}