Merge pull request #125 from powersync-ja/feat/full-text-search

feat: Full text search

demos/example-webpack/src/index.js

@@ -58,4 +58,4 @@ const openDatabase = async () => {
 
 document.addEventListener('DOMContentLoaded', (event) => {
   openDatabase();
-});
+});

demos/react-supabase-todolist/package.json

@@ -11,9 +11,10 @@
   "dependencies": {
     "@journeyapps/powersync-react": "workspace:*",
     "@journeyapps/powersync-sdk-web": "workspace:*",
-    "@journeyapps/wa-sqlite": "~0.1.1",
+    "@journeyapps/wa-sqlite": "~0.2.0",
     "@mui/material": "^5.15.12",
     "@mui/x-data-grid": "^6.19.6",
+    "@mui/icons-material": "^5.15.12",
     "@supabase/supabase-js": "^2.39.7",
     "js-logger": "^1.6.1",
     "lodash": "^4.17.21",

demos/react-supabase-todolist/src/app/utils/fts_helpers.ts

@@ -0,0 +1,39 @@
+import { db } from '@/components/providers/SystemProvider';
+
+/**
+ * adding * to the end of the search term will match any word that starts with the search term
+ * e.g. searching bl will match blue, black, etc.
+ * consult FTS5 Full-text Query Syntax documentation for more options
+ * @param searchTerm
+ * @returns a modified search term with options.
+ */
+function createSearchTermWithOptions(searchTerm: string): string {
+  const searchTermWithOptions: string = `${searchTerm}*`;
+  return searchTermWithOptions;
+}
+
+/**
+ * Search the FTS table for the given searchTerm
+ * @param searchTerm
+ * @param tableName
+ * @returns results from the FTS table
+ */
+export async function searchTable(searchTerm: string, tableName: string): Promise<any[]> {
+  const searchTermWithOptions = createSearchTermWithOptions(searchTerm);
+  return await db.getAll(`SELECT * FROM fts_${tableName} WHERE fts_${tableName} MATCH ? ORDER BY rank`, [
+    searchTermWithOptions
+  ]);
+}
+
+//Used to display the search results in the autocomplete text field
+export class SearchResult {
+  id: string;
+  todoName: string | null;
+  listName: string;
+
+  constructor(id: string, listName: string, todoName: string | null = null) {
+    this.id = id;
+    this.listName = listName;
+    this.todoName = todoName;
+  }
+}

demos/react-supabase-todolist/src/app/utils/fts_setup.ts

@@ -0,0 +1,65 @@
+import { AppSchema } from '../../library/powersync/AppSchema';
+import { Table } from '@journeyapps/powersync-sdk-web';
+import { db } from '@/components/providers/SystemProvider';
+import { ExtractType, generateJsonExtracts } from './helpers';
+
+/**
+ * Create a Full Text Search table for the given table and columns
+ * with an option to use a different tokenizer otherwise it defaults
+ * to unicode61. It also creates the triggers that keep the FTS table
+ * and the PowerSync table in sync.
+ * @param tableName
+ * @param columns
+ * @param tokenizationMethod
+ */
+async function createFtsTable(tableName: string, columns: string[], tokenizationMethod = 'unicode61'): Promise<void> {
+  const internalName = (AppSchema.tables as Table[]).find((table) => table.name === tableName)?.internalName;
+  const stringColumns = columns.join(', ');
+
+  return await db.writeTransaction(async (tx) => {
+    // Add FTS table
+    await tx.execute(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS fts_${tableName}
+      USING fts5(id UNINDEXED, ${stringColumns}, tokenize='${tokenizationMethod}');
+    `);
+    // Copy over records already in table
+    await tx.execute(`
+      INSERT OR REPLACE INTO fts_${tableName}(rowid, id, ${stringColumns})
+      SELECT rowid, id, ${generateJsonExtracts(ExtractType.columnOnly, 'data', columns)} FROM ${internalName};
+    `);
+    // Add INSERT, UPDATE and DELETE and triggers to keep fts table in sync with table
+    await tx.execute(`
+      CREATE TRIGGER IF NOT EXISTS fts_insert_trigger_${tableName} AFTER INSERT ON ${internalName}
+      BEGIN
+        INSERT INTO fts_${tableName}(rowid, id, ${stringColumns})
+        VALUES (
+          NEW.rowid,
+          NEW.id,
+          ${generateJsonExtracts(ExtractType.columnOnly, 'NEW.data', columns)}
+        );
+      END;
+    `);
+    await tx.execute(`
+      CREATE TRIGGER IF NOT EXISTS fts_update_trigger_${tableName} AFTER UPDATE ON ${internalName} BEGIN
+        UPDATE fts_${tableName}
+        SET ${generateJsonExtracts(ExtractType.columnInOperation, 'NEW.data', columns)}
+        WHERE rowid = NEW.rowid;
+      END;
+    `);
+    await tx.execute(`
+      CREATE TRIGGER IF NOT EXISTS fts_delete_trigger_${tableName} AFTER DELETE ON ${internalName} BEGIN
+        DELETE FROM fts_${tableName} WHERE rowid = OLD.rowid;
+      END;
+    `);
+  });
+}
+
+/**
+ * This is where you can add more methods to generate FTS tables in this demo
+ * that correspond to the tables in your schema and populate them
+ * with the data you would like to search on
+ */
+export async function configureFts(): Promise<void> {
+  await createFtsTable('lists', ['name'], 'porter unicode61');
+  await createFtsTable('todos', ['description', 'list_id']);
+}

demos/react-supabase-todolist/src/app/utils/helpers.ts

@@ -0,0 +1,36 @@
+type ExtractGenerator = (jsonColumnName: string, columnName: string) => string;
+
+export enum ExtractType {
+  columnOnly,
+  columnInOperation
+}
+
+type ExtractGeneratorMap = Map<ExtractType, ExtractGenerator>;
+
+function _createExtract(jsonColumnName: string, columnName: string): string {
+  return `json_extract(${jsonColumnName}, '$.${columnName}')`;
+}
+
+const extractGeneratorsMap: ExtractGeneratorMap = new Map<ExtractType, ExtractGenerator>([
+  [ExtractType.columnOnly, (jsonColumnName: string, columnName: string) => _createExtract(jsonColumnName, columnName)],
+  [
+    ExtractType.columnInOperation,
+    (jsonColumnName: string, columnName: string) => {
+      let extract = _createExtract(jsonColumnName, columnName);
+      return `${columnName} = ${extract}`;
+    }
+  ]
+]);
+
+export const generateJsonExtracts = (type: ExtractType, jsonColumnName: string, columns: string[]): string => {
+  const generator = extractGeneratorsMap.get(type);
+  if (generator == null) {
+    throw new Error('Unexpected null generator for key: $type');
+  }
+
+  if (columns.length == 1) {
+    return generator(jsonColumnName, columns[0]);
+  }
+
+  return columns.map((column) => generator(jsonColumnName, column)).join(', ');
+};

demos/react-supabase-todolist/src/app/views/todo-lists/page.tsx

@@ -17,6 +17,7 @@ import { NavigationPage } from '@/components/navigation/NavigationPage';
 import { useSupabase } from '@/components/providers/SystemProvider';
 import { TodoListsWidget } from '@/components/widgets/TodoListsWidget';
 import { LISTS_TABLE } from '@/library/powersync/AppSchema';
+import { SearchBarWidget } from '@/components/widgets/SearchBarWidget';
 
 export default function TodoListsPage() {
   const powerSync = usePowerSync();
@@ -50,6 +51,7 @@ export default function TodoListsPage() {
           <AddIcon />
         </S.FloatingActionButton>
         <Box>
+          <SearchBarWidget />
           <TodoListsWidget />
         </Box>
         {/* TODO use a dialog service in future, this is just a simple example app */}

demos/react-supabase-todolist/src/components/providers/SystemProvider.tsx

@@ -7,6 +7,8 @@ import { CircularProgress } from '@mui/material';
 import Logger from 'js-logger';
 import React, { Suspense } from 'react';
 
+import { configureFts } from '../../app/utils/fts_setup';
+
 const SupabaseContext = React.createContext<SupabaseConnector | null>(null);
 export const useSupabase = () => React.useContext(SupabaseContext);
 
@@ -23,7 +25,6 @@ export const SystemProvider = ({ children }: { children: React.ReactNode }) => {
     // Linting thinks this is a hook due to it's name
     Logger.useDefaults(); // eslint-disable-line
     Logger.setLevel(Logger.DEBUG);
-
     // For console testing purposes
     (window as any)._powersync = powerSync;
 
@@ -37,6 +38,8 @@ export const SystemProvider = ({ children }: { children: React.ReactNode }) => {
 
     connector.init();
 
+    configureFts();
+
     return () => l?.();
   }, [powerSync, connector]);
 

demos/react-supabase-todolist/src/components/widgets/SearchBarWidget.tsx

@@ -0,0 +1,105 @@
+import { SearchResult, searchTable } from '@/app/utils/fts_helpers';
+import { Autocomplete, Box, Card, CardContent, FormControl, Paper, TextField, Typography } from '@mui/material';
+import React from 'react';
+import { useNavigate } from 'react-router-dom';
+import { TODO_LISTS_ROUTE } from '@/app/router';
+import { usePowerSync, usePowerSyncQuery } from '@journeyapps/powersync-react';
+import { LISTS_TABLE, ListRecord } from '@/library/powersync/AppSchema';
+import { todo } from 'node:test';
+
+// This is a simple search bar widget that allows users to search for lists and todo items
+export const SearchBarWidget: React.FC<any> = (props) => {
+  const [searchResults, setSearchResults] = React.useState<SearchResult[]>([]);
+  const [value, setValue] = React.useState<SearchResult | null>(null);
+
+  const navigate = useNavigate();
+  const powersync = usePowerSync();
+
+  const handleInputChange = async (value: string) => {
+    if (value.length !== 0) {
+      let listsSearchResults: any[] = [];
+      let todoItemsSearchResults = await searchTable(value, 'todos');
+      for (let i = 0; i < todoItemsSearchResults.length; i++) {
+        let res = await powersync.get<ListRecord>(`SELECT * FROM ${LISTS_TABLE} WHERE id = ?`, [
+          todoItemsSearchResults[i]['list_id']
+        ]);
+        todoItemsSearchResults[i]['list_name'] = res.name;
+      }
+      if (!todoItemsSearchResults.length) {
+        listsSearchResults = await searchTable(value, 'lists');
+      }
+      let formattedListResults: SearchResult[] = listsSearchResults.map(
+        (result) => new SearchResult(result['id'], result['name'])
+      );
+      let formattedTodoItemsResults: SearchResult[] = todoItemsSearchResults.map((result) => {
+        return new SearchResult(result['list_id'], result['list_name'] ?? '', result['description']);
+      });
+      setSearchResults([...formattedTodoItemsResults, ...formattedListResults]);
+    }
+  };
+
+  return (
+    <div>
+      <FormControl sx={{ my: 1, display: 'flex' }}>
+        <Autocomplete
+          freeSolo
+          id="autocomplete-search"
+          options={searchResults}
+          value={value?.id}
+          getOptionLabel={(option) => {
+            if (option instanceof SearchResult) {
+              return option.todoName ?? option.listName;
+            }
+            return option;
+          }}
+          renderOption={(props, option) => (
+            <Box component="li" {...props}>
+              <Card variant="outlined" sx={{ display: 'flex', width: '100%' }}>
+                <CardContent>
+                  {option.listName && (
+                    <Typography sx={{ fontSize: 18 }} color="text.primary" gutterBottom>
+                      {option.listName}
+                    </Typography>
+                  )}
+                  {option.todoName && (
+                    <Typography sx={{ fontSize: 14 }} color="text.secondary">
+                      {'\u2022'} {option.todoName}
+                    </Typography>
+                  )}
+                </CardContent>
+              </Card>
+            </Box>
+          )}
+          filterOptions={(x) => x}
+          onInputChange={(event, newInputValue, reason) => {
+            if (reason === 'clear') {
+              setValue(null);
+              setSearchResults([]);
+              return;
+            }
+            handleInputChange(newInputValue);
+          }}
+          onChange={(event, newValue, reason) => {
+            if (reason === 'selectOption') {
+              if (newValue instanceof SearchResult) {
+                navigate(TODO_LISTS_ROUTE + '/' + newValue.id);
+              }
+            }
+          }}
+          selectOnFocus
+          clearOnBlur
+          handleHomeEndKeys
+          renderInput={(params) => (
+            <TextField
+              {...params}
+              label="Search..."
+              InputProps={{
+                ...params.InputProps
+              }}
+            />
+          )}
+        />
+      </FormControl>
+    </div>
+  );
+};

packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts

@@ -238,6 +238,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
     this.sdkVersion = version.rows?.item(0)['powersync_rs_version()'] ?? '';
     await this.updateSchema(this.options.schema);
     this.updateHasSynced();
+    await this.database.execute('PRAGMA RECURSIVE_TRIGGERS=TRUE');
     this.ready = true;
     this.iterateListeners((cb) => cb.initialized?.());
   }

packages/powersync-sdk-web/package.json

@@ -33,7 +33,7 @@
   "author": "JOURNEYAPPS",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@journeyapps/wa-sqlite": "~0.1.1",
+    "@journeyapps/wa-sqlite": "~0.2.0",
     "@types/lodash": "^4.14.200",
     "@types/uuid": "^9.0.6",
     "@vitest/browser": "^1.3.1",
@@ -45,7 +45,7 @@
     "webdriverio": "^8.32.3"
   },
   "peerDependencies": {
-    "@journeyapps/wa-sqlite": "~0.1.1"
+    "@journeyapps/wa-sqlite": "~0.2.0"
   },
   "dependencies": {
     "@journeyapps/powersync-sdk-common": "workspace:*",

packages/powersync-sdk-web/src/db/adapters/wa-sqlite/WASQLiteDBAdapter.ts

@@ -105,7 +105,7 @@ export class WASQLiteDBAdapter extends BaseObserver<DBAdapterListener> implement
     const result = await this.workerMethods!.executeBatch!(query, params);
     return {
       ...result,
-      rows: undefined,
+      rows: undefined
     };
   };