feat: Add realtime to update search index

For any change on listened doctypes, the realtime allows us to
dynamically update the related search index.

package.json

@@ -19,7 +19,8 @@
     "cozy-flags": "^4.0.0",
     "cozy-logger": "^1.10.4",
     "cozy-minilog": "^3.3.1",
-    "cozy-pouch-link": "^49.5.0",
+    "cozy-pouch-link": "^49.8.0",
+    "cozy-realtime": "^5.0.2",
     "cozy-tsconfig": "^1.2.0",
     "flexsearch": "^0.7.43",
     "lodash": "^4.17.21",

src/dataproxy/worker/shared-worker.ts

@@ -12,13 +12,17 @@ import {
 import { platformWorker } from '@/dataproxy/worker/platformWorker'
 import schema from '@/doctypes'
 import SearchEngine from '@/search/SearchEngine'
-import { FILES_DOCTYPE, CONTACTS_DOCTYPE, APPS_DOCTYPE } from '@/search/consts'
+import {
+  FILES_DOCTYPE,
+  CONTACTS_DOCTYPE,
+  APPS_DOCTYPE
+} from '@/search/consts'
 
 const log = Minilog('👷‍♂️ [shared-worker]')
 Minilog.enable()
 
 let client: CozyClient | undefined = undefined
-let searchEngine: SearchEngine = null
+let searchEngine: SearchEngine
 
 const broadcastChannel = new BroadcastChannel('DATA_PROXY_BROADCAST_CHANANEL')
 
@@ -32,6 +36,7 @@ const dataProxy: DataProxyWorker = {
     const pouchLinkOptions = {
       doctypes: [FILES_DOCTYPE, CONTACTS_DOCTYPE, APPS_DOCTYPE],
       initialSync: true,
+      periodicSync: false,
       platform: { ...platformWorker },
       doctypesReplicationOptions: {
         [FILES_DOCTYPE]: {

src/search/SearchEngine.ts

@@ -4,17 +4,20 @@ import { encode as encode_balance } from 'flexsearch/dist/module/lang/latin/bala
 
 import CozyClient, { Q } from 'cozy-client'
 import Minilog from 'cozy-minilog'
+import { RealtimePlugin } from 'cozy-realtime'
 
 import {
   SEARCH_SCHEMA,
   APPS_DOCTYPE,
   FILES_DOCTYPE,
   CONTACTS_DOCTYPE,
   DOCTYPE_ORDER,
-  LIMIT_DOCTYPE_SEARCH
+  LIMIT_DOCTYPE_SEARCH,
+  REPLICATION_DEBOUNCE
 } from '@/search/consts'
 import { getPouchLink } from '@/search/helpers/client'
 import { normalizeSearchResult } from '@/search/helpers/normalizeSearchResult'
+import { startReplicationWithDebounce } from '@/search/helpers/replication'
 import {
   queryFilesForSearch,
   queryAllContacts,
@@ -42,28 +45,77 @@ interface FlexSearchResultWithDoctype
 class SearchEngine {
   client: CozyClient
   searchIndexes: SearchIndexes
+  debouncedReplication: () => void
 
   constructor(client: CozyClient) {
     this.client = client
     this.searchIndexes = {}
 
-    this.indexOnReplicationChanges()
+    this.indexOnChanges()
+    this.debouncedReplication = startReplicationWithDebounce(
+      client,
+      REPLICATION_DEBOUNCE
+    )
   }
 
-  indexOnReplicationChanges(): void {
+  indexOnChanges(): void {
     if (!this.client) {
       return
     }
     this.client.on('pouchlink:doctypesync:end', async (doctype: string) => {
-      // TODO: lock to avoid conflict with concurrent index events?
-      const newIndex = await this.indexDocsForSearch(doctype)
-      if (newIndex) {
-        log('debug', `Index updated for doctype ${doctype}`)
-        this.searchIndexes[doctype] = newIndex
-      }
+      await this.indexDocsForSearch(doctype)
+    })
+    this.client.on('login', () => {
+      // Ensure login is done before plugin register
+      this.client.registerPlugin(RealtimePlugin, {})
+      this.handleUpdatedOrCreatedDoc = this.handleUpdatedOrCreatedDoc.bind(this)
+      this.handleDeletedDoc = this.handleDeletedDoc.bind(this)
+
+      this.subscribeDoctype(this.client, FILES_DOCTYPE)
+      this.subscribeDoctype(this.client, CONTACTS_DOCTYPE)
+      this.subscribeDoctype(this.client, APPS_DOCTYPE)
     })
   }
 
+  subscribeDoctype(client: CozyClient, doctype: string): void {
+    const realtime = this.client.plugins.realtime
+    realtime.subscribe('created', doctype, this.handleUpdatedOrCreatedDoc)
+    realtime.subscribe('updated', doctype, this.handleUpdatedOrCreatedDoc)
+    realtime.subscribe('deleted', doctype, this.handleDeletedDoc)
+  }
+
+  handleUpdatedOrCreatedDoc(doc: CozyDoc): void {
+    const doctype: string | undefined = doc._type
+    if (!doctype) {
+      return
+    }
+    const searchIndex = this.searchIndexes?.[doctype]
+    if (!searchIndex) {
+      // No index yet: it will be done by querying the local db after first replication
+      return
+    }
+    log.debug('[REALTIME] index doc after update : ', doc)
+    searchIndex.index.add(doc)
+
+    this.debouncedReplication()
+  }
+
+  handleDeletedDoc(doc: CozyDoc): void {
+    const doctype: string | undefined = doc._type
+    if (!doctype) {
+      return
+    }
+    const searchIndex = this.searchIndexes?.[doctype]
+    if (!searchIndex) {
+      // No index yet: it will be done by querying the local db after first replication
+      return
+    }
+    log.debug('[REALTIME] remove doc from index after update : ', doc)
+    this.searchIndexes[doctype].index.remove(doc._id)
+
+    this.debouncedReplication()
+  }
+
   buildSearchIndex(
     doctype: keyof typeof SEARCH_SCHEMA,
     docs: CozyDoc[]
@@ -88,13 +140,16 @@ class SearchEngine {
     return flexsearchIndex
   }
 
-  async indexDocsForSearch(doctype: string): Promise<SearchIndex> {
+  async indexDocsForSearch(doctype: string): Promise<SearchIndex | null> {
     const searchIndex = this.searchIndexes[doctype]
     const pouchLink = getPouchLink(this.client)
 
-    if (!pouchLink) return null
+    if (!pouchLink) {
+      return null
+    }
 
     if (!searchIndex) {
+      // First creation of search index
       const docs = await this.client.queryAll(Q(doctype).limitBy(null))
       const index = this.buildSearchIndex(doctype, docs)
       const info = await pouchLink.getDbInfo(doctype)
@@ -106,6 +161,10 @@ class SearchEngine {
       return this.searchIndexes[doctype]
     }
 
+    // Incremental index update
+    // At this point, the search index are supposed to be already up-to-date, 
+    // thanks to the realtime.
+    // However, we check it is actually the case for safety, and update the lastSeq
     const lastSeq = searchIndex.lastSeq || 0
     const changes = await pouchLink.getChanges(doctype, {
       include_docs: true,
@@ -139,9 +198,9 @@ class SearchEngine {
 
     log.debug('Finished initializing indexes')
     this.searchIndexes = {
-      [FILES_DOCTYPE]: { index: filesIndex, lastSeq: null },
-      [CONTACTS_DOCTYPE]: { index: contactsIndex, lastSeq: null },
-      [APPS_DOCTYPE]: { index: appsIndex, lastSeq: null }
+      [FILES_DOCTYPE]: { index: filesIndex, lastSeq: 0 },
+      [CONTACTS_DOCTYPE]: { index: contactsIndex, lastSeq: 0 },
+      [APPS_DOCTYPE]: { index: appsIndex, lastSeq: 0 }
     }
     return this.searchIndexes
   }
@@ -172,6 +231,8 @@ class SearchEngine {
         log.warn('[SEARCH] No search index available for ', doctype)
         continue
       }
+      // TODO: do not use flexsearch store and rely on pouch storage?
+      // It's better for memory, but might slow down search queries
       const indexResults = index.index.search(query, LIMIT_DOCTYPE_SEARCH, {
         enrich: true
       })

src/search/consts.ts

@@ -15,6 +15,8 @@ export const SEARCH_SCHEMA = {
   'io.cozy.apps': ['slug', 'name']
 }
 
+export const REPLICATION_DEBOUNCE = 30 * 1000 // 30s
+
 export const FILES_DOCTYPE = 'io.cozy.files'
 export const CONTACTS_DOCTYPE = 'io.cozy.contacts'
 export const APPS_DOCTYPE = 'io.cozy.apps'

src/search/helpers/replication.ts

@@ -0,0 +1,29 @@
+import CozyClient from 'cozy-client'
+import Minilog from 'cozy-minilog'
+
+import { getPouchLink } from '@/search/helpers/client'
+
+const log = Minilog('🗂️ [Replication]')
+
+export const startReplicationWithDebounce = (
+  client: CozyClient,
+  interval: number
+) => {
+  let timeoutId: NodeJS.Timeout | null = null
+
+  return (): void => {
+    if (timeoutId) {
+      log.debug('Reset replication debounce')
+      clearTimeout(timeoutId)
+    }
+
+    timeoutId = setTimeout(() => {
+      const pouchLink = getPouchLink(client)
+      if (!pouchLink) {
+        return
+      }
+      log.debug('Start replication after debounce of ', interval)
+      pouchLink.startReplication()
+    }, interval)
+  }
+}

yarn.lock

@@ -308,7 +308,7 @@
   resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9"
   integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==
 
-"@cozy/[email protected]":
+"@cozy/[email protected]", "@cozy/minilog@^1.0.0":
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/@cozy/minilog/-/minilog-1.0.0.tgz#1acc1aad849261e931e255a5f181b638315f7b84"
   integrity sha512-IkDHF9CLh0kQeSEVsou59ar/VehvenpbEUjLfwhckJyOUqZnKAWmXy8qrBgMT5Loxr8Xjs2wmMnj0D67wP00eQ==
@@ -2198,16 +2198,16 @@ cozy-client@^49.0.0:
     sift "^6.0.0"
     url-search-params-polyfill "^8.0.0"
 
-cozy-client@^49.4.0:
-  version "49.4.0"
-  resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-49.4.0.tgz#248788e5e7a3595dd9137f5d0563ac6a5f5c0231"
-  integrity sha512-jIapZfMzJzCblI4pKnir3nXgOvrXavsRXkm7QB2qbDl9WSCJcpKvMw8Z1yDsma7nB/16E0rKzeEBh+BxXbStTw==
+cozy-client@^49.8.0:
+  version "49.8.0"
+  resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-49.8.0.tgz#d4f6b3a2fb26dc6bfa561046b075eb6e221af73e"
+  integrity sha512-peb2n+buOihzqqNN3/zULmytQI8wOtQZsIqLWEaw2WfNXha1T23wNBrMuE7SaMOof361f011MrM5+WVYhKj+VA==
   dependencies:
     "@cozy/minilog" "1.0.0"
     "@types/jest" "^26.0.20"
     "@types/lodash" "^4.14.170"
     btoa "^1.2.1"
-    cozy-stack-client "^49.4.0"
+    cozy-stack-client "^49.8.0"
     date-fns "2.29.3"
     json-stable-stringify "^1.0.1"
     lodash "^4.17.13"
@@ -2252,16 +2252,24 @@ cozy-minilog@^3.3.1:
   dependencies:
     microee "0.0.6"
 
-cozy-pouch-link@^49.5.0:
-  version "49.5.0"
-  resolved "https://registry.yarnpkg.com/cozy-pouch-link/-/cozy-pouch-link-49.5.0.tgz#ee2b43725ccd63f793800d62562706eaa2ff9888"
-  integrity sha512-T4kcKrwajE0N3URu2lA/YaoKiPRrSVRSFxh9QmAdOhfUHS5qZFrd0ox5gvyWTtNRceeJh1H9jC7LNDV7Qy0q9A==
+cozy-pouch-link@^49.8.0:
+  version "49.8.0"
+  resolved "https://registry.yarnpkg.com/cozy-pouch-link/-/cozy-pouch-link-49.8.0.tgz#a5361bbc0ed23b9bc4d804d31869c615fdf1d13e"
+  integrity sha512-c5+dWx5BNi37JuLVpil9SB91vy3EtB7B8Qa6Krx+znzPYcRoK3Bqjuns5c7hXG5rD8MV2qls1yvxQWUrfy82kA==
   dependencies:
-    cozy-client "^49.4.0"
+    cozy-client "^49.8.0"
     pouchdb-browser "^7.2.2"
     pouchdb-find "^7.2.2"
 
-cozy-stack-client@^49.0.0, cozy-stack-client@^49.4.0:
+cozy-realtime@^5.0.2:
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/cozy-realtime/-/cozy-realtime-5.0.2.tgz#d515e6625e4386c812e8a0ce505c2e3abfea2245"
+  integrity sha512-ncFgsb2BeaYyM+Uax/qbcnDC1n7hpLNAiaqnVD02ApDpN1Uy/2GUzbcAr5xpc1gK4jkp6+fb8FrqyL+vcxXRsg==
+  dependencies:
+    "@cozy/minilog" "^1.0.0"
+    cozy-device-helper "^3.1.0"
+
+cozy-stack-client@^49.0.0:
   version "49.4.0"
   resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-49.4.0.tgz#775d2d48e74182049977e2e423f7a42d39a4ac61"
   integrity sha512-iy2vTpj25vHqErfPeclN3flI99il+WBm+Kt0FWI5YzM4H76+fE07/6Up4zOqtyOuaF7S22OHSim4Zl2hFB/SpA==
@@ -2270,6 +2278,15 @@ cozy-stack-client@^49.0.0, cozy-stack-client@^49.4.0:
     mime "^2.4.0"
     qs "^6.7.0"
 
+cozy-stack-client@^49.8.0:
+  version "49.8.0"
+  resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-49.8.0.tgz#c57dfefe50e47f228fee7e1921c438d35f4e0877"
+  integrity sha512-sYJL2o+DsNs7V5eQghXpWKcMzxc39QAKtM8zAdmWl2MMCyiqO3lBehRomhstcJHtuZrMLXXPQPr1A0ONBlMmZg==
+  dependencies:
+    detect-node "^2.0.4"
+    mime "^2.4.0"
+    qs "^6.7.0"
+
 cozy-tsconfig@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/cozy-tsconfig/-/cozy-tsconfig-1.2.0.tgz#17e61f960f139fae4d26cbac2254b9ab632b269e"
@@ -4742,6 +4759,13 @@ nice-try@^1.0.4:
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
   integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
 
+[email protected]:
+  version "2.6.7"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
+  integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-fetch@^2.6.1:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
@@ -6271,6 +6295,11 @@ util@^0.12.5:
     is-typed-array "^1.1.3"
     which-typed-array "^1.1.2"
 
+[email protected]:
+  version "8.3.2"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
+  integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
+
 uuid@^3.3.2:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"