diff --git a/package.json b/package.json index b0a6aa4..302762f 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "cozy-flags": "^4.0.0", "cozy-logger": "^1.10.4", "cozy-minilog": "^3.3.1", - "cozy-pouch-link": "^49.5.0", + "cozy-pouch-link": "^49.8.0", + "cozy-realtime": "^5.0.2", "cozy-tsconfig": "^1.2.0", "flexsearch": "^0.7.43", "lodash": "^4.17.21", diff --git a/src/dataproxy/worker/shared-worker.ts b/src/dataproxy/worker/shared-worker.ts index 6588431..95f7f78 100644 --- a/src/dataproxy/worker/shared-worker.ts +++ b/src/dataproxy/worker/shared-worker.ts @@ -12,13 +12,17 @@ import { import { platformWorker } from '@/dataproxy/worker/platformWorker' import schema from '@/doctypes' import SearchEngine from '@/search/SearchEngine' -import { FILES_DOCTYPE, CONTACTS_DOCTYPE, APPS_DOCTYPE } from '@/search/consts' +import { + FILES_DOCTYPE, + CONTACTS_DOCTYPE, + APPS_DOCTYPE +} from '@/search/consts' const log = Minilog('👷‍♂️ [shared-worker]') Minilog.enable() let client: CozyClient | undefined = undefined -let searchEngine: SearchEngine = null +let searchEngine: SearchEngine const broadcastChannel = new BroadcastChannel('DATA_PROXY_BROADCAST_CHANANEL') @@ -32,6 +36,7 @@ const dataProxy: DataProxyWorker = { const pouchLinkOptions = { doctypes: [FILES_DOCTYPE, CONTACTS_DOCTYPE, APPS_DOCTYPE], initialSync: true, + periodicSync: false, platform: { ...platformWorker }, doctypesReplicationOptions: { [FILES_DOCTYPE]: { diff --git a/src/search/SearchEngine.ts b/src/search/SearchEngine.ts index 4b6ec9e..de5810f 100644 --- a/src/search/SearchEngine.ts +++ b/src/search/SearchEngine.ts @@ -4,6 +4,7 @@ import { encode as encode_balance } from 'flexsearch/dist/module/lang/latin/bala import CozyClient, { Q } from 'cozy-client' import Minilog from 'cozy-minilog' +import { RealtimePlugin } from 'cozy-realtime' import { SEARCH_SCHEMA, @@ -11,10 +12,12 @@ import { FILES_DOCTYPE, CONTACTS_DOCTYPE, DOCTYPE_ORDER, - LIMIT_DOCTYPE_SEARCH + LIMIT_DOCTYPE_SEARCH, + REPLICATION_DEBOUNCE } from '@/search/consts' import { getPouchLink } from '@/search/helpers/client' import { normalizeSearchResult } from '@/search/helpers/normalizeSearchResult' +import { startReplicationWithDebounce } from '@/search/helpers/replication' import { queryFilesForSearch, queryAllContacts, @@ -42,28 +45,77 @@ interface FlexSearchResultWithDoctype class SearchEngine { client: CozyClient searchIndexes: SearchIndexes + debouncedReplication: () => void constructor(client: CozyClient) { this.client = client this.searchIndexes = {} - this.indexOnReplicationChanges() + this.indexOnChanges() + this.debouncedReplication = startReplicationWithDebounce( + client, + REPLICATION_DEBOUNCE + ) } - indexOnReplicationChanges(): void { + indexOnChanges(): void { if (!this.client) { return } this.client.on('pouchlink:doctypesync:end', async (doctype: string) => { - // TODO: lock to avoid conflict with concurrent index events? - const newIndex = await this.indexDocsForSearch(doctype) - if (newIndex) { - log('debug', `Index updated for doctype ${doctype}`) - this.searchIndexes[doctype] = newIndex - } + await this.indexDocsForSearch(doctype) + }) + this.client.on('login', () => { + // Ensure login is done before plugin register + this.client.registerPlugin(RealtimePlugin, {}) + this.handleUpdatedOrCreatedDoc = this.handleUpdatedOrCreatedDoc.bind(this) + this.handleDeletedDoc = this.handleDeletedDoc.bind(this) + + this.subscribeDoctype(this.client, FILES_DOCTYPE) + this.subscribeDoctype(this.client, CONTACTS_DOCTYPE) + this.subscribeDoctype(this.client, APPS_DOCTYPE) }) } + subscribeDoctype(client: CozyClient, doctype: string): void { + const realtime = this.client.plugins.realtime + realtime.subscribe('created', doctype, this.handleUpdatedOrCreatedDoc) + realtime.subscribe('updated', doctype, this.handleUpdatedOrCreatedDoc) + realtime.subscribe('deleted', doctype, this.handleDeletedDoc) + } + + handleUpdatedOrCreatedDoc(doc: CozyDoc): void { + const doctype: string | undefined = doc._type + if (!doctype) { + return + } + const searchIndex = this.searchIndexes?.[doctype] + if (!searchIndex) { + // No index yet: it will be done by querying the local db after first replication + return + } + log.debug('[REALTIME] index doc after update : ', doc) + searchIndex.index.add(doc) + + this.debouncedReplication() + } + + handleDeletedDoc(doc: CozyDoc): void { + const doctype: string | undefined = doc._type + if (!doctype) { + return + } + const searchIndex = this.searchIndexes?.[doctype] + if (!searchIndex) { + // No index yet: it will be done by querying the local db after first replication + return + } + log.debug('[REALTIME] remove doc from index after update : ', doc) + this.searchIndexes[doctype].index.remove(doc._id) + + this.debouncedReplication() + } + buildSearchIndex( doctype: keyof typeof SEARCH_SCHEMA, docs: CozyDoc[] @@ -88,13 +140,16 @@ class SearchEngine { return flexsearchIndex } - async indexDocsForSearch(doctype: string): Promise { + async indexDocsForSearch(doctype: string): Promise { const searchIndex = this.searchIndexes[doctype] const pouchLink = getPouchLink(this.client) - if (!pouchLink) return null + if (!pouchLink) { + return null + } if (!searchIndex) { + // First creation of search index const docs = await this.client.queryAll(Q(doctype).limitBy(null)) const index = this.buildSearchIndex(doctype, docs) const info = await pouchLink.getDbInfo(doctype) @@ -106,6 +161,10 @@ class SearchEngine { return this.searchIndexes[doctype] } + // Incremental index update + // At this point, the search index are supposed to be already up-to-date, + // thanks to the realtime. + // However, we check it is actually the case for safety, and update the lastSeq const lastSeq = searchIndex.lastSeq || 0 const changes = await pouchLink.getChanges(doctype, { include_docs: true, @@ -139,9 +198,9 @@ class SearchEngine { log.debug('Finished initializing indexes') this.searchIndexes = { - [FILES_DOCTYPE]: { index: filesIndex, lastSeq: null }, - [CONTACTS_DOCTYPE]: { index: contactsIndex, lastSeq: null }, - [APPS_DOCTYPE]: { index: appsIndex, lastSeq: null } + [FILES_DOCTYPE]: { index: filesIndex, lastSeq: 0 }, + [CONTACTS_DOCTYPE]: { index: contactsIndex, lastSeq: 0 }, + [APPS_DOCTYPE]: { index: appsIndex, lastSeq: 0 } } return this.searchIndexes } @@ -172,6 +231,8 @@ class SearchEngine { log.warn('[SEARCH] No search index available for ', doctype) continue } + // TODO: do not use flexsearch store and rely on pouch storage? + // It's better for memory, but might slow down search queries const indexResults = index.index.search(query, LIMIT_DOCTYPE_SEARCH, { enrich: true }) diff --git a/src/search/consts.ts b/src/search/consts.ts index d527ae8..ae2e463 100644 --- a/src/search/consts.ts +++ b/src/search/consts.ts @@ -15,6 +15,8 @@ export const SEARCH_SCHEMA = { 'io.cozy.apps': ['slug', 'name'] } +export const REPLICATION_DEBOUNCE = 30 * 1000 // 30s + export const FILES_DOCTYPE = 'io.cozy.files' export const CONTACTS_DOCTYPE = 'io.cozy.contacts' export const APPS_DOCTYPE = 'io.cozy.apps' diff --git a/src/search/helpers/replication.ts b/src/search/helpers/replication.ts new file mode 100644 index 0000000..8ca86c8 --- /dev/null +++ b/src/search/helpers/replication.ts @@ -0,0 +1,29 @@ +import CozyClient from 'cozy-client' +import Minilog from 'cozy-minilog' + +import { getPouchLink } from '@/search/helpers/client' + +const log = Minilog('🗂️ [Replication]') + +export const startReplicationWithDebounce = ( + client: CozyClient, + interval: number +) => { + let timeoutId: NodeJS.Timeout | null = null + + return (): void => { + if (timeoutId) { + log.debug('Reset replication debounce') + clearTimeout(timeoutId) + } + + timeoutId = setTimeout(() => { + const pouchLink = getPouchLink(client) + if (!pouchLink) { + return + } + log.debug('Start replication after debounce of ', interval) + pouchLink.startReplication() + }, interval) + } +} diff --git a/yarn.lock b/yarn.lock index b0c4e53..0d35211 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,7 +308,7 @@ resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== -"@cozy/minilog@1.0.0": +"@cozy/minilog@1.0.0", "@cozy/minilog@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@cozy/minilog/-/minilog-1.0.0.tgz#1acc1aad849261e931e255a5f181b638315f7b84" integrity sha512-IkDHF9CLh0kQeSEVsou59ar/VehvenpbEUjLfwhckJyOUqZnKAWmXy8qrBgMT5Loxr8Xjs2wmMnj0D67wP00eQ== @@ -2198,16 +2198,16 @@ cozy-client@^49.0.0: sift "^6.0.0" url-search-params-polyfill "^8.0.0" -cozy-client@^49.4.0: - version "49.4.0" - resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-49.4.0.tgz#248788e5e7a3595dd9137f5d0563ac6a5f5c0231" - integrity sha512-jIapZfMzJzCblI4pKnir3nXgOvrXavsRXkm7QB2qbDl9WSCJcpKvMw8Z1yDsma7nB/16E0rKzeEBh+BxXbStTw== +cozy-client@^49.8.0: + version "49.8.0" + resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-49.8.0.tgz#d4f6b3a2fb26dc6bfa561046b075eb6e221af73e" + integrity sha512-peb2n+buOihzqqNN3/zULmytQI8wOtQZsIqLWEaw2WfNXha1T23wNBrMuE7SaMOof361f011MrM5+WVYhKj+VA== dependencies: "@cozy/minilog" "1.0.0" "@types/jest" "^26.0.20" "@types/lodash" "^4.14.170" btoa "^1.2.1" - cozy-stack-client "^49.4.0" + cozy-stack-client "^49.8.0" date-fns "2.29.3" json-stable-stringify "^1.0.1" lodash "^4.17.13" @@ -2252,16 +2252,24 @@ cozy-minilog@^3.3.1: dependencies: microee "0.0.6" -cozy-pouch-link@^49.5.0: - version "49.5.0" - resolved "https://registry.yarnpkg.com/cozy-pouch-link/-/cozy-pouch-link-49.5.0.tgz#ee2b43725ccd63f793800d62562706eaa2ff9888" - integrity sha512-T4kcKrwajE0N3URu2lA/YaoKiPRrSVRSFxh9QmAdOhfUHS5qZFrd0ox5gvyWTtNRceeJh1H9jC7LNDV7Qy0q9A== +cozy-pouch-link@^49.8.0: + version "49.8.0" + resolved "https://registry.yarnpkg.com/cozy-pouch-link/-/cozy-pouch-link-49.8.0.tgz#a5361bbc0ed23b9bc4d804d31869c615fdf1d13e" + integrity sha512-c5+dWx5BNi37JuLVpil9SB91vy3EtB7B8Qa6Krx+znzPYcRoK3Bqjuns5c7hXG5rD8MV2qls1yvxQWUrfy82kA== dependencies: - cozy-client "^49.4.0" + cozy-client "^49.8.0" pouchdb-browser "^7.2.2" pouchdb-find "^7.2.2" -cozy-stack-client@^49.0.0, cozy-stack-client@^49.4.0: +cozy-realtime@^5.0.2: + version "5.0.2" + resolved "https://registry.yarnpkg.com/cozy-realtime/-/cozy-realtime-5.0.2.tgz#d515e6625e4386c812e8a0ce505c2e3abfea2245" + integrity sha512-ncFgsb2BeaYyM+Uax/qbcnDC1n7hpLNAiaqnVD02ApDpN1Uy/2GUzbcAr5xpc1gK4jkp6+fb8FrqyL+vcxXRsg== + dependencies: + "@cozy/minilog" "^1.0.0" + cozy-device-helper "^3.1.0" + +cozy-stack-client@^49.0.0: version "49.4.0" resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-49.4.0.tgz#775d2d48e74182049977e2e423f7a42d39a4ac61" integrity sha512-iy2vTpj25vHqErfPeclN3flI99il+WBm+Kt0FWI5YzM4H76+fE07/6Up4zOqtyOuaF7S22OHSim4Zl2hFB/SpA== @@ -2270,6 +2278,15 @@ cozy-stack-client@^49.0.0, cozy-stack-client@^49.4.0: mime "^2.4.0" qs "^6.7.0" +cozy-stack-client@^49.8.0: + version "49.8.0" + resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-49.8.0.tgz#c57dfefe50e47f228fee7e1921c438d35f4e0877" + integrity sha512-sYJL2o+DsNs7V5eQghXpWKcMzxc39QAKtM8zAdmWl2MMCyiqO3lBehRomhstcJHtuZrMLXXPQPr1A0ONBlMmZg== + dependencies: + detect-node "^2.0.4" + mime "^2.4.0" + qs "^6.7.0" + cozy-tsconfig@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/cozy-tsconfig/-/cozy-tsconfig-1.2.0.tgz#17e61f960f139fae4d26cbac2254b9ab632b269e" @@ -4742,6 +4759,13 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +node-fetch@2.6.7: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" + node-fetch@^2.6.1: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" @@ -6271,6 +6295,11 @@ util@^0.12.5: is-typed-array "^1.1.3" which-typed-array "^1.1.2" +uuid@8.3.2: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"