From 68ee35bf5cd90f30902501f0560af17931bd8597 Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Thu, 4 Nov 2021 17:55:40 +0100 Subject: [PATCH 1/5] deps: Bump cozy-client to v27.1.0 This version brings a new method to fetch `io.cozy.files` changes from the stack without deleted and trashed files. --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 07cc04ba4..9a869f280 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,7 @@ "chai": "^4.2.0", "chai-like": "^1.1.1", "chokidar": "^3.5.0", - "cozy-client": "^27.0.1", + "cozy-client": "^27.1.0", "cozy-client-js": "^0.19.0", "cozy-stack-client": "^24.0.0", "deep-diff": "^1.0.2", diff --git a/yarn.lock b/yarn.lock index 2685b563c..40a9e77fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1730,10 +1730,10 @@ cozy-client-js@^0.19.0: pouchdb-browser "7.0.0" pouchdb-find "7.0.0" -cozy-client@^27.0.1: - version "27.0.1" - resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-27.0.1.tgz#d350adfe7ffed462273bb3a7798bafff707c766d" - integrity sha512-wA/m3HyR2zWybOLuQUyzONsjemJ+MVL66hDZvblTcX0GFNfyw+4snkG/QG6INXZRknIGsH6ZCR8Ah2Pag4mSQQ== +cozy-client@^27.1.0: + version "27.1.0" + resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-27.1.0.tgz#3611c9e407259b5701d6367bcea72f51b5ee423a" + integrity sha512-On/x99pUrZHvkZ3pCr9aSCnyPcVqP5DJAQHw42Gt4u5koOGGr2t8c07k5WrJQmF09n7sXVCZ89pLmwFNQ8JG+A== dependencies: "@cozy/minilog" "1.0.0" "@types/jest" "^26.0.20" @@ -1742,7 +1742,7 @@ cozy-client@^27.0.1: cozy-device-helper "^1.12.0" cozy-flags "2.7.1" cozy-logger "^1.6.0" - cozy-stack-client "^27.0.0" + cozy-stack-client "^27.1.0" json-stable-stringify "^1.0.1" lodash "^4.17.13" microee "^0.0.6" @@ -1793,10 +1793,10 @@ cozy-stack-client@^24.0.0: mime "^2.4.0" qs "^6.7.0" -cozy-stack-client@^27.0.0: - version "27.0.0" - resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-27.0.0.tgz#a3462169aceabf5758679aaacfe030ba2497bc5f" - integrity sha512-iWCsmZmAVanfVdz4g5W7wJqNs5uqOk1SQffvz0M/MxCO3loYM/2fB5fdSYi63VOvrAiaZELqdAVPRE+i0uqNcQ== +cozy-stack-client@^27.1.0: + version "27.1.0" + resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-27.1.0.tgz#facc296e99b666c5fd56e449c65f95decac6ec8b" + integrity sha512-fcZJpoUd5E+rDRwiuQ5Whjg+jJlxlC/xpmDjdph0mlyDshAP5B8pdNpPIm7MYhnUbcS97JEZHLcQntdKZnHz5Q== dependencies: cozy-flags "2.7.1" detect-node "^2.0.4" From c611a9b3b954bb6bd9f383cf10ec4c297a264eef Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Wed, 27 Oct 2021 15:50:29 +0200 Subject: [PATCH 2/5] core/remote: Speed up initial changes fetch When a Cozy has seen a lot of changes, especially deleted documents, fetching all the changes after connecting the Desktop client can be quite long and hit a `cozy-stack` timeout. We will thus make use of a new `cozy-client` method to fetch those changes with deleted and trashed documents filtered by `cozy-stack` instead of CouchDB so we don't hit the time limit. We'll also reduce the number of documents fetched in one batch to decrease load on the server side. Besides, fetching changes via the files collection meant exluded documents would not be filtered out by `cozy-stack` and we'd have to do it on the client side. --- core/remote/cozy.js | 37 +++++++++++++++++-------------------- test/unit/remote/cozy.js | 10 +++++++++- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/core/remote/cozy.js b/core/remote/cozy.js index 5df1b97ef..c2a72501a 100644 --- a/core/remote/cozy.js +++ b/core/remote/cozy.js @@ -306,12 +306,12 @@ class RemoteCozy { async changes( since /*: string */ = '0', - batchSize /*: number */ = 10000 + batchSize /*: number */ = 3000 ) /*: Promise<{last_seq: string, docs: Array}> */ { const client = await this.newClient() const { last_seq, remoteDocs } = since === '0' - ? await fetchInitialChanges(client) + ? await fetchInitialChanges(since, client, batchSize) : await fetchChangesFromFeed(since, this.client, batchSize) const docs = await this.completeRemoteDocs(dropSpecialDocs(remoteDocs)) @@ -591,28 +591,25 @@ async function fetchChangesFromFeed( } async function fetchInitialChanges( - client /*: CozyClient */ -) /*: Promise */ { - const { newLastSeq: last_seq } = await client.stackClient - .collection(FILES_DOCTYPE) - .fetchChanges({ - limit: 1, - descending: true - }) - - let resp = await client.stackClient + since /*: string */, + client /*: CozyClient */, + batchSize /*: number */, + remoteDocs /*: Array */ = [] +) { + const { newLastSeq: last_seq, pending, results } = await client .collection(FILES_DOCTYPE) - .all({ limit: 1000 }) - let remoteDocs = resp.data + .fetchChanges( + { since, includeDocs: true, limit: batchSize }, + { includeFilePath: true, skipDeleted: true, skipTrashed: true } + ) + remoteDocs = remoteDocs.concat(results.map(r => withDefaultValues(r.doc))) - while (resp && resp.next) { - resp = await client.stackClient - .collection(FILES_DOCTYPE) - .all({ limit: 1000, bookmark: resp.bookmark }) - remoteDocs = remoteDocs.concat(resp.data.map(withDefaultValues)) + if (pending === 0) { + return { last_seq, remoteDocs } + } else { + return fetchInitialChanges(last_seq, client, batchSize, remoteDocs) } - return { last_seq, remoteDocs } } module.exports = { diff --git a/test/unit/remote/cozy.js b/test/unit/remote/cozy.js index e52251d5f..f1c0873c8 100644 --- a/test/unit/remote/cozy.js +++ b/test/unit/remote/cozy.js @@ -403,7 +403,7 @@ describe('RemoteCozy', function() { }) context('when no seq given', function() { - it('resolves only with non deleted docs', async function() { + it('resolves only with non trashed, non deleted docs', async function() { const dir = await builders.remoteDir().create() const file = await builders .remoteFile() @@ -414,6 +414,14 @@ describe('RemoteCozy', function() { .inDir(dir) .create() await builders.remoteErased(deletedFile).create() + const trashedFile = await builders + .remoteFile() + .inDir(dir) + .create() + await builders + .remoteFile(trashedFile) + .trashed() + .update() const { docs } = await remoteCozy.changes() From d0754cc975ff46db15dc7f0970f8f7a9dc8f6548 Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Wed, 27 Oct 2021 15:57:09 +0200 Subject: [PATCH 3/5] core/remote: Sort initial changes by path The order of changes synchronization depends on the order of their merge into our PouchDB database. Since it's better to synchronize parent folder creations before their content's, we order the fetched changes by path to make sure we'll merge parents first. --- core/remote/cozy.js | 15 ++++++++++++++- test/unit/remote/cozy.js | 27 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/core/remote/cozy.js b/core/remote/cozy.js index c2a72501a..44d655734 100644 --- a/core/remote/cozy.js +++ b/core/remote/cozy.js @@ -314,7 +314,9 @@ class RemoteCozy { ? await fetchInitialChanges(since, client, batchSize) : await fetchChangesFromFeed(since, this.client, batchSize) - const docs = await this.completeRemoteDocs(dropSpecialDocs(remoteDocs)) + const docs = (await this.completeRemoteDocs( + dropSpecialDocs(remoteDocs) + )).sort(byPath) return { last_seq, docs } } @@ -609,7 +611,18 @@ async function fetchInitialChanges( } else { return fetchInitialChanges(last_seq, client, batchSize, remoteDocs) } +} +function byPath(docA, docB) { + if (!docA._deleted && !docB._deleted) { + if (docA.path < docB.path) return -1 + if (docA.path > docB.path) return 1 + } else if (docA._deleted && !docB._deleted) { + return -1 + } else if (docB._deleted && !docA._deleted) { + return 1 + } + return 0 } module.exports = { diff --git a/test/unit/remote/cozy.js b/test/unit/remote/cozy.js index f1c0873c8..7db2dd77b 100644 --- a/test/unit/remote/cozy.js +++ b/test/unit/remote/cozy.js @@ -432,6 +432,33 @@ describe('RemoteCozy', function() { }) }) + it('resolves with docs ordered by path asc', async function() { + const dirB = await builders + .remoteDir() + .inRootDir() + .name('dirB') + .create() + const fileB = await builders + .remoteFile() + .inRootDir() + .name('fileB') + .create() + const dirA = await builders + .remoteDir() + .inRootDir() + .name('dirA') + .create() + const fileA = await builders + .remoteFile() + .inDir(dirA) + .name('fileA') + .create() + + const { docs } = await remoteCozy.changes() + + should(docs).containDeepOrdered([dirA, fileA, dirB, fileB]) + }) + it('does not swallow errors', function() { this.config.cozyUrl = cozyStackDouble.url() const remoteCozy = new RemoteCozy(this.config) From 438c56f2f111860d4c9aae22f09d0de2ed0d9d4e Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Wed, 27 Oct 2021 15:59:45 +0200 Subject: [PATCH 4/5] core/remote: Fix getDirectoryContent's query index The `RemoteCozy.getDirectoryContent()` method's query to fetch the content of the given remote directory was not declaring `dir_id` as an indexed field although it is used in the selector. This means the CouchDB request won't use the appropriate index and the query will be very slow, at best. Simply declaring the field as part of the indexed fields and the sort fields will improve the query response time. We also reduce the limit of the query as we've encountered timeouts on some Cozy with the 10000 documents limit. --- core/remote/cozy.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/remote/cozy.js b/core/remote/cozy.js index 44d655734..fe3d28c9a 100644 --- a/core/remote/cozy.js +++ b/core/remote/cozy.js @@ -434,9 +434,9 @@ class RemoteCozy { .where({ dir_id: dir._id }) - .indexFields(['name']) - .sortBy([{ name: 'asc' }]) - .limitBy(10000) + .indexFields(['dir_id', 'name']) + .sortBy([{ dir_id: 'asc' }, { name: 'asc' }]) + .limitBy(3000) .offsetBookmark(resp.bookmark) resp = await client.query(queryDef) for (const j of resp.data) { From 9ec359d8d84fcf2e41faa352e1d23fd94e294252 Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Wed, 27 Oct 2021 16:03:05 +0200 Subject: [PATCH 5/5] core/remote/watcher: Emit event before fetching We were not emitting any event in the `RemoteWatcher.watch()` method until we had received all the remote changes and were about to analyse and merge them. This means that when there were a lot of remote changes to fetch (e.g. during the initial changes fetch), the client's status would be "up-to-date" instead of showing that work was underway. We have a specific "buffering" status for this that can start once we have acquired the PouchDB lock but before we start fetching the remote changes and end once we start analysing the changes or at the end of the `watch()` call if no changes were detected or an error was thrown. --- core/remote/watcher/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/remote/watcher/index.js b/core/remote/watcher/index.js index c347bdc37..ace7d1335 100644 --- a/core/remote/watcher/index.js +++ b/core/remote/watcher/index.js @@ -144,6 +144,7 @@ class RemoteWatcher { async watch() /*: Promise */ { const release = await this.pouch.lock(this) try { + this.events.emit('buffering-start') const seq = await this.pouch.getRemoteSeq() const { last_seq, docs } = await this.remoteCozy.changes(seq) this.events.emit('online') @@ -154,6 +155,7 @@ class RemoteWatcher { } this.events.emit('remote-start') + this.events.emit('buffering-end') await this.pullMany(docs) let target = -1 @@ -169,6 +171,7 @@ class RemoteWatcher { return remoteErrors.wrapError(err) } finally { release() + this.events.emit('buffering-end') this.events.emit('remote-end') } }