diff --git a/core/remote/cozy.js b/core/remote/cozy.js index 5df1b97ef..fe3d28c9a 100644 --- a/core/remote/cozy.js +++ b/core/remote/cozy.js @@ -306,15 +306,17 @@ class RemoteCozy { async changes( since /*: string */ = '0', - batchSize /*: number */ = 10000 + batchSize /*: number */ = 3000 ) /*: Promise<{last_seq: string, docs: Array}> */ { const client = await this.newClient() const { last_seq, remoteDocs } = since === '0' - ? await fetchInitialChanges(client) + ? await fetchInitialChanges(since, client, batchSize) : await fetchChangesFromFeed(since, this.client, batchSize) - const docs = await this.completeRemoteDocs(dropSpecialDocs(remoteDocs)) + const docs = (await this.completeRemoteDocs( + dropSpecialDocs(remoteDocs) + )).sort(byPath) return { last_seq, docs } } @@ -432,9 +434,9 @@ class RemoteCozy { .where({ dir_id: dir._id }) - .indexFields(['name']) - .sortBy([{ name: 'asc' }]) - .limitBy(10000) + .indexFields(['dir_id', 'name']) + .sortBy([{ dir_id: 'asc' }, { name: 'asc' }]) + .limitBy(3000) .offsetBookmark(resp.bookmark) resp = await client.query(queryDef) for (const j of resp.data) { @@ -591,28 +593,36 @@ async function fetchChangesFromFeed( } async function fetchInitialChanges( - client /*: CozyClient */ -) /*: Promise */ { - const { newLastSeq: last_seq } = await client.stackClient - .collection(FILES_DOCTYPE) - .fetchChanges({ - limit: 1, - descending: true - }) - - let resp = await client.stackClient + since /*: string */, + client /*: CozyClient */, + batchSize /*: number */, + remoteDocs /*: Array */ = [] +) { + const { newLastSeq: last_seq, pending, results } = await client .collection(FILES_DOCTYPE) - .all({ limit: 1000 }) - let remoteDocs = resp.data + .fetchChanges( + { since, includeDocs: true, limit: batchSize }, + { includeFilePath: true, skipDeleted: true, skipTrashed: true } + ) + remoteDocs = remoteDocs.concat(results.map(r => withDefaultValues(r.doc))) - while (resp && resp.next) { - resp = await client.stackClient - .collection(FILES_DOCTYPE) - .all({ limit: 1000, bookmark: resp.bookmark }) - remoteDocs = remoteDocs.concat(resp.data.map(withDefaultValues)) + if (pending === 0) { + return { last_seq, remoteDocs } + } else { + return fetchInitialChanges(last_seq, client, batchSize, remoteDocs) } +} - return { last_seq, remoteDocs } +function byPath(docA, docB) { + if (!docA._deleted && !docB._deleted) { + if (docA.path < docB.path) return -1 + if (docA.path > docB.path) return 1 + } else if (docA._deleted && !docB._deleted) { + return -1 + } else if (docB._deleted && !docA._deleted) { + return 1 + } + return 0 } module.exports = { diff --git a/core/remote/watcher/index.js b/core/remote/watcher/index.js index c347bdc37..ace7d1335 100644 --- a/core/remote/watcher/index.js +++ b/core/remote/watcher/index.js @@ -144,6 +144,7 @@ class RemoteWatcher { async watch() /*: Promise */ { const release = await this.pouch.lock(this) try { + this.events.emit('buffering-start') const seq = await this.pouch.getRemoteSeq() const { last_seq, docs } = await this.remoteCozy.changes(seq) this.events.emit('online') @@ -154,6 +155,7 @@ class RemoteWatcher { } this.events.emit('remote-start') + this.events.emit('buffering-end') await this.pullMany(docs) let target = -1 @@ -169,6 +171,7 @@ class RemoteWatcher { return remoteErrors.wrapError(err) } finally { release() + this.events.emit('buffering-end') this.events.emit('remote-end') } } diff --git a/package.json b/package.json index 07cc04ba4..9a869f280 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,7 @@ "chai": "^4.2.0", "chai-like": "^1.1.1", "chokidar": "^3.5.0", - "cozy-client": "^27.0.1", + "cozy-client": "^27.1.0", "cozy-client-js": "^0.19.0", "cozy-stack-client": "^24.0.0", "deep-diff": "^1.0.2", diff --git a/test/unit/remote/cozy.js b/test/unit/remote/cozy.js index e52251d5f..7db2dd77b 100644 --- a/test/unit/remote/cozy.js +++ b/test/unit/remote/cozy.js @@ -403,7 +403,7 @@ describe('RemoteCozy', function() { }) context('when no seq given', function() { - it('resolves only with non deleted docs', async function() { + it('resolves only with non trashed, non deleted docs', async function() { const dir = await builders.remoteDir().create() const file = await builders .remoteFile() @@ -414,6 +414,14 @@ describe('RemoteCozy', function() { .inDir(dir) .create() await builders.remoteErased(deletedFile).create() + const trashedFile = await builders + .remoteFile() + .inDir(dir) + .create() + await builders + .remoteFile(trashedFile) + .trashed() + .update() const { docs } = await remoteCozy.changes() @@ -424,6 +432,33 @@ describe('RemoteCozy', function() { }) }) + it('resolves with docs ordered by path asc', async function() { + const dirB = await builders + .remoteDir() + .inRootDir() + .name('dirB') + .create() + const fileB = await builders + .remoteFile() + .inRootDir() + .name('fileB') + .create() + const dirA = await builders + .remoteDir() + .inRootDir() + .name('dirA') + .create() + const fileA = await builders + .remoteFile() + .inDir(dirA) + .name('fileA') + .create() + + const { docs } = await remoteCozy.changes() + + should(docs).containDeepOrdered([dirA, fileA, dirB, fileB]) + }) + it('does not swallow errors', function() { this.config.cozyUrl = cozyStackDouble.url() const remoteCozy = new RemoteCozy(this.config) diff --git a/yarn.lock b/yarn.lock index 2685b563c..40a9e77fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1730,10 +1730,10 @@ cozy-client-js@^0.19.0: pouchdb-browser "7.0.0" pouchdb-find "7.0.0" -cozy-client@^27.0.1: - version "27.0.1" - resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-27.0.1.tgz#d350adfe7ffed462273bb3a7798bafff707c766d" - integrity sha512-wA/m3HyR2zWybOLuQUyzONsjemJ+MVL66hDZvblTcX0GFNfyw+4snkG/QG6INXZRknIGsH6ZCR8Ah2Pag4mSQQ== +cozy-client@^27.1.0: + version "27.1.0" + resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-27.1.0.tgz#3611c9e407259b5701d6367bcea72f51b5ee423a" + integrity sha512-On/x99pUrZHvkZ3pCr9aSCnyPcVqP5DJAQHw42Gt4u5koOGGr2t8c07k5WrJQmF09n7sXVCZ89pLmwFNQ8JG+A== dependencies: "@cozy/minilog" "1.0.0" "@types/jest" "^26.0.20" @@ -1742,7 +1742,7 @@ cozy-client@^27.0.1: cozy-device-helper "^1.12.0" cozy-flags "2.7.1" cozy-logger "^1.6.0" - cozy-stack-client "^27.0.0" + cozy-stack-client "^27.1.0" json-stable-stringify "^1.0.1" lodash "^4.17.13" microee "^0.0.6" @@ -1793,10 +1793,10 @@ cozy-stack-client@^24.0.0: mime "^2.4.0" qs "^6.7.0" -cozy-stack-client@^27.0.0: - version "27.0.0" - resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-27.0.0.tgz#a3462169aceabf5758679aaacfe030ba2497bc5f" - integrity sha512-iWCsmZmAVanfVdz4g5W7wJqNs5uqOk1SQffvz0M/MxCO3loYM/2fB5fdSYi63VOvrAiaZELqdAVPRE+i0uqNcQ== +cozy-stack-client@^27.1.0: + version "27.1.0" + resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-27.1.0.tgz#facc296e99b666c5fd56e449c65f95decac6ec8b" + integrity sha512-fcZJpoUd5E+rDRwiuQ5Whjg+jJlxlC/xpmDjdph0mlyDshAP5B8pdNpPIm7MYhnUbcS97JEZHLcQntdKZnHz5Q== dependencies: cozy-flags "2.7.1" detect-node "^2.0.4"