From 44b9fdf2f118d75ddb3118ddae451cabad295e4a Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Thu, 10 Feb 2022 17:33:51 +0100 Subject: [PATCH 1/2] core/remote: Optimize directory content fetching As part of the selective synchronization feature, when a directory present on the remote Cozy is excluded from a Desktop client's synchronization or re-included, the Desktop client receives a change from the Cozy only for the directory itself. This only change is enough to remove the local directory and its content and prevent the future synchronization of any local directory with this path. However, when the directory is re-included, the change is not enough to detect and thus fetch its remote content. This is why we manually fetch this content. This is a recursive process, fetching its direct content and the content of each single one of its sub-directories and their sub-directories, etc. This process can be quite costly when the directory contains a lot of sub-directories, directly or indirectly, as we send one request for each one of them. We propose to optimize this process by fetching directories "level by level" by requesting at once the content of all the sub-directories accessible after each request. e.g.: With the following hierarchy photos/ photos/2021/holidays/summer/photo.jpg photos/2021/holidays/winter/photo.jpg photos/2022/holidays/winter/photo.jpg photos/2022/work/group-picture.jpg Requesting the content of the `photos` directory with the previous method would require 9 requests (one for each directory) while the new method will require only 4 requests (one for each level of the hierarchy). The new requests might be slightly more complex for the CouchDB server but they're still quite simple and reducing their number will really relieve the Desktop client sending them. --- core/remote/cozy.js | 81 +++++++++++++++++++++++++++------------- test/unit/remote/cozy.js | 37 +++++++++++++++++- 2 files changed, 92 insertions(+), 26 deletions(-) diff --git a/core/remote/cozy.js b/core/remote/cozy.js index bb61bb8b0..77471ecd9 100644 --- a/core/remote/cozy.js +++ b/core/remote/cozy.js @@ -450,34 +450,65 @@ class RemoteCozy { client = client || (await this.newClient()) let dirContent = [] - let resp /*: { next: boolean, bookmark?: string, data: Object[] } */ = { - next: true, - data: [] + let nextDirs = [dir] + while (nextDirs.length) { + const nestedContent = await this.getDirectoriesContent(nextDirs, { + client + }) + dirContent = dirContent.concat(nestedContent.docs) + nextDirs = nestedContent.nextDirs } - while (resp && resp.next) { - const queryDef = Q(FILES_DOCTYPE) - .where({ - dir_id: dir._id - }) - .indexFields(['dir_id', 'name']) - .sortBy([{ dir_id: 'asc' }, { name: 'asc' }]) - .limitBy(3000) - .offsetBookmark(resp.bookmark) - resp = await client.query(queryDef) - for (const j of resp.data) { - const remoteJson = jsonApiToRemoteJsonDoc(j) - if (remoteJson._deleted) continue - - const remoteDoc = await this.toRemoteDoc(remoteJson, dir) - dirContent.push(remoteDoc) - if (remoteDoc.type === DIR_TYPE) { - // Fetch subdir content - dirContent.push(this.getDirectoryContent(remoteDoc, { client })) - } + + return dirContent.sort((a, b) => { + if (a.path < b.path) return -1 + if (a.path > b.path) return 1 + return 0 + }) + } + + async getDirectoriesContent( + dirs /*: $ReadOnlyArray */, + { client } /*: { client: CozyClient } */ + ) /*: Promise<{ nextDirs: $ReadOnlyArray, docs: $ReadOnlyArray }> */ { + const queryDef = Q(FILES_DOCTYPE) + .where({ + dir_id: { $in: dirs.map(dir => dir._id) } + }) + .indexFields(['dir_id', 'name']) + .sortBy([{ dir_id: 'asc' }, { name: 'asc' }]) + .limitBy(3000) + + const resp = await this.queryAll(queryDef, { client }) + + const nextDirs = [] + const docs = [] + for (const j of resp.data) { + const remoteJson = jsonApiToRemoteJsonDoc(j) + if (remoteJson._deleted) continue + + const parentDir = dirs.find( + dir => dir._id === remoteJson.attributes.dir_id + ) + const remoteDoc = await this.toRemoteDoc(remoteJson, parentDir) + docs.push(remoteDoc) + + if (remoteDoc.type === DIR_TYPE) nextDirs.push(remoteDoc) + } + return { nextDirs, docs } + } + + async queryAll(queryDef /*: Q */, { client } /*: { client: CozyClient } */) { + const { data, next, bookmark } = await client.query(queryDef) + + if (next) { + return { + data: data.concat( + await this.queryAll(queryDef.offsetBookmark(bookmark), { client }) + ) } + } else { + return { data } } - // $FlowFixMe Array.prototype.flat is available in NodeJS v12 - return (await Promise.all(dirContent)).flat() } async isEmpty(id /*: string */) /*: Promise */ { diff --git a/test/unit/remote/cozy.js b/test/unit/remote/cozy.js index 8234113cd..7d875970f 100644 --- a/test/unit/remote/cozy.js +++ b/test/unit/remote/cozy.js @@ -878,6 +878,10 @@ describe('RemoteCozy', function() { 'dir/subdir/subsubdir/', 'dir/subdir/file', 'dir/file', + 'dir/other-subdir/', + 'dir/other-subdir/next-level/', + 'dir/other-subdir/next-level/last-level/', + 'dir/other-subdir/next-level/last-level/content', 'dir/subdir/subsubdir/last', 'hello.txt', 'other-dir/', @@ -887,6 +891,10 @@ describe('RemoteCozy', function() { remoteCozy.getDirectoryContent(tree['dir/']) ).be.fulfilledWith([ tree['dir/file'], + tree['dir/other-subdir/'], + tree['dir/other-subdir/next-level/'], + tree['dir/other-subdir/next-level/last-level/'], + tree['dir/other-subdir/next-level/last-level/content'], tree['dir/subdir/'], tree['dir/subdir/file'], tree['dir/subdir/subsubdir/'], @@ -894,6 +902,33 @@ describe('RemoteCozy', function() { ]) }) + it('requests content level by level and not directory by directory', async () => { + const tree = await builders.createRemoteTree([ + 'dir/', + 'dir/file', + 'dir/subdir/', + 'dir/subdir/file', + 'dir/subdir/subsubdir/', + 'dir/subdir/subsubdir/last', + 'dir/other-subdir/', + 'dir/other-subdir/next-level/', + 'dir/other-subdir/next-level/last-level/', + 'dir/other-subdir/next-level/last-level/content', + 'other-dir/', + 'other-dir/content' + ]) + + const client = await remoteCozy.newClient() + const querySpy = sinon.spy(client, 'query') + try { + await remoteCozy.getDirectoryContent(tree['dir/'], { client }) + + should(querySpy).have.callCount(4) + } finally { + querySpy.restore() + } + }) + it('does not fail on an empty directory', async () => { const dir = await builders .remoteDir() @@ -918,7 +953,7 @@ describe('RemoteCozy', function() { const stubbedClient = await remoteCozy.newClient() const originalQuery = stubbedClient.query.bind(stubbedClient) sinon.stub(stubbedClient, 'query').callsFake(async queryDef => { - if (queryDef.selector.dir_id === tree['dir/subdir/']._id) { + if (queryDef.selector.dir_id.$in.includes(tree['dir/subdir/']._id)) { throw new Error('test error') } else { return originalQuery(queryDef) From 56c59e2f3aff6ccf18c2ee96aba3cbe7fc4c5eaf Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Fri, 11 Feb 2022 15:27:32 +0100 Subject: [PATCH 2/2] core/remote/watcher: Optimize first ever retrieval As part of the selective synchronization, for the Desktop client to detect that a directory previously excluded from its synchronization has been re-included and its content must be manually retrieved, the remote change has to meet the following criterias: - it must be a directory change - it must be an "addition" (i.e. the document is not stored in the local PouchDB database) - it must have a short revision number greater than 1, meaning it's been modified on the remote Cozy (i.e. excluding and re-including count as modifications) However, these criterias are not exclusive enough to filter out directories that might have been added and modified on the remote Cozy between the last time the remote changes were fetched on the current retrieval. In this case, it is unnecessary to manually fetch their remote content as we'll retrieve them via their own remote changes but we'll do it anyway as we have no way to tell this situation apart from a folder re-inclusion. Nevertheless, there is one situation where we are sure we will never have to manually fetch a folder's content: the initial remote content retrieval. Indeed, in this case, we are already fetching the remote content and not remote changes so we are sure not to miss anything. In this case, we will simply not try to fetch the content of new directories with a revision greater than 1. --- core/pouch/index.js | 3 +- core/remote/constants.js | 5 +- core/remote/cozy.js | 17 ++-- core/remote/watcher/index.js | 31 +++++-- test/integration/sync_state.js | 4 +- test/support/helpers/remote.js | 2 +- test/unit/remote/watcher.js | 154 ++++++++++++++++++++++++++------- 7 files changed, 167 insertions(+), 49 deletions(-) diff --git a/core/pouch/index.js b/core/pouch/index.js index dbe2a9905..59c53f855 100644 --- a/core/pouch/index.js +++ b/core/pouch/index.js @@ -21,6 +21,7 @@ const { migrate, migrationLog } = require('./migrations') +const remoteConstants = require('../remote/constants') /*:: import type { Config } from '../config' @@ -621,7 +622,7 @@ class Pouch { async getRemoteSeq() /*: Promise */ { const doc = await this.byIdMaybe('_local/remoteSeq') if (doc) return doc.seq - else return '0' + else return remoteConstants.INITIAL_SEQ } // Set last remote replication sequence diff --git a/core/remote/constants.js b/core/remote/constants.js index 6b22e6693..7ad0d9cc4 100644 --- a/core/remote/constants.js +++ b/core/remote/constants.js @@ -39,5 +39,8 @@ module.exports = { // Maximum file size allowed by Swift thus the remote Cozy. // See https://docs.openstack.org/kilo/config-reference/content/object-storage-constraints.html - MAX_FILE_SIZE: FIVE_GIGABYTES + MAX_FILE_SIZE: FIVE_GIGABYTES, + + // Initial CouchDB sequence + INITIAL_SEQ: '0' } diff --git a/core/remote/cozy.js b/core/remote/cozy.js index 77471ecd9..c5c073c92 100644 --- a/core/remote/cozy.js +++ b/core/remote/cozy.js @@ -16,6 +16,7 @@ const { FILES_DOCTYPE, FILE_TYPE, DIR_TYPE, + INITIAL_SEQ, MAX_FILE_SIZE, OAUTH_CLIENTS_DOCTYPE } = require('./constants') @@ -317,20 +318,20 @@ class RemoteCozy { } async changes( - since /*: string */ = '0', + since /*: string */ = INITIAL_SEQ, batchSize /*: number */ = 3000 - ) /*: Promise<{last_seq: string, docs: Array}> */ { + ) /*: Promise<{last_seq: string, docs: Array, isInitialFetch: boolean}> */ { const client = await this.newClient() - const { last_seq, remoteDocs } = - since === '0' - ? await fetchInitialChanges(since, client, batchSize) - : await fetchChangesFromFeed(since, this.client, batchSize) + const isInitialFetch = since === INITIAL_SEQ + const { last_seq, remoteDocs } = isInitialFetch + ? await fetchInitialChanges(since, client, batchSize) + : await fetchChangesFromFeed(since, this.client, batchSize) const docs = (await this.completeRemoteDocs( dropSpecialDocs(remoteDocs) )).sort(byPath) - return { last_seq, docs } + return { last_seq, docs, isInitialFetch } } async fetchLastSeq() { @@ -338,7 +339,7 @@ class RemoteCozy { const { last_seq } = await client .collection(FILES_DOCTYPE) .fetchChangesRaw({ - since: '0', + since: INITIAL_SEQ, descending: true, limit: 1, includeDocs: false diff --git a/core/remote/watcher/index.js b/core/remote/watcher/index.js index 631624984..bccc8708a 100644 --- a/core/remote/watcher/index.js +++ b/core/remote/watcher/index.js @@ -22,7 +22,13 @@ import type EventEmitter from 'events' import type { Pouch } from '../../pouch' import type Prep from '../../prep' import type { RemoteCozy } from '../cozy' -import type { Metadata, MetadataRemoteInfo, SavedMetadata, RemoteRevisionsByID } from '../../metadata' +import type { + Metadata, + MetadataRemoteInfo, + MetadataRemoteDir, + SavedMetadata, + RemoteRevisionsByID +} from '../../metadata' import type { RemoteChange, RemoteFileMove, RemoteDirMove, RemoteDescendantChange } from '../change' import type { RemoteDeletion } from '../document' import type { RemoteError } from '../errors' @@ -42,6 +48,15 @@ const log = logger({ const sideName = 'remote' +const folderMightHaveBeenExcluded = ( + remoteDir /*: MetadataRemoteDir */ +) /*: boolean %checks */ => { + // A folder newly created has a rev number of 1. + // Once exluded, its rev number is at least 2. + // Once re-included, its rev number is at least 3. + return metadata.extractRevNumber(remoteDir) > 2 +} + /** Get changes from the remote Cozy and prepare them for merge */ class RemoteWatcher { /*:: @@ -146,7 +161,9 @@ class RemoteWatcher { try { this.events.emit('buffering-start') const seq = await this.pouch.getRemoteSeq() - const { last_seq, docs } = await this.remoteCozy.changes(seq) + const { last_seq, docs, isInitialFetch } = await this.remoteCozy.changes( + seq + ) this.events.emit('online') if (docs.length === 0) { @@ -156,7 +173,7 @@ class RemoteWatcher { this.events.emit('remote-start') this.events.emit('buffering-end') - await this.pullMany(docs) + await this.pullMany(docs, { isInitialFetch }) let target = -1 target = (await this.pouch.db.changes({ limit: 1, descending: true })) @@ -191,18 +208,20 @@ class RemoteWatcher { * FIXME: Misleading method name? */ async pullMany( - docs /*: $ReadOnlyArray */ + docs /*: $ReadOnlyArray */, + { isInitialFetch } /*: { isInitialFetch: boolean } */ ) /*: Promise */ { let changes = await this.analyse(docs, await this.olds(docs)) for (const change of changes) { if ( + !isInitialFetch && change.type === 'DirAddition' && - metadata.extractRevNumber(change.doc.remote) > 1 + folderMightHaveBeenExcluded(change.doc.remote) ) { log.trace( { path: change.doc.path }, - 'Fetching content of unknwon folder...' + 'Fetching content of unknown folder...' ) const children = (await this.remoteCozy.getDirectoryContent( change.doc.remote diff --git a/test/integration/sync_state.js b/test/integration/sync_state.js index 45b5bec0c..7fa03f192 100644 --- a/test/integration/sync_state.js +++ b/test/integration/sync_state.js @@ -34,7 +34,9 @@ describe('Sync state', () => { it('1 sync error (missing remote file)', async () => { const remoteFile = builders.remoteFile().build() - await helpers._remote.watcher.pullMany([remoteFile]) + await helpers._remote.watcher.pullMany([remoteFile], { + isInitialFetch: true // XXX: avoid unnecessary remote requests + }) await helpers.syncAll() should(events.emit.args).containDeepOrdered([ ['sync-start'], diff --git a/test/support/helpers/remote.js b/test/support/helpers/remote.js index 9a147694e..b4a7cf72b 100644 --- a/test/support/helpers/remote.js +++ b/test/support/helpers/remote.js @@ -179,7 +179,7 @@ class RemoteTestHelpers { } async simulateChanges(docs /*: * */) { - await this.side.watcher.pullMany(docs) + await this.side.watcher.pullMany(docs, { isInitialFetch: false }) } async readFile(path /*: string */) { diff --git a/test/unit/remote/watcher.js b/test/unit/remote/watcher.js index 47663bee5..a4951fe3f 100644 --- a/test/unit/remote/watcher.js +++ b/test/unit/remote/watcher.js @@ -21,7 +21,11 @@ const metadata = require('../../../core/metadata') const Prep = require('../../../core/prep') const { RemoteCozy } = require('../../../core/remote/cozy') const remoteErrors = require('../../../core/remote/errors') -const { FILE_TYPE, DIR_TYPE } = require('../../../core/remote/constants') +const { + FILE_TYPE, + DIR_TYPE, + INITIAL_SEQ +} = require('../../../core/remote/constants') const { RemoteWatcher } = require('../../../core/remote/watcher') const timestamp = require('../../../core/utils/timestamp') @@ -380,7 +384,8 @@ describe('RemoteWatcher', function() { let changes beforeEach(function() { changes = { - last_seq: lastRemoteSeq, + isInitialFetch: false, + last_seq: String(Number(lastRemoteSeq) + 2), // XXX: Include the two changes returned docs: [builders.remoteFile().build(), builders.remoteDir().build()] } }) @@ -407,16 +412,82 @@ describe('RemoteWatcher', function() { it('pulls the changed files/dirs', async function() { await this.watcher.watch() - this.watcher.pullMany.should.be - .calledOnce() - .and.be.calledWithExactly(changes.docs) + should(this.watcher.pullMany) + .have.been.calledOnce() + .and.be.calledWithExactly(changes.docs, { isInitialFetch: false }) }) it('updates the last update sequence in local db', async function() { await this.watcher.watch() - this.pouch.setRemoteSeq.should.be - .calledOnce() - .and.be.calledWithExactly(lastRemoteSeq) + should(this.pouch.setRemoteSeq) + .have.been.calledOnce() + .and.be.calledWithExactly(changes.last_seq) + }) + + context('when a directory has been modified more than once', () => { + it('fetches the content of potentially re-included directories', async function() { + const remoteDocs = [ + builders.remoteFile().build(), + builders + .remoteDir() + .shortRev(3) // XXX: this folder has already been modified twice on the Cozy + .build() + ] + + // Restored in a "parent" afterEach + this.remoteCozy.changes.resolves({ + isInitialFetch: false, + last_seq: String(Number(lastRemoteSeq) + remoteDocs.length), + docs: remoteDocs + }) + // Restore the original pullMany function which is stubbed in beforeEach + this.watcher.pullMany.restore() + // Create a new stub (which calls the original method though) so the + // restore() call in afterEach succeeds. + sinon.stub(this.watcher, 'pullMany').callThrough() + + const spy = sinon.spy(this.remoteCozy, 'getDirectoryContent') + try { + await this.watcher.watch() + should(spy) + .have.been.calledOnce() + .and.calledWith(remoteDocs[1]) + } finally { + spy.restore() + } + }) + + context('when fetching changes for the first time', () => { + it('does not fetch the content of modified directories', async function() { + // Restored in a "parent" afterEach + this.pouch.getRemoteSeq.resolves(INITIAL_SEQ) + // Restored in a "parent" afterEach + this.remoteCozy.changes.resolves({ + isInitialFetch: true, + last_seq: String(Number(INITIAL_SEQ) + 2), + docs: [ + builders.remoteFile().build(), + builders + .remoteDir() + .shortRev(3) // XXX: this folder has already been modified twice on the Cozy + .build() + ] + }) + // Restore the original pullMany function which is stubbed in beforeEach + this.watcher.pullMany.restore() + // Create a new stub (which calls the original method though) so the + // restore() call in afterEach succeeds. + sinon.stub(this.watcher, 'pullMany').callThrough() + + const spy = sinon.spy(this.remoteCozy, 'getDirectoryContent') + try { + await this.watcher.watch() + should(this.remoteCozy.getDirectoryContent).have.not.been.called() + } finally { + spy.restore() + } + }) + }) }) context('on error while fetching changes', () => { @@ -503,7 +574,7 @@ describe('RemoteWatcher', function() { it('pulls many changed files/dirs given their ids', async function() { apply.resolves() - await this.watcher.pullMany(remoteDocs) + await this.watcher.pullMany(remoteDocs, { isInitialFetch: false }) apply.callCount.should.equal(2) // Changes are sorted before applying (first one was given Metadata since @@ -524,13 +595,15 @@ describe('RemoteWatcher', function() { }) it('rejects with the first error', async function() { - await should(this.watcher.pullMany(remoteDocs)).be.rejectedWith( - new Error(remoteDocs[0]) - ) + await should( + this.watcher.pullMany(remoteDocs, { isInitialFetch: false }) + ).be.rejectedWith(new Error(remoteDocs[0])) }) it('still tries to pull other files/dirs', async function() { - await this.watcher.pullMany(remoteDocs).catch(() => {}) + await this.watcher + .pullMany(remoteDocs, { isInitialFetch: false }) + .catch(() => {}) should(apply.args[0][0]).have.properties({ type: 'FileAddition', doc: validMetadata(remoteDocs[0]) @@ -547,7 +620,9 @@ describe('RemoteWatcher', function() { builders.remoteErased().build(), builders.remoteFile().build() ] - await this.watcher.pullMany(remoteDocs).catch(() => {}) + await this.watcher + .pullMany(remoteDocs, { isInitialFetch: false }) + .catch(() => {}) should(apply).have.callCount(5) should(apply.args[0][0]).have.properties({ type: 'FileAddition', @@ -568,14 +643,18 @@ describe('RemoteWatcher', function() { }) it('releases the Pouch lock', async function() { - await this.watcher.pullMany(remoteDocs).catch(() => {}) + await this.watcher + .pullMany(remoteDocs, { isInitialFetch: false }) + .catch(() => {}) const nextLockPromise = this.pouch.lock('nextLock') await should(nextLockPromise).be.fulfilled() }) it('does not update the remote sequence', async function() { const remoteSeq = await this.pouch.getRemoteSeq() - await this.watcher.pullMany(remoteDocs).catch(() => {}) + await this.watcher + .pullMany(remoteDocs, { isInitialFetch: false }) + .catch(() => {}) should(this.pouch.getRemoteSeq()).be.fulfilledWith(remoteSeq) }) }) @@ -586,7 +665,7 @@ describe('RemoteWatcher', function() { .name('whatever') .build() - await this.watcher.pullMany([remoteDoc]) + await this.watcher.pullMany([remoteDoc], { isInitialFetch: false }) should(apply).be.calledOnce() should(apply.args[0][0].doc).deepEqual(validMetadata(remoteDoc)) @@ -599,7 +678,7 @@ describe('RemoteWatcher', function() { _deleted: true } - await this.watcher.pullMany([remoteDeletion]) + await this.watcher.pullMany([remoteDeletion], { isInitialFetch: false }) should(apply).be.calledOnce() should(apply.args[0][0].doc).deepEqual(remoteDeletion) @@ -612,13 +691,18 @@ describe('RemoteWatcher', function() { 'dir/subdir/file' ]) const dir = (tree['dir/'] /*: any */) - const updatedDir = await builders + const dirUpdatedOnce = await builders .remoteDir((dir /*: MetadataRemoteDir */)) .update() + // XXX: we update the directory twice to mimick a potential selective sync + // exclusion/re-inclusion. + const dirUpdatedTwice = await builders + .remoteDir((dirUpdatedOnce /*: MetadataRemoteDir */)) + .update() - await this.watcher.pullMany([updatedDir]) + await this.watcher.pullMany([dirUpdatedTwice], { isInitialFetch: false }) should(apply).be.calledThrice() - should(apply.args[0][0].doc).deepEqual(validMetadata(updatedDir)) + should(apply.args[0][0].doc).deepEqual(validMetadata(dirUpdatedTwice)) should(apply.args[1][0].doc).deepEqual(validMetadata(tree['dir/subdir/'])) should(apply.args[2][0].doc).deepEqual( validMetadata(tree['dir/subdir/file']) @@ -631,7 +715,7 @@ describe('RemoteWatcher', function() { .remoteDir((dir /*: MetadataRemoteDir */)) .update() - await this.watcher.pullMany([updatedDir]) + await this.watcher.pullMany([updatedDir], { isInitialFetch: false }) should(apply).be.calledOnce() should(apply.args[0][0].doc).deepEqual(validMetadata(updatedDir)) }) @@ -644,28 +728,36 @@ describe('RemoteWatcher', function() { 'dir/subdir/file2' ]) const dir = (tree['dir/'] /*: any */) - const updatedDir = await builders + const dirUpdatedOnce = await builders .remoteDir((dir /*: MetadataRemoteDir */)) .update() + // XXX: we update the directory twice to mimick a potential selective sync + // exclusion/re-inclusion. + const dirUpdatedTwice = await builders + .remoteDir((dirUpdatedOnce /*: MetadataRemoteDir */)) + .update() const file = (tree['dir/subdir/file1'] /*: any */) const updatedFile = await builders .remoteFile((file /*: MetadataRemoteFile */)) .update() const newFile = await builders .remoteFile() - .inDir(updatedDir) + .inDir(dirUpdatedTwice) .name('file3') .create() - await this.watcher.pullMany([ - updatedDir, - tree['dir/subdir/'], - tree['dir/subdir/file1'], - tree['dir/subdir/file2'] - ]) + await this.watcher.pullMany( + [ + dirUpdatedTwice, + tree['dir/subdir/'], + tree['dir/subdir/file1'], + tree['dir/subdir/file2'] + ], + { isInitialFetch: false } + ) should(apply.callCount).eql(6) // From Feed - should(apply.args[0][0].doc).deepEqual(validMetadata(updatedDir)) + should(apply.args[0][0].doc).deepEqual(validMetadata(dirUpdatedTwice)) should(apply.args[1][0].doc).deepEqual(validMetadata(tree['dir/subdir/'])) should(apply.args[2][0].doc).deepEqual( validMetadata(tree['dir/subdir/file1'])