From fd00145164812b0a4f62961b0567942e6e5be53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Fri, 22 Dec 2023 08:59:15 +0100 Subject: [PATCH] feat: synchronise all files in the folder if it has not been synchronised before --- .../src/logic/connector/domain/connector.ts | 5 +- .../connectors/folder.connector.ts | 46 +++++++++++++++++-- .../connectors/gdrive.connector.ts | 40 ++++++++++++++-- .../connectors/tests/gdrive.connector.spec.js | 26 ++++++----- .../domain/dto/tests/update-sync.dto.spec.ts | 8 ++-- .../src/logic/sync/domain/nuclia-cloud.ts | 1 - .../src/logic/sync/domain/sync.entity.ts | 42 +++++++++-------- .../use-cases/get-sync-folders.use-case.ts | 2 +- ...se.ts => refresh-access-token.use-case.ts} | 0 .../use-cases/sync-all-data.use-case.ts | 2 +- .../sync-all-folders-data.use-case.ts | 9 +++- .../src/logic/sync/presentation/routes.ts | 2 +- 12 files changed, 134 insertions(+), 49 deletions(-) rename electron-app/src/logic/sync/domain/use-cases/{refresh-acces-token.use-case.ts => refresh-access-token.use-case.ts} (100%) diff --git a/electron-app/src/logic/connector/domain/connector.ts b/electron-app/src/logic/connector/domain/connector.ts index 1865dd3..1ade4bd 100644 --- a/electron-app/src/logic/connector/domain/connector.ts +++ b/electron-app/src/logic/connector/domain/connector.ts @@ -22,7 +22,7 @@ export const SyncItemValidator = z.object({ title: z.string().min(1, { message: 'Required' }), originalId: z.string().min(1, { message: 'Required' }), metadata: z.record(z.string()), - status: z.nativeEnum(FileStatus), + status: z.nativeEnum(FileStatus).optional(), modifiedGMT: z.string().optional(), isFolder: z.boolean().optional(), parents: z.array(z.string()).optional(), @@ -51,7 +51,8 @@ export interface IConnector { getParameters(): ConnectorParameters; getFolders(query?: string): Observable; getFiles(query?: string): Observable; - getLastModified(since: string, folders?: SyncItem[]): Observable; + getFilesFromFolders(folders: SyncItem[]): Observable; + getLastModified(since: string, folders?: SyncItem[]): Observable; // we cannot use the TextField from the SDK because we want to keep connectors independant download(resource: SyncItem): Observable; getLink(resource: SyncItem): Observable; diff --git a/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts index b9f6916..ad48d77 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts +++ b/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts @@ -43,7 +43,38 @@ class FolderImpl implements IConnector { return this._getFiles(this.params.path, query); } - getLastModified(since: string, folders?: SyncItem[]): Observable { + getFilesFromFolders(folders: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { + return forkJoin((folders || []).map((folder) => this._getFiles(folder.originalId))).pipe( + map((results) => { + const result: { items: SyncItem[] } = { + items: [], + }; + results.forEach((res) => { + result.items = [...result.items, ...res.items]; + }); + return result; + }), + ); + } catch (err) { + return of({ + items: [], + }); + } + } + + getLastModified(since: string, folders?: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { return forkJoin( (folders || []).map((folder) => @@ -51,9 +82,18 @@ class FolderImpl implements IConnector { switchMap((results) => this.getFilesModifiedSince(results.items, since)), ), ), - ).pipe(map((results) => results.reduce((acc, result) => acc.concat(result), [] as SyncItem[]))); + ).pipe( + map((results) => { + const items = results.reduce((acc, result) => acc.concat(result), [] as SyncItem[]); + return { + items, + }; + }), + ); } catch (err) { - return of([]); + return of({ + items: [], + }); } } diff --git a/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts index c8a3068..c0fe68f 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts +++ b/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts @@ -35,21 +35,53 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { return true; } - getLastModified(since: string, folders?: SyncItem[] | undefined): Observable { + getLastModified(since: string, folders?: SyncItem[] | undefined): Observable { if ((folders ?? []).length === 0) { - return of([]); + return of({ + items: [], + }); } try { return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe( map((results) => { - return results.reduce( + const items = results.reduce( (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)), [] as SyncItem[], ); + return { + items, + }; + }), + ); + } catch (err) { + return of({ + items: [], + }); + } + } + + getFilesFromFolders(folders: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { + return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe( + map((results) => { + const result: { items: SyncItem[] } = { + items: [], + }; + results.forEach((res) => { + result.items = [...result.items, ...res.items]; + }); + return result; }), ); } catch (err) { - return of([]); + return of({ + items: [], + }); } } diff --git a/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js b/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js index f6c31df..b9fa14f 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js +++ b/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js @@ -117,18 +117,20 @@ describe('Test last modified', () => { ]), ); - expect(lastModified).toEqual([ - { - uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', - title: 'PO6300590983', - originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', - modifiedGMT: '2023-11-29T12:49:27.539Z', - metadata: { - needsPdfConversion: 'yes', - mimeType: 'application/pdf', + expect(lastModified).toEqual({ + items: [ + { + uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + title: 'PO6300590983', + originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + modifiedGMT: '2023-11-29T12:49:27.539Z', + metadata: { + needsPdfConversion: 'yes', + mimeType: 'application/pdf', + }, + status: FileStatus.PENDING, }, - status: 'PENDING', - }, - ]); + ], + }); }); }); diff --git a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts index aaef9fb..1a9addb 100644 --- a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts +++ b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts @@ -96,7 +96,7 @@ describe('Update Sync dto tests', () => { foldersToSync: [{}], }); expect(error).toEqual( - 'Invalid format for foldersToSync: Error: title: Required, originalId: Required, metadata: Required, status: Required', + 'Invalid format for foldersToSync: Error: title: Required, originalId: Required, metadata: Required', ); expect(dto).toBeUndefined(); @@ -104,9 +104,7 @@ describe('Update Sync dto tests', () => { ...props, foldersToSync: [{ title: 'folder1' }], }); - expect(error).toEqual( - 'Invalid format for foldersToSync: Error: originalId: Required, metadata: Required, status: Required', - ); + expect(error).toEqual('Invalid format for foldersToSync: Error: originalId: Required, metadata: Required'); expect(dto).toBeUndefined(); [error, dto] = UpdateSyncDto.create({ @@ -114,7 +112,7 @@ describe('Update Sync dto tests', () => { foldersToSync: [{ title: 'folder1', metadata: 'metadata' }], }); expect(error).toEqual( - 'Invalid format for foldersToSync: Error: originalId: Required, metadata: Expected object, received string, status: Required', + 'Invalid format for foldersToSync: Error: originalId: Required, metadata: Expected object, received string', ); expect(dto).toBeUndefined(); diff --git a/electron-app/src/logic/sync/domain/nuclia-cloud.ts b/electron-app/src/logic/sync/domain/nuclia-cloud.ts index ac656d0..27ab490 100644 --- a/electron-app/src/logic/sync/domain/nuclia-cloud.ts +++ b/electron-app/src/logic/sync/domain/nuclia-cloud.ts @@ -52,7 +52,6 @@ export class NucliaCloud { switchMap((kb) => kb.getResourceBySlug(slug, [], []).pipe( switchMap((resource) => { - console.log('get source from nuclia', resource); if (data.metadata?.labels) { return resource .modify({ usermetadata: { classifications: data.metadata.labels } }) diff --git a/electron-app/src/logic/sync/domain/sync.entity.ts b/electron-app/src/logic/sync/domain/sync.entity.ts index a4e11cb..acaf306 100644 --- a/electron-app/src/logic/sync/domain/sync.entity.ts +++ b/electron-app/src/logic/sync/domain/sync.entity.ts @@ -1,7 +1,7 @@ -import { Observable, catchError, map, of } from 'rxjs'; +import { Observable, catchError, forkJoin, map, of } from 'rxjs'; import { z } from 'zod'; -import { IConnector, SearchResults, SyncItem } from '../../connector/domain/connector'; +import { FileStatus, IConnector, SearchResults, SyncItem } from '../../connector/domain/connector'; import { getConnector } from '../../connector/infrastructure/factory'; export type Connector = { @@ -99,22 +99,28 @@ export class SyncEntity { } getLastModified(): Observable<{ success: boolean; results: SyncItem[]; error?: string }> { - try { - return this.sourceConnector!.getLastModified( - this.lastSyncGMT || '2000-01-01T00:00:00.000Z', - this.foldersToSync, - ).pipe( - map((results) => { - return { success: true, results }; - }), - catchError((err) => { - console.error(`Error on ${this.id}: ${err.message}`); - return of({ success: false, results: [], error: `${err}` }); - }), - ); - } catch (err) { - return of({ success: false, results: [], error: `${err}` }); - } + const foldersToSyncPending: SyncItem[] = (this.foldersToSync ?? []).filter( + (folder) => folder.status === FileStatus.PENDING || folder.status === undefined, + ); + const foldersToSyncUpdated: SyncItem[] = (this.foldersToSync ?? []).filter( + (folder) => folder.status === FileStatus.UPLOADED, + ); + + const getFilesFoldersUpdated = this.sourceConnector!.getLastModified( + this.lastSyncGMT || '2000-01-01T00:00:00.000Z', + foldersToSyncUpdated, + ); + const getFilesFolderPending = this.sourceConnector!.getFilesFromFolders(foldersToSyncPending); + return forkJoin([getFilesFoldersUpdated, getFilesFolderPending]).pipe( + map((results) => { + const [updated, pending] = results; + return { success: true, results: [...updated.items, ...pending.items] }; + }), + catchError((err) => { + console.error(`Error on ${this.id}: ${err.message}`); + return of({ success: false, results: [], error: `${err}` }); + }), + ); } isAccesTokenValid(): Observable { diff --git a/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts index d2032ca..c6e4bda 100644 --- a/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts @@ -4,7 +4,7 @@ import { SearchResults } from '../../../connector/domain/connector'; import { CustomError } from '../../../errors'; import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; -import { RefreshAccessToken } from './refresh-acces-token.use-case'; +import { RefreshAccessToken } from './refresh-access-token.use-case'; export interface GetSyncFoldersUseCase { execute(id: string): Promise; diff --git a/electron-app/src/logic/sync/domain/use-cases/refresh-acces-token.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/refresh-access-token.use-case.ts similarity index 100% rename from electron-app/src/logic/sync/domain/use-cases/refresh-acces-token.use-case.ts rename to electron-app/src/logic/sync/domain/use-cases/refresh-access-token.use-case.ts diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts index c048041..1c8bb0f 100644 --- a/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts @@ -6,7 +6,7 @@ import { eventEmitter } from '../../../../server'; import { UpdateSyncDto } from '../dto/update-sync.dto'; import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; -import { RefreshAccessToken } from './refresh-acces-token.use-case'; +import { RefreshAccessToken } from './refresh-access-token.use-case'; import { SyncSingleFile } from './sync-single-file.use-case'; import { UpdateSync } from './update-sync.use-case'; diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts index f3e3e89..f1e48b9 100644 --- a/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts @@ -3,10 +3,11 @@ import { Observable, delay, forkJoin, map, of, switchMap, tap } from 'rxjs'; import { EVENTS } from '../../../../events/events'; import { eventEmitter } from '../../../../server'; +import { FileStatus } from '../../../connector/domain/connector'; import { UpdateSyncDto } from '../dto/update-sync.dto'; import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; -import { RefreshAccessToken } from './refresh-acces-token.use-case'; +import { RefreshAccessToken } from './refresh-access-token.use-case'; import { SyncSingleFile } from './sync-single-file.use-case'; import { UpdateSync } from './update-sync.use-case'; @@ -29,9 +30,15 @@ export class SyncAllFolders implements SyncAllFoldersUseCase { error, }); + const foldersToSyncCopy = (structuredClone(syncEntity.foldersToSync) ?? []).map((folder) => { + folder.status = FileStatus.UPLOADED; + return folder; + }); + const [, updateSyncDto] = UpdateSyncDto.create({ lastSyncGMT: new Date().toISOString(), id: syncEntity.id, + foldersToSync: foldersToSyncCopy, }); new UpdateSync(this.repository).execute(updateSyncDto!); }; diff --git a/electron-app/src/logic/sync/presentation/routes.ts b/electron-app/src/logic/sync/presentation/routes.ts index 154b58d..a1f95eb 100644 --- a/electron-app/src/logic/sync/presentation/routes.ts +++ b/electron-app/src/logic/sync/presentation/routes.ts @@ -6,13 +6,13 @@ import { UpdateSyncDto } from '../domain/dto/update-sync.dto'; import { CreateSync } from '../domain/use-cases/create-sync.use-case'; import { DeleteSync } from '../domain/use-cases/delete-sync.use-case'; import { GetAllSync } from '../domain/use-cases/get-all-sync.use-case'; +import { GetSyncAuth } from '../domain/use-cases/get-sync-auth.use-case'; import { GetSyncFolders } from '../domain/use-cases/get-sync-folders.use-case'; import { GetSync } from '../domain/use-cases/get-sync.use-case'; import { SyncAllFolders } from '../domain/use-cases/sync-all-folders-data.use-case'; import { UpdateSync } from '../domain/use-cases/update-sync.use-case'; import { FileSystemSyncDatasource } from '../infrastructure/file-system.sync.datasource'; import { SyncRepository } from '../infrastructure/sync.repository'; -import { GetSyncAuth } from '../domain/use-cases/get-sync-auth.use-case'; export class SyncFileSystemRoutes { private readonly basePath: string;