feat: synchronise all files in the folder if it has not been synchron…

…ised before
nuclia · Dec 22, 2023 · fd00145 · fd00145
1 parent 8c8e124
commit fd00145
Show file tree

Hide file tree

Showing 12 changed files with 134 additions and 49 deletions.
diff --git a/electron-app/src/logic/connector/domain/connector.ts b/electron-app/src/logic/connector/domain/connector.ts
@@ -22,7 +22,7 @@ export const SyncItemValidator = z.object({
   title: z.string().min(1, { message: 'Required' }),
   originalId: z.string().min(1, { message: 'Required' }),
   metadata: z.record(z.string()),
-  status: z.nativeEnum(FileStatus),
+  status: z.nativeEnum(FileStatus).optional(),
   modifiedGMT: z.string().optional(),
   isFolder: z.boolean().optional(),
   parents: z.array(z.string()).optional(),
@@ -51,7 +51,8 @@ export interface IConnector {
   getParameters(): ConnectorParameters;
   getFolders(query?: string): Observable<SearchResults>;
   getFiles(query?: string): Observable<SearchResults>;
-  getLastModified(since: string, folders?: SyncItem[]): Observable<SyncItem[]>;
+  getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults>;
+  getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults>;
   // we cannot use the TextField from the SDK because we want to keep connectors independant
   download(resource: SyncItem): Observable<Blob | { body: string; format?: 'PLAIN' | 'MARKDOWN' | 'HTML' } | undefined>;
   getLink(resource: SyncItem): Observable<Link>;

diff --git a/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts
@@ -43,17 +43,57 @@ class FolderImpl implements IConnector {
     return this._getFiles(this.params.path, query);
   }
 
-  getLastModified(since: string, folders?: SyncItem[]): Observable<SyncItem[]> {
+  getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
+    if ((folders ?? []).length === 0) {
+      return of({
+        items: [],
+      });
+    }
+    try {
+      return forkJoin((folders || []).map((folder) => this._getFiles(folder.originalId))).pipe(
+        map((results) => {
+          const result: { items: SyncItem[] } = {
+            items: [],
+          };
+          results.forEach((res) => {
+            result.items = [...result.items, ...res.items];
+          });
+          return result;
+        }),
+      );
+    } catch (err) {
+      return of({
+        items: [],
+      });
+    }
+  }
+
+  getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults> {
+    if ((folders ?? []).length === 0) {
+      return of({
+        items: [],
+      });
+    }
+
     try {
       return forkJoin(
         (folders || []).map((folder) =>
           this._getFiles(folder.originalId).pipe(
             switchMap((results) => this.getFilesModifiedSince(results.items, since)),
           ),
         ),
-      ).pipe(map((results) => results.reduce((acc, result) => acc.concat(result), [] as SyncItem[])));
+      ).pipe(
+        map((results) => {
+          const items = results.reduce((acc, result) => acc.concat(result), [] as SyncItem[]);
+          return {
+            items,
+          };
+        }),
+      );
     } catch (err) {
-      return of([]);
+      return of({
+        items: [],
+      });
     }
   }
 

diff --git a/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts
@@ -35,21 +35,53 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
     return true;
   }
 
-  getLastModified(since: string, folders?: SyncItem[] | undefined): Observable<SyncItem[]> {
+  getLastModified(since: string, folders?: SyncItem[] | undefined): Observable<SearchResults> {
     if ((folders ?? []).length === 0) {
-      return of([]);
+      return of({
+        items: [],
+      });
     }
     try {
       return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe(
         map((results) => {
-          return results.reduce(
+          const items = results.reduce(
             (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)),
             [] as SyncItem[],
           );
+          return {
+            items,
+          };
+        }),
+      );
+    } catch (err) {
+      return of({
+        items: [],
+      });
+    }
+  }
+
+  getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
+    if ((folders ?? []).length === 0) {
+      return of({
+        items: [],
+      });
+    }
+    try {
+      return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe(
+        map((results) => {
+          const result: { items: SyncItem[] } = {
+            items: [],
+          };
+          results.forEach((res) => {
+            result.items = [...result.items, ...res.items];
+          });
+          return result;
         }),
       );
     } catch (err) {
-      return of([]);
+      return of({
+        items: [],
+      });
     }
   }
 

diff --git a/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js b/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js
@@ -117,18 +117,20 @@ describe('Test last modified', () => {
       ]),
     );
 
-    expect(lastModified).toEqual([
-      {
-        uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
-        title: 'PO6300590983',
-        originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
-        modifiedGMT: '2023-11-29T12:49:27.539Z',
-        metadata: {
-          needsPdfConversion: 'yes',
-          mimeType: 'application/pdf',
+    expect(lastModified).toEqual({
+      items: [
+        {
+          uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
+          title: 'PO6300590983',
+          originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
+          modifiedGMT: '2023-11-29T12:49:27.539Z',
+          metadata: {
+            needsPdfConversion: 'yes',
+            mimeType: 'application/pdf',
+          },
+          status: FileStatus.PENDING,
         },
-        status: 'PENDING',
-      },
-    ]);
+      ],
+    });
   });
 });
diff --git a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts
@@ -96,25 +96,23 @@ describe('Update Sync dto tests', () => {
       foldersToSync: [{}],
     });
     expect(error).toEqual(
-      'Invalid format for foldersToSync: Error: title: Required, originalId: Required, metadata: Required, status: Required',
+      'Invalid format for foldersToSync: Error: title: Required, originalId: Required, metadata: Required',
     );
     expect(dto).toBeUndefined();
 
     [error, dto] = UpdateSyncDto.create({
       ...props,
       foldersToSync: [{ title: 'folder1' }],
     });
-    expect(error).toEqual(
-      'Invalid format for foldersToSync: Error: originalId: Required, metadata: Required, status: Required',
-    );
+    expect(error).toEqual('Invalid format for foldersToSync: Error: originalId: Required, metadata: Required');
     expect(dto).toBeUndefined();
 
     [error, dto] = UpdateSyncDto.create({
       ...props,
       foldersToSync: [{ title: 'folder1', metadata: 'metadata' }],
     });
     expect(error).toEqual(
-      'Invalid format for foldersToSync: Error: originalId: Required, metadata: Expected object, received string, status: Required',
+      'Invalid format for foldersToSync: Error: originalId: Required, metadata: Expected object, received string',
     );
     expect(dto).toBeUndefined();
 

diff --git a/electron-app/src/logic/sync/domain/nuclia-cloud.ts b/electron-app/src/logic/sync/domain/nuclia-cloud.ts
@@ -52,7 +52,6 @@ export class NucliaCloud {
         switchMap((kb) =>
           kb.getResourceBySlug(slug, [], []).pipe(
             switchMap((resource) => {
-              console.log('get source from nuclia', resource);
               if (data.metadata?.labels) {
                 return resource
                   .modify({ usermetadata: { classifications: data.metadata.labels } })

diff --git a/electron-app/src/logic/sync/domain/sync.entity.ts b/electron-app/src/logic/sync/domain/sync.entity.ts
@@ -1,7 +1,7 @@
-import { Observable, catchError, map, of } from 'rxjs';
+import { Observable, catchError, forkJoin, map, of } from 'rxjs';
 
 import { z } from 'zod';
-import { IConnector, SearchResults, SyncItem } from '../../connector/domain/connector';
+import { FileStatus, IConnector, SearchResults, SyncItem } from '../../connector/domain/connector';
 import { getConnector } from '../../connector/infrastructure/factory';
 
 export type Connector = {
@@ -99,22 +99,28 @@ export class SyncEntity {
   }
 
   getLastModified(): Observable<{ success: boolean; results: SyncItem[]; error?: string }> {
-    try {
-      return this.sourceConnector!.getLastModified(
-        this.lastSyncGMT || '2000-01-01T00:00:00.000Z',
-        this.foldersToSync,
-      ).pipe(
-        map((results) => {
-          return { success: true, results };
-        }),
-        catchError((err) => {
-          console.error(`Error on ${this.id}: ${err.message}`);
-          return of({ success: false, results: [], error: `${err}` });
-        }),
-      );
-    } catch (err) {
-      return of({ success: false, results: [], error: `${err}` });
-    }
+    const foldersToSyncPending: SyncItem[] = (this.foldersToSync ?? []).filter(
+      (folder) => folder.status === FileStatus.PENDING || folder.status === undefined,
+    );
+    const foldersToSyncUpdated: SyncItem[] = (this.foldersToSync ?? []).filter(
+      (folder) => folder.status === FileStatus.UPLOADED,
+    );
+
+    const getFilesFoldersUpdated = this.sourceConnector!.getLastModified(
+      this.lastSyncGMT || '2000-01-01T00:00:00.000Z',
+      foldersToSyncUpdated,
+    );
+    const getFilesFolderPending = this.sourceConnector!.getFilesFromFolders(foldersToSyncPending);
+    return forkJoin([getFilesFoldersUpdated, getFilesFolderPending]).pipe(
+      map((results) => {
+        const [updated, pending] = results;
+        return { success: true, results: [...updated.items, ...pending.items] };
+      }),
+      catchError((err) => {
+        console.error(`Error on ${this.id}: ${err.message}`);
+        return of({ success: false, results: [], error: `${err}` });
+      }),
+    );
   }
 
   isAccesTokenValid(): Observable<boolean> {

diff --git a/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts
@@ -4,7 +4,7 @@ import { SearchResults } from '../../../connector/domain/connector';
 import { CustomError } from '../../../errors';
 import { SyncEntity } from '../sync.entity';
 import { ISyncRepository } from '../sync.repository';
-import { RefreshAccessToken } from './refresh-acces-token.use-case';
+import { RefreshAccessToken } from './refresh-access-token.use-case';
 
 export interface GetSyncFoldersUseCase {
   execute(id: string): Promise<SearchResults>;

diff --git a/...use-cases/refresh-acces-token.use-case.ts → ...se-cases/refresh-access-token.use-case.ts b/...use-cases/refresh-acces-token.use-case.ts → ...se-cases/refresh-access-token.use-case.ts
diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts
@@ -6,7 +6,7 @@ import { eventEmitter } from '../../../../server';
 import { UpdateSyncDto } from '../dto/update-sync.dto';
 import { SyncEntity } from '../sync.entity';
 import { ISyncRepository } from '../sync.repository';
-import { RefreshAccessToken } from './refresh-acces-token.use-case';
+import { RefreshAccessToken } from './refresh-access-token.use-case';
 import { SyncSingleFile } from './sync-single-file.use-case';
 import { UpdateSync } from './update-sync.use-case';
 

diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts
@@ -3,10 +3,11 @@ import { Observable, delay, forkJoin, map, of, switchMap, tap } from 'rxjs';
 
 import { EVENTS } from '../../../../events/events';
 import { eventEmitter } from '../../../../server';
+import { FileStatus } from '../../../connector/domain/connector';
 import { UpdateSyncDto } from '../dto/update-sync.dto';
 import { SyncEntity } from '../sync.entity';
 import { ISyncRepository } from '../sync.repository';
-import { RefreshAccessToken } from './refresh-acces-token.use-case';
+import { RefreshAccessToken } from './refresh-access-token.use-case';
 import { SyncSingleFile } from './sync-single-file.use-case';
 import { UpdateSync } from './update-sync.use-case';
 
@@ -29,9 +30,15 @@ export class SyncAllFolders implements SyncAllFoldersUseCase {
       error,
     });
 
+    const foldersToSyncCopy = (structuredClone(syncEntity.foldersToSync) ?? []).map((folder) => {
+      folder.status = FileStatus.UPLOADED;
+      return folder;
+    });
+
     const [, updateSyncDto] = UpdateSyncDto.create({
       lastSyncGMT: new Date().toISOString(),
       id: syncEntity.id,
+      foldersToSync: foldersToSyncCopy,
     });
     new UpdateSync(this.repository).execute(updateSyncDto!);
   };

diff --git a/electron-app/src/logic/sync/presentation/routes.ts b/electron-app/src/logic/sync/presentation/routes.ts
@@ -6,13 +6,13 @@ import { UpdateSyncDto } from '../domain/dto/update-sync.dto';
 import { CreateSync } from '../domain/use-cases/create-sync.use-case';
 import { DeleteSync } from '../domain/use-cases/delete-sync.use-case';
 import { GetAllSync } from '../domain/use-cases/get-all-sync.use-case';
+import { GetSyncAuth } from '../domain/use-cases/get-sync-auth.use-case';
 import { GetSyncFolders } from '../domain/use-cases/get-sync-folders.use-case';
 import { GetSync } from '../domain/use-cases/get-sync.use-case';
 import { SyncAllFolders } from '../domain/use-cases/sync-all-folders-data.use-case';
 import { UpdateSync } from '../domain/use-cases/update-sync.use-case';
 import { FileSystemSyncDatasource } from '../infrastructure/file-system.sync.datasource';
 import { SyncRepository } from '../infrastructure/sync.repository';
-import { GetSyncAuth } from '../domain/use-cases/get-sync-auth.use-case';
 
 export class SyncFileSystemRoutes {
   private readonly basePath: string;