From 02a9bd60f64d2a9f2d225b84976f9182805df58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Tue, 19 Dec 2023 09:00:38 +0100 Subject: [PATCH 1/9] chore: add some dependences ( zod, nuclia/core, localstore-polyfill, mime-types) --- electron-app/package-lock.json | 37 +++++++++++++++++++++++++++++++++- electron-app/package.json | 7 ++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/electron-app/package-lock.json b/electron-app/package-lock.json index 05bc694..cb0e01d 100644 --- a/electron-app/package-lock.json +++ b/electron-app/package-lock.json @@ -9,12 +9,16 @@ "version": "0.0.1", "license": "MIT", "dependencies": { + "@nuclia/core": "1.9.0", "compression": "^1.7.4", "electron-squirrel-startup": "^1.0.0", "express": "^4.18.2", + "localstorage-polyfill": "^1.0.1", + "mime-types": "^2.1.35", "rxjs": "^7.8.1", "typescript": "^5.2.2", - "uuid": "^9.0.1" + "uuid": "^9.0.1", + "zod": "^3.22.4" }, "devDependencies": { "@electron-forge/cli": "^6.4.2", @@ -27,6 +31,7 @@ "@electron-forge/publisher-github": "^6.4.2", "@types/compression": "^1.7.5", "@types/express": "^4.17.21", + "@types/mime-types": "^2.1.4", "@types/uuid": "^9.0.7", "@typescript-eslint/eslint-plugin": "^6.12.0", "@typescript-eslint/parser": "^6.12.0", @@ -1169,6 +1174,14 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/@nuclia/core": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@nuclia/core/-/core-1.9.0.tgz", + "integrity": "sha512-UwyotxeyuzhMgNgcfwoPW78vWZHef3KCMNupOE5in3mkMlmZp5dj0pgnOWxvkvq6dLjdmcVpmAqpbzoi/+Yl5g==", + "peerDependencies": { + "rxjs": "^7.8.0" + } + }, "node_modules/@octokit/auth-token": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", @@ -1639,6 +1652,12 @@ "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", "dev": true }, + "node_modules/@types/mime-types": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz", + "integrity": "sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==", + "dev": true + }, "node_modules/@types/minimatch": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", @@ -5578,6 +5597,14 @@ "url": "https://github.com/sponsors/antfu" } }, + "node_modules/localstorage-polyfill": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/localstorage-polyfill/-/localstorage-polyfill-1.0.1.tgz", + "integrity": "sha512-m4iHVZxFH5734oQcPKU08025gIz2+4bjWR9lulP8ZYxEJR0BpA0w32oJmkzh8y3UI9ci7xCBehQDc3oA1X+VHw==", + "engines": { + "node": ">=6" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -8729,6 +8756,14 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.22.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", + "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/electron-app/package.json b/electron-app/package.json index b2a4824..e40995c 100644 --- a/electron-app/package.json +++ b/electron-app/package.json @@ -24,12 +24,16 @@ }, "license": "MIT", "dependencies": { + "@nuclia/core": "1.9.0", "compression": "^1.7.4", "electron-squirrel-startup": "^1.0.0", "express": "^4.18.2", + "localstorage-polyfill": "^1.0.1", + "mime-types": "^2.1.35", "rxjs": "^7.8.1", "typescript": "^5.2.2", - "uuid": "^9.0.1" + "uuid": "^9.0.1", + "zod": "^3.22.4" }, "devDependencies": { "@electron-forge/cli": "^6.4.2", @@ -42,6 +46,7 @@ "@electron-forge/publisher-github": "^6.4.2", "@types/compression": "^1.7.5", "@types/express": "^4.17.21", + "@types/mime-types": "^2.1.4", "@types/uuid": "^9.0.7", "@typescript-eslint/eslint-plugin": "^6.12.0", "@typescript-eslint/parser": "^6.12.0", From ecf78cfdace9d5a9c256940ed89652aec34cd942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Tue, 19 Dec 2023 09:00:45 +0100 Subject: [PATCH 2/9] chore: tsconfig --- electron-app/tsconfig.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/electron-app/tsconfig.json b/electron-app/tsconfig.json index c91ef1e..8dfe0bf 100644 --- a/electron-app/tsconfig.json +++ b/electron-app/tsconfig.json @@ -1,7 +1,6 @@ { "compilerOptions": { - "baseUrl": ".", - "target": "es5", + "target": "es2015", "module": "commonjs", "strict": true, "esModuleInterop": true, From b2b7449c27acfdfd1f2bce7db0f7f879573958a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Tue, 19 Dec 2023 09:01:09 +0100 Subject: [PATCH 3/9] feat: add synchronization events --- electron-app/src/events/events.ts | 3 +++ electron-app/src/subscribers.ts | 37 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/electron-app/src/events/events.ts b/electron-app/src/events/events.ts index f401e83..be0040d 100644 --- a/electron-app/src/events/events.ts +++ b/electron-app/src/events/events.ts @@ -4,6 +4,9 @@ export enum EVENTS { SYNC_CREATED = 'sync-created', SYNC_UPDATED = 'sync-updated', SYNC_DELETED = 'sync-deleted', + START_SYNCHRONIZATION_SYNC_OBJECT = 'start-synchronization-sync-object', + FINISH_SYNCHRONIZATION_SYNC_OBJECT = 'finish-synchronization-sync-object', + FINISH_SYNCHRONIZATION_SINGLE_FILE = 'finish-synchronization-single-file', } export type EVENTS_TYPE = (typeof EVENTS)[keyof typeof EVENTS]; diff --git a/electron-app/src/subscribers.ts b/electron-app/src/subscribers.ts index f79efb6..5ba359d 100644 --- a/electron-app/src/subscribers.ts +++ b/electron-app/src/subscribers.ts @@ -43,4 +43,41 @@ export function initFileSystemSubscribers(basePath: string) { }), ); }); + + eventEmitter.subscribe(EVENTS.START_SYNCHRONIZATION_SYNC_OBJECT, (payload: { [key: string]: string }) => { + const saveLog = new SaveLogs(new LogRepository(new FileSystemLogDatasource(basePath))); + saveLog.execute( + new LogEntity({ + message: 'Synchronization started', + level: LogSeverityLevel.low, + origin: 'electron-app', + action: EVENTS.START_SYNCHRONIZATION_SYNC_OBJECT, + payload, + }), + ); + }); + eventEmitter.subscribe(EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT, (payload: { [key: string]: string }) => { + const saveLog = new SaveLogs(new LogRepository(new FileSystemLogDatasource(basePath))); + saveLog.execute( + new LogEntity({ + message: 'Synchronization finished', + level: LogSeverityLevel.low, + origin: 'electron-app', + action: EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT, + payload, + }), + ); + }); + eventEmitter.subscribe(EVENTS.FINISH_SYNCHRONIZATION_SINGLE_FILE, (payload: { [key: string]: string }) => { + const saveLog = new SaveLogs(new LogRepository(new FileSystemLogDatasource(basePath))); + saveLog.execute( + new LogEntity({ + message: 'Synchronization single file finished', + level: LogSeverityLevel.low, + origin: 'electron-app', + action: EVENTS.FINISH_SYNCHRONIZATION_SINGLE_FILE, + payload, + }), + ); + }); } From e398f0898ea881551ac007230c6d58ee1fef98ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Tue, 19 Dec 2023 09:02:42 +0100 Subject: [PATCH 4/9] feat: add isAccesTokenValid in connector interface --- .../src/logic/connector/domain/connector.ts | 22 ++++++----- .../connectors/folder.connector.ts | 13 +++---- .../connectors/gdrive.connector.ts | 37 +++++++++++++++++-- .../connectors/tests/gdrive.connector.spec.js | 6 +-- 4 files changed, 55 insertions(+), 23 deletions(-) diff --git a/electron-app/src/logic/connector/domain/connector.ts b/electron-app/src/logic/connector/domain/connector.ts index 86884e8..1df5245 100644 --- a/electron-app/src/logic/connector/domain/connector.ts +++ b/electron-app/src/logic/connector/domain/connector.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { Observable } from 'rxjs'; +import { z } from 'zod'; export enum FileStatus { PENDING = 'PENDING', @@ -16,15 +17,17 @@ export interface ConnectorParameters { [key: string]: any; } -export interface SyncItem { - uuid?: string; - title: string; - originalId: string; - metadata: { [key: string]: string }; - status: FileStatus; - modifiedGMT?: string; - isFolder?: boolean; -} +export const SyncItemValidator = z.object({ + uuid: z.string().optional(), + title: z.string().min(1), + originalId: z.string().min(1), + metadata: z.record(z.string()), + status: z.nativeEnum(FileStatus), + modifiedGMT: z.string().optional(), + isFolder: z.boolean().optional(), + parents: z.array(z.string()).optional(), +}); +export type SyncItem = z.infer; export interface SearchResults { items: SyncItem[]; @@ -54,4 +57,5 @@ export interface IConnector { getLink(resource: SyncItem): Observable; hasAuthData(): boolean; refreshAuthentication(): Observable; + isAccesTokenValid(): Observable; } diff --git a/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts index b6d2c64..45c2a2f 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts +++ b/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts @@ -2,14 +2,8 @@ import { Blob as FSBlob } from 'buffer'; import * as fs from 'fs'; import path from 'path'; import { Observable, forkJoin, map, of, switchMap } from 'rxjs'; -import { - ConnectorParameters, - FileStatus, - IConnector, - Link, - SearchResults, - SyncItem, -} from 'src/logic/connector/domain/connector'; + +import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector'; import { SourceConnectorDefinition } from '../factory'; const FILES_TO_IGNORE = ['.DS_Store', 'Thumbs.db']; @@ -136,4 +130,7 @@ class FolderImpl implements IConnector { refreshAuthentication(): Observable { return of(true); } + isAccesTokenValid(): Observable { + return of(true); + } } diff --git a/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts index b26f130..c8a3068 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts +++ b/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts @@ -1,4 +1,4 @@ -import { Observable, concatMap, forkJoin, from, map, of } from 'rxjs'; +import { Observable, catchError, concatMap, forkJoin, from, map, of } from 'rxjs'; import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector'; import { SourceConnectorDefinition } from '../factory'; @@ -29,13 +29,16 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { if (!params?.token) { return false; } - if (!params?.refresh_token) { + if (!params?.refresh) { return false; } return true; } getLastModified(since: string, folders?: SyncItem[] | undefined): Observable { + if ((folders ?? []).length === 0) { + return of([]); + } try { return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe( map((results) => { @@ -58,6 +61,33 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { return this._getItems(query); } + isAccesTokenValid(): Observable { + return from( + fetch('https://www.googleapis.com/drive/v3/about?fields=user', { + headers: { + Authorization: `Bearer ${this.params.token || ''}`, + }, + }).then( + (res) => res.json(), + (err) => { + console.error(`Error fetching about: ${err}`); + throw new Error(err); + }, + ), + ).pipe( + concatMap((res) => { + if (res.error && res.error.status === 'UNAUTHENTICATED') { + return of(false); + } + return of(true); + }), + catchError(() => { + return of(true); + }), + ); + } + + // Script create the tree https://gist.github.com/tanaikech/97b336f04c739ae0181a606eab3dff42 private _getItems( query = '', folder = '', @@ -66,7 +96,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { previous?: SearchResults, ): Observable { let path = - 'https://www.googleapis.com/drive/v3/files?pageSize=50&fields=nextPageToken,files(id,name,mimeType,modifiedTime)'; + 'https://www.googleapis.com/drive/v3/files?pageSize=50&fields=nextPageToken,files(id,name,mimeType,modifiedTime,parents)'; const allDrives = '&corpora=allDrives&supportsAllDrives=true&includeItemsFromAllDrives=true'; path += allDrives; if (query) { @@ -121,6 +151,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { title: item.name, originalId: item.id, modifiedGMT: item.modifiedTime, + parents: item.parents, metadata: { needsPdfConversion: needsPdfConversion ? 'yes' : 'no', mimeType: needsPdfConversion ? 'application/pdf' : item.mimeType, diff --git a/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js b/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js index a8b7d90..f6c31df 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js +++ b/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js @@ -79,7 +79,7 @@ describe('Test validate gdrive params', () => { expect( sourceConnector.areParametersValid({ incorrect: 'test', - refresh_token: 'test', + refresh: 'test', }), ).toBe(false); }); @@ -88,7 +88,7 @@ describe('Test validate gdrive params', () => { expect( sourceConnector.areParametersValid({ token: '', - refresh_token: '', + refresh: '', }), ).toBe(false); }); @@ -97,7 +97,7 @@ describe('Test validate gdrive params', () => { expect( sourceConnector.areParametersValid({ token: 'test', - refresh_token: 'test', + refresh: 'test', }), ).toBe(true); }); From 53a952bff8aeb958c0410d081b8989b67e257431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Tue, 19 Dec 2023 09:02:59 +0100 Subject: [PATCH 5/9] feat: add zod validations, update entity and create sync use cases. --- .../logic/sync/domain/dto/create-sync.dto.ts | 22 +- .../domain/dto/tests/create-sync.dto.spec.ts | 12 +- .../domain/dto/tests/update-sync.dto.spec.ts | 4 +- .../logic/sync/domain/dto/update-sync.dto.ts | 30 ++- .../src/logic/sync/domain/dto/validate.ts | 10 + .../src/logic/sync/domain/nuclia-cloud.ts | 191 ++++++++++++++++++ .../src/logic/sync/domain/sync.entity.ts | 89 +++++--- .../use-cases/get-sync-folders.use-case.ts | 12 +- .../use-cases/refresh-acces-token.use-case.ts | 42 ++++ .../use-cases/sync-all-data.use-case.ts | 103 ++++++++++ .../sync-all-folders-data.use-case.ts | 106 ++++++++++ .../use-cases/sync-single-file.use-case.ts | 93 +++++++++ .../src/logic/sync/presentation/routes.ts | 10 + 13 files changed, 671 insertions(+), 53 deletions(-) create mode 100644 electron-app/src/logic/sync/domain/dto/validate.ts create mode 100644 electron-app/src/logic/sync/domain/nuclia-cloud.ts create mode 100644 electron-app/src/logic/sync/domain/use-cases/refresh-acces-token.use-case.ts create mode 100644 electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts create mode 100644 electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts create mode 100644 electron-app/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts diff --git a/electron-app/src/logic/sync/domain/dto/create-sync.dto.ts b/electron-app/src/logic/sync/domain/dto/create-sync.dto.ts index f359acf..bbc8469 100644 --- a/electron-app/src/logic/sync/domain/dto/create-sync.dto.ts +++ b/electron-app/src/logic/sync/domain/dto/create-sync.dto.ts @@ -1,7 +1,8 @@ import { v4 as uuidv4 } from 'uuid'; import { MakeOptional } from '../../../../types/server'; import { getConnector } from '../../../connector/infrastructure/factory'; -import { ISyncEntity } from '../sync.entity'; +import { ISyncEntity, NucliaOptionsValidator } from '../sync.entity'; +import { validateZodSchema } from './validate'; export class CreateSyncDto { private constructor(public readonly options: ISyncEntity) {} @@ -38,20 +39,17 @@ export class CreateSyncDto { } if (props.kb) { - if (!props.kb.knowledgeBox) { - return ['knowledgeBox is mandatory']; - } - if (!props.kb.zone) { - return ['zone is mandatory']; - } - if (!props.kb.backend) { - return ['backend is mandatory']; - } - if (!props.kb.apiKey) { - return ['apiKey is mandatory']; + try { + validateZodSchema(NucliaOptionsValidator, props.kb); + } catch (error) { + return [`Invalid format for kb: ${error}`]; } } + if (props.foldersToSync) { + return ['You can not create a sync with foldersToSync']; + } + return [undefined, new CreateSyncDto({ ...props, id })]; } } diff --git a/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts b/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts index 94d7e8d..994c3a1 100644 --- a/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts +++ b/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts @@ -74,7 +74,9 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('knowledgeBox is mandatory'); + expect(error).toEqual( + 'Invalid format for kb: Error: backend is required, zone is required, knowledgeBox is required, apiKey is required', + ); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -85,7 +87,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('zone is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: backend is required, zone is required, apiKey is required'); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -97,7 +99,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('zone is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: backend is required, zone is required, apiKey is required'); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -109,7 +111,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('backend is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: backend is required, apiKey is required'); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -122,7 +124,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('apiKey is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: apiKey is required'); expect(dto).toBeUndefined(); }); }); diff --git a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts index dde02eb..3111709 100644 --- a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts +++ b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts @@ -65,7 +65,7 @@ describe('Create Sync dto tests', () => { knowledgeBox: '', }, }); - expect(error).toEqual('knowledgeBox is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: knowledgeBox is required'); expect(dto).toBeUndefined(); [error, dto] = UpdateSyncDto.create({ @@ -74,7 +74,7 @@ describe('Create Sync dto tests', () => { apiKey: '', }, }); - expect(error).toEqual('apiKey is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: apiKey is required'); expect(dto).toBeUndefined(); }); }); diff --git a/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts b/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts index 43b97d8..95eabd4 100644 --- a/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts +++ b/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts @@ -1,5 +1,7 @@ +import { SyncItemValidator } from '../../../connector/domain/connector'; import { getConnector } from '../../../connector/infrastructure/factory'; import { ISyncEntity } from '../sync.entity'; +import { validateZodSchema } from './validate'; type Values = Partial & { id: string }; export class UpdateSyncDto { @@ -14,12 +16,14 @@ export class UpdateSyncDto { if (this.options.kb) returnObj.kb = this.options.kb; if (this.options.labels) returnObj.labels = this.options.labels; if (this.options.title) returnObj.title = this.options.title; + if (this.options.lastSyncGMT) returnObj.lastSyncGMT = this.options.lastSyncGMT; + if (this.options.foldersToSync) returnObj.foldersToSync = this.options.foldersToSync; return returnObj; } static create(props: Values): [string?, UpdateSyncDto?] { - const { id, connector, kb } = props; + const { id, connector, kb, foldersToSync } = props; if (!id) { return ['id is mandatory']; @@ -34,21 +38,37 @@ export class UpdateSyncDto { return [`Connector ${connector.name} parameters are not valid`]; } } + const isDefined = (value: unknown) => value !== null && value !== undefined; if (kb) { const { knowledgeBox, zone, backend, apiKey } = kb; if (isDefined(knowledgeBox) && !knowledgeBox) { - return ['knowledgeBox is mandatory']; + return ['Invalid format for kb: Error: knowledgeBox is required']; } if (isDefined(zone) && !zone) { - return ['zone is mandatory']; + return ['Invalid format for kb: Error: zone is required']; } if (isDefined(backend) && !backend) { - return ['backend is mandatory']; + return ['Invalid format for kb: Error: backend is required']; } if (isDefined(apiKey) && !apiKey) { - return ['apiKey is mandatory']; + return ['Invalid format for kb: Error: apiKey is required']; + } + } + + if (foldersToSync) { + let errorMsg = ''; + const valid = foldersToSync.some((folder) => { + try { + validateZodSchema(SyncItemValidator, folder); + return true; + } catch (error) { + errorMsg = `Invalid format for foldersToSync: ${error}`; + } + }); + if (!valid) { + return [errorMsg]; } } diff --git a/electron-app/src/logic/sync/domain/dto/validate.ts b/electron-app/src/logic/sync/domain/dto/validate.ts new file mode 100644 index 0000000..dd8ec12 --- /dev/null +++ b/electron-app/src/logic/sync/domain/dto/validate.ts @@ -0,0 +1,10 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { ZodIssue, ZodObject } from 'zod'; + +export const validateZodSchema = (schema: ZodObject, data: any) => { + const result = schema.safeParse(data); + if (!result.success) { + throw new Error(result.error.issues.map((issue: ZodIssue) => issue.message).join(', ')); + } + return result.data; +}; diff --git a/electron-app/src/logic/sync/domain/nuclia-cloud.ts b/electron-app/src/logic/sync/domain/nuclia-cloud.ts new file mode 100644 index 0000000..ac656d0 --- /dev/null +++ b/electron-app/src/logic/sync/domain/nuclia-cloud.ts @@ -0,0 +1,191 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { lookup } from 'mime-types'; +import { createHash } from 'node:crypto'; +import { Observable, catchError, delay, map, of, retry, switchMap, throwError, timer } from 'rxjs'; + +import { + FIELD_TYPE, + ICreateResource, + INuclia, + Nuclia, + NucliaOptions, + Resource, + TextField, + UploadResponse, + WritableKnowledgeBox, +} from '@nuclia/core'; +import { Link } from '../../connector/domain/connector'; + +function sha256(message: string): string { + return createHash('sha256').update(message).digest('hex'); +} + +const retryDelays = [1000, 5000, 20000]; +const RETRY_CONFIG = { + count: 3, + delay: (error: unknown, retryCount: number) => { + // failing operator will be retried once this delay function emits, + // retryDelays is an array containing the delay to wait before retrying + return timer(retryDelays[retryCount <= retryDelays.length ? retryCount - 1 : retryDelays.length - 1]); + }, +}; + +export class NucliaCloud { + nuclia: INuclia; + private kb?: WritableKnowledgeBox; + + constructor(options: NucliaOptions) { + this.nuclia = new Nuclia(options); + } + + upload( + originalId: string, + filename: string, + data: { buffer?: ArrayBuffer; text?: TextField; metadata?: any }, + ): Observable<{ success: boolean; message?: string }> { + const slug = sha256(originalId); + const text = data.text; + const buffer = data.buffer; + if (buffer || text) { + return this.getKb().pipe( + switchMap((kb) => + kb.getResourceBySlug(slug, [], []).pipe( + switchMap((resource) => { + console.log('get source from nuclia', resource); + if (data.metadata?.labels) { + return resource + .modify({ usermetadata: { classifications: data.metadata.labels } }) + .pipe(map(() => resource)); + } else { + return of(resource); + } + }), + catchError((error) => { + if (error.status === 404) { + const resourceData: ICreateResource = { slug, title: filename }; + if (data.metadata.labels) { + resourceData.usermetadata = { classifications: data.metadata?.labels }; + } + return kb.createResource(resourceData, true).pipe( + retry(RETRY_CONFIG), + map((data) => kb.getResourceFromData({ id: data.uuid })), + ); + } else { + console.error(`Problem creating ${slug}, status ${error.status}`); + return of(undefined); + } + }), + ), + ), + catchError(() => of(undefined)), + delay(500), + switchMap((resource) => { + if (!resource) { + return of({ success: false }); + } + if (buffer) { + try { + return resource + .upload('file', buffer, false, { + contentType: data.metadata.mimeType || lookup(filename) || 'application/octet-stream', + filename, + }) + .pipe( + catchError((error: any) => { + console.error(`Problem uploading ${filename} to ${slug}, error: ${JSON.stringify(error)}`); + return of({ success: false, message: error.body?.detail || JSON.stringify(error) }); + }), + switchMap((res) => { + if (res && (res as UploadResponse).completed) { + return of({ success: true }); + } else { + return this.deleteResource(slug, resource).pipe( + map(() => + (res as any).success === false + ? (res as { success: boolean; message: string }) + : { success: false, message: 'Upload failed' }, + ), + ); + } + }), + ); + } catch (error) { + console.error(`Error uploading ${filename} to ${slug}, status ${error}`); + return this.deleteResource(slug, resource).pipe(map(() => ({ success: false }))); + } + } else if (text) { + try { + return resource.setField(FIELD_TYPE.text, 'text', text).pipe( + catchError((error: any) => { + console.error(`Problem adding ${filename} to ${slug}, status ${error}`); + return of({ success: false }); + }), + switchMap((res) => { + if (res) { + return of({ success: true }); + } else { + return this.deleteResource(slug, resource).pipe(map(() => ({ success: false }))); + } + }), + ); + } catch (error) { + console.error(`Error adding ${filename} to ${slug}, status ${error}`); + return this.deleteResource(slug, resource).pipe(map(() => ({ success: false }))); + } + } else { + return of({ success: false }); + } + }), + ); + } else { + return of({ success: false }); + } + } + + private deleteResource(slug: string, resource: Resource): Observable { + try { + return resource.delete().pipe(map(() => false)); + } catch (error) { + console.error(`Problem deleting ${slug}, status ${error}`); + return of(false); + } + } + + uploadLink(originalId: string, filename: string, data: Link): Observable { + const slug = sha256(originalId); + return this.getKb().pipe( + switchMap((kb) => + kb + .createOrUpdateResource({ + title: filename, + slug, + links: { link: { uri: data.uri } }, + origin: { url: data.uri }, + icon: 'application/stf-link', + }) + .pipe( + retry(RETRY_CONFIG), + catchError((error) => { + console.log('createOrUpdateResource – error:', JSON.stringify(error)); + return throwError(() => new Error('Resource creation/modification failed')); + }), + ), + ), + map(() => undefined), + ); + } + + private getKb(): Observable { + if (this.kb) { + return of(this.kb); + } else { + return this.nuclia.db.getKnowledgeBox().pipe( + map((kb) => { + this.kb = kb; + return kb; + }), + ); + } + } +} diff --git a/electron-app/src/logic/sync/domain/sync.entity.ts b/electron-app/src/logic/sync/domain/sync.entity.ts index 0e8679e..52588e5 100644 --- a/electron-app/src/logic/sync/domain/sync.entity.ts +++ b/electron-app/src/logic/sync/domain/sync.entity.ts @@ -1,6 +1,7 @@ -import { Observable, catchError, concatMap, of } from 'rxjs'; +import { Observable, catchError, map, of } from 'rxjs'; -import { IConnector, SearchResults } from '../../connector/domain/connector'; +import { z } from 'zod'; +import { IConnector, SearchResults, SyncItem } from '../../connector/domain/connector'; import { getConnector } from '../../connector/infrastructure/factory'; export type Connector = { @@ -10,29 +11,33 @@ export type Connector = { parameters: { [key: string]: any }; }; -export interface NucliaOptions { +export const NucliaOptionsValidator = z.object({ /** * The Nuclia backend to use. * * Example: `https://nuclia.cloud/api` */ - backend: string; + backend: z.string({ required_error: 'backend is required' }).min(1, { message: 'backend is required' }), /** * The geographical zone for the regional API calls. * * Example: `europe-1` */ - zone: string; + zone: z.string({ required_error: 'zone is required' }).min(1, { message: 'zone is required' }), /** * The Nuclia Knowledge Box unique id. * * Example: `17815eb2-06a5-40ee-a5aa-b2f9dbc5da70` */ - knowledgeBox: string; + knowledgeBox: z + .string({ required_error: 'knowledgeBox is required' }) + .min(1, { message: 'knowledgeBox is required' }), /** * Allows you to make calls to a private Knowledge Box. * * It can be used in a server-side app, but never in a web app. */ - apiKey: string; -} + apiKey: z.string({ required_error: 'apiKey is required' }).min(1, { message: 'apiKey is required' }), +}); + +export type NucliaOptions = z.infer; export type Classification = {}; @@ -42,6 +47,8 @@ export interface ISyncEntity { labels?: Classification[]; title: string; id: string; + lastSyncGMT?: string; + foldersToSync?: SyncItem[]; } export class SyncEntity { @@ -50,15 +57,19 @@ export class SyncEntity { public labels?: Classification[]; public title: string; public id: string; - private sourceConnector?: IConnector; + public sourceConnector?: IConnector; + public lastSyncGMT?: string; + public foldersToSync?: SyncItem[] = []; constructor(options: ISyncEntity) { - const { connector, kb, labels, title, id } = options; + const { connector, kb, labels, title, id, lastSyncGMT, foldersToSync } = options; this.connector = connector; this.kb = kb; this.labels = labels; this.title = title; this.id = id; + this.lastSyncGMT = lastSyncGMT; + this.foldersToSync = foldersToSync; this.setConnectorDefinition(); } @@ -71,26 +82,52 @@ export class SyncEntity { this.sourceConnector.setParameters(this.connector?.parameters ?? {}); } - get folders(): Observable { + get allFolders(): Observable { + if (!this.sourceConnector) { + return of({ + items: [], + }); + } + return this.sourceConnector.getFolders(); + } + + get files(): Observable { if (!this.sourceConnector) { return of({ items: [], }); } - return this.sourceConnector.getFolders().pipe( - catchError(() => { - return this.sourceConnector!.refreshAuthentication().pipe( - concatMap((success) => { - if (success) { - const newParams = this.sourceConnector!.getParameters(); - this.connector.parameters = newParams; - return this.sourceConnector!.getFolders(); - } else { - throw new Error('Failed to refresh authentication'); - } - }), - ); - }), - ); + return this.sourceConnector.getFiles(); + } + + getLastModified(): Observable<{ success: boolean; results: SyncItem[]; error?: string }> { + try { + return this.sourceConnector!.getLastModified( + this.lastSyncGMT || '2000-01-01T00:00:00.000Z', + this.foldersToSync, + ).pipe( + map((results) => { + return { success: true, results }; + }), + catchError((err) => { + console.error(`Error on ${this.id}: ${err.message}`); + return of({ success: false, results: [], error: `${err}` }); + }), + ); + } catch (err) { + return of({ success: false, results: [], error: `${err}` }); + } + } + + isAccesTokenValid(): Observable { + return this.sourceConnector!.isAccesTokenValid(); + } + + refreshAuthentication(): Observable { + return this.sourceConnector!.refreshAuthentication(); + } + + getConnectorParameters() { + return this.sourceConnector!.getParameters(); } } diff --git a/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts index db0687a..d2032ca 100644 --- a/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts @@ -4,6 +4,7 @@ import { SearchResults } from '../../../connector/domain/connector'; import { CustomError } from '../../../errors'; import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; +import { RefreshAccessToken } from './refresh-acces-token.use-case'; export interface GetSyncFoldersUseCase { execute(id: string): Promise; @@ -17,8 +18,13 @@ export class GetSyncFolders implements GetSyncFoldersUseCase { if (data === null) { throw new CustomError(`Sync with id ${id} not found`, 404); } - const syncEntity = new SyncEntity(data); - const dataFolders = await firstValueFrom(syncEntity.folders.pipe()); - return dataFolders; + + try { + const syncEntity = new SyncEntity(data); + const updatedEntity = await firstValueFrom(new RefreshAccessToken(this.repository).execute(syncEntity)); + return await firstValueFrom(updatedEntity.allFolders.pipe()); + } catch (error) { + throw new CustomError(`Error getting folders for sync with id ${id}`, 500); + } } } diff --git a/electron-app/src/logic/sync/domain/use-cases/refresh-acces-token.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/refresh-acces-token.use-case.ts new file mode 100644 index 0000000..5adbf2a --- /dev/null +++ b/electron-app/src/logic/sync/domain/use-cases/refresh-acces-token.use-case.ts @@ -0,0 +1,42 @@ +import { Observable, concatMap, of } from 'rxjs'; +import { UpdateSyncDto } from '../dto/update-sync.dto'; +import { SyncEntity } from '../sync.entity'; +import { ISyncRepository } from '../sync.repository'; +import { UpdateSync } from './update-sync.use-case'; + +export interface RefreshAccessTokenUseCase { + execute(entity: SyncEntity): Observable; +} + +export class RefreshAccessToken implements RefreshAccessTokenUseCase { + constructor(private readonly repository: ISyncRepository) {} + + execute(entity: SyncEntity) { + const observable = entity.isAccesTokenValid().pipe( + concatMap((isValid) => { + if (!isValid) { + return entity.refreshAuthentication().pipe( + concatMap(async (success) => { + if (success) { + const newParams = entity.getConnectorParameters(); + const [, updateSyncDto] = UpdateSyncDto.create({ + connector: { + ...entity.connector, + parameters: newParams, + }, + id: entity.id, + }); + await new UpdateSync(this.repository).execute(updateSyncDto!); + } else { + throw new Error('Failed to refresh authentication'); + } + return entity; + }), + ); + } + return of(entity); + }), + ); + return observable; + } +} diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts new file mode 100644 index 0000000..c048041 --- /dev/null +++ b/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts @@ -0,0 +1,103 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Observable, catchError, delay, forkJoin, map, of, switchMap, tap } from 'rxjs'; + +import { EVENTS } from '../../../../events/events'; +import { eventEmitter } from '../../../../server'; +import { UpdateSyncDto } from '../dto/update-sync.dto'; +import { SyncEntity } from '../sync.entity'; +import { ISyncRepository } from '../sync.repository'; +import { RefreshAccessToken } from './refresh-acces-token.use-case'; +import { SyncSingleFile } from './sync-single-file.use-case'; +import { UpdateSync } from './update-sync.use-case'; + +require('localstorage-polyfill'); + +export interface SyncAllDataUseCase { + execute(since: Date | undefined): Promise; +} + +export class SyncAllData implements SyncAllDataUseCase { + constructor(private readonly repository: ISyncRepository) {} + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async execute(since: Date | undefined) { + const syncObjects = await this.repository.getAllSync(); + of(Object.values(syncObjects)) + .pipe( + switchMap((syncObjectValues) => { + if (syncObjectValues.length === 0) { + return of(undefined); + } else { + return forkJoin( + syncObjectValues.map((syncObj) => + of(syncObj).pipe( + switchMap((syncObj) => new RefreshAccessToken(this.repository).execute(new SyncEntity(syncObj))), + switchMap((syncEntity) => + syncEntity.files.pipe( + map((files) => { + return { files, syncEntity }; + }), + ), + ), + switchMap(({ files, syncEntity }) => { + const items = files.items.slice(0, 10); + // addActiveSyncLog(id, source); + eventEmitter.emit(EVENTS.START_SYNCHRONIZATION_SYNC_OBJECT, { + from: syncEntity.id, + to: syncEntity.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + total: items?.length || 0, + }); + + const batch: Observable<{ id: string; success: boolean }>[] = items.map((item) => + of(item).pipe( + switchMap((item) => + new SyncSingleFile(syncEntity, item).execute().pipe( + map((res) => ({ id: item.originalId, success: res.success })), + // do not overwhelm the source + delay(500), + ), + ), + ), + ); + + return forkJoin(batch).pipe( + tap((result) => { + if (result) { + const processed = result.map((res) => res.id); + const successCount = result.filter((res) => res.success).length; + + console.log('processed', processed); + console.log('successCount', successCount); + eventEmitter.emit(EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT, { + from: syncEntity.id, + to: syncEntity.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + processed, + successCount, + }); + + const [, updateSyncDto] = UpdateSyncDto.create({ + lastSyncGMT: new Date().toISOString(), + id: syncEntity.id, + }); + new UpdateSync(this.repository).execute(updateSyncDto!); + } + }), + ); + }), + catchError(() => { + // emit error event + return of(undefined); + }), + ), + ), + ); + } + }), + ) + .subscribe(() => console.log('Finish sync all data')); + + return true; + } +} diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts new file mode 100644 index 0000000..f3e3e89 --- /dev/null +++ b/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts @@ -0,0 +1,106 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Observable, delay, forkJoin, map, of, switchMap, tap } from 'rxjs'; + +import { EVENTS } from '../../../../events/events'; +import { eventEmitter } from '../../../../server'; +import { UpdateSyncDto } from '../dto/update-sync.dto'; +import { SyncEntity } from '../sync.entity'; +import { ISyncRepository } from '../sync.repository'; +import { RefreshAccessToken } from './refresh-acces-token.use-case'; +import { SyncSingleFile } from './sync-single-file.use-case'; +import { UpdateSync } from './update-sync.use-case'; + +require('localstorage-polyfill'); + +export interface SyncAllFoldersUseCase { + execute(since: Date | undefined): Promise; +} + +export class SyncAllFolders implements SyncAllFoldersUseCase { + constructor(private readonly repository: ISyncRepository) {} + + callbackFinishSync = (syncEntity: SyncEntity, processed: string[], successCount: number, error?: string) => { + eventEmitter.emit(EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT, { + from: syncEntity.id, + to: syncEntity.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + processed, + successCount, + error, + }); + + const [, updateSyncDto] = UpdateSyncDto.create({ + lastSyncGMT: new Date().toISOString(), + id: syncEntity.id, + }); + new UpdateSync(this.repository).execute(updateSyncDto!); + }; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async execute() { + const syncObjects = await this.repository.getAllSync(); + of(Object.values(syncObjects)) + .pipe( + switchMap((syncObjectValues) => { + if (syncObjectValues.length === 0) { + return of(undefined); + } else { + return forkJoin( + syncObjectValues.map((syncObj) => + of(syncObj).pipe( + switchMap((syncObj) => new RefreshAccessToken(this.repository).execute(new SyncEntity(syncObj))), + switchMap((syncEntity) => + syncEntity.getLastModified().pipe( + map((result) => { + return { result, syncEntity }; + }), + ), + ), + switchMap(({ result, syncEntity }) => { + eventEmitter.emit(EVENTS.START_SYNCHRONIZATION_SYNC_OBJECT, { + from: syncEntity.id, + to: syncEntity.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + total: result.results?.length || 0, + }); + + if (!result.success || result.results.length === 0) { + this.callbackFinishSync(syncEntity, [], 0, result.error); + return of(undefined); + } + + const batch: Observable<{ id: string; success: boolean }>[] = result.results.map((item) => + of(item).pipe( + switchMap((item) => + new SyncSingleFile(syncEntity, item).execute().pipe( + map((res) => ({ id: item.originalId, success: res.success })), + // do not overwhelm the source + delay(500), + ), + ), + ), + ); + + return forkJoin(batch).pipe( + tap((result) => { + if (result) { + const processed = result.map((res) => res.id); + const successCount = result.filter((res) => res.success).length; + + console.log('processed', processed); + console.log('successCount', successCount); + this.callbackFinishSync(syncEntity, processed, successCount, ''); + } + }), + ); + }), + ), + ), + ); + } + }), + ) + .subscribe(() => console.log('Finish sync folders data')); + + return true; + } +} diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts new file mode 100644 index 0000000..c927bb2 --- /dev/null +++ b/electron-app/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts @@ -0,0 +1,93 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { TextField } from '@nuclia/core'; +import { Observable, from, map, of, switchMap, tap } from 'rxjs'; + +import { EVENTS } from '../../../../events/events'; +import { eventEmitter } from '../../../../server'; +import { SyncItem } from '../../../connector/domain/connector'; +import { NucliaCloud } from '../nuclia-cloud'; +import { SyncEntity } from '../sync.entity'; + +require('localstorage-polyfill'); + +export interface SyncSingleFileUseCase { + execute(): Observable<{ success: boolean; message?: string }>; +} + +function downloadFileOrLink( + sync: SyncEntity, + item: SyncItem, +): Observable<{ type: 'blob' | 'link' | 'text'; blob?: Blob; link?: any; text?: TextField }> { + const connector = sync.sourceConnector; + return connector! + .download(item) + .pipe(map((res) => (res instanceof Blob ? { type: 'blob', blob: res } : { type: 'text', text: res }))); +} + +export class SyncSingleFile implements SyncSingleFileUseCase { + constructor( + private readonly sync: SyncEntity, + private readonly item: SyncItem, + ) {} + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + execute() { + const sync = this.sync; + const item = this.item; + + if (!sync.kb) { + return of({ success: false }); + } + const nucliaConnector = new NucliaCloud(sync.kb); + return downloadFileOrLink(sync, item).pipe( + switchMap((data) => { + try { + if (data.type === 'blob' && data.blob) { + return from(data.blob.arrayBuffer()).pipe( + switchMap((arrayBuffer) => { + return nucliaConnector.upload(item.originalId, item.title, { + buffer: arrayBuffer, + metadata: { ...item.metadata, labels: sync.labels }, + }); + }), + ); + } else if (data.type === 'text' && data.text) { + return nucliaConnector.upload(item.originalId, item.title, { + text: data.text, + metadata: { labels: sync.labels }, + }); + } else if (data.type === 'link' && data.link) { + return nucliaConnector + .uploadLink(item.originalId, item.title, data.link) + .pipe(map(() => ({ success: true, message: '' }))); + } else { + return of({ success: false, message: '' }); + } + } catch (err) { + return of({ success: false, message: `${err}` }); + } + }), + tap((res) => { + if (res.success) { + console.log(`Uploaded ${item.originalId} with success`); + eventEmitter.emit(EVENTS.FINISH_SYNCHRONIZATION_SINGLE_FILE, { + from: sync.id, + to: sync.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + status: 'success', + message: `Uploaded ${item.originalId} with success`, + }); + } else { + console.warn(`Failed to upload ${item.originalId}`); + eventEmitter.emit(EVENTS.FINISH_SYNCHRONIZATION_SINGLE_FILE, { + from: sync.id, + to: sync.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + status: 'failed', + message: `Failed to upload ${item.originalId} ${res.message || ''}`, + }); + } + }), + ); + } +} diff --git a/electron-app/src/logic/sync/presentation/routes.ts b/electron-app/src/logic/sync/presentation/routes.ts index 48735a2..6ad71f7 100644 --- a/electron-app/src/logic/sync/presentation/routes.ts +++ b/electron-app/src/logic/sync/presentation/routes.ts @@ -8,6 +8,7 @@ import { DeleteSync } from '../domain/use-cases/delete-sync.use-case'; import { GetAllSync } from '../domain/use-cases/get-all-sync.use-case'; import { GetSyncFolders } from '../domain/use-cases/get-sync-folders.use-case'; import { GetSync } from '../domain/use-cases/get-sync.use-case'; +import { SyncAllFolders } from '../domain/use-cases/sync-all-folders-data.use-case'; import { UpdateSync } from '../domain/use-cases/update-sync.use-case'; import { FileSystemSyncDatasource } from '../infrastructure/file-system.sync.datasource'; import { SyncRepository } from '../infrastructure/sync.repository'; @@ -50,6 +51,15 @@ export class SyncFileSystemRoutes { } }); + router.get('/execute', async (_req, res) => { + try { + await new SyncAllFolders(syncRepository).execute(); + res.status(200).send({ success: true }); + } catch (error) { + this.handleError(res, error); + } + }); + router.post('/', async (req, res) => { const [error, createSyncDto] = CreateSyncDto.create(req.body); if (error) return res.status(400).json({ error }); From 3088de663754ac312addcf52a0cfa543666ebbc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Tue, 19 Dec 2023 10:07:19 +0100 Subject: [PATCH 6/9] refactor: get auth --- electron-app/src/logic/sync/domain/sync.entity.ts | 4 ++++ .../sync/domain/use-cases/get-sync-auth.use-case.ts | 10 ++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/electron-app/src/logic/sync/domain/sync.entity.ts b/electron-app/src/logic/sync/domain/sync.entity.ts index 86ef062..8d11720 100644 --- a/electron-app/src/logic/sync/domain/sync.entity.ts +++ b/electron-app/src/logic/sync/domain/sync.entity.ts @@ -130,4 +130,8 @@ export class SyncEntity { getConnectorParameters() { return this.sourceConnector!.getParameters(); } + + hasAuthData() { + return this.sourceConnector!.hasAuthData(); + } } diff --git a/electron-app/src/logic/sync/domain/use-cases/get-sync-auth.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/get-sync-auth.use-case.ts index 0925f70..68d212d 100644 --- a/electron-app/src/logic/sync/domain/use-cases/get-sync-auth.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/get-sync-auth.use-case.ts @@ -1,6 +1,6 @@ import { CustomError } from '../../../errors'; +import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; -import { getConnector } from '../../../connector/infrastructure/factory'; export interface GetSyncAuthUseCase { execute(id: string): Promise; @@ -14,12 +14,6 @@ export class GetSyncAuth implements GetSyncAuthUseCase { if (data === null) { throw new CustomError(`Sync with id ${id} not found`, 404); } - const connectorDefinition = getConnector(data.connector?.name || ''); - if (!connectorDefinition) { - return false; - } - const sourceConnector = connectorDefinition.factory(); - sourceConnector.setParameters(data.connector?.parameters ?? {}); - return sourceConnector.hasAuthData(); + return new SyncEntity(data).hasAuthData(); } } From 07c81e45f6315808c1c16fd33f22398e29f685e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Wed, 20 Dec 2023 08:24:14 +0100 Subject: [PATCH 7/9] feat: error messages validate sync obj --- .../src/logic/connector/domain/connector.ts | 4 +- .../domain/dto/tests/create-sync.dto.spec.ts | 10 ++-- .../domain/dto/tests/update-sync.dto.spec.ts | 55 ++++++++++++++++++- .../logic/sync/domain/dto/update-sync.dto.ts | 2 +- .../src/logic/sync/domain/dto/validate.ts | 8 ++- .../src/logic/sync/domain/sync.entity.ts | 10 ++-- 6 files changed, 72 insertions(+), 17 deletions(-) diff --git a/electron-app/src/logic/connector/domain/connector.ts b/electron-app/src/logic/connector/domain/connector.ts index 1df5245..1865dd3 100644 --- a/electron-app/src/logic/connector/domain/connector.ts +++ b/electron-app/src/logic/connector/domain/connector.ts @@ -19,8 +19,8 @@ export interface ConnectorParameters { export const SyncItemValidator = z.object({ uuid: z.string().optional(), - title: z.string().min(1), - originalId: z.string().min(1), + title: z.string().min(1, { message: 'Required' }), + originalId: z.string().min(1, { message: 'Required' }), metadata: z.record(z.string()), status: z.nativeEnum(FileStatus), modifiedGMT: z.string().optional(), diff --git a/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts b/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts index a08657f..3206c02 100644 --- a/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts +++ b/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts @@ -75,7 +75,7 @@ describe('Create Sync dto tests', () => { }); expect(error).toEqual( - 'Invalid format for kb: Error: backend is required, zone is required, knowledgeBox is required, apiKey is required', + 'Invalid format for kb: Error: backend: Required, zone: Required, knowledgeBox: Required, apiKey: Required', ); expect(dto).toBeUndefined(); @@ -87,7 +87,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('Invalid format for kb: Error: backend is required, zone is required, apiKey is required'); + expect(error).toEqual('Invalid format for kb: Error: backend: Required, zone: Required, apiKey: Required'); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -99,7 +99,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('Invalid format for kb: Error: backend is required, zone is required, apiKey is required'); + expect(error).toEqual('Invalid format for kb: Error: backend: Required, zone: Required, apiKey: Required'); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -111,7 +111,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('Invalid format for kb: Error: backend is required, apiKey is required'); + expect(error).toEqual('Invalid format for kb: Error: backend: Required, apiKey: Required'); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -124,7 +124,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('Invalid format for kb: Error: apiKey is required'); + expect(error).toEqual('Invalid format for kb: Error: apiKey: Required'); expect(dto).toBeUndefined(); }); }); diff --git a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts index 3111709..aaef9fb 100644 --- a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts +++ b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts @@ -5,8 +5,8 @@ import { UpdateSyncDto } from '../update-sync.dto'; const props: any = { id: 'test', }; -describe('Create Sync dto tests', () => { - test('should create a valid dto', () => { +describe('Update Sync dto tests', () => { + test('should update a valid dto', () => { let [error, dto] = UpdateSyncDto.create(props); expect(error).toBeUndefined(); expect(dto).toBeDefined(); @@ -15,18 +15,30 @@ describe('Create Sync dto tests', () => { ...props, connector: undefined, kb: undefined, + foldersToSync: undefined, }); expect(error).toBeUndefined(); expect(dto).toBeDefined(); [error, dto] = UpdateSyncDto.create({ ...props, + foldersToSync: [], kb: { backend: 'backend', }, }); expect(error).toBeUndefined(); expect(dto).toBeDefined(); + + [error, dto] = UpdateSyncDto.create({ + ...props, + kb: { + backend: 'backend', + }, + foldersToSync: [{ title: 'folder1', metadata: {}, status: 'PENDING', originalId: 'id' }], + }); + expect(error).toBeUndefined(); + expect(dto).toBeDefined(); }); test('should not update a valid dto - id is mandatory', () => { @@ -77,4 +89,43 @@ describe('Create Sync dto tests', () => { expect(error).toEqual('Invalid format for kb: Error: apiKey is required'); expect(dto).toBeUndefined(); }); + + test('should not update a valid dto - folders to sync params error', () => { + let [error, dto] = UpdateSyncDto.create({ + ...props, + foldersToSync: [{}], + }); + expect(error).toEqual( + 'Invalid format for foldersToSync: Error: title: Required, originalId: Required, metadata: Required, status: Required', + ); + expect(dto).toBeUndefined(); + + [error, dto] = UpdateSyncDto.create({ + ...props, + foldersToSync: [{ title: 'folder1' }], + }); + expect(error).toEqual( + 'Invalid format for foldersToSync: Error: originalId: Required, metadata: Required, status: Required', + ); + expect(dto).toBeUndefined(); + + [error, dto] = UpdateSyncDto.create({ + ...props, + foldersToSync: [{ title: 'folder1', metadata: 'metadata' }], + }); + expect(error).toEqual( + 'Invalid format for foldersToSync: Error: originalId: Required, metadata: Expected object, received string, status: Required', + ); + expect(dto).toBeUndefined(); + + [error, dto] = UpdateSyncDto.create({ + ...props, + foldersToSync: [{ title: 'folder1', metadata: {}, status: '' }], + }); + expect(error).toEqual( + // eslint-disable-next-line quotes + "Invalid format for foldersToSync: Error: originalId: Required, status: Invalid enum value. Expected 'PENDING' | 'PROCESSING' | 'UPLOADED', received ''", + ); + expect(dto).toBeUndefined(); + }); }); diff --git a/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts b/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts index 95eabd4..aec2a8d 100644 --- a/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts +++ b/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts @@ -57,7 +57,7 @@ export class UpdateSyncDto { } } - if (foldersToSync) { + if (foldersToSync && foldersToSync.length > 0) { let errorMsg = ''; const valid = foldersToSync.some((folder) => { try { diff --git a/electron-app/src/logic/sync/domain/dto/validate.ts b/electron-app/src/logic/sync/domain/dto/validate.ts index dd8ec12..777eaa4 100644 --- a/electron-app/src/logic/sync/domain/dto/validate.ts +++ b/electron-app/src/logic/sync/domain/dto/validate.ts @@ -4,7 +4,13 @@ import { ZodIssue, ZodObject } from 'zod'; export const validateZodSchema = (schema: ZodObject, data: any) => { const result = schema.safeParse(data); if (!result.success) { - throw new Error(result.error.issues.map((issue: ZodIssue) => issue.message).join(', ')); + throw new Error( + result.error.issues + .map((issue: ZodIssue) => { + return `${issue.path.join('.')}: ${issue.message}`; + }) + .join(', '), + ); } return result.data; }; diff --git a/electron-app/src/logic/sync/domain/sync.entity.ts b/electron-app/src/logic/sync/domain/sync.entity.ts index 8d11720..a4e11cb 100644 --- a/electron-app/src/logic/sync/domain/sync.entity.ts +++ b/electron-app/src/logic/sync/domain/sync.entity.ts @@ -16,25 +16,23 @@ export const NucliaOptionsValidator = z.object({ * The Nuclia backend to use. * * Example: `https://nuclia.cloud/api` */ - backend: z.string({ required_error: 'backend is required' }).min(1, { message: 'backend is required' }), + backend: z.string().min(1, { message: 'Required' }), /** * The geographical zone for the regional API calls. * * Example: `europe-1` */ - zone: z.string({ required_error: 'zone is required' }).min(1, { message: 'zone is required' }), + zone: z.string().min(1, { message: 'Required' }), /** * The Nuclia Knowledge Box unique id. * * Example: `17815eb2-06a5-40ee-a5aa-b2f9dbc5da70` */ - knowledgeBox: z - .string({ required_error: 'knowledgeBox is required' }) - .min(1, { message: 'knowledgeBox is required' }), + knowledgeBox: z.string().min(1, { message: 'Required' }), /** * Allows you to make calls to a private Knowledge Box. * * It can be used in a server-side app, but never in a web app. */ - apiKey: z.string({ required_error: 'apiKey is required' }).min(1, { message: 'apiKey is required' }), + apiKey: z.string().min(1, { message: 'Required' }), }); export type NucliaOptions = z.infer; From 8c8e124b602c8eec6287305fe3bf630949c45024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Fri, 22 Dec 2023 08:57:57 +0100 Subject: [PATCH 8/9] test: get auth endpoint --- electron-app/tests/fixtures.js | 42 +++++++++++++++++++++++++++++++++ electron-app/tests/sync.spec.js | 16 +++++++++++-- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/electron-app/tests/fixtures.js b/electron-app/tests/fixtures.js index f7bc5da..a6199ed 100644 --- a/electron-app/tests/fixtures.js +++ b/electron-app/tests/fixtures.js @@ -41,6 +41,48 @@ export const serverTest = test.extend({ expect(response.status).toBe(201); await use(server); }, + serverWithOAuthSync: async ({ server }, use) => { + let response = await request(server.app) + .post('/sync') + .send({ + id: 'sync_oauth_gdrive', + connector: { + name: 'gdrive', + logo: '', + parameters: { + token: 'token_test', + refresh: 'refresh_token_test', + }, + }, + kb: { + knowledgeBox: 'test', + zone: 'local', + backend: 'http://localhost:8000', + apiKey: 'apiKey', + }, + }); + expect(response.status).toBe(201); + await use(server); + }, + serverWithSyncWithoutConnector: async ({ server }, use) => { + let response = await request(server.app) + .post('/sync') + .send({ + id: 'sync_without_connector', + title: 'Sync without connector', + connector: { + name: 'gdrive', + }, + kb: { + knowledgeBox: 'test', + zone: 'local', + backend: 'http://localhost:8000', + apiKey: 'apiKey', + }, + }); + expect(response.status).toBe(201); + await use(server); + }, }); export const serverTestWithoutFolder = test.extend({ diff --git a/electron-app/tests/sync.spec.js b/electron-app/tests/sync.spec.js index afe6aac..50c2282 100644 --- a/electron-app/tests/sync.spec.js +++ b/electron-app/tests/sync.spec.js @@ -68,10 +68,22 @@ describe('Test Sync object', () => { }); }); + serverTest('Get entity auth', async ({ serverWithOAuthSync }) => { + const responseAuth = await request(serverWithOAuthSync.app).get('/sync/sync_oauth_gdrive/auth'); + expect(responseAuth.status).toBe(200); + expect(responseAuth.body.hasAuth).toBe(true); + }); + + serverTest('Get entity auth', async ({ serverWithSyncWithoutConnector }) => { + const responseAuth = await request(serverWithSyncWithoutConnector.app).get('/sync/sync_without_connector/auth'); + expect(responseAuth.status).toBe(200); + expect(responseAuth.body.hasAuth).toBe(false); + }); + serverTest('Delete a sync', async ({ serverWithSync }) => { let response = await request(serverWithSync.app).get('/sync'); expect(response.status).toBe(200); - expect(Object.keys(response.body).length).toEqual(1); + expect(Object.keys(response.body).length).toEqual(3); const id = Object.keys(response.body)[0]; const responseDelete = await request(serverWithSync.app).delete(`/sync/${id}`); @@ -79,6 +91,6 @@ describe('Test Sync object', () => { response = await request(serverWithSync.app).get('/sync'); expect(response.status).toBe(200); - expect(Object.keys(response.body).length).toEqual(0); + expect(Object.keys(response.body).length).toEqual(2); }); }); From fd00145164812b0a4f62961b0567942e6e5be53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Boixader=20G=C3=BCell?= Date: Fri, 22 Dec 2023 08:59:15 +0100 Subject: [PATCH 9/9] feat: synchronise all files in the folder if it has not been synchronised before --- .../src/logic/connector/domain/connector.ts | 5 +- .../connectors/folder.connector.ts | 46 +++++++++++++++++-- .../connectors/gdrive.connector.ts | 40 ++++++++++++++-- .../connectors/tests/gdrive.connector.spec.js | 26 ++++++----- .../domain/dto/tests/update-sync.dto.spec.ts | 8 ++-- .../src/logic/sync/domain/nuclia-cloud.ts | 1 - .../src/logic/sync/domain/sync.entity.ts | 42 +++++++++-------- .../use-cases/get-sync-folders.use-case.ts | 2 +- ...se.ts => refresh-access-token.use-case.ts} | 0 .../use-cases/sync-all-data.use-case.ts | 2 +- .../sync-all-folders-data.use-case.ts | 9 +++- .../src/logic/sync/presentation/routes.ts | 2 +- 12 files changed, 134 insertions(+), 49 deletions(-) rename electron-app/src/logic/sync/domain/use-cases/{refresh-acces-token.use-case.ts => refresh-access-token.use-case.ts} (100%) diff --git a/electron-app/src/logic/connector/domain/connector.ts b/electron-app/src/logic/connector/domain/connector.ts index 1865dd3..1ade4bd 100644 --- a/electron-app/src/logic/connector/domain/connector.ts +++ b/electron-app/src/logic/connector/domain/connector.ts @@ -22,7 +22,7 @@ export const SyncItemValidator = z.object({ title: z.string().min(1, { message: 'Required' }), originalId: z.string().min(1, { message: 'Required' }), metadata: z.record(z.string()), - status: z.nativeEnum(FileStatus), + status: z.nativeEnum(FileStatus).optional(), modifiedGMT: z.string().optional(), isFolder: z.boolean().optional(), parents: z.array(z.string()).optional(), @@ -51,7 +51,8 @@ export interface IConnector { getParameters(): ConnectorParameters; getFolders(query?: string): Observable; getFiles(query?: string): Observable; - getLastModified(since: string, folders?: SyncItem[]): Observable; + getFilesFromFolders(folders: SyncItem[]): Observable; + getLastModified(since: string, folders?: SyncItem[]): Observable; // we cannot use the TextField from the SDK because we want to keep connectors independant download(resource: SyncItem): Observable; getLink(resource: SyncItem): Observable; diff --git a/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts index b9f6916..ad48d77 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts +++ b/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts @@ -43,7 +43,38 @@ class FolderImpl implements IConnector { return this._getFiles(this.params.path, query); } - getLastModified(since: string, folders?: SyncItem[]): Observable { + getFilesFromFolders(folders: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { + return forkJoin((folders || []).map((folder) => this._getFiles(folder.originalId))).pipe( + map((results) => { + const result: { items: SyncItem[] } = { + items: [], + }; + results.forEach((res) => { + result.items = [...result.items, ...res.items]; + }); + return result; + }), + ); + } catch (err) { + return of({ + items: [], + }); + } + } + + getLastModified(since: string, folders?: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { return forkJoin( (folders || []).map((folder) => @@ -51,9 +82,18 @@ class FolderImpl implements IConnector { switchMap((results) => this.getFilesModifiedSince(results.items, since)), ), ), - ).pipe(map((results) => results.reduce((acc, result) => acc.concat(result), [] as SyncItem[]))); + ).pipe( + map((results) => { + const items = results.reduce((acc, result) => acc.concat(result), [] as SyncItem[]); + return { + items, + }; + }), + ); } catch (err) { - return of([]); + return of({ + items: [], + }); } } diff --git a/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts index c8a3068..c0fe68f 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts +++ b/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts @@ -35,21 +35,53 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { return true; } - getLastModified(since: string, folders?: SyncItem[] | undefined): Observable { + getLastModified(since: string, folders?: SyncItem[] | undefined): Observable { if ((folders ?? []).length === 0) { - return of([]); + return of({ + items: [], + }); } try { return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe( map((results) => { - return results.reduce( + const items = results.reduce( (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)), [] as SyncItem[], ); + return { + items, + }; + }), + ); + } catch (err) { + return of({ + items: [], + }); + } + } + + getFilesFromFolders(folders: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { + return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe( + map((results) => { + const result: { items: SyncItem[] } = { + items: [], + }; + results.forEach((res) => { + result.items = [...result.items, ...res.items]; + }); + return result; }), ); } catch (err) { - return of([]); + return of({ + items: [], + }); } } diff --git a/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js b/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js index f6c31df..b9fa14f 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js +++ b/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js @@ -117,18 +117,20 @@ describe('Test last modified', () => { ]), ); - expect(lastModified).toEqual([ - { - uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', - title: 'PO6300590983', - originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', - modifiedGMT: '2023-11-29T12:49:27.539Z', - metadata: { - needsPdfConversion: 'yes', - mimeType: 'application/pdf', + expect(lastModified).toEqual({ + items: [ + { + uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + title: 'PO6300590983', + originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + modifiedGMT: '2023-11-29T12:49:27.539Z', + metadata: { + needsPdfConversion: 'yes', + mimeType: 'application/pdf', + }, + status: FileStatus.PENDING, }, - status: 'PENDING', - }, - ]); + ], + }); }); }); diff --git a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts index aaef9fb..1a9addb 100644 --- a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts +++ b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts @@ -96,7 +96,7 @@ describe('Update Sync dto tests', () => { foldersToSync: [{}], }); expect(error).toEqual( - 'Invalid format for foldersToSync: Error: title: Required, originalId: Required, metadata: Required, status: Required', + 'Invalid format for foldersToSync: Error: title: Required, originalId: Required, metadata: Required', ); expect(dto).toBeUndefined(); @@ -104,9 +104,7 @@ describe('Update Sync dto tests', () => { ...props, foldersToSync: [{ title: 'folder1' }], }); - expect(error).toEqual( - 'Invalid format for foldersToSync: Error: originalId: Required, metadata: Required, status: Required', - ); + expect(error).toEqual('Invalid format for foldersToSync: Error: originalId: Required, metadata: Required'); expect(dto).toBeUndefined(); [error, dto] = UpdateSyncDto.create({ @@ -114,7 +112,7 @@ describe('Update Sync dto tests', () => { foldersToSync: [{ title: 'folder1', metadata: 'metadata' }], }); expect(error).toEqual( - 'Invalid format for foldersToSync: Error: originalId: Required, metadata: Expected object, received string, status: Required', + 'Invalid format for foldersToSync: Error: originalId: Required, metadata: Expected object, received string', ); expect(dto).toBeUndefined(); diff --git a/electron-app/src/logic/sync/domain/nuclia-cloud.ts b/electron-app/src/logic/sync/domain/nuclia-cloud.ts index ac656d0..27ab490 100644 --- a/electron-app/src/logic/sync/domain/nuclia-cloud.ts +++ b/electron-app/src/logic/sync/domain/nuclia-cloud.ts @@ -52,7 +52,6 @@ export class NucliaCloud { switchMap((kb) => kb.getResourceBySlug(slug, [], []).pipe( switchMap((resource) => { - console.log('get source from nuclia', resource); if (data.metadata?.labels) { return resource .modify({ usermetadata: { classifications: data.metadata.labels } }) diff --git a/electron-app/src/logic/sync/domain/sync.entity.ts b/electron-app/src/logic/sync/domain/sync.entity.ts index a4e11cb..acaf306 100644 --- a/electron-app/src/logic/sync/domain/sync.entity.ts +++ b/electron-app/src/logic/sync/domain/sync.entity.ts @@ -1,7 +1,7 @@ -import { Observable, catchError, map, of } from 'rxjs'; +import { Observable, catchError, forkJoin, map, of } from 'rxjs'; import { z } from 'zod'; -import { IConnector, SearchResults, SyncItem } from '../../connector/domain/connector'; +import { FileStatus, IConnector, SearchResults, SyncItem } from '../../connector/domain/connector'; import { getConnector } from '../../connector/infrastructure/factory'; export type Connector = { @@ -99,22 +99,28 @@ export class SyncEntity { } getLastModified(): Observable<{ success: boolean; results: SyncItem[]; error?: string }> { - try { - return this.sourceConnector!.getLastModified( - this.lastSyncGMT || '2000-01-01T00:00:00.000Z', - this.foldersToSync, - ).pipe( - map((results) => { - return { success: true, results }; - }), - catchError((err) => { - console.error(`Error on ${this.id}: ${err.message}`); - return of({ success: false, results: [], error: `${err}` }); - }), - ); - } catch (err) { - return of({ success: false, results: [], error: `${err}` }); - } + const foldersToSyncPending: SyncItem[] = (this.foldersToSync ?? []).filter( + (folder) => folder.status === FileStatus.PENDING || folder.status === undefined, + ); + const foldersToSyncUpdated: SyncItem[] = (this.foldersToSync ?? []).filter( + (folder) => folder.status === FileStatus.UPLOADED, + ); + + const getFilesFoldersUpdated = this.sourceConnector!.getLastModified( + this.lastSyncGMT || '2000-01-01T00:00:00.000Z', + foldersToSyncUpdated, + ); + const getFilesFolderPending = this.sourceConnector!.getFilesFromFolders(foldersToSyncPending); + return forkJoin([getFilesFoldersUpdated, getFilesFolderPending]).pipe( + map((results) => { + const [updated, pending] = results; + return { success: true, results: [...updated.items, ...pending.items] }; + }), + catchError((err) => { + console.error(`Error on ${this.id}: ${err.message}`); + return of({ success: false, results: [], error: `${err}` }); + }), + ); } isAccesTokenValid(): Observable { diff --git a/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts index d2032ca..c6e4bda 100644 --- a/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts @@ -4,7 +4,7 @@ import { SearchResults } from '../../../connector/domain/connector'; import { CustomError } from '../../../errors'; import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; -import { RefreshAccessToken } from './refresh-acces-token.use-case'; +import { RefreshAccessToken } from './refresh-access-token.use-case'; export interface GetSyncFoldersUseCase { execute(id: string): Promise; diff --git a/electron-app/src/logic/sync/domain/use-cases/refresh-acces-token.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/refresh-access-token.use-case.ts similarity index 100% rename from electron-app/src/logic/sync/domain/use-cases/refresh-acces-token.use-case.ts rename to electron-app/src/logic/sync/domain/use-cases/refresh-access-token.use-case.ts diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts index c048041..1c8bb0f 100644 --- a/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts @@ -6,7 +6,7 @@ import { eventEmitter } from '../../../../server'; import { UpdateSyncDto } from '../dto/update-sync.dto'; import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; -import { RefreshAccessToken } from './refresh-acces-token.use-case'; +import { RefreshAccessToken } from './refresh-access-token.use-case'; import { SyncSingleFile } from './sync-single-file.use-case'; import { UpdateSync } from './update-sync.use-case'; diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts index f3e3e89..f1e48b9 100644 --- a/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts @@ -3,10 +3,11 @@ import { Observable, delay, forkJoin, map, of, switchMap, tap } from 'rxjs'; import { EVENTS } from '../../../../events/events'; import { eventEmitter } from '../../../../server'; +import { FileStatus } from '../../../connector/domain/connector'; import { UpdateSyncDto } from '../dto/update-sync.dto'; import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; -import { RefreshAccessToken } from './refresh-acces-token.use-case'; +import { RefreshAccessToken } from './refresh-access-token.use-case'; import { SyncSingleFile } from './sync-single-file.use-case'; import { UpdateSync } from './update-sync.use-case'; @@ -29,9 +30,15 @@ export class SyncAllFolders implements SyncAllFoldersUseCase { error, }); + const foldersToSyncCopy = (structuredClone(syncEntity.foldersToSync) ?? []).map((folder) => { + folder.status = FileStatus.UPLOADED; + return folder; + }); + const [, updateSyncDto] = UpdateSyncDto.create({ lastSyncGMT: new Date().toISOString(), id: syncEntity.id, + foldersToSync: foldersToSyncCopy, }); new UpdateSync(this.repository).execute(updateSyncDto!); }; diff --git a/electron-app/src/logic/sync/presentation/routes.ts b/electron-app/src/logic/sync/presentation/routes.ts index 154b58d..a1f95eb 100644 --- a/electron-app/src/logic/sync/presentation/routes.ts +++ b/electron-app/src/logic/sync/presentation/routes.ts @@ -6,13 +6,13 @@ import { UpdateSyncDto } from '../domain/dto/update-sync.dto'; import { CreateSync } from '../domain/use-cases/create-sync.use-case'; import { DeleteSync } from '../domain/use-cases/delete-sync.use-case'; import { GetAllSync } from '../domain/use-cases/get-all-sync.use-case'; +import { GetSyncAuth } from '../domain/use-cases/get-sync-auth.use-case'; import { GetSyncFolders } from '../domain/use-cases/get-sync-folders.use-case'; import { GetSync } from '../domain/use-cases/get-sync.use-case'; import { SyncAllFolders } from '../domain/use-cases/sync-all-folders-data.use-case'; import { UpdateSync } from '../domain/use-cases/update-sync.use-case'; import { FileSystemSyncDatasource } from '../infrastructure/file-system.sync.datasource'; import { SyncRepository } from '../infrastructure/sync.repository'; -import { GetSyncAuth } from '../domain/use-cases/get-sync-auth.use-case'; export class SyncFileSystemRoutes { private readonly basePath: string;