diff --git a/electron-app/package-lock.json b/electron-app/package-lock.json index 372a70e..2ba0638 100644 --- a/electron-app/package-lock.json +++ b/electron-app/package-lock.json @@ -9,13 +9,17 @@ "version": "0.0.1", "license": "MIT", "dependencies": { + "@nuclia/core": "1.9.0", "compression": "^1.7.4", "cors": "^2.8.5", "electron-squirrel-startup": "^1.0.0", "express": "^4.18.2", + "localstorage-polyfill": "^1.0.1", + "mime-types": "^2.1.35", "rxjs": "^7.8.1", "typescript": "^5.2.2", - "uuid": "^9.0.1" + "uuid": "^9.0.1", + "zod": "^3.22.4" }, "devDependencies": { "@electron-forge/cli": "^6.4.2", @@ -29,6 +33,7 @@ "@types/compression": "^1.7.5", "@types/cors": "^2.8.17", "@types/express": "^4.17.21", + "@types/mime-types": "^2.1.4", "@types/uuid": "^9.0.7", "@typescript-eslint/eslint-plugin": "^6.12.0", "@typescript-eslint/parser": "^6.12.0", @@ -1171,6 +1176,14 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/@nuclia/core": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@nuclia/core/-/core-1.9.0.tgz", + "integrity": "sha512-UwyotxeyuzhMgNgcfwoPW78vWZHef3KCMNupOE5in3mkMlmZp5dj0pgnOWxvkvq6dLjdmcVpmAqpbzoi/+Yl5g==", + "peerDependencies": { + "rxjs": "^7.8.0" + } + }, "node_modules/@octokit/auth-token": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", @@ -1650,6 +1663,12 @@ "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", "dev": true }, + "node_modules/@types/mime-types": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz", + "integrity": "sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==", + "dev": true + }, "node_modules/@types/minimatch": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", @@ -5601,6 +5620,14 @@ "url": "https://github.com/sponsors/antfu" } }, + "node_modules/localstorage-polyfill": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/localstorage-polyfill/-/localstorage-polyfill-1.0.1.tgz", + "integrity": "sha512-m4iHVZxFH5734oQcPKU08025gIz2+4bjWR9lulP8ZYxEJR0BpA0w32oJmkzh8y3UI9ci7xCBehQDc3oA1X+VHw==", + "engines": { + "node": ">=6" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -8760,6 +8787,14 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.22.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", + "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/electron-app/package.json b/electron-app/package.json index 4f94191..12eb2f5 100644 --- a/electron-app/package.json +++ b/electron-app/package.json @@ -24,13 +24,17 @@ }, "license": "MIT", "dependencies": { + "@nuclia/core": "1.9.0", "compression": "^1.7.4", "cors": "^2.8.5", "electron-squirrel-startup": "^1.0.0", "express": "^4.18.2", + "localstorage-polyfill": "^1.0.1", + "mime-types": "^2.1.35", "rxjs": "^7.8.1", "typescript": "^5.2.2", - "uuid": "^9.0.1" + "uuid": "^9.0.1", + "zod": "^3.22.4" }, "devDependencies": { "@electron-forge/cli": "^6.4.2", @@ -44,6 +48,7 @@ "@types/compression": "^1.7.5", "@types/cors": "^2.8.17", "@types/express": "^4.17.21", + "@types/mime-types": "^2.1.4", "@types/uuid": "^9.0.7", "@typescript-eslint/eslint-plugin": "^6.12.0", "@typescript-eslint/parser": "^6.12.0", diff --git a/electron-app/src/events/events.ts b/electron-app/src/events/events.ts index f401e83..be0040d 100644 --- a/electron-app/src/events/events.ts +++ b/electron-app/src/events/events.ts @@ -4,6 +4,9 @@ export enum EVENTS { SYNC_CREATED = 'sync-created', SYNC_UPDATED = 'sync-updated', SYNC_DELETED = 'sync-deleted', + START_SYNCHRONIZATION_SYNC_OBJECT = 'start-synchronization-sync-object', + FINISH_SYNCHRONIZATION_SYNC_OBJECT = 'finish-synchronization-sync-object', + FINISH_SYNCHRONIZATION_SINGLE_FILE = 'finish-synchronization-single-file', } export type EVENTS_TYPE = (typeof EVENTS)[keyof typeof EVENTS]; diff --git a/electron-app/src/logic/connector/domain/connector.ts b/electron-app/src/logic/connector/domain/connector.ts index 86884e8..1ade4bd 100644 --- a/electron-app/src/logic/connector/domain/connector.ts +++ b/electron-app/src/logic/connector/domain/connector.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { Observable } from 'rxjs'; +import { z } from 'zod'; export enum FileStatus { PENDING = 'PENDING', @@ -16,15 +17,17 @@ export interface ConnectorParameters { [key: string]: any; } -export interface SyncItem { - uuid?: string; - title: string; - originalId: string; - metadata: { [key: string]: string }; - status: FileStatus; - modifiedGMT?: string; - isFolder?: boolean; -} +export const SyncItemValidator = z.object({ + uuid: z.string().optional(), + title: z.string().min(1, { message: 'Required' }), + originalId: z.string().min(1, { message: 'Required' }), + metadata: z.record(z.string()), + status: z.nativeEnum(FileStatus).optional(), + modifiedGMT: z.string().optional(), + isFolder: z.boolean().optional(), + parents: z.array(z.string()).optional(), +}); +export type SyncItem = z.infer; export interface SearchResults { items: SyncItem[]; @@ -48,10 +51,12 @@ export interface IConnector { getParameters(): ConnectorParameters; getFolders(query?: string): Observable; getFiles(query?: string): Observable; - getLastModified(since: string, folders?: SyncItem[]): Observable; + getFilesFromFolders(folders: SyncItem[]): Observable; + getLastModified(since: string, folders?: SyncItem[]): Observable; // we cannot use the TextField from the SDK because we want to keep connectors independant download(resource: SyncItem): Observable; getLink(resource: SyncItem): Observable; hasAuthData(): boolean; refreshAuthentication(): Observable; + isAccesTokenValid(): Observable; } diff --git a/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts index 8ae1d2d..ad48d77 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts +++ b/electron-app/src/logic/connector/infrastructure/connectors/folder.connector.ts @@ -43,7 +43,38 @@ class FolderImpl implements IConnector { return this._getFiles(this.params.path, query); } - getLastModified(since: string, folders?: SyncItem[]): Observable { + getFilesFromFolders(folders: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { + return forkJoin((folders || []).map((folder) => this._getFiles(folder.originalId))).pipe( + map((results) => { + const result: { items: SyncItem[] } = { + items: [], + }; + results.forEach((res) => { + result.items = [...result.items, ...res.items]; + }); + return result; + }), + ); + } catch (err) { + return of({ + items: [], + }); + } + } + + getLastModified(since: string, folders?: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { return forkJoin( (folders || []).map((folder) => @@ -51,9 +82,18 @@ class FolderImpl implements IConnector { switchMap((results) => this.getFilesModifiedSince(results.items, since)), ), ), - ).pipe(map((results) => results.reduce((acc, result) => acc.concat(result), [] as SyncItem[]))); + ).pipe( + map((results) => { + const items = results.reduce((acc, result) => acc.concat(result), [] as SyncItem[]); + return { + items, + }; + }), + ); } catch (err) { - return of([]); + return of({ + items: [], + }); } } @@ -129,4 +169,7 @@ class FolderImpl implements IConnector { refreshAuthentication(): Observable { return of(true); } + isAccesTokenValid(): Observable { + return of(true); + } } diff --git a/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts index d2dc7f5..c0fe68f 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts +++ b/electron-app/src/logic/connector/infrastructure/connectors/gdrive.connector.ts @@ -1,4 +1,4 @@ -import { Observable, concatMap, forkJoin, from, map, of } from 'rxjs'; +import { Observable, catchError, concatMap, forkJoin, from, map, of } from 'rxjs'; import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector'; import { SourceConnectorDefinition } from '../factory'; @@ -35,18 +35,53 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { return true; } - getLastModified(since: string, folders?: SyncItem[] | undefined): Observable { + getLastModified(since: string, folders?: SyncItem[] | undefined): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } try { return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe( map((results) => { - return results.reduce( + const items = results.reduce( (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)), [] as SyncItem[], ); + return { + items, + }; }), ); } catch (err) { - return of([]); + return of({ + items: [], + }); + } + } + + getFilesFromFolders(folders: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { + return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe( + map((results) => { + const result: { items: SyncItem[] } = { + items: [], + }; + results.forEach((res) => { + result.items = [...result.items, ...res.items]; + }); + return result; + }), + ); + } catch (err) { + return of({ + items: [], + }); } } @@ -58,6 +93,33 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { return this._getItems(query); } + isAccesTokenValid(): Observable { + return from( + fetch('https://www.googleapis.com/drive/v3/about?fields=user', { + headers: { + Authorization: `Bearer ${this.params.token || ''}`, + }, + }).then( + (res) => res.json(), + (err) => { + console.error(`Error fetching about: ${err}`); + throw new Error(err); + }, + ), + ).pipe( + concatMap((res) => { + if (res.error && res.error.status === 'UNAUTHENTICATED') { + return of(false); + } + return of(true); + }), + catchError(() => { + return of(true); + }), + ); + } + + // Script create the tree https://gist.github.com/tanaikech/97b336f04c739ae0181a606eab3dff42 private _getItems( query = '', folder = '', @@ -66,7 +128,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { previous?: SearchResults, ): Observable { let path = - 'https://www.googleapis.com/drive/v3/files?pageSize=50&fields=nextPageToken,files(id,name,mimeType,modifiedTime)'; + 'https://www.googleapis.com/drive/v3/files?pageSize=50&fields=nextPageToken,files(id,name,mimeType,modifiedTime,parents)'; const allDrives = '&corpora=allDrives&supportsAllDrives=true&includeItemsFromAllDrives=true'; path += allDrives; if (query) { @@ -121,6 +183,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector { title: item.name, originalId: item.id, modifiedGMT: item.modifiedTime, + parents: item.parents, metadata: { needsPdfConversion: needsPdfConversion ? 'yes' : 'no', mimeType: needsPdfConversion ? 'application/pdf' : item.mimeType, diff --git a/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js b/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js index f6c31df..b9fa14f 100644 --- a/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js +++ b/electron-app/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js @@ -117,18 +117,20 @@ describe('Test last modified', () => { ]), ); - expect(lastModified).toEqual([ - { - uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', - title: 'PO6300590983', - originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', - modifiedGMT: '2023-11-29T12:49:27.539Z', - metadata: { - needsPdfConversion: 'yes', - mimeType: 'application/pdf', + expect(lastModified).toEqual({ + items: [ + { + uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + title: 'PO6300590983', + originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + modifiedGMT: '2023-11-29T12:49:27.539Z', + metadata: { + needsPdfConversion: 'yes', + mimeType: 'application/pdf', + }, + status: FileStatus.PENDING, }, - status: 'PENDING', - }, - ]); + ], + }); }); }); diff --git a/electron-app/src/logic/sync/domain/dto/create-sync.dto.ts b/electron-app/src/logic/sync/domain/dto/create-sync.dto.ts index 8b6e425..21616ea 100644 --- a/electron-app/src/logic/sync/domain/dto/create-sync.dto.ts +++ b/electron-app/src/logic/sync/domain/dto/create-sync.dto.ts @@ -1,7 +1,8 @@ import { v4 as uuidv4 } from 'uuid'; import { MakeOptional } from '../../../../types/server'; import { getConnector } from '../../../connector/infrastructure/factory'; -import { ISyncEntity } from '../sync.entity'; +import { ISyncEntity, NucliaOptionsValidator } from '../sync.entity'; +import { validateZodSchema } from './validate'; export class CreateSyncDto { private constructor(public readonly options: ISyncEntity) {} @@ -34,20 +35,17 @@ export class CreateSyncDto { } if (props.kb) { - if (!props.kb.knowledgeBox) { - return ['knowledgeBox is mandatory']; - } - if (!props.kb.zone) { - return ['zone is mandatory']; - } - if (!props.kb.backend) { - return ['backend is mandatory']; - } - if (!props.kb.apiKey) { - return ['apiKey is mandatory']; + try { + validateZodSchema(NucliaOptionsValidator, props.kb); + } catch (error) { + return [`Invalid format for kb: ${error}`]; } } + if (props.foldersToSync) { + return ['You can not create a sync with foldersToSync']; + } + return [undefined, new CreateSyncDto({ ...props, id })]; } } diff --git a/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts b/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts index 35fb7f8..3206c02 100644 --- a/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts +++ b/electron-app/src/logic/sync/domain/dto/tests/create-sync.dto.spec.ts @@ -74,7 +74,9 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('knowledgeBox is mandatory'); + expect(error).toEqual( + 'Invalid format for kb: Error: backend: Required, zone: Required, knowledgeBox: Required, apiKey: Required', + ); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -85,7 +87,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('zone is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: backend: Required, zone: Required, apiKey: Required'); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -97,7 +99,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('zone is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: backend: Required, zone: Required, apiKey: Required'); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -109,7 +111,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('backend is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: backend: Required, apiKey: Required'); expect(dto).toBeUndefined(); [error, dto] = CreateSyncDto.create({ @@ -122,7 +124,7 @@ describe('Create Sync dto tests', () => { }, }); - expect(error).toEqual('apiKey is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: apiKey: Required'); expect(dto).toBeUndefined(); }); }); diff --git a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts index dde02eb..1a9addb 100644 --- a/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts +++ b/electron-app/src/logic/sync/domain/dto/tests/update-sync.dto.spec.ts @@ -5,8 +5,8 @@ import { UpdateSyncDto } from '../update-sync.dto'; const props: any = { id: 'test', }; -describe('Create Sync dto tests', () => { - test('should create a valid dto', () => { +describe('Update Sync dto tests', () => { + test('should update a valid dto', () => { let [error, dto] = UpdateSyncDto.create(props); expect(error).toBeUndefined(); expect(dto).toBeDefined(); @@ -15,18 +15,30 @@ describe('Create Sync dto tests', () => { ...props, connector: undefined, kb: undefined, + foldersToSync: undefined, }); expect(error).toBeUndefined(); expect(dto).toBeDefined(); [error, dto] = UpdateSyncDto.create({ ...props, + foldersToSync: [], kb: { backend: 'backend', }, }); expect(error).toBeUndefined(); expect(dto).toBeDefined(); + + [error, dto] = UpdateSyncDto.create({ + ...props, + kb: { + backend: 'backend', + }, + foldersToSync: [{ title: 'folder1', metadata: {}, status: 'PENDING', originalId: 'id' }], + }); + expect(error).toBeUndefined(); + expect(dto).toBeDefined(); }); test('should not update a valid dto - id is mandatory', () => { @@ -65,7 +77,7 @@ describe('Create Sync dto tests', () => { knowledgeBox: '', }, }); - expect(error).toEqual('knowledgeBox is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: knowledgeBox is required'); expect(dto).toBeUndefined(); [error, dto] = UpdateSyncDto.create({ @@ -74,7 +86,44 @@ describe('Create Sync dto tests', () => { apiKey: '', }, }); - expect(error).toEqual('apiKey is mandatory'); + expect(error).toEqual('Invalid format for kb: Error: apiKey is required'); + expect(dto).toBeUndefined(); + }); + + test('should not update a valid dto - folders to sync params error', () => { + let [error, dto] = UpdateSyncDto.create({ + ...props, + foldersToSync: [{}], + }); + expect(error).toEqual( + 'Invalid format for foldersToSync: Error: title: Required, originalId: Required, metadata: Required', + ); + expect(dto).toBeUndefined(); + + [error, dto] = UpdateSyncDto.create({ + ...props, + foldersToSync: [{ title: 'folder1' }], + }); + expect(error).toEqual('Invalid format for foldersToSync: Error: originalId: Required, metadata: Required'); + expect(dto).toBeUndefined(); + + [error, dto] = UpdateSyncDto.create({ + ...props, + foldersToSync: [{ title: 'folder1', metadata: 'metadata' }], + }); + expect(error).toEqual( + 'Invalid format for foldersToSync: Error: originalId: Required, metadata: Expected object, received string', + ); + expect(dto).toBeUndefined(); + + [error, dto] = UpdateSyncDto.create({ + ...props, + foldersToSync: [{ title: 'folder1', metadata: {}, status: '' }], + }); + expect(error).toEqual( + // eslint-disable-next-line quotes + "Invalid format for foldersToSync: Error: originalId: Required, status: Invalid enum value. Expected 'PENDING' | 'PROCESSING' | 'UPLOADED', received ''", + ); expect(dto).toBeUndefined(); }); }); diff --git a/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts b/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts index 43b97d8..aec2a8d 100644 --- a/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts +++ b/electron-app/src/logic/sync/domain/dto/update-sync.dto.ts @@ -1,5 +1,7 @@ +import { SyncItemValidator } from '../../../connector/domain/connector'; import { getConnector } from '../../../connector/infrastructure/factory'; import { ISyncEntity } from '../sync.entity'; +import { validateZodSchema } from './validate'; type Values = Partial & { id: string }; export class UpdateSyncDto { @@ -14,12 +16,14 @@ export class UpdateSyncDto { if (this.options.kb) returnObj.kb = this.options.kb; if (this.options.labels) returnObj.labels = this.options.labels; if (this.options.title) returnObj.title = this.options.title; + if (this.options.lastSyncGMT) returnObj.lastSyncGMT = this.options.lastSyncGMT; + if (this.options.foldersToSync) returnObj.foldersToSync = this.options.foldersToSync; return returnObj; } static create(props: Values): [string?, UpdateSyncDto?] { - const { id, connector, kb } = props; + const { id, connector, kb, foldersToSync } = props; if (!id) { return ['id is mandatory']; @@ -34,21 +38,37 @@ export class UpdateSyncDto { return [`Connector ${connector.name} parameters are not valid`]; } } + const isDefined = (value: unknown) => value !== null && value !== undefined; if (kb) { const { knowledgeBox, zone, backend, apiKey } = kb; if (isDefined(knowledgeBox) && !knowledgeBox) { - return ['knowledgeBox is mandatory']; + return ['Invalid format for kb: Error: knowledgeBox is required']; } if (isDefined(zone) && !zone) { - return ['zone is mandatory']; + return ['Invalid format for kb: Error: zone is required']; } if (isDefined(backend) && !backend) { - return ['backend is mandatory']; + return ['Invalid format for kb: Error: backend is required']; } if (isDefined(apiKey) && !apiKey) { - return ['apiKey is mandatory']; + return ['Invalid format for kb: Error: apiKey is required']; + } + } + + if (foldersToSync && foldersToSync.length > 0) { + let errorMsg = ''; + const valid = foldersToSync.some((folder) => { + try { + validateZodSchema(SyncItemValidator, folder); + return true; + } catch (error) { + errorMsg = `Invalid format for foldersToSync: ${error}`; + } + }); + if (!valid) { + return [errorMsg]; } } diff --git a/electron-app/src/logic/sync/domain/dto/validate.ts b/electron-app/src/logic/sync/domain/dto/validate.ts new file mode 100644 index 0000000..777eaa4 --- /dev/null +++ b/electron-app/src/logic/sync/domain/dto/validate.ts @@ -0,0 +1,16 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { ZodIssue, ZodObject } from 'zod'; + +export const validateZodSchema = (schema: ZodObject, data: any) => { + const result = schema.safeParse(data); + if (!result.success) { + throw new Error( + result.error.issues + .map((issue: ZodIssue) => { + return `${issue.path.join('.')}: ${issue.message}`; + }) + .join(', '), + ); + } + return result.data; +}; diff --git a/electron-app/src/logic/sync/domain/nuclia-cloud.ts b/electron-app/src/logic/sync/domain/nuclia-cloud.ts new file mode 100644 index 0000000..27ab490 --- /dev/null +++ b/electron-app/src/logic/sync/domain/nuclia-cloud.ts @@ -0,0 +1,190 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { lookup } from 'mime-types'; +import { createHash } from 'node:crypto'; +import { Observable, catchError, delay, map, of, retry, switchMap, throwError, timer } from 'rxjs'; + +import { + FIELD_TYPE, + ICreateResource, + INuclia, + Nuclia, + NucliaOptions, + Resource, + TextField, + UploadResponse, + WritableKnowledgeBox, +} from '@nuclia/core'; +import { Link } from '../../connector/domain/connector'; + +function sha256(message: string): string { + return createHash('sha256').update(message).digest('hex'); +} + +const retryDelays = [1000, 5000, 20000]; +const RETRY_CONFIG = { + count: 3, + delay: (error: unknown, retryCount: number) => { + // failing operator will be retried once this delay function emits, + // retryDelays is an array containing the delay to wait before retrying + return timer(retryDelays[retryCount <= retryDelays.length ? retryCount - 1 : retryDelays.length - 1]); + }, +}; + +export class NucliaCloud { + nuclia: INuclia; + private kb?: WritableKnowledgeBox; + + constructor(options: NucliaOptions) { + this.nuclia = new Nuclia(options); + } + + upload( + originalId: string, + filename: string, + data: { buffer?: ArrayBuffer; text?: TextField; metadata?: any }, + ): Observable<{ success: boolean; message?: string }> { + const slug = sha256(originalId); + const text = data.text; + const buffer = data.buffer; + if (buffer || text) { + return this.getKb().pipe( + switchMap((kb) => + kb.getResourceBySlug(slug, [], []).pipe( + switchMap((resource) => { + if (data.metadata?.labels) { + return resource + .modify({ usermetadata: { classifications: data.metadata.labels } }) + .pipe(map(() => resource)); + } else { + return of(resource); + } + }), + catchError((error) => { + if (error.status === 404) { + const resourceData: ICreateResource = { slug, title: filename }; + if (data.metadata.labels) { + resourceData.usermetadata = { classifications: data.metadata?.labels }; + } + return kb.createResource(resourceData, true).pipe( + retry(RETRY_CONFIG), + map((data) => kb.getResourceFromData({ id: data.uuid })), + ); + } else { + console.error(`Problem creating ${slug}, status ${error.status}`); + return of(undefined); + } + }), + ), + ), + catchError(() => of(undefined)), + delay(500), + switchMap((resource) => { + if (!resource) { + return of({ success: false }); + } + if (buffer) { + try { + return resource + .upload('file', buffer, false, { + contentType: data.metadata.mimeType || lookup(filename) || 'application/octet-stream', + filename, + }) + .pipe( + catchError((error: any) => { + console.error(`Problem uploading ${filename} to ${slug}, error: ${JSON.stringify(error)}`); + return of({ success: false, message: error.body?.detail || JSON.stringify(error) }); + }), + switchMap((res) => { + if (res && (res as UploadResponse).completed) { + return of({ success: true }); + } else { + return this.deleteResource(slug, resource).pipe( + map(() => + (res as any).success === false + ? (res as { success: boolean; message: string }) + : { success: false, message: 'Upload failed' }, + ), + ); + } + }), + ); + } catch (error) { + console.error(`Error uploading ${filename} to ${slug}, status ${error}`); + return this.deleteResource(slug, resource).pipe(map(() => ({ success: false }))); + } + } else if (text) { + try { + return resource.setField(FIELD_TYPE.text, 'text', text).pipe( + catchError((error: any) => { + console.error(`Problem adding ${filename} to ${slug}, status ${error}`); + return of({ success: false }); + }), + switchMap((res) => { + if (res) { + return of({ success: true }); + } else { + return this.deleteResource(slug, resource).pipe(map(() => ({ success: false }))); + } + }), + ); + } catch (error) { + console.error(`Error adding ${filename} to ${slug}, status ${error}`); + return this.deleteResource(slug, resource).pipe(map(() => ({ success: false }))); + } + } else { + return of({ success: false }); + } + }), + ); + } else { + return of({ success: false }); + } + } + + private deleteResource(slug: string, resource: Resource): Observable { + try { + return resource.delete().pipe(map(() => false)); + } catch (error) { + console.error(`Problem deleting ${slug}, status ${error}`); + return of(false); + } + } + + uploadLink(originalId: string, filename: string, data: Link): Observable { + const slug = sha256(originalId); + return this.getKb().pipe( + switchMap((kb) => + kb + .createOrUpdateResource({ + title: filename, + slug, + links: { link: { uri: data.uri } }, + origin: { url: data.uri }, + icon: 'application/stf-link', + }) + .pipe( + retry(RETRY_CONFIG), + catchError((error) => { + console.log('createOrUpdateResource – error:', JSON.stringify(error)); + return throwError(() => new Error('Resource creation/modification failed')); + }), + ), + ), + map(() => undefined), + ); + } + + private getKb(): Observable { + if (this.kb) { + return of(this.kb); + } else { + return this.nuclia.db.getKnowledgeBox().pipe( + map((kb) => { + this.kb = kb; + return kb; + }), + ); + } + } +} diff --git a/electron-app/src/logic/sync/domain/sync.entity.ts b/electron-app/src/logic/sync/domain/sync.entity.ts index 31c8cc3..acaf306 100644 --- a/electron-app/src/logic/sync/domain/sync.entity.ts +++ b/electron-app/src/logic/sync/domain/sync.entity.ts @@ -1,6 +1,7 @@ -import { Observable, catchError, concatMap, of } from 'rxjs'; +import { Observable, catchError, forkJoin, map, of } from 'rxjs'; -import { IConnector, SearchResults } from '../../connector/domain/connector'; +import { z } from 'zod'; +import { FileStatus, IConnector, SearchResults, SyncItem } from '../../connector/domain/connector'; import { getConnector } from '../../connector/infrastructure/factory'; export type Connector = { @@ -10,29 +11,31 @@ export type Connector = { parameters: { [key: string]: any }; }; -export interface NucliaOptions { +export const NucliaOptionsValidator = z.object({ /** * The Nuclia backend to use. * * Example: `https://nuclia.cloud/api` */ - backend: string; + backend: z.string().min(1, { message: 'Required' }), /** * The geographical zone for the regional API calls. * * Example: `europe-1` */ - zone: string; + zone: z.string().min(1, { message: 'Required' }), /** * The Nuclia Knowledge Box unique id. * * Example: `17815eb2-06a5-40ee-a5aa-b2f9dbc5da70` */ - knowledgeBox: string; + knowledgeBox: z.string().min(1, { message: 'Required' }), /** * Allows you to make calls to a private Knowledge Box. * * It can be used in a server-side app, but never in a web app. */ - apiKey: string; -} + apiKey: z.string().min(1, { message: 'Required' }), +}); + +export type NucliaOptions = z.infer; export type Classification = {}; @@ -42,6 +45,8 @@ export interface ISyncEntity { labels?: Classification[]; title: string; id: string; + lastSyncGMT?: string; + foldersToSync?: SyncItem[]; } export class SyncEntity { @@ -50,15 +55,19 @@ export class SyncEntity { public labels?: Classification[]; public title: string; public id: string; - private sourceConnector?: IConnector; + public sourceConnector?: IConnector; + public lastSyncGMT?: string; + public foldersToSync?: SyncItem[] = []; constructor(options: ISyncEntity) { - const { connector, kb, labels, title, id } = options; + const { connector, kb, labels, title, id, lastSyncGMT, foldersToSync } = options; this.connector = connector; this.kb = kb; this.labels = labels; this.title = title; this.id = id; + this.lastSyncGMT = lastSyncGMT; + this.foldersToSync = foldersToSync; this.setConnectorDefinition(); } @@ -71,26 +80,62 @@ export class SyncEntity { this.sourceConnector.setParameters(this.connector?.parameters ?? {}); } - get folders(): Observable { + get allFolders(): Observable { + if (!this.sourceConnector) { + return of({ + items: [], + }); + } + return this.sourceConnector.getFolders(); + } + + get files(): Observable { if (!this.sourceConnector) { return of({ items: [], }); } - return this.sourceConnector.getFolders().pipe( - catchError(() => { - return this.sourceConnector!.refreshAuthentication().pipe( - concatMap((success) => { - if (success) { - const newParams = this.sourceConnector!.getParameters(); - this.connector.parameters = newParams; - return this.sourceConnector!.getFolders(); - } else { - throw new Error('Failed to refresh authentication'); - } - }), - ); + return this.sourceConnector.getFiles(); + } + + getLastModified(): Observable<{ success: boolean; results: SyncItem[]; error?: string }> { + const foldersToSyncPending: SyncItem[] = (this.foldersToSync ?? []).filter( + (folder) => folder.status === FileStatus.PENDING || folder.status === undefined, + ); + const foldersToSyncUpdated: SyncItem[] = (this.foldersToSync ?? []).filter( + (folder) => folder.status === FileStatus.UPLOADED, + ); + + const getFilesFoldersUpdated = this.sourceConnector!.getLastModified( + this.lastSyncGMT || '2000-01-01T00:00:00.000Z', + foldersToSyncUpdated, + ); + const getFilesFolderPending = this.sourceConnector!.getFilesFromFolders(foldersToSyncPending); + return forkJoin([getFilesFoldersUpdated, getFilesFolderPending]).pipe( + map((results) => { + const [updated, pending] = results; + return { success: true, results: [...updated.items, ...pending.items] }; + }), + catchError((err) => { + console.error(`Error on ${this.id}: ${err.message}`); + return of({ success: false, results: [], error: `${err}` }); }), ); } + + isAccesTokenValid(): Observable { + return this.sourceConnector!.isAccesTokenValid(); + } + + refreshAuthentication(): Observable { + return this.sourceConnector!.refreshAuthentication(); + } + + getConnectorParameters() { + return this.sourceConnector!.getParameters(); + } + + hasAuthData() { + return this.sourceConnector!.hasAuthData(); + } } diff --git a/electron-app/src/logic/sync/domain/use-cases/get-sync-auth.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/get-sync-auth.use-case.ts index 0925f70..68d212d 100644 --- a/electron-app/src/logic/sync/domain/use-cases/get-sync-auth.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/get-sync-auth.use-case.ts @@ -1,6 +1,6 @@ import { CustomError } from '../../../errors'; +import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; -import { getConnector } from '../../../connector/infrastructure/factory'; export interface GetSyncAuthUseCase { execute(id: string): Promise; @@ -14,12 +14,6 @@ export class GetSyncAuth implements GetSyncAuthUseCase { if (data === null) { throw new CustomError(`Sync with id ${id} not found`, 404); } - const connectorDefinition = getConnector(data.connector?.name || ''); - if (!connectorDefinition) { - return false; - } - const sourceConnector = connectorDefinition.factory(); - sourceConnector.setParameters(data.connector?.parameters ?? {}); - return sourceConnector.hasAuthData(); + return new SyncEntity(data).hasAuthData(); } } diff --git a/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts index db0687a..c6e4bda 100644 --- a/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts +++ b/electron-app/src/logic/sync/domain/use-cases/get-sync-folders.use-case.ts @@ -4,6 +4,7 @@ import { SearchResults } from '../../../connector/domain/connector'; import { CustomError } from '../../../errors'; import { SyncEntity } from '../sync.entity'; import { ISyncRepository } from '../sync.repository'; +import { RefreshAccessToken } from './refresh-access-token.use-case'; export interface GetSyncFoldersUseCase { execute(id: string): Promise; @@ -17,8 +18,13 @@ export class GetSyncFolders implements GetSyncFoldersUseCase { if (data === null) { throw new CustomError(`Sync with id ${id} not found`, 404); } - const syncEntity = new SyncEntity(data); - const dataFolders = await firstValueFrom(syncEntity.folders.pipe()); - return dataFolders; + + try { + const syncEntity = new SyncEntity(data); + const updatedEntity = await firstValueFrom(new RefreshAccessToken(this.repository).execute(syncEntity)); + return await firstValueFrom(updatedEntity.allFolders.pipe()); + } catch (error) { + throw new CustomError(`Error getting folders for sync with id ${id}`, 500); + } } } diff --git a/electron-app/src/logic/sync/domain/use-cases/refresh-access-token.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/refresh-access-token.use-case.ts new file mode 100644 index 0000000..5adbf2a --- /dev/null +++ b/electron-app/src/logic/sync/domain/use-cases/refresh-access-token.use-case.ts @@ -0,0 +1,42 @@ +import { Observable, concatMap, of } from 'rxjs'; +import { UpdateSyncDto } from '../dto/update-sync.dto'; +import { SyncEntity } from '../sync.entity'; +import { ISyncRepository } from '../sync.repository'; +import { UpdateSync } from './update-sync.use-case'; + +export interface RefreshAccessTokenUseCase { + execute(entity: SyncEntity): Observable; +} + +export class RefreshAccessToken implements RefreshAccessTokenUseCase { + constructor(private readonly repository: ISyncRepository) {} + + execute(entity: SyncEntity) { + const observable = entity.isAccesTokenValid().pipe( + concatMap((isValid) => { + if (!isValid) { + return entity.refreshAuthentication().pipe( + concatMap(async (success) => { + if (success) { + const newParams = entity.getConnectorParameters(); + const [, updateSyncDto] = UpdateSyncDto.create({ + connector: { + ...entity.connector, + parameters: newParams, + }, + id: entity.id, + }); + await new UpdateSync(this.repository).execute(updateSyncDto!); + } else { + throw new Error('Failed to refresh authentication'); + } + return entity; + }), + ); + } + return of(entity); + }), + ); + return observable; + } +} diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts new file mode 100644 index 0000000..1c8bb0f --- /dev/null +++ b/electron-app/src/logic/sync/domain/use-cases/sync-all-data.use-case.ts @@ -0,0 +1,103 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Observable, catchError, delay, forkJoin, map, of, switchMap, tap } from 'rxjs'; + +import { EVENTS } from '../../../../events/events'; +import { eventEmitter } from '../../../../server'; +import { UpdateSyncDto } from '../dto/update-sync.dto'; +import { SyncEntity } from '../sync.entity'; +import { ISyncRepository } from '../sync.repository'; +import { RefreshAccessToken } from './refresh-access-token.use-case'; +import { SyncSingleFile } from './sync-single-file.use-case'; +import { UpdateSync } from './update-sync.use-case'; + +require('localstorage-polyfill'); + +export interface SyncAllDataUseCase { + execute(since: Date | undefined): Promise; +} + +export class SyncAllData implements SyncAllDataUseCase { + constructor(private readonly repository: ISyncRepository) {} + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async execute(since: Date | undefined) { + const syncObjects = await this.repository.getAllSync(); + of(Object.values(syncObjects)) + .pipe( + switchMap((syncObjectValues) => { + if (syncObjectValues.length === 0) { + return of(undefined); + } else { + return forkJoin( + syncObjectValues.map((syncObj) => + of(syncObj).pipe( + switchMap((syncObj) => new RefreshAccessToken(this.repository).execute(new SyncEntity(syncObj))), + switchMap((syncEntity) => + syncEntity.files.pipe( + map((files) => { + return { files, syncEntity }; + }), + ), + ), + switchMap(({ files, syncEntity }) => { + const items = files.items.slice(0, 10); + // addActiveSyncLog(id, source); + eventEmitter.emit(EVENTS.START_SYNCHRONIZATION_SYNC_OBJECT, { + from: syncEntity.id, + to: syncEntity.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + total: items?.length || 0, + }); + + const batch: Observable<{ id: string; success: boolean }>[] = items.map((item) => + of(item).pipe( + switchMap((item) => + new SyncSingleFile(syncEntity, item).execute().pipe( + map((res) => ({ id: item.originalId, success: res.success })), + // do not overwhelm the source + delay(500), + ), + ), + ), + ); + + return forkJoin(batch).pipe( + tap((result) => { + if (result) { + const processed = result.map((res) => res.id); + const successCount = result.filter((res) => res.success).length; + + console.log('processed', processed); + console.log('successCount', successCount); + eventEmitter.emit(EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT, { + from: syncEntity.id, + to: syncEntity.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + processed, + successCount, + }); + + const [, updateSyncDto] = UpdateSyncDto.create({ + lastSyncGMT: new Date().toISOString(), + id: syncEntity.id, + }); + new UpdateSync(this.repository).execute(updateSyncDto!); + } + }), + ); + }), + catchError(() => { + // emit error event + return of(undefined); + }), + ), + ), + ); + } + }), + ) + .subscribe(() => console.log('Finish sync all data')); + + return true; + } +} diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts new file mode 100644 index 0000000..f1e48b9 --- /dev/null +++ b/electron-app/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts @@ -0,0 +1,113 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Observable, delay, forkJoin, map, of, switchMap, tap } from 'rxjs'; + +import { EVENTS } from '../../../../events/events'; +import { eventEmitter } from '../../../../server'; +import { FileStatus } from '../../../connector/domain/connector'; +import { UpdateSyncDto } from '../dto/update-sync.dto'; +import { SyncEntity } from '../sync.entity'; +import { ISyncRepository } from '../sync.repository'; +import { RefreshAccessToken } from './refresh-access-token.use-case'; +import { SyncSingleFile } from './sync-single-file.use-case'; +import { UpdateSync } from './update-sync.use-case'; + +require('localstorage-polyfill'); + +export interface SyncAllFoldersUseCase { + execute(since: Date | undefined): Promise; +} + +export class SyncAllFolders implements SyncAllFoldersUseCase { + constructor(private readonly repository: ISyncRepository) {} + + callbackFinishSync = (syncEntity: SyncEntity, processed: string[], successCount: number, error?: string) => { + eventEmitter.emit(EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT, { + from: syncEntity.id, + to: syncEntity.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + processed, + successCount, + error, + }); + + const foldersToSyncCopy = (structuredClone(syncEntity.foldersToSync) ?? []).map((folder) => { + folder.status = FileStatus.UPLOADED; + return folder; + }); + + const [, updateSyncDto] = UpdateSyncDto.create({ + lastSyncGMT: new Date().toISOString(), + id: syncEntity.id, + foldersToSync: foldersToSyncCopy, + }); + new UpdateSync(this.repository).execute(updateSyncDto!); + }; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async execute() { + const syncObjects = await this.repository.getAllSync(); + of(Object.values(syncObjects)) + .pipe( + switchMap((syncObjectValues) => { + if (syncObjectValues.length === 0) { + return of(undefined); + } else { + return forkJoin( + syncObjectValues.map((syncObj) => + of(syncObj).pipe( + switchMap((syncObj) => new RefreshAccessToken(this.repository).execute(new SyncEntity(syncObj))), + switchMap((syncEntity) => + syncEntity.getLastModified().pipe( + map((result) => { + return { result, syncEntity }; + }), + ), + ), + switchMap(({ result, syncEntity }) => { + eventEmitter.emit(EVENTS.START_SYNCHRONIZATION_SYNC_OBJECT, { + from: syncEntity.id, + to: syncEntity.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + total: result.results?.length || 0, + }); + + if (!result.success || result.results.length === 0) { + this.callbackFinishSync(syncEntity, [], 0, result.error); + return of(undefined); + } + + const batch: Observable<{ id: string; success: boolean }>[] = result.results.map((item) => + of(item).pipe( + switchMap((item) => + new SyncSingleFile(syncEntity, item).execute().pipe( + map((res) => ({ id: item.originalId, success: res.success })), + // do not overwhelm the source + delay(500), + ), + ), + ), + ); + + return forkJoin(batch).pipe( + tap((result) => { + if (result) { + const processed = result.map((res) => res.id); + const successCount = result.filter((res) => res.success).length; + + console.log('processed', processed); + console.log('successCount', successCount); + this.callbackFinishSync(syncEntity, processed, successCount, ''); + } + }), + ); + }), + ), + ), + ); + } + }), + ) + .subscribe(() => console.log('Finish sync folders data')); + + return true; + } +} diff --git a/electron-app/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts b/electron-app/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts new file mode 100644 index 0000000..c927bb2 --- /dev/null +++ b/electron-app/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts @@ -0,0 +1,93 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { TextField } from '@nuclia/core'; +import { Observable, from, map, of, switchMap, tap } from 'rxjs'; + +import { EVENTS } from '../../../../events/events'; +import { eventEmitter } from '../../../../server'; +import { SyncItem } from '../../../connector/domain/connector'; +import { NucliaCloud } from '../nuclia-cloud'; +import { SyncEntity } from '../sync.entity'; + +require('localstorage-polyfill'); + +export interface SyncSingleFileUseCase { + execute(): Observable<{ success: boolean; message?: string }>; +} + +function downloadFileOrLink( + sync: SyncEntity, + item: SyncItem, +): Observable<{ type: 'blob' | 'link' | 'text'; blob?: Blob; link?: any; text?: TextField }> { + const connector = sync.sourceConnector; + return connector! + .download(item) + .pipe(map((res) => (res instanceof Blob ? { type: 'blob', blob: res } : { type: 'text', text: res }))); +} + +export class SyncSingleFile implements SyncSingleFileUseCase { + constructor( + private readonly sync: SyncEntity, + private readonly item: SyncItem, + ) {} + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + execute() { + const sync = this.sync; + const item = this.item; + + if (!sync.kb) { + return of({ success: false }); + } + const nucliaConnector = new NucliaCloud(sync.kb); + return downloadFileOrLink(sync, item).pipe( + switchMap((data) => { + try { + if (data.type === 'blob' && data.blob) { + return from(data.blob.arrayBuffer()).pipe( + switchMap((arrayBuffer) => { + return nucliaConnector.upload(item.originalId, item.title, { + buffer: arrayBuffer, + metadata: { ...item.metadata, labels: sync.labels }, + }); + }), + ); + } else if (data.type === 'text' && data.text) { + return nucliaConnector.upload(item.originalId, item.title, { + text: data.text, + metadata: { labels: sync.labels }, + }); + } else if (data.type === 'link' && data.link) { + return nucliaConnector + .uploadLink(item.originalId, item.title, data.link) + .pipe(map(() => ({ success: true, message: '' }))); + } else { + return of({ success: false, message: '' }); + } + } catch (err) { + return of({ success: false, message: `${err}` }); + } + }), + tap((res) => { + if (res.success) { + console.log(`Uploaded ${item.originalId} with success`); + eventEmitter.emit(EVENTS.FINISH_SYNCHRONIZATION_SINGLE_FILE, { + from: sync.id, + to: sync.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + status: 'success', + message: `Uploaded ${item.originalId} with success`, + }); + } else { + console.warn(`Failed to upload ${item.originalId}`); + eventEmitter.emit(EVENTS.FINISH_SYNCHRONIZATION_SINGLE_FILE, { + from: sync.id, + to: sync.kb?.knowledgeBox || 'Unknown kb', + date: new Date().toISOString(), + status: 'failed', + message: `Failed to upload ${item.originalId} ${res.message || ''}`, + }); + } + }), + ); + } +} diff --git a/electron-app/src/logic/sync/presentation/routes.ts b/electron-app/src/logic/sync/presentation/routes.ts index 5bd374b..a1f95eb 100644 --- a/electron-app/src/logic/sync/presentation/routes.ts +++ b/electron-app/src/logic/sync/presentation/routes.ts @@ -6,12 +6,13 @@ import { UpdateSyncDto } from '../domain/dto/update-sync.dto'; import { CreateSync } from '../domain/use-cases/create-sync.use-case'; import { DeleteSync } from '../domain/use-cases/delete-sync.use-case'; import { GetAllSync } from '../domain/use-cases/get-all-sync.use-case'; +import { GetSyncAuth } from '../domain/use-cases/get-sync-auth.use-case'; import { GetSyncFolders } from '../domain/use-cases/get-sync-folders.use-case'; import { GetSync } from '../domain/use-cases/get-sync.use-case'; +import { SyncAllFolders } from '../domain/use-cases/sync-all-folders-data.use-case'; import { UpdateSync } from '../domain/use-cases/update-sync.use-case'; import { FileSystemSyncDatasource } from '../infrastructure/file-system.sync.datasource'; import { SyncRepository } from '../infrastructure/sync.repository'; -import { GetSyncAuth } from '../domain/use-cases/get-sync-auth.use-case'; export class SyncFileSystemRoutes { private readonly basePath: string; @@ -51,6 +52,15 @@ export class SyncFileSystemRoutes { } }); + router.get('/execute', async (_req, res) => { + try { + await new SyncAllFolders(syncRepository).execute(); + res.status(200).send({ success: true }); + } catch (error) { + this.handleError(res, error); + } + }); + router.post('/', async (req, res) => { const [error, createSyncDto] = CreateSyncDto.create(req.body); if (error) return res.status(400).json({ error }); diff --git a/electron-app/src/subscribers.ts b/electron-app/src/subscribers.ts index f79efb6..5ba359d 100644 --- a/electron-app/src/subscribers.ts +++ b/electron-app/src/subscribers.ts @@ -43,4 +43,41 @@ export function initFileSystemSubscribers(basePath: string) { }), ); }); + + eventEmitter.subscribe(EVENTS.START_SYNCHRONIZATION_SYNC_OBJECT, (payload: { [key: string]: string }) => { + const saveLog = new SaveLogs(new LogRepository(new FileSystemLogDatasource(basePath))); + saveLog.execute( + new LogEntity({ + message: 'Synchronization started', + level: LogSeverityLevel.low, + origin: 'electron-app', + action: EVENTS.START_SYNCHRONIZATION_SYNC_OBJECT, + payload, + }), + ); + }); + eventEmitter.subscribe(EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT, (payload: { [key: string]: string }) => { + const saveLog = new SaveLogs(new LogRepository(new FileSystemLogDatasource(basePath))); + saveLog.execute( + new LogEntity({ + message: 'Synchronization finished', + level: LogSeverityLevel.low, + origin: 'electron-app', + action: EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT, + payload, + }), + ); + }); + eventEmitter.subscribe(EVENTS.FINISH_SYNCHRONIZATION_SINGLE_FILE, (payload: { [key: string]: string }) => { + const saveLog = new SaveLogs(new LogRepository(new FileSystemLogDatasource(basePath))); + saveLog.execute( + new LogEntity({ + message: 'Synchronization single file finished', + level: LogSeverityLevel.low, + origin: 'electron-app', + action: EVENTS.FINISH_SYNCHRONIZATION_SINGLE_FILE, + payload, + }), + ); + }); } diff --git a/electron-app/tests/fixtures.js b/electron-app/tests/fixtures.js index f7bc5da..a6199ed 100644 --- a/electron-app/tests/fixtures.js +++ b/electron-app/tests/fixtures.js @@ -41,6 +41,48 @@ export const serverTest = test.extend({ expect(response.status).toBe(201); await use(server); }, + serverWithOAuthSync: async ({ server }, use) => { + let response = await request(server.app) + .post('/sync') + .send({ + id: 'sync_oauth_gdrive', + connector: { + name: 'gdrive', + logo: '', + parameters: { + token: 'token_test', + refresh: 'refresh_token_test', + }, + }, + kb: { + knowledgeBox: 'test', + zone: 'local', + backend: 'http://localhost:8000', + apiKey: 'apiKey', + }, + }); + expect(response.status).toBe(201); + await use(server); + }, + serverWithSyncWithoutConnector: async ({ server }, use) => { + let response = await request(server.app) + .post('/sync') + .send({ + id: 'sync_without_connector', + title: 'Sync without connector', + connector: { + name: 'gdrive', + }, + kb: { + knowledgeBox: 'test', + zone: 'local', + backend: 'http://localhost:8000', + apiKey: 'apiKey', + }, + }); + expect(response.status).toBe(201); + await use(server); + }, }); export const serverTestWithoutFolder = test.extend({ diff --git a/electron-app/tests/sync.spec.js b/electron-app/tests/sync.spec.js index afe6aac..50c2282 100644 --- a/electron-app/tests/sync.spec.js +++ b/electron-app/tests/sync.spec.js @@ -68,10 +68,22 @@ describe('Test Sync object', () => { }); }); + serverTest('Get entity auth', async ({ serverWithOAuthSync }) => { + const responseAuth = await request(serverWithOAuthSync.app).get('/sync/sync_oauth_gdrive/auth'); + expect(responseAuth.status).toBe(200); + expect(responseAuth.body.hasAuth).toBe(true); + }); + + serverTest('Get entity auth', async ({ serverWithSyncWithoutConnector }) => { + const responseAuth = await request(serverWithSyncWithoutConnector.app).get('/sync/sync_without_connector/auth'); + expect(responseAuth.status).toBe(200); + expect(responseAuth.body.hasAuth).toBe(false); + }); + serverTest('Delete a sync', async ({ serverWithSync }) => { let response = await request(serverWithSync.app).get('/sync'); expect(response.status).toBe(200); - expect(Object.keys(response.body).length).toEqual(1); + expect(Object.keys(response.body).length).toEqual(3); const id = Object.keys(response.body)[0]; const responseDelete = await request(serverWithSync.app).delete(`/sync/${id}`); @@ -79,6 +91,6 @@ describe('Test Sync object', () => { response = await request(serverWithSync.app).get('/sync'); expect(response.status).toBe(200); - expect(Object.keys(response.body).length).toEqual(0); + expect(Object.keys(response.body).length).toEqual(2); }); }); diff --git a/electron-app/tsconfig.json b/electron-app/tsconfig.json index c91ef1e..8dfe0bf 100644 --- a/electron-app/tsconfig.json +++ b/electron-app/tsconfig.json @@ -1,7 +1,6 @@ { "compilerOptions": { - "baseUrl": ".", - "target": "es5", + "target": "es2015", "module": "commonjs", "strict": true, "esModuleInterop": true,