From e774f57638bc920c7edceb932278f20ac0f3b837 Mon Sep 17 00:00:00 2001 From: Ralf Aron Date: Mon, 21 Oct 2024 07:27:26 +0200 Subject: [PATCH] fix: pagination --- package-lock.json | 23 +- package.json | 1 + projects/aas-core/src/lib/server-message.ts | 6 +- projects/aas-core/src/lib/types.ts | 1 + .../start/add-endpoint-form.component.spec.ts | 16 +- projects/aas-server/package.json | 1 + .../aas-server/src/app/aas-index/aas-index.ts | 8 +- .../src/app/aas-index/lowdb/lowdb-index.ts | 48 ++- .../src/app/aas-index/mysql/mysql-index.ts | 31 +- .../src/app/aas-provider/aas-provider.ts | 67 +++-- .../src/app/aas-provider/aas-resource-scan.ts | 8 +- .../src/app/aas-provider/aas-server-scan.ts | 8 +- .../src/app/aas-provider/directory-scan.ts | 9 +- .../src/app/aas-provider/opcua-server-scan.ts | 8 +- .../src/app/aas-provider/parallel.ts | 21 +- .../src/app/aas-provider/scan-result.ts | 21 +- .../src/app/aas-provider/task-handler.ts | 6 +- .../src/app/aas-provider/worker-data.ts | 16 +- .../{scan-container.ts => endpoint-scan.ts} | 57 ++-- projects/aas-server/src/app/http-client.ts | 283 ++++-------------- .../src/app/packages/aas-resource.ts | 11 +- .../packages/aas-server/aas-api-client-v0.ts | 10 +- .../packages/aas-server/aas-api-client-v1.ts | 8 +- .../packages/aas-server/aas-api-client-v3.ts | 113 ++++--- .../app/packages/aas-server/aas-api-client.ts | 3 +- .../packages/file-system/aasx-directory.ts | 8 +- .../src/app/template/template-storage.ts | 4 +- .../aas-server/src/app/types/paged-result.ts | 16 + projects/aas-server/src/app/worker-app.ts | 68 ++++- .../test/aas-index/lowdb/lowdb-index.spec.ts | 4 +- .../packages/aas-server/aasx-package.spec.ts | 6 +- .../test/packages/opcua/opcua-client.spec.ts | 8 +- projects/aas-server/tsoa.json | 2 +- 33 files changed, 488 insertions(+), 412 deletions(-) rename projects/aas-server/src/app/{scan-container.ts => endpoint-scan.ts} (69%) create mode 100644 projects/aas-server/src/app/types/paged-result.ts diff --git a/package-lock.json b/package-lock.json index ef1cb648..ad67e2cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "aas-portal-project", - "version": "3.0.0-development.94", + "version": "3.0.0-development.97", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "aas-portal-project", - "version": "3.0.0-development.94", + "version": "3.0.0-development.97", "license": "Apache-2.0", "workspaces": [ "projects/fhg-jest", @@ -32,6 +32,7 @@ "@popperjs/core": "^2.11.8", "@xmldom/xmldom": "^0.8.10", "aas-core": "projects/aas-core", + "axios": "^1.7.7", "bcryptjs": "^2.4.3", "bootstrap": "^5.3.3", "bootstrap-icons": "^1.11.3", @@ -9124,6 +9125,17 @@ "node": ">= 6.0.0" } }, + "node_modules/axios": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/axobject-query": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", @@ -13389,7 +13401,6 @@ "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", - "dev": true, "funding": [ { "type": "individual", @@ -26136,6 +26147,12 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", diff --git a/package.json b/package.json index e8105371..1d8a25cb 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@popperjs/core": "^2.11.8", "@xmldom/xmldom": "^0.8.10", "aas-core": "projects/aas-core", + "axios": "^1.7.7", "bcryptjs": "^2.4.3", "bootstrap": "^5.3.3", "bootstrap-icons": "^1.11.3", diff --git a/projects/aas-core/src/lib/server-message.ts b/projects/aas-core/src/lib/server-message.ts index 4f19cd86..5e31ce8c 100644 --- a/projects/aas-core/src/lib/server-message.ts +++ b/projects/aas-core/src/lib/server-message.ts @@ -6,7 +6,7 @@ * *****************************************************************************/ -import { AASContainer, AASDocument, AASEndpoint } from './types.js'; +import { AASDocument, AASEndpoint } from './types.js'; /** Defines the message types. */ export type AASServerMessageType = @@ -22,9 +22,7 @@ export type AASServerMessageType = export interface AASServerMessage { /** The type of change. */ type: AASServerMessageType; - /** The container if type `ContainerAdded` and `ContainerRemoved`. */ - container?: AASContainer; - /** The endpoint if type `ContainerAdded`, `ContainerRemoved`, `EndpointAdded`, `EndpointRemoved`. */ + /** The endpoint if type `EndpointAdded`, `EndpointRemoved`. */ endpoint?: AASEndpoint; /** The document if type `Added`, `Removed` or `Changed` */ document?: AASDocument; diff --git a/projects/aas-core/src/lib/types.ts b/projects/aas-core/src/lib/types.ts index a0746c97..3e1b2047 100644 --- a/projects/aas-core/src/lib/types.ts +++ b/projects/aas-core/src/lib/types.ts @@ -59,6 +59,7 @@ export type AASEndpoint = { /** Represents a server (AASX, OPC-UA) or file directory (AASX package files). */ export interface AASContainer extends AASEndpoint { + cursor?: string; documents?: AASDocument[]; } diff --git a/projects/aas-portal/src/test/start/add-endpoint-form.component.spec.ts b/projects/aas-portal/src/test/start/add-endpoint-form.component.spec.ts index 4e5b4de0..3e39d753 100644 --- a/projects/aas-portal/src/test/start/add-endpoint-form.component.spec.ts +++ b/projects/aas-portal/src/test/start/add-endpoint-form.component.spec.ts @@ -51,7 +51,7 @@ describe('AddEndpointFormComponent', () => { let endpoint: AASEndpoint | undefined; spyOn(modal, 'close').and.callFake(result => (endpoint = result)); - component.item.set({ ...component.items()[3], value: 'file:///my-endpoint' }); + component.selectItem(component.items()[3]); component.name.set('My endpoint'); form.dispatchEvent(new Event('submit')); @@ -65,7 +65,7 @@ describe('AddEndpointFormComponent', () => { let endpoint: AASEndpoint | undefined; spyOn(modal, 'close').and.callFake(result => (endpoint = result)); - component.item.set({ ...component.items()[3], value: 'file:///a\\b\\my-endpoint' }); + component.selectItem(component.items()[3]); component.name.set('My endpoint'); form.dispatchEvent(new Event('submit')); @@ -78,7 +78,7 @@ describe('AddEndpointFormComponent', () => { it('ignores AAS endpoint: Name: "", URL: "file:///my-endpoint"', () => { spyOn(modal, 'close'); - component.item.set({ ...component.items()[3], value: 'file:///my-endpoint' }); + component.selectItem(component.items()[3]); component.name.set(''); form.dispatchEvent(new Event('submit')); @@ -89,7 +89,7 @@ describe('AddEndpointFormComponent', () => { it('ignores AAS endpoint Name: "My endpoint", URL: "file:///"', () => { spyOn(modal, 'close'); - component.item.set({ ...component.items()[3], value: 'file:///' }); + component.selectItem(component.items()[3]); component.name.set('My endpoint'); form.dispatchEvent(new Event('submit')); @@ -101,7 +101,7 @@ describe('AddEndpointFormComponent', () => { let endpoint: AASEndpoint | undefined; spyOn(modal, 'close').and.callFake(result => (endpoint = result)); - component.item.set({ ...component.items()[1], value: 'opc.tcp://localhost:30001/I4AASServer' }); + component.selectItem(component.items()[1]); component.name.set('I4AAS Server'); form.dispatchEvent(new Event('submit')); @@ -114,7 +114,7 @@ describe('AddEndpointFormComponent', () => { it('ignores AAS endpoint Name: "I4AAS Server", URL: "opc.tcp://"', () => { spyOn(modal, 'close'); - component.item.set(component.items()[1]); + component.selectItem(component.items()[1]); fixture.detectChanges(); inputNameElement.value = 'I4AAS Server'; @@ -131,7 +131,7 @@ describe('AddEndpointFormComponent', () => { let endpoint: AASEndpoint | undefined; spyOn(modal, 'close').and.callFake(result => (endpoint = result)); - component.item.set({ ...component.items()[0], value: 'http://localhost:50001/' }); + component.selectItem(component.items()[0]); component.name.set('AASX Server'); form.dispatchEvent(new Event('submit')); @@ -145,7 +145,7 @@ describe('AddEndpointFormComponent', () => { let endpoint: AASEndpoint | undefined; spyOn(modal, 'close').and.callFake(result => (endpoint = result)); - component.item.set({ ...component.items()[2], value: 'http://localhost:8080/root/folder' }); + component.selectItem(component.items()[2]); component.name.set('WebDAV Server'); form.dispatchEvent(new Event('submit')); diff --git a/projects/aas-server/package.json b/projects/aas-server/package.json index 683b53d3..4f754ceb 100644 --- a/projects/aas-server/package.json +++ b/projects/aas-server/package.json @@ -22,6 +22,7 @@ "dependencies": { "@babel/polyfill": "^7.4.4", "@xmldom/xmldom": "^0.8.10", + "axios": "^1.7.7", "bcryptjs": "^2.4.3", "chalk": "^2.4.1", "cors": "^2.8.5", diff --git a/projects/aas-server/src/app/aas-index/aas-index.ts b/projects/aas-server/src/app/aas-index/aas-index.ts index 36045caf..a63a1b24 100644 --- a/projects/aas-server/src/app/aas-index/aas-index.ts +++ b/projects/aas-server/src/app/aas-index/aas-index.ts @@ -20,6 +20,8 @@ import { toBoolean, } from 'aas-core'; +import { PagedResult } from '../types/paged-result.js'; + export abstract class AASIndex { public abstract getCount(query?: string): Promise; @@ -35,7 +37,11 @@ export abstract class AASIndex { public abstract getDocuments(cursor: AASCursor, query?: string, language?: string): Promise; - public abstract getContainerDocuments(endpointName: string): Promise; + public abstract nextPage( + endpointName: string, + cursor: string | undefined, + limit?: number, + ): Promise>; public abstract update(document: AASDocument): Promise; diff --git a/projects/aas-server/src/app/aas-index/lowdb/lowdb-index.ts b/projects/aas-server/src/app/aas-index/lowdb/lowdb-index.ts index 8cf486eb..4cdcf8ed 100644 --- a/projects/aas-server/src/app/aas-index/lowdb/lowdb-index.ts +++ b/projects/aas-server/src/app/aas-index/lowdb/lowdb-index.ts @@ -26,6 +26,8 @@ import { Variable } from '../../variable.js'; import { urlToEndpoint } from '../../configuration.js'; import { ERRORS } from '../../errors.js'; import { LowDbData, LowDbDocument, LowDbElement } from './lowdb-types.js'; +import { decodeBase64Url, encodeBase64Url } from '../../convert.js'; +import { PagedResult } from '../../types/paged-result.js'; export class LowDbIndex extends AASIndex { private readonly promise: Promise; @@ -88,9 +90,51 @@ export class LowDbIndex extends AASIndex { return true; } - public override async getContainerDocuments(endpointName: string): Promise { + public override async nextPage( + endpointName: string, + cursor: string | undefined, + limit: number = 10, + ): Promise> { await this.promise; - return this.db.data.documents.filter(document => document.endpoint === endpointName); + + if (cursor) { + cursor = decodeBase64Url(cursor); + } + + const documents: AASDocument[] = []; + if (this.db.data.documents.length === 0) { + return { result: documents, paging_metadata: {} }; + } + + const items = this.db.data.documents; + const index = items.findIndex(item => { + if (item.endpoint !== endpointName) { + return false; + } + + return cursor === undefined || cursor.localeCompare(item.id) <= 0; + }); + + if (index < 0) { + return { result: documents, paging_metadata: {} }; + } + + const result: AASDocument[] = []; + for (let i = 0, j = index, n = items.length; i < limit && j < n; i++, j++) { + const item = items[j]; + if (item.endpoint !== endpointName) { + break; + } + + result.push(item); + } + + const k = index + limit + 1; + if (k >= items.length || items[k].endpoint !== endpointName) { + return { result, paging_metadata: {} }; + } + + return { result, paging_metadata: { cursor: encodeBase64Url(items[k].id) } }; } public override async getDocuments(cursor: AASCursor, query?: string, language?: string): Promise { diff --git a/projects/aas-server/src/app/aas-index/mysql/mysql-index.ts b/projects/aas-server/src/app/aas-index/mysql/mysql-index.ts index da7fd15e..80dbd517 100644 --- a/projects/aas-server/src/app/aas-index/mysql/mysql-index.ts +++ b/projects/aas-server/src/app/aas-index/mysql/mysql-index.ts @@ -15,6 +15,8 @@ import { Variable } from '../../variable.js'; import { urlToEndpoint } from '../../configuration.js'; import { MySqlQuery } from './mysql-query.js'; import { DocumentCount, MySqlDocument, MySqlEndpoint } from './mysql-types.js'; +import { PagedResult } from '../../types/paged-result.js'; +import { encodeBase64Url } from '../../convert.js'; export class MySqlIndex extends AASIndex { private readonly connection: Promise; @@ -117,12 +119,29 @@ export class MySqlIndex extends AASIndex { return this.getLastPage(cursor.limit); } - public override async getContainerDocuments(endpointName: string): Promise { - return ( - await ( - await this.connection - ).query('SELECT * FROM `documents` WHERE endpoint = ?', [endpointName]) - )[0].map(row => this.toDocument(row)); + public override async nextPage( + endpointName: string, + cursor: string | undefined, + limit: number = 100, + ): Promise> { + const connection = await this.connection; + let sql: string; + const values: unknown[] = [endpointName + cursor]; + if (cursor) { + values.push(limit + 1); + sql = 'SELECT * FROM `documents` WHERE CONCAT(endpoint, id) > ? ORDER BY endpoint ASC, id ASC LIMIT ?;'; + } else { + sql = 'SELECT * FROM `documents` ORDER BY endpoint ASC, id ASC LIMIT ?;'; + } + + const [results] = await connection.query(sql, values); + const documents = results.map(result => this.toDocument(result)); + return { + result: documents.slice(0, limit), + paging_metadata: { + cursor: documents.length >= limit + 1 ? encodeBase64Url(documents[limit].id) : undefined, + }, + }; } public override async update(document: AASDocument): Promise { diff --git a/projects/aas-server/src/app/aas-provider/aas-provider.ts b/projects/aas-server/src/app/aas-provider/aas-provider.ts index ec528b33..b8f4e33e 100644 --- a/projects/aas-server/src/app/aas-provider/aas-provider.ts +++ b/projects/aas-server/src/app/aas-provider/aas-provider.ts @@ -26,10 +26,10 @@ import { import { ImageProcessing } from '../image-processing.js'; import { AASIndex } from '../aas-index/aas-index.js'; -import { ScanResultType, ScanResult, ScanContainerResult } from './scan-result.js'; +import { ScanResultType, ScanResult, ScanEndpointResult, ScanNextPageResult } from './scan-result.js'; import { Logger } from '../logging/logger.js'; import { Parallel } from './parallel.js'; -import { ScanContainerData } from './worker-data.js'; +import { ScanEndpointData } from './worker-data.js'; import { SocketClient } from '../live/socket-client.js'; import { EmptySubscription } from '../live/empty-subscription.js'; import { SocketSubscription } from '../live/socket-subscription.js'; @@ -61,6 +61,7 @@ export class AASProvider { this.timeout = variable.SCAN_CONTAINER_TIMEOUT; this.parallel.on('message', this.parallelOnMessage); this.parallel.on('end', this.parallelOnEnd); + this.parallel.on('nextPage', this.parallelOnNextPage); } /** @@ -232,7 +233,7 @@ export class AASProvider { } as AASServerMessage, }); - setTimeout(this.scanContainer, 0, this.taskHandler.createTaskId(), endpoint); + setTimeout(this.scanEndpoint, 0, this.taskHandler.createTaskId(), endpoint); } /** @@ -376,8 +377,8 @@ export class AASProvider { public async scanAsync(factory: AASResourceScanFactory): Promise { for (const endpoint of await this.index.getEndpoints()) { if (endpoint.type === 'FileSystem') { - const documents = await factory.create(endpoint).scanAsync(); - documents.forEach(async document => await this.index.add(document)); + const result = await factory.create(endpoint).scanAsync(); + result.result.forEach(async document => await this.index.add(document)); } } } @@ -463,22 +464,23 @@ export class AASProvider { private startScan = async (): Promise => { try { for (const endpoint of await this.index.getEndpoints()) { - setTimeout(this.scanContainer, 0, this.taskHandler.createTaskId(), endpoint); + setTimeout(this.scanEndpoint, 0, this.taskHandler.createTaskId(), endpoint); } } catch (error) { this.logger.error(error); } }; - private scanContainer = async (taskId: number, endpoint: AASEndpoint) => { - const documents = await this.index.getContainerDocuments(endpoint.name); - const data: ScanContainerData = { - type: 'ScanContainerData', + private scanEndpoint = async (taskId: number, endpoint: AASEndpoint) => { + const result = await this.index.nextPage(endpoint.name, undefined); + const data: ScanEndpointData = { + type: 'ScanEndpointData', taskId, - container: { ...endpoint, documents }, + endpoint, + documents: result.result, }; - this.taskHandler.set(taskId, { name: endpoint.name, owner: this, type: 'ScanContainer' }); + this.taskHandler.set(taskId, { endpointName: endpoint.name, owner: this, type: 'ScanEndpoint' }); this.parallel.execute(data); }; @@ -496,13 +498,13 @@ export class AASProvider { } this.taskHandler.delete(result.taskId); - if ((await await this.index.hasEndpoint(task.name)) === true) { - const endpoint = await this.index.getEndpoint(task.name); - setTimeout(this.scanContainer, this.timeout, result.taskId, endpoint); + if ((await await this.index.hasEndpoint(task.endpointName)) === true) { + const endpoint = await this.index.getEndpoint(task.endpointName); + setTimeout(this.scanEndpoint, this.timeout, result.taskId, endpoint); } if (result.messages) { - this.logger.start(`scan ${task.name ?? 'undefined'}`); + this.logger.start(`scan ${task.endpointName ?? 'undefined'}`); result.messages.forEach(message => this.logger.log(message)); this.logger.stop(); } @@ -512,17 +514,17 @@ export class AASProvider { } }; - private parallelOnMessage = async (result: ScanResult) => { + private parallelOnMessage = async (result: ScanEndpointResult) => { try { switch (result.type) { case ScanResultType.Changed: - await this.onChanged(result as ScanContainerResult); + await this.onChanged(result); break; case ScanResultType.Added: - await this.onAdded(result as ScanContainerResult); + await this.onAdded(result); break; case ScanResultType.Removed: - await this.onRemoved(result as ScanContainerResult); + await this.onRemoved(result); break; } } catch (error) { @@ -530,7 +532,20 @@ export class AASProvider { } }; - private async onChanged(result: ScanContainerResult): Promise { + private parallelOnNextPage = async (result: ScanNextPageResult, worker: Worker) => { + const a = await this.index.nextPage(result.endpoint.name, result.cursor); + const data: ScanEndpointData = { + type: 'ScanEndpointData', + taskId: result.taskId, + endpoint: result.endpoint, + documents: a.result, + cursor: a.paging_metadata.cursor, + }; + + worker.postMessage(data); + }; + + private async onChanged(result: ScanEndpointResult): Promise { const document = result.document; if ((await this.index.hasEndpoint(document.endpoint)) === false) { return; @@ -544,25 +559,25 @@ export class AASProvider { this.sendMessage({ type: 'Changed', document: { ...document, content: null } }); } - private async onAdded(result: ScanContainerResult): Promise { + private async onAdded(result: ScanEndpointResult): Promise { if ((await this.index.hasEndpoint(result.document.endpoint)) === false) { return; } await this.index.add(result.document); - this.logger.info(`Added: AAS ${result.document.idShort} [${result.document.id}] in ${result.container.url}`); + this.logger.info(`Added: AAS ${result.document.idShort} [${result.document.id}] in ${result.endpoint.url}`); this.sendMessage({ type: 'Added', document: result.document }); } - private async onRemoved(result: ScanContainerResult): Promise { + private async onRemoved(result: ScanEndpointResult): Promise { const document = result.document; if ((await this.index.hasEndpoint(document.endpoint)) === false) { return; } - await this.index.remove(result.container.name, document.id); + await this.index.remove(result.endpoint.name, document.id); this.cache.remove(document.endpoint, document.id); - this.logger.info(`Removed: AAS ${document.idShort} [${document.id}] in ${result.container.url}`); + this.logger.info(`Removed: AAS ${document.idShort} [${document.id}] in ${result.endpoint.url}`); this.sendMessage({ type: 'Removed', document: { ...document, content: null } }); } diff --git a/projects/aas-server/src/app/aas-provider/aas-resource-scan.ts b/projects/aas-server/src/app/aas-provider/aas-resource-scan.ts index 45806610..23be2d98 100644 --- a/projects/aas-server/src/app/aas-provider/aas-resource-scan.ts +++ b/projects/aas-server/src/app/aas-provider/aas-resource-scan.ts @@ -8,9 +8,13 @@ import EventEmitter from 'events'; import { AASDocument } from 'aas-core'; +import { PagedResult } from '../types/paged-result.js'; /** Defines an automate to scan an AAS resource for Asset Administration Shells. */ export abstract class AASResourceScan extends EventEmitter { - /** Gets all documents of the current container. */ - public abstract scanAsync(): Promise; + /** + * Gets all documents of the current container. + * @param cursor ToDo. + * */ + public abstract scanAsync(cursor?: string): Promise>; } diff --git a/projects/aas-server/src/app/aas-provider/aas-server-scan.ts b/projects/aas-server/src/app/aas-provider/aas-server-scan.ts index b34f2acb..5fc28902 100644 --- a/projects/aas-server/src/app/aas-provider/aas-server-scan.ts +++ b/projects/aas-server/src/app/aas-provider/aas-server-scan.ts @@ -11,6 +11,7 @@ import { Logger } from '../logging/logger.js'; import { AASApiClient } from '../packages/aas-server/aas-api-client.js'; import { AASServerPackage } from '../packages/aas-server/aas-server-package.js'; import { AASResourceScan } from './aas-resource-scan.js'; +import { PagedResult } from '../types/paged-result.js'; export class AASServerScan extends AASResourceScan { private readonly logger: Logger; @@ -23,11 +24,12 @@ export class AASServerScan extends AASResourceScan { this.server = server; } - public async scanAsync(): Promise { + public async scanAsync(cursor?: string): Promise> { try { await this.server.openAsync(); const documents: AASDocument[] = []; - const ids = new Set(await this.server.getShellsAsync()); + const result = await this.server.getShellsAsync(cursor); + const ids = new Set(result.result); for (const id of ids) { try { const aasPackage = new AASServerPackage(this.logger, this.server, id); @@ -39,7 +41,7 @@ export class AASServerScan extends AASResourceScan { } } - return documents; + return { result: documents, paging_metadata: { cursor: result.paging_metadata.cursor } }; } finally { await this.server.closeAsync(); } diff --git a/projects/aas-server/src/app/aas-provider/directory-scan.ts b/projects/aas-server/src/app/aas-provider/directory-scan.ts index 65b418f7..e3ffddbd 100644 --- a/projects/aas-server/src/app/aas-provider/directory-scan.ts +++ b/projects/aas-server/src/app/aas-provider/directory-scan.ts @@ -11,6 +11,7 @@ import { Logger } from '../logging/logger.js'; import { AasxPackage } from '../packages/file-system/aasx-package.js'; import { AasxDirectory } from '../packages/file-system/aasx-directory.js'; import { AASResourceScan } from './aas-resource-scan.js'; +import { PagedResult } from '../types/paged-result.js'; export class DirectoryScan extends AASResourceScan { public constructor( @@ -20,12 +21,12 @@ export class DirectoryScan extends AASResourceScan { super(); } - public async scanAsync(): Promise { + public async scanAsync(cursor?: string): Promise> { try { await this.source.openAsync(); - const files = await this.source.getFiles(); + const result = await this.source.getFiles(cursor); const documents: AASDocument[] = []; - for (const file of files) { + for (const file of result.result) { try { const aasxPackage = new AasxPackage(this.logger, this.source, file); const document = await aasxPackage.createDocumentAsync(); @@ -36,7 +37,7 @@ export class DirectoryScan extends AASResourceScan { } } - return documents; + return { result: documents, paging_metadata: { cursor: result.paging_metadata.cursor } }; } finally { await this.source.closeAsync(); } diff --git a/projects/aas-server/src/app/aas-provider/opcua-server-scan.ts b/projects/aas-server/src/app/aas-provider/opcua-server-scan.ts index fb39e0e0..0c405de6 100644 --- a/projects/aas-server/src/app/aas-provider/opcua-server-scan.ts +++ b/projects/aas-server/src/app/aas-provider/opcua-server-scan.ts @@ -7,12 +7,13 @@ *****************************************************************************/ import { AttributeIds, BrowseDescriptionLike, QualifiedName, ReferenceDescription } from 'node-opcua'; -import { AASDocument } from 'aas-core'; +import { AASDocument, noop } from 'aas-core'; import { Logger } from '../logging/logger.js'; import { OpcuaDataTypeDictionary } from '../packages/opcua/opcua-data-type-dictionary.js'; import { OpcuaClient } from '../packages/opcua/opcua-client.js'; import { OpcuaPackage } from '../packages/opcua/opcua-package.js'; import { AASResourceScan } from './aas-resource-scan.js'; +import { PagedResult } from '../types/paged-result.js'; export class OpcuaServerScan extends AASResourceScan { private readonly logger: Logger; @@ -25,8 +26,9 @@ export class OpcuaServerScan extends AASResourceScan { this.server = server; } - public async scanAsync(): Promise { + public async scanAsync(cursor?: string): Promise> { try { + noop(cursor); await this.server.openAsync(); const documents: AASDocument[] = []; const dataTypes = new OpcuaDataTypeDictionary(); @@ -43,7 +45,7 @@ export class OpcuaServerScan extends AASResourceScan { } } - return documents; + return { result: documents, paging_metadata: {} }; } finally { await this.server.closeAsync(); } diff --git a/projects/aas-server/src/app/aas-provider/parallel.ts b/projects/aas-server/src/app/aas-provider/parallel.ts index 052e84b3..f10a9660 100644 --- a/projects/aas-server/src/app/aas-provider/parallel.ts +++ b/projects/aas-server/src/app/aas-provider/parallel.ts @@ -53,10 +53,16 @@ class WorkerTask extends EventEmitter { private workerOnMessage = (value: Uint8Array) => { const result: ScanResult = JSON.parse(Buffer.from(value).toString()); - if (result.type === ScanResultType.End) { - this.emit('end', this, result); - } else { - this.emit('message', result); + switch (result.type) { + case ScanResultType.End: + this.emit('end', result, this); + break; + case ScanResultType.NextPage: + this.emit('nextPage', result, this); + break; + default: + this.emit('message', result); + break; } }; @@ -95,6 +101,7 @@ export class Parallel extends EventEmitter { public execute(data: WorkerData): void { const task = new WorkerTask(data); task.on('message', this.taskOnMessage); + task.on('nextPage', this.taskOnNextPage); task.on('end', this.taskOnEnd); task.on('error', this.taskOnError); @@ -127,7 +134,11 @@ export class Parallel extends EventEmitter { this.emit('message', result); }; - private taskOnEnd = (task: WorkerTask, result: ScanResult) => { + private taskOnNextPage = (result: ScanResult, task: WorkerTask) => { + this.emit('nextPage', result, task.worker); + }; + + private taskOnEnd = (result: ScanResult, task: WorkerTask) => { this.emit('end', result); if (task) { diff --git a/projects/aas-server/src/app/aas-provider/scan-result.ts b/projects/aas-server/src/app/aas-provider/scan-result.ts index 3ba9db13..ce2262ed 100644 --- a/projects/aas-server/src/app/aas-provider/scan-result.ts +++ b/projects/aas-server/src/app/aas-provider/scan-result.ts @@ -6,13 +6,14 @@ * *****************************************************************************/ -import { AASDocument, AASContainer, Message, TemplateDescriptor } from 'aas-core'; +import { AASDocument, Message, TemplateDescriptor, AASEndpoint } from 'aas-core'; export enum ScanResultType { Added, Removed, Changed, Update, + NextPage, End, } @@ -26,15 +27,21 @@ export interface ScanResult { messages?: Message[]; } -/** The result of a container scan. */ -export interface ScanContainerResult extends ScanResult { - /** The AAS container. */ - container: AASContainer; - /** The result subject. */ +/** The result of an endpoint scan. */ +export interface ScanEndpointResult extends ScanResult { + endpoint: AASEndpoint; + documents: AASDocument[]; + cursor?: string; document: AASDocument; } -/** The result of a container scan. */ +/** The result of a template scan. */ export interface ScanTemplatesResult extends ScanResult { templates: TemplateDescriptor[]; } + +/** The result for a next page request. */ +export interface ScanNextPageResult extends ScanResult { + endpoint: AASEndpoint; + cursor: string; +} diff --git a/projects/aas-server/src/app/aas-provider/task-handler.ts b/projects/aas-server/src/app/aas-provider/task-handler.ts index 3ca0cf51..760ec767 100644 --- a/projects/aas-server/src/app/aas-provider/task-handler.ts +++ b/projects/aas-server/src/app/aas-provider/task-handler.ts @@ -9,9 +9,9 @@ import { singleton } from 'tsyringe'; export interface Task { - name: string; + endpointName: string; owner: object; - type: 'ScanContainer' | 'ScanTemplates'; + type: 'ScanEndpoint' | 'ScanTemplates'; } @singleton() @@ -34,7 +34,7 @@ export class TaskHandler { public empty(owner: object, name?: string): boolean { for (const task of this.tasks.values()) { - if (task.owner === owner && (!name || task.name === name)) { + if (task.owner === owner && (!name || task.endpointName === name)) { return false; } } diff --git a/projects/aas-server/src/app/aas-provider/worker-data.ts b/projects/aas-server/src/app/aas-provider/worker-data.ts index a8757836..c34b749f 100644 --- a/projects/aas-server/src/app/aas-provider/worker-data.ts +++ b/projects/aas-server/src/app/aas-provider/worker-data.ts @@ -6,24 +6,26 @@ * *****************************************************************************/ -import { AASContainer } from 'aas-core'; +import { AASDocument, AASEndpoint } from 'aas-core'; export interface WorkerData { taskId: number; - type: 'ScanContainerData' | 'ScanTemplatesData'; + type: 'ScanEndpointData' | 'ScanTemplatesData'; } -export interface ScanContainerData extends WorkerData { - type: 'ScanContainerData'; - container: AASContainer; +export interface ScanEndpointData extends WorkerData { + type: 'ScanEndpointData'; + endpoint: AASEndpoint; + documents: AASDocument[]; + cursor?: string; } export interface ScanTemplatesData extends WorkerData { type: 'ScanTemplatesData'; } -export function isScanContainerData(data: WorkerData): data is ScanContainerData { - return data.type === 'ScanContainerData'; +export function isScanEndpointData(data: WorkerData): data is ScanEndpointData { + return data.type === 'ScanEndpointData'; } export function isScanTemplatesData(data: WorkerData): data is ScanTemplatesData { diff --git a/projects/aas-server/src/app/scan-container.ts b/projects/aas-server/src/app/endpoint-scan.ts similarity index 69% rename from projects/aas-server/src/app/scan-container.ts rename to projects/aas-server/src/app/endpoint-scan.ts index e96a99b8..a2b1ca52 100644 --- a/projects/aas-server/src/app/scan-container.ts +++ b/projects/aas-server/src/app/endpoint-scan.ts @@ -10,15 +10,15 @@ import { inject, singleton } from 'tsyringe'; import { parentPort } from 'worker_threads'; import { Logger } from './logging/logger.js'; import { AASDocument } from 'aas-core'; -import { ScanContainerData } from './aas-provider/worker-data.js'; -import { ScanContainerResult, ScanResultType } from './aas-provider/scan-result.js'; +import { ScanEndpointData } from './aas-provider/worker-data.js'; +import { ScanEndpointResult, ScanResultType } from './aas-provider/scan-result.js'; import { toUint8Array } from './convert.js'; import { AASResourceScanFactory } from './aas-provider/aas-resource-scan-factory.js'; import { Variable } from './variable.js'; @singleton() -export class ScanContainer { - private data!: ScanContainerData; +export class EndpointScan { + private data!: ScanEndpointData; public constructor( @inject('Logger') private readonly logger: Logger, @@ -26,15 +26,15 @@ export class ScanContainer { @inject(Variable) private readonly variable: Variable, ) {} - public async scanAsync(data: ScanContainerData): Promise { + public async scanAsync(data: ScanEndpointData): Promise { this.data = data; - let documents: AASDocument[]; - const scan = this.resourceScanFactory.create(data.container); + const scan = this.resourceScanFactory.create(data.endpoint); try { scan.on('scanned', this.onDocumentScanned); scan.on('error', this.onError); - documents = await scan.scanAsync(); - this.computeDeleted(documents); + const result = await scan.scanAsync(data.cursor); + this.computeDeleted(result.result); + return result.paging_metadata.cursor; } finally { scan.off('scanned', this.onDocumentScanned); scan.off('error', this.onError); @@ -42,10 +42,12 @@ export class ScanContainer { } private computeDeleted(documents: AASDocument[]): void { - if (!this.data.container.documents) return; + if (this.data.documents === undefined) { + return; + } const current = new Map(documents.map(item => [item.id, item])); - for (const document of this.data.container.documents) { + for (const document of this.data.documents) { if (!current.has(document.id)) { this.postDeleted(document); } @@ -53,7 +55,7 @@ export class ScanContainer { } private onDocumentScanned = (document: AASDocument): void => { - const reference = this.data.container.documents?.find(item => item.id === document.id); + const reference = this.data.documents?.find(item => item.id === document.id); if (reference) { if (this.documentChanged(document, reference)) { this.postChanged(document); @@ -68,10 +70,12 @@ export class ScanContainer { }; private postChanged(document: AASDocument): void { - const value: ScanContainerResult = { + const value: ScanEndpointResult = { taskId: this.data.taskId, type: ScanResultType.Changed, - container: this.data.container, + endpoint: this.data.endpoint, + documents: this.data.documents, + cursor: this.data.cursor, document: document, }; @@ -80,10 +84,12 @@ export class ScanContainer { } private postDeleted(document: AASDocument): void { - const value: ScanContainerResult = { + const value: ScanEndpointResult = { taskId: this.data.taskId, type: ScanResultType.Removed, - container: this.data.container, + endpoint: this.data.endpoint, + documents: this.data.documents, + cursor: this.data.cursor, document: document, }; @@ -92,10 +98,12 @@ export class ScanContainer { } private postAdded(document: AASDocument): void { - const value: ScanContainerResult = { + const value: ScanEndpointResult = { taskId: this.data.taskId, type: ScanResultType.Added, - container: this.data.container, + endpoint: this.data.endpoint, + documents: this.data.documents, + cursor: this.data.cursor, document: document, }; @@ -113,17 +121,4 @@ export class ScanContainer { return true; } - - // private equalContent(a: aas.Environment | null, b: aas.Environment | null): boolean { - // let equals: boolean; - // if (a === b) { - // equals = true; - // } else if (a !== null && b !== null) { - // equals = isDeepEqual(a, b); - // } else { - // equals = false; - // } - - // return equals; - // } } diff --git a/projects/aas-server/src/app/http-client.ts b/projects/aas-server/src/app/http-client.ts index 6b42cc00..ced44f64 100644 --- a/projects/aas-server/src/app/http-client.ts +++ b/projects/aas-server/src/app/http-client.ts @@ -7,11 +7,11 @@ *****************************************************************************/ import http from 'http'; -import https from 'https'; import net from 'net'; import FormData from 'form-data'; -import { parseUrl } from './convert.js'; +import axios, { AxiosResponse } from 'axios'; import { singleton } from 'tsyringe'; +import { parseUrl } from './convert.js'; @singleton() export class HttpClient { @@ -22,43 +22,12 @@ export class HttpClient { * @param headers Additional outgoing http headers. * @returns The requested object. */ - public get(url: URL, headers: http.OutgoingHttpHeaders = {}): Promise { - return new Promise((result, reject) => { - const options: http.RequestOptions = { - host: url.hostname, - port: url.port, - path: url.pathname + url.search, - method: 'GET', - headers: { - connection: 'keep-alive', - ...headers, - }, - }; - - const requester = url.protocol === 'https:' ? https.request : http.request; - const request = requester(options, response => { - let data = ''; - response.on('data', (chunk: string) => { - data += chunk; - }); - - response.on('end', () => { - try { - HttpClient.checkStatusCode(response, data); - result(JSON.parse(data)); - } catch (error) { - reject(error); - } - }); - - response.on('error', error => reject(error)); - }); - - request - .on('timeout', () => request.destroy()) - .on('error', error => reject(error)) - .end(); + public async get(url: URL, headers?: Record): Promise { + const response: AxiosResponse = await axios.get(url.toString(), { + headers, }); + + return response.data; } /** @@ -67,27 +36,13 @@ export class HttpClient { * @param headers Additional outgoing http headers. * @returns The request. */ - public getResponse(url: URL, headers?: http.OutgoingHttpHeaders): Promise { - return new Promise((result, reject) => { - const options: http.RequestOptions = { - host: url.hostname, - port: url.port, - path: url.pathname + url.search, - method: 'GET', - timeout: 3000, - }; - - if (headers) { - options.headers = { ...headers }; - } - - const requester = url.protocol === 'https:' ? https.request : http.request; - const request = requester(options, response => result(response)); - request - .on('timeout', () => request.destroy()) - .on('error', error => reject(error)) - .end(); + public async getResponse(url: URL, headers?: Record): Promise { + const response = await axios.get(url.toString(), { + headers, + responseType: 'stream', }); + + return response.data; } /** @@ -96,43 +51,18 @@ export class HttpClient { * @param obj The object to send. * @param headers Additional outgoing http headers. */ - public put(url: URL, obj: object, headers: http.OutgoingHttpHeaders = {}): Promise { - return new Promise((result, reject) => { - const data = JSON.stringify(obj); - const options: http.RequestOptions = { - hostname: url.hostname, - port: url.port, - path: url.pathname + url.search, - method: 'PUT', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(data), - ...headers, - }, - }; - - const requester = url.protocol === 'https:' ? https.request : http.request; - const request = requester(options, response => { - let responseData = ''; - response.on('data', (chunk: string) => { - responseData += chunk; - }); - - response.on('end', () => { - try { - HttpClient.checkStatusCode(response, responseData); - result(responseData); - } catch (error) { - reject(error); - } - }); + public async put(url: URL, obj: object, headers?: Record): Promise { + const response = await axios.put(url.toString(), { + obj, + headers, + }); - response.on('error', error => reject(error)); - }).on('error', error => reject(error)); + const data = response.data; + if (typeof data === 'string') { + return data; + } - request.write(data); - request.end(); - }); + return response.statusText; } /** @@ -141,7 +71,7 @@ export class HttpClient { * @param obj The object to send. * @param headers Additional outgoing http headers. */ - public post(url: URL, obj: FormData | object, headers?: http.OutgoingHttpHeaders): Promise { + public post(url: URL, obj: FormData | object, headers?: Record): Promise { return obj instanceof FormData ? this.postFormData(url, obj, headers) : this.postObject(url, obj, headers); } @@ -150,40 +80,17 @@ export class HttpClient { * @param url The URL of the object to delete. * @param headers Additional outgoing http headers. */ - public delete(url: URL, headers: http.OutgoingHttpHeaders = {}): Promise { - return new Promise((result, reject) => { - const options: https.RequestOptions = { - hostname: url.hostname, - port: url.port, - path: url.pathname + url.search, - method: 'DELETE', - headers: { - 'Content-Type': 'application/json', - ...headers, - }, - }; - - const requester = url.protocol === 'https:' ? https.request : http.request; - requester(options, response => { - let responseData = ''; - response.on('data', (chunk: string) => { - responseData += chunk; - }); + public async delete(url: URL, headers?: Record): Promise { + const response = await axios.delete(url.toString(), { + headers, + }); - response.on('end', function () { - try { - HttpClient.checkStatusCode(response, responseData); - result(responseData); - } catch (error) { - reject(error); - } - }); + const data = response.data; + if (typeof data === 'string') { + return data; + } - response.on('error', error => reject(error)); - }) - .on('error', error => reject(error)) - .end(); - }); + return response.statusText; } /** @@ -196,110 +103,42 @@ export class HttpClient { const port = Number(temp.port ? temp.port : temp.protocol === 'http:' ? 80 : 443); const socket = net.createConnection(port, temp.hostname); socket.setTimeout(3000); - socket.on('connect', () => { - socket.end(); - }); - - socket.on('end', () => { - socket.destroy(); - resolve(); - }); - - socket.on('timeout', () => { - socket.destroy(); - reject(new Error(`${url} does not exist.`)); - }); - - socket.on('error', () => { - socket.destroy(); - reject(new Error(`${url} does not exist.`)); - }); - }); - } - - private postObject(url: URL, obj: object, headers: http.OutgoingHttpHeaders = {}): Promise { - return new Promise((result, reject) => { - const data = JSON.stringify(obj); - const options: http.RequestOptions = { - hostname: url.hostname, - port: url.port, - path: url.pathname + url.search, - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(data), - ...headers, - }, - }; - - const requester = url.protocol === 'https:' ? https.request : http.request; - const request = requester(options, response => { - let responseData = ''; - response.on('data', (chunk: string) => { - responseData += chunk; - }); - - response.on('end', () => { - try { - HttpClient.checkStatusCode(response, responseData); - result(responseData); - } catch (error) { - reject(error); - } + socket + .on('connect', () => { + socket.end(); + }) + .on('end', () => { + socket.destroy(); + resolve(); + }) + .on('timeout', () => { + socket.destroy(); + reject(new Error(`${url} does not exist.`)); + }) + .on('error', () => { + socket.destroy(); + reject(new Error(`${url} does not exist.`)); }); - - response.on('error', error => reject(error)); - }).on('error', error => reject(error)); - - request.write(data); - request.end(); }); } - private postFormData(url: URL, formData: FormData, headers: http.OutgoingHttpHeaders = {}): Promise { - return new Promise((result, reject) => { - const options: http.RequestOptions = { - hostname: url.hostname, - port: url.port, - path: url.pathname + url.search, - method: 'POST', - headers: { ...formData.getHeaders(), ...headers }, - }; - - const requester = url.protocol === 'https:' ? https.request : http.request; - const request = requester(options, response => { - let responseData = ''; - response.on('data', (chunk: string) => { - responseData += chunk; - }); - - response.on('end', function () { - try { - HttpClient.checkStatusCode(response, responseData); - result(responseData); - } catch (error) { - reject(error); - } - }); - - response.on('error', error => reject(error)); - }).on('error', error => reject(error)); + private async postObject(url: URL, obj: object, headers: Record | undefined): Promise { + const response = await axios.post(url.toString(), obj, { headers }); + const data = response.data; + if (typeof data === 'string') { + return data; + } - formData.pipe(request); - }); + return response.statusText; } - private static checkStatusCode(response: http.IncomingMessage, data?: string): void { - if (!response.statusCode) { - throw new Error(data ? `Unknown status code: ${data}` : 'Unknown status code.'); + private async postFormData(url: URL, formData: FormData, headers?: Record): Promise { + const response = await axios.post(url.toString(), formData, { headers }); + const data = response.data; + if (typeof data === 'string') { + return data; } - if (response.statusCode < 200 || response.statusCode >= 300) { - throw new Error( - data - ? `(${response.statusCode}) ${response.statusMessage}: ${data}` - : `(${response.statusCode}) ${response.statusMessage}.`, - ); - } + return response.statusText; } } diff --git a/projects/aas-server/src/app/packages/aas-resource.ts b/projects/aas-server/src/app/packages/aas-resource.ts index f1b4f6bf..381eb6bc 100644 --- a/projects/aas-server/src/app/packages/aas-resource.ts +++ b/projects/aas-server/src/app/packages/aas-resource.ts @@ -100,7 +100,14 @@ export abstract class AASResource { * @param url The URL. * @returns A new URL. */ - protected resolve(url: string): URL { - return new URL(url, this.endpoint.url); + protected resolve(url: string, searchParams?: Record): URL { + const resolvedUrl = new URL(url, this.endpoint.url); + if (searchParams) { + for (const name in searchParams) { + resolvedUrl.searchParams.set(name, String(searchParams[name])); + } + } + + return resolvedUrl; } } diff --git a/projects/aas-server/src/app/packages/aas-server/aas-api-client-v0.ts b/projects/aas-server/src/app/packages/aas-server/aas-api-client-v0.ts index 997bd2f0..72cec3ca 100644 --- a/projects/aas-server/src/app/packages/aas-server/aas-api-client-v0.ts +++ b/projects/aas-server/src/app/packages/aas-server/aas-api-client-v0.ts @@ -6,13 +6,14 @@ * *****************************************************************************/ -import { aas, AASEndpoint, DifferenceItem, selectSubmodel } from 'aas-core'; +import { aas, AASEndpoint, DifferenceItem, noop, selectSubmodel } from 'aas-core'; import { Logger } from '../../logging/logger.js'; import { JsonReaderV2 } from '../json-reader-v2.js'; import { AASApiClient } from './aas-api-client.js'; import { JsonWriterV2 } from '../json-writer-v2.js'; import * as aasV2 from '../../types/aas-v2.js'; import { HttpClient } from '../../http-client.js'; +import { PagedResult } from '../../types/paged-result.js'; interface AASList { aaslist: string[]; @@ -29,9 +30,12 @@ export class AASApiClientV0 extends AASApiClient { public readonly onlineReady = true; - public async getShellsAsync(): Promise { + public async getShellsAsync(cursor?: string): Promise> { const value = await this.http.get(this.resolve('/server/listaas')); - return value.aaslist.map(item => item.split(' : ')[1].trim()); + + noop(cursor); + + return { result: value.aaslist.map(item => item.split(' : ')[1].trim()), paging_metadata: {} }; } public override async readEnvironmentAsync(id: string): Promise { diff --git a/projects/aas-server/src/app/packages/aas-server/aas-api-client-v1.ts b/projects/aas-server/src/app/packages/aas-server/aas-api-client-v1.ts index 82e97431..803cc8b7 100644 --- a/projects/aas-server/src/app/packages/aas-server/aas-api-client-v1.ts +++ b/projects/aas-server/src/app/packages/aas-server/aas-api-client-v1.ts @@ -17,6 +17,7 @@ import { isAssetAdministrationShell, isSubmodel, isSubmodelElement, + noop, selectSubmodel, } from 'aas-core'; @@ -29,6 +30,7 @@ import { JsonWriterV2 } from '../json-writer-v2.js'; import { ERRORS } from '../../errors.js'; import { JsonReaderV3 } from '../json-reader-v3.js'; import { HttpClient } from '../../http-client.js'; +import { PagedResult } from '../../types/paged-result.js'; interface PackageDescriptor { aasIds: string[]; @@ -71,13 +73,15 @@ export class AASApiClientV1 extends AASApiClient { public readonly onlineReady = true; - public async getShellsAsync(): Promise { + public async getShellsAsync(cursor?: string): Promise> { const result = await this.http.get( this.resolve('shells'), this.endpoint.headers, ); - return result.map(shell => shell.identification.id); + noop(cursor); + + return { result: result.map(shell => shell.identification.id), paging_metadata: {} }; } public async readEnvironmentAsync(id: string): Promise { diff --git a/projects/aas-server/src/app/packages/aas-server/aas-api-client-v3.ts b/projects/aas-server/src/app/packages/aas-server/aas-api-client-v3.ts index 8252d4f2..04801c21 100644 --- a/projects/aas-server/src/app/packages/aas-server/aas-api-client-v3.ts +++ b/projects/aas-server/src/app/packages/aas-server/aas-api-client-v3.ts @@ -9,12 +9,6 @@ import FormData from 'form-data'; import cloneDeep from 'lodash-es/cloneDeep.js'; import { createReadStream } from 'fs'; -import { encodeBase64Url } from '../../convert.js'; -import { AASApiClient } from './aas-api-client.js'; -import { Logger } from '../../logging/logger.js'; -import { JsonReaderV3 } from '../json-reader-v3.js'; -import { JsonWriterV3 } from '../json-writer-v3.js'; -import { ERRORS } from '../../errors.js'; import { aas, AASEndpoint, @@ -26,7 +20,15 @@ import { isSubmodelElement, selectSubmodel, } from 'aas-core'; + +import { encodeBase64Url } from '../../convert.js'; +import { AASApiClient } from './aas-api-client.js'; +import { Logger } from '../../logging/logger.js'; +import { JsonReaderV3 } from '../json-reader-v3.js'; +import { JsonWriterV3 } from '../json-writer-v3.js'; +import { ERRORS } from '../../errors.js'; import { HttpClient } from '../../http-client.js'; +import { PagedResult } from '../../types/paged-result.js'; interface PackageDescriptor { aasIds: string[]; @@ -55,15 +57,6 @@ export interface OperationResult { inoutputVariables?: aas.OperationVariable[]; } -interface PagedResultPagingMetadata { - cursor?: string; -} - -interface PagedResult { - result: T[]; - paging_metadata: PagedResultPagingMetadata; -} - export class AASApiClientV3 extends AASApiClient { public constructor(logger: Logger, http: HttpClient, endpoint: AASEndpoint) { super(logger, http, endpoint); @@ -75,13 +68,21 @@ export class AASApiClientV3 extends AASApiClient { public readonly onlineReady = true; - public async getShellsAsync(): Promise { + public async getShellsAsync(cursor?: string): Promise> { + const searchParams: Record = { limit: 10 }; + if (cursor) { + searchParams.cursor = cursor; + } + const result = await this.http.get>( - this.resolve('shells'), + this.resolve('shells', searchParams), this.endpoint.headers, ); - return result.result.map(shell => shell.id); + return { + result: result.result.map(shell => shell.id), + paging_metadata: { cursor: result.paging_metadata.cursor }, + }; } public async readEnvironmentAsync(id: string): Promise { @@ -91,42 +92,34 @@ export class AASApiClientV3 extends AASApiClient { this.endpoint.headers, ); - const submodels: aas.Submodel[] = []; - if (shell.submodels) { - for (const reference of shell.submodels) { - const submodelId = encodeBase64Url(reference.keys[0].value); - try { - submodels.push( - await this.http.get( - this.resolve(`shells/${aasId}/submodels/${submodelId}`), - this.endpoint.headers, - ), - ); - } catch (error) { - this.logger.error(`Unable to read Submodel "${reference.keys[0].value}": ${error?.message}`); - } + const env: aas.Environment = { + assetAdministrationShells: [shell], + submodels: await this.readSubmodels(aasId, shell.submodels), + conceptDescriptions: [], + }; + + return new JsonReaderV3(env).readEnvironment(); + } + + public async readConceptDescriptionsAsync(): Promise { + const conceptDescriptions: aas.ConceptDescription[] = []; + let cursor: string | undefined; + const searchParams: Record = { limit: 100 }; + do { + if (cursor) { + searchParams.cursor = cursor; } - } - let conceptDescriptions: aas.ConceptDescription[]; - try { const pagedResult = await this.http.get>( this.resolve(`concept-descriptions`), this.endpoint.headers, ); - conceptDescriptions = pagedResult.result; - } catch { - conceptDescriptions = []; - } + conceptDescriptions.push(...pagedResult.result); + cursor = pagedResult.paging_metadata.cursor; + } while (cursor); - const sourceEnv: aas.Environment = { - assetAdministrationShells: [shell], - submodels, - conceptDescriptions, - }; - - return new JsonReaderV3(sourceEnv).readEnvironment(); + return conceptDescriptions; } public override getThumbnailAsync(id: string): Promise { @@ -181,10 +174,11 @@ export class AASApiClientV3 extends AASApiClient { return messages; } - public async openFileAsync(_: aas.AssetAdministrationShell, file: aas.File): Promise { + public async openFileAsync(shell: aas.AssetAdministrationShell, file: aas.File): Promise { + const aasId = encodeBase64Url(shell.id); const smId = encodeBase64Url(file.parent!.keys[0].value); const path = getIdShortPath(file); - const url = this.resolve(`submodels/${smId}/submodel-elements/${path}/attachment`); + const url = this.resolve(`shells/${aasId}/submodels/${smId}/submodel-elements/${path}/attachment`); return await this.http.getResponse(url, this.endpoint.headers); } @@ -278,6 +272,29 @@ export class AASApiClientV3 extends AASApiClient { return blob.value; } + private async readSubmodels(aasId: string, submodelRefs: aas.Reference[] | undefined): Promise { + const submodels: aas.Submodel[] = []; + if (submodelRefs === undefined) { + return submodels; + } + + for (const reference of submodelRefs) { + const smId = encodeBase64Url(reference.keys[0].value); + try { + submodels.push( + await this.http.get( + this.resolve(`shells/${aasId}/submodels/${smId}`), + this.endpoint.headers, + ), + ); + } catch (error) { + this.logger.error(`Unable to read Submodel "${reference.keys[0].value}": ${error?.message}`); + } + } + + return submodels; + } + private async putShellAsync(shell: aas.AssetAdministrationShell): Promise { const aasId = encodeBase64Url(shell.id); return await this.http.put(this.resolve(`shells/${aasId}`), new JsonWriterV3().convert(shell)); diff --git a/projects/aas-server/src/app/packages/aas-server/aas-api-client.ts b/projects/aas-server/src/app/packages/aas-server/aas-api-client.ts index 5645a83a..203e60e7 100644 --- a/projects/aas-server/src/app/packages/aas-server/aas-api-client.ts +++ b/projects/aas-server/src/app/packages/aas-server/aas-api-client.ts @@ -15,6 +15,7 @@ import { AASPackage } from '../aas-package.js'; import { AASResource } from '../aas-resource.js'; import { AASServerPackage } from './aas-server-package.js'; import { SocketSubscription } from '../../live/socket-subscription.js'; +import { PagedResult } from '../../types/paged-result.js'; interface PropertyValue { value: string; @@ -91,7 +92,7 @@ export abstract class AASApiClient extends AASResource { * Gets the names of the Asset Administration Shells contained in the current AASX server. * @returns The names of the AASs contained in the current AASX server. */ - public abstract getShellsAsync(): Promise; + public abstract getShellsAsync(cursor?: string): Promise>; /** * ToDo diff --git a/projects/aas-server/src/app/packages/file-system/aasx-directory.ts b/projects/aas-server/src/app/packages/file-system/aasx-directory.ts index 69b63e6d..823ccb6a 100644 --- a/projects/aas-server/src/app/packages/file-system/aasx-directory.ts +++ b/projects/aas-server/src/app/packages/file-system/aasx-directory.ts @@ -6,7 +6,7 @@ * *****************************************************************************/ -import { aas, AASEndpoint, ApplicationError } from 'aas-core'; +import { aas, AASEndpoint, ApplicationError, noop } from 'aas-core'; import { extname, join } from 'path/posix'; import { readFile } from 'fs/promises'; import { ERRORS } from '../../errors.js'; @@ -16,6 +16,7 @@ import { AASPackage } from '../aas-package.js'; import { AASResource } from '../aas-resource.js'; import { AasxPackage } from './aasx-package.js'; import { SocketSubscription } from '../../live/socket-subscription.js'; +import { PagedResult } from '../../types/paged-result.js'; export class AasxDirectory extends AASResource { private readonly root: string; @@ -46,10 +47,11 @@ export class AasxDirectory extends AASResource { return this.fileStorage.readFile(join(this.root, name)); } - public async getFiles(): Promise { + public async getFiles(cursor?: string): Promise> { + noop(cursor); const files: string[] = []; await this.readDirAsync(this.root, '', files); - return files; + return { result: files, paging_metadata: {} }; } public async testAsync(): Promise { diff --git a/projects/aas-server/src/app/template/template-storage.ts b/projects/aas-server/src/app/template/template-storage.ts index a9eb9e5c..2ef64b83 100644 --- a/projects/aas-server/src/app/template/template-storage.ts +++ b/projects/aas-server/src/app/template/template-storage.ts @@ -81,7 +81,7 @@ export class TemplateStorage { taskId, }; - this.taskHandler.set(taskId, { name: 'TemplateStorage', owner: this, type: 'ScanTemplates' }); + this.taskHandler.set(taskId, { endpointName: 'TemplateStorage', owner: this, type: 'ScanTemplates' }); this.parallel.execute(data); }; @@ -127,7 +127,7 @@ export class TemplateStorage { setTimeout(this.scanTemplates, this.timeout, result.taskId); if (result.messages) { - this.logger.start(`scan ${task?.name ?? 'undefined'}`); + this.logger.start(`scan ${task?.endpointName ?? 'undefined'}`); result.messages.forEach(message => this.logger.log(message)); this.logger.stop(); } diff --git a/projects/aas-server/src/app/types/paged-result.ts b/projects/aas-server/src/app/types/paged-result.ts new file mode 100644 index 00000000..91f4808a --- /dev/null +++ b/projects/aas-server/src/app/types/paged-result.ts @@ -0,0 +1,16 @@ +/****************************************************************************** + * + * Copyright (c) 2019-2024 Fraunhofer IOSB-INA Lemgo, + * eine rechtlich nicht selbstaendige Einrichtung der Fraunhofer-Gesellschaft + * zur Foerderung der angewandten Forschung e.V. + * + *****************************************************************************/ + +export interface PagedResultPagingMetadata { + cursor?: string; +} + +export interface PagedResult { + result: T[]; + paging_metadata: PagedResultPagingMetadata; +} diff --git a/projects/aas-server/src/app/worker-app.ts b/projects/aas-server/src/app/worker-app.ts index d2d68985..366b743e 100644 --- a/projects/aas-server/src/app/worker-app.ts +++ b/projects/aas-server/src/app/worker-app.ts @@ -9,17 +9,23 @@ import { inject, singleton } from 'tsyringe'; import { parentPort } from 'worker_threads'; import { Logger } from './logging/logger.js'; -import { WorkerData, isScanContainerData, isScanTemplatesData } from './aas-provider/worker-data.js'; -import { ScanResult, ScanResultType } from './aas-provider/scan-result.js'; +import { ScanNextPageResult, ScanResult, ScanResultType } from './aas-provider/scan-result.js'; import { toUint8Array } from './convert.js'; -import { ScanContainer } from './scan-container.js'; +import { EndpointScan } from './endpoint-scan.js'; import { TemplateScan } from './template/template-scan.js'; +import { + ScanEndpointData, + ScanTemplatesData, + WorkerData, + isScanEndpointData, + isScanTemplatesData, +} from './aas-provider/worker-data.js'; @singleton() export class WorkerApp { public constructor( @inject('Logger') private readonly logger: Logger, - @inject(ScanContainer) private readonly scanContainer: ScanContainer, + @inject(EndpointScan) private readonly endpointScan: EndpointScan, @inject(TemplateScan) private readonly templateScan: TemplateScan, ) {} @@ -28,16 +34,29 @@ export class WorkerApp { } private parentPortOnMessage = async (data: WorkerData) => { + if (parentPort === null) { + return; + } + try { this.logger.start(`Scan ${data.taskId}`); - if (isScanContainerData(data)) { - await this.scanContainer.scanAsync(data); + let result: ScanResult; + + if (isScanEndpointData(data)) { + result = await this.scanEndpoint(data); } else if (isScanTemplatesData(data)) { - await this.templateScan.scanAsync(data); + result = await this.scanTemplates(data); + } else { + result = { + taskId: data.taskId, + type: ScanResultType.End, + messages: this.logger.getMessages(), + }; } + + parentPort.postMessage(toUint8Array(result)); } catch (error) { this.logger.error(error); - } finally { this.logger.stop(); const result: ScanResult = { taskId: data.taskId, @@ -45,7 +64,38 @@ export class WorkerApp { messages: this.logger.getMessages(), }; - parentPort?.postMessage(toUint8Array(result)); + parentPort.postMessage(toUint8Array(result)); } }; + + private async scanEndpoint(data: ScanEndpointData): Promise { + const cursor = await this.endpointScan.scanAsync(data); + this.logger.stop(); + + if (cursor) { + return { + taskId: data.taskId, + type: ScanResultType.NextPage, + cursor, + messages: this.logger.getMessages(), + } as ScanNextPageResult; + } + + return { + taskId: data.taskId, + type: ScanResultType.End, + messages: this.logger.getMessages(), + } as ScanResult; + } + + private async scanTemplates(data: ScanTemplatesData): Promise { + await this.templateScan.scanAsync(data); + this.logger.stop(); + + return { + taskId: data.taskId, + type: ScanResultType.End, + messages: this.logger.getMessages(), + } as ScanResult; + } } diff --git a/projects/aas-server/src/test/aas-index/lowdb/lowdb-index.spec.ts b/projects/aas-server/src/test/aas-index/lowdb/lowdb-index.spec.ts index 5d25d9b1..732aed54 100644 --- a/projects/aas-server/src/test/aas-index/lowdb/lowdb-index.spec.ts +++ b/projects/aas-server/src/test/aas-index/lowdb/lowdb-index.spec.ts @@ -44,8 +44,8 @@ describe('LowDbIndex', () => { describe('getContainerDocuments', () => { it('returns all documents that belongs to a container', async () => { - const array = await index.getContainerDocuments('Samples'); - expect(array).toEqual(db.data.documents.filter(document => document.endpoint === 'Samples')); + const result = await index.nextPage('Samples', undefined); + expect(result.result).toEqual(db.data.documents.filter(document => document.endpoint === 'Samples')); }); }); diff --git a/projects/aas-server/src/test/packages/aas-server/aasx-package.spec.ts b/projects/aas-server/src/test/packages/aas-server/aasx-package.spec.ts index 361b73b2..1e13df72 100644 --- a/projects/aas-server/src/test/packages/aas-server/aasx-package.spec.ts +++ b/projects/aas-server/src/test/packages/aas-server/aasx-package.spec.ts @@ -22,7 +22,11 @@ describe('AasxPackage', function () { beforeEach(function () { logger = createSpyObj(['error', 'warning', 'info', 'debug', 'start', 'stop']); fileStorage = new LocalFileStorage('file:///samples', './src/test/assets/'); - source = new AasxDirectory(logger, fileStorage, 'file:///samples', 'Samples'); + source = new AasxDirectory(logger, fileStorage, { + url: 'file:///samples', + name: 'Samples', + type: 'FileSystem', + }); }); describe('createDocumentAsync', function () { diff --git a/projects/aas-server/src/test/packages/opcua/opcua-client.spec.ts b/projects/aas-server/src/test/packages/opcua/opcua-client.spec.ts index b3c2ecc6..b425ba34 100644 --- a/projects/aas-server/src/test/packages/opcua/opcua-client.spec.ts +++ b/projects/aas-server/src/test/packages/opcua/opcua-client.spec.ts @@ -23,7 +23,11 @@ describe('OpcuaClient', function () { beforeEach(function () { logger = createSpyObj(['error', 'warning', 'info', 'debug', 'start', 'stop']); - server = new OpcuaClient(logger, 'opc.tcp://localhost:1234/I4AASServer', 'OPCUA Server'); + server = new OpcuaClient(logger, { + url: 'opc.tcp://localhost:1234/I4AASServer', + name: 'OPCUA Server', + type: 'OPC_UA', + }); }); afterEach(() => { @@ -167,6 +171,7 @@ describe('OpcuaClient', function () { return new Promise(resolve => resolve(result)); }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any session.call.mockImplementation(call as any); const operation: aas.Operation = { idShort: 'add', @@ -208,6 +213,7 @@ describe('OpcuaClient', function () { return new Promise(resolve => resolve(result)); }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any session.call.mockImplementation(call as any); const operation: aas.Operation = { idShort: 'add', diff --git a/projects/aas-server/tsoa.json b/projects/aas-server/tsoa.json index 85b383fe..9e4e81aa 100644 --- a/projects/aas-server/tsoa.json +++ b/projects/aas-server/tsoa.json @@ -1,6 +1,6 @@ { "entryFile": "src/app/aas-server.ts", - "noImplicitAdditionalProperties": "silently-remove-extras", + "noImplicitAdditionalProperties": "ignore", "controllerPathGlobs": [ "src/app/controller/**/*controller.ts" ],