From 4382915fbc29b6b491f6bd5912b24f86caf894d8 Mon Sep 17 00:00:00 2001 From: jolevesq Date: Mon, 13 Jan 2025 16:29:35 -0500 Subject: [PATCH] Add fetch web worker --- .../src/core/workers/abstract-worker-pool.ts | 34 +++++++++ .../src/core/workers/abstract-worker.ts | 4 +- .../core/workers/fetch-esri-worker-pool.ts | 51 ++++++++++++++ .../core/workers/fetch-esri-worker-script.ts | 69 +++++++++++++++++++ .../src/core/workers/fetch-esri-worker.ts | 42 +++++++++++ .../src/core/workers/fetch-worker-type.ts | 10 +++ .../src/core/workers/json-export-script.ts | 10 +-- .../layer/gv-layers/raster/gv-esri-dynamic.ts | 33 ++++++++- .../hover-feature-info-layer-set.ts | 2 +- 9 files changed, 246 insertions(+), 9 deletions(-) create mode 100644 packages/geoview-core/src/core/workers/abstract-worker-pool.ts create mode 100644 packages/geoview-core/src/core/workers/fetch-esri-worker-pool.ts create mode 100644 packages/geoview-core/src/core/workers/fetch-esri-worker-script.ts create mode 100644 packages/geoview-core/src/core/workers/fetch-esri-worker.ts create mode 100644 packages/geoview-core/src/core/workers/fetch-worker-type.ts diff --git a/packages/geoview-core/src/core/workers/abstract-worker-pool.ts b/packages/geoview-core/src/core/workers/abstract-worker-pool.ts new file mode 100644 index 00000000000..fe866c33093 --- /dev/null +++ b/packages/geoview-core/src/core/workers/abstract-worker-pool.ts @@ -0,0 +1,34 @@ +import { AbstractWorker } from './abstract-worker'; + +export abstract class AbstractWorkerPool { + protected workers: AbstractWorker[] = []; + + protected busyWorkers = new Set>(); + + protected WorkerClass: new () => AbstractWorker; + + protected name: string; + + constructor(name: string, workerClass: new () => AbstractWorker, numWorkers = navigator.hardwareConcurrency || 4) { + this.name = name; + this.WorkerClass = workerClass; + this.initializeWorkers(numWorkers); + } + + protected initializeWorkers(numWorkers: number): void { + for (let i = 0; i < numWorkers; i++) { + const worker = new this.WorkerClass(); + this.workers.push(worker); + } + } + + public abstract init(): Promise; + + public abstract process(params: unknown): Promise; + + public terminate(): void { + this.workers.forEach((worker) => worker.terminate()); + this.workers = []; + this.busyWorkers.clear(); + } +} diff --git a/packages/geoview-core/src/core/workers/abstract-worker.ts b/packages/geoview-core/src/core/workers/abstract-worker.ts index 26369a7ec48..7a9312abc08 100644 --- a/packages/geoview-core/src/core/workers/abstract-worker.ts +++ b/packages/geoview-core/src/core/workers/abstract-worker.ts @@ -83,14 +83,14 @@ export abstract class AbstractWorker { * @param args - Arguments to pass to the worker for initialization. * @returns A promise that resolves when the worker is initialized. */ - protected abstract init(...args: unknown[]): Promise; + public abstract init(...args: unknown[]): Promise; /** * Process the worker. This method should be implemented by subclasses. * @param args - Arguments to pass to the worker for process. * @returns A promise that resolves when the worker is processed. */ - protected abstract process(...args: unknown[]): Promise; + public abstract process(...args: unknown[]): Promise; /** * Terminates the worker. diff --git a/packages/geoview-core/src/core/workers/fetch-esri-worker-pool.ts b/packages/geoview-core/src/core/workers/fetch-esri-worker-pool.ts new file mode 100644 index 00000000000..f5cd3dc8888 --- /dev/null +++ b/packages/geoview-core/src/core/workers/fetch-esri-worker-pool.ts @@ -0,0 +1,51 @@ +import { AbstractWorkerPool } from './abstract-worker-pool'; +import { FetchEsriWorker, FetchEsriWorkerType } from './fetch-esri-worker'; +import { QueryParams } from './fetch-esri-worker-script'; +import { createWorkerLogger } from './helper/logger-worker'; + +import { TypeJsonObject } from '@/api/config/types/config-types'; + +export class FetchEsriWorkerPool extends AbstractWorkerPool { + #logger = createWorkerLogger('FetchEsriWorkerPool'); + + constructor(numWorkers = navigator.hardwareConcurrency || 4) { + super('FetchEsriWorkerPool', FetchEsriWorker, numWorkers); + this.#logger.logInfo('Worker pool created', `Number of workers: ${numWorkers}`); + } + + public async init(): Promise { + try { + this.#logger.logTrace('Initializing worker pool'); + await Promise.all(this.workers.map((worker) => worker.init())); + this.#logger.logTrace('Worker pool initialized'); + } catch (error) { + this.#logger.logError('Worker pool initialization failed', error); + throw error; + } + } + + public async process(params: QueryParams): Promise { + const availableWorker = this.workers.find((w) => !this.busyWorkers.has(w)); + if (!availableWorker) { + throw new Error('No available workers'); + } + + const result = await availableWorker.process(params); + return result as TypeJsonObject; + } + + // /** + // * Process an ESRI query and transform features using a worker from the pool + // */ + // public async processQuery(params: QueryParams): Promise { + // try { + // this.#logger.logTrace('Starting query process', params.url); + // const result = await this.process(params); + // this.#logger.logTrace('Query process completed'); + // return result as TypeJsonObject; + // } catch (error) { + // this.#logger.logError('Query process failed', error); + // throw error; + // } + // } +} diff --git a/packages/geoview-core/src/core/workers/fetch-esri-worker-script.ts b/packages/geoview-core/src/core/workers/fetch-esri-worker-script.ts new file mode 100644 index 00000000000..c90fa5f13af --- /dev/null +++ b/packages/geoview-core/src/core/workers/fetch-esri-worker-script.ts @@ -0,0 +1,69 @@ +import { expose } from 'comlink'; + +import { createWorkerLogger } from './helper/logger-worker'; + +import { TypeJsonObject } from '@/api/config/types/config-types'; +import { TypeStyleGeometry } from '@/api/config/types/map-schema-types'; + +export interface QueryParams { + url: string; + geometryType: TypeStyleGeometry; + objectIds: number[]; + queryGeometry: boolean; + projection: number; + maxAllowableOffset: number; +} + +const logger = createWorkerLogger('FetchEsriWorker'); + +// Move the ESRI query function directly into the worker to avoid circular dependencies +async function queryEsriFeatures(params: QueryParams): Promise { + const response = await fetch(`${params.url}/query`, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: new URLSearchParams({ + f: 'json', + geometryType: params.geometryType, + objectIds: params.objectIds.join(','), + outFields: '*', + returnGeometry: params.queryGeometry.toString(), + outSR: params.projection.toString(), + maxAllowableOffset: params.maxAllowableOffset.toString(), + }), + }); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return response.json(); +} + +const worker = { + // eslint-disable-next-line require-await + async init(): Promise { + try { + logger.logTrace('init worker', 'FetchEsriWorker initialized'); + } catch { + logger.logError('init worker', 'FetchEsriWorker failed to initialize'); + } + }, + + async process(params: QueryParams): Promise { + try { + logger.logTrace('process worker - Starting query processing', params.url); + const response = await queryEsriFeatures(params); + logger.logDebug('process worker - Query completed'); + return response; + } catch (error) { + logger.logError('process worker - Query processing failed', error); + throw error; + } + }, +}; + +// Expose the worker methods to be accessible from the main thread +expose(worker); +export default {} as typeof Worker & { new (): Worker }; diff --git a/packages/geoview-core/src/core/workers/fetch-esri-worker.ts b/packages/geoview-core/src/core/workers/fetch-esri-worker.ts new file mode 100644 index 00000000000..3e9b6630fa7 --- /dev/null +++ b/packages/geoview-core/src/core/workers/fetch-esri-worker.ts @@ -0,0 +1,42 @@ +import { AbstractWorker } from './abstract-worker'; +import { QueryParams } from './fetch-esri-worker-script'; +import { TypeJsonObject } from '@/api/config/types/config-types'; + +export interface FetchEsriWorkerType { + /** + * Initializes the worker - empty for now. + */ + init: () => Promise; + + /** + * Processes an ESRI query JSON export. + * @param {QueryParams} queryParams - The query parameters for the fetch. + * @returns {TypeJsonObject} A promise that resolves to the response fetch as JSON string. + */ + process: (queryParams: QueryParams) => Promise; +} + +export class FetchEsriWorker extends AbstractWorker { + constructor() { + super('FetchEsriWorker', new Worker(new URL('./fetch-esri-worker-script.ts', import.meta.url))); + } + + /** + * Initializes the worker - empty for now. + * @returns A promise that resolves when initialization is complete. + */ + public async init(): Promise { + const result = await this.proxy.init(); + return result; + } + + /** + * Processes a JSON fetch for an esri query. + * @param {QueryParams} queryParams - The query parameters for the fetch. + * @returns A promise that resolves to the processed JSON string. + */ + public async process(queryParams: QueryParams): Promise { + const result = await this.proxy.process(queryParams); + return result; + } +} diff --git a/packages/geoview-core/src/core/workers/fetch-worker-type.ts b/packages/geoview-core/src/core/workers/fetch-worker-type.ts new file mode 100644 index 00000000000..0fa206e849c --- /dev/null +++ b/packages/geoview-core/src/core/workers/fetch-worker-type.ts @@ -0,0 +1,10 @@ +// import { TypeStyleGeometry } from '@/api/config/types/map-schema-types'; + +// export interface QueryParams { +// url: string; +// geometryType: TypeStyleGeometry; +// objectIds: number[]; +// queryGeometry: boolean; +// projection: number; +// maxAllowableOffset: number; +// } diff --git a/packages/geoview-core/src/core/workers/json-export-script.ts b/packages/geoview-core/src/core/workers/json-export-script.ts index 2b82b8952bd..0d869aa9eef 100644 --- a/packages/geoview-core/src/core/workers/json-export-script.ts +++ b/packages/geoview-core/src/core/workers/json-export-script.ts @@ -137,9 +137,9 @@ const worker = { try { sourceCRS = projectionInfo.sourceCRS; targetCRS = projectionInfo.targetCRS; - logger.logTrace('init', `Worker initialized with sourceCRS: ${sourceCRS}, targetCRS: ${targetCRS}`); + logger.logTrace('init worker', `Worker initialized with sourceCRS: ${sourceCRS}, targetCRS: ${targetCRS}`); } catch (error) { - logger.logError('init', error); + logger.logError('init worker', error); } }, @@ -151,7 +151,7 @@ const worker = { */ process(chunk: TypeWorkerExportChunk[], isFirst: boolean): string { try { - logger.logTrace('process', `Processing chunk of ${chunk.length} items`); + logger.logTrace('process worker', `Processing chunk of ${chunk.length} items`); let result = ''; if (isFirst) { result += '{"type":"FeatureCollection","features":['; @@ -171,10 +171,10 @@ const worker = { result += processedChunk.join(','); - logger.logTrace('process', `Finished processing`); + logger.logTrace('process worker', `Finished processing`); return result; } catch (error) { - logger.logError('process', error); + logger.logError('process worker', error); return ''; } }, diff --git a/packages/geoview-core/src/geo/layer/gv-layers/raster/gv-esri-dynamic.ts b/packages/geoview-core/src/geo/layer/gv-layers/raster/gv-esri-dynamic.ts index 47d21bf5129..2bfaa15fe0a 100644 --- a/packages/geoview-core/src/geo/layer/gv-layers/raster/gv-esri-dynamic.ts +++ b/packages/geoview-core/src/geo/layer/gv-layers/raster/gv-esri-dynamic.ts @@ -30,6 +30,8 @@ import { CONST_LAYER_TYPES } from '../../geoview-layers/abstract-geoview-layers' import { TypeLegend } from '@/core/stores/store-interface-and-intial-values/layer-state'; import { TypeEsriImageLayerLegend } from './gv-esri-image'; import { TypeJsonObject } from '@/api/config/types/config-types'; +import { FetchEsriWorkerPool } from '@/core/workers/fetch-esri-worker-pool'; +import { QueryParams } from '@/core/workers/fetch-esri-worker-script'; type TypeFieldOfTheSameValue = { value: string | number | Date; nbOccurence: number }; type TypeQueryTree = { fieldValue: string | number | Date; nextField: TypeQueryTree }[]; @@ -41,6 +43,8 @@ type TypeQueryTree = { fieldValue: string | number | Date; nextField: TypeQueryT * @class GVEsriDynamic */ export class GVEsriDynamic extends AbstractGVRaster { + #fetchWorkerPool: FetchEsriWorkerPool; + // The default hit tolerance the query should be using static override DEFAULT_HIT_TOLERANCE: number = 7; @@ -56,6 +60,10 @@ export class GVEsriDynamic extends AbstractGVRaster { public constructor(mapId: string, olSource: ImageArcGISRest, layerConfig: EsriDynamicLayerEntryConfig) { super(mapId, olSource, layerConfig); + // Setup the worker pool + this.#fetchWorkerPool = new FetchEsriWorkerPool(); + this.#fetchWorkerPool.init().then(() => logger.logTraceCore('Worker pool for fetch ESRI initialized')); + // TODO: Investigate to see if we can call the export map for the whole service at once instead of making many call // TODO.CONT: We can use the layers and layersDef parameters to set what should be visible. // TODO.CONT: layers=show:layerId ; layerDefs={ "layerId": "layer def" } @@ -261,6 +269,26 @@ export class GVEsriDynamic extends AbstractGVRaster { return this.getFeatureInfoAtLongLat(projCoordinate, queryGeometry); } + async yourQueryMethod(layerConfig: any, objectIds: number[], queryGeometry: boolean): Promise { + try { + const params: QueryParams = { + url: layerConfig.source.dataAccessPath + layerConfig.layerId, + geometryType: (layerConfig.getLayerMetadata()!.geometryType as string).replace('esriGeometry', ''), + objectIds, + queryGeometry, + projection: 3978, + maxAllowableOffset: 7000, + }; + + const response = await this.#fetchWorkerPool.process(params); + const features = new EsriJSON().readFeatures({ features: response }) as Feature[]; + return await this.formatFeatureInfoResult(features, layerConfig); + } catch (error) { + console.error('Query processing failed:', error); + throw error; + } + } + /** * Overrides the return of feature information at the provided long lat coordinate. * @param {Coordinate} lnglat - The coordinate that will be used by the query. @@ -325,6 +353,8 @@ export class GVEsriDynamic extends AbstractGVRaster { // Get meters per pixel to set the maxAllowableOffset to simplify return geometry const maxAllowableOffset = queryGeometry ? getMetersPerPixel(mapViewer, lnglat[1]) : 0; + this.yourQueryMethod(layerConfig, objectIds, true).then((features) => console.log('Features worker', features)); + // TODO: We need to separate the query attribute from geometry. We can use the attributes returned by identify to show details panel // TODO.CONT: or create 2 distinc query one for attributes and one for geometry. This way we can display the anel faster and wait later for geometry // TODO.CONT: We need to see if we can fetch in async mode without freezing the ui. If not we will need a web worker for the fetch. @@ -335,13 +365,14 @@ export class GVEsriDynamic extends AbstractGVRaster { (layerConfig.getLayerMetadata()!.geometryType as string).replace('esriGeometry', '') as TypeStyleGeometry, objectIds, '*', - queryGeometry, + false, mapViewer.getMapState().currentProjection, maxAllowableOffset, false ); // TODO: This is also time consuming, the creation of the feature can take several seconds, check web worker + // TODO.CONT: Because web worker can only use sereialize date and not object with function it may be difficult for this... // Transform the features in an OL feature const features = new EsriJSON().readFeatures({ features: response }) as Feature[]; const arrayOfFeatureInfoEntries = await this.formatFeatureInfoResult(features, layerConfig); diff --git a/packages/geoview-core/src/geo/layer/layer-sets/hover-feature-info-layer-set.ts b/packages/geoview-core/src/geo/layer/layer-sets/hover-feature-info-layer-set.ts index 9662e2051a4..3dfdd74927e 100644 --- a/packages/geoview-core/src/geo/layer/layer-sets/hover-feature-info-layer-set.ts +++ b/packages/geoview-core/src/geo/layer/layer-sets/hover-feature-info-layer-set.ts @@ -107,7 +107,7 @@ export class HoverFeatureInfoLayerSet extends AbstractLayerSet { // If layer was found if (layer && layer instanceof AbstractGVLayer) { // If state is not queryable - if (!AbstractLayerSet.isStateQueryable(layer)) return; + return; // if (!AbstractLayerSet.isStateQueryable(layer)) return; // Flag processing this.resultSet[layerPath].feature = undefined;