compatibility updates for api rest [email protected] (#227)

* #204 added preferences for admin area to cache stream connection information * #204 refactored some admin stream conf properties, added new admin prefs * #204 minor admin prefs refactoring * #204 * #204 added path/method props back in from 204.2 * #204 added getServerInfoMetadata method to admin service * #204 updates for changes in [email protected] * #226 rest-api-client-ng updated to 2.3.0 * #226 rest-api-client-ng updated to 2.3.0
senzing-garage · Aug 5, 2021 · 3ed74b6 · 3ed74b6
1 parent 201a04d
commit 3ed74b6
Show file tree

Hide file tree

Showing 11 changed files with 195 additions and 42 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 [markdownlint](https://dlaa.me/markdownlint/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.2.4] - 2021-08-04
+
+- [API Client package](https://github.com/Senzing/rest-api-client-ng/releases) updated to version [2.3.0](https://github.com/Senzing/rest-api-client-ng/releases/tag/2.3.0)
+- Compatibility updates for [Senzing OAS 2.7.0](https://github.com/Senzing/senzing-rest-api-specification/blob/master/CHANGELOG.md#270---2021-07-22) specification
+- Compatibility updates for [Senzing Rest [email protected]](https://github.com/Senzing/senzing-api-server/blob/master/CHANGELOG.md#270---2021-07-22)
+- Preferences has several models added to facilitate stateful storage of **Stream Loading** features found in the [POC Server] extensions:
+  - `AdminStreamConnProperties`, `AdminStreamAnalysisConfig`, `AdminStreamLoadConfig`, `AdminStreamUploadRates` in src/lib/models/data-admin.ts
+  - `SzAdminPrefs` added to `src/lib/services/sz-prefs.service.ts`
+
+relevant tickets: #204 #226
+
 ## [2.2.3] - 2021-06-14
 
 - Passing additional or custom headers are necessary in certain operation scenario's, ie: passing `X-Amz-Security-Token` to a [Cognito](https://aws.amazon.com/cognito/) enabled [API Gateway](https://aws.amazon.com/api-gateway/) address after user authentication.

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@senzing/sdk-components-ng",
-  "version": "2.2.3",
+  "version": "2.2.4",
   "scripts": {
     "ng": "ng",
     "start": "ng build @senzing/sdk-components-ng && ng serve",
@@ -59,7 +59,7 @@
     "@angular/platform-browser": "~10.0.9",
     "@angular/platform-browser-dynamic": "~10.0.9",
     "@angular/router": "~10.0.9",
-    "@senzing/rest-api-client-ng": "^2.2.2",
+    "@senzing/rest-api-client-ng": "^2.3.0",
     "@senzing/sdk-graph-components": "^2.2.0",
     "d3": "^5.9.1",
     "document-register-element": "^1.14.5",

diff --git a/src/lib/models/data-admin.ts b/src/lib/models/data-admin.ts
@@ -0,0 +1,31 @@
+export interface AdminStreamConnProperties {
+    connected: boolean;
+    clientId?: string;
+    hostname: string;
+    port?: number;
+    secure?: boolean;
+    connectionTest: boolean;
+    reconnectOnClose: boolean;
+    reconnectConsecutiveAttemptLimit: number;
+    path: string;
+    method?: string;
+}
+
+export interface AdminStreamAnalysisConfig {
+    sampleSize: number;
+    uploadRate: number;
+}
+export interface AdminStreamLoadConfig {
+    autoCreateMissingDataSources: boolean | undefined;
+    assignMissingDataSourceRecordsToStaticTarget?: string | undefined | boolean;
+    uploadRate: number;
+}
+export const AdminStreamUploadRates = {
+    unlimited: -1,
+    100: 100,
+    1000: 1000,
+    5000: 5000,
+    10000: 10000,
+    20000: 20000,
+    50000: 50000
+};
diff --git a/src/lib/sdk.module.ts b/src/lib/sdk.module.ts
@@ -30,6 +30,7 @@ import { MatDialogModule } from '@angular/material/dialog';
 // import { SzBulkLoadStatus } from './models/data-importing';
 // import { SzDataSourceRecordAnalysis } from './models/data-sources';
 // import { SzFolioItem, SzSearchParamsFolio, SzSearchParamsFolioItem } from './models/folio';
+// import { AdminStreamConnProperties } from './lib/models/data-admin';
 
 /** services */
 import { SzMessageBundleService } from './services/sz-message-bundle.service';

diff --git a/src/lib/services/sz-admin.service.ts b/src/lib/services/sz-admin.service.ts
@@ -2,13 +2,13 @@ import { Injectable } from '@angular/core';
 import { Observable, Subject, BehaviorSubject } from 'rxjs';
 import {
   AdminService, ConfigService,
-  Body,
-  Body2,
+  DatasourcesBody,
+  EntitytypesBody,
   SzAttributeClass, SzAttributeTypesResponse, SzAttributeTypesResponseData,
   SzEntityClassDescriptor,
   SzEntityTypeResponse, SzEntityTypeResponseData, SzEntityTypeDescriptor,
   SzDataSourcesResponse,
-  SzBaseResponse, SzBaseResponseMeta,
+  SzBaseResponse, SzMeta,
   SzBulkLoadResponse,
   SzLicenseResponse, SzLicenseInfo,
   SzVersionResponse, SzVersionInfo, SzAttributeTypeResponse, SzAttributeTypeResponseData,
@@ -95,7 +95,7 @@ export class SzAdminService {
   }
 
   /** get diagnostic information from the rest-api-server host */
-  public getHeartbeat(): Observable<SzBaseResponseMeta> {
+  public getHeartbeat(): Observable<SzMeta> {
     return this.adminService.heartbeat()
     .pipe(
       map( (resp: SzBaseResponse) => resp.meta )
@@ -125,7 +125,13 @@ export class SzAdminService {
       tap( (data: SzServerInfo ) => { this.serverInfo = data; })
     );
   }
-  public addDataSources(body?: Body | string, dataSource?: string[], withRaw?: boolean, observe?: 'body', reportProgress?: boolean): Observable<SzDataSourcesResponseData> {
+  public getServerInfoMetadata(): Observable<SzMeta> {
+    return this.adminService.getServerInfo()
+    .pipe(
+      map( (resp: SzServerInfoResponse) => resp.meta )
+    );
+  }
+  public addDataSources(body?: DatasourcesBody | string, dataSource?: string[], withRaw?: boolean, observe?: 'body', reportProgress?: boolean): Observable<SzDataSourcesResponseData> {
     if (!this.adminEnabled || this.readOnly) {
       throw new Error('admin operation not permitted.');
     }
@@ -149,7 +155,7 @@ export class SzAdminService {
       map( (resp: SzEntityClassesResponse) => resp.data )
     );*/
   }
-  public addEntityTypes(body?: Body2 | string, entityType?: string | string[], entityClass?: string, observe?: 'body', reportProgress?: boolean): Observable<string[]> {
+  public addEntityTypes(body?: EntitytypesBody | string, entityType?: string | string[], entityClass?: string, observe?: 'body', reportProgress?: boolean): Observable<string[]> {
     if (!this.adminEnabled || this.readOnly) {
       throw new Error('admin operation not permitted.');
     }
@@ -250,16 +256,16 @@ export class SzAdminService {
     if (!this.adminEnabled || this.readOnly) {
       throw new Error('admin operation not permitted.');
     }
-    return this.bulkDataService.analyzeBulkRecords(body, progressPeriod, observe, reportProgress)
+    return this.bulkDataService.analyzeBulkRecords(body, progressPeriod, undefined, observe, reportProgress)
     .pipe(
       map( (resp: SzBulkDataAnalysisResponse) => resp )
     );
   }
-  public loadBulkRecords(body: string | Blob | File | { [key: string]: any}[], dataSource?: string, mapDataSources?: string, mapDataSource?: string[], entityType?: string, mapEntityTypes?: string, mapEntityType?: string[], progressPeriod?: string, observe?: 'body', reportProgress?: boolean): Observable<SzBulkLoadResponse> {
+  public loadBulkRecords(body: string | Blob | File | { [key: string]: any}[], dataSource?: string, mapDataSources?: string, mapDataSource?: string[], entityType?: string, mapEntityTypes?: string, mapEntityType?: string[], maxFailures?: string | number, progressPeriod?: string, eofSendFileTimeout?: string | number, observe?: 'body', reportProgress?: boolean): Observable<SzBulkLoadResponse> {
     if (!this.adminEnabled || this.readOnly) {
       throw new Error('admin operation not permitted.');
     }
-    return this.bulkDataService.loadBulkRecords(body, dataSource, mapDataSources, mapDataSource, entityType, mapEntityTypes, mapEntityType, progressPeriod, observe, reportProgress);
+    return this.bulkDataService.loadBulkRecords(body, dataSource, mapDataSources, mapDataSource, entityType, mapEntityTypes, mapEntityType, maxFailures, progressPeriod, eofSendFileTimeout, observe, reportProgress);
   }
 
 }
diff --git a/src/lib/services/sz-bulk-data.service.ts b/src/lib/services/sz-bulk-data.service.ts
@@ -183,25 +183,6 @@ export class SzBulkDataService {
     console.log('SzBulkDataService.createEntityTypes: ', entityTypes);
     return this.entityTypesService.addEntityTypes(entityTypes, "ACTOR");
   }
-  /*
-  public analyzeBulkRecords(body: string | Blob | File | { [key: string]: any}[], progressPeriod?: string, observe?: 'body', reportProgress?: boolean): Observable<SzBulkDataAnalysisResponse> {
-    //public analyzeBulkRecords(body: string | Blob, observe?: "body", reportProgress?: boolean): Observable<SzBulkDataAnalysisResponse> {
-      if (!this.adminEnabled || this.readOnly) {
-        throw new Error('admin operation not permitted.');
-      }
-      return this.bulkDataService.analyzeBulkRecords(body, progressPeriod, observe, reportProgress)
-      .pipe(
-        map( (resp: SzBulkDataAnalysisResponse) => resp )
-      );
-    }
-    public loadBulkRecords(body: string | Blob | File | { [key: string]: any}[], dataSource?: string, mapDataSources?: string, mapDataSource?: string[], entityType?: string, mapEntityTypes?: string, mapEntityType?: string[], progressPeriod?: string, observe?: 'body', reportProgress?: boolean): Observable<SzBulkLoadResponse> {
-    //public loadBulkRecords(body: string | Blob, dataSource?: string, entityType?: string, observe?: "body", reportProgress?: boolean): Observable<SzBulkDataLoadResponse> {
-      if (!this.adminEnabled || this.readOnly) {
-        throw new Error('admin operation not permitted.');
-      }
-      return this.bulkDataService.loadBulkRecords(body, dataSource, mapDataSources, mapDataSource, entityType, mapEntityTypes, mapEntityType, progressPeriod, observe, reportProgress);
-  }
-  */
   /** analze a file and prep for mapping */
   public analyze(file: File): Observable<SzBulkDataAnalysisResponse> {
     //console.log('SzBulkDataService.analyze: ', file);
@@ -270,10 +251,7 @@ export class SzBulkDataService {
       // no new datasources or already avail
       this.loadingFile.next(true);
       promise.then(() => {
-        //this.bulkDataService.loadBulkRecords(file, dataSource?: string, mapDataSources?: string, mapDataSource?: Array<string>, entityType?: string, mapEntityTypes?: string, mapEntityType?: Array<string>, progressPeriod?: string, observe?: 'body', reportProgress?: boolean)
-        //this.bulkDataService.loadBulkRecords(file, dataSource, mapDataSources, mapDataSource, entityType?: string, mapEntityTypes, mapEntityType, progressPeriod, observe, reportProgress)
         this.bulkDataService.loadBulkRecords  (file, undefined,  JSON.stringify(dataSourceMap),  undefined,     undefined,           JSON.stringify(entityTypeMap)).pipe(
-        //this.bulkDataService.loadBulkRecords(file, dataSourceMap, entityTypeMap ).pipe(
           catchError((err: Error) => {
             console.warn('Handling error locally and rethrowing it...', err);
             this.loadingFile.next(false);

diff --git a/src/lib/services/sz-prefs.service.ts b/src/lib/services/sz-prefs.service.ts
@@ -3,6 +3,8 @@ import { Subject, BehaviorSubject, merge, timer } from 'rxjs';
 import { takeUntil, debounce } from 'rxjs/operators';
 import { SzDataSourceComposite } from '../models/data-sources';
 import { SzSearchHistoryFolio, SzSearchHistoryFolioItem, SzSearchParamsFolio } from '../models/folio';
+import { AdminStreamAnalysisConfig, AdminStreamConnProperties, AdminStreamLoadConfig } from '../models/data-admin';
+//import { Configuration as SzRestConfiguration, ConfigurationParameters as SzRestConfigurationParameters } from '@senzing/rest-api-client-ng';
 
 /**
  * preferences bus base class. provides common methods for
@@ -71,6 +73,115 @@ export class SzSdkPrefsBase {
   }
 }
 
+/**
+ * search form related preferences bus class.
+ * used by {@link SzPrefsService} to store it's
+ * admin area related prefs.
+ * Should really be used from {@link SzPrefsService} context, not on its own.
+ *
+ * @example
+ * this.prefs.admin.streamConnectionProperties = {
+    connected: boolean;
+    clientId?: string;
+    hostname: string;
+    sampleSize: number;
+    port?: number;
+    connectionTest: boolean;
+    reconnectOnClose: boolean;
+ * };
+ *
+ * @example
+ * this.prefs.searchResults.prefsChanged.subscribe( (prefs) => { console.log('search form pref change happened.', prefs); })
+
+ */
+export class SzAdminPrefs extends SzSdkPrefsBase {
+  // --------------- private vars
+  /** @internal */
+  private _streamAnalysisConfig: AdminStreamAnalysisConfig = {
+    sampleSize: 10000,
+    uploadRate: -1
+  };
+  /** @internal */
+  private _streamConnectionProperties: AdminStreamConnProperties | undefined;
+  /** @internal */
+  private _streamLoadConfig: AdminStreamLoadConfig = {
+    autoCreateMissingDataSources: false,
+    uploadRate: -1
+  };
+  /** @internal */
+  private _useStreamingForAnalysis: boolean = false;
+  /** @internal */
+  private _useStreamingForLoad: boolean = false;
+
+  /** the keys of member setters or variables in the object
+   * to output in json, or to take as json input
+   */
+  jsonKeys = [
+    'streamAnalysisConfig',
+    'streamConnectionProperties',
+    'streamLoadConfig',
+    'useStreamingForAnalysis',
+    'useStreamingForLoad',
+  ]
+
+  // ------------------- getters and setters
+  /** configuration parameters for doing analysis on a file stream prior to importing */
+  public get streamAnalysisConfig(): AdminStreamAnalysisConfig {
+    return this._streamAnalysisConfig;
+  }
+  /** configuration parameters for doing analysis on a file stream prior to importing */
+  public set streamAnalysisConfig(value: AdminStreamAnalysisConfig) {
+    this._streamAnalysisConfig = value;
+    if(!this.bulkSet) this.prefsChanged.next( this.toJSONObject() );
+  }
+  /** connection parameters defining how and where to stream to bulk-loading endpoints */
+  public get streamConnectionProperties(): AdminStreamConnProperties | undefined {
+    return this._streamConnectionProperties;
+  }
+  /** connection parameters defining how and where to stream to bulk-loading endpoints */
+  public set streamConnectionProperties(value: AdminStreamConnProperties | undefined) {
+    this._streamConnectionProperties = value;
+    if(!this.bulkSet) this.prefsChanged.next( this.toJSONObject() );
+  }
+  /** configuration parameters for related to importing records using the stream connection */
+  public get streamLoadConfig(): AdminStreamLoadConfig {
+    return this._streamLoadConfig;
+  }
+  /** configuration parameters for related to importing records using the stream connection */
+  public set streamLoadConfig(value: AdminStreamLoadConfig) {
+    this._streamLoadConfig = value;
+    if(!this.bulkSet) this.prefsChanged.next( this.toJSONObject() );
+  }
+  /** whether or not to use the streamConnectionProperties to do analysis through websocket stream. */
+  public get useStreamingForAnalysis(): boolean {
+    return this._useStreamingForAnalysis;
+  }
+  /** whether or not to use the streamConnectionProperties to do analysis through websocket stream. */
+  public set useStreamingForAnalysis(value: boolean) {
+    this._useStreamingForAnalysis = value;
+    if(!this.bulkSet) this.prefsChanged.next( this.toJSONObject() );
+  }
+  /** whether or not to use the streamConnectionProperties to do analysis through websocket stream. */
+  public get useStreamingForLoad(): boolean {
+    return this._useStreamingForLoad;
+  }
+  /** whether or not to use the streamConnectionProperties to do record importing through websocket stream. */
+  public set useStreamingForLoad(value: boolean) {
+    this._useStreamingForLoad = value;
+    if(!this.bulkSet) this.prefsChanged.next( this.toJSONObject() );
+  }
+
+  constructor(){
+    super();
+    /**
+     * publish out a "first" real payload so that
+     * subscribers get an initial payload from this subclass
+     * instead of the empty superclass
+     **/
+    this.prefsChanged.next( this.toJSONObject() );
+  }
+}
+
 /**
  * search form related preferences bus class.
  * used by {@link SzPrefsService} to store it's
@@ -871,7 +982,8 @@ export interface SzSdkPrefsModel {
   searchForm?: any,
   searchResults?: any,
   entityDetail?: any,
-  graph?: any
+  graph?: any,
+  admin?: any
 };
 
 
@@ -929,6 +1041,8 @@ export class SzPrefsService implements OnDestroy {
   public entityDetail?: SzEntityDetailPrefs   = new SzEntityDetailPrefs();
   /** instance of {@link SzGraphPrefs} */
   public graph?: SzGraphPrefs                 = new SzGraphPrefs();
+  /** instance of {@link SzAdminPrefs} */
+  public admin?: SzAdminPrefs                 = new SzAdminPrefs();
 
   /**
    * subscribe for state change representation. */
@@ -957,6 +1071,9 @@ export class SzPrefsService implements OnDestroy {
     if(this.graph){
       retObj.graph = this.graph.toJSONObject();
     }
+    if(this.admin){
+      retObj.admin = this.admin.toJSONObject();
+    }
     return retObj;
   }
   /** populate values from JSON. bulk import basically. */
@@ -1004,6 +1121,12 @@ export class SzPrefsService implements OnDestroy {
     if(_sVal.entityDetail){
       this.entityDetail.fromJSONObject( _sVal.entityDetail );
     }
+    if(_sVal.graph){
+      this.graph.fromJSONObject( _sVal.graph );
+    }
+    if(_sVal.admin){
+      this.entityDetail.fromJSONObject( _sVal.admin );
+    }
   }
   /** get a serialized JSON string from current instance. bulk export. */
   public toJSONString(): string {
@@ -1017,7 +1140,8 @@ export class SzPrefsService implements OnDestroy {
       this.searchForm.prefsChanged,
       this.searchResults.prefsChanged,
       this.entityDetail.prefsChanged,
-      this.graph.prefsChanged
+      this.graph.prefsChanged,
+      this.admin.prefsChanged,
     );
     // now filter and debounce
     // so that any back to back changes are

diff --git a/src/package.json b/src/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@senzing/sdk-components-ng",
-  "version": "2.2.3",
+  "version": "2.2.4",
   "private": false,
   "keywords" :["Angular","Library","Senzing"],
   "license" : "SEE LICENSE IN LICENSE",
@@ -11,7 +11,7 @@
   "homepage" :"https://senzing.github.io/sdk-components-ng/",
   "dependencies": {
     "schematics-utilities": "^2.0.2",
-    "@senzing/rest-api-client-ng": "^2.2.2",
+    "@senzing/rest-api-client-ng": "^2.3.0",
     "d3": "^5.9.1",
     "jspdf": "^2.3.1",
     "html2canvas": "^1.0.0-rc.3",