From 1c0627d950c0a7fa3c7094219eb7674caddf27d1 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Fri, 8 Nov 2024 14:03:54 -0500 Subject: [PATCH 01/50] Prototype of creating external source/event types thru Gateway --- .../modals/CreateGroupsOrTypesModal.svelte | 234 +++++++----------- src/utilities/effects.ts | 77 +++--- 2 files changed, 136 insertions(+), 175 deletions(-) diff --git a/src/components/modals/CreateGroupsOrTypesModal.svelte b/src/components/modals/CreateGroupsOrTypesModal.svelte index eebb69ba40..851adf7421 100644 --- a/src/components/modals/CreateGroupsOrTypesModal.svelte +++ b/src/components/modals/CreateGroupsOrTypesModal.svelte @@ -4,192 +4,144 @@ import { createEventDispatcher } from 'svelte'; import { createExternalEventTypeError, resetExternalEventStores } from '../../stores/external-event'; import { - createDerivationGroupError, createExternalSourceTypeError, - externalSourceTypes, - resetExternalSourceStores, + + resetExternalSourceStores } from '../../stores/external-source'; - import type { User } from '../../types/app'; + import type { RadioButtonId } from '../../types/radio-buttons'; import effects from '../../utilities/effects'; - import { permissionHandler } from '../../utilities/permissionHandler'; - import { featurePermissions } from '../../utilities/permissions'; + import { parseJSONStream } from '../../utilities/generic'; import AlertError from '../ui/AlertError.svelte'; - import Tab from '../ui/Tabs/Tab.svelte'; - import TabPanel from '../ui/Tabs/TabPanel.svelte'; - import Tabs from '../ui/Tabs/Tabs.svelte'; + import RadioButton from '../ui/RadioButtons/RadioButton.svelte'; + import RadioButtons from '../ui/RadioButtons/RadioButtons.svelte'; import Modal from './Modal.svelte'; import ModalContent from './ModalContent.svelte'; import ModalFooter from './ModalFooter.svelte'; import ModalHeader from './ModalHeader.svelte'; - export let user: User | null; + // export let user: User | null; const dispatch = createEventDispatcher<{ close: void; }>(); + const EXTERNAL_EVENT_TYPE = 'External Event Type'; + const EXTERNAL_SOURCE_TYPE = 'External Source Type'; + + let definitionType: RadioButtonId = EXTERNAL_EVENT_TYPE; let newTypeName: string = ''; - let newTypeSourceType: string = ''; let newTypeError: string | null = null; + let fileInput: HTMLInputElement; + let errors: string[] = []; + let files: FileList | undefined; + let file: File | undefined; + let parsedJSONSchema: object | undefined; + + /** TODO - Re-add permissions let hasCreateDerivationGroupPermission: boolean = false; let hasCreateExternalSourceTypePermission: boolean = false; let hasCreateExternalEventTypePermission: boolean = false; $: hasCreateDerivationGroupPermission = featurePermissions.derivationGroup.canCreate(user); - $: hasCreateExternalSourceTypePermission = featurePermissions.externalSourceType.canCreate(user); - $: hasCreateExternalEventTypePermission = featurePermissions.externalEventType.canCreate(user); + */ - function onCreateDerivationGroup() { - if (newTypeName === '') { - newTypeError = 'Please select a source type.'; - } else if (newTypeSourceType === '') { - newTypeError = 'Please enter a new type name.'; - } else { - effects.createDerivationGroup({ name: newTypeName, source_type_name: newTypeSourceType }, user); - newTypeName = ''; - newTypeSourceType = ''; - } + function onClick() { + fileInput.value = ''; + errors = []; } - function onCreateExternalSourceType() { - if (newTypeName === '') { - newTypeError = 'Please enter a new type name.'; - } else { - effects.createExternalSourceType({ name: newTypeName }, user); - newTypeName = ''; - } + async function onChange() { + console.log("onChange"); } - function onCreateExternalEventType() { - if (newTypeName === '') { - newTypeError = 'Please enter a new type name.'; - } else { - effects.createExternalEventType({ name: newTypeName }, user); - newTypeName = ''; - } - } function handleChange() { resetExternalSourceStores(); resetExternalEventStores(); newTypeError = null; } + + async function handleUpload() { + if (files) { + file = files[0]; + if (file !== undefined && /\.json$/.test(file.name)) { + try { + parsedJSONSchema = await parseJSONStream(file.stream()); + if (definitionType === EXTERNAL_EVENT_TYPE) { + effects.createExternalEventType(newTypeName, parsedJSONSchema, null); + } else if (definitionType === EXTERNAL_SOURCE_TYPE) { + effects.createExternalSourceType(newTypeName, parsedJSONSchema, null); + } + } catch (error) { + throw new Error('JSON Schema could not be read.'); + } + } + } + } + + function onSelectDefinitionType(event: CustomEvent<{ id: RadioButtonId }>) { + const { + detail: { id }, + } = event; + definitionType = id; + } - Create Derivation Groups or Types + Create New External Source/Event Types
-
- - - Derivation Group - External Source Type - External Event Type - - -
-

Provide a name and an external source type for the new derivation group.

-

- The newly created group will be empty, though you can upload sources into it. -

-
-
- - - -
- - -
-

Provide a name for the new external source type.

-

- The newly created external source type will be empty, though you can upload sources into it. -

-
-
- - -
- - -
-

Provide a name for the new external event type.

-

- The newly created external event type will be empty, though you can upload events into it. -

+
+ + +
+ {EXTERNAL_EVENT_TYPE}
-
- - + + +
+ {EXTERNAL_SOURCE_TYPE}
- - + +
-
- - - - +
+
+ +
+ + +
+
+
+ + +
+ @@ -252,4 +204,8 @@ .source-type-selection { width: 200px; } + + .type-creation-input { + padding-bottom: 12px; + } diff --git a/src/utilities/effects.ts b/src/utilities/effects.ts index 4ab9d6d76f..9e5c7728cb 100644 --- a/src/utilities/effects.ts +++ b/src/utilities/effects.ts @@ -29,7 +29,6 @@ import { createExternalEventTypeError, creatingExternalEventType } from '../stor import { createDerivationGroupError, createExternalSourceError, - createExternalSourceTypeError, creatingExternalSource, derivationGroupPlanLinkError, parsingError, @@ -99,7 +98,6 @@ import type { ExternalSourceInsertInput, ExternalSourcePkey, ExternalSourceSlim, - ExternalSourceType, ExternalSourceTypeInsertInput, PlanDerivationGroup, } from '../types/external-source'; @@ -863,25 +861,21 @@ const effects = { } }, - async createExternalEventType(eventType: ExternalEventTypeInsertInput, user: User | null) { + async createExternalEventType(eventTypeName: string, eventTypeAttributesSchema: object, user: User | null) { try { creatingExternalEventType.set(true); createExternalEventTypeError.set(null); - if (eventType) { - const { createExternalEventType: created } = await reqHasura( - gql.CREATE_EXTERNAL_EVENT_TYPE, - { eventType }, - user, - ); - if (created) { - showSuccessToast('External Event Type Created Successfully'); - creatingExternalEventType.set(false); - return created.name; - } else { - throw Error('Unable to create external event type'); - } - } else { - throw Error('Unable to create external event type'); + + const body = JSON.stringify({ + attribute_schema: eventTypeAttributesSchema, + external_event_type_name: eventTypeName, + }); + + try { + await reqGateway(`/uploadExternalEventType`, 'POST', body, user, false); + showSuccessToast('External Event Type Created Successfully'); + } catch (e) { + catchError(e as Error); } } catch (e) { catchError('External Event Type Create Failed', e as Error); @@ -1031,28 +1025,27 @@ const effects = { } }, - async createExternalSourceType( - sourceType: ExternalSourceTypeInsertInput, - user: User | null, - ): Promise { + async createExternalSourceType(sourceTypeName: string, sourceTypeAttributesSchema: object, user: User | null) { try { - createExternalSourceTypeError.set(null); - const { createExternalSourceType: created } = await reqHasura( - gql.CREATE_EXTERNAL_SOURCE_TYPE, - { sourceType }, - user, - ); - if (created !== null) { + creatingExternalEventType.set(true); + createExternalEventTypeError.set(null); + + const body = JSON.stringify({ + attribute_schema: sourceTypeAttributesSchema, + external_source_type_name: sourceTypeName, + }); + + try { + await reqGateway(`/uploadExternalSourceType`, 'POST', body, user, false); showSuccessToast('External Source Type Created Successfully'); - return created as ExternalSourceType; - } else { - throw Error(`Unable to create external source type`); + } catch (e) { + catchError(e as Error); } } catch (e) { - catchError('External Source Type Create Failed', e as Error); - showFailureToast('External Source Type Create Failed'); - createExternalSourceTypeError.set((e as Error).message); - return undefined; + catchError('External Event Type Create Failed', e as Error); + showFailureToast('External Event Type Create Failed'); + createExternalEventTypeError.set((e as Error).message); + creatingExternalEventType.set(false); } }, @@ -6567,6 +6560,18 @@ const effects = { } }, + async validateSchema(jsonSchema: object): Promise<{ errors?: []; valid: boolean }> { + try { + const result = await reqGateway('/validation/schema', 'POST', JSON.stringify(jsonSchema), null, false); + return result; + } catch (e) { + catchError(e as Error); + console.log(e); + // TODO: Fix errors + return { errors: [], valid: false }; + } + }, + async validateViewJSON(unValidatedView: unknown): Promise<{ errors?: string[]; valid: boolean }> { try { const { errors, valid } = validateViewJSONAgainstSchema(unValidatedView); From dc8e578b71e0a06581b4bcaab4e64bfc68d36ccf Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Fri, 8 Nov 2024 15:33:11 -0500 Subject: [PATCH 02/50] Prototype WIP --- .../ExternalSourceManager.svelte | 1 + src/types/external-event.ts | 8 +++- src/types/external-source.ts | 7 +++- src/utilities/effects.ts | 41 ++++--------------- 4 files changed, 20 insertions(+), 37 deletions(-) diff --git a/src/components/external-source/ExternalSourceManager.svelte b/src/components/external-source/ExternalSourceManager.svelte index a805e334f2..466c987f39 100644 --- a/src/components/external-source/ExternalSourceManager.svelte +++ b/src/components/external-source/ExternalSourceManager.svelte @@ -328,6 +328,7 @@ $endTimeDoyField.value, parsedExternalSource.events, parsedExternalSource.source.key, + parsedExternalSource.source.attributes, $validAtDoyField.value, user, ); diff --git a/src/types/external-event.ts b/src/types/external-event.ts index 8d7344a658..014169162e 100644 --- a/src/types/external-event.ts +++ b/src/types/external-event.ts @@ -19,6 +19,7 @@ export type ExternalEventId = string; // This is the type that conforms with the database schema. export type ExternalEventDB = { + attributes: object; derivation_group_name: string; duration: string; event_type_name: string; @@ -30,6 +31,7 @@ export type ExternalEventDB = { // This is the JSON type that the user can upload. export type ExternalEventJson = { + attributes: object; duration: string; event_type: string; key: string; @@ -39,6 +41,7 @@ export type ExternalEventJson = { // no analogue to ExternalSourceSlim as we have no subevents or anything of the sort that we may elect to exclude export type ExternalEvent = { + attributes: object; duration: string; duration_ms: number; pkey: ExternalEventPkey; @@ -50,6 +53,7 @@ export type ExternalEvent = { // no analgoue to PlanExternalSource as such a link doesn't exist for external events export type ExternalEventType = { + attribute_schema: object; name: string; }; @@ -57,7 +61,7 @@ export type ExternalEventType = { // this doesn't do any actual filtering. extra keys in surplus of this are NOT checked. // Typescript doesn't really allow us to check these, so ensuring we don't push additional and unnecessary data to the DB should be caught // https://stackoverflow.com/questions/64263271/typescript-validate-excess-keys-on-value-returned-from-function -export type ExternalEventInsertInput = Pick & +export type ExternalEventInsertInput = Pick & Pick; -export type ExternalEventTypeInsertInput = Pick; +export type ExternalEventTypeInsertInput = ExternalEventType; diff --git a/src/types/external-source.ts b/src/types/external-source.ts index df68118a24..af524238ab 100644 --- a/src/types/external-source.ts +++ b/src/types/external-source.ts @@ -9,6 +9,7 @@ export type ExternalSourcePkey = { // This is the type that conforms with the database schema. We don't really use it, as it is pretty heavyweight - instead we derive lighter types from it. export type ExternalSourceDB = { + attributes: object; created_at: string; derivation_group_name: string; end_time: string; @@ -24,6 +25,7 @@ export type ExternalSourceDB = { export type ExternalSourceJson = { events: ExternalEventJson[]; source: { + attributes: object; key: string; period: { end_time: string; @@ -48,6 +50,7 @@ export type PlanDerivationGroup = { }; export type ExternalSourceType = { + attribute_schema: object; name: string; }; @@ -62,7 +65,7 @@ export type DerivationGroup = { // This is used for the GraphQL mutation. export type ExternalSourceInsertInput = Pick< ExternalSourceDB, - 'source_type_name' | 'start_time' | 'end_time' | 'valid_at' + 'attributes' | 'source_type_name' | 'start_time' | 'end_time' | 'valid_at' > & Pick & { external_events: { @@ -70,7 +73,7 @@ export type ExternalSourceInsertInput = Pick< }; }; -export type ExternalSourceTypeInsertInput = Pick; +export type ExternalSourceTypeInsertInput = ExternalSourceType; export type DerivationGroupInsertInput = Pick; diff --git a/src/utilities/effects.ts b/src/utilities/effects.ts index 9e5c7728cb..641f51a9f8 100644 --- a/src/utilities/effects.ts +++ b/src/utilities/effects.ts @@ -90,7 +90,6 @@ import type { ExternalEventInsertInput, ExternalEventJson, ExternalEventType, - ExternalEventTypeInsertInput, } from '../types/external-event'; import type { DerivationGroup, @@ -98,7 +97,6 @@ import type { ExternalSourceInsertInput, ExternalSourcePkey, ExternalSourceSlim, - ExternalSourceTypeInsertInput, PlanDerivationGroup, } from '../types/external-source'; import type { Model, ModelInsertInput, ModelLog, ModelSchema, ModelSetInput, ModelSlim } from '../types/model'; @@ -892,6 +890,7 @@ const effects = { endTime: string, externalEvents: ExternalEventJson[], externalSourceKey: string, + externalSourceAttributes: object, validAt: string, user: User | null, ) { @@ -903,9 +902,6 @@ const effects = { createExternalSourceError.set(null); // Create mutation inputs for Hasura - const externalSourceTypeInsert: ExternalSourceTypeInsertInput = { - name: externalSourceTypeName, - }; const derivationGroupInsert: DerivationGroupInsertInput = { name: derivationGroupName !== '' ? derivationGroupName : `${externalSourceTypeName} Default`, source_type_name: externalSourceTypeName, @@ -935,6 +931,7 @@ const effects = { // Create external source mutation input for Hasura const externalSourceInsert: ExternalSourceInsertInput = { + attributes: externalSourceAttributes, derivation_group_name: derivationGroupInsert.name, end_time: endTimeFormatted, external_events: { @@ -947,13 +944,8 @@ const effects = { }; // Create external events + external event types mutation inputs for Hasura - const externalEventTypeInserts: ExternalEventTypeInsertInput[] = []; let externalEventsCreated: ExternalEventInsertInput[] = []; for (const externalEvent of externalEvents) { - externalEventTypeInserts.push({ - name: externalEvent.event_type, - } as ExternalEventTypeInsertInput); - // Ensure the duration is valid try { getIntervalInMs(externalEvent.duration); @@ -985,6 +977,7 @@ const effects = { externalEvent.duration !== undefined ) { externalEventsCreated.push({ + attributes: externalEvent.attributes, duration: externalEvent.duration, event_type_name: externalEvent.event_type, key: externalEvent.key, @@ -996,23 +989,9 @@ const effects = { externalSourceInsert.external_events.data = externalEventsCreated; externalEventsCreated = []; - const { createExternalSource: createExternalSourceResponse } = await reqHasura( - gql.CREATE_EXTERNAL_SOURCE, - { - derivation_group: derivationGroupInsert, - event_type: externalEventTypeInserts, - source: externalSourceInsert, - source_type: externalSourceTypeInsert, - }, - user, - ); - if (createExternalSourceResponse !== undefined && createExternalSourceResponse !== null) { - showSuccessToast('External Source Created Successfully'); - creatingExternalSource.set(false); - return createExternalSourceResponse as ExternalSourceSlim; - } else { - throw Error(`Unable to create external source`); - } + const body = JSON.stringify(externalSourceInsert); + await reqGateway(`/uploadExternalSource`, 'POST', body, user, false); + showSuccessToast('External Source Type Created Successfully'); } catch (e) { catchError('External Source Create Failed', e as Error); showFailureToast('External Source Create Failed'); @@ -1035,12 +1014,8 @@ const effects = { external_source_type_name: sourceTypeName, }); - try { - await reqGateway(`/uploadExternalSourceType`, 'POST', body, user, false); - showSuccessToast('External Source Type Created Successfully'); - } catch (e) { - catchError(e as Error); - } + await reqGateway(`/uploadExternalSourceType`, 'POST', body, user, false); + showSuccessToast('External Source Type Created Successfully'); } catch (e) { catchError('External Event Type Create Failed', e as Error); showFailureToast('External Event Type Create Failed'); From d4090027fc9b6ac426bc30534968f6028c063489 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Fri, 15 Nov 2024 10:58:22 -0500 Subject: [PATCH 03/50] WIP Prototype of visualizing attributes --- .../external-events/ExternalEventForm.svelte | 16 ++- .../ExternalSourceManager.svelte | 72 ++++++++++- .../modals/CreateGroupsOrTypesModal.svelte | 67 +++++----- .../modals/ManageGroupsAndTypesModal.svelte | 118 +++++++++++++++++- .../parameters/ParameterBase.svelte | 2 +- src/components/parameters/ParameterRec.svelte | 2 +- .../parameters/ParameterRecStruct.svelte | 58 ++++++--- src/components/parameters/Parameters.svelte | 2 +- src/types/external-event.ts | 3 +- src/types/external-source.ts | 3 +- src/types/parameter.ts | 11 +- src/types/schema.ts | 7 ++ src/utilities/effects.ts | 53 ++++---- src/utilities/gql.ts | 67 ++-------- src/utilities/parameters.ts | 18 ++- src/utilities/timeline.test.ts | 1 + src/utilities/view.test.ts | 9 +- 17 files changed, 352 insertions(+), 157 deletions(-) diff --git a/src/components/external-events/ExternalEventForm.svelte b/src/components/external-events/ExternalEventForm.svelte index e8eca5c11c..0f3785b1c1 100644 --- a/src/components/external-events/ExternalEventForm.svelte +++ b/src/components/external-events/ExternalEventForm.svelte @@ -1,9 +1,10 @@ @@ -65,6 +68,17 @@ /> + + +
+ {#each Object.entries(externalEvent.attributes) as attribute} +
+
{attribute[0]}
+
{attribute[1]} ({externalEventType?.attribute_schema.properties[attribute[0]].type})
+
+ {/each} +
+
diff --git a/src/components/external-source/ExternalSourceManager.svelte b/src/components/external-source/ExternalSourceManager.svelte index 466c987f39..a576965c4c 100644 --- a/src/components/external-source/ExternalSourceManager.svelte +++ b/src/components/external-source/ExternalSourceManager.svelte @@ -13,20 +13,23 @@ createExternalSourceTypeError, creatingExternalSource, externalSources, + externalSourceTypes, parsingError, - planDerivationGroupLinks, + planDerivationGroupLinks } from '../../stores/external-source'; import { field } from '../../stores/form'; import { plans } from '../../stores/plans'; import { plugins } from '../../stores/plugins'; import type { User } from '../../types/app'; import type { DataGridColumnDef } from '../../types/data-grid'; - import type { ExternalEvent, ExternalEventId } from '../../types/external-event'; + import type { ExternalEvent, ExternalEventId, ExternalEventType } from '../../types/external-event'; import { type ExternalSourceJson, type ExternalSourceSlim, - type PlanDerivationGroup, + type ExternalSourceType, + type PlanDerivationGroup } from '../../types/external-source'; + import type { ArgumentsMap, JSONTypeSchema, ParametersMap } from '../../types/parameter'; import effects from '../../utilities/effects'; import { getExternalEventRowId, @@ -34,6 +37,7 @@ getExternalSourceSlimRowId, } from '../../utilities/externalEvents'; import { parseJSONStream } from '../../utilities/generic'; + import { getFormParameters } from '../../utilities/parameters'; import { permissionHandler } from '../../utilities/permissionHandler'; import { featurePermissions } from '../../utilities/permissions'; import { formatDate } from '../../utilities/time'; @@ -46,6 +50,7 @@ import DatePickerField from '../form/DatePickerField.svelte'; import Field from '../form/Field.svelte'; import Input from '../form/Input.svelte'; + import Parameters from '../parameters/Parameters.svelte'; import AlertError from '../ui/AlertError.svelte'; import CssGrid from '../ui/CssGrid.svelte'; import CssGridGutter from '../ui/CssGridGutter.svelte'; @@ -131,8 +136,12 @@ // source detail variables let selectedSource: ExternalSourceSlim | null = null; + let selectedSourceAttributes: ArgumentsMap = {}; + let selectedSourceType: ExternalSourceType | undefined = undefined; + let selectedSourceTypeAttributes: Record | undefined = undefined; + let selectedSourceTypeParametersMap: ParametersMap = {}; let selectedSourceId: string | null = null; - let selectedSourceEventTypes: string[] = []; + let selectedSourceEventTypes: ExternalEventType[] = []; // Selected element variables let selectedEvent: ExternalEvent | null = null; @@ -172,6 +181,24 @@ ]); } + $: if (selectedSource !== null) { + // Create an ArgumentsMap for the External Source + selectedSourceAttributes = selectedSource.attributes as ArgumentsMap; + // Create a ParametersMap for the External Source Type + selectedSourceType = $externalSourceTypes.find(sourceType => sourceType.name === selectedSource?.source_type_name); + selectedSourceTypeAttributes = selectedSourceType?.attribute_schema.properties as Record; + selectedSourceTypeParametersMap = Object.entries(selectedSourceTypeAttributes).reduce((acc: ParametersMap, currentAttribute: [string, JSONTypeSchema]) => { + acc[currentAttribute[0]] = { + order: 0, + schema: { + properties: currentAttribute[1]?.properties, + type: currentAttribute[1].type, + } + } + return acc; + }, {} as ParametersMap); + } + $: selectedSourceId = selectedSource ? getExternalSourceRowId({ derivation_group_name: selectedSource.derivation_group_name, key: selectedSource.key }) : null; @@ -529,7 +556,7 @@ {#if selectedSourceEventTypes.length > 0} {#each selectedSourceEventTypes as eventType}
- {eventType} + {eventType.name}
{/each} {:else} @@ -556,7 +583,19 @@
Not used in any plans
{/if} - + +
+ +
+
- - - - - - diff --git a/src/components/modals/ManageGroupsAndTypesModal.svelte b/src/components/modals/ManageGroupsAndTypesModal.svelte index 55e7398871..1f0f702711 100644 --- a/src/components/modals/ManageGroupsAndTypesModal.svelte +++ b/src/components/modals/ManageGroupsAndTypesModal.svelte @@ -1,24 +1,42 @@ Manage Derivation Groups and Types + {#if selectedDerivationGroup === undefined && selectedExternalSourceType === undefined && selectedExternalEventType === undefined} + + + Upload Type Definition + + +
+
+ + +
+ {#if file !== undefined} + + {#if parsedExternalSourceEventTypeSchema !== undefined} +
+
+ +
+ Source & Event Type Attribute Schema Parsed +
+ {:else} + +
+ Source & Event Type Attribute Schema Could Not Be Parsed +
+ {/if} + {/if} + {#if parsedExternalSourceEventTypeSchema !== undefined} +
+
The following External Source Type(s) will be created
+ {#each Object.keys(parsedExternalSourceEventTypeSchema.source_types) as newSourceTypeName} +
  • {newSourceTypeName}
    • + {/each} +
    The following External Event Type(s) will be created
    + {#each Object.keys(parsedExternalSourceEventTypeSchema.event_types) as newEventTypeName} +
  • {newEventTypeName}
    • + {/each} +
    + {/if} +
    + {#each uploadResponseErrors as currentError} + + {/each} + +
    +
    +     +     + + {/if}
    - + - Derivation Group - External Source Type - External Event Type + Derivation Group + External Source Type + External Event Type @@ -447,12 +626,21 @@
    {#if selectedDerivationGroup !== undefined} - - + + Sources in '{selectedDerivationGroup.name}' + {#if selectedDerivationGroupSources.length > 0} @@ -501,12 +689,21 @@ {:else if selectedExternalSourceType !== undefined} - - + + '{selectedExternalSourceType.name}' Details + {#if selectedExternalSourceTypeDerivationGroups.length > 0} @@ -567,12 +764,21 @@ {:else if selectedExternalEventType !== undefined} - - + + '{selectedExternalEventType.name}' Details + diff --git a/src/types/external-source.ts b/src/types/external-source.ts index 456033b385..90528ddb6d 100644 --- a/src/types/external-source.ts +++ b/src/types/external-source.ts @@ -50,6 +50,11 @@ export type PlanDerivationGroup = { plan_id: number; }; +export type ExternalSourceEventTypeSchema = { + event_types: SchemaObject; + source_types: SchemaObject; +}; + export type ExternalSourceType = { attribute_schema: SchemaObject; name: string; diff --git a/src/utilities/effects.ts b/src/utilities/effects.ts index 4eca123209..16d175d7ee 100644 --- a/src/utilities/effects.ts +++ b/src/utilities/effects.ts @@ -215,7 +215,6 @@ import { compare, convertToQuery, getSearchParameterNumber, setQueryParam } from import gql, { convertToGQLArray } from './gql'; import { showConfirmModal, - showCreateGroupsOrTypes, showCreatePlanBranchModal, showCreatePlanSnapshotModal, showCreateViewModal, @@ -985,11 +984,11 @@ const effects = { key: externalSourceKey, period: { end_time: endTimeFormatted, - start_time: startTimeFormatted + start_time: startTimeFormatted, }, source_type_name: externalSourceTypeName, - valid_at: validAtFormatted - } + valid_at: validAtFormatted, + }; const body = new FormData(); body.append('source', JSON.stringify(sourceData)); body.append('events', JSON.stringify(externalEventsCreated)); @@ -1074,15 +1073,6 @@ const effects = { } }, - async createGroupsOrTypes(user: User | null): Promise { - try { - await showCreateGroupsOrTypes(user); - } catch (e) { - catchError('Unable To Be View Derivation Groups and External Types', e as Error); - showFailureToast('Derivation Group/External Type Viewing Failed'); - } - }, - async createModel( name: string, version: string, diff --git a/src/utilities/modal.ts b/src/utilities/modal.ts index 34cb179fb6..9d5e5eca1f 100644 --- a/src/utilities/modal.ts +++ b/src/utilities/modal.ts @@ -2,7 +2,6 @@ import { browser } from '$app/environment'; import AboutModal from '../components/modals/AboutModal.svelte'; import ConfirmActivityCreationModal from '../components/modals/ConfirmActivityCreationModal.svelte'; import ConfirmModal from '../components/modals/ConfirmModal.svelte'; -import CreateGroupsOrTypesModal from '../components/modals/CreateGroupsOrTypesModal.svelte'; import CreatePlanBranchModal from '../components/modals/CreatePlanBranchModal.svelte'; import CreatePlanSnapshotModal from '../components/modals/CreatePlanSnapshotModal.svelte'; import CreateViewModal from '../components/modals/CreateViewModal.svelte'; @@ -156,40 +155,6 @@ export async function showConfirmModal( }); } -/** - * Shows a CreateGroupsOrTypesModal component with the supplied arguments. - */ -export async function showCreateGroupsOrTypes(user: User | null): Promise { - return new Promise(resolve => { - if (browser) { - const target: ModalElement | null = document.querySelector('#svelte-modal'); - - if (target) { - const manageGroupsAndTypesModal = new CreateGroupsOrTypesModal({ - props: { user }, - target, - }); - target.resolve = resolve; - - manageGroupsAndTypesModal.$on('close', () => { - target.replaceChildren(); - target.resolve = null; - target.removeAttribute('data-dismissible'); - manageGroupsAndTypesModal.$destroy(); - }); - manageGroupsAndTypesModal.$on('add', (e: CustomEvent<{ derivationGroupName: string }[]>) => { - target.replaceChildren(); - target.resolve = null; - resolve({ confirm: true, value: e.detail }); - manageGroupsAndTypesModal.$destroy(); - }); - } - } else { - resolve({ confirm: false }); - } - }); -} - /** * Shows a DeleteExternalSourceModal component with the supplied arguments. */ From dcefe3e3571127de9c73fad11e215d9fd4b9a967 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 26 Nov 2024 10:30:46 -0500 Subject: [PATCH 26/50] Move external type management modal to sub-page of External Sources --- .../ExternalSourceManager.svelte | 18 - .../ExternalTypeManager.svelte | 894 ++++++++++++++++++ src/routes/external-sources/+layout.svelte | 27 +- src/routes/external-sources/+layout.ts | 6 + src/routes/external-sources/+page.ts | 10 +- .../{ => sources}/+page.svelte | 8 +- src/routes/external-sources/sources/+page.ts | 9 + .../external-sources/types/+page.svelte | 14 + src/routes/external-sources/types/+page.ts | 9 + src/utilities/effects.ts | 10 - src/utilities/modal.ts | 35 - 11 files changed, 965 insertions(+), 75 deletions(-) create mode 100644 src/components/external-source/ExternalTypeManager.svelte create mode 100644 src/routes/external-sources/+layout.ts rename src/routes/external-sources/{ => sources}/+page.svelte (53%) create mode 100644 src/routes/external-sources/sources/+page.ts create mode 100644 src/routes/external-sources/types/+page.svelte create mode 100644 src/routes/external-sources/types/+page.ts diff --git a/src/components/external-source/ExternalSourceManager.svelte b/src/components/external-source/ExternalSourceManager.svelte index c6750dde3b..810a75640f 100644 --- a/src/components/external-source/ExternalSourceManager.svelte +++ b/src/components/external-source/ExternalSourceManager.svelte @@ -465,10 +465,6 @@ } } - function onManageGroupsAndTypes() { - effects.manageGroupsAndTypes(user); - } - function hasDeleteExternalSourcePermissionOnRow(user: User | null, externalSource: ExternalSourceSlim | undefined) { if (externalSource === undefined) { return false; @@ -782,20 +778,6 @@
    - - - {#if $externalSources.length} diff --git a/src/components/external-source/ExternalTypeManager.svelte b/src/components/external-source/ExternalTypeManager.svelte new file mode 100644 index 0000000000..efc3871e70 --- /dev/null +++ b/src/components/external-source/ExternalTypeManager.svelte @@ -0,0 +1,894 @@ + + + + + + {#if selectedDerivationGroup === undefined && selectedExternalSourceType === undefined && selectedExternalEventType === undefined} + + + Upload Type Definition + + +
    +
    + + +
    + {#if file !== undefined} + + {#if parsedExternalSourceEventTypeSchema !== undefined} +
    +
    + +
    + Source & Event Type Attribute Schema Parsed +
    + {:else} + +
    Source & Event Type Attribute Schema Could Not Be Parsed
    + {/if} + {/if} + {#if parsedExternalSourceEventTypeSchema !== undefined} +
    +
    The following External Source Type(s) will be created
    + {#each Object.keys(parsedExternalSourceEventTypeSchema.source_types) as newSourceTypeName} +
  • {newSourceTypeName}
    • + {/each} +
    The following External Event Type(s) will be created
    + {#each Object.keys(parsedExternalSourceEventTypeSchema.event_types) as newEventTypeName} +
  • {newEventTypeName}
    • + {/each} +
    + {/if} +
    + {#each uploadResponseErrors as currentError} + + {/each} + +
    +
    +     +     + + {:else if selectedDerivationGroup !== undefined} + + + + Sources in '{selectedDerivationGroup.name}' + + + + + {#if selectedDerivationGroupSources.length > 0} + {#each selectedDerivationGroupSources as source} + + + +

    + {selectedDerivationGroup.sources.get(source.key)?.event_counts} events +

    +     +
    +
    Key:
    + {source.key} +
    + +
    +
    Source Type:
    + {source.source_type_name} +
    + +
    +
    Start Time:
    + {source.start_time} +
    + +
    +
    End Time:
    + {source.end_time} +
    + +
    +
    Valid At:
    + {source.valid_at} +
    + +
    +
    Created At:
    + {source.created_at} +
    +     + {/each} + {:else} +

    No sources in this group.

    + {/if} +     +     + + {:else if selectedExternalSourceType !== undefined} + + + + '{selectedExternalSourceType.name}' Details + + + + + {#if selectedExternalSourceTypeDerivationGroups.length > 0} + {#each selectedExternalSourceTypeDerivationGroups as associatedDerivationGroup} + + + +

    + {associatedDerivationGroup.derived_event_total} events +

    +     +
    +
    Name:
    + {associatedDerivationGroup.name} +
    + + + {#each associatedDerivationGroup.sources as source} + {source[0]} + {/each} + +     + {/each} + {:else} +

    No sources associated with this External Source Type.

    + {/if} + + {#each Object.entries(selectedExternalSourceType.attribute_schema) as attribute} + {#if attribute[0] !== 'properties'} +
    +
    {attribute[0]}
    +
    {attribute[1]}
    +
    + {/if} + {/each} +     + +
    + +
    +     +     +     + + {:else if selectedExternalEventType !== undefined} + + + + '{selectedExternalEventType.name}' Details + + + + + + {#each Object.entries(selectedExternalEventType.attribute_schema) as attribute} + {#if attribute[0] !== 'properties'} +
    +
    {attribute[0]}
    +
    {attribute[1]}
    +
    + {/if} + {/each} +     + +
    + +
    +     +     +     + + {/if} +
    +
    + + + Derivation Group + External Source Type + External Event Type + + + + + + + + + + + +
    +
    +     + + diff --git a/src/routes/external-sources/+layout.svelte b/src/routes/external-sources/+layout.svelte index ca8db138de..482b67a107 100644 --- a/src/routes/external-sources/+layout.svelte +++ b/src/routes/external-sources/+layout.svelte @@ -1,16 +1,39 @@ diff --git a/src/routes/external-sources/+layout.ts b/src/routes/external-sources/+layout.ts new file mode 100644 index 0000000000..2cf6a75f8a --- /dev/null +++ b/src/routes/external-sources/+layout.ts @@ -0,0 +1,6 @@ +import type { LayoutLoad } from './$types'; + +export const load: LayoutLoad = async ({ parent }) => { + const { user } = await parent(); + return { user }; +}; diff --git a/src/routes/external-sources/+page.ts b/src/routes/external-sources/+page.ts index ee8329053d..e0113b4ad1 100644 --- a/src/routes/external-sources/+page.ts +++ b/src/routes/external-sources/+page.ts @@ -1,9 +1,7 @@ +import { base } from '$app/paths'; +import { redirect } from '@sveltejs/kit'; import type { PageLoad } from './$types'; -export const load: PageLoad = async ({ parent }) => { - const { user } = await parent(); - - return { - user, - }; +export const load: PageLoad = () => { + redirect(302, `${base}/external-sources/sources`); }; diff --git a/src/routes/external-sources/+page.svelte b/src/routes/external-sources/sources/+page.svelte similarity index 53% rename from src/routes/external-sources/+page.svelte rename to src/routes/external-sources/sources/+page.svelte index bbb88e3176..a2447c9984 100644 --- a/src/routes/external-sources/+page.svelte +++ b/src/routes/external-sources/sources/+page.svelte @@ -2,10 +2,10 @@ + + + + diff --git a/src/routes/external-sources/types/+page.ts b/src/routes/external-sources/types/+page.ts new file mode 100644 index 0000000000..ee8329053d --- /dev/null +++ b/src/routes/external-sources/types/+page.ts @@ -0,0 +1,9 @@ +import type { PageLoad } from './$types'; + +export const load: PageLoad = async ({ parent }) => { + const { user } = await parent(); + + return { + user, + }; +}; diff --git a/src/utilities/effects.ts b/src/utilities/effects.ts index 16d175d7ee..8a0d87910e 100644 --- a/src/utilities/effects.ts +++ b/src/utilities/effects.ts @@ -221,7 +221,6 @@ import { showDeleteActivitiesModal, showDeleteExternalSourceModal, showEditViewModal, - showManageGroupsAndTypes, showManagePlanConstraintsModal, showManagePlanDerivationGroups, showManagePlanSchedulingConditionsModal, @@ -4945,15 +4944,6 @@ const effects = { } }, - async manageGroupsAndTypes(user: User | null): Promise { - try { - await showManageGroupsAndTypes(user); - } catch (e) { - catchError('Unable To Be View Derivation Groups and External Types', e as Error); - showFailureToast('Derivation Group/External Type Viewing Failed'); - } - }, - async managePlanConstraints(user: User | null): Promise { try { await showManagePlanConstraintsModal(user); diff --git a/src/utilities/modal.ts b/src/utilities/modal.ts index 9d5e5eca1f..7777b0bbe6 100644 --- a/src/utilities/modal.ts +++ b/src/utilities/modal.ts @@ -11,7 +11,6 @@ import DeleteExternalEventSourceTypeModal from '../components/modals/DeleteExter import DeleteExternalSourceModal from '../components/modals/DeleteExternalSourceModal.svelte'; import EditViewModal from '../components/modals/EditViewModal.svelte'; import ExpansionSequenceModal from '../components/modals/ExpansionSequenceModal.svelte'; -import ManageGroupsAndTypesModal from '../components/modals/ManageGroupsAndTypesModal.svelte'; import ManagePlanConstraintsModal from '../components/modals/ManagePlanConstraintsModal.svelte'; import ManagePlanDerivationGroupsModal from '../components/modals/ManagePlanDerivationGroupsModal.svelte'; import ManagePlanSchedulingConditionsModal from '../components/modals/ManagePlanSchedulingConditionsModal.svelte'; @@ -348,40 +347,6 @@ export async function showManagePlanDerivationGroups(user: User | null): Promise }); } -/** - * Shows a ManageGroupsAndTypes component with the supplied arguments. - */ -export async function showManageGroupsAndTypes(user: User | null): Promise { - return new Promise(resolve => { - if (browser) { - const target: ModalElement | null = document.querySelector('#svelte-modal'); - - if (target) { - const manageGroupsAndTypesModal = new ManageGroupsAndTypesModal({ - props: { user }, - target, - }); - target.resolve = resolve; - - manageGroupsAndTypesModal.$on('close', () => { - target.replaceChildren(); - target.resolve = null; - target.removeAttribute('data-dismissible'); - manageGroupsAndTypesModal.$destroy(); - }); - manageGroupsAndTypesModal.$on('add', (e: CustomEvent<{ derivationGroupName: string }[]>) => { - target.replaceChildren(); - target.resolve = null; - resolve({ confirm: true, value: e.detail }); - manageGroupsAndTypesModal.$destroy(); - }); - } - } else { - resolve({ confirm: false }); - } - }); -} - /** * Shows a ManagePlanSchedulingConditionsModal component with the supplied arguments. */ From 893f84fab34c09475ea889b5eed32dbfe5c956c1 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 26 Nov 2024 12:29:15 -0500 Subject: [PATCH 27/50] Update e2e tests for new 'type' page --- .../data/Schema_Example-External-Source.json | 14 ----- e2e-tests/data/Schema_ExampleEvent.json | 14 ----- e2e-tests/data/Schema_Example_Source.json | 40 +++++++++++++ e2e-tests/fixtures/ExternalSources.ts | 59 +++++++++---------- e2e-tests/tests/external-sources.test.ts | 45 +++----------- 5 files changed, 75 insertions(+), 97 deletions(-) delete mode 100644 e2e-tests/data/Schema_Example-External-Source.json delete mode 100644 e2e-tests/data/Schema_ExampleEvent.json create mode 100644 e2e-tests/data/Schema_Example_Source.json diff --git a/e2e-tests/data/Schema_Example-External-Source.json b/e2e-tests/data/Schema_Example-External-Source.json deleted file mode 100644 index 4a6df9a6fa..0000000000 --- a/e2e-tests/data/Schema_Example-External-Source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "Example External Source", - "description": "Schema for the attributes of the Example External Source External Source Type.", - "type": "object", - "properties": { - "version": { - "type": "number" - }, - "spacecraft": { - "type": "string" - } - } -} \ No newline at end of file diff --git a/e2e-tests/data/Schema_ExampleEvent.json b/e2e-tests/data/Schema_ExampleEvent.json deleted file mode 100644 index a53a5b02de..0000000000 --- a/e2e-tests/data/Schema_ExampleEvent.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "ExampleEvent", - "description": "Schema for the attributes of the ExampleEvent External Event Type.", - "type": "object", - "properties": { - "example1": { - "type": "string" - }, - "example2": { - "type": "number" - } - } -} \ No newline at end of file diff --git a/e2e-tests/data/Schema_Example_Source.json b/e2e-tests/data/Schema_Example_Source.json new file mode 100644 index 0000000000..9e01ad06e5 --- /dev/null +++ b/e2e-tests/data/Schema_Example_Source.json @@ -0,0 +1,40 @@ +{ + "event_types": { + "ExampleEvent": { + "title": "ExampleEvent", + "description": "Schema for the attributes of the ExampleEvent External Event Type.", + "type": "object", + "properties": { + "example1": { + "type": "string" + }, + "example2": { + "type": "number" + } + }, + "required": [ + "example1", + "example2" + ] + } + }, + "source_types": { + "Example External Source": { + "title": "Example External Source", + "description": "Schema for the attributes of the Example External Source External Source Type.", + "type": "object", + "properties": { + "version": { + "type": "number" + }, + "spacecraft": { + "type": "string" + } + }, + "required": [ + "version", + "spacecraft" + ] + } + } +} diff --git a/e2e-tests/fixtures/ExternalSources.ts b/e2e-tests/fixtures/ExternalSources.ts index 083fc26441..32ae296eab 100644 --- a/e2e-tests/fixtures/ExternalSources.ts +++ b/e2e-tests/fixtures/ExternalSources.ts @@ -3,8 +3,6 @@ import { expect, type Locator, type Page } from '@playwright/test'; export class ExternalSources { alertError: Locator; closeButton: Locator; - createTypesButton: Locator; - createTypesModal: Locator; deleteSourceButton: Locator; deleteSourceButtonConfirmation: Locator; derivationATypeName: string = 'DerivationA'; @@ -32,23 +30,21 @@ export class ExternalSources { exampleDerivationGroup: string = 'Example External Source Default'; exampleEventType: string = 'ExampleEvent'; exampleSourceType: string = 'Example External Source'; + exampleTypeSchema: string = 'e2e-tests/data/Schema_Example_Source.json'; externalEventSelectedForm: Locator; externalEventTableHeaderDuration: Locator; externalEventTableHeaderEventType: Locator; externalEventTableRow: Locator; externalEventTypeName: string = 'ExampleEvent'; - externalEventTypeSchema: string = 'e2e-tests/data/Schema_ExampleEvent.json'; externalSourceFileName: string = 'example-external-source.json'; externalSourceFilePath: string = 'e2e-tests/data/example-external-source.json'; externalSourceFilePathMissingField: string = 'e2e-tests/data/example-external-source-missing-field.json'; externalSourceFilePathSyntaxError: string = 'e2e-tests/data/example-external-source-syntax-error.json'; externalSourceSelectedForm: Locator; externalSourceTypeName: string = 'Example External Source'; - externalSourceTypeSchema: string = 'e2e-tests/data/Schema_Example-External-Source.json'; externalSourceUpload: Locator; externalSourcesTable: Locator; inputFile: Locator; - manageGroupsAndTypesButton: Locator; nameInput: Locator; panelExternalEventsTable: Locator; saveButton: Locator; @@ -66,23 +62,16 @@ export class ExternalSources { await this.closeButton.click(); } - async createType(typeName: string, typeSchema: string, isSourceType: boolean) { - await this.createTypesButton.click(); - await expect(this.createTypesModal).toBeVisible(); - if (isSourceType) { - await this.page.getByRole('radio', { name: 'External Source Type' }).click(); + async createType(typeSchema: string, typeName: string, isExternalEventType: boolean) { + await this.gotoTypeManager(); + await this.page.getByRole('textbox').focus(); + await this.page.getByRole('textbox').setInputFiles(typeSchema); + await this.page.getByRole('textbox').evaluate(e => e.blur()); + await this.page.getByLabel('Upload External Source & Event Type(s)').click(); + if (isExternalEventType) { + await this.page.getByRole('button', { name: 'External Event Type' }).click(); } - await this.page.getByPlaceholder('New Type Name').fill(typeName); - await this.page.locator('#svelte-modal input[name="file"]').focus(); - await this.page.locator('#svelte-modal input[name="file"]').setInputFiles(typeSchema); - await this.page.locator('#svelte-modal input[name="file"]').evaluate(e => e.blur()); - await this.page.getByRole('button', { exact: true, name: 'Create' }).click(); - if (isSourceType) { - await this.waitForToast('External Source Type Created Successfully'); - } else { - await this.waitForToast('External Event Type Created Successfully'); - } - await expect(this.createTypesModal).not.toBeVisible(); + await expect(this.page.getByRole('row', { name: typeName })).toBeVisible(); } async deleteDerivationGroup(derivationGroupName: string) { await this.page.getByRole('button', { exact: true, name: 'Derivation Group' }).click(); @@ -110,7 +99,7 @@ export class ExternalSources { async deleteSource(sourceName: string) { // Only delete a source if its visible in the table - if (await this.page.getByRole('gridcell', { name: sourceName }).first().isVisible()) { + if (await this.page.getByRole('gridcell', { name: sourceName }).isVisible()) { await this.selectSource(sourceName); await this.deleteSourceButton.click(); await this.deleteSourceButtonConfirmation.click(); @@ -143,6 +132,11 @@ export class ExternalSources { await this.page.waitForTimeout(250); } + async gotoTypeManager() { + await this.page.goto('/external-sources/types', { waitUntil: 'networkidle' }); + await this.page.waitForTimeout(250); + } + async linkDerivationGroup(derivationGroupName: string, sourceTypeName: string) { // Assumes the Manage Derivation Groups modal is already showing await this.page.getByRole('row', { name: derivationGroupName }).getByRole('checkbox').click(); @@ -152,15 +146,15 @@ export class ExternalSources { } async selectEvent(eventName: string, sourceName: string = 'example-external-source.json') { - // Assumes the selected source was the test source, and selects the specific event from it - // NOTE: This may not be the case, and should be re-visited when we implement deletion for External Sources! + await this.goto(); await this.selectSource(sourceName); await this.page.getByRole('gridcell', { name: eventName }).click(); } async selectSource(sourceName: string = 'example-external-source.json') { - // Always selects the first source with the example's source type in the table - await this.page.getByRole('gridcell', { name: sourceName }).first().click(); + await this.goto(); + await this.page.getByRole('gridcell', { name: sourceName }).click(); + await expect(this.page.getByText('Selected External Source')).toBeVisible(); } async unlinkDerivationGroup(derivationGroupName: string, sourceTypeName: string) { @@ -195,23 +189,24 @@ export class ExternalSources { this.viewEventSourceMetadata = page.getByRole('button', { name: 'View Event Source Metadata' }); this.panelExternalEventsTable = page.locator('[data-component-name="ExternalEventsTablePanel"]'); this.externalSourcesTable = page.locator('#external-sources-table'); - this.createTypesButton = page.getByLabel('Create external source types or external event types.'); - this.manageGroupsAndTypesButton = page.getByLabel('Manage and inspect existing'); - this.createTypesModal = page.locator(`.modal:has-text("Create New External Source/Event Types")`); } async uploadExternalSource( inputFilePath: string = this.externalSourceFilePath, inputFileName: string = this.externalSourceFileName, + validateUpload: boolean = true, ) { + await this.goto(); await this.fillInputFile(inputFilePath); // Wait for all errors to disappear, assuming stores are just taking time to load await this.page.getByLabel('please create one before uploading an external source').waitFor({ state: 'hidden' }); await this.page.getByLabel('Please create it!').waitFor({ state: 'hidden' }); await this.uploadButton.click(); - await this.waitForToast('External Source Created Successfully'); - await expect(this.externalSourcesTable).toBeVisible(); - await expect(this.externalSourcesTable.getByRole('gridcell', { name: inputFileName })).toBeVisible(); + if (validateUpload) { + await expect(this.externalSourcesTable).toBeVisible(); + await expect(this.externalSourcesTable.getByRole('gridcell', { name: inputFileName })).toBeVisible(); + await this.waitForToast('External Source Created Successfully'); + } } async waitForToast(message: string) { diff --git a/e2e-tests/tests/external-sources.test.ts b/e2e-tests/tests/external-sources.test.ts index 1efcd0c4ce..4ad2066239 100644 --- a/e2e-tests/tests/external-sources.test.ts +++ b/e2e-tests/tests/external-sources.test.ts @@ -23,24 +23,10 @@ test.beforeEach(async () => { test.describe.serial('External Sources', () => { test('Uploading an external source', async () => { - await externalSources.createType( - externalSources.externalEventTypeName, - externalSources.externalEventTypeSchema, - false, - ); - await externalSources.createType( - externalSources.externalSourceTypeName, - externalSources.externalSourceTypeSchema, - true, - ); + await externalSources.createType(externalSources.exampleTypeSchema, 'Example External Source', false); await externalSources.uploadExternalSource(); }); - test('Upload button should be enabled after entering a filepath', async () => { - await externalSources.fillInputFile(externalSources.externalSourceFilePath); - await expect(externalSources.uploadButton).toBeVisible(); - }); - test('External event form should be shown when an event is selected', async () => { await externalSources.selectEvent('ExampleEvent:1/sc/sc1:1'); await expect(externalSources.inputFile).not.toBeVisible(); @@ -69,15 +55,6 @@ test.describe.serial('External Sources', () => { await expect(externalSources.externalSourceSelectedForm).not.toBeVisible(); }); - // TODO: Metadata will be implemented in a future batch of work! - // test('Selected external source should show metadata in a collapsible', async () => { - // await externalSources.selectSource(); - // await externalSources.viewEventSourceMetadata.click(); - // await expect(page.getByText('0', { exact: true })).toBeVisible(); - // await expect(page.getByText('1', { exact: true }).first()).toBeVisible(); - // await expect(page.getByText('version')).toBeVisible(); - // }); - test('Selected external source should show event types in a collapsible', async () => { await externalSources.selectSource(); await externalSources.viewContainedEventTypes.click(); @@ -97,7 +74,7 @@ test.describe.serial('External Sources', () => { await expect(externalSources.inputFile).toBeVisible(); await expect(externalSources.externalEventSelectedForm).not.toBeVisible(); await expect(externalSources.externalSourceSelectedForm).not.toBeVisible(); - await externalSources.manageGroupsAndTypesButton.click(); + await externalSources.gotoTypeManager(); await externalSources.deleteDerivationGroup(externalSources.exampleDerivationGroup); await externalSources.deleteExternalEventType(externalSources.exampleEventType); await externalSources.deleteExternalSourceType(externalSources.exampleSourceType); @@ -106,23 +83,17 @@ test.describe.serial('External Sources', () => { test.describe.serial('External Source Error Handling', () => { test('Duplicate keys is handled gracefully', async () => { - await externalSources.createType( - externalSources.externalEventTypeName, - externalSources.externalEventTypeSchema, - false, - ); - await externalSources.createType( - externalSources.externalSourceTypeName, - externalSources.externalSourceTypeSchema, - true, - ); + await externalSources.createType(externalSources.exampleTypeSchema, 'Example External Source', false); await externalSources.uploadExternalSource(); - await externalSources.deselectSourceButton.click(); await expect(externalSources.externalSourcesTable).toBeVisible(); await expect( externalSources.externalSourcesTable.getByRole('gridcell', { name: externalSources.externalSourceFileName }), ).toBeVisible(); - await externalSources.uploadExternalSource(); + await externalSources.uploadExternalSource( + externalSources.externalSourceFilePath, + externalSources.externalSourceFileName, + false, + ); await expect(page.getByLabel('Uniqueness violation.')).toBeVisible(); await externalSources.waitForToast('External Source Create Failed'); await expect(page.getByRole('gridcell', { name: externalSources.externalSourceFileName })).toHaveCount(1); From 9e909030825f5521d3ab6937af103ff271861892 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 26 Nov 2024 16:15:30 -0500 Subject: [PATCH 28/50] Updating derivation examples for e2e tests --- e2e-tests/data/Schema_Derivation-Test.json | 11 --- e2e-tests/data/Schema_DerivationAjson | 17 ---- e2e-tests/data/Schema_DerivationB.json | 17 ---- e2e-tests/data/Schema_DerivationC.json | 17 ---- e2e-tests/data/Schema_DerivationD.json | 17 ---- e2e-tests/data/Schema_Example_Derivation.json | 93 +++++++++++++++++++ .../data/external-event-derivation-1.json | 2 +- .../data/external-event-derivation-2.json | 2 +- .../data/external-event-derivation-3.json | 2 +- .../data/external-event-derivation-4.json | 2 +- e2e-tests/fixtures/ExternalSources.ts | 8 +- e2e-tests/tests/plan-external-source.test.ts | 17 +--- 12 files changed, 104 insertions(+), 101 deletions(-) delete mode 100644 e2e-tests/data/Schema_Derivation-Test.json delete mode 100644 e2e-tests/data/Schema_DerivationAjson delete mode 100644 e2e-tests/data/Schema_DerivationB.json delete mode 100644 e2e-tests/data/Schema_DerivationC.json delete mode 100644 e2e-tests/data/Schema_DerivationD.json create mode 100644 e2e-tests/data/Schema_Example_Derivation.json diff --git a/e2e-tests/data/Schema_Derivation-Test.json b/e2e-tests/data/Schema_Derivation-Test.json deleted file mode 100644 index 07164341fd..0000000000 --- a/e2e-tests/data/Schema_Derivation-Test.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "Derivation Test", - "description": "Schema for the attributes of the Derivation Test External Source Type.", - "type": "object", - "properties": { - "exampleAttribute": { - "type": "string" - } - } -} diff --git a/e2e-tests/data/Schema_DerivationAjson b/e2e-tests/data/Schema_DerivationAjson deleted file mode 100644 index 8cfae18496..0000000000 --- a/e2e-tests/data/Schema_DerivationAjson +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "DerivationAC", - "description": "Schema for the attributes of the DerivationA External Event Type.", - "type": "object", - "properties": { - "rules": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "should_present": { - "type": "boolean" - } - } -} diff --git a/e2e-tests/data/Schema_DerivationB.json b/e2e-tests/data/Schema_DerivationB.json deleted file mode 100644 index 2063fb7665..0000000000 --- a/e2e-tests/data/Schema_DerivationB.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "DerivationB", - "description": "Schema for the attributes of the DerivationB External Event Type.", - "type": "object", - "properties": { - "rules": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "should_present": { - "type": "boolean" - } - } -} diff --git a/e2e-tests/data/Schema_DerivationC.json b/e2e-tests/data/Schema_DerivationC.json deleted file mode 100644 index eacd137223..0000000000 --- a/e2e-tests/data/Schema_DerivationC.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "DerivationC", - "description": "Schema for the attributes of the DerivationC External Event Type.", - "type": "object", - "properties": { - "rules": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "should_present": { - "type": "boolean" - } - } -} diff --git a/e2e-tests/data/Schema_DerivationD.json b/e2e-tests/data/Schema_DerivationD.json deleted file mode 100644 index d06b8024ba..0000000000 --- a/e2e-tests/data/Schema_DerivationD.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "DerivationD", - "description": "Schema for the attributes of the DerivationD External Event Type.", - "type": "object", - "properties": { - "rules": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "should_present": { - "type": "boolean" - } - } -} diff --git a/e2e-tests/data/Schema_Example_Derivation.json b/e2e-tests/data/Schema_Example_Derivation.json new file mode 100644 index 0000000000..24c6ab1f57 --- /dev/null +++ b/e2e-tests/data/Schema_Example_Derivation.json @@ -0,0 +1,93 @@ +{ + "event_types": { + "DerivationA": { + "type": "object", + "required": [ + "rules", + "notes", + "should_present" + ], + "properties": { + "rules": { + "type": "string" + }, + "notes": { + "type": "string" + }, + "should_present": { + "type": "boolean" + } + } + }, + "DerivationB": { + "type": "object", + "required": [ + "rules", + "notes", + "should_present" + ], + "properties": { + "rules": { + "type": "string" + }, + "notes": { + "type": "string" + }, + "should_present": { + "type": "boolean" + } + } + }, + "DerivationC": { + "type": "object", + "required": [ + "rules", + "notes", + "should_present" + ], + "properties": { + "rules": { + "type": "string" + }, + "notes": { + "type": "string" + }, + "should_present": { + "type": "boolean" + } + } + }, + "DerivationD": { + "type": "object", + "required": [ + "rules", + "notes", + "should_present" + ], + "properties": { + "rules": { + "type": "string" + }, + "notes": { + "type": "string" + }, + "should_present": { + "type": "boolean" + } + } + } + }, + "source_types": { + "DerivationTest": { + "type": "object", + "required": [ + "exampleAttribute" + ], + "properties": { + "exampleAttribute": { + "type": "string" + } + } + } + } +} diff --git a/e2e-tests/data/external-event-derivation-1.json b/e2e-tests/data/external-event-derivation-1.json index cd5891b5d6..e9b16123d9 100644 --- a/e2e-tests/data/external-event-derivation-1.json +++ b/e2e-tests/data/external-event-derivation-1.json @@ -1,7 +1,7 @@ { "source": { "key": "external-event-derivation-1.json", - "source_type": "Derivation Test", + "source_type": "DerivationTest", "valid_at": "2022-018T00:00:00Z", "period": { "start_time": "2022-005T00:00:00Z", diff --git a/e2e-tests/data/external-event-derivation-2.json b/e2e-tests/data/external-event-derivation-2.json index 3175d22b3d..0db0d92553 100644 --- a/e2e-tests/data/external-event-derivation-2.json +++ b/e2e-tests/data/external-event-derivation-2.json @@ -1,7 +1,7 @@ { "source": { "key": "external-event-derivation-2.json", - "source_type": "Derivation Test", + "source_type": "DerivationTest", "valid_at": "2022-019T00:00:00Z", "period": { "start_time": "2022-001T00:00:00Z", diff --git a/e2e-tests/data/external-event-derivation-3.json b/e2e-tests/data/external-event-derivation-3.json index b6563ea76e..06a7e5d137 100644 --- a/e2e-tests/data/external-event-derivation-3.json +++ b/e2e-tests/data/external-event-derivation-3.json @@ -1,7 +1,7 @@ { "source": { "key": "external-event-derivation-3.json", - "source_type": "Derivation Test", + "source_type": "DerivationTest", "valid_at": "2022-020T00:00:00Z", "period": { "start_time": "2022-003T00:00:00Z", diff --git a/e2e-tests/data/external-event-derivation-4.json b/e2e-tests/data/external-event-derivation-4.json index 6e1085e5b1..50adf4844a 100644 --- a/e2e-tests/data/external-event-derivation-4.json +++ b/e2e-tests/data/external-event-derivation-4.json @@ -1,7 +1,7 @@ { "source": { "key": "external-event-derivation-4.json", - "source_type": "Derivation Test", + "source_type": "DerivationTest", "valid_at": "2022-021T00:00:00Z", "period": { "start_time": "2022-001T12:00:00Z", diff --git a/e2e-tests/fixtures/ExternalSources.ts b/e2e-tests/fixtures/ExternalSources.ts index 32ae296eab..a7e56a5d84 100644 --- a/e2e-tests/fixtures/ExternalSources.ts +++ b/e2e-tests/fixtures/ExternalSources.ts @@ -21,10 +21,10 @@ export class ExternalSources { derivationTestFileKey2: string = 'external-event-derivation-2.json'; derivationTestFileKey3: string = 'external-event-derivation-3.json'; derivationTestFileKey4: string = 'external-event-derivation-4.json'; - derivationTestGroupName: string = 'Derivation Test Default'; - derivationTestSourceType: string = 'Derivation Test'; - derivationTestSourceTypeName: string = 'Derivation Test'; - derivationTestSourceTypeSchema: string = 'e2e-tests/data/Schema_Derivation-Test.json'; + derivationTestGroupName: string = 'DerivationTest Default'; + derivationTestSourceType: string = 'DerivationTest'; + derivationTestSourceTypeName: string = 'DerivationTest'; + derivationTestTypeSchema: string = 'e2e-tests/data/Schema_Example_Derivation.json'; deselectEventButton: Locator; deselectSourceButton: Locator; exampleDerivationGroup: string = 'Example External Source Default'; diff --git a/e2e-tests/tests/plan-external-source.test.ts b/e2e-tests/tests/plan-external-source.test.ts index 658b7f769a..6c79e4df12 100644 --- a/e2e-tests/tests/plan-external-source.test.ts +++ b/e2e-tests/tests/plan-external-source.test.ts @@ -39,16 +39,7 @@ test.beforeAll(async ({ baseURL, browser }) => { await plans.goto(); await plans.createPlan(); await externalSources.goto(); - await externalSources.createType( - externalSources.externalEventTypeName, - externalSources.externalEventTypeSchema, - false, - ); - await externalSources.createType( - externalSources.externalSourceTypeName, - externalSources.externalSourceTypeSchema, - true, - ); + await externalSources.createType(externalSources.exampleTypeSchema, externalSources.externalSourceTypeName, false); await externalSources.uploadExternalSource(); }); @@ -154,11 +145,10 @@ test.describe.serial('Plan External Sources', () => { test('Cards should be shown when a new external source is uploaded', async () => { // Upload a test file and link its derivation group to the plan await externalSources.goto(); - await externalSources.createType(externalSources.derivationATypeName, externalSources.derivationATypeSchema, false); await externalSources.createType( + externalSources.derivationTestTypeSchema, externalSources.derivationTestSourceTypeName, - externalSources.derivationTestSourceTypeSchema, - true, + false, ); await externalSources.uploadExternalSource( externalSources.derivationTestFile1, @@ -174,7 +164,6 @@ test.describe.serial('Plan External Sources', () => { // Upload another test await externalSources.goto(); - await externalSources.createType(externalSources.derivationBTypeName, externalSources.derivationBTypeSchema, false); await externalSources.uploadExternalSource( externalSources.derivationTestFile2, externalSources.derivationTestFileKey2, From 8fcbd48026457dc3b129150e795317fc0dc0550e Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 26 Nov 2024 16:43:16 -0500 Subject: [PATCH 29/50] --amend --- e2e-tests/tests/external-sources.test.ts | 6 ++++++ e2e-tests/tests/plan-external-source.test.ts | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/e2e-tests/tests/external-sources.test.ts b/e2e-tests/tests/external-sources.test.ts index 4ad2066239..79631488bb 100644 --- a/e2e-tests/tests/external-sources.test.ts +++ b/e2e-tests/tests/external-sources.test.ts @@ -13,6 +13,12 @@ test.beforeAll(async ({ browser }) => { }); test.afterAll(async () => { + await externalSources.goto(); + await externalSources.deleteSource(externalSources.externalSourceFileName); + await externalSources.gotoTypeManager(); + await externalSources.deleteDerivationGroup(externalSources.exampleDerivationGroup); + await externalSources.deleteExternalSourceType(externalSources.exampleSourceType); + await externalSources.deleteExternalEventType(externalSources.exampleEventType); await page.close(); await context.close(); }); diff --git a/e2e-tests/tests/plan-external-source.test.ts b/e2e-tests/tests/plan-external-source.test.ts index 6c79e4df12..bc1b47c427 100644 --- a/e2e-tests/tests/plan-external-source.test.ts +++ b/e2e-tests/tests/plan-external-source.test.ts @@ -50,13 +50,23 @@ test.afterAll(async () => { await models.deleteModel(); await externalSources.goto(); - // Cleanup all test files that *may* have been uploaded await externalSources.deleteSource(externalSources.externalSourceFileName); await externalSources.deleteSource(externalSources.derivationTestFileKey1); await externalSources.deleteSource(externalSources.derivationTestFileKey2); await externalSources.deleteSource(externalSources.derivationTestFileKey3); await externalSources.deleteSource(externalSources.derivationTestFileKey4); + await externalSources.gotoTypeManager(); + await externalSources.deleteDerivationGroup(externalSources.exampleDerivationGroup); + await externalSources.deleteDerivationGroup(externalSources.derivationTestGroupName); + await externalSources.deleteExternalSourceType(externalSources.exampleSourceType); + await externalSources.deleteExternalSourceType(externalSources.derivationTestSourceTypeName); + await externalSources.deleteExternalEventType(externalSources.exampleEventType); + await externalSources.deleteExternalEventType(externalSources.derivationATypeName); + await externalSources.deleteExternalEventType(externalSources.derivationBTypeName); + await externalSources.deleteExternalEventType(externalSources.derivationCTypeName); + await externalSources.deleteExternalEventType(externalSources.derivationDTypeName); + await page.close(); await context.close(); }); From 38e080307d598f9ca029a8c6f7db63b4f6602148 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 27 Nov 2024 10:48:30 -0500 Subject: [PATCH 30/50] Add support for arrays as series --- src/utilities/parameters.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/utilities/parameters.ts b/src/utilities/parameters.ts index 92caa3b6f9..a8e334f10f 100644 --- a/src/utilities/parameters.ts +++ b/src/utilities/parameters.ts @@ -147,7 +147,6 @@ export function translateJsonSchemaArgumentsToValueSchema(jsonArguments: Argumen /** * Returns a list of ValueSchema objects that represent a JSON schema's properties. - * @param schema */ export function translateJsonSchemaToValueSchema(jsonSchema: SchemaObject | undefined): Record { if (jsonSchema === undefined) { @@ -162,11 +161,16 @@ export function translateJsonSchemaToValueSchema(jsonSchema: SchemaObject | unde // Handle nested objects, 'properties' => 'items' const propName: string = property[0]; if ('type' in property[1]) { - const { type: propType, properties: propProperties } = property[1] as { + const { + type: propType, + properties: propProperties, + items: propItems, + } = property[1] as { + items?: Record<'type', JSONType>; properties?: Record; type: JSONType; }; - const propTranslated = translateJsonSchemaTypeToValueSchema(propType as JSONType, propProperties); + const propTranslated = translateJsonSchemaTypeToValueSchema(propType as JSONType, propProperties, propItems); propertiesAsValueSchema[propName] = propTranslated; } else { throw new Error('Cannot convert invalid JSON schema property - no "type" field exists'); @@ -178,6 +182,7 @@ export function translateJsonSchemaToValueSchema(jsonSchema: SchemaObject | unde function translateJsonSchemaTypeToValueSchema( jsonSchemaType: JSONType, jsonSchemaProperties?: Record, + jsonSchemaItems?: Record<'type', JSONType>, ): ValueSchema { if (jsonSchemaType === 'number' || jsonSchemaType === 'integer') { return { type: 'int' } as ValueSchemaInt; @@ -189,10 +194,15 @@ function translateJsonSchemaTypeToValueSchema( } return { items: jsonSchemaProperties, type: 'struct' } as ValueSchemaStruct; } else if (jsonSchemaType === 'array') { - if (jsonSchemaProperties === undefined) { - throw new Error('Cannot convert "array" from JSON Schema without any nested "properties" defined'); + if (jsonSchemaItems === undefined) { + throw new Error('Cannot convert "array" from JSON Schema without any nested "items" defined'); + } else if (Object.keys(jsonSchemaItems).length === 0) { + throw new Error('Cannot convert "array" from JSON Schema without an "items" field defined'); } - return { type: 'series' } as ValueSchemaSeries; + // ValueSchema expects a singular type for the series where JSON Schema allows multiple. Take the first if the user gave multiple + const firstItem = Object.entries(jsonSchemaItems)[0]; + const translatedItem: ValueSchema = translateJsonSchemaTypeToValueSchema(firstItem[1]); + return { items: translatedItem, type: 'series' } as ValueSchemaSeries; } else { return { type: jsonSchemaType } as ValueSchema; } From 29d7b7061c46b9ac8b9eb48135343269704fa444 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 27 Nov 2024 11:18:15 -0500 Subject: [PATCH 31/50] Fix issue with auto-selecting source after upload --- .../external-source/ExternalSourceManager.svelte | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/components/external-source/ExternalSourceManager.svelte b/src/components/external-source/ExternalSourceManager.svelte index 810a75640f..ee9e6a48b3 100644 --- a/src/components/external-source/ExternalSourceManager.svelte +++ b/src/components/external-source/ExternalSourceManager.svelte @@ -372,7 +372,7 @@ async function onFormSubmit(_e: SubmitEvent) { if (parsedExternalSource && file) { - const createExternalSourceResponse: { data: { createExternalSource: ExternalSourceSlim } } | undefined = + const requestResponse: { createExternalSource: ExternalSourceSlim, upsertDerivationGroup: { name: string } | null } | undefined = await effects.createExternalSource( $sourceTypeField.value, $derivationGroupField.value, @@ -384,17 +384,18 @@ $validAtDoyField.value, user, ); - // Following a successful mutation... - if (createExternalSourceResponse !== undefined) { + if (requestResponse !== undefined) { + const { createExternalSource: createExternalSourceResponse } = requestResponse; // Auto-select the new source selectedSource = { - ...createExternalSourceResponse.data.createExternalSource, + ...createExternalSourceResponse, created_at: new Date().toISOString().replace('Z', '+00:00'), // technically not the exact time it shows up in the database }; gridRowSizes = gridRowSizesBottomPanel; } } + // Reset the form behind the source parsedExternalSource = undefined; file = undefined; From 2fba23251dae13c9e8c1a1c85d73221c2652806f Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 27 Nov 2024 11:38:10 -0500 Subject: [PATCH 32/50] Formatting --- .../ExternalSourceManager.svelte | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/components/external-source/ExternalSourceManager.svelte b/src/components/external-source/ExternalSourceManager.svelte index ee9e6a48b3..877f39d9db 100644 --- a/src/components/external-source/ExternalSourceManager.svelte +++ b/src/components/external-source/ExternalSourceManager.svelte @@ -372,18 +372,19 @@ async function onFormSubmit(_e: SubmitEvent) { if (parsedExternalSource && file) { - const requestResponse: { createExternalSource: ExternalSourceSlim, upsertDerivationGroup: { name: string } | null } | undefined = - await effects.createExternalSource( - $sourceTypeField.value, - $derivationGroupField.value, - $startTimeDoyField.value, - $endTimeDoyField.value, - parsedExternalSource.events, - parsedExternalSource.source.key, - parsedExternalSource.source.attributes, - $validAtDoyField.value, - user, - ); + const requestResponse: + | { createExternalSource: ExternalSourceSlim; upsertDerivationGroup: { name: string } | null } + | undefined = await effects.createExternalSource( + $sourceTypeField.value, + $derivationGroupField.value, + $startTimeDoyField.value, + $endTimeDoyField.value, + parsedExternalSource.events, + parsedExternalSource.source.key, + parsedExternalSource.source.attributes, + $validAtDoyField.value, + user, + ); // Following a successful mutation... if (requestResponse !== undefined) { const { createExternalSource: createExternalSourceResponse } = requestResponse; From ff2495b431eddadf02e1b0caaab7ac6a40ca5792 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 3 Dec 2024 11:31:13 -0500 Subject: [PATCH 33/50] Always show all 3 tables in type manager --- .../ExternalTypeManager.svelte | 134 ++++++++---------- 1 file changed, 59 insertions(+), 75 deletions(-) diff --git a/src/components/external-source/ExternalTypeManager.svelte b/src/components/external-source/ExternalTypeManager.svelte index efc3871e70..9b63d79012 100644 --- a/src/components/external-source/ExternalTypeManager.svelte +++ b/src/components/external-source/ExternalTypeManager.svelte @@ -6,10 +6,12 @@ import type { ICellRendererParams } from 'ag-grid-community'; import XIcon from 'bootstrap-icons/icons/x.svg?component'; import ExternalSourceIcon from '../../assets/external-source-box.svg?component'; + import { externalEventTypes } from '../../stores/external-event'; import { createExternalSourceEventTypeError, derivationGroups, externalSources, + externalSourceTypes, sourcesUsingExternalEventTypes, } from '../../stores/external-source'; import type { User } from '../../types/app'; @@ -24,6 +26,7 @@ import type { ParametersMap } from '../../types/parameter'; import type { ValueSchema } from '../../types/schema'; import effects from '../../utilities/effects'; + import { getDerivationGroupRowId, getExternalEventTypeRowId, getExternalSourceTypeRowId } from '../../utilities/externalEvents'; import { parseJSONStream } from '../../utilities/generic'; import { showDeleteDerivationGroupModal, showDeleteExternalEventSourceTypeModal } from '../../utilities/modal'; import { getFormParameters, translateJsonSchemaToValueSchema } from '../../utilities/parameters'; @@ -31,19 +34,14 @@ import { featurePermissions } from '../../utilities/permissions'; import { tooltip } from '../../utilities/tooltip'; import Collapse from '../Collapse.svelte'; - import ExternalEventTypeManagementTab from '../external-events/ExternalEventTypeManagementTab.svelte'; - import DerivationGroupManagementTab from '../external-source/DerivationGroupManagementTab.svelte'; - import ExternalSourceTypeManagementTab from '../external-source/ExternalSourceTypeManagementTab.svelte'; import Parameters from '../parameters/Parameters.svelte'; import AlertError from '../ui/AlertError.svelte'; import CssGrid from '../ui/CssGrid.svelte'; import CssGridGutter from '../ui/CssGridGutter.svelte'; + import DataGrid from '../ui/DataGrid/DataGrid.svelte'; import DataGridActions from '../ui/DataGrid/DataGridActions.svelte'; import Panel from '../ui/Panel.svelte'; import SectionTitle from '../ui/SectionTitle.svelte'; - import Tab from '../ui/Tabs/Tab.svelte'; - import TabPanel from '../ui/Tabs/TabPanel.svelte'; - import Tabs from '../ui/Tabs/Tabs.svelte'; export let user: User | null; @@ -59,10 +57,6 @@ type ModalCellRendererParamsExternalSourceType = ICellRendererParams & CellRendererParams; type ModalCellRendererParamsExternalEventType = ICellRendererParams & CellRendererParams; - const derivationGroupTabName: string = 'Derivation Group'; - const externalSourceTypeTabName: string = 'External Source Type'; - const externalEventTypeTabName: string = 'External Event Type'; - const columnSize: string = '.55fr 3px 1.5fr'; const creationPermissionError: string = 'You do not have permission to upload External Source & Event Types.'; @@ -134,6 +128,10 @@ let externalSourceTypeColumnDefs: DataGridColumnDef[] = externalSourceTypeBaseColumnDefs; let externalEventTypeColumnDefs: DataGridColumnDef[] = externalEventTypeBaseColumnDefs; + let derivationGroupDataGrid: DataGrid; + let externalSourceTypeDataGrid: DataGrid; + let externalEventTypeDataGrid: DataGrid; + let hasDeleteExternalSourceTypePermission: boolean = false; let hasDeleteExternalEventTypePermission: boolean = false; @@ -151,10 +149,8 @@ let selectedExternalEventTypeAttributesSchema: Record; let selectedExternalEventTypeParametersMap: ParametersMap = {}; - let groupsAndTypesTabs: Tabs; - // File upload variables - let fileInput: HTMLInputElement; + let fileInput: HTMLInputElement | null; let uploadResponseErrors: string[] = []; let files: FileList | undefined; let file: File | undefined; @@ -461,7 +457,9 @@ } function resetUploadForm() { - fileInput.value = ''; + if (fileInput !== null) { + fileInput.value = ''; + } file = undefined; files = undefined; uploadResponseErrors = []; @@ -478,17 +476,16 @@ if (file !== undefined && /\.json$/.test(file.name)) { uploadResponseErrors = []; const combinedSchema = await parseJSONStream<{ event_types: object; source_types: object }>(file.stream()); - const creationResponse = await effects.createExternalSourceEventTypes( + await effects.createExternalSourceEventTypes( combinedSchema.event_types, combinedSchema.source_types, user, ); - if (creationResponse !== null) { - groupsAndTypesTabs.selectTab(externalSourceTypeTabName); - } files = undefined; file = undefined; - fileInput.value = ''; + if (fileInput != null) { + fileInput.value = ''; + } parsedExternalSourceEventTypeSchema = undefined; } } @@ -762,29 +759,53 @@ {/if} -
    -
    - - - Derivation Group - External Source Type - External Event Type - - - - - - - - - - - +
    + +
    + +
    + +
    + +
    + +
    +
    From 8afe2762f7d23ef428c42e2585406bbd50df4380 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 4 Dec 2024 12:16:17 -0500 Subject: [PATCH 36/50] Update formatting for Arrays in attribute schemas, add associated sources to event types --- .../ExternalSourceManager.svelte | 2 +- .../ExternalTypeManager.svelte | 80 ++++++++++++++++--- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/components/external-source/ExternalSourceManager.svelte b/src/components/external-source/ExternalSourceManager.svelte index 877f39d9db..3437943b63 100644 --- a/src/components/external-source/ExternalSourceManager.svelte +++ b/src/components/external-source/ExternalSourceManager.svelte @@ -344,7 +344,7 @@ uploadDisabledMessage = `No external event types are currently defined, please create one before uploading an external source!`; isUploadDisabled = true; } else if ($externalSourceTypes.length === 0) { - uploadDisabledMessage = `External Source Type "${parsedExternalSource.source.source_type}" is not defined. Please create it!`; + uploadDisabledMessage = `No external source types are currently defined, please create one before uploading an external source!`; isUploadDisabled = true; } isUploadDisabled = doesSourceTypeAndEventTypesExist(parsedExternalSource); diff --git a/src/components/external-source/ExternalTypeManager.svelte b/src/components/external-source/ExternalTypeManager.svelte index 8f5cc21d8b..b8562fd899 100644 --- a/src/components/external-source/ExternalTypeManager.svelte +++ b/src/components/external-source/ExternalTypeManager.svelte @@ -139,6 +139,9 @@ let hasDeleteExternalSourceTypePermission: boolean = false; let hasDeleteExternalEventTypePermission: boolean = false; + let hasCreateExternalSourceTypePermission: boolean = false; + let hasCreateExternalEventTypePermission: boolean = false; + let hasCreationPermission: boolean = false; let derivationGroupFilterString: string = ''; let externalSourceTypeFilterString: string = ''; @@ -153,21 +156,16 @@ let selectedExternalSourceTypeParametersMap: ParametersMap = {}; let selectedExternalEventType: ExternalEventType | undefined = undefined; + let selectedExternalEventTypeSources: string[] = []; let selectedExternalEventTypeAttributesSchema: Record; let selectedExternalEventTypeParametersMap: ParametersMap = {}; - // File upload variables let fileInput: HTMLInputElement | null; let uploadResponseErrors: string[] = []; let files: FileList | undefined; let file: File | undefined; let parsedExternalSourceEventTypeSchema: ExternalSourceEventTypeSchema | undefined = undefined; - // Upload permissions - let hasCreateExternalSourceTypePermission: boolean = false; - let hasCreateExternalEventTypePermission: boolean = false; - let hasCreationPermission: boolean = false; - $: hasDeleteExternalSourceTypePermission = featurePermissions.externalSourceType.canDelete(user); $: hasDeleteExternalEventTypePermission = featurePermissions.externalEventType.canDelete(user); $: hasCreateExternalSourceTypePermission = featurePermissions.externalSourceType.canCreate(user); @@ -178,6 +176,12 @@ source => selectedDerivationGroup?.name === source.derivation_group_name, ); + $: if (selectedExternalEventType !== undefined) { + selectedExternalEventTypeSources = getAssociatedExternalSourcesByEventType(selectedExternalEventType.name); + } else { + selectedExternalEventTypeSources = []; + } + $: if (selectedExternalEventType !== undefined) { selectedExternalEventTypeAttributesSchema = translateJsonSchemaToValueSchema( selectedExternalEventType?.attribute_schema, @@ -193,6 +197,7 @@ {} as ParametersMap, ); } + $: if (selectedExternalSourceType !== undefined) { selectedExternalSourceTypeAttributeSchema = translateJsonSchemaToValueSchema( selectedExternalSourceType?.attribute_schema, @@ -323,6 +328,15 @@ $: externalEventTypeColumnDefs = [ ...externalEventTypeBaseColumnDefs, + { + filter: 'number', + headerName: 'Associated External Sources', + sortable: true, + valueFormatter: params => { + const associatedExternalSources = getAssociatedExternalSourcesByEventType(params.data?.name); + return `${associatedExternalSources.length}`; + }, + }, { cellClass: 'action-cell-container', cellRenderer: (params: ModalCellRendererParamsExternalEventType) => { @@ -365,7 +379,7 @@ } function deleteExternalSourceType(sourceType: ExternalSourceType) { - // makes sure all associated derivation groups are deleted before this + // Makes sure all associated derivation groups are deleted before this showDeleteExternalEventSourceTypeModal( sourceType, 'External Source Type', @@ -375,7 +389,7 @@ } function deleteExternalEventType(eventType: ExternalEventType) { - // makes sure all associated sources (and therefore events, as orphans are not possible) are deleted before this + // Makes sure all associated sources (and therefore events, as orphans are not possible) are deleted before this // NOTE: does not update in derivation_group_comp after removing a EE type; derivation_group_comp defaults to 0 event types after its last external source removed, // as it has no awareness of external source type or paired events (as the latter don't even exist). showDeleteExternalEventSourceTypeModal( @@ -386,6 +400,14 @@ ); } + function getAssociatedExternalSourcesByEventType(eventType: string | undefined) { + if (eventType === undefined) { + return []; + } + const associatedSources = $sourcesUsingExternalEventTypes.filter(entry => entry.types.includes(eventType)).map(entry => entry.key); // NOTE: MAY NEED TO REMOVE THIS - COULD BE A VERY SLOW OPERATION. + return associatedSources; + } + function getAssociatedExternalSourcesBySourceType(sourceType: string | undefined) { if (sourceType === undefined) { return []; @@ -694,7 +716,15 @@ {#if attribute[0] !== 'properties'}
    {attribute[0]}
    -
    {attribute[1]}
    + {#if Array.isArray(attribute[1])} +
      + {#each attribute[1] as attributeValue} +
    • {attributeValue}
      • + {/each} +
    + {:else} +
    {attribute[1]}
    + {/if}
    {/if} {/each} @@ -731,6 +761,19 @@ + + {#if selectedExternalEventTypeSources.length > 0} + {#each selectedExternalEventTypeSources as associatedSource} +
  • {associatedSource}
    • + {/each} + {:else} + {`No External Sources using ${selectedExternalEventType.name}`} + {/if} +
    {attribute[0]}
    -
    {attribute[1]}
    + {#if Array.isArray(attribute[1])} +
      + {#each attribute[1] as attributeValue} +
    • {attributeValue}
      • + {/each} +
    + {:else} +
    {attribute[1]}
    + {/if}
    {/if} {/each} @@ -837,6 +888,10 @@ diff --git a/src/components/external-source/ExternalTypeManager.svelte b/src/components/external-source/ExternalTypeManager.svelte index bca960a42c..6aa48bf4a0 100644 --- a/src/components/external-source/ExternalTypeManager.svelte +++ b/src/components/external-source/ExternalTypeManager.svelte @@ -506,7 +506,7 @@ file = files[0]; if (file !== undefined && /\.json$/.test(file.name)) { uploadResponseErrors = []; - const combinedSchema = await parseJSONStream<{ event_types: object; source_types: object }>(file.stream()); + const combinedSchema = await parseJSONStream(file.stream()); await effects.createExternalSourceEventTypes(combinedSchema.event_types, combinedSchema.source_types, user); files = undefined; file = undefined; @@ -523,7 +523,7 @@ try { parsedExternalSourceEventTypeSchema = await parseJSONStream(stream); - if (!parsedExternalSourceEventTypeSchema.event_types || !parsedExternalSourceEventTypeSchema.source_types) { + if (!parsedExternalSourceEventTypeSchema.event_types && !parsedExternalSourceEventTypeSchema.source_types) { parsedExternalSourceEventTypeSchema = undefined; throw new Error('External Source & Event Type Schema has Invalid Format'); } @@ -584,17 +584,21 @@ {#if parsedExternalSourceEventTypeSchema !== undefined}
    The following External Source Type(s) will be created
    -
      - {#each Object.keys(parsedExternalSourceEventTypeSchema.source_types) as newSourceTypeName} -
    • {newSourceTypeName}
      • - {/each} -
    + {#if parsedExternalSourceEventTypeSchema.source_types} +
      + {#each Object.keys(parsedExternalSourceEventTypeSchema.source_types) as newSourceTypeName} +
    • {newSourceTypeName}
      • + {/each} +
    + {/if}
    The following External Event Type(s) will be created
    -
      - {#each Object.keys(parsedExternalSourceEventTypeSchema.event_types) as newEventTypeName} -
    • {newEventTypeName}
      • - {/each} -
    + {#if parsedExternalSourceEventTypeSchema.event_types} +
      + {#each Object.keys(parsedExternalSourceEventTypeSchema.event_types) as newEventTypeName} +
    • {newEventTypeName}
      • + {/each} +
    + {/if}
    {/if}
    diff --git a/src/types/external-source.ts b/src/types/external-source.ts index 90528ddb6d..6030496be4 100644 --- a/src/types/external-source.ts +++ b/src/types/external-source.ts @@ -51,8 +51,8 @@ export type PlanDerivationGroup = { }; export type ExternalSourceEventTypeSchema = { - event_types: SchemaObject; - source_types: SchemaObject; + event_types?: SchemaObject; + source_types?: SchemaObject; }; export type ExternalSourceType = { diff --git a/src/utilities/effects.ts b/src/utilities/effects.ts index 8b68ab449d..e5b89b8d75 100644 --- a/src/utilities/effects.ts +++ b/src/utilities/effects.ts @@ -1010,28 +1010,40 @@ const effects = { } }, - async createExternalSourceEventTypes(eventTypes: object, sourceTypes: object, user: User | null) { + async createExternalSourceEventTypes(eventTypes: object | undefined, sourceTypes: object | undefined, user: User | null) { if (!gatewayPermissions.CREATE_EXTERNAL_EVENT_TYPE(user) || !gatewayPermissions.CREATE_EXTERNAL_SOURCE_TYPE(user)) { throwPermissionError('create en external source or event type'); } + if (eventTypes === undefined && sourceTypes === undefined) { + showFailureToast('Neither External Source Nor Event Type Not Specified'); + return false; + } createExternalSourceEventTypeError.set(null); try { // Extract External Source Type schema & format for request to Gateway const body = new FormData(); - body.append('event_types', JSON.stringify(eventTypes)); - body.append('source_types', JSON.stringify(sourceTypes)); + if (eventTypes !== undefined) { + body.append('event_types', JSON.stringify(eventTypes)); + } + if (sourceTypes !== undefined) { + body.append('source_types', JSON.stringify(sourceTypes)); + } const response = await reqGateway(`/uploadExternalSourceEventTypes`, 'POST', body, user, true); if (response?.errors === undefined) { showSuccessToast('External Source & Event Type Created Successfully'); + return true; } else { showFailureToast('External Source & Event Type Create Failed'); + console.log(response.errors) + return false; } } catch (e) { showFailureToast('External Source & Event Type Create Failed'); createExternalSourceEventTypeError.set((e as Error).message); catchError(e as Error); + return false; } }, From efc7a1dc9de6a7a7b2c2a8afab90df00469a6721 Mon Sep 17 00:00:00 2001 From: psubram3 Date: Fri, 3 Jan 2025 09:20:20 -0800 Subject: [PATCH 44/50] show required attributes --- .../external-events/ExternalEventForm.svelte | 12 +++++++++++- src/components/parameters/ParameterInfo.svelte | 10 +++++++++- src/types/parameter.ts | 1 + src/utilities/parameters.ts | 1 + 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/components/external-events/ExternalEventForm.svelte b/src/components/external-events/ExternalEventForm.svelte index ff05b69aee..ac7f334896 100644 --- a/src/components/external-events/ExternalEventForm.svelte +++ b/src/components/external-events/ExternalEventForm.svelte @@ -27,6 +27,7 @@ let externalEventType: ExternalEventType | undefined = undefined; let externalEventTypeParametersMap: ParametersMap = {}; let startTimeField: FieldStore; + let requiredAttributes: string[] = []; $: externalEventType = $externalEventTypes.find(eventType => eventType.name === externalEvent.pkey.event_type_name); $: externalEventAttributes = translateJsonSchemaArgumentsToValueSchema(externalEvent.attributes); @@ -44,6 +45,11 @@ }, {} as ParametersMap, ); + + // Get the set of required attribute names + requiredAttributes = + $externalEventTypes.find(eventType => eventType.name === externalEvent?.pkey.event_type_name)?.attribute_schema + .required ?? []; } $: startTimeField = field(`${formatDate(new Date(externalEvent.start_time), $plugins.time.primary.format)}`); @@ -101,7 +107,11 @@
    diff --git a/src/components/parameters/ParameterInfo.svelte b/src/components/parameters/ParameterInfo.svelte index 55baf914eb..208829d9c0 100644 --- a/src/components/parameters/ParameterInfo.svelte +++ b/src/components/parameters/ParameterInfo.svelte @@ -20,10 +20,14 @@ let leaveTimeout: NodeJS.Timeout | null = null; let source: ValueSource; let unit: string | undefined = undefined; + let required: boolean = true; + let externalEvent: boolean = false; $: if (formParameter) { source = formParameter.valueSource; unit = formParameter.schema?.metadata?.unit?.value; + externalEvent = formParameter.externalEvent ?? false; + required = formParameter.required ?? true; } function leaveCallback() { @@ -75,7 +79,7 @@ } -{#if unit || source !== 'none'} +{#if externalEvent || unit || source !== 'none'} {/if} + {#if externalEvent} +
    Required
    +
    {required}
    + {/if}
    diff --git a/src/types/parameter.ts b/src/types/parameter.ts index 33f7929755..63e8f465fe 100644 --- a/src/types/parameter.ts +++ b/src/types/parameter.ts @@ -8,6 +8,7 @@ export type EffectiveArguments = { export type FormParameter = { errors: string[] | null; + externalEvent?: boolean; file?: File; index?: number; key?: string; diff --git a/src/utilities/parameters.ts b/src/utilities/parameters.ts index 123fbd51d9..c8e0632798 100644 --- a/src/utilities/parameters.ts +++ b/src/utilities/parameters.ts @@ -73,6 +73,7 @@ export function getFormParameters( const formParameter: FormParameter = { errors: null, + externalEvent: true, name, order, required, From e04e89c1d813b53284bbc4b00bd36f36620482a9 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Fri, 3 Jan 2025 14:17:19 -0500 Subject: [PATCH 45/50] Fix bug with not displaying objects/structs nested within arrays/series --- src/utilities/parameters.ts | 80 ++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 24 deletions(-) diff --git a/src/utilities/parameters.ts b/src/utilities/parameters.ts index c8e0632798..7ef7a0eadf 100644 --- a/src/utilities/parameters.ts +++ b/src/utilities/parameters.ts @@ -173,27 +173,51 @@ export function translateJsonSchemaToValueSchema(jsonSchema: SchemaObject | unde }; const propTranslated = translateJsonSchemaTypeToValueSchema(propType as JSONType, propProperties, propItems); if ('items' in propTranslated) { - propTranslated.items = Object.entries(propTranslated.items).reduce( - (acc: Record, currentItem: [string, ValueSchema]) => { - const { - type: currentType, - properties: currentProperties, - items: currentItems, - } = currentItem[1] as { - items?: Record<'type', JSONType>; - properties?: Record; - type: JSONType; - }; - const translatedItem = translateJsonSchemaTypeToValueSchema( - currentType as JSONType, - currentProperties, - currentItems, - ); - acc[currentItem[0]] = translatedItem; - return acc; - }, - {} as Record, - ); + if (propTranslated.items.type === 'struct') { + propTranslated.items.items = Object.entries(propTranslated.items.items).reduce( + (acc: Record, currentItem: [string, ValueSchema]) => { + const { + type: currentType, + properties: currentProperties, + items: currentItems, + } = currentItem[1] as { + items?: Record<'type', JSONType>; + properties?: Record; + type: JSONType; + }; + const translatedItem = translateJsonSchemaTypeToValueSchema( + currentType as JSONType, + currentProperties, + currentItems, + ); + acc[currentItem[0]] = translatedItem; + return acc; + }, + {} as Record, + ); + } else { + propTranslated.items = Object.entries(propTranslated.items).reduce( + (acc: Record, currentItem: [string, ValueSchema]) => { + const { + type: currentType, + properties: currentProperties, + items: currentItems, + } = currentItem[1] as { + items?: Record<'type', JSONType>; + properties?: Record; + type: JSONType; + }; + const translatedItem = translateJsonSchemaTypeToValueSchema( + currentType as JSONType, + currentProperties, + currentItems, + ); + acc[currentItem[0]] = translatedItem; + return acc; + }, + {} as Record, + ); + } } propertiesAsValueSchema[propName] = propTranslated; } else { @@ -206,7 +230,7 @@ export function translateJsonSchemaToValueSchema(jsonSchema: SchemaObject | unde function translateJsonSchemaTypeToValueSchema( jsonSchemaType: JSONType, jsonSchemaProperties?: Record, - jsonSchemaItems?: Record<'type', JSONType>, + jsonSchemaItems?: Record | { properties: Record, type: string } ): ValueSchema { if (jsonSchemaType === 'number' || jsonSchemaType === 'integer') { return { type: 'int' } as ValueSchemaInt; @@ -224,8 +248,16 @@ function translateJsonSchemaTypeToValueSchema( throw new Error('Cannot convert "array" from JSON Schema without an "items" field defined'); } // ValueSchema expects a singular type for the series where JSON Schema allows multiple. Take the first if the user gave multiple - const firstItem = Object.entries(jsonSchemaItems)[0]; - const translatedItem: ValueSchema = translateJsonSchemaTypeToValueSchema(firstItem[1]); + const firstItemType = jsonSchemaItems.type; + + let translatedItem: ValueSchema | undefined = undefined; + // Required to properly nest objects within arrays (i.e., structs within series) + if (firstItemType === 'object') { + const nestedProperties = jsonSchemaItems.properties as Record; + translatedItem = translateJsonSchemaTypeToValueSchema(firstItemType, nestedProperties); + } else { + translatedItem = translateJsonSchemaTypeToValueSchema(firstItemType as JSONType); + } return { items: translatedItem, type: 'series' } as ValueSchemaSeries; } else { return { type: jsonSchemaType } as ValueSchema; From 7273047f20c9de88b22eb71d6b9746238df17f10 Mon Sep 17 00:00:00 2001 From: psubram3 Date: Fri, 3 Jan 2025 11:46:36 -0800 Subject: [PATCH 46/50] update tests --- e2e-tests/data/Schema_Example_Source.json | 8 +++- e2e-tests/data/example-external-source.json | 2 +- .../data/example-external-source_no-attr.json | 21 +++++++++ e2e-tests/fixtures/ExternalSources.ts | 5 ++ e2e-tests/tests/external-sources.test.ts | 47 ++++++++++++++++++- 5 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 e2e-tests/data/example-external-source_no-attr.json diff --git a/e2e-tests/data/Schema_Example_Source.json b/e2e-tests/data/Schema_Example_Source.json index 9e01ad06e5..7835931784 100644 --- a/e2e-tests/data/Schema_Example_Source.json +++ b/e2e-tests/data/Schema_Example_Source.json @@ -10,6 +10,9 @@ }, "example2": { "type": "number" + }, + "optional": { + "type": "string" } }, "required": [ @@ -24,6 +27,9 @@ "description": "Schema for the attributes of the Example External Source External Source Type.", "type": "object", "properties": { + "optional": { + "type": "string" + }, "version": { "type": "number" }, @@ -37,4 +43,4 @@ ] } } -} +} \ No newline at end of file diff --git a/e2e-tests/data/example-external-source.json b/e2e-tests/data/example-external-source.json index 5eaa9dff1b..11ad80a570 100644 --- a/e2e-tests/data/example-external-source.json +++ b/e2e-tests/data/example-external-source.json @@ -24,4 +24,4 @@ } } ] -} +} \ No newline at end of file diff --git a/e2e-tests/data/example-external-source_no-attr.json b/e2e-tests/data/example-external-source_no-attr.json new file mode 100644 index 0000000000..e08e9762fc --- /dev/null +++ b/e2e-tests/data/example-external-source_no-attr.json @@ -0,0 +1,21 @@ +{ + "source": { + "key": "ExampleExternalSource:example-external-source_no-attr.json", + "source_type": "Empty External Source", + "valid_at": "2024-001T00:00:00Z", + "period": { + "start_time": "2022-001T00:00:00Z", + "end_time": "2022-002T00:00:00Z" + }, + "attributes": {} + }, + "events": [ + { + "key": "EmptyEvent:1/sc/sc1:1:X/1", + "event_type": "EmptyEvent", + "start_time": "2022-001T12:00:00Z", + "duration": "00:00:00", + "attributes": {} + } + ] +} \ No newline at end of file diff --git a/e2e-tests/fixtures/ExternalSources.ts b/e2e-tests/fixtures/ExternalSources.ts index 69b5378003..8421b6b0f6 100644 --- a/e2e-tests/fixtures/ExternalSources.ts +++ b/e2e-tests/fixtures/ExternalSources.ts @@ -30,6 +30,9 @@ export class ExternalSources { deselectEventButton: Locator; deselectSourceButton: Locator; exampleDerivationGroup: string = 'Example External Source Default'; + exampleEmptyDerivationGroup: string = 'Empty External Source Default'; + exampleEmptyEventType: string = 'EmptyEvent'; + exampleEmptySourceType: string = 'Empty External Source'; exampleEventType: string = 'ExampleEvent'; exampleSourceType: string = 'Example External Source'; exampleTypeSchema: string = 'e2e-tests/data/Schema_Example_Source.json'; @@ -44,6 +47,8 @@ export class ExternalSources { externalSourceFilePath: string = 'e2e-tests/data/example-external-source.json'; externalSourceFilePathMissingField: string = 'e2e-tests/data/example-external-source-missing-field.json'; externalSourceFilePathSyntaxError: string = 'e2e-tests/data/example-external-source-syntax-error.json'; + externalSourceNoAttributeFileName: string = 'example-external-source_no-attr.json'; + externalSourceNoAttributeFilePath: string = 'e2e-tests/data/example-external-source_no-attr.json'; externalSourceSelectedForm: Locator; externalSourceTypeName: string = 'Example External Source'; externalSourceUpload: Locator; diff --git a/e2e-tests/tests/external-sources.test.ts b/e2e-tests/tests/external-sources.test.ts index 6bdf9ff2b6..eac018302a 100644 --- a/e2e-tests/tests/external-sources.test.ts +++ b/e2e-tests/tests/external-sources.test.ts @@ -42,6 +42,16 @@ test.describe.serial('External Sources', () => { await expect(externalSources.inputFile).not.toBeVisible(); }); + test('Optional argument should be marked in external event form', async () => { + await externalSources.selectEvent('ExampleEvent:1/sc/sc1:1'); + await page.click('text="Attributes"') + const parameter = page.locator('.parameter').filter({ hasText: 'optional' }).first(); + parameter.hover(); + const parameterInfo = parameter.getByRole('contentinfo'); + await parameterInfo.hover(); + await expect(page.locator('.parameter-info-values').filter({ hasText: 'Required' }).filter({ hasText: 'false' })).toBeVisible(); + }); + test('External source form should be shown when a source is selected', async () => { await externalSources.selectSource(); await expect(page.locator('.external-source-header-title-value')).toBeVisible(); @@ -77,17 +87,52 @@ test.describe.serial('External Sources', () => { await expect(externalSources.externalEventTableHeaderDuration).toBeVisible(); }); - test('Deleting an external source', async () => { + test('Create Empty Source and Event Type', async () => { + await externalSources.uploadExternalSource( + externalSources.externalSourceNoAttributeFilePath, + externalSources.externalSourceNoAttributeFileName, + false + ); + + await externalSources.gotoTypeManager(); + + const externalSourceTypeTable = await externalSources.page.locator('.external-source-type-table'); + const externalEventTypeTable = await externalSources.page.locator('.external-event-type-table'); + + const sourceType = await externalSourceTypeTable.getByRole('gridcell').filter({ hasText: 'Empty External Source' }); + await sourceType.hover(); + await page.getByRole('button', { name: 'View External Source Type' }).click(); + await expect(page.locator('text="Attribute Schema - Properties"')).toBeVisible() + const sourceTypeAttributes = await page.locator('text="Attribute Schema - Properties"'); + await sourceTypeAttributes.click(); + await expect(page.locator('.parameter')).toHaveCount(0); + + const eventType = await externalEventTypeTable.getByRole('gridcell').filter({ hasText: 'EmptyEvent' }); + await eventType.hover(); + await page.getByRole('button', { name: 'View External Event Type' }).click(); + await expect(page.locator('text="Attribute Schema - Properties"')).toBeVisible() + const eventTypeAttributes = await page.locator('text="Attribute Schema - Properties"'); + await eventTypeAttributes.click(); + await expect(page.locator('.parameter')).toHaveCount(0); + + await externalSources.goto(); + }) + + test('Deleting all external sources', async () => { await expect(externalSources.externalSourcesTable).toBeVisible(); await externalSources.deleteSource(externalSources.externalSourceFileName); + await externalSources.deleteSource(externalSources.externalSourceNoAttributeFileName); await expect(page.getByText('External Source Deleted Successfully')).toBeVisible(); await expect(externalSources.inputFile).toBeVisible(); await expect(externalSources.externalEventSelectedForm).not.toBeVisible(); await expect(externalSources.externalSourceSelectedForm).not.toBeVisible(); await externalSources.gotoTypeManager(); await externalSources.deleteDerivationGroup(externalSources.exampleDerivationGroup); + await externalSources.deleteDerivationGroup(externalSources.exampleEmptyDerivationGroup); await externalSources.deleteExternalSourceType(externalSources.exampleSourceType); + await externalSources.deleteExternalSourceType(externalSources.exampleEmptySourceType); await externalSources.deleteExternalEventType(externalSources.exampleEventType); + await externalSources.deleteExternalEventType(externalSources.exampleEmptyEventType); }); }); From ea032522e3b4b848d59a8d6791b1143e95ca09df Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 19 Nov 2024 06:27:05 -0800 Subject: [PATCH 47/50] handle sources with some defined and some undefined types more clearly, improve error handling --- .../ExternalSourceManager.svelte | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/components/external-source/ExternalSourceManager.svelte b/src/components/external-source/ExternalSourceManager.svelte index e3048399f0..c3d04169f3 100644 --- a/src/components/external-source/ExternalSourceManager.svelte +++ b/src/components/external-source/ExternalSourceManager.svelte @@ -343,8 +343,12 @@ }); $: if (parsedExternalSource !== undefined) { - doesSourceTypeAndEventTypesExist(parsedExternalSource); - isUploadDisabled = false; + if (doesSourceTypeAndEventTypesExist(parsedExternalSource)) { + isUploadDisabled = false; + uploadDisabledMessage = null; + } else { + isUploadDisabled = true; + } } else { isUploadDisabled = true; uploadDisabledMessage = null; @@ -511,24 +515,37 @@ const externalSourceType = $externalSourceTypes.find( sourceType => sourceType.name === externalSource.source.source_type, ); - if (externalSourceType === undefined) { + if (externalSourceType === undefined && Object.keys(externalSource.source.attributes).length === 0) { newExternalSourceType = externalSource.source.source_type; } // Check that all the External Event Types for the source to-be-uploaded exist - const newSourceExternalEventTypes: string[] = Array.from( - new Set(externalSource.events.map(event => event.event_type)), - ); + const newSourceExternalEventTypes: { [event_type: string]: string[] } = {}; + for (const event of externalSource.events) { + if (newSourceExternalEventTypes[event.event_type] === undefined) { + newSourceExternalEventTypes[event.event_type] = Object.keys(event.attributes); + } else if (newSourceExternalEventTypes[event.event_type] !== Object.keys(event.attributes)) { + // if there is a mismatch, cause error + console.log(newSourceExternalEventTypes[event.event_type], Object.keys(event.attributes)); + uploadDisabledMessage = 'Event attributes are inconsistent across events of type ' + event.event_type + '.'; + return false; + } + } let eventTypes: string[] = []; - for (const eventType of newSourceExternalEventTypes) { - if ($externalEventTypes.find(eventTypeFromDB => eventTypeFromDB.name === eventType) === undefined) { - eventTypes.push(eventType); + for (const entry of Object.entries(newSourceExternalEventTypes)) { + if ( + $externalEventTypes.find(eventTypeFromDB => eventTypeFromDB.name === entry[0]) === undefined && + entry[1].length === 0 // only create new types if they don't have attributes. otherwise, a schema is needed as we do not infer one + ) { + eventTypes.push(entry[0]); } } if (eventTypes.length > 0) { + console.log(externalSource.events); newExternalEventTypes = eventTypes; } + return true; } From 8614e5e2b08b42e9a0b031a0468a66daad4c808f Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 8 Jan 2025 09:27:14 -0500 Subject: [PATCH 48/50] Formatting fix --- e2e-tests/tests/external-sources.test.ts | 14 ++++++++------ src/utilities/effects.ts | 8 ++++++-- src/utilities/parameters.ts | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/e2e-tests/tests/external-sources.test.ts b/e2e-tests/tests/external-sources.test.ts index eac018302a..33455e5c7c 100644 --- a/e2e-tests/tests/external-sources.test.ts +++ b/e2e-tests/tests/external-sources.test.ts @@ -44,12 +44,14 @@ test.describe.serial('External Sources', () => { test('Optional argument should be marked in external event form', async () => { await externalSources.selectEvent('ExampleEvent:1/sc/sc1:1'); - await page.click('text="Attributes"') + await page.click('text="Attributes"'); const parameter = page.locator('.parameter').filter({ hasText: 'optional' }).first(); parameter.hover(); const parameterInfo = parameter.getByRole('contentinfo'); await parameterInfo.hover(); - await expect(page.locator('.parameter-info-values').filter({ hasText: 'Required' }).filter({ hasText: 'false' })).toBeVisible(); + await expect( + page.locator('.parameter-info-values').filter({ hasText: 'Required' }).filter({ hasText: 'false' }), + ).toBeVisible(); }); test('External source form should be shown when a source is selected', async () => { @@ -91,7 +93,7 @@ test.describe.serial('External Sources', () => { await externalSources.uploadExternalSource( externalSources.externalSourceNoAttributeFilePath, externalSources.externalSourceNoAttributeFileName, - false + false, ); await externalSources.gotoTypeManager(); @@ -102,7 +104,7 @@ test.describe.serial('External Sources', () => { const sourceType = await externalSourceTypeTable.getByRole('gridcell').filter({ hasText: 'Empty External Source' }); await sourceType.hover(); await page.getByRole('button', { name: 'View External Source Type' }).click(); - await expect(page.locator('text="Attribute Schema - Properties"')).toBeVisible() + await expect(page.locator('text="Attribute Schema - Properties"')).toBeVisible(); const sourceTypeAttributes = await page.locator('text="Attribute Schema - Properties"'); await sourceTypeAttributes.click(); await expect(page.locator('.parameter')).toHaveCount(0); @@ -110,13 +112,13 @@ test.describe.serial('External Sources', () => { const eventType = await externalEventTypeTable.getByRole('gridcell').filter({ hasText: 'EmptyEvent' }); await eventType.hover(); await page.getByRole('button', { name: 'View External Event Type' }).click(); - await expect(page.locator('text="Attribute Schema - Properties"')).toBeVisible() + await expect(page.locator('text="Attribute Schema - Properties"')).toBeVisible(); const eventTypeAttributes = await page.locator('text="Attribute Schema - Properties"'); await eventTypeAttributes.click(); await expect(page.locator('.parameter')).toHaveCount(0); await externalSources.goto(); - }) + }); test('Deleting all external sources', async () => { await expect(externalSources.externalSourcesTable).toBeVisible(); diff --git a/src/utilities/effects.ts b/src/utilities/effects.ts index e5b89b8d75..799d459740 100644 --- a/src/utilities/effects.ts +++ b/src/utilities/effects.ts @@ -1010,7 +1010,11 @@ const effects = { } }, - async createExternalSourceEventTypes(eventTypes: object | undefined, sourceTypes: object | undefined, user: User | null) { + async createExternalSourceEventTypes( + eventTypes: object | undefined, + sourceTypes: object | undefined, + user: User | null, + ) { if (!gatewayPermissions.CREATE_EXTERNAL_EVENT_TYPE(user) || !gatewayPermissions.CREATE_EXTERNAL_SOURCE_TYPE(user)) { throwPermissionError('create en external source or event type'); } @@ -1036,7 +1040,7 @@ const effects = { return true; } else { showFailureToast('External Source & Event Type Create Failed'); - console.log(response.errors) + console.log(response.errors); return false; } } catch (e) { diff --git a/src/utilities/parameters.ts b/src/utilities/parameters.ts index 7ef7a0eadf..4ceb37a635 100644 --- a/src/utilities/parameters.ts +++ b/src/utilities/parameters.ts @@ -230,7 +230,7 @@ export function translateJsonSchemaToValueSchema(jsonSchema: SchemaObject | unde function translateJsonSchemaTypeToValueSchema( jsonSchemaType: JSONType, jsonSchemaProperties?: Record, - jsonSchemaItems?: Record | { properties: Record, type: string } + jsonSchemaItems?: Record | { properties: Record; type: string }, ): ValueSchema { if (jsonSchemaType === 'number' || jsonSchemaType === 'integer') { return { type: 'int' } as ValueSchemaInt; From 1b6584ba183a4d642fb40c371444d991840f2427 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 8 Jan 2025 09:41:21 -0500 Subject: [PATCH 49/50] Add null condition for source/event types to be created --- src/components/modals/ManageGroupsAndTypesModal.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/modals/ManageGroupsAndTypesModal.svelte b/src/components/modals/ManageGroupsAndTypesModal.svelte index 1f0f702711..16e9ad4b44 100644 --- a/src/components/modals/ManageGroupsAndTypesModal.svelte +++ b/src/components/modals/ManageGroupsAndTypesModal.svelte @@ -585,11 +585,11 @@ {#if parsedExternalSourceEventTypeSchema !== undefined}
    The following External Source Type(s) will be created
    - {#each Object.keys(parsedExternalSourceEventTypeSchema.source_types) as newSourceTypeName} + {#each Object.keys(parsedExternalSourceEventTypeSchema.source_types ?? {}) as newSourceTypeName}
  • {newSourceTypeName}
    • {/each}
    The following External Event Type(s) will be created
    - {#each Object.keys(parsedExternalSourceEventTypeSchema.event_types) as newEventTypeName} + {#each Object.keys(parsedExternalSourceEventTypeSchema.event_types ?? {}) as newEventTypeName}
  • {newEventTypeName}
    • {/each}
    From a284cae09498110a246e6bf7fd00ee2511752f45 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 8 Jan 2025 09:55:34 -0500 Subject: [PATCH 50/50] Remove console.logs --- src/components/external-source/ExternalSourceManager.svelte | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/components/external-source/ExternalSourceManager.svelte b/src/components/external-source/ExternalSourceManager.svelte index c3d04169f3..30be4efcb8 100644 --- a/src/components/external-source/ExternalSourceManager.svelte +++ b/src/components/external-source/ExternalSourceManager.svelte @@ -403,8 +403,6 @@ ? await effects.createExternalSourceEventTypes(newEventTypes, newSourceType, user) : true; - console.log('CREATED TYPES: ', createdTypes); - if (createdTypes) { const requestResponse: | { createExternalSource: ExternalSourceSlim; upsertDerivationGroup: { name: string } | null } @@ -526,7 +524,6 @@ newSourceExternalEventTypes[event.event_type] = Object.keys(event.attributes); } else if (newSourceExternalEventTypes[event.event_type] !== Object.keys(event.attributes)) { // if there is a mismatch, cause error - console.log(newSourceExternalEventTypes[event.event_type], Object.keys(event.attributes)); uploadDisabledMessage = 'Event attributes are inconsistent across events of type ' + event.event_type + '.'; return false; } @@ -542,7 +539,6 @@ } } if (eventTypes.length > 0) { - console.log(externalSource.events); newExternalEventTypes = eventTypes; } return true;