From 081ff5e82cf95e3a60ae9bcbbeb2e5261499d3a7 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Thu, 12 Dec 2024 20:25:52 +0200 Subject: [PATCH 01/17] updated graphql schema --- resources/schema.graphql | 142 ++-- src/app/api/dataset-flow.api.spec.ts | 16 +- src/app/api/dataset-flow.api.ts | 153 ++--- .../account-dataset-flows-paused.graphql | 2 +- .../gql/account/account-pause-flows.graphql | 2 +- .../gql/account/account-resume-flows.graphql | 2 +- .../dataset-flow-compacting.graphql | 84 +-- .../dataset-all-flows-paused.graphql | 2 +- .../flows-dataset/dataset-pause-flows.graphql | 2 +- .../dataset-resume-flows.graphql | 2 +- .../fragments/fragment-dataset-flow.graphql | 18 +- .../fragment-flow-history-data.graphql | 2 +- .../dataset-flow-batching.graphql | 90 +-- .../dataset-flow-configs.graphql | 21 +- .../dataset-flow-schedule.graphql | 90 +-- src/app/api/kamu.graphql.interface.ts | 622 +++++------------- .../flows-table/flows-table.component.ts | 57 +- .../flows-table/flows-table.helpers.mock.ts | 7 +- .../flows-table/flows-table.helpers.spec.ts | 2 +- .../flow-details-history-tab.helpers.ts | 8 +- .../services/dataset-compaction.service.ts | 68 +- .../services/dataset-scheduling.service.ts | 105 ++- ...taset-settings-compacting-tab.component.ts | 72 +- ...taset-settings-scheduling-tab.component.ts | 172 +++-- .../services/dataset-flows.service.ts | 6 +- .../overview-component/overview.component.ts | 5 + src/app/services/account.service.ts | 6 +- 27 files changed, 711 insertions(+), 1047 deletions(-) diff --git a/resources/schema.graphql b/resources/schema.graphql index 1dfc33404..c8d2c19f4 100644 --- a/resources/schema.graphql +++ b/resources/schema.graphql @@ -80,18 +80,6 @@ type AccountEdge { node: Account! } -type AccountFlowConfigs { - """ - Checks if all configs of all datasets in account are disabled - """ - allPaused: Boolean! -} - -type AccountFlowConfigsMut { - resumeAccountDatasetFlows: Boolean! - pauseAccountDatasetFlows: Boolean! -} - input AccountFlowFilters { byFlowType: DatasetFlowType byStatus: FlowStatus @@ -104,19 +92,31 @@ type AccountFlowRuns { listDatasetsWithFlow: DatasetConnection! } +type AccountFlowTriggers { + """ + Checks if all triggers of all datasets in account are disabled + """ + allPaused: Boolean! +} + +type AccountFlowTriggersMut { + resumeAccountDatasetFlows: Boolean! + pauseAccountDatasetFlows: Boolean! +} + type AccountFlows { """ Returns interface for flow runs queries """ runs: AccountFlowRuns! """ - Returns interface for flow configurations queries + Returns interface for flow triggers queries """ - configs: AccountFlowConfigs! + triggers: AccountFlowTriggers! } type AccountFlowsMut { - configs: AccountFlowConfigsMut! + triggers: AccountFlowTriggersMut! } scalar AccountID @@ -200,6 +200,11 @@ type AuthMut { revokeAccessToken(tokenId: AccessTokenID!): RevokeResult! } +input BatchingInput { + minRecordsToAwait: Int! + maxBatchingInterval: TimeDeltaInput! +} + type BlockRef { name: String! blockHash: Multihash! @@ -565,18 +570,10 @@ type DatasetFlowConfigs { Returns defined configuration for a flow of specified type """ byType(datasetFlowType: DatasetFlowType!): FlowConfiguration - """ - Checks if all configs of this dataset are disabled - """ - allPaused: Boolean! } type DatasetFlowConfigsMut { - setConfigIngest(datasetFlowType: DatasetFlowType!, paused: Boolean!, ingest: IngestConditionInput!): SetFlowConfigResult! - setConfigTransform(datasetFlowType: DatasetFlowType!, paused: Boolean!, transform: TransformConditionInput!): SetFlowTransformConfigResult! - setConfigCompaction(datasetFlowType: DatasetFlowType!, compactionArgs: CompactionConditionInput!): SetFlowCompactionConfigResult! - pauseFlows(datasetFlowType: DatasetFlowType): Boolean! - resumeFlows(datasetFlowType: DatasetFlowType): Boolean! + setConfig(datasetFlowType: DatasetFlowType!, configInput: FlowConfigurationInput!): SetFlowConfigResult! } input DatasetFlowFilters { @@ -596,6 +593,23 @@ type DatasetFlowRunsMut { cancelScheduledTasks(flowId: FlowID!): CancelScheduledTasksResult! } +type DatasetFlowTriggers { + """ + Returns defined trigger for a flow of specified type + """ + byType(datasetFlowType: DatasetFlowType!): FlowTrigger + """ + Checks if all triggers of this dataset are disabled + """ + allPaused: Boolean! +} + +type DatasetFlowTriggersMut { + setTrigger(datasetFlowType: DatasetFlowType!, paused: Boolean!, triggerInput: FlowTriggerInput!): SetFlowTriggerResult! + pauseFlows(datasetFlowType: DatasetFlowType): Boolean! + resumeFlows(datasetFlowType: DatasetFlowType): Boolean! +} + enum DatasetFlowType { INGEST EXECUTE_TRANSFORM @@ -609,6 +623,10 @@ type DatasetFlows { """ configs: DatasetFlowConfigs! """ + Returns interface for flow triggers queries + """ + triggers: DatasetFlowTriggers! + """ Returns interface for flow runs queries """ runs: DatasetFlowRuns! @@ -617,6 +635,7 @@ type DatasetFlows { type DatasetFlowsMut { configs: DatasetFlowConfigsMut! runs: DatasetFlowRunsMut! + triggers: DatasetFlowTriggersMut! } scalar DatasetID @@ -978,7 +997,7 @@ type Flow { """ Primary flow trigger """ - primaryTrigger: FlowTrigger! + primaryTrigger: FlowTriggerType! """ Start condition """ @@ -994,9 +1013,7 @@ type FlowAbortedResult { } type FlowConfiguration { - paused: Boolean! ingest: FlowConfigurationIngest - transform: FlowConfigurationTransform compaction: FlowConfigurationCompaction reset: FlowConfigurationReset } @@ -1008,10 +1025,14 @@ type FlowConfigurationCompactionRule { } type FlowConfigurationIngest { - schedule: FlowConfigurationSchedule! fetchUncacheable: Boolean! } +input FlowConfigurationInput @oneOf { + ingest: IngestConditionInput + compaction: CompactionConditionInput +} + type FlowConfigurationReset { mode: SnapshotPropagationMode! oldHeadHash: Multihash @@ -1026,14 +1047,7 @@ input FlowConfigurationResetToSeedDummy { dummy: String! } -union FlowConfigurationSchedule = TimeDelta | Cron5ComponentExpression - -union FlowConfigurationSnapshot = FlowConfigurationTransform | FlowConfigurationCompactionRule | FlowConfigurationIngest | FlowConfigurationReset - -type FlowConfigurationTransform { - minRecordsToAwait: Int! - maxBatchingInterval: TimeDelta! -} +union FlowConfigurationSnapshot = FlowConfigurationCompactionRule | FlowConfigurationIngest | FlowConfigurationReset type FlowConnection { """ @@ -1133,7 +1147,7 @@ type FlowEventAborted implements FlowEvent { type FlowEventInitiated implements FlowEvent { eventId: EventID! eventTime: DateTime! - trigger: FlowTrigger! + trigger: FlowTriggerType! } type FlowEventScheduledForActivation implements FlowEvent { @@ -1159,7 +1173,7 @@ type FlowEventTaskChanged implements FlowEvent { type FlowEventTriggerAdded implements FlowEvent { eventId: EventID! eventTime: DateTime! - trigger: FlowTrigger! + trigger: FlowTriggerType! } type FlowFailedError { @@ -1179,13 +1193,13 @@ type FlowFailureReasonInputDatasetCompacted { scalar FlowID -type FlowIncompatibleDatasetKind implements SetFlowConfigResult & SetFlowTransformConfigResult & SetFlowCompactionConfigResult & TriggerFlowResult { +type FlowIncompatibleDatasetKind implements SetFlowConfigResult & TriggerFlowResult & SetFlowTriggerResult { expectedDatasetKind: DatasetKind! actualDatasetKind: DatasetKind! message: String! } -type FlowInvalidCompactionConfig implements SetFlowCompactionConfigResult { +type FlowInvalidConfigInputError implements SetFlowConfigResult { reason: String! message: String! } @@ -1195,7 +1209,7 @@ type FlowInvalidRunConfigurations implements TriggerFlowResult { message: String! } -type FlowInvalidTransformConfig implements SetFlowTransformConfigResult { +type FlowInvalidTriggerInputError implements SetFlowTriggerResult { reason: String! message: String! } @@ -1207,13 +1221,12 @@ type FlowNotFound implements GetFlowResult & CancelScheduledTasksResult { union FlowOutcome = FlowSuccessResult | FlowFailedError | FlowAbortedResult -type FlowPreconditionsNotMet implements SetFlowConfigResult & SetFlowTransformConfigResult & TriggerFlowResult { +type FlowPreconditionsNotMet implements SetFlowConfigResult & TriggerFlowResult & SetFlowTriggerResult { preconditions: String! message: String! } input FlowRunConfiguration @oneOf { - transform: TransformConditionInput compaction: CompactionConditionInput ingest: IngestConditionInput reset: ResetConditionInput @@ -1222,7 +1235,7 @@ input FlowRunConfiguration @oneOf { union FlowStartCondition = FlowStartConditionSchedule | FlowStartConditionThrottling | FlowStartConditionBatching | FlowStartConditionExecutor type FlowStartConditionBatching { - activeTransformRule: FlowConfigurationTransform! + activeBatchingRule: FlowTriggerBatchingRule! batchingDeadline: DateTime! accumulatedRecordsCount: Int! watermarkModified: Boolean! @@ -1268,12 +1281,26 @@ type FlowTimingRecords { finishedAt: DateTime } -union FlowTrigger = FlowTriggerManual | FlowTriggerAutoPolling | FlowTriggerPush | FlowTriggerInputDatasetFlow +type FlowTrigger { + paused: Boolean! + schedule: FlowTriggerScheduleRule + batching: FlowTriggerBatchingRule +} type FlowTriggerAutoPolling { dummy: Boolean! } +type FlowTriggerBatchingRule { + minRecordsToAwait: Int! + maxBatchingInterval: TimeDelta! +} + +input FlowTriggerInput @oneOf { + schedule: ScheduleInput + batching: BatchingInput +} + type FlowTriggerInputDatasetFlow { dataset: Dataset! flowType: DatasetFlowType! @@ -1288,7 +1315,11 @@ type FlowTriggerPush { dummy: Boolean! } -type FlowTypeIsNotSupported implements SetFlowConfigResult & SetFlowTransformConfigResult & SetFlowCompactionConfigResult { +union FlowTriggerScheduleRule = TimeDelta | Cron5ComponentExpression + +union FlowTriggerType = FlowTriggerManual | FlowTriggerAutoPolling | FlowTriggerPush | FlowTriggerInputDatasetFlow + +type FlowTypeIsNotSupported implements SetFlowConfigResult & SetFlowTriggerResult { message: String! } @@ -1307,7 +1338,6 @@ input IngestConditionInput { Flag indicates to ignore cache during ingest step for API calls """ fetchUncacheable: Boolean! - schedule: ScheduleInput! } input InitiatorFilterInput @oneOf { @@ -1727,20 +1757,21 @@ type SetDataSchema { schema: DataSchema! } -interface SetFlowCompactionConfigResult { +interface SetFlowConfigResult { message: String! } -interface SetFlowConfigResult { +type SetFlowConfigSuccess implements SetFlowConfigResult { + config: FlowConfiguration! message: String! } -type SetFlowConfigSuccess implements SetFlowConfigResult & SetFlowTransformConfigResult & SetFlowCompactionConfigResult { - config: FlowConfiguration! +interface SetFlowTriggerResult { message: String! } -interface SetFlowTransformConfigResult { +type SetFlowTriggerSuccess implements SetFlowTriggerResult { + trigger: FlowTrigger! message: String! } @@ -1925,11 +1956,6 @@ enum TimeUnit { union Transform = TransformSql -input TransformConditionInput { - minRecordsToAwait: Int! - maxBatchingInterval: TimeDeltaInput! -} - type TransformInput { datasetRef: DatasetRef! alias: String! @@ -2030,4 +2056,4 @@ directive @specifiedBy(url: String!) on SCALAR schema { query: Query mutation: Mutation -} +} \ No newline at end of file diff --git a/src/app/api/dataset-flow.api.spec.ts b/src/app/api/dataset-flow.api.spec.ts index c457147b1..1462f3ec8 100644 --- a/src/app/api/dataset-flow.api.spec.ts +++ b/src/app/api/dataset-flow.api.spec.ts @@ -83,14 +83,14 @@ describe("DatasetFlowApi", () => { .subscribe((res: GetDatasetFlowConfigsQuery) => { const configType = res.datasets.byId?.flows.configs.byType; const mockConfigType = mockIngestGetDatasetFlowConfigsSuccess.datasets.byId?.flows.configs.byType; - expect(configType?.paused).toEqual(mockConfigType?.paused); - expect(configType?.transform).toEqual(null); - if ( - configType?.ingest?.schedule.__typename === "TimeDelta" && - mockConfigType?.ingest?.schedule.__typename === "TimeDelta" - ) { - expect(configType.ingest).toEqual(mockConfigType.ingest); - } + + // expect(configType?.ingest).toEqual(null); + // if ( + // configType?.ingest?.schedule.__typename === "TimeDelta" && + // mockConfigType?.ingest?.schedule.__typename === "TimeDelta" + // ) { + // expect(configType.ingest).toEqual(mockConfigType.ingest); + // } }); const op = controller.expectOne(GetDatasetFlowConfigsDocument); diff --git a/src/app/api/dataset-flow.api.ts b/src/app/api/dataset-flow.api.ts index 1a1fd652a..71dde9373 100644 --- a/src/app/api/dataset-flow.api.ts +++ b/src/app/api/dataset-flow.api.ts @@ -3,16 +3,9 @@ import { inject, Injectable } from "@angular/core"; import { CancelScheduledTasksGQL, CancelScheduledTasksMutation, - CompactionConditionInput, DatasetAllFlowsPausedGQL, DatasetAllFlowsPausedQuery, - DatasetFlowBatchingGQL, - DatasetFlowBatchingMutation, - DatasetFlowCompactionGQL, - DatasetFlowCompactionMutation, DatasetFlowFilters, - DatasetFlowScheduleGQL, - DatasetFlowScheduleMutation, DatasetFlowType, DatasetFlowsInitiatorsGQL, DatasetFlowsInitiatorsQuery, @@ -29,8 +22,6 @@ import { GetDatasetListFlowsQuery, GetFlowByIdGQL, GetFlowByIdQuery, - IngestConditionInput, - TransformConditionInput, } from "./kamu.graphql.interface"; import { Observable, first, map } from "rxjs"; import { ApolloQueryResult } from "@apollo/client"; @@ -40,8 +31,8 @@ import { noCacheFetchPolicy } from "../common/data.helpers"; @Injectable({ providedIn: "root" }) export class DatasetFlowApi { private getDatasetFlowConfigsGQL = inject(GetDatasetFlowConfigsGQL); - private datasetFlowScheduleGQL = inject(DatasetFlowScheduleGQL); - private datasetFlowBatchingGQL = inject(DatasetFlowBatchingGQL); + // private datasetFlowScheduleGQL = inject(DatasetFlowScheduleGQL); + // private datasetFlowBatchingGQL = inject(DatasetFlowBatchingGQL); private getDatasetListFlowsGQL = inject(GetDatasetListFlowsGQL); private datasetPauseFlowsGQL = inject(DatasetPauseFlowsGQL); private datasetResumeFlowsGQL = inject(DatasetResumeFlowsGQL); @@ -49,7 +40,7 @@ export class DatasetFlowApi { private datasetTriggerFlowGQL = inject(DatasetTriggerFlowGQL); private datasetFlowByIdGQL = inject(GetFlowByIdGQL); private cancelScheduledTasksGQL = inject(CancelScheduledTasksGQL); - private datasetFlowCompactionGQL = inject(DatasetFlowCompactionGQL); + // private datasetFlowCompactionGQL = inject(DatasetFlowCompactionGQL); private datasetFlowsInitiatorsGQL = inject(DatasetFlowsInitiatorsGQL); public datasetTriggerFlow(params: { @@ -84,53 +75,53 @@ export class DatasetFlowApi { ); } - public setDatasetFlowSchedule(params: { - accountId: string; - datasetId: string; - datasetFlowType: DatasetFlowType; - paused: boolean; - ingest: IngestConditionInput; - }): Observable { - return this.datasetFlowScheduleGQL - .mutate({ - ...params, - }) - .pipe( - first(), - map((result: MutationResult) => { - /* istanbul ignore else */ - if (result.data) { - return result.data; - } else { - throw new DatasetOperationError(result.errors ?? []); - } - }), - ); - } + // public setDatasetFlowSchedule(params: { + // accountId: string; + // datasetId: string; + // datasetFlowType: DatasetFlowType; + // paused: boolean; + // ingest: IngestConditionInput; + // }): Observable { + // return this.datasetFlowScheduleGQL + // .mutate({ + // ...params, + // }) + // .pipe( + // first(), + // map((result: MutationResult) => { + // /* istanbul ignore else */ + // if (result.data) { + // return result.data; + // } else { + // throw new DatasetOperationError(result.errors ?? []); + // } + // }), + // ); + // } - public setDatasetFlowBatching(params: { - accountId: string; - datasetId: string; - datasetFlowType: DatasetFlowType; - paused: boolean; - transform: TransformConditionInput; - }): Observable { - return this.datasetFlowBatchingGQL - .mutate({ - ...params, - }) - .pipe( - first(), - map((result: MutationResult) => { - /* istanbul ignore else */ - if (result.data) { - return result.data; - } else { - throw new DatasetOperationError(result.errors ?? []); - } - }), - ); - } + // public setDatasetFlowBatching(params: { + // accountId: string; + // datasetId: string; + // datasetFlowType: DatasetFlowType; + // paused: boolean; + // transform: TransformConditionInput; + // }): Observable { + // return this.datasetFlowBatchingGQL + // .mutate({ + // ...params, + // }) + // .pipe( + // first(), + // map((result: MutationResult) => { + // /* istanbul ignore else */ + // if (result.data) { + // return result.data; + // } else { + // throw new DatasetOperationError(result.errors ?? []); + // } + // }), + // ); + // } public getDatasetListFlows(params: { datasetId: string; @@ -246,29 +237,29 @@ export class DatasetFlowApi { ); } - public setDatasetFlowCompaction(params: { - datasetId: string; - datasetFlowType: DatasetFlowType; - compactionArgs: CompactionConditionInput; - }): Observable { - return this.datasetFlowCompactionGQL - .mutate({ - datasetId: params.datasetId, - datasetFlowType: params.datasetFlowType, - compactionArgs: params.compactionArgs, - }) - .pipe( - first(), - map((result: MutationResult) => { - /* istanbul ignore else */ - if (result.data) { - return result.data; - } else { - throw new DatasetOperationError(result.errors ?? []); - } - }), - ); - } + // public setDatasetFlowCompaction(params: { + // datasetId: string; + // datasetFlowType: DatasetFlowType; + // compactionArgs: CompactionConditionInput; + // }): Observable { + // return this.datasetFlowCompactionGQL + // .mutate({ + // datasetId: params.datasetId, + // datasetFlowType: params.datasetFlowType, + // compactionArgs: params.compactionArgs, + // }) + // .pipe( + // first(), + // map((result: MutationResult) => { + // /* istanbul ignore else */ + // if (result.data) { + // return result.data; + // } else { + // throw new DatasetOperationError(result.errors ?? []); + // } + // }), + // ); + // } public getDatasetFlowsInitiators(datasetId: string): Observable { return this.datasetFlowsInitiatorsGQL.watch({ datasetId }, noCacheFetchPolicy).valueChanges.pipe( diff --git a/src/app/api/gql/account/account-dataset-flows-paused.graphql b/src/app/api/gql/account/account-dataset-flows-paused.graphql index c208e4e56..33f8a7b5d 100644 --- a/src/app/api/gql/account/account-dataset-flows-paused.graphql +++ b/src/app/api/gql/account/account-dataset-flows-paused.graphql @@ -2,7 +2,7 @@ query accountDatasetFlowsPaused($accountName: AccountName!) { accounts { byName(name: $accountName) { flows { - configs { + triggers { allPaused } } diff --git a/src/app/api/gql/account/account-pause-flows.graphql b/src/app/api/gql/account/account-pause-flows.graphql index 1b3db6c66..7ae88e282 100644 --- a/src/app/api/gql/account/account-pause-flows.graphql +++ b/src/app/api/gql/account/account-pause-flows.graphql @@ -2,7 +2,7 @@ mutation accountPauseFlows($accountName: AccountName!) { accounts { byName(accountName: $accountName) { flows { - configs { + triggers { pauseAccountDatasetFlows } } diff --git a/src/app/api/gql/account/account-resume-flows.graphql b/src/app/api/gql/account/account-resume-flows.graphql index bde50b6b3..e681a6a5a 100644 --- a/src/app/api/gql/account/account-resume-flows.graphql +++ b/src/app/api/gql/account/account-resume-flows.graphql @@ -2,7 +2,7 @@ mutation accountResumeFlows($accountName: AccountName!) { accounts { byName(accountName: $accountName) { flows { - configs { + triggers { resumeAccountDatasetFlows } } diff --git a/src/app/api/gql/compacting-dataset/dataset-flow-compacting.graphql b/src/app/api/gql/compacting-dataset/dataset-flow-compacting.graphql index ea61596de..ee265c938 100644 --- a/src/app/api/gql/compacting-dataset/dataset-flow-compacting.graphql +++ b/src/app/api/gql/compacting-dataset/dataset-flow-compacting.graphql @@ -1,46 +1,46 @@ -mutation datasetFlowCompaction( - $datasetId: DatasetID! - $datasetFlowType: DatasetFlowType! - $compactionArgs: CompactionConditionInput! -) { - datasets { - byId(datasetId: $datasetId) { - flows { - configs { - setConfigCompaction(datasetFlowType: $datasetFlowType, compactionArgs: $compactionArgs) { - ... on SetFlowConfigSuccess { - message - config { - compaction { - ... on CompactionFull { - maxSliceSize - maxSliceRecords - } +# mutation datasetFlowCompaction( +# $datasetId: DatasetID! +# $datasetFlowType: DatasetFlowType! +# $compactionArgs: CompactionConditionInput! +# ) { +# datasets { +# byId(datasetId: $datasetId) { +# flows { +# configs { +# setConfigCompaction(datasetFlowType: $datasetFlowType, compactionArgs: $compactionArgs) { +# ... on SetFlowConfigSuccess { +# message +# config { +# compaction { +# ... on CompactionFull { +# maxSliceSize +# maxSliceRecords +# } - ... on CompactionMetadataOnly { - recursive - } - } - } - } +# ... on CompactionMetadataOnly { +# recursive +# } +# } +# } +# } - ... on FlowIncompatibleDatasetKind { - message - expectedDatasetKind - actualDatasetKind - } +# ... on FlowIncompatibleDatasetKind { +# message +# expectedDatasetKind +# actualDatasetKind +# } - ... on FlowTypeIsNotSupported { - message - } +# ... on FlowTypeIsNotSupported { +# message +# } - ... on FlowInvalidCompactionConfig { - reason - message - } - } - } - } - } - } -} +# ... on FlowInvalidCompactionConfig { +# reason +# message +# } +# } +# } +# } +# } +# } +# } diff --git a/src/app/api/gql/flows-dataset/dataset-all-flows-paused.graphql b/src/app/api/gql/flows-dataset/dataset-all-flows-paused.graphql index 40029ce5a..1306f0540 100644 --- a/src/app/api/gql/flows-dataset/dataset-all-flows-paused.graphql +++ b/src/app/api/gql/flows-dataset/dataset-all-flows-paused.graphql @@ -2,7 +2,7 @@ query datasetAllFlowsPaused($datasetId: DatasetID!) { datasets { byId(datasetId: $datasetId) { flows { - configs { + triggers { allPaused } } diff --git a/src/app/api/gql/flows-dataset/dataset-pause-flows.graphql b/src/app/api/gql/flows-dataset/dataset-pause-flows.graphql index ed28c480d..f27096b05 100644 --- a/src/app/api/gql/flows-dataset/dataset-pause-flows.graphql +++ b/src/app/api/gql/flows-dataset/dataset-pause-flows.graphql @@ -2,7 +2,7 @@ mutation datasetPauseFlows($datasetId: DatasetID!, $datasetFlowType: DatasetFlow datasets { byId(datasetId: $datasetId) { flows { - configs { + triggers { pauseFlows(datasetFlowType: $datasetFlowType) } } diff --git a/src/app/api/gql/flows-dataset/dataset-resume-flows.graphql b/src/app/api/gql/flows-dataset/dataset-resume-flows.graphql index 9b9cfc4fb..7f8cc6911 100644 --- a/src/app/api/gql/flows-dataset/dataset-resume-flows.graphql +++ b/src/app/api/gql/flows-dataset/dataset-resume-flows.graphql @@ -2,7 +2,7 @@ mutation datasetResumeFlows($datasetId: DatasetID!, $datasetFlowType: DatasetFlo datasets { byId(datasetId: $datasetId) { flows { - configs { + triggers { resumeFlows(datasetFlowType: $datasetFlowType) } } diff --git a/src/app/api/gql/flows-dataset/fragments/fragment-dataset-flow.graphql b/src/app/api/gql/flows-dataset/fragments/fragment-dataset-flow.graphql index 6fffd112d..b8290f3b4 100644 --- a/src/app/api/gql/flows-dataset/fragments/fragment-dataset-flow.graphql +++ b/src/app/api/gql/flows-dataset/fragments/fragment-dataset-flow.graphql @@ -96,7 +96,7 @@ fragment FlowSummaryData on Flow { } ... on FlowStartConditionBatching { - activeTransformRule { + activeBatchingRule { minRecordsToAwait maxBatchingInterval { ...TimeDeltaData @@ -118,14 +118,14 @@ fragment FlowSummaryData on Flow { configSnapshot { ... on FlowConfigurationIngest { - schedule { - ... on TimeDelta { - ...TimeDeltaData - } - ... on Cron5ComponentExpression { - cron5ComponentExpression - } - } + # schedule { + # ... on TimeDelta { + # ...TimeDeltaData + # } + # ... on Cron5ComponentExpression { + # cron5ComponentExpression + # } + # } fetchUncacheable } diff --git a/src/app/api/gql/flows-dataset/fragments/fragment-flow-history-data.graphql b/src/app/api/gql/flows-dataset/fragments/fragment-flow-history-data.graphql index b27c31af5..a343f6d7e 100644 --- a/src/app/api/gql/flows-dataset/fragments/fragment-flow-history-data.graphql +++ b/src/app/api/gql/flows-dataset/fragments/fragment-flow-history-data.graphql @@ -38,7 +38,7 @@ fragment FlowHistoryData on FlowEvent { } ... on FlowStartConditionBatching { - activeTransformRule { + activeBatchingRule { minRecordsToAwait maxBatchingInterval { ...TimeDeltaData diff --git a/src/app/api/gql/scheduling-dataset/dataset-flow-batching.graphql b/src/app/api/gql/scheduling-dataset/dataset-flow-batching.graphql index 3fc4044f4..2b8dc50f1 100644 --- a/src/app/api/gql/scheduling-dataset/dataset-flow-batching.graphql +++ b/src/app/api/gql/scheduling-dataset/dataset-flow-batching.graphql @@ -1,49 +1,49 @@ -mutation datasetFlowBatching( - $datasetId: DatasetID! - $datasetFlowType: DatasetFlowType! - $paused: Boolean! - $transform: TransformConditionInput! -) { - datasets { - byId(datasetId: $datasetId) { - flows { - configs { - setConfigTransform(datasetFlowType: $datasetFlowType, paused: $paused, transform: $transform) { - __typename - ... on SetFlowConfigSuccess { - message - config { - transform { - maxBatchingInterval { - ...TimeDeltaData - } - minRecordsToAwait - } - } - } +# mutation datasetFlowBatching( +# $datasetId: DatasetID! +# $datasetFlowType: DatasetFlowType! +# $paused: Boolean! +# $transform: TransformConditionInput! +# ) { +# datasets { +# byId(datasetId: $datasetId) { +# flows { +# configs { +# setConfigTransform(datasetFlowType: $datasetFlowType, paused: $paused, transform: $transform) { +# __typename +# ... on SetFlowConfigSuccess { +# message +# config { +# transform { +# maxBatchingInterval { +# ...TimeDeltaData +# } +# minRecordsToAwait +# } +# } +# } - ... on FlowIncompatibleDatasetKind { - message - expectedDatasetKind - actualDatasetKind - } +# ... on FlowIncompatibleDatasetKind { +# message +# expectedDatasetKind +# actualDatasetKind +# } - ... on FlowInvalidTransformConfig { - message - reason - } +# ... on FlowInvalidTransformConfig { +# message +# reason +# } - ... on FlowPreconditionsNotMet { - message - preconditions - } +# ... on FlowPreconditionsNotMet { +# message +# preconditions +# } - ... on FlowTypeIsNotSupported { - message - } - } - } - } - } - } -} +# ... on FlowTypeIsNotSupported { +# message +# } +# } +# } +# } +# } +# } +# } diff --git a/src/app/api/gql/scheduling-dataset/dataset-flow-configs.graphql b/src/app/api/gql/scheduling-dataset/dataset-flow-configs.graphql index 5682b01ac..3dd680df1 100644 --- a/src/app/api/gql/scheduling-dataset/dataset-flow-configs.graphql +++ b/src/app/api/gql/scheduling-dataset/dataset-flow-configs.graphql @@ -8,23 +8,18 @@ query getDatasetFlowConfigs($datasetId: DatasetID!, $datasetFlowType: DatasetFlo configs { __typename byType(datasetFlowType: $datasetFlowType) { - paused ingest { - schedule { - ... on TimeDelta { - ...TimeDeltaData - } - ... on Cron5ComponentExpression { - cron5ComponentExpression - } - } fetchUncacheable } - transform { - maxBatchingInterval { - ...TimeDeltaData + + reset { + oldHeadHash + recursive + mode { + ... on SnapshotConfigurationResetToSeedDummy { + dummy + } } - minRecordsToAwait } compaction { diff --git a/src/app/api/gql/scheduling-dataset/dataset-flow-schedule.graphql b/src/app/api/gql/scheduling-dataset/dataset-flow-schedule.graphql index 23bef8121..00020c049 100644 --- a/src/app/api/gql/scheduling-dataset/dataset-flow-schedule.graphql +++ b/src/app/api/gql/scheduling-dataset/dataset-flow-schedule.graphql @@ -1,48 +1,48 @@ -mutation DatasetFlowSchedule( - $datasetId: DatasetID! - $datasetFlowType: DatasetFlowType! - $paused: Boolean! - $ingest: IngestConditionInput! -) { - datasets { - byId(datasetId: $datasetId) { - flows { - configs { - setConfigIngest(datasetFlowType: $datasetFlowType, paused: $paused, ingest: $ingest) { - __typename - ... on SetFlowConfigSuccess { - message - config { - ingest { - schedule { - ... on TimeDelta { - ...TimeDeltaData - } - ... on Cron5ComponentExpression { - cron5ComponentExpression - } - } - fetchUncacheable - } - } - } +# mutation DatasetFlowSchedule( +# $datasetId: DatasetID! +# $datasetFlowType: DatasetFlowType! +# $paused: Boolean! +# $ingest: IngestConditionInput! +# ) { +# datasets { +# byId(datasetId: $datasetId) { +# flows { +# configs { +# setConfigIngest(datasetFlowType: $datasetFlowType, paused: $paused, ingest: $ingest) { +# __typename +# ... on SetFlowConfigSuccess { +# message +# config { +# ingest { +# schedule { +# ... on TimeDelta { +# ...TimeDeltaData +# } +# ... on Cron5ComponentExpression { +# cron5ComponentExpression +# } +# } +# fetchUncacheable +# } +# } +# } - ... on FlowIncompatibleDatasetKind { - message - expectedDatasetKind - actualDatasetKind - } +# ... on FlowIncompatibleDatasetKind { +# message +# expectedDatasetKind +# actualDatasetKind +# } - ... on FlowPreconditionsNotMet { - message - } +# ... on FlowPreconditionsNotMet { +# message +# } - ... on FlowTypeIsNotSupported { - message - } - } - } - } - } - } -} +# ... on FlowTypeIsNotSupported { +# message +# } +# } +# } +# } +# } +# } +# } diff --git a/src/app/api/kamu.graphql.interface.ts b/src/app/api/kamu.graphql.interface.ts index bade652fe..a8519d1f9 100644 --- a/src/app/api/kamu.graphql.interface.ts +++ b/src/app/api/kamu.graphql.interface.ts @@ -86,18 +86,6 @@ export type AccountEdge = { node: Account; }; -export type AccountFlowConfigs = { - __typename?: "AccountFlowConfigs"; - /** Checks if all configs of all datasets in account are disabled */ - allPaused: Scalars["Boolean"]; -}; - -export type AccountFlowConfigsMut = { - __typename?: "AccountFlowConfigsMut"; - pauseAccountDatasetFlows: Scalars["Boolean"]; - resumeAccountDatasetFlows: Scalars["Boolean"]; -}; - export type AccountFlowFilters = { byDatasetIds: Array; byFlowType?: InputMaybe; @@ -117,17 +105,29 @@ export type AccountFlowRunsListFlowsArgs = { perPage?: InputMaybe; }; +export type AccountFlowTriggers = { + __typename?: "AccountFlowTriggers"; + /** Checks if all triggers of all datasets in account are disabled */ + allPaused: Scalars["Boolean"]; +}; + +export type AccountFlowTriggersMut = { + __typename?: "AccountFlowTriggersMut"; + pauseAccountDatasetFlows: Scalars["Boolean"]; + resumeAccountDatasetFlows: Scalars["Boolean"]; +}; + export type AccountFlows = { __typename?: "AccountFlows"; - /** Returns interface for flow configurations queries */ - configs: AccountFlowConfigs; /** Returns interface for flow runs queries */ runs: AccountFlowRuns; + /** Returns interface for flow triggers queries */ + triggers: AccountFlowTriggers; }; export type AccountFlowsMut = { __typename?: "AccountFlowsMut"; - configs: AccountFlowConfigsMut; + triggers: AccountFlowTriggersMut; }; export type AccountMut = { @@ -247,6 +247,11 @@ export type AuthMutRevokeAccessTokenArgs = { tokenId: Scalars["AccessTokenID"]; }; +export type BatchingInput = { + maxBatchingInterval: TimeDeltaInput; + minRecordsToAwait: Scalars["Int"]; +}; + export type BlockRef = { __typename?: "BlockRef"; blockHash: Scalars["Multihash"]; @@ -634,8 +639,6 @@ export type DatasetEnvVarsMutSaveEnvVariableArgs = { export type DatasetFlowConfigs = { __typename?: "DatasetFlowConfigs"; - /** Checks if all configs of this dataset are disabled */ - allPaused: Scalars["Boolean"]; /** Returns defined configuration for a flow of specified type */ byType?: Maybe; }; @@ -646,38 +649,14 @@ export type DatasetFlowConfigsByTypeArgs = { export type DatasetFlowConfigsMut = { __typename?: "DatasetFlowConfigsMut"; - pauseFlows: Scalars["Boolean"]; - resumeFlows: Scalars["Boolean"]; - setConfigCompaction: SetFlowCompactionConfigResult; - setConfigIngest: SetFlowConfigResult; - setConfigTransform: SetFlowTransformConfigResult; -}; - -export type DatasetFlowConfigsMutPauseFlowsArgs = { - datasetFlowType?: InputMaybe; + setConfig: SetFlowConfigResult; }; -export type DatasetFlowConfigsMutResumeFlowsArgs = { - datasetFlowType?: InputMaybe; -}; - -export type DatasetFlowConfigsMutSetConfigCompactionArgs = { - compactionArgs: CompactionConditionInput; +export type DatasetFlowConfigsMutSetConfigArgs = { + configInput: FlowConfigurationInput; datasetFlowType: DatasetFlowType; }; -export type DatasetFlowConfigsMutSetConfigIngestArgs = { - datasetFlowType: DatasetFlowType; - ingest: IngestConditionInput; - paused: Scalars["Boolean"]; -}; - -export type DatasetFlowConfigsMutSetConfigTransformArgs = { - datasetFlowType: DatasetFlowType; - paused: Scalars["Boolean"]; - transform: TransformConditionInput; -}; - export type DatasetFlowFilters = { byFlowType?: InputMaybe; byInitiator?: InputMaybe; @@ -716,6 +695,39 @@ export type DatasetFlowRunsMutTriggerFlowArgs = { flowRunConfiguration?: InputMaybe; }; +export type DatasetFlowTriggers = { + __typename?: "DatasetFlowTriggers"; + /** Checks if all triggers of this dataset are disabled */ + allPaused: Scalars["Boolean"]; + /** Returns defined trigger for a flow of specified type */ + byType?: Maybe; +}; + +export type DatasetFlowTriggersByTypeArgs = { + datasetFlowType: DatasetFlowType; +}; + +export type DatasetFlowTriggersMut = { + __typename?: "DatasetFlowTriggersMut"; + pauseFlows: Scalars["Boolean"]; + resumeFlows: Scalars["Boolean"]; + setTrigger: SetFlowTriggerResult; +}; + +export type DatasetFlowTriggersMutPauseFlowsArgs = { + datasetFlowType?: InputMaybe; +}; + +export type DatasetFlowTriggersMutResumeFlowsArgs = { + datasetFlowType?: InputMaybe; +}; + +export type DatasetFlowTriggersMutSetTriggerArgs = { + datasetFlowType: DatasetFlowType; + paused: Scalars["Boolean"]; + triggerInput: FlowTriggerInput; +}; + export enum DatasetFlowType { ExecuteTransform = "EXECUTE_TRANSFORM", HardCompaction = "HARD_COMPACTION", @@ -729,12 +741,15 @@ export type DatasetFlows = { configs: DatasetFlowConfigs; /** Returns interface for flow runs queries */ runs: DatasetFlowRuns; + /** Returns interface for flow triggers queries */ + triggers: DatasetFlowTriggers; }; export type DatasetFlowsMut = { __typename?: "DatasetFlowsMut"; configs: DatasetFlowConfigsMut; runs: DatasetFlowRunsMut; + triggers: DatasetFlowTriggersMut; }; export enum DatasetKind { @@ -1080,7 +1095,7 @@ export type Flow = { /** Outcome of the flow (Finished state only) */ outcome?: Maybe; /** Primary flow trigger */ - primaryTrigger: FlowTrigger; + primaryTrigger: FlowTriggerType; /** Start condition */ startCondition?: Maybe; /** Status of the flow */ @@ -1100,9 +1115,7 @@ export type FlowConfiguration = { __typename?: "FlowConfiguration"; compaction?: Maybe; ingest?: Maybe; - paused: Scalars["Boolean"]; reset?: Maybe; - transform?: Maybe; }; export type FlowConfigurationCompaction = CompactionFull | CompactionMetadataOnly; @@ -1115,9 +1128,12 @@ export type FlowConfigurationCompactionRule = { export type FlowConfigurationIngest = { __typename?: "FlowConfigurationIngest"; fetchUncacheable: Scalars["Boolean"]; - schedule: FlowConfigurationSchedule; }; +export type FlowConfigurationInput = + | { compaction: CompactionConditionInput; ingest?: never } + | { compaction?: never; ingest: IngestConditionInput }; + export type FlowConfigurationReset = { __typename?: "FlowConfigurationReset"; mode: SnapshotPropagationMode; @@ -1133,19 +1149,10 @@ export type FlowConfigurationResetToSeedDummy = { dummy: Scalars["String"]; }; -export type FlowConfigurationSchedule = Cron5ComponentExpression | TimeDelta; - export type FlowConfigurationSnapshot = | FlowConfigurationCompactionRule | FlowConfigurationIngest - | FlowConfigurationReset - | FlowConfigurationTransform; - -export type FlowConfigurationTransform = { - __typename?: "FlowConfigurationTransform"; - maxBatchingInterval: TimeDelta; - minRecordsToAwait: Scalars["Int"]; -}; + | FlowConfigurationReset; export type FlowConnection = { __typename?: "FlowConnection"; @@ -1260,7 +1267,7 @@ export type FlowEventInitiated = FlowEvent & { __typename?: "FlowEventInitiated"; eventId: Scalars["EventID"]; eventTime: Scalars["DateTime"]; - trigger: FlowTrigger; + trigger: FlowTriggerType; }; export type FlowEventScheduledForActivation = FlowEvent & { @@ -1290,7 +1297,7 @@ export type FlowEventTriggerAdded = FlowEvent & { __typename?: "FlowEventTriggerAdded"; eventId: Scalars["EventID"]; eventTime: Scalars["DateTime"]; - trigger: FlowTrigger; + trigger: FlowTriggerType; }; export type FlowFailedError = { @@ -1311,9 +1318,8 @@ export type FlowFailureReasonInputDatasetCompacted = { message: Scalars["String"]; }; -export type FlowIncompatibleDatasetKind = SetFlowCompactionConfigResult & - SetFlowConfigResult & - SetFlowTransformConfigResult & +export type FlowIncompatibleDatasetKind = SetFlowConfigResult & + SetFlowTriggerResult & TriggerFlowResult & { __typename?: "FlowIncompatibleDatasetKind"; actualDatasetKind: DatasetKind; @@ -1321,8 +1327,8 @@ export type FlowIncompatibleDatasetKind = SetFlowCompactionConfigResult & message: Scalars["String"]; }; -export type FlowInvalidCompactionConfig = SetFlowCompactionConfigResult & { - __typename?: "FlowInvalidCompactionConfig"; +export type FlowInvalidConfigInputError = SetFlowConfigResult & { + __typename?: "FlowInvalidConfigInputError"; message: Scalars["String"]; reason: Scalars["String"]; }; @@ -1333,8 +1339,8 @@ export type FlowInvalidRunConfigurations = TriggerFlowResult & { message: Scalars["String"]; }; -export type FlowInvalidTransformConfig = SetFlowTransformConfigResult & { - __typename?: "FlowInvalidTransformConfig"; +export type FlowInvalidTriggerInputError = SetFlowTriggerResult & { + __typename?: "FlowInvalidTriggerInputError"; message: Scalars["String"]; reason: Scalars["String"]; }; @@ -1349,7 +1355,7 @@ export type FlowNotFound = CancelScheduledTasksResult & export type FlowOutcome = FlowAbortedResult | FlowFailedError | FlowSuccessResult; export type FlowPreconditionsNotMet = SetFlowConfigResult & - SetFlowTransformConfigResult & + SetFlowTriggerResult & TriggerFlowResult & { __typename?: "FlowPreconditionsNotMet"; message: Scalars["String"]; @@ -1357,10 +1363,9 @@ export type FlowPreconditionsNotMet = SetFlowConfigResult & }; export type FlowRunConfiguration = - | { compaction: CompactionConditionInput; ingest?: never; reset?: never; transform?: never } - | { compaction?: never; ingest: IngestConditionInput; reset?: never; transform?: never } - | { compaction?: never; ingest?: never; reset: ResetConditionInput; transform?: never } - | { compaction?: never; ingest?: never; reset?: never; transform: TransformConditionInput }; + | { compaction: CompactionConditionInput; ingest?: never; reset?: never } + | { compaction?: never; ingest: IngestConditionInput; reset?: never } + | { compaction?: never; ingest?: never; reset: ResetConditionInput }; export type FlowStartCondition = | FlowStartConditionBatching @@ -1371,7 +1376,7 @@ export type FlowStartCondition = export type FlowStartConditionBatching = { __typename?: "FlowStartConditionBatching"; accumulatedRecordsCount: Scalars["Int"]; - activeTransformRule: FlowConfigurationTransform; + activeBatchingRule: FlowTriggerBatchingRule; batchingDeadline: Scalars["DateTime"]; watermarkModified: Scalars["Boolean"]; }; @@ -1417,13 +1422,28 @@ export type FlowTimingRecords = { runningSince?: Maybe; }; -export type FlowTrigger = FlowTriggerAutoPolling | FlowTriggerInputDatasetFlow | FlowTriggerManual | FlowTriggerPush; +export type FlowTrigger = { + __typename?: "FlowTrigger"; + batching?: Maybe; + paused: Scalars["Boolean"]; + schedule?: Maybe; +}; export type FlowTriggerAutoPolling = { __typename?: "FlowTriggerAutoPolling"; dummy: Scalars["Boolean"]; }; +export type FlowTriggerBatchingRule = { + __typename?: "FlowTriggerBatchingRule"; + maxBatchingInterval: TimeDelta; + minRecordsToAwait: Scalars["Int"]; +}; + +export type FlowTriggerInput = + | { batching: BatchingInput; schedule?: never } + | { batching?: never; schedule: ScheduleInput }; + export type FlowTriggerInputDatasetFlow = { __typename?: "FlowTriggerInputDatasetFlow"; dataset: Dataset; @@ -1441,9 +1461,16 @@ export type FlowTriggerPush = { dummy: Scalars["Boolean"]; }; -export type FlowTypeIsNotSupported = SetFlowCompactionConfigResult & - SetFlowConfigResult & - SetFlowTransformConfigResult & { +export type FlowTriggerScheduleRule = Cron5ComponentExpression | TimeDelta; + +export type FlowTriggerType = + | FlowTriggerAutoPolling + | FlowTriggerInputDatasetFlow + | FlowTriggerManual + | FlowTriggerPush; + +export type FlowTypeIsNotSupported = SetFlowConfigResult & + SetFlowTriggerResult & { __typename?: "FlowTypeIsNotSupported"; message: Scalars["String"]; }; @@ -1461,7 +1488,6 @@ export type GetFlowSuccess = GetFlowResult & { export type IngestConditionInput = { /** Flag indicates to ignore cache during ingest step for API calls */ fetchUncacheable: Scalars["Boolean"]; - schedule: ScheduleInput; }; export type InitiatorFilterInput = @@ -1929,24 +1955,24 @@ export type SetDataSchema = { schema: DataSchema; }; -export type SetFlowCompactionConfigResult = { +export type SetFlowConfigResult = { message: Scalars["String"]; }; -export type SetFlowConfigResult = { +export type SetFlowConfigSuccess = SetFlowConfigResult & { + __typename?: "SetFlowConfigSuccess"; + config: FlowConfiguration; message: Scalars["String"]; }; -export type SetFlowConfigSuccess = SetFlowCompactionConfigResult & - SetFlowConfigResult & - SetFlowTransformConfigResult & { - __typename?: "SetFlowConfigSuccess"; - config: FlowConfiguration; - message: Scalars["String"]; - }; +export type SetFlowTriggerResult = { + message: Scalars["String"]; +}; -export type SetFlowTransformConfigResult = { +export type SetFlowTriggerSuccess = SetFlowTriggerResult & { + __typename?: "SetFlowTriggerSuccess"; message: Scalars["String"]; + trigger: FlowTrigger; }; export type SetInfo = { @@ -2113,11 +2139,6 @@ export enum TimeUnit { export type Transform = TransformSql; -export type TransformConditionInput = { - maxBatchingInterval: TimeDeltaInput; - minRecordsToAwait: Scalars["Int"]; -}; - export type TransformInput = { __typename?: "TransformInput"; alias: Scalars["String"]; @@ -2274,7 +2295,7 @@ export type AccountDatasetFlowsPausedQuery = { __typename?: "Account"; flows?: { __typename?: "AccountFlows"; - configs: { __typename?: "AccountFlowConfigs"; allPaused: boolean }; + triggers: { __typename?: "AccountFlowTriggers"; allPaused: boolean }; } | null; } | null; }; @@ -2339,7 +2360,7 @@ export type AccountPauseFlowsMutation = { __typename?: "AccountMut"; flows: { __typename?: "AccountFlowsMut"; - configs: { __typename?: "AccountFlowConfigsMut"; pauseAccountDatasetFlows: boolean }; + triggers: { __typename?: "AccountFlowTriggersMut"; pauseAccountDatasetFlows: boolean }; }; } | null; }; @@ -2357,7 +2378,7 @@ export type AccountResumeFlowsMutation = { __typename?: "AccountMut"; flows: { __typename?: "AccountFlowsMut"; - configs: { __typename?: "AccountFlowConfigsMut"; resumeAccountDatasetFlows: boolean }; + triggers: { __typename?: "AccountFlowTriggersMut"; resumeAccountDatasetFlows: boolean }; }; } | null; }; @@ -2415,48 +2436,6 @@ export type FetchAccountDetailsMutation = { auth: { __typename?: "AuthMut"; accountDetails: { __typename?: "Account" } & AccountFragment }; }; -export type DatasetFlowCompactionMutationVariables = Exact<{ - datasetId: Scalars["DatasetID"]; - datasetFlowType: DatasetFlowType; - compactionArgs: CompactionConditionInput; -}>; - -export type DatasetFlowCompactionMutation = { - __typename?: "Mutation"; - datasets: { - __typename?: "DatasetsMut"; - byId?: { - __typename?: "DatasetMut"; - flows: { - __typename?: "DatasetFlowsMut"; - configs: { - __typename?: "DatasetFlowConfigsMut"; - setConfigCompaction: - | { - __typename?: "FlowIncompatibleDatasetKind"; - message: string; - expectedDatasetKind: DatasetKind; - actualDatasetKind: DatasetKind; - } - | { __typename?: "FlowInvalidCompactionConfig"; reason: string; message: string } - | { __typename?: "FlowTypeIsNotSupported"; message: string } - | { - __typename?: "SetFlowConfigSuccess"; - message: string; - config: { - __typename?: "FlowConfiguration"; - compaction?: - | { __typename?: "CompactionFull"; maxSliceSize: number; maxSliceRecords: number } - | { __typename?: "CompactionMetadataOnly"; recursive: boolean } - | null; - }; - }; - }; - }; - } | null; - }; -}; - export type CommitEventToDatasetMutationVariables = Exact<{ datasetId: Scalars["DatasetID"]; event: Scalars["String"]; @@ -2996,7 +2975,10 @@ export type DatasetAllFlowsPausedQuery = { __typename?: "Datasets"; byId?: { __typename?: "Dataset"; - flows: { __typename?: "DatasetFlows"; configs: { __typename?: "DatasetFlowConfigs"; allPaused: boolean } }; + flows: { + __typename?: "DatasetFlows"; + triggers: { __typename?: "DatasetFlowTriggers"; allPaused: boolean }; + }; } | null; }; }; @@ -3144,7 +3126,7 @@ export type DatasetPauseFlowsMutation = { __typename?: "DatasetMut"; flows: { __typename?: "DatasetFlowsMut"; - configs: { __typename?: "DatasetFlowConfigsMut"; pauseFlows: boolean }; + triggers: { __typename?: "DatasetFlowTriggersMut"; pauseFlows: boolean }; }; } | null; }; @@ -3163,7 +3145,7 @@ export type DatasetResumeFlowsMutation = { __typename?: "DatasetMut"; flows: { __typename?: "DatasetFlowsMut"; - configs: { __typename?: "DatasetFlowConfigsMut"; resumeFlows: boolean }; + triggers: { __typename?: "DatasetFlowTriggersMut"; resumeFlows: boolean }; }; } | null; }; @@ -3288,8 +3270,8 @@ export type FlowSummaryDataFragment = { batchingDeadline: string; accumulatedRecordsCount: number; watermarkModified: boolean; - activeTransformRule: { - __typename?: "FlowConfigurationTransform"; + activeBatchingRule: { + __typename?: "FlowTriggerBatchingRule"; minRecordsToAwait: number; maxBatchingInterval: { __typename?: "TimeDelta" } & TimeDeltaDataFragment; }; @@ -3303,15 +3285,8 @@ export type FlowSummaryDataFragment = { __typename?: "FlowConfigurationCompactionRule"; compactionRule: { __typename: "CompactionFull" } | { __typename: "CompactionMetadataOnly" }; } - | { - __typename?: "FlowConfigurationIngest"; - fetchUncacheable: boolean; - schedule: - | { __typename?: "Cron5ComponentExpression"; cron5ComponentExpression: string } - | ({ __typename?: "TimeDelta" } & TimeDeltaDataFragment); - } + | { __typename?: "FlowConfigurationIngest"; fetchUncacheable: boolean } | { __typename?: "FlowConfigurationReset" } - | { __typename?: "FlowConfigurationTransform" } | null; }; @@ -3379,8 +3354,8 @@ type FlowHistoryData_FlowEventStartConditionUpdated_Fragment = { batchingDeadline: string; accumulatedRecordsCount: number; watermarkModified: boolean; - activeTransformRule: { - __typename?: "FlowConfigurationTransform"; + activeBatchingRule: { + __typename?: "FlowTriggerBatchingRule"; minRecordsToAwait: number; maxBatchingInterval: { __typename?: "TimeDelta" } & TimeDeltaDataFragment; }; @@ -4075,51 +4050,6 @@ export type RenameDatasetMutation = { }; }; -export type DatasetFlowBatchingMutationVariables = Exact<{ - datasetId: Scalars["DatasetID"]; - datasetFlowType: DatasetFlowType; - paused: Scalars["Boolean"]; - transform: TransformConditionInput; -}>; - -export type DatasetFlowBatchingMutation = { - __typename?: "Mutation"; - datasets: { - __typename?: "DatasetsMut"; - byId?: { - __typename?: "DatasetMut"; - flows: { - __typename?: "DatasetFlowsMut"; - configs: { - __typename?: "DatasetFlowConfigsMut"; - setConfigTransform: - | { - __typename: "FlowIncompatibleDatasetKind"; - message: string; - expectedDatasetKind: DatasetKind; - actualDatasetKind: DatasetKind; - } - | { __typename: "FlowInvalidTransformConfig"; message: string; reason: string } - | { __typename: "FlowPreconditionsNotMet"; message: string; preconditions: string } - | { __typename: "FlowTypeIsNotSupported"; message: string } - | { - __typename: "SetFlowConfigSuccess"; - message: string; - config: { - __typename?: "FlowConfiguration"; - transform?: { - __typename?: "FlowConfigurationTransform"; - minRecordsToAwait: number; - maxBatchingInterval: { __typename?: "TimeDelta" } & TimeDeltaDataFragment; - } | null; - }; - }; - }; - }; - } | null; - }; -}; - export type GetDatasetFlowConfigsQueryVariables = Exact<{ datasetId: Scalars["DatasetID"]; datasetFlowType: DatasetFlowType; @@ -4138,18 +4068,14 @@ export type GetDatasetFlowConfigsQuery = { __typename: "DatasetFlowConfigs"; byType?: { __typename?: "FlowConfiguration"; - paused: boolean; - ingest?: { - __typename?: "FlowConfigurationIngest"; - fetchUncacheable: boolean; - schedule: - | { __typename?: "Cron5ComponentExpression"; cron5ComponentExpression: string } - | ({ __typename?: "TimeDelta" } & TimeDeltaDataFragment); - } | null; - transform?: { - __typename?: "FlowConfigurationTransform"; - minRecordsToAwait: number; - maxBatchingInterval: { __typename?: "TimeDelta" } & TimeDeltaDataFragment; + ingest?: { __typename?: "FlowConfigurationIngest"; fetchUncacheable: boolean } | null; + reset?: { + __typename?: "FlowConfigurationReset"; + oldHeadHash?: string | null; + recursive: boolean; + mode: + | { __typename?: "SnapshotConfigurationResetCustom" } + | { __typename?: "SnapshotConfigurationResetToSeedDummy"; dummy: string }; } | null; compaction?: | { @@ -4168,55 +4094,6 @@ export type GetDatasetFlowConfigsQuery = { }; }; -export type DatasetFlowScheduleMutationVariables = Exact<{ - datasetId: Scalars["DatasetID"]; - datasetFlowType: DatasetFlowType; - paused: Scalars["Boolean"]; - ingest: IngestConditionInput; -}>; - -export type DatasetFlowScheduleMutation = { - __typename?: "Mutation"; - datasets: { - __typename?: "DatasetsMut"; - byId?: { - __typename?: "DatasetMut"; - flows: { - __typename?: "DatasetFlowsMut"; - configs: { - __typename?: "DatasetFlowConfigsMut"; - setConfigIngest: - | { - __typename: "FlowIncompatibleDatasetKind"; - message: string; - expectedDatasetKind: DatasetKind; - actualDatasetKind: DatasetKind; - } - | { __typename: "FlowPreconditionsNotMet"; message: string } - | { __typename: "FlowTypeIsNotSupported"; message: string } - | { - __typename: "SetFlowConfigSuccess"; - message: string; - config: { - __typename?: "FlowConfiguration"; - ingest?: { - __typename?: "FlowConfigurationIngest"; - fetchUncacheable: boolean; - schedule: - | { - __typename?: "Cron5ComponentExpression"; - cron5ComponentExpression: string; - } - | ({ __typename?: "TimeDelta" } & TimeDeltaDataFragment); - } | null; - }; - }; - }; - }; - } | null; - }; -}; - export type TimeDeltaDataFragment = { __typename?: "TimeDelta"; every: number; unit: TimeUnit }; export type SearchDatasetsAutocompleteQueryVariables = Exact<{ @@ -4524,7 +4401,7 @@ export const FlowSummaryDataFragmentDoc = gql` shiftedFrom } ... on FlowStartConditionBatching { - activeTransformRule { + activeBatchingRule { minRecordsToAwait maxBatchingInterval { ...TimeDeltaData @@ -4543,14 +4420,6 @@ export const FlowSummaryDataFragmentDoc = gql` } configSnapshot { ... on FlowConfigurationIngest { - schedule { - ... on TimeDelta { - ...TimeDeltaData - } - ... on Cron5ComponentExpression { - cron5ComponentExpression - } - } fetchUncacheable } ... on FlowConfigurationCompactionRule { @@ -4630,7 +4499,7 @@ export const FlowHistoryDataFragmentDoc = gql` shiftedFrom } ... on FlowStartConditionBatching { - activeTransformRule { + activeBatchingRule { minRecordsToAwait maxBatchingInterval { ...TimeDeltaData @@ -5551,7 +5420,7 @@ export const AccountDatasetFlowsPausedDocument = gql` accounts { byName(name: $accountName) { flows { - configs { + triggers { allPaused } } @@ -5649,7 +5518,7 @@ export const AccountPauseFlowsDocument = gql` accounts { byName(accountName: $accountName) { flows { - configs { + triggers { pauseAccountDatasetFlows } } @@ -5676,7 +5545,7 @@ export const AccountResumeFlowsDocument = gql` accounts { byName(accountName: $accountName) { flows { - configs { + triggers { resumeAccountDatasetFlows } } @@ -5746,64 +5615,6 @@ export class FetchAccountDetailsGQL extends Apollo.Mutation< super(apollo); } } -export const DatasetFlowCompactionDocument = gql` - mutation datasetFlowCompaction( - $datasetId: DatasetID! - $datasetFlowType: DatasetFlowType! - $compactionArgs: CompactionConditionInput! - ) { - datasets { - byId(datasetId: $datasetId) { - flows { - configs { - setConfigCompaction(datasetFlowType: $datasetFlowType, compactionArgs: $compactionArgs) { - ... on SetFlowConfigSuccess { - message - config { - compaction { - ... on CompactionFull { - maxSliceSize - maxSliceRecords - } - ... on CompactionMetadataOnly { - recursive - } - } - } - } - ... on FlowIncompatibleDatasetKind { - message - expectedDatasetKind - actualDatasetKind - } - ... on FlowTypeIsNotSupported { - message - } - ... on FlowInvalidCompactionConfig { - reason - message - } - } - } - } - } - } - } -`; - -@Injectable({ - providedIn: "root", -}) -export class DatasetFlowCompactionGQL extends Apollo.Mutation< - DatasetFlowCompactionMutation, - DatasetFlowCompactionMutationVariables -> { - document = DatasetFlowCompactionDocument; - - constructor(apollo: Apollo.Apollo) { - super(apollo); - } -} export const CommitEventToDatasetDocument = gql` mutation commitEventToDataset($datasetId: DatasetID!, $event: String!) { datasets { @@ -6620,7 +6431,7 @@ export const DatasetAllFlowsPausedDocument = gql` datasets { byId(datasetId: $datasetId) { flows { - configs { + triggers { allPaused } } @@ -6801,7 +6612,7 @@ export const DatasetPauseFlowsDocument = gql` datasets { byId(datasetId: $datasetId) { flows { - configs { + triggers { pauseFlows(datasetFlowType: $datasetFlowType) } } @@ -6828,7 +6639,7 @@ export const DatasetResumeFlowsDocument = gql` datasets { byId(datasetId: $datasetId) { flows { - configs { + triggers { resumeFlows(datasetFlowType: $datasetFlowType) } } @@ -6966,68 +6777,6 @@ export class RenameDatasetGQL extends Apollo.Mutation { - document = DatasetFlowBatchingDocument; - - constructor(apollo: Apollo.Apollo) { - super(apollo); - } -} export const GetDatasetFlowConfigsDocument = gql` query getDatasetFlowConfigs($datasetId: DatasetID!, $datasetFlowType: DatasetFlowType!) { datasets { @@ -7037,23 +6786,17 @@ export const GetDatasetFlowConfigsDocument = gql` configs { __typename byType(datasetFlowType: $datasetFlowType) { - paused ingest { - schedule { - ... on TimeDelta { - ...TimeDeltaData - } - ... on Cron5ComponentExpression { - cron5ComponentExpression - } - } fetchUncacheable } - transform { - maxBatchingInterval { - ...TimeDeltaData + reset { + oldHeadHash + recursive + mode { + ... on SnapshotConfigurationResetToSeedDummy { + dummy + } } - minRecordsToAwait } compaction { ... on CompactionFull { @@ -7069,7 +6812,6 @@ export const GetDatasetFlowConfigsDocument = gql` } } ${DatasetBasicsFragmentDoc} - ${TimeDeltaDataFragmentDoc} `; @Injectable({ @@ -7085,68 +6827,6 @@ export class GetDatasetFlowConfigsGQL extends Apollo.Query< super(apollo); } } -export const DatasetFlowScheduleDocument = gql` - mutation DatasetFlowSchedule( - $datasetId: DatasetID! - $datasetFlowType: DatasetFlowType! - $paused: Boolean! - $ingest: IngestConditionInput! - ) { - datasets { - byId(datasetId: $datasetId) { - flows { - configs { - setConfigIngest(datasetFlowType: $datasetFlowType, paused: $paused, ingest: $ingest) { - __typename - ... on SetFlowConfigSuccess { - message - config { - ingest { - schedule { - ... on TimeDelta { - ...TimeDeltaData - } - ... on Cron5ComponentExpression { - cron5ComponentExpression - } - } - fetchUncacheable - } - } - } - ... on FlowIncompatibleDatasetKind { - message - expectedDatasetKind - actualDatasetKind - } - ... on FlowPreconditionsNotMet { - message - } - ... on FlowTypeIsNotSupported { - message - } - } - } - } - } - } - } - ${TimeDeltaDataFragmentDoc} -`; - -@Injectable({ - providedIn: "root", -}) -export class DatasetFlowScheduleGQL extends Apollo.Mutation< - DatasetFlowScheduleMutation, - DatasetFlowScheduleMutationVariables -> { - document = DatasetFlowScheduleDocument; - - constructor(apollo: Apollo.Apollo) { - super(apollo); - } -} export const SearchDatasetsAutocompleteDocument = gql` query searchDatasetsAutocomplete($query: String!, $perPage: Int, $page: Int) { search { diff --git a/src/app/common/components/flows-table/flows-table.component.ts b/src/app/common/components/flows-table/flows-table.component.ts index 7c1df635b..b10679b71 100644 --- a/src/app/common/components/flows-table/flows-table.component.ts +++ b/src/app/common/components/flows-table/flows-table.component.ts @@ -22,7 +22,6 @@ import { Dataset, DatasetListFlowsDataFragment, DatasetFlowType, - IngestConditionInput, } from "src/app/api/kamu.graphql.interface"; import AppValues from "src/app/common/app.values"; import { MatTableDataSource } from "@angular/material/table"; @@ -213,7 +212,9 @@ export class FlowsTableComponent extends BaseComponent implements OnInit, OnChan datasetId: node.description.datasetId, datasetFlowType: DatasetFlowType.Ingest, flowRunConfiguration: { - ingest: this.setScheduleOptions(node), + ingest: { + fetchUncacheable: true, + }, }, }) .pipe(takeUntilDestroyed(this.destroyRef)) @@ -227,35 +228,29 @@ export class FlowsTableComponent extends BaseComponent implements OnInit, OnChan } } - private setScheduleOptions(node: FlowSummaryDataFragment): IngestConditionInput { - /* istanbul ignore else */ - if (node.configSnapshot?.__typename === "FlowConfigurationIngest") { - switch (node.configSnapshot.schedule.__typename) { - case "TimeDelta": - return { - schedule: { - timeDelta: { - every: node.configSnapshot.schedule.every, - unit: node.configSnapshot.schedule.unit, - }, - }, - fetchUncacheable: true, - }; - case "Cron5ComponentExpression": - return { - schedule: { - cron5ComponentExpression: node.configSnapshot.schedule.cron5ComponentExpression, - }, - fetchUncacheable: true, - }; - /* istanbul ignore next */ - default: - throw new Error("Unknown configuration schedule type"); - } - } else { - throw new Error("The type for the configuration is not FlowConfigurationIngest"); - } - } + // private setScheduleOptions(node: FlowSummaryDataFragment): IngestConditionInput { + // /* istanbul ignore else */ + // if (node.configSnapshot?.__typename === "FlowConfigurationIngest") { + // switch (node.configSnapshot.schedule.__typename) { + // case "TimeDelta": + // return { + // fetchUncacheable: true, + // }; + // case "Cron5ComponentExpression": + // return { + // schedule: { + // cron5ComponentExpression: node.configSnapshot.schedule.cron5ComponentExpression, + // }, + // fetchUncacheable: true, + // }; + // /* istanbul ignore next */ + // default: + // throw new Error("Unknown configuration schedule type"); + // } + // } else { + // throw new Error("The type for the configuration is not FlowConfigurationIngest"); + // } + // } private initializeFilters(): void { this.dropdownDatasetList = this.involvedDatasets.slice(0, this.FILTERED_ITEMS_COUNT); diff --git a/src/app/common/components/flows-table/flows-table.helpers.mock.ts b/src/app/common/components/flows-table/flows-table.helpers.mock.ts index 99ed7484b..5e8529bd8 100644 --- a/src/app/common/components/flows-table/flows-table.helpers.mock.ts +++ b/src/app/common/components/flows-table/flows-table.helpers.mock.ts @@ -310,7 +310,7 @@ export const mockFlowSummaryDataFragmentTooltipAndDurationText: FlowSummaryDataF }, startCondition: { __typename: "FlowStartConditionBatching", - activeTransformRule: { + activeBatchingRule: { minRecordsToAwait: 500, maxBatchingInterval: { every: 5, @@ -580,11 +580,6 @@ export const mockFlowSummaryDataFragmentShowForceLink: FlowSummaryDataFragment = }, startCondition: null, configSnapshot: { - schedule: { - every: 1, - unit: TimeUnit.Minutes, - __typename: "TimeDelta", - }, fetchUncacheable: false, __typename: "FlowConfigurationIngest", }, diff --git a/src/app/common/components/flows-table/flows-table.helpers.spec.ts b/src/app/common/components/flows-table/flows-table.helpers.spec.ts index cc0d93e34..50f000914 100644 --- a/src/app/common/components/flows-table/flows-table.helpers.spec.ts +++ b/src/app/common/components/flows-table/flows-table.helpers.spec.ts @@ -50,7 +50,7 @@ describe("DatasetFlowTableHelpers", () => { expect( DatasetFlowTableHelpers.waitingBlockText({ __typename: "FlowStartConditionBatching", - activeTransformRule: { + activeBatchingRule: { minRecordsToAwait: 500, maxBatchingInterval: { every: 5, diff --git a/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts b/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts index 23d9e4bdd..989b355ec 100644 --- a/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts +++ b/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts @@ -11,7 +11,7 @@ import { FlowStartCondition, FlowStatus, FlowSummaryDataFragment, - FlowTrigger, + FlowTriggerType, TaskStatus, } from "src/app/api/kamu.graphql.interface"; import AppValues from "src/app/common/app.values"; @@ -230,7 +230,7 @@ export class DatasetFlowDetailsHelpers { } } - private static describeTrigger(trigger: FlowTrigger): string { + private static describeTrigger(trigger: FlowTriggerType): string { switch (trigger.__typename) { case "FlowTriggerAutoPolling": return "automatically"; @@ -246,7 +246,7 @@ export class DatasetFlowDetailsHelpers { } } - private static describeTriggerDetails(trigger: FlowTrigger): string { + private static describeTriggerDetails(trigger: FlowTriggerType): string { switch (trigger.__typename) { case "FlowTriggerAutoPolling": return ""; @@ -287,7 +287,7 @@ export class DatasetFlowDetailsHelpers { )}, shifted from ${moment(startCondition.shiftedFrom).format(AppValues.TIME_FORMAT)}`; case "FlowStartConditionBatching": return `Accumulated ${startCondition.accumulatedRecordsCount}/${ - startCondition.activeTransformRule.minRecordsToAwait + startCondition.activeBatchingRule.minRecordsToAwait } records. Watermark ${ startCondition.watermarkModified ? "modified" : "unchanged" }. Deadline at ${moment(startCondition.batchingDeadline).format( diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-compaction.service.ts b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-compaction.service.ts index 6e4381dba..3de88df75 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-compaction.service.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-compaction.service.ts @@ -2,13 +2,7 @@ import { inject, Injectable } from "@angular/core"; import { ToastrService } from "ngx-toastr"; import { Observable, map, of, switchMap } from "rxjs"; import { DatasetFlowApi } from "src/app/api/dataset-flow.api"; -import { - CompactionConditionInput, - DatasetFlowCompactionMutation, - DatasetFlowType, - DatasetTriggerFlowMutation, - FlowRunConfiguration, -} from "src/app/api/kamu.graphql.interface"; +import { DatasetFlowType, DatasetTriggerFlowMutation, FlowRunConfiguration } from "src/app/api/kamu.graphql.interface"; import { DatasetFlowsService } from "../../flows-component/services/dataset-flows.service"; @Injectable({ @@ -19,36 +13,36 @@ export class DatasetCompactionService { private toastrService = inject(ToastrService); private flowsService = inject(DatasetFlowsService); - public runHardCompaction(params: { - datasetId: string; - datasetFlowType: DatasetFlowType; - compactionArgs: CompactionConditionInput; - }): Observable { - return this.datasetFlowApi.setDatasetFlowCompaction(params).pipe( - map((data: DatasetFlowCompactionMutation) => { - if (data.datasets.byId?.flows.configs.setConfigCompaction.__typename === "SetFlowConfigSuccess") { - return true; - } else { - this.toastrService.error(data.datasets.byId?.flows.configs.setConfigCompaction.message); - return false; - } - }), - switchMap((success: boolean) => { - if (success && params.compactionArgs.full) { - return this.flowsService.datasetTriggerFlow({ - datasetId: params.datasetId, - datasetFlowType: params.datasetFlowType, - flowRunConfiguration: { - compaction: { - full: params.compactionArgs.full, - }, - }, - }); - } - return of(false); - }), - ); - } + // public runHardCompaction(params: { + // datasetId: string; + // datasetFlowType: DatasetFlowType; + // compactionArgs: CompactionConditionInput; + // }): Observable { + // return this.datasetFlowApi.setDatasetFlowCompaction(params).pipe( + // map((data: DatasetFlowCompactionMutation) => { + // if (data.datasets.byId?.flows.configs.setConfigCompaction.__typename === "SetFlowConfigSuccess") { + // return true; + // } else { + // this.toastrService.error(data.datasets.byId?.flows.configs.setConfigCompaction.message); + // return false; + // } + // }), + // switchMap((success: boolean) => { + // if (success && params.compactionArgs.full) { + // return this.flowsService.datasetTriggerFlow({ + // datasetId: params.datasetId, + // datasetFlowType: params.datasetFlowType, + // flowRunConfiguration: { + // compaction: { + // full: params.compactionArgs.full, + // }, + // }, + // }); + // } + // return of(false); + // }), + // ); + // } public resetToSeed(params: { accountId: string; diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts index 441199bb4..acc04e6d8 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts @@ -3,14 +3,7 @@ import { inject, Injectable } from "@angular/core"; import { ToastrService } from "ngx-toastr"; import { Observable, map } from "rxjs"; import { DatasetFlowApi } from "src/app/api/dataset-flow.api"; -import { - DatasetFlowBatchingMutation, - DatasetFlowScheduleMutation, - DatasetFlowType, - GetDatasetFlowConfigsQuery, - IngestConditionInput, - TransformConditionInput, -} from "src/app/api/kamu.graphql.interface"; +import { DatasetFlowType, GetDatasetFlowConfigsQuery } from "src/app/api/kamu.graphql.interface"; import AppValues from "src/app/common/app.values"; import { DatasetViewTypeEnum } from "src/app/dataset-view/dataset-view.interface"; import { DatasetInfo } from "src/app/interface/navigation.interface"; @@ -30,53 +23,53 @@ export class DatasetSchedulingService { return this.datasetFlowApi.getDatasetFlowConfigs({ datasetId, datasetFlowType }); } - public setDatasetFlowSchedule(params: { - accountId: string; - datasetId: string; - datasetFlowType: DatasetFlowType; - paused: boolean; - ingest: IngestConditionInput; - datasetInfo: DatasetInfo; - }): Observable { - return this.datasetFlowApi.setDatasetFlowSchedule(params).pipe( - map((data: DatasetFlowScheduleMutation) => { - const setConfigSchedule = data.datasets.byId?.flows.configs.setConfigIngest; - if (setConfigSchedule?.__typename === "SetFlowConfigSuccess") { - setTimeout(() => { - this.navigationService.navigateToDatasetView({ - accountName: params.datasetInfo.accountName, - datasetName: params.datasetInfo.datasetName, - tab: DatasetViewTypeEnum.Flows, - }); - }, AppValues.SIMULATION_START_CONDITION_DELAY_MS); - } else if (setConfigSchedule?.__typename === "FlowIncompatibleDatasetKind") { - this.toastrService.error(setConfigSchedule.message); - } - }), - ); - } + // public setDatasetFlowSchedule(params: { + // accountId: string; + // datasetId: string; + // datasetFlowType: DatasetFlowType; + // paused: boolean; + // ingest: IngestConditionInput; + // datasetInfo: DatasetInfo; + // }): Observable { + // return this.datasetFlowApi.setDatasetFlowSchedule(params).pipe( + // map((data: DatasetFlowScheduleMutation) => { + // const setConfigSchedule = data.datasets.byId?.flows.configs.setConfigIngest; + // if (setConfigSchedule?.__typename === "SetFlowConfigSuccess") { + // setTimeout(() => { + // this.navigationService.navigateToDatasetView({ + // accountName: params.datasetInfo.accountName, + // datasetName: params.datasetInfo.datasetName, + // tab: DatasetViewTypeEnum.Flows, + // }); + // }, AppValues.SIMULATION_START_CONDITION_DELAY_MS); + // } else if (setConfigSchedule?.__typename === "FlowIncompatibleDatasetKind") { + // this.toastrService.error(setConfigSchedule.message); + // } + // }), + // ); + // } - public setDatasetFlowBatching(params: { - accountId: string; - datasetId: string; - datasetFlowType: DatasetFlowType; - paused: boolean; - transform: TransformConditionInput; - datasetInfo: DatasetInfo; - }): Observable { - return this.datasetFlowApi.setDatasetFlowBatching(params).pipe( - map((data: DatasetFlowBatchingMutation) => { - const setConfigBatching = data.datasets.byId?.flows.configs.setConfigTransform; - setConfigBatching?.__typename === "SetFlowConfigSuccess" - ? setTimeout(() => { - this.navigationService.navigateToDatasetView({ - accountName: params.datasetInfo.accountName, - datasetName: params.datasetInfo.datasetName, - tab: DatasetViewTypeEnum.Flows, - }); - }, AppValues.SIMULATION_START_CONDITION_DELAY_MS) - : this.toastrService.error(setConfigBatching?.message); - }), - ); - } + // public setDatasetFlowBatching(params: { + // accountId: string; + // datasetId: string; + // datasetFlowType: DatasetFlowType; + // paused: boolean; + // transform: TransformConditionInput; + // datasetInfo: DatasetInfo; + // }): Observable { + // return this.datasetFlowApi.setDatasetFlowBatching(params).pipe( + // map((data: DatasetFlowBatchingMutation) => { + // const setConfigBatching = data.datasets.byId?.flows.configs.setConfigTransform; + // setConfigBatching?.__typename === "SetFlowConfigSuccess" + // ? setTimeout(() => { + // this.navigationService.navigateToDatasetView({ + // accountName: params.datasetInfo.accountName, + // datasetName: params.datasetInfo.datasetName, + // tab: DatasetViewTypeEnum.Flows, + // }); + // }, AppValues.SIMULATION_START_CONDITION_DELAY_MS) + // : this.toastrService.error(setConfigBatching?.message); + // }), + // ); + // } } diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/compacting/dataset-settings-compacting-tab.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/compacting/dataset-settings-compacting-tab.component.ts index b8022a37c..4ad83d663 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/compacting/dataset-settings-compacting-tab.component.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/compacting/dataset-settings-compacting-tab.component.ts @@ -92,41 +92,41 @@ export class DatasetSettingsCompactingTabComponent extends BaseComponent impleme } public onRunCompaction(): void { - promiseWithCatch( - this.modalService.error({ - title: "Run compaction", - message: "Do you want to run hard compaction?", - yesButtonText: "Ok", - noButtonText: "Cancel", - handler: (ok) => { - if (ok) { - this.datasetCompactionService - .runHardCompaction({ - datasetId: this.datasetBasics.id, - datasetFlowType: DatasetFlowType.HardCompaction, - compactionArgs: { - full: { - maxSliceSize: this.sliceSizeInBytes, - maxSliceRecords: this.recordsCount.value as number, - recursive: this.recursive.value as boolean, - }, - }, - }) - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe((result: boolean) => { - if (result) { - setTimeout(() => { - this.navigationService.navigateToDatasetView({ - accountName: this.datasetBasics.owner.accountName, - datasetName: this.datasetBasics.name, - tab: DatasetViewTypeEnum.Flows, - }); - }, AppValues.SIMULATION_START_CONDITION_DELAY_MS); - } - }); - } - }, - }), - ); + // promiseWithCatch( + // this.modalService.error({ + // title: "Run compaction", + // message: "Do you want to run hard compaction?", + // yesButtonText: "Ok", + // noButtonText: "Cancel", + // handler: (ok) => { + // if (ok) { + // this.datasetCompactionService + // .runHardCompaction({ + // datasetId: this.datasetBasics.id, + // datasetFlowType: DatasetFlowType.HardCompaction, + // compactionArgs: { + // full: { + // maxSliceSize: this.sliceSizeInBytes, + // maxSliceRecords: this.recordsCount.value as number, + // recursive: this.recursive.value as boolean, + // }, + // }, + // }) + // .pipe(takeUntilDestroyed(this.destroyRef)) + // .subscribe((result: boolean) => { + // if (result) { + // setTimeout(() => { + // this.navigationService.navigateToDatasetView({ + // accountName: this.datasetBasics.owner.accountName, + // datasetName: this.datasetBasics.name, + // tab: DatasetViewTypeEnum.Flows, + // }); + // }, AppValues.SIMULATION_START_CONDITION_DELAY_MS); + // } + // }); + // } + // }, + // }), + // ); } } diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts index d173345a8..2fb3e5941 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts @@ -163,110 +163,100 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme } private checkStatusSection(): void { - if (this.datasetBasics.kind === DatasetKind.Root) { - this.batchingForm.disable(); - this.pollingGroup.enable(); - this.cronExpression.disable(); - this.datasetSchedulingService - .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.Ingest) - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe((data) => { - const flowConfiguration = data.datasets.byId?.flows.configs.byType?.ingest; - const paused = data.datasets.byId?.flows.configs.byType?.paused; - if (flowConfiguration?.schedule) { - this.pollingForm.patchValue({ updatesState: !paused }); - this.pollingGroup.patchValue({ - ...flowConfiguration.schedule, - fetchUncacheable: flowConfiguration.fetchUncacheable, - }); - if (flowConfiguration.schedule.__typename === "Cron5ComponentExpression") { - this.pollingGroup.patchValue({ - // splice for sync with cron parser - cronExpression: flowConfiguration.schedule.cron5ComponentExpression, - }); - } - } - }); - } else { - this.pollingGroup.disable(); - this.batchingForm.enable(); - this.datasetSchedulingService - .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.ExecuteTransform) - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe((data) => { - const flowConfiguration = data.datasets.byId?.flows.configs.byType; - const paused = data.datasets.byId?.flows.configs.byType?.paused; - if (flowConfiguration?.transform) { - const batchingConfig = flowConfiguration.transform; - this.pollingForm.patchValue({ updatesState: !paused }); - this.batchingForm.patchValue({ - ...batchingConfig.maxBatchingInterval, - minRecordsToAwait: batchingConfig.minRecordsToAwait, - }); - } - }); - } + // if (this.datasetBasics.kind === DatasetKind.Root) { + // this.batchingForm.disable(); + // this.pollingGroup.enable(); + // this.cronExpression.disable(); + // this.datasetSchedulingService + // .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.Ingest) + // .pipe(takeUntilDestroyed(this.destroyRef)) + // .subscribe((data) => { + // const flowConfiguration = data.datasets.byId?.flows.configs.byType?.ingest; + // const paused = data.datasets.byId?.flows.configs.byType?.paused; + // if (flowConfiguration?.schedule) { + // this.pollingForm.patchValue({ updatesState: !paused }); + // this.pollingGroup.patchValue({ + // ...flowConfiguration.schedule, + // fetchUncacheable: flowConfiguration.fetchUncacheable, + // }); + // if (flowConfiguration.schedule.__typename === "Cron5ComponentExpression") { + // this.pollingGroup.patchValue({ + // // splice for sync with cron parser + // cronExpression: flowConfiguration.schedule.cron5ComponentExpression, + // }); + // } + // } + // }); + // } else { + // this.pollingGroup.disable(); + // this.batchingForm.enable(); + // this.datasetSchedulingService + // .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.ExecuteTransform) + // .pipe(takeUntilDestroyed(this.destroyRef)) + // .subscribe((data) => { + // const flowConfiguration = data.datasets.byId?.flows.configs.byType; + // const paused = data.datasets.byId?.flows.configs.byType?.paused; + // if (flowConfiguration?.transform) { + // const batchingConfig = flowConfiguration.transform; + // this.pollingForm.patchValue({ updatesState: !paused }); + // this.batchingForm.patchValue({ + // ...batchingConfig.maxBatchingInterval, + // minRecordsToAwait: batchingConfig.minRecordsToAwait, + // }); + // } + // }); + // } } public onSubmit(): void { - if (this.datasetBasics.kind === DatasetKind.Root) { - this.setScheduleOptions(); - this.datasetSchedulingService - .setDatasetFlowSchedule({ - accountId: this.datasetBasics.owner.id, - datasetId: this.datasetBasics.id, - datasetFlowType: DatasetFlowType.Ingest, - paused: !(this.updateState.value as boolean), - ingest: this.scheduleOptions, - datasetInfo: { - accountName: this.datasetBasics.owner.accountName, - datasetName: this.datasetBasics.name, - }, - }) - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe(); - } else { - this.datasetSchedulingService - .setDatasetFlowBatching({ - accountId: this.datasetBasics.owner.id, - datasetId: this.datasetBasics.id, - datasetFlowType: DatasetFlowType.ExecuteTransform, - paused: !(this.updateState.value as boolean), - transform: { - minRecordsToAwait: this.batchingMinRecordsToAwait.value as number, - maxBatchingInterval: { - every: this.batchingEveryTime.value as number, - unit: this.batchingUnitTime.value as TimeUnit, - }, - }, - datasetInfo: { - accountName: this.datasetBasics.owner.accountName, - datasetName: this.datasetBasics.name, - }, - }) - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe(); - } + // if (this.datasetBasics.kind === DatasetKind.Root) { + // this.setScheduleOptions(); + // this.datasetSchedulingService + // .setDatasetFlowSchedule({ + // accountId: this.datasetBasics.owner.id, + // datasetId: this.datasetBasics.id, + // datasetFlowType: DatasetFlowType.Ingest, + // paused: !(this.updateState.value as boolean), + // ingest: this.scheduleOptions, + // datasetInfo: { + // accountName: this.datasetBasics.owner.accountName, + // datasetName: this.datasetBasics.name, + // }, + // }) + // .pipe(takeUntilDestroyed(this.destroyRef)) + // .subscribe(); + // } else { + // this.datasetSchedulingService + // .setDatasetFlowBatching({ + // accountId: this.datasetBasics.owner.id, + // datasetId: this.datasetBasics.id, + // datasetFlowType: DatasetFlowType.ExecuteTransform, + // paused: !(this.updateState.value as boolean), + // transform: { + // minRecordsToAwait: this.batchingMinRecordsToAwait.value as number, + // maxBatchingInterval: { + // every: this.batchingEveryTime.value as number, + // unit: this.batchingUnitTime.value as TimeUnit, + // }, + // }, + // datasetInfo: { + // accountName: this.datasetBasics.owner.accountName, + // datasetName: this.datasetBasics.name, + // }, + // }) + // .pipe(takeUntilDestroyed(this.destroyRef)) + // .subscribe(); + // } } private setScheduleOptions(): void { if (this.pollingGroup.controls.__typename.value === PollingGroupEnum.TIME_DELTA) { this.scheduleOptions = { - schedule: { - timeDelta: { - every: this.pollingEveryTime.value as number, - unit: this.pollingUnitTime.value as TimeUnit, - }, - }, fetchUncacheable: this.pollingFetchUncacheable.value as boolean, }; } if (this.pollingGroup.controls.__typename.value === PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION) { this.scheduleOptions = { - schedule: { - // sync with server validator - cron5ComponentExpression: this.cronExpression.value as string, - }, fetchUncacheable: this.pollingFetchUncacheable.value as boolean, }; } diff --git a/src/app/dataset-view/additional-components/flows-component/services/dataset-flows.service.ts b/src/app/dataset-view/additional-components/flows-component/services/dataset-flows.service.ts index 69715b9c6..07a075a3f 100644 --- a/src/app/dataset-view/additional-components/flows-component/services/dataset-flows.service.ts +++ b/src/app/dataset-view/additional-components/flows-component/services/dataset-flows.service.ts @@ -89,7 +89,7 @@ export class DatasetFlowsService { public datasetPauseFlows(params: { datasetId: string; datasetFlowType?: DatasetFlowType }): Observable { return this.datasetFlowApi.datasetPauseFlows(params).pipe( map((data: DatasetPauseFlowsMutation) => { - const result = data.datasets.byId?.flows.configs.pauseFlows; + const result = data.datasets.byId?.flows.triggers.pauseFlows; result ? this.toastrService.success("Flows paused") : this.toastrService.error("Error, flows not paused"); @@ -100,7 +100,7 @@ export class DatasetFlowsService { public datasetResumeFlows(params: { datasetId: string; datasetFlowType?: DatasetFlowType }): Observable { return this.datasetFlowApi.datasetResumeFlows(params).pipe( map((data: DatasetResumeFlowsMutation) => { - const result = data.datasets.byId?.flows.configs.resumeFlows; + const result = data.datasets.byId?.flows.triggers.resumeFlows; result ? this.toastrService.success("Flows resumed") : this.toastrService.error("Error, flows not resumed"); @@ -111,7 +111,7 @@ export class DatasetFlowsService { public allFlowsPaused(datasetId: string): Observable> { return this.datasetFlowApi.allFlowsPaused(datasetId).pipe( map((data: DatasetAllFlowsPausedQuery) => { - return data.datasets.byId?.flows.configs.allPaused; + return data.datasets.byId?.flows.triggers.allPaused; }), ); } diff --git a/src/app/dataset-view/additional-components/overview-component/overview.component.ts b/src/app/dataset-view/additional-components/overview-component/overview.component.ts index 087d121fc..18ef4ca53 100644 --- a/src/app/dataset-view/additional-components/overview-component/overview.component.ts +++ b/src/app/dataset-view/additional-components/overview-component/overview.component.ts @@ -289,6 +289,11 @@ export class OverviewComponent extends BaseComponent implements OnInit { this.datasetBasics.kind === DatasetKind.Root ? DatasetFlowType.Ingest : DatasetFlowType.ExecuteTransform, + flowRunConfiguration: { + ingest: { + fetchUncacheable: true, + }, + }, }) .pipe(takeUntilDestroyed(this.destroyRef)) .subscribe((success: boolean) => { diff --git a/src/app/services/account.service.ts b/src/app/services/account.service.ts index 0e734d08a..fb160cebd 100644 --- a/src/app/services/account.service.ts +++ b/src/app/services/account.service.ts @@ -96,7 +96,7 @@ export class AccountService { public accountAllFlowsPaused(accountName: string): Observable> { return this.accountApi.accountFlowsPaused(accountName).pipe( map((data: AccountDatasetFlowsPausedQuery) => { - return data.accounts.byName?.flows?.configs.allPaused; + return data.accounts.byName?.flows?.triggers.allPaused; }), ); } @@ -104,7 +104,7 @@ export class AccountService { public accountPauseFlows(accountName: string): Observable { return this.accountApi.accountPauseFlows(accountName).pipe( map((data: AccountPauseFlowsMutation) => { - const result = data.accounts.byName?.flows.configs.pauseAccountDatasetFlows; + const result = data.accounts.byName?.flows.triggers.pauseAccountDatasetFlows; result ? this.toastrService.success("Flows paused") : this.toastrService.error("Error, flows not paused"); @@ -115,7 +115,7 @@ export class AccountService { public accountResumeFlows(accountName: string): Observable { return this.accountApi.accountResumeFlows(accountName).pipe( map((data: AccountResumeFlowsMutation) => { - const result = data.accounts.byName?.flows.configs.resumeAccountDatasetFlows; + const result = data.accounts.byName?.flows.triggers.resumeAccountDatasetFlows; result ? this.toastrService.success("Flows resumed") : this.toastrService.error("Error, flows not resumed"); From cfaa025823bf272a30e50ee4babdf6207dd71def Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Fri, 13 Dec 2024 11:07:00 +0200 Subject: [PATCH 02/17] fixed css style --- .../common/components/flows-table/flows-table.component.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app/common/components/flows-table/flows-table.component.html b/src/app/common/components/flows-table/flows-table.component.html index 6111a66a6..a432a2af3 100644 --- a/src/app/common/components/flows-table/flows-table.component.html +++ b/src/app/common/components/flows-table/flows-table.component.html @@ -107,7 +107,9 @@
- calendar_today + calendar_today {{ durationBlockText(element) }} From 55b59d03f73bd860febf0c6e4a4e0952ac7ad4b7 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Fri, 13 Dec 2024 14:53:35 +0200 Subject: [PATCH 03/17] implemented force-update link --- .../components/flows-table/flows-table.component.html | 6 +++--- .../common/components/flows-table/flows-table.component.ts | 6 +----- .../common/components/flows-table/flows-table.helpers.ts | 3 +-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/app/common/components/flows-table/flows-table.component.html b/src/app/common/components/flows-table/flows-table.component.html index a432a2af3..8809cc87e 100644 --- a/src/app/common/components/flows-table/flows-table.component.html +++ b/src/app/common/components/flows-table/flows-table.component.html @@ -86,13 +86,13 @@ " >
  - - + >
diff --git a/src/app/common/components/flows-table/flows-table.component.ts b/src/app/common/components/flows-table/flows-table.component.ts index b10679b71..1bd32986c 100644 --- a/src/app/common/components/flows-table/flows-table.component.ts +++ b/src/app/common/components/flows-table/flows-table.component.ts @@ -195,7 +195,6 @@ export class FlowsTableComponent extends BaseComponent implements OnInit, OnChan return ( node.description.__typename === "FlowDescriptionDatasetPollingIngest" && node.description.ingestResult?.__typename === "FlowDescriptionUpdateResultUpToDate" && - node.configSnapshot?.__typename === "FlowConfigurationIngest" && node.description.ingestResult.uncacheable && ((node.configSnapshot?.__typename === "FlowConfigurationIngest" && !node.configSnapshot.fetchUncacheable) || !node.configSnapshot) @@ -203,10 +202,7 @@ export class FlowsTableComponent extends BaseComponent implements OnInit, OnChan } public onForceUpdate(node: FlowSummaryDataFragment): void { - if ( - node.description.__typename === "FlowDescriptionDatasetPollingIngest" && - node.configSnapshot?.__typename === "FlowConfigurationIngest" - ) { + if (node.description.__typename === "FlowDescriptionDatasetPollingIngest") { this.datasetFlowsService .datasetTriggerFlow({ datasetId: node.description.datasetId, diff --git a/src/app/common/components/flows-table/flows-table.helpers.ts b/src/app/common/components/flows-table/flows-table.helpers.ts index a774fe557..8fd6b4949 100644 --- a/src/app/common/components/flows-table/flows-table.helpers.ts +++ b/src/app/common/components/flows-table/flows-table.helpers.ts @@ -101,8 +101,7 @@ export class DatasetFlowTableHelpers { ((element.configSnapshot?.__typename === "FlowConfigurationIngest" && !element.configSnapshot.fetchUncacheable) || !element.configSnapshot) - ? // TODO: Replace when will be new API - `Source is uncacheable: to re-scan the data, use force update` + ? `Source is uncacheable: to re-scan the data, use` : "Dataset is up-to-date"; case "FlowDescriptionDatasetExecuteTransform": From baf569e413f1409b7bbaaf3d95a59453a937dc06 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Fri, 13 Dec 2024 19:11:33 +0200 Subject: [PATCH 04/17] created new layout --- .../fragments/fragment-dataset-flow.graphql | 8 - .../dataset-settings.component.scss | 2 +- .../dataset-settings.model.ts | 2 +- ...set-settings-scheduling-tab.component.html | 223 ++++++++++-------- ...taset-settings-scheduling-tab.component.ts | 14 +- 5 files changed, 138 insertions(+), 111 deletions(-) diff --git a/src/app/api/gql/flows-dataset/fragments/fragment-dataset-flow.graphql b/src/app/api/gql/flows-dataset/fragments/fragment-dataset-flow.graphql index b8290f3b4..3e6d3377f 100644 --- a/src/app/api/gql/flows-dataset/fragments/fragment-dataset-flow.graphql +++ b/src/app/api/gql/flows-dataset/fragments/fragment-dataset-flow.graphql @@ -118,14 +118,6 @@ fragment FlowSummaryData on Flow { configSnapshot { ... on FlowConfigurationIngest { - # schedule { - # ... on TimeDelta { - # ...TimeDeltaData - # } - # ... on Cron5ComponentExpression { - # cron5ComponentExpression - # } - # } fetchUncacheable } diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.component.scss b/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.component.scss index b9081d45e..3d26fabf2 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.component.scss +++ b/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.component.scss @@ -6,7 +6,7 @@ p { .custom-container { display: grid; - grid-template-columns: 300px 1fr; + grid-template-columns: 275px 1fr; } .content-container { diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.model.ts b/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.model.ts index dc4c9781c..fdbc7eb06 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.model.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.model.ts @@ -36,7 +36,7 @@ export const datasetSettingsSidePanelData: DatasetSettingsSidePanelItem[] = [ id: "general", }, { - name: "Scheduling", + name: "Scheduled updates", iconName: "clock", showDivider: false, activeTab: SettingsTabsEnum.SCHEDULING, diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html index e198d7ee8..932413733 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html @@ -1,105 +1,141 @@ -
-

Schedule

+
+

Scheduled updates

-

Enable/Disable updates

+

Enable/Disable update schedule

{{ pollingForm.controls.updatesState.value ? "enabled" : "disabled" }}
-
-

Root polling source

- - -
- Update every: - - - -
- {{ pollingGroup.get("every")?.errors?.range.message }} +
+
+
+
+ Triggers +
+ + +
+ Launch every: + + + +
+ {{ pollingGroup.get("every")?.errors?.range.message }} +
+ +
+ + +
+
+ Cron expression : +
+ + + +
+ Invalid expression +
+ + + Next time: {{ nextTime }} + +
+ + +
+ Cron expression accepted values +
1. minutes: 0-59 * , -
+
2. hours: 0-23 * , -
+
3. day of month: 1-31 * , - ?
+
4. months: (JAN-DEC or 1-12) * , -
+
5. day of week: (SUN-SAT or 1-7) * , - ?
- -
- - -
-
- Cron expression :
- +
- +
- - -
- Cron expression accepted values -
1. minutes: 0-59 * , -
-
2. hours: 0-23 * , -
-
3. day of month: 1-31 * , - ?
-
4. months: (JAN-DEC or 1-12) * , -
-
5. day of week: (SUN-SAT or 1-7) * , - ?
-
-
- - Fetch uncacheable +
+
+ Configuration + +
+ + Fetch uncacheable +
+
+
+ +
+
-
+
-

Derivative (batching configuration)

Max batching interval: @@ -151,17 +187,12 @@

Derivative (batching configuration)

+
+
+ +
-
- -
diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts index 2fb3e5941..d4122fd97 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts @@ -36,11 +36,11 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme updatesState: new FormControl(false, { nonNullable: true }), pollingGroup: new FormGroup({ __typename: new FormControl(PollingGroupEnum.TIME_DELTA, [Validators.required]), - every: new FormControl>({ value: null, disabled: true }, [ + every: new FormControl>({ value: null, disabled: false }, [ Validators.required, Validators.min(1), ]), - unit: new FormControl>({ value: null, disabled: true }, [Validators.required]), + unit: new FormControl>({ value: null, disabled: false }, [Validators.required]), cronExpression: new FormControl>({ value: "", disabled: true }, [ Validators.required, cronExpressionValidator(), @@ -50,12 +50,12 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme }); public batchingForm = new FormGroup({ - every: new FormControl>({ value: null, disabled: true }, [ + every: new FormControl>({ value: null, disabled: false }, [ Validators.required, Validators.min(1), ]), - unit: new FormControl>({ value: null, disabled: true }, [Validators.required]), - minRecordsToAwait: new FormControl>({ value: null, disabled: true }, [ + unit: new FormControl>({ value: null, disabled: false }, [Validators.required]), + minRecordsToAwait: new FormControl>({ value: null, disabled: false }, [ Validators.required, Validators.min(1), ]), @@ -107,6 +107,10 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme return cronExpressionNextTime(this.cronExpression.value as string); } + public get isRootDataset(): boolean { + return this.datasetBasics.kind === DatasetKind.Root; + } + public ngOnInit() { if (!this.datasetPermissions.permissions.canSchedule) { this.pollingForm.disable(); From 9dafe8a3e98cc7b85a692d92fd31afbc4434ffc5 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Fri, 13 Dec 2024 19:41:53 +0200 Subject: [PATCH 05/17] Changed label buttons --- .../dataset-settings-scheduling-tab.component.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html index 932413733..18a1332b8 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html @@ -108,7 +108,7 @@

Enable/Disable update schedule

(click)="onSubmit()" data-test-id="save-config-options" > - Trigger flow + Save Trigger
@@ -127,7 +127,7 @@

Enable/Disable update schedule

(click)="onSubmit()" data-test-id="save-config-options" > - Set configuration + Save configuration
@@ -190,7 +190,7 @@

Enable/Disable update schedule

From 9e5c38cd5b1360ac365f68925943e2ced3ea917e Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Mon, 16 Dec 2024 17:55:34 +0200 Subject: [PATCH 06/17] integrated setDatasetFlowConfigs method --- src/app/api/dataset-flow.api.ts | 28 ++++++ .../dataset-flow-set-configs.graphql | 39 ++++++++ src/app/api/kamu.graphql.interface.ts | 84 ++++++++++++++++ .../services/dataset-scheduling.service.ts | 24 ++++- ...set-settings-scheduling-tab.component.html | 41 ++++---- ...taset-settings-scheduling-tab.component.ts | 98 ++++++++++++------- ...settings-scheduling-tab.component.types.ts | 7 +- src/assets/runtime-config.json | 2 +- 8 files changed, 263 insertions(+), 60 deletions(-) create mode 100644 src/app/api/gql/scheduling-dataset/dataset-flow-set-configs.graphql diff --git a/src/app/api/dataset-flow.api.ts b/src/app/api/dataset-flow.api.ts index 71dde9373..eebf6d4b0 100644 --- a/src/app/api/dataset-flow.api.ts +++ b/src/app/api/dataset-flow.api.ts @@ -15,6 +15,7 @@ import { DatasetResumeFlowsMutation, DatasetTriggerFlowGQL, DatasetTriggerFlowMutation, + FlowConfigurationInput, FlowRunConfiguration, GetDatasetFlowConfigsGQL, GetDatasetFlowConfigsQuery, @@ -22,6 +23,8 @@ import { GetDatasetListFlowsQuery, GetFlowByIdGQL, GetFlowByIdQuery, + SetDatasetFlowConfigGQL, + SetDatasetFlowConfigMutation, } from "./kamu.graphql.interface"; import { Observable, first, map } from "rxjs"; import { ApolloQueryResult } from "@apollo/client"; @@ -42,6 +45,7 @@ export class DatasetFlowApi { private cancelScheduledTasksGQL = inject(CancelScheduledTasksGQL); // private datasetFlowCompactionGQL = inject(DatasetFlowCompactionGQL); private datasetFlowsInitiatorsGQL = inject(DatasetFlowsInitiatorsGQL); + private setDatasetFlowConfigGQL = inject(SetDatasetFlowConfigGQL); public datasetTriggerFlow(params: { accountId: string; @@ -75,6 +79,30 @@ export class DatasetFlowApi { ); } + public setDatasetFlowConfigs(params: { + datasetId: string; + datasetFlowType: DatasetFlowType; + configInput: FlowConfigurationInput; + }): Observable { + return this.setDatasetFlowConfigGQL + .mutate({ + datasetId: params.datasetId, + datasetFlowType: params.datasetFlowType, + configInput: params.configInput, + }) + .pipe( + first(), + map((result: MutationResult) => { + /* istanbul ignore else */ + if (result.data) { + return result.data; + } else { + throw new DatasetOperationError(result.errors ?? []); + } + }), + ); + } + // public setDatasetFlowSchedule(params: { // accountId: string; // datasetId: string; diff --git a/src/app/api/gql/scheduling-dataset/dataset-flow-set-configs.graphql b/src/app/api/gql/scheduling-dataset/dataset-flow-set-configs.graphql new file mode 100644 index 000000000..0646f2f4f --- /dev/null +++ b/src/app/api/gql/scheduling-dataset/dataset-flow-set-configs.graphql @@ -0,0 +1,39 @@ +mutation setDatasetFlowConfig( + $datasetId: DatasetID! + $datasetFlowType: DatasetFlowType! + $configInput: FlowConfigurationInput! +) { + datasets { + byId(datasetId: $datasetId) { + flows { + configs { + setConfig(datasetFlowType: $datasetFlowType, configInput: $configInput) { + ... on SetFlowConfigSuccess { + message + } + + ... on FlowTypeIsNotSupported { + message + } + + ... on FlowPreconditionsNotMet { + message + preconditions + } + + ... on FlowInvalidConfigInputError { + message + reason + } + + ... on FlowIncompatibleDatasetKind { + message + actualDatasetKind + expectedDatasetKind + } + } + } + } + } + } +} diff --git a/src/app/api/kamu.graphql.interface.ts b/src/app/api/kamu.graphql.interface.ts index a8519d1f9..e5f19d552 100644 --- a/src/app/api/kamu.graphql.interface.ts +++ b/src/app/api/kamu.graphql.interface.ts @@ -4094,6 +4094,39 @@ export type GetDatasetFlowConfigsQuery = { }; }; +export type SetDatasetFlowConfigMutationVariables = Exact<{ + datasetId: Scalars["DatasetID"]; + datasetFlowType: DatasetFlowType; + configInput: FlowConfigurationInput; +}>; + +export type SetDatasetFlowConfigMutation = { + __typename?: "Mutation"; + datasets: { + __typename?: "DatasetsMut"; + byId?: { + __typename?: "DatasetMut"; + flows: { + __typename?: "DatasetFlowsMut"; + configs: { + __typename?: "DatasetFlowConfigsMut"; + setConfig: + | { + __typename?: "FlowIncompatibleDatasetKind"; + message: string; + actualDatasetKind: DatasetKind; + expectedDatasetKind: DatasetKind; + } + | { __typename?: "FlowInvalidConfigInputError"; message: string; reason: string } + | { __typename?: "FlowPreconditionsNotMet"; message: string; preconditions: string } + | { __typename?: "FlowTypeIsNotSupported"; message: string } + | { __typename?: "SetFlowConfigSuccess"; message: string }; + }; + }; + } | null; + }; +}; + export type TimeDeltaDataFragment = { __typename?: "TimeDelta"; every: number; unit: TimeUnit }; export type SearchDatasetsAutocompleteQueryVariables = Exact<{ @@ -6827,6 +6860,57 @@ export class GetDatasetFlowConfigsGQL extends Apollo.Query< super(apollo); } } +export const SetDatasetFlowConfigDocument = gql` + mutation setDatasetFlowConfig( + $datasetId: DatasetID! + $datasetFlowType: DatasetFlowType! + $configInput: FlowConfigurationInput! + ) { + datasets { + byId(datasetId: $datasetId) { + flows { + configs { + setConfig(datasetFlowType: $datasetFlowType, configInput: $configInput) { + ... on SetFlowConfigSuccess { + message + } + ... on FlowTypeIsNotSupported { + message + } + ... on FlowPreconditionsNotMet { + message + preconditions + } + ... on FlowInvalidConfigInputError { + message + reason + } + ... on FlowIncompatibleDatasetKind { + message + actualDatasetKind + expectedDatasetKind + } + } + } + } + } + } + } +`; + +@Injectable({ + providedIn: "root", +}) +export class SetDatasetFlowConfigGQL extends Apollo.Mutation< + SetDatasetFlowConfigMutation, + SetDatasetFlowConfigMutationVariables +> { + document = SetDatasetFlowConfigDocument; + + constructor(apollo: Apollo.Apollo) { + super(apollo); + } +} export const SearchDatasetsAutocompleteDocument = gql` query searchDatasetsAutocomplete($query: String!, $perPage: Int, $page: Int) { search { diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts index acc04e6d8..f2b6dd16b 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts @@ -3,7 +3,12 @@ import { inject, Injectable } from "@angular/core"; import { ToastrService } from "ngx-toastr"; import { Observable, map } from "rxjs"; import { DatasetFlowApi } from "src/app/api/dataset-flow.api"; -import { DatasetFlowType, GetDatasetFlowConfigsQuery } from "src/app/api/kamu.graphql.interface"; +import { + DatasetFlowType, + FlowConfigurationInput, + GetDatasetFlowConfigsQuery, + SetDatasetFlowConfigMutation, +} from "src/app/api/kamu.graphql.interface"; import AppValues from "src/app/common/app.values"; import { DatasetViewTypeEnum } from "src/app/dataset-view/dataset-view.interface"; import { DatasetInfo } from "src/app/interface/navigation.interface"; @@ -23,6 +28,23 @@ export class DatasetSchedulingService { return this.datasetFlowApi.getDatasetFlowConfigs({ datasetId, datasetFlowType }); } + public setDatasetFlowConfigs(params: { + datasetId: string; + datasetFlowType: DatasetFlowType; + configInput: FlowConfigurationInput; + }): Observable { + return this.datasetFlowApi.setDatasetFlowConfigs(params).pipe( + map((data: SetDatasetFlowConfigMutation) => { + const setConfig = data.datasets.byId?.flows.configs.setConfig; + if (setConfig?.__typename === "SetFlowConfigSuccess") { + this.toastrService.success("Configuration saved"); + } else { + this.toastrService.error(setConfig?.message); + } + }), + ); + } + // public setDatasetFlowSchedule(params: { // accountId: string; // datasetId: string; diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html index 18a1332b8..ff59918f9 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html @@ -1,22 +1,24 @@

Scheduled updates

-
-
-

Enable/Disable update schedule

- - {{ pollingForm.controls.updatesState.value ? "enabled" : "disabled" }} - -
- -
-
-
-
+
+
+
+ +
Triggers
+
+ + Enable/Disable update schedule + +
Enable/Disable update schedule Save Trigger
-
-
+ +
+
+
Configuration
@@ -124,16 +128,17 @@

Enable/Disable update schedule

-
+
- +
+
diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts index d4122fd97..3b66c03bc 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts @@ -14,7 +14,12 @@ import { import { DatasetSchedulingService } from "../../services/dataset-scheduling.service"; import { cronExpressionValidator, everyTimeMapperValidators } from "src/app/common/data.helpers"; import { cronExpressionNextTime, logError } from "src/app/common/app.helpers"; -import { BatchingFormType, PollingFormType, PollingGroupType } from "./dataset-settings-scheduling-tab.component.types"; +import { + BatchingFormType, + IngestConfigurationFormType, + PollingFormType, + PollingGroupType, +} from "./dataset-settings-scheduling-tab.component.types"; import { takeUntilDestroyed } from "@angular/core/rxjs-interop"; @Component({ @@ -32,9 +37,13 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme private scheduleOptions: IngestConditionInput; private everyTimeMapperValidators: Record = everyTimeMapperValidators; + public ingestConfigurationForm = new FormGroup({ + fetchUncacheable: new FormControl(false, { nonNullable: true }), + }); + public pollingForm = new FormGroup({ - updatesState: new FormControl(false, { nonNullable: true }), pollingGroup: new FormGroup({ + updatesState: new FormControl(false, { nonNullable: true }), __typename: new FormControl(PollingGroupEnum.TIME_DELTA, [Validators.required]), every: new FormControl>({ value: null, disabled: false }, [ Validators.required, @@ -45,7 +54,6 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme Validators.required, cronExpressionValidator(), ]), - fetchUncacheable: new FormControl(false, { nonNullable: true }), }), }); @@ -67,18 +75,10 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme return this.pollingForm.get("pollingGroup") as FormGroup; } - public get updateState(): AbstractControl { - return this.pollingForm.controls.updatesState; - } - public get pollingType(): AbstractControl { return this.pollingGroup.controls.__typename; } - public get pollingFetchUncacheable(): AbstractControl { - return this.pollingGroup.controls.fetchUncacheable; - } - public get batchingEveryTime(): AbstractControl { return this.batchingForm.controls.every; } @@ -111,6 +111,10 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme return this.datasetBasics.kind === DatasetKind.Root; } + public get fetchUncacheable(): FormControl { + return this.ingestConfigurationForm.controls.fetchUncacheable; + } + public ngOnInit() { if (!this.datasetPermissions.permissions.canSchedule) { this.pollingForm.disable(); @@ -122,6 +126,21 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme } } + public saveIngestConfiguration(): void { + this.datasetSchedulingService + .setDatasetFlowConfigs({ + datasetId: this.datasetBasics.id, + datasetFlowType: DatasetFlowType.Ingest, + configInput: { + ingest: { + fetchUncacheable: this.fetchUncacheable.value, + }, + }, + }) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(); + } + private pollingTypeChanges(): void { this.pollingType.valueChanges.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((value: PollingGroupEnum) => { switch (value) { @@ -167,31 +186,34 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme } private checkStatusSection(): void { - // if (this.datasetBasics.kind === DatasetKind.Root) { - // this.batchingForm.disable(); - // this.pollingGroup.enable(); - // this.cronExpression.disable(); - // this.datasetSchedulingService - // .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.Ingest) - // .pipe(takeUntilDestroyed(this.destroyRef)) - // .subscribe((data) => { - // const flowConfiguration = data.datasets.byId?.flows.configs.byType?.ingest; - // const paused = data.datasets.byId?.flows.configs.byType?.paused; - // if (flowConfiguration?.schedule) { - // this.pollingForm.patchValue({ updatesState: !paused }); - // this.pollingGroup.patchValue({ - // ...flowConfiguration.schedule, - // fetchUncacheable: flowConfiguration.fetchUncacheable, - // }); - // if (flowConfiguration.schedule.__typename === "Cron5ComponentExpression") { - // this.pollingGroup.patchValue({ - // // splice for sync with cron parser - // cronExpression: flowConfiguration.schedule.cron5ComponentExpression, - // }); - // } - // } - // }); - // } else { + if (this.datasetBasics.kind === DatasetKind.Root) { + this.batchingForm.disable(); + this.pollingGroup.enable(); + this.cronExpression.disable(); + this.datasetSchedulingService + .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.Ingest) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe((data) => { + const flowConfiguration = data.datasets.byId?.flows.configs.byType?.ingest; + this.ingestConfigurationForm.patchValue({ fetchUncacheable: flowConfiguration?.fetchUncacheable }); + console.log("==>", flowConfiguration); + // const paused = data.datasets.byId?.flows.configs.byType?.paused; + // if (flowConfiguration?.schedule) { + // this.pollingForm.patchValue({ updatesState: !paused }); + // this.pollingGroup.patchValue({ + // ...flowConfiguration.schedule, + // fetchUncacheable: flowConfiguration.fetchUncacheable, + // }); + // if (flowConfiguration.schedule.__typename === "Cron5ComponentExpression") { + // this.pollingGroup.patchValue({ + // // splice for sync with cron parser + // cronExpression: flowConfiguration.schedule.cron5ComponentExpression, + // }); + // } + // } + }); + } + // else { // this.pollingGroup.disable(); // this.batchingForm.enable(); // this.datasetSchedulingService @@ -256,12 +278,12 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme private setScheduleOptions(): void { if (this.pollingGroup.controls.__typename.value === PollingGroupEnum.TIME_DELTA) { this.scheduleOptions = { - fetchUncacheable: this.pollingFetchUncacheable.value as boolean, + fetchUncacheable: this.fetchUncacheable.value, }; } if (this.pollingGroup.controls.__typename.value === PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION) { this.scheduleOptions = { - fetchUncacheable: this.pollingFetchUncacheable.value as boolean, + fetchUncacheable: this.fetchUncacheable.value, }; } } diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.types.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.types.ts index 1dbd062b9..dfd21e7b9 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.types.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.types.ts @@ -4,16 +4,15 @@ import { TimeUnit } from "src/app/api/kamu.graphql.interface"; import { MaybeNull } from "src/app/common/app.types"; export interface PollingFormType { - updatesState: FormControl; pollingGroup: FormGroup; } export interface PollingGroupType { + updatesState: FormControl; __typename: FormControl>; every: FormControl>; unit: FormControl>; cronExpression: FormControl>; - fetchUncacheable: FormControl; } export interface BatchingFormType { @@ -21,3 +20,7 @@ export interface BatchingFormType { unit: FormControl>; minRecordsToAwait: FormControl>; } + +export interface IngestConfigurationFormType { + fetchUncacheable: FormControl; +} diff --git a/src/assets/runtime-config.json b/src/assets/runtime-config.json index 459ef669d..cf29289f7 100644 --- a/src/assets/runtime-config.json +++ b/src/assets/runtime-config.json @@ -3,7 +3,7 @@ "apiServerHttpUrl": "http://localhost:8080", "githubClientId": "361a3b4fda86d0234d2f", "grafanaLogs": { - "taskDetailsUrl": " https://grafana.sha.stg.internal.kamu.dev/explore?schemaVersion=1&panes=%7B%22LYy%22%3A%7B%22datasource%22%3A%22365dd1cc-3781-43c1-86a1-a3f78c6ab420%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22expr%22%3A%22%7Bjob%3D%5C%22europort%2Fkamu-api-server%5C%22%7D+%7C+json+root_span%2C+task_id%3D%5C%22log.spans%5B0%5D.task_id%5C%22%2C+target%3D%5C%22log.target%5C%22%2C+message%3D%5C%22log.fields.message%5C%22+%7C+task_id+%3D+%60{{taskId}}%60+%7C%3D+%60%60%22%2C%22queryType%22%3A%22range%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22365dd1cc-3781-43c1-86a1-a3f78c6ab420%22%7D%2C%22editorMode%22%3A%22builder%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22{{fromTime}}%22%2C%22to%22%3A%22{{toTime}}%22%7D%7D%7D&orgId=1", + "taskDetailsUrl": "https://grafana.sha.stg.internal.kamu.dev/explore?schemaVersion=1&panes=%7B%22LYy%22%3A%7B%22datasource%22%3A%22365dd1cc-3781-43c1-86a1-a3f78c6ab420%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22expr%22%3A%22%7Bjob%3D%5C%22europort%2Fkamu-api-server%5C%22%7D+%7C+json+root_span%2C+task_id%3D%5C%22log.spans%5B0%5D.task_id%5C%22%2C+target%3D%5C%22log.target%5C%22%2C+message%3D%5C%22log.fields.message%5C%22+%7C+task_id+%3D+%60{{taskId}}%60+%7C%3D+%60%60%22%2C%22queryType%22%3A%22range%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22365dd1cc-3781-43c1-86a1-a3f78c6ab420%22%7D%2C%22editorMode%22%3A%22builder%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22{{fromTime}}%22%2C%22to%22%3A%22{{toTime}}%22%7D%7D%7D&orgId=1", "flowHistoryUrl": "" }, "features": { From c3e2b4eb48e796c9f7cc81732e2d2d970e3a953b Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Mon, 16 Dec 2024 23:19:36 +0200 Subject: [PATCH 07/17] integrated scheduling tab --- resources/schema.graphql | 6 + src/app/api/dataset-flow.api.ts | 44 ++++ .../fragment-flow-history-data.graphql | 7 + .../dataset-flow-batching.graphql | 49 ---- .../dataset-flow-schedule.graphql | 48 ---- .../dataset-flow-set-triggers.graphql | 39 +++ .../dataset-flow-triggers.graphql | 29 +++ src/app/api/kamu.graphql.interface.ts | 189 ++++++++++++++ .../flow-details-history-tab.helpers.ts | 19 ++ .../services/dataset-scheduling.service.ts | 35 +++ ...set-settings-scheduling-tab.component.html | 134 +++++----- ...taset-settings-scheduling-tab.component.ts | 235 ++++++++++-------- ...settings-scheduling-tab.component.types.ts | 1 + 13 files changed, 568 insertions(+), 267 deletions(-) delete mode 100644 src/app/api/gql/scheduling-dataset/dataset-flow-batching.graphql delete mode 100644 src/app/api/gql/scheduling-dataset/dataset-flow-schedule.graphql create mode 100644 src/app/api/gql/scheduling-dataset/dataset-flow-set-triggers.graphql create mode 100644 src/app/api/gql/scheduling-dataset/dataset-flow-triggers.graphql diff --git a/resources/schema.graphql b/resources/schema.graphql index c8d2c19f4..b725fa1a8 100644 --- a/resources/schema.graphql +++ b/resources/schema.graphql @@ -1012,6 +1012,12 @@ type FlowAbortedResult { message: String! } +type FlowConfigSnapshotModified implements FlowEvent { + eventId: EventID! + eventTime: DateTime! + configSnapshot: FlowConfigurationSnapshot! +} + type FlowConfiguration { ingest: FlowConfigurationIngest compaction: FlowConfigurationCompaction diff --git a/src/app/api/dataset-flow.api.ts b/src/app/api/dataset-flow.api.ts index eebf6d4b0..683ea9e82 100644 --- a/src/app/api/dataset-flow.api.ts +++ b/src/app/api/dataset-flow.api.ts @@ -17,14 +17,19 @@ import { DatasetTriggerFlowMutation, FlowConfigurationInput, FlowRunConfiguration, + FlowTriggerInput, GetDatasetFlowConfigsGQL, GetDatasetFlowConfigsQuery, + GetDatasetFlowTriggersGQL, + GetDatasetFlowTriggersQuery, GetDatasetListFlowsGQL, GetDatasetListFlowsQuery, GetFlowByIdGQL, GetFlowByIdQuery, SetDatasetFlowConfigGQL, SetDatasetFlowConfigMutation, + SetDatasetFlowTriggersGQL, + SetDatasetFlowTriggersMutation, } from "./kamu.graphql.interface"; import { Observable, first, map } from "rxjs"; import { ApolloQueryResult } from "@apollo/client"; @@ -46,6 +51,8 @@ export class DatasetFlowApi { // private datasetFlowCompactionGQL = inject(DatasetFlowCompactionGQL); private datasetFlowsInitiatorsGQL = inject(DatasetFlowsInitiatorsGQL); private setDatasetFlowConfigGQL = inject(SetDatasetFlowConfigGQL); + private setDatasetFlowTriggersGQL = inject(SetDatasetFlowTriggersGQL); + private getDatasetFlowTriggersGQL = inject(GetDatasetFlowTriggersGQL); public datasetTriggerFlow(params: { accountId: string; @@ -103,6 +110,43 @@ export class DatasetFlowApi { ); } + public setDatasetFlowTriggers(params: { + datasetId: string; + datasetFlowType: DatasetFlowType; + paused: boolean; + triggerInput: FlowTriggerInput; + }): Observable { + return this.setDatasetFlowTriggersGQL.mutate(params).pipe( + first(), + map((result: MutationResult) => { + /* istanbul ignore else */ + if (result.data) { + return result.data; + } else { + throw new DatasetOperationError(result.errors ?? []); + } + }), + ); + } + + public getDatasetFlowTriggers(params: { + datasetId: string; + datasetFlowType: DatasetFlowType; + }): Observable { + return this.getDatasetFlowTriggersGQL + .watch(params, { + ...noCacheFetchPolicy, + context: { + skipLoading: true, + }, + }) + .valueChanges.pipe( + map((result: ApolloQueryResult) => { + return result.data; + }), + ); + } + // public setDatasetFlowSchedule(params: { // accountId: string; // datasetId: string; diff --git a/src/app/api/gql/flows-dataset/fragments/fragment-flow-history-data.graphql b/src/app/api/gql/flows-dataset/fragments/fragment-flow-history-data.graphql index a343f6d7e..aef0b53c6 100644 --- a/src/app/api/gql/flows-dataset/fragments/fragment-flow-history-data.graphql +++ b/src/app/api/gql/flows-dataset/fragments/fragment-flow-history-data.graphql @@ -5,6 +5,7 @@ fragment FlowHistoryData on FlowEvent { ... on FlowEventAborted { __typename } + ... on FlowEventInitiated { trigger { __typename @@ -90,4 +91,10 @@ fragment FlowHistoryData on FlowEvent { } } } + + ... on FlowConfigSnapshotModified { + configSnapshot { + __typename + } + } } diff --git a/src/app/api/gql/scheduling-dataset/dataset-flow-batching.graphql b/src/app/api/gql/scheduling-dataset/dataset-flow-batching.graphql deleted file mode 100644 index 2b8dc50f1..000000000 --- a/src/app/api/gql/scheduling-dataset/dataset-flow-batching.graphql +++ /dev/null @@ -1,49 +0,0 @@ -# mutation datasetFlowBatching( -# $datasetId: DatasetID! -# $datasetFlowType: DatasetFlowType! -# $paused: Boolean! -# $transform: TransformConditionInput! -# ) { -# datasets { -# byId(datasetId: $datasetId) { -# flows { -# configs { -# setConfigTransform(datasetFlowType: $datasetFlowType, paused: $paused, transform: $transform) { -# __typename -# ... on SetFlowConfigSuccess { -# message -# config { -# transform { -# maxBatchingInterval { -# ...TimeDeltaData -# } -# minRecordsToAwait -# } -# } -# } - -# ... on FlowIncompatibleDatasetKind { -# message -# expectedDatasetKind -# actualDatasetKind -# } - -# ... on FlowInvalidTransformConfig { -# message -# reason -# } - -# ... on FlowPreconditionsNotMet { -# message -# preconditions -# } - -# ... on FlowTypeIsNotSupported { -# message -# } -# } -# } -# } -# } -# } -# } diff --git a/src/app/api/gql/scheduling-dataset/dataset-flow-schedule.graphql b/src/app/api/gql/scheduling-dataset/dataset-flow-schedule.graphql deleted file mode 100644 index 00020c049..000000000 --- a/src/app/api/gql/scheduling-dataset/dataset-flow-schedule.graphql +++ /dev/null @@ -1,48 +0,0 @@ -# mutation DatasetFlowSchedule( -# $datasetId: DatasetID! -# $datasetFlowType: DatasetFlowType! -# $paused: Boolean! -# $ingest: IngestConditionInput! -# ) { -# datasets { -# byId(datasetId: $datasetId) { -# flows { -# configs { -# setConfigIngest(datasetFlowType: $datasetFlowType, paused: $paused, ingest: $ingest) { -# __typename -# ... on SetFlowConfigSuccess { -# message -# config { -# ingest { -# schedule { -# ... on TimeDelta { -# ...TimeDeltaData -# } -# ... on Cron5ComponentExpression { -# cron5ComponentExpression -# } -# } -# fetchUncacheable -# } -# } -# } - -# ... on FlowIncompatibleDatasetKind { -# message -# expectedDatasetKind -# actualDatasetKind -# } - -# ... on FlowPreconditionsNotMet { -# message -# } - -# ... on FlowTypeIsNotSupported { -# message -# } -# } -# } -# } -# } -# } -# } diff --git a/src/app/api/gql/scheduling-dataset/dataset-flow-set-triggers.graphql b/src/app/api/gql/scheduling-dataset/dataset-flow-set-triggers.graphql new file mode 100644 index 000000000..c7de87853 --- /dev/null +++ b/src/app/api/gql/scheduling-dataset/dataset-flow-set-triggers.graphql @@ -0,0 +1,39 @@ +mutation setDatasetFlowTriggers( + $datasetId: DatasetID! + $datasetFlowType: DatasetFlowType! + $paused: Boolean! + $triggerInput: FlowTriggerInput! +) { + datasets { + byId(datasetId: $datasetId) { + flows { + triggers { + setTrigger(datasetFlowType: $datasetFlowType, paused: $paused, triggerInput: $triggerInput) { + ... on SetFlowTriggerSuccess { + message + } + + ... on FlowIncompatibleDatasetKind { + message + expectedDatasetKind + actualDatasetKind + } + + ... on FlowPreconditionsNotMet { + message + } + + ... on FlowTypeIsNotSupported { + message + } + + ... on FlowInvalidTriggerInputError { + message + reason + } + } + } + } + } + } +} diff --git a/src/app/api/gql/scheduling-dataset/dataset-flow-triggers.graphql b/src/app/api/gql/scheduling-dataset/dataset-flow-triggers.graphql new file mode 100644 index 000000000..106afc88b --- /dev/null +++ b/src/app/api/gql/scheduling-dataset/dataset-flow-triggers.graphql @@ -0,0 +1,29 @@ +query getDatasetFlowTriggers($datasetId: DatasetID!, $datasetFlowType: DatasetFlowType!) { + datasets { + byId(datasetId: $datasetId) { + flows { + triggers { + byType(datasetFlowType: $datasetFlowType) { + paused + schedule { + ... on TimeDelta { + ...TimeDeltaData + } + + ... on Cron5ComponentExpression { + cron5ComponentExpression + } + } + + batching { + maxBatchingInterval { + ...TimeDeltaData + } + minRecordsToAwait + } + } + } + } + } + } +} diff --git a/src/app/api/kamu.graphql.interface.ts b/src/app/api/kamu.graphql.interface.ts index e5f19d552..700ba1f66 100644 --- a/src/app/api/kamu.graphql.interface.ts +++ b/src/app/api/kamu.graphql.interface.ts @@ -1111,6 +1111,13 @@ export type FlowAbortedResult = { message: Scalars["String"]; }; +export type FlowConfigSnapshotModified = FlowEvent & { + __typename?: "FlowConfigSnapshotModified"; + configSnapshot: FlowConfigurationSnapshot; + eventId: Scalars["EventID"]; + eventTime: Scalars["DateTime"]; +}; + export type FlowConfiguration = { __typename?: "FlowConfiguration"; compaction?: Maybe; @@ -3033,6 +3040,9 @@ export type GetFlowByIdQuery = { flow: { __typename?: "Flow"; history: Array< + | ({ + __typename?: "FlowConfigSnapshotModified"; + } & FlowHistoryData_FlowConfigSnapshotModified_Fragment) | ({ __typename?: "FlowEventAborted"; } & FlowHistoryData_FlowEventAborted_Fragment) @@ -3319,6 +3329,16 @@ export type FlowConnectionDataFragment = { edges: Array<{ __typename?: "FlowEdge"; node: { __typename?: "Flow" } & FlowSummaryDataFragment }>; }; +type FlowHistoryData_FlowConfigSnapshotModified_Fragment = { + __typename: "FlowConfigSnapshotModified"; + eventId: string; + eventTime: string; + configSnapshot: + | { __typename: "FlowConfigurationCompactionRule" } + | { __typename: "FlowConfigurationIngest" } + | { __typename: "FlowConfigurationReset" }; +}; + type FlowHistoryData_FlowEventAborted_Fragment = { __typename: "FlowEventAborted"; eventId: string; eventTime: string }; type FlowHistoryData_FlowEventInitiated_Fragment = { @@ -3390,6 +3410,7 @@ type FlowHistoryData_FlowEventTriggerAdded_Fragment = { }; export type FlowHistoryDataFragment = + | FlowHistoryData_FlowConfigSnapshotModified_Fragment | FlowHistoryData_FlowEventAborted_Fragment | FlowHistoryData_FlowEventInitiated_Fragment | FlowHistoryData_FlowEventScheduledForActivation_Fragment @@ -4127,6 +4148,74 @@ export type SetDatasetFlowConfigMutation = { }; }; +export type SetDatasetFlowTriggersMutationVariables = Exact<{ + datasetId: Scalars["DatasetID"]; + datasetFlowType: DatasetFlowType; + paused: Scalars["Boolean"]; + triggerInput: FlowTriggerInput; +}>; + +export type SetDatasetFlowTriggersMutation = { + __typename?: "Mutation"; + datasets: { + __typename?: "DatasetsMut"; + byId?: { + __typename?: "DatasetMut"; + flows: { + __typename?: "DatasetFlowsMut"; + triggers: { + __typename?: "DatasetFlowTriggersMut"; + setTrigger: + | { + __typename?: "FlowIncompatibleDatasetKind"; + message: string; + expectedDatasetKind: DatasetKind; + actualDatasetKind: DatasetKind; + } + | { __typename?: "FlowInvalidTriggerInputError"; message: string; reason: string } + | { __typename?: "FlowPreconditionsNotMet"; message: string } + | { __typename?: "FlowTypeIsNotSupported"; message: string } + | { __typename?: "SetFlowTriggerSuccess"; message: string }; + }; + }; + } | null; + }; +}; + +export type GetDatasetFlowTriggersQueryVariables = Exact<{ + datasetId: Scalars["DatasetID"]; + datasetFlowType: DatasetFlowType; +}>; + +export type GetDatasetFlowTriggersQuery = { + __typename?: "Query"; + datasets: { + __typename?: "Datasets"; + byId?: { + __typename?: "Dataset"; + flows: { + __typename?: "DatasetFlows"; + triggers: { + __typename?: "DatasetFlowTriggers"; + byType?: { + __typename?: "FlowTrigger"; + paused: boolean; + schedule?: + | { __typename?: "Cron5ComponentExpression"; cron5ComponentExpression: string } + | ({ __typename?: "TimeDelta" } & TimeDeltaDataFragment) + | null; + batching?: { + __typename?: "FlowTriggerBatchingRule"; + minRecordsToAwait: number; + maxBatchingInterval: { __typename?: "TimeDelta" } & TimeDeltaDataFragment; + } | null; + } | null; + }; + }; + } | null; + }; +}; + export type TimeDeltaDataFragment = { __typename?: "TimeDelta"; every: number; unit: TimeUnit }; export type SearchDatasetsAutocompleteQueryVariables = Exact<{ @@ -4582,6 +4671,11 @@ export const FlowHistoryDataFragmentDoc = gql` } } } + ... on FlowConfigSnapshotModified { + configSnapshot { + __typename + } + } } ${AccountFragmentDoc} ${DatasetBasicsFragmentDoc} @@ -6911,6 +7005,101 @@ export class SetDatasetFlowConfigGQL extends Apollo.Mutation< super(apollo); } } +export const SetDatasetFlowTriggersDocument = gql` + mutation setDatasetFlowTriggers( + $datasetId: DatasetID! + $datasetFlowType: DatasetFlowType! + $paused: Boolean! + $triggerInput: FlowTriggerInput! + ) { + datasets { + byId(datasetId: $datasetId) { + flows { + triggers { + setTrigger(datasetFlowType: $datasetFlowType, paused: $paused, triggerInput: $triggerInput) { + ... on SetFlowTriggerSuccess { + message + } + ... on FlowIncompatibleDatasetKind { + message + expectedDatasetKind + actualDatasetKind + } + ... on FlowPreconditionsNotMet { + message + } + ... on FlowTypeIsNotSupported { + message + } + ... on FlowInvalidTriggerInputError { + message + reason + } + } + } + } + } + } + } +`; + +@Injectable({ + providedIn: "root", +}) +export class SetDatasetFlowTriggersGQL extends Apollo.Mutation< + SetDatasetFlowTriggersMutation, + SetDatasetFlowTriggersMutationVariables +> { + document = SetDatasetFlowTriggersDocument; + + constructor(apollo: Apollo.Apollo) { + super(apollo); + } +} +export const GetDatasetFlowTriggersDocument = gql` + query getDatasetFlowTriggers($datasetId: DatasetID!, $datasetFlowType: DatasetFlowType!) { + datasets { + byId(datasetId: $datasetId) { + flows { + triggers { + byType(datasetFlowType: $datasetFlowType) { + paused + schedule { + ... on TimeDelta { + ...TimeDeltaData + } + ... on Cron5ComponentExpression { + cron5ComponentExpression + } + } + batching { + maxBatchingInterval { + ...TimeDeltaData + } + minRecordsToAwait + } + } + } + } + } + } + } + ${TimeDeltaDataFragmentDoc} +`; + +@Injectable({ + providedIn: "root", +}) +export class GetDatasetFlowTriggersGQL extends Apollo.Query< + GetDatasetFlowTriggersQuery, + GetDatasetFlowTriggersQueryVariables +> { + document = GetDatasetFlowTriggersDocument; + + constructor(apollo: Apollo.Apollo) { + super(apollo); + } +} export const SearchDatasetsAutocompleteDocument = gql` query searchDatasetsAutocomplete($query: String!, $perPage: Int, $page: Int) { search { diff --git a/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts b/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts index 989b355ec..e78d0e341 100644 --- a/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts +++ b/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts @@ -1,5 +1,6 @@ import moment from "moment"; import { + FlowConfigSnapshotModified, FlowEventInitiated, FlowEventScheduledForActivation, FlowEventStartConditionUpdated, @@ -42,6 +43,8 @@ export class DatasetFlowDetailsHelpers { } case "FlowEventScheduledForActivation": return "Flow scheduled for activation"; + case "FlowConfigSnapshotModified": + return "Flow configuration was modified"; /* istanbul ignore next */ default: throw new Error("Unknown event typename"); @@ -64,6 +67,8 @@ export class DatasetFlowDetailsHelpers { return { icon: "downloading", class: "text-muted" }; case "FlowEventScheduledForActivation": return { icon: "timer", class: "text-muted" }; + case "FlowConfigSnapshotModified": + return { icon: "downloading", class: "text-muted" }; case "FlowEventTaskChanged": { const event = flowEvent as FlowEventTaskChanged; switch (event.taskStatus) { @@ -108,6 +113,20 @@ export class DatasetFlowDetailsHelpers { case "FlowEventInitiated": case "FlowEventTriggerAdded": return this.describeTriggerDetails((flowEvent as FlowEventInitiated).trigger); + case "FlowConfigSnapshotModified": { + const event = flowEvent as FlowConfigSnapshotModified; + switch (event.configSnapshot.__typename) { + case "FlowConfigurationCompactionRule": + return "Modified by compaction rule"; + case "FlowConfigurationIngest": + return "Modified by ingest rule"; + case "FlowConfigurationReset": + return "Modified by reset rule"; + /* istanbul ignore next */ + default: + throw new Error("Unknown configSnapshot typename"); + } + } case "FlowEventAborted": return ""; case "FlowEventScheduledForActivation": { diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts index f2b6dd16b..37cb76618 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.ts @@ -6,8 +6,11 @@ import { DatasetFlowApi } from "src/app/api/dataset-flow.api"; import { DatasetFlowType, FlowConfigurationInput, + FlowTriggerInput, GetDatasetFlowConfigsQuery, + GetDatasetFlowTriggersQuery, SetDatasetFlowConfigMutation, + SetDatasetFlowTriggersMutation, } from "src/app/api/kamu.graphql.interface"; import AppValues from "src/app/common/app.values"; import { DatasetViewTypeEnum } from "src/app/dataset-view/dataset-view.interface"; @@ -45,6 +48,38 @@ export class DatasetSchedulingService { ); } + public fetchDatasetFlowTriggers( + datasetId: string, + datasetFlowType: DatasetFlowType, + ): Observable { + return this.datasetFlowApi.getDatasetFlowTriggers({ datasetId, datasetFlowType }); + } + + public setDatasetTriggers(params: { + datasetId: string; + datasetFlowType: DatasetFlowType; + paused: boolean; + triggerInput: FlowTriggerInput; + datasetInfo: DatasetInfo; + }): Observable { + return this.datasetFlowApi.setDatasetFlowTriggers(params).pipe( + map((data: SetDatasetFlowTriggersMutation) => { + const triggers = data.datasets.byId?.flows.triggers.setTrigger; + if (triggers?.__typename === "SetFlowTriggerSuccess") { + setTimeout(() => { + this.navigationService.navigateToDatasetView({ + accountName: params.datasetInfo.accountName, + datasetName: params.datasetInfo.datasetName, + tab: DatasetViewTypeEnum.Flows, + }); + }, AppValues.SIMULATION_START_CONDITION_DELAY_MS); + } else { + this.toastrService.error(triggers?.message); + } + }), + ); + } + // public setDatasetFlowSchedule(params: { // accountId: string; // datasetId: string; diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html index ff59918f9..a5534ea0b 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html @@ -5,7 +5,7 @@

Scheduled updates

-
+
Triggers
@@ -45,12 +45,12 @@

Scheduled updates

/>
- {{ pollingGroup.get("every")?.errors?.range.message }} + {{ pollingForm.get("every")?.errors?.range.message }}
@@ -71,9 +71,9 @@

Scheduled updates

@@ -85,7 +85,7 @@

Scheduled updates

- Next time: {{ nextTime }} @@ -107,7 +107,7 @@

Scheduled updates

@@ -141,60 +141,72 @@

Scheduled updates

-
-
- Max batching interval: - - - -
- {{ batchingForm.get("every")?.errors?.range.message }} -
- -
-
- Min records to await: - - -
- Minimum value 1 -
- +
+ Triggers +
+
+ + Enable/Disable update schedule + +
+
+ Max batching interval: + + + +
+ {{ batchingForm.get("every")?.errors?.range.message }} +
+ +
+
+ Min records to await: + + +
+ Minimum value 1 +
+ +
-
-
diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts index 3b66c03bc..bb68617c3 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts @@ -8,6 +8,9 @@ import { DatasetFlowType, DatasetKind, DatasetPermissionsFragment, + FlowTriggerInput, + GetDatasetFlowConfigsQuery, + GetDatasetFlowTriggersQuery, IngestConditionInput, TimeUnit, } from "src/app/api/kamu.graphql.interface"; @@ -17,7 +20,6 @@ import { cronExpressionNextTime, logError } from "src/app/common/app.helpers"; import { BatchingFormType, IngestConfigurationFormType, - PollingFormType, PollingGroupType, } from "./dataset-settings-scheduling-tab.component.types"; import { takeUntilDestroyed } from "@angular/core/rxjs-interop"; @@ -34,30 +36,28 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme public readonly pollingGroupEnum: typeof PollingGroupEnum = PollingGroupEnum; public readonly throttlingGroupEnum: typeof ThrottlingGroupEnum = ThrottlingGroupEnum; public readonly timeUnit: typeof TimeUnit = TimeUnit; - private scheduleOptions: IngestConditionInput; private everyTimeMapperValidators: Record = everyTimeMapperValidators; public ingestConfigurationForm = new FormGroup({ fetchUncacheable: new FormControl(false, { nonNullable: true }), }); - public pollingForm = new FormGroup({ - pollingGroup: new FormGroup({ - updatesState: new FormControl(false, { nonNullable: true }), - __typename: new FormControl(PollingGroupEnum.TIME_DELTA, [Validators.required]), - every: new FormControl>({ value: null, disabled: false }, [ - Validators.required, - Validators.min(1), - ]), - unit: new FormControl>({ value: null, disabled: false }, [Validators.required]), - cronExpression: new FormControl>({ value: "", disabled: true }, [ - Validators.required, - cronExpressionValidator(), - ]), - }), + public pollingForm = new FormGroup({ + updatesState: new FormControl(false, { nonNullable: true }), + __typename: new FormControl(PollingGroupEnum.TIME_DELTA, [Validators.required]), + every: new FormControl>({ value: null, disabled: false }, [ + Validators.required, + Validators.min(1), + ]), + unit: new FormControl>({ value: null, disabled: false }, [Validators.required]), + cronExpression: new FormControl>({ value: "", disabled: true }, [ + Validators.required, + cronExpressionValidator(), + ]), }); public batchingForm = new FormGroup({ + updatesState: new FormControl(false, { nonNullable: true }), every: new FormControl>({ value: null, disabled: false }, [ Validators.required, Validators.min(1), @@ -71,12 +71,8 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme private datasetSchedulingService = inject(DatasetSchedulingService); - public get pollingGroup(): FormGroup { - return this.pollingForm.get("pollingGroup") as FormGroup; - } - public get pollingType(): AbstractControl { - return this.pollingGroup.controls.__typename; + return this.pollingForm.controls.__typename; } public get batchingEveryTime(): AbstractControl { @@ -91,16 +87,20 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme return this.batchingForm.controls.minRecordsToAwait; } + public get batchingUpdateState(): AbstractControl { + return this.batchingForm.controls.updatesState; + } + public get pollingEveryTime(): AbstractControl { - return this.pollingGroup.controls.every; + return this.pollingForm.controls.every; } public get pollingUnitTime(): AbstractControl { - return this.pollingGroup.controls.unit; + return this.pollingForm.controls.unit; } public get cronExpression(): AbstractControl { - return this.pollingGroup.controls.cronExpression; + return this.pollingForm.controls.cronExpression; } public get nextTime(): string { @@ -115,6 +115,10 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme return this.ingestConfigurationForm.controls.fetchUncacheable; } + public get pollingUpdateState(): AbstractControl { + return this.pollingForm.controls.updatesState; + } + public ngOnInit() { if (!this.datasetPermissions.permissions.canSchedule) { this.pollingForm.disable(); @@ -187,104 +191,117 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme private checkStatusSection(): void { if (this.datasetBasics.kind === DatasetKind.Root) { - this.batchingForm.disable(); - this.pollingGroup.enable(); - this.cronExpression.disable(); + // this.cronExpression.disable(); + //Init configs this.datasetSchedulingService .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.Ingest) .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe((data) => { + .subscribe((data: GetDatasetFlowConfigsQuery) => { const flowConfiguration = data.datasets.byId?.flows.configs.byType?.ingest; this.ingestConfigurationForm.patchValue({ fetchUncacheable: flowConfiguration?.fetchUncacheable }); - console.log("==>", flowConfiguration); - // const paused = data.datasets.byId?.flows.configs.byType?.paused; - // if (flowConfiguration?.schedule) { - // this.pollingForm.patchValue({ updatesState: !paused }); - // this.pollingGroup.patchValue({ - // ...flowConfiguration.schedule, - // fetchUncacheable: flowConfiguration.fetchUncacheable, - // }); - // if (flowConfiguration.schedule.__typename === "Cron5ComponentExpression") { - // this.pollingGroup.patchValue({ - // // splice for sync with cron parser - // cronExpression: flowConfiguration.schedule.cron5ComponentExpression, - // }); - // } - // } + }); + //Init triggers + this.datasetSchedulingService + .fetchDatasetFlowTriggers(this.datasetBasics.id, DatasetFlowType.Ingest) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe((data: GetDatasetFlowTriggersQuery) => { + const flowTriggers = data.datasets.byId?.flows.triggers.byType; + const schedule = flowTriggers?.schedule; + if (schedule && schedule.__typename === PollingGroupEnum.TIME_DELTA) { + this.pollingForm.patchValue({ + updatesState: !flowTriggers.paused, + __typename: schedule?.__typename as PollingGroupEnum, + every: schedule.every, + unit: schedule.unit, + }); + } + if (schedule && schedule.__typename === PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION) { + this.pollingForm.patchValue({ + updatesState: !flowTriggers.paused, + __typename: schedule.__typename as PollingGroupEnum, + cronExpression: schedule.cron5ComponentExpression, + }); + } + }); + } else { + this.datasetSchedulingService + .fetchDatasetFlowTriggers(this.datasetBasics.id, DatasetFlowType.ExecuteTransform) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe((data: GetDatasetFlowTriggersQuery) => { + const flowTriggers = data.datasets.byId?.flows.triggers.byType; + const batching = flowTriggers?.batching; + if (batching) { + this.batchingForm.patchValue({ + ...batching.maxBatchingInterval, + minRecordsToAwait: batching.minRecordsToAwait, + updatesState: !flowTriggers.paused, + }); + } }); } - // else { - // this.pollingGroup.disable(); - // this.batchingForm.enable(); - // this.datasetSchedulingService - // .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.ExecuteTransform) - // .pipe(takeUntilDestroyed(this.destroyRef)) - // .subscribe((data) => { - // const flowConfiguration = data.datasets.byId?.flows.configs.byType; - // const paused = data.datasets.byId?.flows.configs.byType?.paused; - // if (flowConfiguration?.transform) { - // const batchingConfig = flowConfiguration.transform; - // this.pollingForm.patchValue({ updatesState: !paused }); - // this.batchingForm.patchValue({ - // ...batchingConfig.maxBatchingInterval, - // minRecordsToAwait: batchingConfig.minRecordsToAwait, - // }); - // } - // }); - // } } - public onSubmit(): void { - // if (this.datasetBasics.kind === DatasetKind.Root) { - // this.setScheduleOptions(); - // this.datasetSchedulingService - // .setDatasetFlowSchedule({ - // accountId: this.datasetBasics.owner.id, - // datasetId: this.datasetBasics.id, - // datasetFlowType: DatasetFlowType.Ingest, - // paused: !(this.updateState.value as boolean), - // ingest: this.scheduleOptions, - // datasetInfo: { - // accountName: this.datasetBasics.owner.accountName, - // datasetName: this.datasetBasics.name, - // }, - // }) - // .pipe(takeUntilDestroyed(this.destroyRef)) - // .subscribe(); - // } else { - // this.datasetSchedulingService - // .setDatasetFlowBatching({ - // accountId: this.datasetBasics.owner.id, - // datasetId: this.datasetBasics.id, - // datasetFlowType: DatasetFlowType.ExecuteTransform, - // paused: !(this.updateState.value as boolean), - // transform: { - // minRecordsToAwait: this.batchingMinRecordsToAwait.value as number, - // maxBatchingInterval: { - // every: this.batchingEveryTime.value as number, - // unit: this.batchingUnitTime.value as TimeUnit, - // }, - // }, - // datasetInfo: { - // accountName: this.datasetBasics.owner.accountName, - // datasetName: this.datasetBasics.name, - // }, - // }) - // .pipe(takeUntilDestroyed(this.destroyRef)) - // .subscribe(); - // } + public saveBatchingTriggers(): void { + this.datasetSchedulingService + .setDatasetTriggers({ + datasetId: this.datasetBasics.id, + datasetFlowType: DatasetFlowType.ExecuteTransform, + paused: !(this.batchingUpdateState.value as boolean), + triggerInput: this.setBatchingTriggerInput(), + datasetInfo: { + accountName: this.datasetBasics.owner.accountName, + datasetName: this.datasetBasics.name, + }, + }) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(); } - private setScheduleOptions(): void { - if (this.pollingGroup.controls.__typename.value === PollingGroupEnum.TIME_DELTA) { - this.scheduleOptions = { - fetchUncacheable: this.fetchUncacheable.value, + public savePollingTriggers(): void { + this.datasetSchedulingService + .setDatasetTriggers({ + datasetId: this.datasetBasics.id, + datasetFlowType: DatasetFlowType.Ingest, + paused: !(this.pollingUpdateState.value as boolean), + triggerInput: this.setPollingTriggerInput(), + datasetInfo: { + accountName: this.datasetBasics.owner.accountName, + datasetName: this.datasetBasics.name, + }, + }) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(); + } + + private setPollingTriggerInput(): FlowTriggerInput { + if (this.pollingForm.controls.__typename.value === PollingGroupEnum.TIME_DELTA) { + return { + schedule: { + timeDelta: { + every: this.pollingEveryTime.value as number, + unit: this.pollingUnitTime.value as TimeUnit, + }, + }, }; - } - if (this.pollingGroup.controls.__typename.value === PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION) { - this.scheduleOptions = { - fetchUncacheable: this.fetchUncacheable.value, + } else { + return { + schedule: { + // sync with server validator + cron5ComponentExpression: this.cronExpression.value as string, + }, }; } } + + private setBatchingTriggerInput(): FlowTriggerInput { + return { + batching: { + minRecordsToAwait: this.batchingMinRecordsToAwait.value as number, + maxBatchingInterval: { + every: this.batchingEveryTime.value as number, + unit: this.batchingUnitTime.value as TimeUnit, + }, + }, + }; + } } diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.types.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.types.ts index dfd21e7b9..629fd69f7 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.types.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.types.ts @@ -16,6 +16,7 @@ export interface PollingGroupType { } export interface BatchingFormType { + updatesState: FormControl; every: FormControl>; unit: FormControl>; minRecordsToAwait: FormControl>; From cf1ec5dd5a26aea7e4c2eb09f73eee324d91c085 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Tue, 17 Dec 2024 13:32:47 +0200 Subject: [PATCH 08/17] added disabled conditions --- .../dataset-settings-scheduling-tab.component.html | 6 ++++-- .../scheduling/dataset-settings-scheduling-tab.component.ts | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html index a5534ea0b..08b536878 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html @@ -14,7 +14,7 @@

Scheduled updates

class="mat-elevation-z0" formControlName="updatesState" > - Enable/Disable update schedule + Enable/Disable updates
Scheduled updates diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.spec.ts index 423e750da..21e2857ea 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.spec.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.spec.ts @@ -3,7 +3,7 @@ import { DatasetSettingsSchedulingTabComponent } from "./dataset-settings-schedu import { Apollo } from "apollo-angular"; import { ApolloTestingModule } from "apollo-angular/testing"; import { BrowserAnimationsModule } from "@angular/platform-browser/animations"; -import { ToastrModule } from "ngx-toastr"; +import { ToastrModule, ToastrService } from "ngx-toastr"; import { SharedTestModule } from "src/app/common/shared-test.module"; import { MatDividerModule } from "@angular/material/divider"; import { MatSlideToggleModule } from "@angular/material/slide-toggle"; @@ -26,12 +26,17 @@ import { PollingGroupEnum } from "../../dataset-settings.model"; import _ from "lodash"; import { TimeDelta, TimeUnit } from "src/app/api/kamu.graphql.interface"; import { of } from "rxjs"; -import { mockIngestGetDatasetFlowConfigsSuccess } from "src/app/api/mock/dataset-flow.mock"; +import { + mockGetDatasetFlowTriggersBatchingQuery, + mockGetDatasetFlowTriggersQuery, + mockIngestGetDatasetFlowConfigsSuccess, +} from "src/app/api/mock/dataset-flow.mock"; describe("DatasetSettingsSchedulingTabComponent", () => { let component: DatasetSettingsSchedulingTabComponent; let fixture: ComponentFixture; let datasetSchedulingService: DatasetSchedulingService; + let toastrService: ToastrService; const MOCK_PARAM_EVERY = 10; const MOCK_PARAM_UNIT = TimeUnit.Minutes; @@ -61,6 +66,8 @@ describe("DatasetSettingsSchedulingTabComponent", () => { fixture = TestBed.createComponent(DatasetSettingsSchedulingTabComponent); datasetSchedulingService = TestBed.inject(DatasetSchedulingService); + toastrService = TestBed.inject(ToastrService); + component = fixture.componentInstance; component.datasetBasics = mockDatasetBasicsRootFragment; }); @@ -71,6 +78,31 @@ describe("DatasetSettingsSchedulingTabComponent", () => { expect(component).toBeTruthy(); }); + it("should check initial state", () => { + component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); + spyOn(datasetSchedulingService, "fetchDatasetFlowConfigs").and.returnValue( + of(mockIngestGetDatasetFlowConfigsSuccess), + ); + spyOn(datasetSchedulingService, "fetchDatasetFlowTriggers").and.returnValue( + of(mockGetDatasetFlowTriggersQuery), + ); + fixture.detectChanges(); + const fetchUncacheableCheckBox = findElementByDataTestId(fixture, "fetchUncacheable") as HTMLInputElement; + expect(fetchUncacheableCheckBox.checked).toBeFalsy(); + }); + + it("should check initial state for derivative dataset", () => { + component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); + component.datasetBasics = mockDatasetBasicsDerivedFragment; + + spyOn(datasetSchedulingService, "fetchDatasetFlowTriggers").and.returnValue( + of(mockGetDatasetFlowTriggersBatchingQuery), + ); + fixture.detectChanges(); + const fetchUncacheableCheckBox = findElementByDataTestId(fixture, "batching-interval-unit") as HTMLInputElement; + expect(fetchUncacheableCheckBox.value).toEqual(TimeUnit.Hours); + }); + it("should check have permission to canSchedule", () => { component.datasetPermissions = _.cloneDeep(mockNotScheduleDatasetPermissionsFragment); fixture.detectChanges(); @@ -92,8 +124,8 @@ describe("DatasetSettingsSchedulingTabComponent", () => { expect(component.pollingType.value).toEqual(PollingGroupEnum.TIME_DELTA); }); - it("should check 'Save' button works for ROOT dataset", () => { - const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetFlowSchedule").and.callThrough(); + it("should check 'Save trigger' button works for ROOT dataset", () => { + const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetTriggers").and.callThrough(); component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); fixture.detectChanges(); setFieldValue(fixture, "polling-group-every", MOCK_PARAM_EVERY.toString()); @@ -101,11 +133,9 @@ describe("DatasetSettingsSchedulingTabComponent", () => { fixture.detectChanges(); emitClickOnElementByDataTestId(fixture, "save-config-options"); - expect(setDatasetFlowScheduleSpy).toHaveBeenCalledWith( jasmine.objectContaining({ - ingest: { - fetchUncacheable: false, + triggerInput: { schedule: { timeDelta: MOCK_INPUT_TIME_DELTA, }, @@ -114,8 +144,8 @@ describe("DatasetSettingsSchedulingTabComponent", () => { ); }); - it("should check 'Save' button works for ROOT dataset with time delta", () => { - const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetFlowSchedule").and.callThrough(); + it("should check 'Save trigger' button works for ROOT dataset with time delta", () => { + const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetTriggers").and.callThrough(); component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); fixture.detectChanges(); setFieldValue(fixture, "polling-group-every", MOCK_PARAM_EVERY.toString()); @@ -126,8 +156,7 @@ describe("DatasetSettingsSchedulingTabComponent", () => { expect(setDatasetFlowScheduleSpy).toHaveBeenCalledWith( jasmine.objectContaining({ - ingest: { - fetchUncacheable: false, + triggerInput: { schedule: { timeDelta: MOCK_INPUT_TIME_DELTA, }, @@ -136,8 +165,8 @@ describe("DatasetSettingsSchedulingTabComponent", () => { ); }); - it("should check 'Save' button works for ROOT dataset with cron expression", () => { - const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetFlowSchedule").and.callThrough(); + it("should check 'Save trigger' button works for ROOT dataset with cron expression", () => { + const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetTriggers").and.callThrough(); component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); fixture.detectChanges(); emitClickOnElementByDataTestId(fixture, "button-cron-expression"); @@ -148,8 +177,7 @@ describe("DatasetSettingsSchedulingTabComponent", () => { expect(setDatasetFlowScheduleSpy).toHaveBeenCalledWith( jasmine.objectContaining({ - ingest: { - fetchUncacheable: false, + triggerInput: { schedule: { cron5ComponentExpression: `${MOCK_CRON_EXPRESSION}`, }, @@ -158,8 +186,8 @@ describe("DatasetSettingsSchedulingTabComponent", () => { ); }); - it("should check 'Save' button works for DERIVATIVE dataset", () => { - const setDatasetFlowBatchingSpy = spyOn(datasetSchedulingService, "setDatasetFlowBatching").and.callThrough(); + it("should check 'Save triger' button works for DERIVATIVE dataset", () => { + const setDatasetFlowBatchingSpy = spyOn(datasetSchedulingService, "setDatasetTriggers").and.callThrough(); component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); component.datasetBasics = mockDatasetBasicsDerivedFragment; fixture.detectChanges(); @@ -168,15 +196,17 @@ describe("DatasetSettingsSchedulingTabComponent", () => { setFieldValue(fixture, "batching-min-records", MOCK_MIN_RECORDS_TO_AWAIT.toString()); fixture.detectChanges(); - emitClickOnElementByDataTestId(fixture, "save-config-options"); + emitClickOnElementByDataTestId(fixture, "save-batching-triggers"); expect(setDatasetFlowBatchingSpy).toHaveBeenCalledWith( jasmine.objectContaining({ - transform: { - minRecordsToAwait: MOCK_MIN_RECORDS_TO_AWAIT, - maxBatchingInterval: { - every: MOCK_PARAM_EVERY, - unit: MOCK_PARAM_UNIT, + triggerInput: { + batching: { + minRecordsToAwait: MOCK_MIN_RECORDS_TO_AWAIT, + maxBatchingInterval: { + every: MOCK_PARAM_EVERY, + unit: MOCK_PARAM_UNIT, + }, }, }, }), @@ -203,19 +233,19 @@ describe("DatasetSettingsSchedulingTabComponent", () => { expect(errorMessageElem?.textContent?.trim()).toEqual("Invalid expression"); }); - it("should check init form with schedule when configuration is exist", () => { + it("should check save ingest configuration", () => { component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); - const fetchDatasetFlowConfigsSpy = spyOn(datasetSchedulingService, "fetchDatasetFlowConfigs").and.returnValue( - of(mockIngestGetDatasetFlowConfigsSuccess), + const toastrServiceSpy = spyOn(toastrService, "success"); + const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetFlowConfigs").and.returnValue( + of(true), ); - component.ngOnInit(); - - expect(fetchDatasetFlowConfigsSpy).toHaveBeenCalledTimes(1); - expect(component.pollingGroup.value).toEqual({ - __typename: "TimeDelta", - every: 3, - unit: TimeUnit.Hours, - fetchUncacheable: false, - }); + fixture.detectChanges(); + emitClickOnElementByDataTestId(fixture, "fetchUncacheable"); + fixture.detectChanges(); + + emitClickOnElementByDataTestId(fixture, "save-polling-configuration"); + + expect(setDatasetFlowScheduleSpy).toHaveBeenCalledTimes(1); + expect(toastrServiceSpy).toHaveBeenCalledTimes(1); }); }); diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts index d4a119a34..1753e63b7 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts @@ -165,6 +165,7 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme this.disableAndClearControl(this.pollingUnitTime); break; } + /* istanbul ignore next */ default: { logError("Unknown PollingGroupEnum key"); } diff --git a/src/app/dataset-view/additional-components/flows-component/services/dataset-flows.service.spec.ts b/src/app/dataset-view/additional-components/flows-component/services/dataset-flows.service.spec.ts index 9a1553383..a1fa545a7 100644 --- a/src/app/dataset-view/additional-components/flows-component/services/dataset-flows.service.spec.ts +++ b/src/app/dataset-view/additional-components/flows-component/services/dataset-flows.service.spec.ts @@ -177,17 +177,7 @@ describe("DatasetFlowsService", () => { spyOn(datasetFlowApi, "allFlowsPaused").and.returnValue(of(mockDatasetAllFlowsPausedQuery)); const subscription$ = service.allFlowsPaused(MOCK_DATASET_ID).subscribe((result) => { - expect(result).toEqual(mockDatasetAllFlowsPausedQuery.datasets.byId?.flows.configs.allPaused); - }); - - expect(subscription$.closed).toBeTrue(); - }); - - it("should check get flow by id", () => { - spyOn(datasetFlowApi, "allFlowsPaused").and.returnValue(of(mockDatasetAllFlowsPausedQuery)); - - const subscription$ = service.allFlowsPaused(MOCK_DATASET_ID).subscribe((result) => { - expect(result).toEqual(mockDatasetAllFlowsPausedQuery.datasets.byId?.flows.configs.allPaused); + expect(result).toEqual(mockDatasetAllFlowsPausedQuery.datasets.byId?.flows.triggers.allPaused); }); expect(subscription$.closed).toBeTrue(); From e655d8ea8c18aadab4c45e426d9dd714d6a1d3b2 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Wed, 18 Dec 2024 10:09:41 +0200 Subject: [PATCH 12/17] changed CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fff0cb875..11f4c2896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `demo` mode: unimplemented features are disabled - `develop` mode: unimplemented features are available - Made a tile element clickable +- Made `force update` link for a flow table +### Changed +- Flow configuration separation ## [0.33.0] - 2024-12-03 ### Added From 4a1a1760ff11bf4378d468f9bc7a30d01adcdc58 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Wed, 18 Dec 2024 15:51:51 +0200 Subject: [PATCH 13/17] added separated components for trigers and configuration --- .../directives/feature-flag.directive.ts | 2 - .../flow-details-history-tab.helpers.ts | 2 +- .../dataset-scheduling.service.spec.ts | 21 -- .../batching-trigger-form.component.html | 74 ++++++ .../batching-trigger-form.component.scss | 18 ++ .../batching-trigger-form.component.spec.ts | 21 ++ .../batching-trigger-form.component.ts | 92 +++++++ .../batching-trigger.module.ts | 12 + ...set-settings-scheduling-tab.component.html | 224 ++--------------- ...set-settings-scheduling-tab.component.scss | 28 --- ...taset-settings-scheduling-tab.component.ts | 229 ++---------------- .../ingest-configuration-form.component.html | 26 ++ ...ngest-configuration-form.component.spec.ts | 21 ++ .../ingest-configuration-form.component.ts | 37 +++ .../ingest-configuration.module.ts | 11 + .../ingest-trigger-form.component.html | 106 ++++++++ .../ingest-trigger-form.component.scss | 33 +++ .../ingest-trigger-form.component.spec.ts | 21 ++ .../ingest-trigger-form.component.ts | 140 +++++++++++ .../ingest-trigger.module.ts | 14 ++ src/app/dataset-view/dataset.module.ts | 7 +- 21 files changed, 667 insertions(+), 472 deletions(-) create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.html create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.scss create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.spec.ts create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.ts create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger.module.ts create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.html create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.spec.ts create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.ts create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration.module.ts create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.html create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.scss create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.spec.ts create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.ts create mode 100644 src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger.module.ts diff --git a/src/app/common/directives/feature-flag.directive.ts b/src/app/common/directives/feature-flag.directive.ts index 52684cef4..73271c62c 100644 --- a/src/app/common/directives/feature-flag.directive.ts +++ b/src/app/common/directives/feature-flag.directive.ts @@ -15,7 +15,6 @@ export class FeatureFlagDirective { @Input("appFeatureFlag") public featureName: string; - /* istanbul ignore next */ public ngOnInit(): void { if (!this.el.nativeElement) { return; @@ -38,7 +37,6 @@ export class FeatureFlagDirective { } } - /* istanbul ignore next */ private deduceFeatureVisibility(feature: Feature): FeatureVisibility { const showMode = this.featureFlagsService.getEffectiveFeatureShowMode(); switch (showMode) { diff --git a/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts b/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts index e78d0e341..0ab911498 100644 --- a/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts +++ b/src/app/dataset-flow/dataset-flow-details/tabs/flow-details-history-tab/flow-details-history-tab.helpers.ts @@ -68,7 +68,7 @@ export class DatasetFlowDetailsHelpers { case "FlowEventScheduledForActivation": return { icon: "timer", class: "text-muted" }; case "FlowConfigSnapshotModified": - return { icon: "downloading", class: "text-muted" }; + return { icon: "outbound", class: "text-muted" }; case "FlowEventTaskChanged": { const event = flowEvent as FlowEventTaskChanged; switch (event.taskStatus) { diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.spec.ts index 3c1aaf53e..7152ea019 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.spec.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/services/dataset-scheduling.service.spec.ts @@ -171,25 +171,4 @@ describe("DatasetSchedulingService", () => { expect(subscription$.closed).toBeTrue(); })); - - // it("should check set dataset flow batching with error", () => { - // const errorMessage = mockSetDatasetFlowBatchingError.datasets.byId?.flows.configs.setConfigTransform.message; - // spyOn(datasetFlowApi, "setDatasetFlowBatching").and.returnValue(of(mockSetDatasetFlowBatchingError)); - // const toastrServiceErrorSpy = spyOn(toastService, "error"); - - // const subscription$ = service - // .setDatasetFlowBatching({ - // accountId: TEST_ACCOUNT_ID, - // datasetId: MOCK_DATASET_ID, - // datasetFlowType: DatasetFlowType.ExecuteTransform, - // paused: false, - // transform: MOCK_BATCHING_CONFIG, - // datasetInfo: mockDatasetInfo, - // }) - // .subscribe(() => { - // expect(toastrServiceErrorSpy).toHaveBeenCalledWith(errorMessage); - // }); - - // expect(subscription$.closed).toBeTrue(); - // }); }); diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.html new file mode 100644 index 000000000..2e6881842 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.html @@ -0,0 +1,74 @@ + +
+
+ Triggers +
+
+ + Enable/Disable updates + +
+
+ Max batching interval: + + + +
+ {{ batchingForm.get("every")?.errors?.range.message }} +
+ +
+
+ Min records to await: + + +
+ Minimum value 1 +
+ +
+
+
+
+ +
+
+ diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.scss b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.scss new file mode 100644 index 000000000..1f9d41661 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.scss @@ -0,0 +1,18 @@ +h4 { + font-weight: 500; + letter-spacing: normal; +} + +.form-control-width { + width: 132px; +} + +.form-control-label { + min-width: 140px; +} + +@media (width <=1085px) { + .form-control-width { + width: 80px; + } +} diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.spec.ts new file mode 100644 index 000000000..5f6405a44 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.spec.ts @@ -0,0 +1,21 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { BatchingTriggerFormComponent } from './batching-trigger-form.component'; + +describe('BatchingTriggerFormComponent', () => { + let component: BatchingTriggerFormComponent; + let fixture: ComponentFixture; + + beforeEach(() => { + TestBed.configureTestingModule({ + declarations: [BatchingTriggerFormComponent] + }); + fixture = TestBed.createComponent(BatchingTriggerFormComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.ts new file mode 100644 index 000000000..3b2530db3 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.ts @@ -0,0 +1,92 @@ +import { ChangeDetectionStrategy, Component, EventEmitter, inject, Input, OnInit, Output } from "@angular/core"; +import { + DatasetBasicsFragment, + DatasetFlowType, + GetDatasetFlowTriggersQuery, + TimeUnit, +} from "src/app/api/kamu.graphql.interface"; +import { BaseComponent } from "src/app/common/base.component"; +import { DatasetSchedulingService } from "../../../services/dataset-scheduling.service"; +import { FormGroup, FormControl, Validators, AbstractControl, ValidatorFn } from "@angular/forms"; +import { MaybeNull } from "src/app/common/app.types"; +import { BatchingFormType } from "../dataset-settings-scheduling-tab.component.types"; +import { takeUntilDestroyed } from "@angular/core/rxjs-interop"; +import { everyTimeMapperValidators } from "src/app/common/data.helpers"; + +@Component({ + selector: "app-batching-trigger-form", + templateUrl: "./batching-trigger-form.component.html", + styleUrls: ["./batching-trigger-form.component.scss"], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class BatchingTriggerFormComponent extends BaseComponent implements OnInit { + @Input({ required: true }) public datasetBasics: DatasetBasicsFragment; + @Output() public saveTriggerEmit = new EventEmitter>(); + public readonly timeUnit: typeof TimeUnit = TimeUnit; + private everyTimeMapperValidators: Record = everyTimeMapperValidators; + + public batchingForm = new FormGroup({ + updatesState: new FormControl(false, { nonNullable: true }), + every: new FormControl>({ value: null, disabled: false }, [ + Validators.required, + Validators.min(1), + ]), + unit: new FormControl>({ value: null, disabled: false }, [Validators.required]), + minRecordsToAwait: new FormControl>({ value: null, disabled: false }, [ + Validators.required, + Validators.min(1), + ]), + }); + + private datasetSchedulingService = inject(DatasetSchedulingService); + + public get batchingEveryTime(): AbstractControl { + return this.batchingForm.controls.every; + } + + public get batchingUnitTime(): AbstractControl { + return this.batchingForm.controls.unit; + } + + public get batchingMinRecordsToAwait(): AbstractControl { + return this.batchingForm.controls.minRecordsToAwait; + } + + public get batchingUpdateState(): AbstractControl { + return this.batchingForm.controls.updatesState; + } + + public saveBatchingTriggers(): void { + this.saveTriggerEmit.emit(this.batchingForm); + } + + private setBatchingEveryTimeValidator(): void { + this.batchingUnitTime.valueChanges.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((data: TimeUnit) => { + if (data) { + this.batchingEveryTime.setValidators([this.everyTimeMapperValidators[data], Validators.required]); + } + }); + } + + public ngOnInit(): void { + this.setBatchingEveryTimeValidator(); + this.initBatchingForm(); + } + + public initBatchingForm(): void { + this.datasetSchedulingService + .fetchDatasetFlowTriggers(this.datasetBasics.id, DatasetFlowType.ExecuteTransform) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe((data: GetDatasetFlowTriggersQuery) => { + const flowTriggers = data.datasets.byId?.flows.triggers.byType; + const batching = flowTriggers?.batching; + if (batching) { + this.batchingForm.patchValue({ + ...batching.maxBatchingInterval, + minRecordsToAwait: batching.minRecordsToAwait, + updatesState: !flowTriggers.paused, + }); + } + }); + } +} diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger.module.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger.module.ts new file mode 100644 index 000000000..9750f0078 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger.module.ts @@ -0,0 +1,12 @@ +import { MatSlideToggleModule } from "@angular/material/slide-toggle"; +import { NgModule } from "@angular/core"; +import { CommonModule } from "@angular/common"; +import { BatchingTriggerFormComponent } from "./batching-trigger-form.component"; +import { FormsModule, ReactiveFormsModule } from "@angular/forms"; + +@NgModule({ + declarations: [BatchingTriggerFormComponent], + exports: [BatchingTriggerFormComponent], + imports: [CommonModule, FormsModule, ReactiveFormsModule, MatSlideToggleModule], +}) +export class BatchingTriggerModule {} diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html index 275fe3fa4..ff5b87da8 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.html @@ -3,220 +3,24 @@

Scheduled updates

-
-
-
- Triggers -
-
- - Enable/Disable updates - -
- - -
- Launch every: - - - -
- {{ pollingForm.get("every")?.errors?.range.message }} -
- -
- - -
-
- Cron expression : -
- - - -
- Invalid expression -
- - - Next time: {{ nextTime }} - -
- - -
- Cron expression accepted values -
1. minutes: 0-59 * , -
-
2. hours: 0-23 * , -
-
3. day of month: 1-31 * , - ?
-
4. months: (JAN-DEC or 1-12) * , -
-
5. day of week: (SUN-SAT or 1-7) * , - ?
-
-
-
- -
- -
-
+
+
-
-
- Configuration -
- - Fetch uncacheable -
-
-
- -
-
+
+
-
-
-
- Triggers -
-
- - Enable/Disable updates - -
-
- Max batching interval: - - - -
- {{ batchingForm.get("every")?.errors?.range.message }} -
- -
-
- Min records to await: - - -
- Minimum value 1 -
- -
-
-
-
- -
-
-
+
+ +
diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.scss b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.scss index 90519ab2b..bf5c15641 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.scss +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.scss @@ -25,34 +25,6 @@ .custom-container { padding-bottom: 100px; - - h4 { - font-weight: 500; - letter-spacing: normal; - } - - .form-control-width { - width: 132px; - } - - .form-control-expression { - width: 132px; - } - - .form-control-label { - min-width: 140px; - } - - .cron-expression-hint { - padding-left: 150px; - } - - .checkbox-uncacheable { - width: 16px; - height: 16px; - accent-color: rgba(63 81 181); - margin-left: 0.85rem; - } } @media (width <=1085px) { diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts index 1753e63b7..7f70b1f8d 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.ts @@ -1,6 +1,5 @@ -import { MaybeNull } from "../../../../../common/app.types"; -import { ChangeDetectionStrategy, Component, inject, Input, OnInit } from "@angular/core"; -import { AbstractControl, FormControl, FormGroup, ValidatorFn, Validators } from "@angular/forms"; +import { ChangeDetectionStrategy, Component, inject, Input } from "@angular/core"; +import { FormGroup } from "@angular/forms"; import { BaseComponent } from "../../../../../common/base.component"; import { PollingGroupEnum, ThrottlingGroupEnum } from "../../dataset-settings.model"; import { @@ -9,13 +8,9 @@ import { DatasetKind, DatasetPermissionsFragment, FlowTriggerInput, - GetDatasetFlowConfigsQuery, - GetDatasetFlowTriggersQuery, TimeUnit, } from "src/app/api/kamu.graphql.interface"; import { DatasetSchedulingService } from "../../services/dataset-scheduling.service"; -import { cronExpressionValidator, everyTimeMapperValidators } from "src/app/common/data.helpers"; -import { cronExpressionNextTime, logError } from "src/app/common/app.helpers"; import { BatchingFormType, IngestConfigurationFormType, @@ -30,115 +25,27 @@ import { ToastrService } from "ngx-toastr"; styleUrls: ["./dataset-settings-scheduling-tab.component.scss"], changeDetection: ChangeDetectionStrategy.OnPush, }) -export class DatasetSettingsSchedulingTabComponent extends BaseComponent implements OnInit { +export class DatasetSettingsSchedulingTabComponent extends BaseComponent { @Input({ required: true }) public datasetBasics: DatasetBasicsFragment; @Input({ required: true }) public datasetPermissions: DatasetPermissionsFragment; - public readonly pollingGroupEnum: typeof PollingGroupEnum = PollingGroupEnum; public readonly throttlingGroupEnum: typeof ThrottlingGroupEnum = ThrottlingGroupEnum; public readonly timeUnit: typeof TimeUnit = TimeUnit; - private everyTimeMapperValidators: Record = everyTimeMapperValidators; - - public ingestConfigurationForm = new FormGroup({ - fetchUncacheable: new FormControl(false, { nonNullable: true }), - }); - - public pollingForm = new FormGroup({ - updatesState: new FormControl(false, { nonNullable: true }), - __typename: new FormControl(PollingGroupEnum.TIME_DELTA, [Validators.required]), - every: new FormControl>({ value: null, disabled: false }, [ - Validators.required, - Validators.min(1), - ]), - unit: new FormControl>({ value: null, disabled: false }, [Validators.required]), - cronExpression: new FormControl>({ value: "", disabled: true }, [ - Validators.required, - cronExpressionValidator(), - ]), - }); - - public batchingForm = new FormGroup({ - updatesState: new FormControl(false, { nonNullable: true }), - every: new FormControl>({ value: null, disabled: false }, [ - Validators.required, - Validators.min(1), - ]), - unit: new FormControl>({ value: null, disabled: false }, [Validators.required]), - minRecordsToAwait: new FormControl>({ value: null, disabled: false }, [ - Validators.required, - Validators.min(1), - ]), - }); private datasetSchedulingService = inject(DatasetSchedulingService); private toastrService = inject(ToastrService); - public get pollingType(): AbstractControl { - return this.pollingForm.controls.__typename; - } - - public get batchingEveryTime(): AbstractControl { - return this.batchingForm.controls.every; - } - - public get batchingUnitTime(): AbstractControl { - return this.batchingForm.controls.unit; - } - - public get batchingMinRecordsToAwait(): AbstractControl { - return this.batchingForm.controls.minRecordsToAwait; - } - - public get batchingUpdateState(): AbstractControl { - return this.batchingForm.controls.updatesState; - } - - public get pollingEveryTime(): AbstractControl { - return this.pollingForm.controls.every; - } - - public get pollingUnitTime(): AbstractControl { - return this.pollingForm.controls.unit; - } - - public get cronExpression(): AbstractControl { - return this.pollingForm.controls.cronExpression; - } - - public get nextTime(): string { - return cronExpressionNextTime(this.cronExpression.value as string); - } - public get isRootDataset(): boolean { return this.datasetBasics.kind === DatasetKind.Root; } - public get fetchUncacheable(): FormControl { - return this.ingestConfigurationForm.controls.fetchUncacheable; - } - - public get pollingUpdateState(): AbstractControl { - return this.pollingForm.controls.updatesState; - } - - public ngOnInit() { - if (!this.datasetPermissions.permissions.canSchedule) { - this.pollingForm.disable(); - } else { - this.checkStatusSection(); - this.pollingTypeChanges(); - this.setPollingEveryTimeValidator(); - this.setBatchingEveryTimeValidator(); - } - } - - public saveIngestConfiguration(): void { + public saveIngestConfiguration(ingestConfigurationForm: FormGroup): void { this.datasetSchedulingService .setDatasetFlowConfigs({ datasetId: this.datasetBasics.id, datasetFlowType: DatasetFlowType.Ingest, configInput: { ingest: { - fetchUncacheable: this.fetchUncacheable.value, + fetchUncacheable: ingestConfigurationForm.controls.fetchUncacheable.value, }, }, }) @@ -150,109 +57,13 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme }); } - private pollingTypeChanges(): void { - this.pollingType.valueChanges.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((value: PollingGroupEnum) => { - switch (value) { - case PollingGroupEnum.TIME_DELTA: { - this.pollingEveryTime.enable(); - this.pollingUnitTime.enable(); - this.disableAndClearControl(this.cronExpression); - break; - } - case PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION: { - this.cronExpression.enable(); - this.disableAndClearControl(this.pollingEveryTime); - this.disableAndClearControl(this.pollingUnitTime); - break; - } - /* istanbul ignore next */ - default: { - logError("Unknown PollingGroupEnum key"); - } - } - }); - } - - private setPollingEveryTimeValidator(): void { - this.pollingUnitTime.valueChanges.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((data: TimeUnit) => { - if (data) { - this.pollingEveryTime.setValidators([this.everyTimeMapperValidators[data], Validators.required]); - } - }); - } - - private setBatchingEveryTimeValidator(): void { - this.batchingUnitTime.valueChanges.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((data: TimeUnit) => { - if (data) { - this.batchingEveryTime.setValidators([this.everyTimeMapperValidators[data], Validators.required]); - } - }); - } - - private disableAndClearControl(control: AbstractControl): void { - control.disable(); - control.markAsUntouched(); - control.markAsPristine(); - } - - private checkStatusSection(): void { - if (this.datasetBasics.kind === DatasetKind.Root) { - //Init configs - this.datasetSchedulingService - .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.Ingest) - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe((data: GetDatasetFlowConfigsQuery) => { - const flowConfiguration = data.datasets.byId?.flows.configs.byType?.ingest; - this.ingestConfigurationForm.patchValue({ fetchUncacheable: flowConfiguration?.fetchUncacheable }); - }); - //Init triggers - this.datasetSchedulingService - .fetchDatasetFlowTriggers(this.datasetBasics.id, DatasetFlowType.Ingest) - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe((data: GetDatasetFlowTriggersQuery) => { - const flowTriggers = data.datasets.byId?.flows.triggers.byType; - const schedule = flowTriggers?.schedule; - if (schedule && schedule.__typename === PollingGroupEnum.TIME_DELTA) { - this.pollingForm.patchValue({ - updatesState: !flowTriggers.paused, - __typename: schedule?.__typename as PollingGroupEnum, - every: schedule.every, - unit: schedule.unit, - }); - } - if (schedule && schedule.__typename === PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION) { - this.pollingForm.patchValue({ - updatesState: !flowTriggers.paused, - __typename: schedule.__typename as PollingGroupEnum, - cronExpression: schedule.cron5ComponentExpression, - }); - } - }); - } else { - this.datasetSchedulingService - .fetchDatasetFlowTriggers(this.datasetBasics.id, DatasetFlowType.ExecuteTransform) - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe((data: GetDatasetFlowTriggersQuery) => { - const flowTriggers = data.datasets.byId?.flows.triggers.byType; - const batching = flowTriggers?.batching; - if (batching) { - this.batchingForm.patchValue({ - ...batching.maxBatchingInterval, - minRecordsToAwait: batching.minRecordsToAwait, - updatesState: !flowTriggers.paused, - }); - } - }); - } - } - - public saveBatchingTriggers(): void { + public saveBatchingTriggers(batchingTriggerForm: FormGroup): void { this.datasetSchedulingService .setDatasetTriggers({ datasetId: this.datasetBasics.id, datasetFlowType: DatasetFlowType.ExecuteTransform, - paused: !(this.batchingUpdateState.value as boolean), - triggerInput: this.setBatchingTriggerInput(), + paused: !(batchingTriggerForm.controls.updatesState.value as boolean), + triggerInput: this.setBatchingTriggerInput(batchingTriggerForm), datasetInfo: { accountName: this.datasetBasics.owner.accountName, datasetName: this.datasetBasics.name, @@ -262,13 +73,13 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme .subscribe(); } - public savePollingTriggers(): void { + public savePollingTriggers(pollingForm: FormGroup): void { this.datasetSchedulingService .setDatasetTriggers({ datasetId: this.datasetBasics.id, datasetFlowType: DatasetFlowType.Ingest, - paused: !(this.pollingUpdateState.value as boolean), - triggerInput: this.setPollingTriggerInput(), + paused: !(pollingForm.controls.updatesState.value as boolean), + triggerInput: this.setPollingTriggerInput(pollingForm), datasetInfo: { accountName: this.datasetBasics.owner.accountName, datasetName: this.datasetBasics.name, @@ -278,13 +89,13 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme .subscribe(); } - private setPollingTriggerInput(): FlowTriggerInput { - if (this.pollingForm.controls.__typename.value === PollingGroupEnum.TIME_DELTA) { + private setPollingTriggerInput(pollingForm: FormGroup): FlowTriggerInput { + if (pollingForm.controls.__typename.value === PollingGroupEnum.TIME_DELTA) { return { schedule: { timeDelta: { - every: this.pollingEveryTime.value as number, - unit: this.pollingUnitTime.value as TimeUnit, + every: pollingForm.controls.every.value as number, + unit: pollingForm.controls.unit.value as TimeUnit, }, }, }; @@ -292,19 +103,19 @@ export class DatasetSettingsSchedulingTabComponent extends BaseComponent impleme return { schedule: { // sync with server validator - cron5ComponentExpression: this.cronExpression.value as string, + cron5ComponentExpression: pollingForm.controls.cronExpression.value as string, }, }; } } - private setBatchingTriggerInput(): FlowTriggerInput { + private setBatchingTriggerInput(batchingTriggerForm: FormGroup): FlowTriggerInput { return { batching: { - minRecordsToAwait: this.batchingMinRecordsToAwait.value as number, + minRecordsToAwait: batchingTriggerForm.controls.minRecordsToAwait.value as number, maxBatchingInterval: { - every: this.batchingEveryTime.value as number, - unit: this.batchingUnitTime.value as TimeUnit, + every: batchingTriggerForm.controls.every.value as number, + unit: batchingTriggerForm.controls.unit.value as TimeUnit, }, }, }; diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.html new file mode 100644 index 000000000..f755926a3 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.html @@ -0,0 +1,26 @@ +
+
+ Configuration + +
+ + Fetch uncacheable +
+
+
+ +
+
+
diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.spec.ts new file mode 100644 index 000000000..cc2eebd4e --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.spec.ts @@ -0,0 +1,21 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { IngestConfigurationFormComponent } from './ingest-configuration-form.component'; + +describe('IngestConfigurationFormComponent', () => { + let component: IngestConfigurationFormComponent; + let fixture: ComponentFixture; + + beforeEach(() => { + TestBed.configureTestingModule({ + declarations: [IngestConfigurationFormComponent] + }); + fixture = TestBed.createComponent(IngestConfigurationFormComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.ts new file mode 100644 index 000000000..bfdf55cfc --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.ts @@ -0,0 +1,37 @@ +import { ChangeDetectionStrategy, Component, EventEmitter, inject, Input, OnInit, Output } from "@angular/core"; +import { FormControl, FormGroup } from "@angular/forms"; +import { IngestConfigurationFormType } from "../dataset-settings-scheduling-tab.component.types"; +import { BaseComponent } from "src/app/common/base.component"; +import { takeUntilDestroyed } from "@angular/core/rxjs-interop"; +import { DatasetBasicsFragment, DatasetFlowType, GetDatasetFlowConfigsQuery } from "src/app/api/kamu.graphql.interface"; +import { DatasetSchedulingService } from "../../../services/dataset-scheduling.service"; + +@Component({ + selector: "app-ingest-configuration-form", + templateUrl: "./ingest-configuration-form.component.html", + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class IngestConfigurationFormComponent extends BaseComponent implements OnInit { + @Input({ required: true }) public datasetBasics: DatasetBasicsFragment; + @Output() public saveConfigurationEmit = new EventEmitter>(); + + private datasetSchedulingService = inject(DatasetSchedulingService); + + public ingestConfigurationForm = new FormGroup({ + fetchUncacheable: new FormControl(false, { nonNullable: true }), + }); + + public ngOnInit(): void { + this.datasetSchedulingService + .fetchDatasetFlowConfigs(this.datasetBasics.id, DatasetFlowType.Ingest) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe((data: GetDatasetFlowConfigsQuery) => { + const flowConfiguration = data.datasets.byId?.flows.configs.byType?.ingest; + this.ingestConfigurationForm.patchValue({ fetchUncacheable: flowConfiguration?.fetchUncacheable }); + }); + } + + public saveIngestConfiguration(): void { + this.saveConfigurationEmit.emit(this.ingestConfigurationForm); + } +} diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration.module.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration.module.ts new file mode 100644 index 000000000..f3afe8a7f --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration.module.ts @@ -0,0 +1,11 @@ +import { NgModule } from "@angular/core"; +import { CommonModule } from "@angular/common"; +import { IngestConfigurationFormComponent } from "./ingest-configuration-form.component"; +import { FormsModule, ReactiveFormsModule } from "@angular/forms"; + +@NgModule({ + declarations: [IngestConfigurationFormComponent], + exports: [IngestConfigurationFormComponent], + imports: [CommonModule, FormsModule, ReactiveFormsModule], +}) +export class IngestConfigurationModule {} diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.html new file mode 100644 index 000000000..0c581ad5e --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.html @@ -0,0 +1,106 @@ +
+
+
+ Triggers +
+
+ + {{ updateStateToggleLabel }} + +
+ + +
+ Launch every: + + + +
+ {{ pollingForm.get("every")?.errors?.range.message }} +
+ +
+ + +
+
+ Cron expression : +
+ + + +
+ Invalid expression +
+ + + Next time: {{ nextTime }} + +
+ + +
+ Cron expression accepted values +
1. minutes: 0-59 * , -
+
2. hours: 0-23 * , -
+
3. day of month: 1-31 * , - ?
+
4. months: (JAN-DEC or 1-12) * , -
+
5. day of week: (SUN-SAT or 1-7) * , - ?
+
+
+
+ +
+ +
+
+
diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.scss b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.scss new file mode 100644 index 000000000..7f6259b42 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.scss @@ -0,0 +1,33 @@ +h4 { + font-weight: 500; + letter-spacing: normal; +} + +.form-control-width { + width: 132px; +} + +.form-control-expression { + width: 132px; +} + +.form-control-label { + min-width: 140px; +} + +.cron-expression-hint { + padding-left: 150px; +} + +.checkbox-uncacheable { + width: 16px; + height: 16px; + accent-color: rgba(63 81 181); + margin-left: 0.85rem; +} + +@media (width <=1085px) { + .form-control-width { + width: 80px; + } +} diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.spec.ts new file mode 100644 index 000000000..bec3bbcc8 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.spec.ts @@ -0,0 +1,21 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { IngestTriggerFormComponent } from './ingest-trigger-form.component'; + +describe('IngestTriggerFormComponent', () => { + let component: IngestTriggerFormComponent; + let fixture: ComponentFixture; + + beforeEach(() => { + TestBed.configureTestingModule({ + declarations: [IngestTriggerFormComponent] + }); + fixture = TestBed.createComponent(IngestTriggerFormComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.ts new file mode 100644 index 000000000..765472340 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.ts @@ -0,0 +1,140 @@ +import { ChangeDetectionStrategy, Component, EventEmitter, inject, Input, OnInit, Output } from "@angular/core"; +import { AbstractControl, FormControl, FormGroup, ValidatorFn, Validators } from "@angular/forms"; +import { + DatasetBasicsFragment, + DatasetFlowType, + GetDatasetFlowTriggersQuery, + TimeUnit, +} from "src/app/api/kamu.graphql.interface"; +import { PollingGroupType } from "../dataset-settings-scheduling-tab.component.types"; +import { PollingGroupEnum } from "../../../dataset-settings.model"; +import { cronExpressionNextTime, logError } from "src/app/common/app.helpers"; +import { BaseComponent } from "src/app/common/base.component"; +import { takeUntilDestroyed } from "@angular/core/rxjs-interop"; +import { cronExpressionValidator, everyTimeMapperValidators } from "src/app/common/data.helpers"; +import { MaybeNull } from "src/app/common/app.types"; +import { DatasetSchedulingService } from "../../../services/dataset-scheduling.service"; + +@Component({ + selector: "app-ingest-trigger-form", + templateUrl: "./ingest-trigger-form.component.html", + styleUrls: ["./ingest-trigger-form.component.scss"], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class IngestTriggerFormComponent extends BaseComponent implements OnInit { + @Input({ required: true }) public datasetBasics: DatasetBasicsFragment; + @Output() public saveTriggerEmit = new EventEmitter>(); + @Input({ required: true }) public updateStateToggleLabel: string; + + public readonly timeUnit: typeof TimeUnit = TimeUnit; + public readonly pollingGroupEnum: typeof PollingGroupEnum = PollingGroupEnum; + private everyTimeMapperValidators: Record = everyTimeMapperValidators; + + private datasetSchedulingService = inject(DatasetSchedulingService); + + public pollingForm = new FormGroup({ + updatesState: new FormControl(false, { nonNullable: true }), + __typename: new FormControl(PollingGroupEnum.TIME_DELTA, [Validators.required]), + every: new FormControl>({ value: null, disabled: false }, [ + Validators.required, + Validators.min(1), + ]), + unit: new FormControl>({ value: null, disabled: false }, [Validators.required]), + cronExpression: new FormControl>({ value: "", disabled: true }, [ + Validators.required, + cronExpressionValidator(), + ]), + }); + + ngOnInit(): void { + this.setPollingEveryTimeValidator(); + this.initPollingForm(); + this.pollingTypeChanges(); + } + + public savePollingTriggers(): void { + this.saveTriggerEmit.emit(this.pollingForm); + } + + public initPollingForm(): void { + this.datasetSchedulingService + .fetchDatasetFlowTriggers(this.datasetBasics.id, DatasetFlowType.Ingest) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe((data: GetDatasetFlowTriggersQuery) => { + const flowTriggers = data.datasets.byId?.flows.triggers.byType; + const schedule = flowTriggers?.schedule; + if (schedule && schedule.__typename === PollingGroupEnum.TIME_DELTA) { + this.pollingForm.patchValue({ + updatesState: !flowTriggers.paused, + __typename: schedule?.__typename as PollingGroupEnum, + every: schedule.every, + unit: schedule.unit, + }); + } + if (schedule && schedule.__typename === PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION) { + this.pollingForm.patchValue({ + updatesState: !flowTriggers.paused, + __typename: schedule.__typename as PollingGroupEnum, + cronExpression: schedule.cron5ComponentExpression, + }); + } + }); + } + + public get nextTime(): string { + return cronExpressionNextTime(this.cronExpression.value as string); + } + + public get cronExpression(): AbstractControl { + return this.pollingForm.controls.cronExpression; + } + + public get pollingEveryTime(): AbstractControl { + return this.pollingForm.controls.every; + } + + public get pollingUnitTime(): AbstractControl { + return this.pollingForm.controls.unit; + } + + private disableAndClearControl(control: AbstractControl): void { + control.disable(); + control.markAsUntouched(); + control.markAsPristine(); + } + + private setPollingEveryTimeValidator(): void { + this.pollingUnitTime.valueChanges.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((data: TimeUnit) => { + if (data) { + this.pollingEveryTime.setValidators([this.everyTimeMapperValidators[data], Validators.required]); + } + }); + } + + public get pollingType(): AbstractControl { + return this.pollingForm.controls.__typename; + } + + private pollingTypeChanges(): void { + this.pollingType.valueChanges.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((value: PollingGroupEnum) => { + switch (value) { + case PollingGroupEnum.TIME_DELTA: { + this.pollingEveryTime.enable(); + this.pollingUnitTime.enable(); + this.disableAndClearControl(this.cronExpression); + break; + } + case PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION: { + this.cronExpression.enable(); + this.disableAndClearControl(this.pollingEveryTime); + this.disableAndClearControl(this.pollingUnitTime); + break; + } + /* istanbul ignore next */ + default: { + logError("Unknown PollingGroupEnum key"); + } + } + }); + } +} diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger.module.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger.module.ts new file mode 100644 index 000000000..6294b41a7 --- /dev/null +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger.module.ts @@ -0,0 +1,14 @@ +import { NgModule } from "@angular/core"; +import { CommonModule } from "@angular/common"; +import { IngestTriggerFormComponent } from "./ingest-trigger-form.component"; +import { FormsModule, ReactiveFormsModule } from "@angular/forms"; +import { MatRadioModule } from "@angular/material/radio"; +import { MatSlideToggleModule } from "@angular/material/slide-toggle"; + +@NgModule({ + declarations: [IngestTriggerFormComponent], + exports: [IngestTriggerFormComponent], + + imports: [CommonModule, FormsModule, ReactiveFormsModule, MatRadioModule, MatSlideToggleModule], +}) +export class IngestTriggerModule {} diff --git a/src/app/dataset-view/dataset.module.ts b/src/app/dataset-view/dataset.module.ts index 6c9a16ab5..dff2cdeaf 100644 --- a/src/app/dataset-view/dataset.module.ts +++ b/src/app/dataset-view/dataset.module.ts @@ -85,7 +85,9 @@ import { AngularMultiSelectModule } from "angular2-multiselect-dropdown"; import { MatTabsModule } from "@angular/material/tabs"; import { MatChipsModule } from "@angular/material/chips"; import { QuerySharedModule } from "../query/shared/query-shared/query-shared.module"; - +import { IngestConfigurationModule } from "./additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration.module"; +import { IngestTriggerModule } from "./additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger.module"; +import { BatchingTriggerModule } from "./additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger.module"; @NgModule({ imports: [ CommonModule, @@ -141,6 +143,9 @@ import { QuerySharedModule } from "../query/shared/query-shared/query-shared.mod MatSortModule, AngularMultiSelectModule, QuerySharedModule, + IngestConfigurationModule, + IngestTriggerModule, + BatchingTriggerModule, ], exports: [ DatasetViewHeaderComponent, From 7f8f4d5b89706496079ed30fdfa4d697bfbbda0a Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Wed, 18 Dec 2024 17:37:57 +0200 Subject: [PATCH 14/17] refactored unit tests --- src/app/api/mock/dataset-flow.mock.ts | 22 +- .../batching-trigger-form.component.spec.ts | 55 +++-- .../batching-trigger-form.component.ts | 8 - ...-settings-scheduling-tab.component.spec.ts | 195 ++++++++---------- ...ngest-configuration-form.component.spec.ts | 55 +++-- .../ingest-trigger-form.component.html | 1 - .../ingest-trigger-form.component.spec.ts | 123 +++++++++-- 7 files changed, 294 insertions(+), 165 deletions(-) diff --git a/src/app/api/mock/dataset-flow.mock.ts b/src/app/api/mock/dataset-flow.mock.ts index 2f4028e51..4e7d259de 100644 --- a/src/app/api/mock/dataset-flow.mock.ts +++ b/src/app/api/mock/dataset-flow.mock.ts @@ -139,7 +139,27 @@ export const mockGetDatasetFlowTriggersQuery: GetDatasetFlowTriggersQuery = { paused: true, schedule: { __typename: "Cron5ComponentExpression", - cron5ComponentExpression: "", + cron5ComponentExpression: "* * * * ?", + }, + }, + }, + }, + }, + }, +}; + +export const mockGetDatasetFlowTriggersTimeDeltaQuery: GetDatasetFlowTriggersQuery = { + datasets: { + __typename: "Datasets", + byId: { + flows: { + triggers: { + byType: { + paused: true, + schedule: { + __typename: "TimeDelta", + every: 10, + unit: TimeUnit.Minutes, }, }, }, diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.spec.ts index 5f6405a44..747d99e96 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.spec.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.spec.ts @@ -1,21 +1,46 @@ -import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { ComponentFixture, TestBed } from "@angular/core/testing"; +import { BatchingTriggerFormComponent } from "./batching-trigger-form.component"; +import { FormsModule, ReactiveFormsModule } from "@angular/forms"; +import { MatSlideToggleModule } from "@angular/material/slide-toggle"; +import { Apollo } from "apollo-angular"; +import { SharedTestModule } from "src/app/common/shared-test.module"; +import { DatasetSchedulingService } from "../../../services/dataset-scheduling.service"; +import { mockGetDatasetFlowTriggersBatchingQuery } from "src/app/api/mock/dataset-flow.mock"; +import { of } from "rxjs"; +import { ToastrModule } from "ngx-toastr"; +import { mockDatasetBasicsDerivedFragment } from "src/app/search/mock.data"; +import { emitClickOnElementByDataTestId } from "src/app/common/base-test.helpers.spec"; -import { BatchingTriggerFormComponent } from './batching-trigger-form.component'; +describe("BatchingTriggerFormComponent", () => { + let component: BatchingTriggerFormComponent; + let fixture: ComponentFixture; + let datasetSchedulingService: DatasetSchedulingService; -describe('BatchingTriggerFormComponent', () => { - let component: BatchingTriggerFormComponent; - let fixture: ComponentFixture; + beforeEach(() => { + TestBed.configureTestingModule({ + declarations: [BatchingTriggerFormComponent], + providers: [Apollo], + imports: [FormsModule, ReactiveFormsModule, ToastrModule.forRoot(), MatSlideToggleModule, SharedTestModule], + }); + fixture = TestBed.createComponent(BatchingTriggerFormComponent); + component = fixture.componentInstance; + component.datasetBasics = mockDatasetBasicsDerivedFragment; + datasetSchedulingService = TestBed.inject(DatasetSchedulingService); + spyOn(datasetSchedulingService, "fetchDatasetFlowTriggers").and.returnValue( + of(mockGetDatasetFlowTriggersBatchingQuery), + ); + fixture.detectChanges(); + }); - beforeEach(() => { - TestBed.configureTestingModule({ - declarations: [BatchingTriggerFormComponent] + it("should create", () => { + expect(component).toBeTruthy(); }); - fixture = TestBed.createComponent(BatchingTriggerFormComponent); - component = fixture.componentInstance; - fixture.detectChanges(); - }); - it('should create', () => { - expect(component).toBeTruthy(); - }); + it("should check save batching triggers", () => { + const saveTriggerEmitSpy = spyOn(component.saveTriggerEmit, "emit"); + + emitClickOnElementByDataTestId(fixture, "save-batching-triggers"); + + expect(saveTriggerEmitSpy).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.ts index 3b2530db3..4c16bb92b 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/batching-trigger-form/batching-trigger-form.component.ts @@ -48,14 +48,6 @@ export class BatchingTriggerFormComponent extends BaseComponent implements OnIni return this.batchingForm.controls.unit; } - public get batchingMinRecordsToAwait(): AbstractControl { - return this.batchingForm.controls.minRecordsToAwait; - } - - public get batchingUpdateState(): AbstractControl { - return this.batchingForm.controls.updatesState; - } - public saveBatchingTriggers(): void { this.saveTriggerEmit.emit(this.batchingForm); } diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.spec.ts index 21e2857ea..a71ba43c1 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.spec.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/dataset-settings-scheduling-tab.component.spec.ts @@ -12,17 +12,10 @@ import { mockDatasetBasicsDerivedFragment, mockDatasetBasicsRootFragment, mockFullPowerDatasetPermissionsFragment, - mockNotScheduleDatasetPermissionsFragment, } from "src/app/search/mock.data"; -import { ReactiveFormsModule } from "@angular/forms"; +import { FormControl, FormGroup, ReactiveFormsModule, Validators } from "@angular/forms"; import { DatasetSchedulingService } from "../../services/dataset-scheduling.service"; -import { - checkButtonDisabled, - emitClickOnElementByDataTestId, - findElementByDataTestId, - setFieldValue, -} from "src/app/common/base-test.helpers.spec"; -import { PollingGroupEnum } from "../../dataset-settings.model"; +import { findElementByDataTestId } from "src/app/common/base-test.helpers.spec"; import _ from "lodash"; import { TimeDelta, TimeUnit } from "src/app/api/kamu.graphql.interface"; import { of } from "rxjs"; @@ -31,6 +24,17 @@ import { mockGetDatasetFlowTriggersQuery, mockIngestGetDatasetFlowConfigsSuccess, } from "src/app/api/mock/dataset-flow.mock"; +import { BatchingTriggerModule } from "./batching-trigger-form/batching-trigger.module"; +import { IngestConfigurationModule } from "./ingest-configuration-form/ingest-configuration.module"; +import { IngestTriggerModule } from "./ingest-trigger-form/ingest-trigger.module"; +import { + BatchingFormType, + IngestConfigurationFormType, + PollingGroupType, +} from "./dataset-settings-scheduling-tab.component.types"; +import { MaybeNull } from "src/app/common/app.types"; +import { PollingGroupEnum } from "../../dataset-settings.model"; +import { cronExpressionValidator } from "src/app/common/data.helpers"; describe("DatasetSettingsSchedulingTabComponent", () => { let component: DatasetSettingsSchedulingTabComponent; @@ -42,7 +46,6 @@ describe("DatasetSettingsSchedulingTabComponent", () => { const MOCK_PARAM_UNIT = TimeUnit.Minutes; const MOCK_MIN_RECORDS_TO_AWAIT = 40; const MOCK_CRON_EXPRESSION = "* * * * ?"; - const MOCK_INVALID_CRON_EXPRESSION = "* *"; const MOCK_INPUT_TIME_DELTA: TimeDelta = { every: MOCK_PARAM_EVERY, unit: MOCK_PARAM_UNIT, @@ -61,6 +64,9 @@ describe("DatasetSettingsSchedulingTabComponent", () => { MatSlideToggleModule, MatRadioModule, ReactiveFormsModule, + IngestConfigurationModule, + IngestTriggerModule, + BatchingTriggerModule, ], }).compileComponents(); @@ -103,41 +109,53 @@ describe("DatasetSettingsSchedulingTabComponent", () => { expect(fetchUncacheableCheckBox.value).toEqual(TimeUnit.Hours); }); - it("should check have permission to canSchedule", () => { - component.datasetPermissions = _.cloneDeep(mockNotScheduleDatasetPermissionsFragment); - fixture.detectChanges(); - expect(component.pollingForm.disabled).toEqual(true); - }); - - it("should check the initial state of the 'Save' button", () => { + it("should check save ingest configuration", () => { + const mockConfigurationForm = new FormGroup({ + fetchUncacheable: new FormControl(false, { nonNullable: true }), + }); component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); fixture.detectChanges(); - checkButtonDisabled(fixture, "save-config-options", true); - }); + const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetFlowConfigs").and.returnValue( + of(true), + ); + const toastrServiceSpy = spyOn(toastrService, "success"); - it("should check switch polling options", () => { - component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); - fixture.detectChanges(); - emitClickOnElementByDataTestId(fixture, "button-cron-expression"); - expect(component.pollingType.value).toEqual(PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION); - emitClickOnElementByDataTestId(fixture, "button-time-delta"); - expect(component.pollingType.value).toEqual(PollingGroupEnum.TIME_DELTA); + component.saveIngestConfiguration(mockConfigurationForm); + + expect(setDatasetFlowScheduleSpy).toHaveBeenCalledTimes(1); + expect(toastrServiceSpy).toHaveBeenCalledTimes(1); }); - it("should check 'Save trigger' button works for ROOT dataset", () => { - const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetTriggers").and.callThrough(); + it("should check 'Save triger' button works for DERIVATIVE dataset", () => { + const setDatasetFlowBatchingSpy = spyOn(datasetSchedulingService, "setDatasetTriggers").and.callThrough(); component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); - fixture.detectChanges(); - setFieldValue(fixture, "polling-group-every", MOCK_PARAM_EVERY.toString()); - setFieldValue(fixture, "polling-group-unit", MOCK_PARAM_UNIT); - fixture.detectChanges(); + component.datasetBasics = mockDatasetBasicsDerivedFragment; + const mockBatchingTriggerForm = new FormGroup({ + updatesState: new FormControl(false, { nonNullable: true }), + every: new FormControl>({ value: MOCK_PARAM_EVERY, disabled: false }, [ + Validators.required, + Validators.min(1), + ]), + unit: new FormControl>({ value: MOCK_PARAM_UNIT, disabled: false }, [ + Validators.required, + ]), + minRecordsToAwait: new FormControl>( + { value: MOCK_MIN_RECORDS_TO_AWAIT, disabled: false }, + [Validators.required, Validators.min(1)], + ), + }); + + component.saveBatchingTriggers(mockBatchingTriggerForm); - emitClickOnElementByDataTestId(fixture, "save-config-options"); - expect(setDatasetFlowScheduleSpy).toHaveBeenCalledWith( + expect(setDatasetFlowBatchingSpy).toHaveBeenCalledWith( jasmine.objectContaining({ triggerInput: { - schedule: { - timeDelta: MOCK_INPUT_TIME_DELTA, + batching: { + minRecordsToAwait: MOCK_MIN_RECORDS_TO_AWAIT, + maxBatchingInterval: { + every: MOCK_PARAM_EVERY, + unit: MOCK_PARAM_UNIT, + }, }, }, }), @@ -147,12 +165,24 @@ describe("DatasetSettingsSchedulingTabComponent", () => { it("should check 'Save trigger' button works for ROOT dataset with time delta", () => { const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetTriggers").and.callThrough(); component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); - fixture.detectChanges(); - setFieldValue(fixture, "polling-group-every", MOCK_PARAM_EVERY.toString()); - setFieldValue(fixture, "polling-group-unit", MOCK_PARAM_UNIT); - fixture.detectChanges(); - emitClickOnElementByDataTestId(fixture, "save-config-options"); + const mockPollingTriggerForm = new FormGroup({ + updatesState: new FormControl(false, { nonNullable: true }), + __typename: new FormControl(PollingGroupEnum.TIME_DELTA, [Validators.required]), + every: new FormControl>({ value: MOCK_PARAM_EVERY, disabled: false }, [ + Validators.required, + Validators.min(1), + ]), + unit: new FormControl>({ value: MOCK_PARAM_UNIT, disabled: false }, [ + Validators.required, + ]), + cronExpression: new FormControl>({ value: "", disabled: true }, [ + Validators.required, + cronExpressionValidator(), + ]), + }); + + component.savePollingTriggers(mockPollingTriggerForm); expect(setDatasetFlowScheduleSpy).toHaveBeenCalledWith( jasmine.objectContaining({ @@ -168,12 +198,22 @@ describe("DatasetSettingsSchedulingTabComponent", () => { it("should check 'Save trigger' button works for ROOT dataset with cron expression", () => { const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetTriggers").and.callThrough(); component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); - fixture.detectChanges(); - emitClickOnElementByDataTestId(fixture, "button-cron-expression"); - setFieldValue(fixture, "polling-group-cron-expression", MOCK_CRON_EXPRESSION); - fixture.detectChanges(); - emitClickOnElementByDataTestId(fixture, "save-config-options"); + const mockPollingTriggerForm = new FormGroup({ + updatesState: new FormControl(false, { nonNullable: true }), + __typename: new FormControl(PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION, [Validators.required]), + every: new FormControl>({ value: null, disabled: false }, [ + Validators.required, + Validators.min(1), + ]), + unit: new FormControl>({ value: null, disabled: false }, [Validators.required]), + cronExpression: new FormControl>({ value: MOCK_CRON_EXPRESSION, disabled: true }, [ + Validators.required, + cronExpressionValidator(), + ]), + }); + + component.savePollingTriggers(mockPollingTriggerForm); expect(setDatasetFlowScheduleSpy).toHaveBeenCalledWith( jasmine.objectContaining({ @@ -185,67 +225,4 @@ describe("DatasetSettingsSchedulingTabComponent", () => { }), ); }); - - it("should check 'Save triger' button works for DERIVATIVE dataset", () => { - const setDatasetFlowBatchingSpy = spyOn(datasetSchedulingService, "setDatasetTriggers").and.callThrough(); - component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); - component.datasetBasics = mockDatasetBasicsDerivedFragment; - fixture.detectChanges(); - setFieldValue(fixture, "batching-interval-every", MOCK_PARAM_EVERY.toString()); - setFieldValue(fixture, "batching-interval-unit", MOCK_PARAM_UNIT); - setFieldValue(fixture, "batching-min-records", MOCK_MIN_RECORDS_TO_AWAIT.toString()); - fixture.detectChanges(); - - emitClickOnElementByDataTestId(fixture, "save-batching-triggers"); - - expect(setDatasetFlowBatchingSpy).toHaveBeenCalledWith( - jasmine.objectContaining({ - triggerInput: { - batching: { - minRecordsToAwait: MOCK_MIN_RECORDS_TO_AWAIT, - maxBatchingInterval: { - every: MOCK_PARAM_EVERY, - unit: MOCK_PARAM_UNIT, - }, - }, - }, - }), - ); - }); - - it("should check cron expression error", () => { - component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); - fixture.detectChanges(); - emitClickOnElementByDataTestId(fixture, "button-cron-expression"); - setFieldValue(fixture, "polling-group-cron-expression", MOCK_INVALID_CRON_EXPRESSION); - fixture.detectChanges(); - const errorMessageElem = findElementByDataTestId(fixture, "cronExpression-error"); - expect(errorMessageElem?.textContent?.trim()).toEqual("Invalid expression"); - }); - - it("should check init form with schedule", () => { - component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); - fixture.detectChanges(); - emitClickOnElementByDataTestId(fixture, "button-cron-expression"); - setFieldValue(fixture, "polling-group-cron-expression", MOCK_INVALID_CRON_EXPRESSION); - fixture.detectChanges(); - const errorMessageElem = findElementByDataTestId(fixture, "cronExpression-error"); - expect(errorMessageElem?.textContent?.trim()).toEqual("Invalid expression"); - }); - - it("should check save ingest configuration", () => { - component.datasetPermissions = _.cloneDeep(mockFullPowerDatasetPermissionsFragment); - const toastrServiceSpy = spyOn(toastrService, "success"); - const setDatasetFlowScheduleSpy = spyOn(datasetSchedulingService, "setDatasetFlowConfigs").and.returnValue( - of(true), - ); - fixture.detectChanges(); - emitClickOnElementByDataTestId(fixture, "fetchUncacheable"); - fixture.detectChanges(); - - emitClickOnElementByDataTestId(fixture, "save-polling-configuration"); - - expect(setDatasetFlowScheduleSpy).toHaveBeenCalledTimes(1); - expect(toastrServiceSpy).toHaveBeenCalledTimes(1); - }); }); diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.spec.ts index cc2eebd4e..25af2b1d0 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.spec.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-configuration-form/ingest-configuration-form.component.spec.ts @@ -1,21 +1,46 @@ -import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { ComponentFixture, TestBed } from "@angular/core/testing"; +import { IngestConfigurationFormComponent } from "./ingest-configuration-form.component"; +import { mockDatasetBasicsRootFragment } from "src/app/search/mock.data"; +import { emitClickOnElementByDataTestId } from "src/app/common/base-test.helpers.spec"; +import { SharedTestModule } from "src/app/common/shared-test.module"; +import { Apollo } from "apollo-angular"; +import { ToastrModule } from "ngx-toastr"; +import { DatasetSchedulingService } from "../../../services/dataset-scheduling.service"; +import { of } from "rxjs"; +import { mockIngestGetDatasetFlowConfigsSuccess } from "src/app/api/mock/dataset-flow.mock"; +import { FormsModule, ReactiveFormsModule } from "@angular/forms"; -import { IngestConfigurationFormComponent } from './ingest-configuration-form.component'; +describe("IngestConfigurationFormComponent", () => { + let component: IngestConfigurationFormComponent; + let fixture: ComponentFixture; + let datasetSchedulingService: DatasetSchedulingService; -describe('IngestConfigurationFormComponent', () => { - let component: IngestConfigurationFormComponent; - let fixture: ComponentFixture; + beforeEach(() => { + TestBed.configureTestingModule({ + declarations: [IngestConfigurationFormComponent], + providers: [Apollo], + imports: [SharedTestModule, FormsModule, ReactiveFormsModule, ToastrModule.forRoot()], + }); + fixture = TestBed.createComponent(IngestConfigurationFormComponent); + component = fixture.componentInstance; + datasetSchedulingService = TestBed.inject(DatasetSchedulingService); + component.datasetBasics = mockDatasetBasicsRootFragment; + spyOn(datasetSchedulingService, "fetchDatasetFlowConfigs").and.returnValue( + of(mockIngestGetDatasetFlowConfigsSuccess), + ); + fixture.detectChanges(); + }); - beforeEach(() => { - TestBed.configureTestingModule({ - declarations: [IngestConfigurationFormComponent] + it("should create", () => { + expect(component).toBeTruthy(); }); - fixture = TestBed.createComponent(IngestConfigurationFormComponent); - component = fixture.componentInstance; - fixture.detectChanges(); - }); - it('should create', () => { - expect(component).toBeTruthy(); - }); + it("should check save ingest configuration", () => { + const saveConfigurationEmitSpy = spyOn(component.saveConfigurationEmit, "emit"); + emitClickOnElementByDataTestId(fixture, "fetchUncacheable"); + + emitClickOnElementByDataTestId(fixture, "save-polling-configuration"); + + expect(saveConfigurationEmitSpy).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.html index 0c581ad5e..3f2aa3d99 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.html +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.html @@ -56,7 +56,6 @@ formControlName="cronExpression" placeholder="Example: * * * * ?" class="form-control form-control-expression" - data-test-id="cron-expression-input" data-test-id="polling-group-cron-expression" /> diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.spec.ts index bec3bbcc8..6ef8f75ed 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.spec.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/scheduling/ingest-trigger-form/ingest-trigger-form.component.spec.ts @@ -1,21 +1,112 @@ -import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { ComponentFixture, TestBed } from "@angular/core/testing"; +import { IngestTriggerFormComponent } from "./ingest-trigger-form.component"; +import { MatSlideToggleModule } from "@angular/material/slide-toggle"; +import { FormsModule, ReactiveFormsModule } from "@angular/forms"; +import { MatRadioModule } from "@angular/material/radio"; +import { mockDatasetBasicsRootFragment } from "src/app/search/mock.data"; +import { SharedTestModule } from "src/app/common/shared-test.module"; +import { Apollo } from "apollo-angular"; +import { ToastrModule } from "ngx-toastr"; +import { DatasetSchedulingService } from "../../../services/dataset-scheduling.service"; +import { of } from "rxjs"; +import { + mockGetDatasetFlowTriggersQuery, + mockGetDatasetFlowTriggersTimeDeltaQuery, +} from "src/app/api/mock/dataset-flow.mock"; +import { + emitClickOnElementByDataTestId, + findElementByDataTestId, + setFieldValue, +} from "src/app/common/base-test.helpers.spec"; +import { TimeUnit } from "src/app/api/kamu.graphql.interface"; +import { PollingGroupEnum } from "../../../dataset-settings.model"; -import { IngestTriggerFormComponent } from './ingest-trigger-form.component'; +describe("IngestTriggerFormComponent", () => { + let component: IngestTriggerFormComponent; + let fixture: ComponentFixture; + let datasetSchedulingService: DatasetSchedulingService; -describe('IngestTriggerFormComponent', () => { - let component: IngestTriggerFormComponent; - let fixture: ComponentFixture; + const MOCK_INVALID_CRON_EXPRESSION = "* *"; - beforeEach(() => { - TestBed.configureTestingModule({ - declarations: [IngestTriggerFormComponent] + beforeEach(() => { + TestBed.configureTestingModule({ + declarations: [IngestTriggerFormComponent], + providers: [Apollo], + imports: [ + FormsModule, + ReactiveFormsModule, + ToastrModule.forRoot(), + MatRadioModule, + SharedTestModule, + MatSlideToggleModule, + ], + }); + fixture = TestBed.createComponent(IngestTriggerFormComponent); + component = fixture.componentInstance; + datasetSchedulingService = TestBed.inject(DatasetSchedulingService); + component.datasetBasics = mockDatasetBasicsRootFragment; + }); + + it("should create", () => { + expect(component).toBeTruthy(); + }); + + it("should check init form with cron expression", () => { + spyOn(datasetSchedulingService, "fetchDatasetFlowTriggers").and.returnValue( + of(mockGetDatasetFlowTriggersQuery), + ); + fixture.detectChanges(); + + const cronExpessionControl = findElementByDataTestId( + fixture, + "polling-group-cron-expression", + ) as HTMLInputElement; + expect(cronExpessionControl.value.trim()).toEqual("* * * * ?"); + }); + + it("should check init form with time delta", () => { + spyOn(datasetSchedulingService, "fetchDatasetFlowTriggers").and.returnValue( + of(mockGetDatasetFlowTriggersTimeDeltaQuery), + ); + fixture.detectChanges(); + + const everyControl = findElementByDataTestId(fixture, "polling-group-every") as HTMLInputElement; + expect(everyControl.value.trim()).toEqual("10"); + const unitControl = findElementByDataTestId(fixture, "polling-group-unit") as HTMLInputElement; + expect(unitControl.value.trim()).toEqual(TimeUnit.Minutes); + }); + + it("should check switch polling options", () => { + spyOn(datasetSchedulingService, "fetchDatasetFlowTriggers").and.returnValue( + of(mockGetDatasetFlowTriggersTimeDeltaQuery), + ); + fixture.detectChanges(); + emitClickOnElementByDataTestId(fixture, "button-cron-expression"); + expect(component.pollingType.value).toEqual(PollingGroupEnum.CRON_5_COMPONENT_EXPRESSION); + emitClickOnElementByDataTestId(fixture, "button-time-delta"); + expect(component.pollingType.value).toEqual(PollingGroupEnum.TIME_DELTA); + }); + + it("should check cron expression error", () => { + spyOn(datasetSchedulingService, "fetchDatasetFlowTriggers").and.returnValue( + of(mockGetDatasetFlowTriggersTimeDeltaQuery), + ); + fixture.detectChanges(); + emitClickOnElementByDataTestId(fixture, "button-cron-expression"); + setFieldValue(fixture, "polling-group-cron-expression", MOCK_INVALID_CRON_EXPRESSION); + fixture.detectChanges(); + const errorMessageElem = findElementByDataTestId(fixture, "cronExpression-error"); + expect(errorMessageElem?.textContent?.trim()).toEqual("Invalid expression"); + }); + + it("should check save pollingTriggers", () => { + spyOn(datasetSchedulingService, "fetchDatasetFlowTriggers").and.returnValue( + of(mockGetDatasetFlowTriggersTimeDeltaQuery), + ); + fixture.detectChanges(); + const saveTriggerEmitSpy = spyOn(component.saveTriggerEmit, "emit"); + emitClickOnElementByDataTestId(fixture, "save-config-options"); + + expect(saveTriggerEmitSpy).toHaveBeenCalledTimes(1); }); - fixture = TestBed.createComponent(IngestTriggerFormComponent); - component = fixture.componentInstance; - fixture.detectChanges(); - }); - - it('should create', () => { - expect(component).toBeTruthy(); - }); }); From 26751c9d590fe3393f2bdc314c6ee6f645f7b1e4 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Thu, 19 Dec 2024 19:10:00 +0200 Subject: [PATCH 15/17] minor changes --- .../dataset-settings.component.spec.ts | 4 ++++ .../dataset-settings-general-tab.component.html | 12 +++++------- .../dataset-settings-general-tab.component.scss | 4 ++++ .../dataset-settings-general-tab.component.spec.ts | 5 ++++- .../overview-component/overview.component.ts | 2 +- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.component.spec.ts index 3bfdd966a..cc3a4992d 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.component.spec.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/dataset-settings.component.spec.ts @@ -25,6 +25,8 @@ import { mockMetadataRootUpdate, mockOverviewDataUpdate } from "../data-tabs.moc import { TooltipIconComponent } from "src/app/dataset-block/metadata-block/components/tooltip-icon/tooltip-icon.component"; import { NgbTooltipModule } from "@ng-bootstrap/ng-bootstrap"; import { SharedModule } from "src/app/shared/shared/shared.module"; +import { MatCheckboxModule } from "@angular/material/checkbox"; +import { BatchingTriggerModule } from "./tabs/scheduling/batching-trigger-form/batching-trigger.module"; describe("DatasetSettingsComponent", () => { let component: DatasetSettingsComponent; @@ -77,6 +79,8 @@ describe("DatasetSettingsComponent", () => { MatIconModule, NgbTooltipModule, SharedModule, + MatCheckboxModule, + BatchingTriggerModule, ], }) .overrideComponent(DatasetSettingsComponent, { diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.html b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.html index a0ab093f4..15acabda0 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.html +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.html @@ -77,13 +77,11 @@

Danger Zone

- - Recursive: - + +
+ Recursive +
+
diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.scss b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.scss index c6d1c9673..d9721ba14 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.scss +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.scss @@ -7,6 +7,10 @@ --mat-radio-ripple-color: transparent; } + .mdc-checkbox__ripple { + display: none; + } + .mat-mdc-radio-label { white-space: normal; } diff --git a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.spec.ts b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.spec.ts index 3ef3add4a..2e2424055 100644 --- a/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.spec.ts +++ b/src/app/dataset-view/additional-components/dataset-settings-component/tabs/general/dataset-settings-general-tab.component.spec.ts @@ -6,7 +6,7 @@ import { DatasetSettingsGeneralTabComponent } from "./dataset-settings-general-t import { DatasetSettingsService } from "../../services/dataset-settings.service"; import { ModalService } from "../../../../../components/modal/modal.service"; import { ApolloModule } from "apollo-angular"; -import { FormBuilder, ReactiveFormsModule } from "@angular/forms"; +import { FormBuilder, FormsModule, ReactiveFormsModule } from "@angular/forms"; import { HttpClientTestingModule } from "@angular/common/http/testing"; import { MatDividerModule } from "@angular/material/divider"; import { MatIconModule } from "@angular/material/icon"; @@ -33,6 +33,7 @@ import { DatasetResetMode } from "./dataset-settings-general-tab.types"; import AppValues from "src/app/common/app.values"; import { DatasetFlowsService } from "../../../flows-component/services/dataset-flows.service"; import { DatasetService } from "../../../../dataset.service"; +import { MatCheckboxModule } from "@angular/material/checkbox"; describe("DatasetSettingsGeneralTabComponent", () => { let component: DatasetSettingsGeneralTabComponent; @@ -59,6 +60,8 @@ describe("DatasetSettingsGeneralTabComponent", () => { MatRadioModule, MatIconModule, NgbTooltipModule, + MatCheckboxModule, + FormsModule, ], providers: [FormBuilder], }).compileComponents(); diff --git a/src/app/dataset-view/additional-components/overview-component/overview.component.ts b/src/app/dataset-view/additional-components/overview-component/overview.component.ts index 18ef4ca53..3da70cebb 100644 --- a/src/app/dataset-view/additional-components/overview-component/overview.component.ts +++ b/src/app/dataset-view/additional-components/overview-component/overview.component.ts @@ -224,7 +224,7 @@ export class OverviewComponent extends BaseComponent implements OnInit { } public get visibleUpdateButton(): boolean { - return this.isUserLogged && this.enableScheduling; + return this.isUserLogged && this.enableScheduling && this.canSchedule; } public get hasSetPollingSource(): boolean { From a9590ca727088c5e15dcc73f542ed4d5eafac475 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Thu, 19 Dec 2024 20:21:47 +0200 Subject: [PATCH 16/17] changed CHANGELOG.md --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c77065440..69955acb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +### Added +- Made `force update` link for a flow table +### Changed +- Flow configuration separation + ## [0.34.0] - 2024-12-18 ### Added - Handling feature elements in 3 modes @@ -12,9 +18,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `demo` mode: unimplemented features are disabled - `develop` mode: unimplemented features are available - Made a tile element clickable -- Made `force update` link for a flow table -### Changed -- Flow configuration separation ### Fixed - Fixed error handling for the query explainer From 18196ca5fa20cbfa16de6e1f8f92c841f653f5d5 Mon Sep 17 00:00:00 2001 From: Dmitriy Borzenko Date: Fri, 20 Dec 2024 10:30:23 +0200 Subject: [PATCH 17/17] added modal window for a force update --- .../flows-table/flows-table.component.spec.ts | 15 ++++-- .../flows-table/flows-table.component.ts | 50 ++++++++++++------- .../flows-table/flows-table.helpers.ts | 9 +++- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/app/common/components/flows-table/flows-table.component.spec.ts b/src/app/common/components/flows-table/flows-table.component.spec.ts index f26c6d4d8..abecc5d2b 100644 --- a/src/app/common/components/flows-table/flows-table.component.spec.ts +++ b/src/app/common/components/flows-table/flows-table.component.spec.ts @@ -1,5 +1,5 @@ import { Apollo } from "apollo-angular"; -import { ComponentFixture, TestBed } from "@angular/core/testing"; +import { ComponentFixture, fakeAsync, flush, TestBed, tick } from "@angular/core/testing"; import { FlowsTableComponent } from "./flows-table.component"; import { MatTableModule } from "@angular/material/table"; import { MatMenuModule } from "@angular/material/menu"; @@ -133,13 +133,22 @@ describe("FlowsTableComponent", () => { expect(component.showForceUpdateLink(mockFlowSummaryDataFragmentShowForceLink)).toEqual(true); }); - it("should check trigger flow with force udate option", () => { + it("should check trigger flow with force udate option", fakeAsync(() => { const datasetTriggerFlowSpy = spyOn(datasetFlowsService, "datasetTriggerFlow").and.returnValue(of(true)); + const forceUpdateModalSpy = spyOn(modalService, "error").and.callFake((options) => { + options.handler?.call(undefined, true); + return Promise.resolve(""); + }); const toastrServiceSuccessSpy = spyOn(toastService, "success"); component.onForceUpdate(mockFlowSummaryDataFragmentShowForceLink); + fixture.detectChanges(); + tick(); + + expect(forceUpdateModalSpy).toHaveBeenCalledTimes(1); expect(datasetTriggerFlowSpy).toHaveBeenCalledTimes(1); expect(toastrServiceSuccessSpy).toHaveBeenCalledWith("Force update started"); - }); + flush(); + })); }); diff --git a/src/app/common/components/flows-table/flows-table.component.ts b/src/app/common/components/flows-table/flows-table.component.ts index f9f852b18..e21f04df2 100644 --- a/src/app/common/components/flows-table/flows-table.component.ts +++ b/src/app/common/components/flows-table/flows-table.component.ts @@ -200,26 +200,38 @@ export class FlowsTableComponent extends BaseComponent implements OnInit, OnChan } public onForceUpdate(node: FlowSummaryDataFragment): void { - if (node.description.__typename === "FlowDescriptionDatasetPollingIngest") { - this.datasetFlowsService - .datasetTriggerFlow({ - datasetId: node.description.datasetId, - datasetFlowType: DatasetFlowType.Ingest, - flowRunConfiguration: { - ingest: { - fetchUncacheable: true, - }, - }, - }) - .pipe(takeUntilDestroyed(this.destroyRef)) - .subscribe((result: boolean) => { - if (result) { - this.toastrService.success("Force update started"); + promiseWithCatch( + this.modalService.error({ + title: "Run force update", + message: "Do you want to run a force update?", + yesButtonText: "Ok", + noButtonText: "Cancel", + handler: (ok) => { + if (ok) { + if (node.description.__typename === "FlowDescriptionDatasetPollingIngest") { + this.datasetFlowsService + .datasetTriggerFlow({ + datasetId: node.description.datasetId, + datasetFlowType: DatasetFlowType.Ingest, + flowRunConfiguration: { + ingest: { + fetchUncacheable: true, + }, + }, + }) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe((result: boolean) => { + if (result) { + this.toastrService.success("Force update started"); + } + }); + } else { + throw new Error("Configuration snapshot is undefined"); + } } - }); - } else { - throw new Error("Configuration snapshot is undefined"); - } + }, + }), + ); } private initializeFilters(): void { diff --git a/src/app/common/components/flows-table/flows-table.helpers.ts b/src/app/common/components/flows-table/flows-table.helpers.ts index 8fd6b4949..a20198b1d 100644 --- a/src/app/common/components/flows-table/flows-table.helpers.ts +++ b/src/app/common/components/flows-table/flows-table.helpers.ts @@ -117,6 +117,13 @@ export class DatasetFlowTableHelpers { case "FlowDescriptionDatasetHardCompaction": switch (element.description.compactionResult?.__typename) { case "FlowDescriptionHardCompactionSuccess": + if ( + element.configSnapshot?.__typename === "FlowConfigurationCompactionRule" && + element.configSnapshot.compactionRule.__typename === + "CompactionMetadataOnly" + ) { + return "All data except metadata has been deleted"; + } return `Compacted ${element.description.compactionResult.originalBlocksCount} original blocks to ${element.description.compactionResult.resultingBlocksCount} resulting blocks`; case "FlowDescriptionHardCompactionNothingToDo": @@ -129,7 +136,7 @@ export class DatasetFlowTableHelpers { case "FlowDescriptionDatasetReset": switch (element.description.__typename) { case "FlowDescriptionDatasetReset": - return "All dataset history has been cleared."; + return "All dataset history has been cleared"; /* istanbul ignore next */ default: return "Unknown reset result typename";