From d3e02c9607663e67033139f9c684e8659fb555e0 Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Fri, 3 May 2024 16:49:16 +0530 Subject: [PATCH 1/6] data flow service --- src/getStepStartStates.ts | 3 + src/google-cloud/types.ts | 1 + src/ingestSources.ts | 2 + src/steps/data-flow/client.ts | 34 +++ src/steps/data-flow/constants.ts | 115 ++++++++++ src/steps/data-flow/converters.ts | 65 ++++++ src/steps/data-flow/index.ts | 369 ++++++++++++++++++++++++++++++ src/steps/steps.ts | 2 + 8 files changed, 591 insertions(+) create mode 100644 src/steps/data-flow/client.ts create mode 100644 src/steps/data-flow/constants.ts create mode 100644 src/steps/data-flow/converters.ts create mode 100644 src/steps/data-flow/index.ts diff --git a/src/getStepStartStates.ts b/src/getStepStartStates.ts index d5ec871a..5db07da4 100644 --- a/src/getStepStartStates.ts +++ b/src/getStepStartStates.ts @@ -201,6 +201,7 @@ import { WebSecurityScannerSteps } from './steps/web-security-scanner/constants' import { IntegrationConfig } from './types'; import { isMasterOrganizationInstance } from './utils/isMasterOrganizationInstance'; import { isSingleProjectInstance } from './utils/isSingleProjectInstance'; +import { STEP_GOOGLE_CLOUD_DATAFLOW_JOB } from './steps/data-flow/constants'; function makeStepStartStates( stepIds: string[], @@ -333,6 +334,7 @@ function getDefaultStepStartStates(params: { [STEP_AUDIT_CONFIG_IAM_POLICY]: { disabled: !!config.configureOrganizationProjects, }, + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB]: { disabled: false }, [STEP_COMPUTE_DISKS]: { disabled: false }, [STEP_COMPUTE_REGION_DISKS]: { disabled: false }, [STEP_COMPUTE_IMAGES]: { disabled: false }, @@ -648,6 +650,7 @@ async function getStepStartStatesUsingServiceEnablements(params: { [STEP_AUDIT_CONFIG_IAM_POLICY]: config.configureOrganizationProjects ? { disabled: true } : createStepStartState(ServiceUsageName.RESOURCE_MANAGER), + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB]: createStepStartState(ServiceUsageName.DATA_FLOW), [STEP_COMPUTE_DISKS]: createStepStartState(ServiceUsageName.COMPUTE), [STEP_COMPUTE_REGION_DISKS]: createStepStartState(ServiceUsageName.COMPUTE), [STEP_COMPUTE_IMAGES]: createStepStartState(ServiceUsageName.COMPUTE), diff --git a/src/google-cloud/types.ts b/src/google-cloud/types.ts index 86351bf8..e96ad8a6 100644 --- a/src/google-cloud/types.ts +++ b/src/google-cloud/types.ts @@ -39,4 +39,5 @@ export enum ServiceUsageName { CLOUD_SOURCE_REPOSITORIES = 'sourcerepo.googleapis.com', WEB_SECURITY_SCANNER = 'websecurityscanner.googleapis.com', ORG_POLICY = 'orgpolicy.googleapis.com', + DATA_FLOW = 'dataflow.googleapis.com' } diff --git a/src/ingestSources.ts b/src/ingestSources.ts index 1e2ce3cc..618fca0c 100644 --- a/src/ingestSources.ts +++ b/src/ingestSources.ts @@ -31,6 +31,7 @@ import { SpannerIngestionConfig } from './steps/spanner/constants'; import { SQLAdminIngestionConfig } from './steps/sql-admin'; import { StorageIngestionConfig } from './steps/storage/constants'; import { WebSecurityScannerIngestionConfig } from './steps/web-security-scanner/constants'; +import { DataflowIngestionConfig } from './steps/data-flow/constants'; export const ingestionConfig: IntegrationIngestionConfigFieldMap = { ...AccessContextManagerIngestionConfig, @@ -66,4 +67,5 @@ export const ingestionConfig: IntegrationIngestionConfigFieldMap = { ...SQLAdminIngestionConfig, ...StorageIngestionConfig, ...WebSecurityScannerIngestionConfig, + ...DataflowIngestionConfig }; diff --git a/src/steps/data-flow/client.ts b/src/steps/data-flow/client.ts new file mode 100644 index 00000000..5c55da8e --- /dev/null +++ b/src/steps/data-flow/client.ts @@ -0,0 +1,34 @@ +import { google, dataflow_v1b3 } from 'googleapis'; +import { Client } from '../../google-cloud/client'; +import { + DataFlowPermissions, + STEP_GOOGLE_CLOUD_DATAFLOW, + +} from './constants'; + +export class dataFlowClient extends Client { + private client = google.dataflow({ version: 'v1b3', retry: false }); + + async iterateGoogleCloudDataFlowJob( + callback: (data: dataflow_v1b3.Schema$Job) => Promise, + ) { + const auth = await this.getAuthenticatedServiceClient(); + + await this.iterateApi( + async (nextPageToken) => { + return this.client.projects.jobs.list({ + projectId: this.projectId, + auth, + pageToken: nextPageToken, + }); + }, + async (data: dataflow_v1b3.Schema$ListJobsResponse) => { + for (const job of data.jobs || []) { + await callback(job); + } + }, + STEP_GOOGLE_CLOUD_DATAFLOW, + DataFlowPermissions.STEP_GOOGLE_CLOUD_DATAFLOW_JOB, + ); + } +} diff --git a/src/steps/data-flow/constants.ts b/src/steps/data-flow/constants.ts new file mode 100644 index 00000000..3c736888 --- /dev/null +++ b/src/steps/data-flow/constants.ts @@ -0,0 +1,115 @@ +export const STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE = 'fetch-google-cloud-dataflow-datastore'; +export const GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS = 'Datastore'; +export const GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE = 'google_cloud_dataflow_datastore'; + +export const STEP_GOOGLE_CLOUD_DATAFLOW = 'fetch-google-cloud-dataflow'; +export const GOOGLE_CLOUD_DATAFLOW_CLASS = 'Service'; +export const GOOGLE_CLOUD_DATAFLOW_TYPE = 'google_cloud_dataflow'; + +export const STEP_GOOGLE_CLOUD_DATAFLOW_JOB = 'fetch-google-cloud-dataflow-job'; +export const GOOGLE_CLOUD_DATAFLOW_JOB_CLASS = ['Workflow']; +export const GOOGLE_CLOUD_DATAFLOW_JOB_TYPE = 'google_cloud_dataflow_job'; + +export const STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT = 'fetch-google-cloud-dataflow-snapshot'; +export const GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_CLASS = 'Database, DataStore, Image, Backup'; +export const GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE = 'google_cloud_dataflow_snapshot'; + +export const STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE = + 'fetch-project-has-google-cloud-dataflow-datastore'; +export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE = + 'google_cloud_project_has_google_cloud_dataflow_datastore'; + +export const STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW = + 'fetch-project-has-google-cloud-dataflow'; +export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW = + 'google_cloud_project_has_dataflow'; + +export const STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB = + 'fetch-project-has-google-cloud-dataflow-job'; +export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB = + 'google_cloud_project_has_dataflow_job'; + +export const STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE = + 'fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore'; +export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE = + 'google_cloud_dataflow_job_uses_google_cloud_dataflow_datastore'; + +export const STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT = + 'fetch-google-cloud-dataflow-job-has-google-cloud-dataflow-snapshot'; +export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT = + 'google_cloud_dataflow_job_uses_google_cloud_dataflow_snapshot'; + +export const STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC = + 'fetch-google-cloud-dataflow-job-uses-google-pubsub-topic'; +export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC = + 'google_cloud_dataflow_job_uses_google_pubsub_topic'; + +export const IngestionSources = { + GOOGLE_CLOUD_DATAFLOW_DATASTORE: 'google-cloud-dataflow-datastore', + GOOGLE_CLOUD_DATAFLOW: 'google-cloud-dataflow', + GOOGLE_CLOUD_DATAFLOW_JOB: 'google-cloud-dataflow-job', + GOOGLE_CLOUD_DATAFLOW_SNAPSHOT: 'google-cloud-dataflow-snapshot', + GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE: 'google-cloud-project-has-google-cloud-dataflow-datastore-job', + GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW: 'google-cloud-project-has-google-cloud-dataflow', + GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB: 'google-cloud-project-has-google-cloud-dataflow-job', + GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE: 'google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore', + GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT: 'google-cloud-dataflow-job-has-google-cloud-dataflow-snapshot', + GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC: 'google-cloud-dataflow-snapshot-uses-google-pubsub-topic' +}; + +export const DataflowIngestionConfig = { + [IngestionSources.GOOGLE_CLOUD_DATAFLOW]: { + title: 'Google Cloud Dataflow', + description: '', + defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { + title: 'Google Cloud Dataflow Datastore', + description: '', + defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_DATAFLOW_JOB]: { + title: 'Google Cloud Dataflow Job', + description: '', + defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: { + title: 'Google Cloud Dataflow Snapshot', + description: '', + defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { + title: 'Google Cloud Project has Google Cloud Dataflow Datastore Job', + description: '', + defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW]: { + title: 'Google Cloud Project has Google Cloud Dataflow', + description: '', + defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB]: { + title: 'Google Cloud Project has Google Cloud Dataflow Job', + description: '', + defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { + title: 'Google Cloud Dataflow Job Uses Google Cloud Dataflow Datastore', + description: '', + defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: { + title: 'Google Cloud Dataflow Job has Google Cloud Dataflow Snapshot', + description: '', + defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC]: { + title: 'Google Cloud Dataflow Snapshot Uses Google Pub/Sub Topic', + description: '', + defaultsToDisabled: false, + } +}; + +export const DataFlowPermissions = { + STEP_GOOGLE_CLOUD_DATAFLOW_JOB: ['dataflow.jobs.list'], +}; diff --git a/src/steps/data-flow/converters.ts b/src/steps/data-flow/converters.ts new file mode 100644 index 00000000..05510b7d --- /dev/null +++ b/src/steps/data-flow/converters.ts @@ -0,0 +1,65 @@ +import { dataflow_v1b3 } from 'googleapis'; +import { createGoogleCloudIntegrationEntity } from '../../utils/entity'; +import { GOOGLE_CLOUD_DATAFLOW_CLASS, GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS, GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, GOOGLE_CLOUD_DATAFLOW_JOB_CLASS, GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, GOOGLE_CLOUD_DATAFLOW_TYPE } from './constants'; + +export function createGoogleCloudDataFlowEntity(serviceObj) { + const data = { + ...serviceObj, // organizationId, projectId, instanceId + name: 'Google Cloud Data Flow', + }; + return createGoogleCloudIntegrationEntity(serviceObj, { + entityData: { + source: {}, + assign: { + _class: GOOGLE_CLOUD_DATAFLOW_CLASS, + _type: GOOGLE_CLOUD_DATAFLOW_TYPE, + _key: `${GOOGLE_CLOUD_DATAFLOW_TYPE}:${serviceObj}`, + name: data.name, + function: ['workflow', 'networking'], + category: ['network', 'security'], + }, + }, + }); +} + +export function createGoogleCloudDataFlowJobEntity( + data: dataflow_v1b3.Schema$Job, + projectId: string, +) { + return createGoogleCloudIntegrationEntity(data, { + entityData: { + source: data, + assign: { + _key: data.id as string, + _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, + _class: GOOGLE_CLOUD_DATAFLOW_JOB_CLASS, + id: data.id as string, + name: data.name, + displayName: data.name as string, + description: data.environment as string, + }, + }, + }); +} + +export function createGoogleCloudDataFlowDataStoreEntity( + data: dataflow_v1b3.Schema$DatastoreIODetails, + jobId:string, + projectId: string, +) { + return createGoogleCloudIntegrationEntity(data, { + entityData: { + source: data, + assign: { + _key: `projects/${projectId}${data.namespace}`, + _type: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + _class: GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS, + projectId: data.projectId as string, + name: data.namespace, + classification: data.namespace, + encrypted: false, + jobId: jobId + }, + }, + }); +} diff --git a/src/steps/data-flow/index.ts b/src/steps/data-flow/index.ts new file mode 100644 index 00000000..96f252d2 --- /dev/null +++ b/src/steps/data-flow/index.ts @@ -0,0 +1,369 @@ +import { + createDirectRelationship, + getRawData, + IntegrationMissingKeyError, + RelationshipClass, +} from '@jupiterone/integration-sdk-core'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; +import { dataFlowClient } from './client' +import { + IngestionSources, + STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS, + STEP_GOOGLE_CLOUD_DATAFLOW, + GOOGLE_CLOUD_DATAFLOW_TYPE, + GOOGLE_CLOUD_DATAFLOW_CLASS, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + GOOGLE_CLOUD_DATAFLOW_JOB_CLASS, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, + GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_CLASS, +} from './constants'; +import { + createGoogleCloudDataFlowEntity, + createGoogleCloudDataFlowDataStoreEntity, + createGoogleCloudDataFlowJobEntity, +} from './converters'; +import { PROJECT_ENTITY_TYPE, STEP_RESOURCE_MANAGER_PROJECT } from '../resource-manager'; +import { getProjectEntity } from '../../utils/project'; +import { dataflow_v1b3 } from 'googleapis'; + +export async function fetchGoogleCloudDataFlowDataStore( + context: IntegrationStepContext, +): Promise { + const { + jobState, + instance: { config }, + logger, + } = context; + + const client = new dataFlowClient({ config }, logger); + await jobState.iterateEntities( + { _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE }, + async (dataflowEntity) => { + const dataflow = getRawData(dataflowEntity); + + if (dataflow?.jobMetadata?.datastoreDetails) { + await jobState.addEntity(createGoogleCloudDataFlowDataStoreEntity(dataflow, dataflow.id as string, client.projectId)); + } + }, + ); +} + +export async function fetchGoogleCloudDataFlowJob( + context: IntegrationStepContext, +): Promise { + const { + jobState, + instance: { config }, + logger, + } = context; + + const client = new dataFlowClient({ config }, logger); + + await client.iterateGoogleCloudDataFlowJob(async (dataflow) => { + await jobState.addEntity(createGoogleCloudDataFlowJobEntity(dataflow, client.projectId)); + }); +} + +export async function fetchGoogleCloudDataFlowService( + context: IntegrationStepContext, +): Promise { + const { + jobState, + instance: { config }, + logger, + } = context; + + const client = new dataFlowClient({ config }, logger); + await jobState.addEntity(createGoogleCloudDataFlowEntity(client.projectId)); +} + +export async function fetchGoogleCloudDataFlowSnapshot( + context: IntegrationStepContext, +): Promise { + const { + jobState, + instance: { config }, + logger, + } = context; + + const client = new dataFlowClient({ config }, logger); + await jobState.addEntity(createGoogleCloudDataFlowEntity(client.projectId)); +} + +export async function buildProjectHasDataflowDatastoreRelationship( + context: IntegrationStepContext, +): Promise { + const { jobState } = context; + + const projectEntity = await getProjectEntity(jobState); + + if (!projectEntity) return; + + await jobState.iterateEntities( + { _type: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE }, + async (dataflowJob) => { + await jobState.addRelationship( + createDirectRelationship({ + _class: RelationshipClass.HAS, + fromKey: projectEntity._key as string, + fromType: PROJECT_ENTITY_TYPE, + toKey: dataflowJob._key as string, + toType: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + }), + ); + }, + ); +} + +export async function buildProjectHasDataflowRelationship( + context: IntegrationStepContext, +): Promise { + const { jobState } = context; + + const projectEntity = await getProjectEntity(jobState); + + if (!projectEntity) return; + + await jobState.iterateEntities( + { _type: GOOGLE_CLOUD_DATAFLOW_TYPE }, + async (dataflow) => { + await jobState.addRelationship( + createDirectRelationship({ + _class: RelationshipClass.HAS, + fromKey: projectEntity._key as string, + fromType: PROJECT_ENTITY_TYPE, + toKey: dataflow._key as string, + toType: GOOGLE_CLOUD_DATAFLOW_TYPE, + }), + ); + }, + ); +} + +export async function buildProjectHasDataflowJobRelationship( + context: IntegrationStepContext, +): Promise { + const { jobState } = context; + + const projectEntity = await getProjectEntity(jobState); + + if (!projectEntity) return; + + await jobState.iterateEntities( + { _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE }, + async (dataflowJob) => { + await jobState.addRelationship( + createDirectRelationship({ + _class: RelationshipClass.HAS, + fromKey: projectEntity._key as string, + fromType: PROJECT_ENTITY_TYPE, + toKey: dataflowJob._key as string, + toType: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, + }), + ); + }, + ); +} +export async function buildDataflowUsesDataflowDatastoreRelationship( + executionContext: IntegrationStepContext, +) { + const { jobState } = executionContext; + + await jobState.iterateEntities( + { _type: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE }, + async (datastoreEntity) => { + const dataflowJobKey = datastoreEntity.jobId as string + + const hasDataflowJobKey = jobState.hasKey(dataflowJobKey); + + if (!hasDataflowJobKey) { + throw new IntegrationMissingKeyError( + `Cannot build Relationship. + Error: Missing Key. + dataflowJobKey : ${dataflowJobKey}`, + ); + } + + await jobState.addRelationship( + createDirectRelationship({ + _class: RelationshipClass.HAS, + fromKey: dataflowJobKey, + fromType: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, + toKey: datastoreEntity._key, + toType: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + }), + ); + }, + ); +} + +export const dataFlowSteps: GoogleCloudIntegrationStep[] = [ + { + id: STEP_GOOGLE_CLOUD_DATAFLOW, + ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW, + name: 'Google Cloud Dataflow', + entities: [ + { + resourceName: 'Google Cloud Dataflow', + _type: GOOGLE_CLOUD_DATAFLOW_TYPE, + _class: GOOGLE_CLOUD_DATAFLOW_CLASS, + }, + ], + relationships: [], + dependsOn: [], + executionHandler: fetchGoogleCloudDataFlowService, + apis: ['dataflow.googleapis.com'], + }, + { + id: STEP_GOOGLE_CLOUD_DATAFLOW_JOB, + ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW_JOB, + name: 'Google Cloud Dataflow Job', + entities: [ + { + resourceName: 'Google Cloud Dataflow Job', + _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, + _class: ['Workflow'], + }, + ], + relationships: [], + dependsOn: [], + executionHandler: fetchGoogleCloudDataFlowJob, + apis: ['dataflow.googleapis.com'], + }, + { + id: STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW_DATASTORE, + name: 'Google Cloud Dataflow Datastore', + entities: [ + { + resourceName: 'Google Cloud Dataflow Datastore', + _type: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + _class: GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS, + }, + ], + relationships: [], + dependsOn: [STEP_GOOGLE_CLOUD_DATAFLOW_JOB], + executionHandler: fetchGoogleCloudDataFlowDataStore, + apis: ['dataflow.googleapis.com'], + }, + { + id: STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + ingestionSourceId: + IngestionSources.GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + name: 'Project HAS DataFlow Datastore', + entities: [], + relationships: [ + { + _class: RelationshipClass.HAS, + _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + sourceType: PROJECT_ENTITY_TYPE, + targetType: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + }, + ], + dependsOn: [ + STEP_RESOURCE_MANAGER_PROJECT, + STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + ], + executionHandler: buildProjectHasDataflowDatastoreRelationship, + permissions: [], + apis: ['dataflow.googleapis.com'], + }, + { + id: STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + ingestionSourceId: + IngestionSources.GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + name: 'Project HAS DataFlow', + entities: [], + relationships: [ + { + _class: RelationshipClass.HAS, + _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + sourceType: PROJECT_ENTITY_TYPE, + targetType: GOOGLE_CLOUD_DATAFLOW_TYPE, + }, + ], + dependsOn: [ + STEP_RESOURCE_MANAGER_PROJECT, + STEP_GOOGLE_CLOUD_DATAFLOW, + ], + executionHandler: buildProjectHasDataflowRelationship, + permissions: [], + apis: ['dataflow.googleapis.com'], + }, + { + id: STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + ingestionSourceId: + IngestionSources.GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + name: 'Project HAS DataFlow Job', + entities: [], + relationships: [ + { + _class: RelationshipClass.HAS, + _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + sourceType: PROJECT_ENTITY_TYPE, + targetType: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, + }, + ], + dependsOn: [ + STEP_RESOURCE_MANAGER_PROJECT, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB, + ], + executionHandler: buildProjectHasDataflowJobRelationship, + permissions: [], + apis: ['dataflow.googleapis.com'], + }, + { + id: STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + ingestionSourceId: + IngestionSources.GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + name: 'DataFlow HAS DataFlow Datastore', + entities: [], + relationships: [ + { + _class: RelationshipClass.HAS, + _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + sourceType: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, + targetType: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + }, + ], + dependsOn: [ + STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB, + ], + executionHandler: buildDataflowUsesDataflowDatastoreRelationship, + permissions: [], + apis: ['dataflow.googleapis.com'], + }, + { + id: STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + name: 'Google Cloud Dataflow Snapshot', + entities: [ + { + resourceName: 'Google Cloud Dataflow Datastore', + _type: GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, + _class: GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_CLASS, + }, + ], + relationships: [], + dependsOn: [STEP_GOOGLE_CLOUD_DATAFLOW_JOB], + executionHandler: fetchGoogleCloudDataFlowSnapshot, + apis: ['dataflow.googleapis.com'], + }, +]; + diff --git a/src/steps/steps.ts b/src/steps/steps.ts index 3b7f0068..b9b9e064 100644 --- a/src/steps/steps.ts +++ b/src/steps/steps.ts @@ -37,6 +37,7 @@ import { spannerSteps } from './spanner'; import { sqlAdminSteps } from './sql-admin'; import { storageSteps } from './storage'; import { webSecurityScannerSteps } from './web-security-scanner'; +import { dataFlowSteps } from './data-flow'; const steps: GoogleCloudIntegrationStep[] = wrapStepExecutionHandlers([ ...functionsSteps, @@ -71,6 +72,7 @@ const steps: GoogleCloudIntegrationStep[] = wrapStepExecutionHandlers([ ...cloudBuildSteps, ...cloudSourceRepositoriesSteps, ...webSecurityScannerSteps, + ...dataFlowSteps ]); function wrapStepExecutionHandlers( From c58dd8666b5b3343750fac864ffa98dd4639a992 Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Tue, 7 May 2024 15:59:23 +0530 Subject: [PATCH 2/6] added unit tests --- src/steps/data-flow/converters.ts | 25 ++- src/steps/data-flow/index.test.ts | 226 +++++++++++++++++++++++ src/steps/data-flow/index.ts | 21 ++- src/steps/data-flow/relationship.test.ts | 205 ++++++++++++++++++++ 4 files changed, 467 insertions(+), 10 deletions(-) create mode 100644 src/steps/data-flow/index.test.ts create mode 100644 src/steps/data-flow/relationship.test.ts diff --git a/src/steps/data-flow/converters.ts b/src/steps/data-flow/converters.ts index 05510b7d..67c29ffb 100644 --- a/src/steps/data-flow/converters.ts +++ b/src/steps/data-flow/converters.ts @@ -44,19 +44,36 @@ export function createGoogleCloudDataFlowJobEntity( export function createGoogleCloudDataFlowDataStoreEntity( data: dataflow_v1b3.Schema$DatastoreIODetails, - jobId:string, - projectId: string, ) { return createGoogleCloudIntegrationEntity(data, { entityData: { source: data, assign: { - _key: `projects/${projectId}${data.namespace}`, + _key: data.namespace as string, _type: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, _class: GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS, projectId: data.projectId as string, name: data.namespace, - classification: data.namespace, + encrypted: false, + }, + }, + }); +} + +export function createGoogleCloudDataFlowSnapshotEntity( + data: dataflow_v1b3.Schema$Snapshot, + jobId:string, +) { + return createGoogleCloudIntegrationEntity(data, { + entityData: { + source: data, + assign: { + _key: data.id as string, + _type: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + _class: GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS, + projectId: data.projectId as string, + name: data.diskSizeBytes, + classification: data.diskSizeBytes, encrypted: false, jobId: jobId }, diff --git a/src/steps/data-flow/index.test.ts b/src/steps/data-flow/index.test.ts new file mode 100644 index 00000000..7a7acbbd --- /dev/null +++ b/src/steps/data-flow/index.test.ts @@ -0,0 +1,226 @@ +import { + createMockStepExecutionContext, + Recording, + } from '@jupiterone/integration-sdk-testing'; + import { + fetchGoogleCloudDataFlowDataStore, + fetchGoogleCloudDataFlowJob, + fetchGoogleCloudDataFlowService, + fetchGoogleCloudDataFlowSnapshot, + } from '.'; + import { integrationConfig } from '../../../test/config'; + import { setupGoogleCloudRecording } from '../../../test/recording'; + import { IntegrationConfig } from '../../types'; + import { + GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + GOOGLE_CLOUD_DATAFLOW_JOB_CLASS, + GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, + GOOGLE_CLOUD_DATAFLOW_TYPE, + } from './constants'; + + const tempNewAccountConfig = { + ...integrationConfig, + serviceAccountKeyFile: integrationConfig.serviceAccountKeyFile.replace( + 'j1-gc-integration-dev-v2', + 'j1-gc-integration-dev-v3', + ), + serviceAccountKeyConfig: { + ...integrationConfig.serviceAccountKeyConfig, + project_id: 'j1-gc-integration-dev-v3', + }, + }; + + describe('#fetchGoogleCloudDataFlowService', () => { + let recording: Recording; + + beforeEach(() => { + recording = setupGoogleCloudRecording({ + directory: __dirname, + name: 'fetchGoogleCloudDataFlowService', + }); + }); + + afterEach(async () => { + await recording.stop(); + }); + + test('should collect data', async () => { + const context = createMockStepExecutionContext({ + instanceConfig: tempNewAccountConfig, + }); + + await fetchGoogleCloudDataFlowService(context); + + expect({ + numCollectedEntities: context.jobState.collectedEntities.length, + numCollectedRelationships: context.jobState.collectedRelationships.length, + collectedEntities: context.jobState.collectedEntities, + collectedRelationships: context.jobState.collectedRelationships, + encounteredTypes: context.jobState.encounteredTypes, + }).toMatchSnapshot(); + + expect( + context.jobState.collectedEntities.filter( + (e) => e.type === GOOGLE_CLOUD_DATAFLOW_TYPE, + ), + ).toMatchGraphObjectSchema({ + _class: ['Service'], + schema: { + additionalProperties: false, + properties: { + _type: { const: 'google_cloud_dataflow' }, + name: { type: 'string' }, + category: { type: 'array' }, + function: { type: 'array' }, + }, + }, + }); + }); + }); + + describe('#fetchGoogleCloudDataFlowJob', () => { + let recording: Recording; + + beforeEach(() => { + recording = setupGoogleCloudRecording({ + directory: __dirname, + name: 'fetchGoogleCloudDataFlowJob', + }); + }); + + afterEach(async () => { + await recording.stop(); + }); + + test('should collect data', async () => { + const context = createMockStepExecutionContext({ + instanceConfig: tempNewAccountConfig, + }); + + await fetchGoogleCloudDataFlowJob(context); + + expect({ + numCollectedEntities: context.jobState.collectedEntities.length, + numCollectedRelationships: context.jobState.collectedRelationships.length, + collectedEntities: context.jobState.collectedEntities, + collectedRelationships: context.jobState.collectedRelationships, + encounteredTypes: context.jobState.encounteredTypes, + }).toMatchSnapshot(); + + expect( + context.jobState.collectedEntities.filter( + (e) => e.type === GOOGLE_CLOUD_DATAFLOW_JOB_CLASS, + ), + ).toMatchGraphObjectSchema({ + _class: ['Workflow'], + schema: { + additionalProperties: false, + properties: { + _type: { const: 'google_cloud_dataflow_job' }, + name: { type: 'string' }, + displayname: { type: 'string' }, + description: { type: 'string' }, + }, + }, + }); + }); + }); + + describe('#fetchGoogleCloudDataFlowDataStore', () => { + let recording: Recording; + + beforeEach(() => { + recording = setupGoogleCloudRecording({ + directory: __dirname, + name: 'fetchGoogleCloudDataFlowDataStore', + }); + }); + + afterEach(async () => { + await recording.stop(); + }); + + test('should collect data', async () => { + const context = createMockStepExecutionContext({ + instanceConfig: tempNewAccountConfig, + }); + + await fetchGoogleCloudDataFlowDataStore(context); + + expect({ + numCollectedEntities: context.jobState.collectedEntities.length, + numCollectedRelationships: context.jobState.collectedRelationships.length, + collectedEntities: context.jobState.collectedEntities, + collectedRelationships: context.jobState.collectedRelationships, + encounteredTypes: context.jobState.encounteredTypes, + }).toMatchSnapshot(); + + expect( + context.jobState.collectedEntities.filter( + (e) => e.type === GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, + ), + ).toMatchGraphObjectSchema({ + _class: ['Datastore'], + schema: { + additionalProperties: false, + properties: { + _type: { const: 'google_cloud_dataflow_datastore' }, + name: { type: 'string' }, + encrypted: { type: 'string' }, + projectId: { type: 'string' }, + }, + }, + }); + }); + }); + + describe('#fetchGoogleCloudDataFlowSnapshot', () => { + let recording: Recording; + + beforeEach(() => { + recording = setupGoogleCloudRecording({ + directory: __dirname, + name: 'fetchGoogleCloudDataFlowSnapshot', + }); + }); + + afterEach(async () => { + await recording.stop(); + }); + + test('should collect data', async () => { + const context = createMockStepExecutionContext({ + instanceConfig: tempNewAccountConfig, + }); + + await fetchGoogleCloudDataFlowSnapshot(context); + expect({ + numCollectedEntities: context.jobState.collectedEntities.length, + numCollectedRelationships: context.jobState.collectedRelationships.length, + collectedEntities: context.jobState.collectedEntities, + collectedRelationships: context.jobState.collectedRelationships, + encounteredTypes: context.jobState.encounteredTypes, + }).toMatchSnapshot(); + + expect( + context.jobState.collectedEntities.filter( + (e) => e.type === GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, + ), + ).toMatchGraphObjectSchema({ + _class: ['Database'], + schema: { + additionalProperties: false, + properties: { + _type: { const: 'google_bigtable_backup' }, + name: { type: 'string' }, + projectId: { type: 'string' }, + classification: { type: 'string' }, + encrypted: { type: 'boolean' }, + jobId: { type: 'string' }, + }, + }, + }); + }); + }); + + \ No newline at end of file diff --git a/src/steps/data-flow/index.ts b/src/steps/data-flow/index.ts index 96f252d2..4d5f8d52 100644 --- a/src/steps/data-flow/index.ts +++ b/src/steps/data-flow/index.ts @@ -25,7 +25,6 @@ import { STEP_GOOGLE_CLOUD_DATAFLOW_JOB, STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, - GOOGLE_CLOUD_DATAFLOW_JOB_CLASS, STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, @@ -36,6 +35,7 @@ import { createGoogleCloudDataFlowEntity, createGoogleCloudDataFlowDataStoreEntity, createGoogleCloudDataFlowJobEntity, + createGoogleCloudDataFlowSnapshotEntity, } from './converters'; import { PROJECT_ENTITY_TYPE, STEP_RESOURCE_MANAGER_PROJECT } from '../resource-manager'; import { getProjectEntity } from '../../utils/project'; @@ -46,18 +46,15 @@ export async function fetchGoogleCloudDataFlowDataStore( ): Promise { const { jobState, - instance: { config }, - logger, } = context; - const client = new dataFlowClient({ config }, logger); await jobState.iterateEntities( { _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE }, async (dataflowEntity) => { const dataflow = getRawData(dataflowEntity); if (dataflow?.jobMetadata?.datastoreDetails) { - await jobState.addEntity(createGoogleCloudDataFlowDataStoreEntity(dataflow, dataflow.id as string, client.projectId)); + await jobState.addEntity(createGoogleCloudDataFlowDataStoreEntity(dataflow)); } }, ); @@ -102,9 +99,21 @@ export async function fetchGoogleCloudDataFlowSnapshot( } = context; const client = new dataFlowClient({ config }, logger); - await jobState.addEntity(createGoogleCloudDataFlowEntity(client.projectId)); + await jobState.iterateEntities( + { _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE }, + async (dataflowEntity) => { + const dataflow = getRawData(dataflowEntity); + + if (dataflow?.snapshots) { + for (const snapshot of dataflow.snapshots) { + await jobState.addEntity(createGoogleCloudDataFlowSnapshotEntity(snapshot, client.projectId)); + } + } + }, + ); } + export async function buildProjectHasDataflowDatastoreRelationship( context: IntegrationStepContext, ): Promise { diff --git a/src/steps/data-flow/relationship.test.ts b/src/steps/data-flow/relationship.test.ts new file mode 100644 index 00000000..5b31b32a --- /dev/null +++ b/src/steps/data-flow/relationship.test.ts @@ -0,0 +1,205 @@ +import { + executeStepWithDependencies, + Recording, + StepTestConfig, + } from '@jupiterone/integration-sdk-testing'; + import { invocationConfig } from '../../..'; + import { integrationConfig } from '../../../test/config'; + import { + getMatchRequestsBy, + setupGoogleCloudRecording, + } from '../../../test/recording'; +import { STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + } from './constants'; + + describe(STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, () => { + let recording: Recording; + afterEach(async () => { + if (recording) await recording.stop(); + }); + + jest.setTimeout(450000); + + test( + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + async () => { + recording = setupGoogleCloudRecording({ + name: STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + directory: __dirname, + options: { + matchRequestsBy: getMatchRequestsBy(integrationConfig), + }, + }); + + const stepTestConfig: StepTestConfig = { + stepId: STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + instanceConfig: integrationConfig, + invocationConfig: invocationConfig as any, + }; + + const result = await executeStepWithDependencies(stepTestConfig); + expect(result).toMatchStepMetadata(stepTestConfig); + }, + ); + }); + + describe(STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, () => { + let recording: Recording; + afterEach(async () => { + if (recording) await recording.stop(); + }); + + jest.setTimeout(450000); + + test( + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + async () => { + recording = setupGoogleCloudRecording({ + name: STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + directory: __dirname, + options: { + matchRequestsBy: getMatchRequestsBy(integrationConfig), + }, + }); + + const stepTestConfig: StepTestConfig = { + stepId: STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + instanceConfig: integrationConfig, + invocationConfig: invocationConfig as any, + }; + + const result = await executeStepWithDependencies(stepTestConfig); + expect(result).toMatchStepMetadata(stepTestConfig); + }, + ); + }); + + describe(STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, () => { + let recording: Recording; + afterEach(async () => { + if (recording) await recording.stop(); + }); + + jest.setTimeout(450000); + + test( + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + async () => { + recording = setupGoogleCloudRecording({ + name: STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + directory: __dirname, + options: { + matchRequestsBy: getMatchRequestsBy(integrationConfig), + }, + }); + + const stepTestConfig: StepTestConfig = { + stepId: STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + instanceConfig: integrationConfig, + invocationConfig: invocationConfig as any, + }; + + const result = await executeStepWithDependencies(stepTestConfig); + expect(result).toMatchStepMetadata(stepTestConfig); + }, + ); + }); + + describe(STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, () => { + let recording: Recording; + afterEach(async () => { + if (recording) await recording.stop(); + }); + + jest.setTimeout(450000); + + test( + STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + async () => { + recording = setupGoogleCloudRecording({ + name: STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + directory: __dirname, + options: { + matchRequestsBy: getMatchRequestsBy(integrationConfig), + }, + }); + + const stepTestConfig: StepTestConfig = { + stepId: STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + instanceConfig: integrationConfig, + invocationConfig: invocationConfig as any, + }; + + const result = await executeStepWithDependencies(stepTestConfig); + expect(result).toMatchStepMetadata(stepTestConfig); + }, + ); + }); + + describe(STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE, () => { + let recording: Recording; + afterEach(async () => { + if (recording) await recording.stop(); + }); + + jest.setTimeout(450000); + + test( + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + async () => { + recording = setupGoogleCloudRecording({ + name: STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + directory: __dirname, + options: { + matchRequestsBy: getMatchRequestsBy(integrationConfig), + }, + }); + + const stepTestConfig: StepTestConfig = { + stepId: STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + instanceConfig: integrationConfig, + invocationConfig: invocationConfig as any, + }; + + const result = await executeStepWithDependencies(stepTestConfig); + expect(result).toMatchStepMetadata(stepTestConfig); + }, + ); + }); + + describe(STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, () => { + let recording: Recording; + afterEach(async () => { + if (recording) await recording.stop(); + }); + + jest.setTimeout(450000); + + test( + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + async () => { + recording = setupGoogleCloudRecording({ + name: STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + directory: __dirname, + options: { + matchRequestsBy: getMatchRequestsBy(integrationConfig), + }, + }); + + const stepTestConfig: StepTestConfig = { + stepId: STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + instanceConfig: integrationConfig, + invocationConfig: invocationConfig as any, + }; + + const result = await executeStepWithDependencies(stepTestConfig); + expect(result).toMatchStepMetadata(stepTestConfig); + }, + ); + }); + \ No newline at end of file From 5644be27cd18bbe86131d64b218f1f027a38fca5 Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Fri, 10 May 2024 15:41:19 +0530 Subject: [PATCH 3/6] updated unit test --- src/getStepStartStates.ts | 37 +- .../recording.har | 250 +++++++++ .../recording.har | 486 ++++++++++++++++++ .../recording.har | 486 ++++++++++++++++++ .../recording.har | 250 +++++++++ .../recording.har | 250 +++++++++ .../__snapshots__/index.test.ts.snap | 155 ++++++ src/steps/data-flow/index.ts | 20 + src/steps/data-flow/relationship.test.ts | 2 +- 9 files changed, 1930 insertions(+), 6 deletions(-) create mode 100644 src/steps/data-flow/__recordings__/fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore_3496021278/recording.har create mode 100644 src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-datastore_1189976537/recording.har create mode 100644 src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-job_3299039843/recording.har create mode 100644 src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow_982301565/recording.har create mode 100644 src/steps/data-flow/__recordings__/fetchGoogleCloudDataFlowJob_1170190852/recording.har create mode 100644 src/steps/data-flow/__snapshots__/index.test.ts.snap diff --git a/src/getStepStartStates.ts b/src/getStepStartStates.ts index 5db07da4..097ceac1 100644 --- a/src/getStepStartStates.ts +++ b/src/getStepStartStates.ts @@ -201,7 +201,18 @@ import { WebSecurityScannerSteps } from './steps/web-security-scanner/constants' import { IntegrationConfig } from './types'; import { isMasterOrganizationInstance } from './utils/isMasterOrganizationInstance'; import { isSingleProjectInstance } from './utils/isSingleProjectInstance'; -import { STEP_GOOGLE_CLOUD_DATAFLOW_JOB } from './steps/data-flow/constants'; +import { + STEP_GOOGLE_CLOUD_DATAFLOW_JOB, + STEP_GOOGLE_CLOUD_DATAFLOW, + STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + // STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE +} from './steps/data-flow/constants'; function makeStepStartStates( stepIds: string[], @@ -244,9 +255,8 @@ export default async function getStepStartStates( logger.publishEvent({ name: 'integration_config', - description: `Starting Google Cloud integration with service account (email=${ - config.serviceAccountKeyConfig.client_email - }, configureOrganizationProjects=${!!config.configureOrganizationProjects})`, + description: `Starting Google Cloud integration with service account (email=${config.serviceAccountKeyConfig.client_email + }, configureOrganizationProjects=${!!config.configureOrganizationProjects})`, }); const masterOrgInstance = isMasterOrganizationInstance(config); @@ -334,7 +344,15 @@ function getDefaultStepStartStates(params: { [STEP_AUDIT_CONFIG_IAM_POLICY]: { disabled: !!config.configureOrganizationProjects, }, - [STEP_GOOGLE_CLOUD_DATAFLOW_JOB]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: { disabled: false }, + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { disabled: false }, + // [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC]: { disabled: false }, + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW]: { disabled: false }, + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { disabled: false }, [STEP_COMPUTE_DISKS]: { disabled: false }, [STEP_COMPUTE_REGION_DISKS]: { disabled: false }, [STEP_COMPUTE_IMAGES]: { disabled: false }, @@ -651,6 +669,15 @@ async function getStepStartStatesUsingServiceEnablements(params: { ? { disabled: true } : createStepStartState(ServiceUsageName.RESOURCE_MANAGER), [STEP_GOOGLE_CLOUD_DATAFLOW_JOB]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: createStepStartState(ServiceUsageName.DATA_FLOW), + // [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: createStepStartState(ServiceUsageName.DATA_FLOW), [STEP_COMPUTE_DISKS]: createStepStartState(ServiceUsageName.COMPUTE), [STEP_COMPUTE_REGION_DISKS]: createStepStartState(ServiceUsageName.COMPUTE), [STEP_COMPUTE_IMAGES]: createStepStartState(ServiceUsageName.COMPUTE), diff --git a/src/steps/data-flow/__recordings__/fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore_3496021278/recording.har b/src/steps/data-flow/__recordings__/fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore_3496021278/recording.har new file mode 100644 index 00000000..7869f432 --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore_3496021278/recording.har @@ -0,0 +1,250 @@ +{ + "log": { + "_recordingName": "fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1222, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1222, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:39 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:38.864Z", + "time": 187, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 187 + } + }, + { + "_id": "45fdb1f844a17b6d6ec371710879c815", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "dataflow.googleapis.com" + } + ], + "headersSize": 1305, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://dataflow.googleapis.com/v1b3/projects/j1-gc-integration-dev-v3/jobs" + }, + "response": { + "bodySize": 763, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 763, + "text": "{\"jobs\":[{\"id\":\"2024-05-02_05_25_02-17305733872805367843\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-job3\",\"type\":\"JOB_TYPE_BATCH\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-05-02T13:28:54.170517Z\",\"createTime\":\"2024-05-02T12:25:06.098727Z\",\"location\":\"us-central1\",\"jobMetadata\":{\"sdkVersion\":{\"version\":\"2.55.1\",\"versionDisplayName\":\"Apache Beam SDK for Java\",\"sdkSupportStatus\":\"SUPPORTED\"}},\"startTime\":\"2024-05-02T12:25:06.098727Z\"},{\"id\":\"2024-04-29_06_54_21-11420805499054724452\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job2\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T14:01:46.604199Z\",\"createTime\":\"2024-04-29T13:54:22.225266Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:54:22.225266Z\"},{\"id\":\"2024-04-29_06_40_47-701576625030004708\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job1\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T13:46:31.741443Z\",\"createTime\":\"2024-04-29T13:40:48.311738Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:40:48.311738Z\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:40 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:39.060Z", + "time": 1774, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1774 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-datastore_1189976537/recording.har b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-datastore_1189976537/recording.har new file mode 100644 index 00000000..f7a91e1c --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-datastore_1189976537/recording.har @@ -0,0 +1,486 @@ +{ + "log": { + "_recordingName": "fetch-project-has-google-cloud-dataflow-datastore", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1231, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1231, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:34 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:34.765Z", + "time": 234, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 234 + } + }, + { + "_id": "23a717153d5a0e7db6eb50c807895f67", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "cloudresourcemanager.googleapis.com" + } + ], + "headersSize": 1322, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://cloudresourcemanager.googleapis.com/v3/projects/j1-gc-integration-dev-v3" + }, + "response": { + "bodySize": 336, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 336, + "text": "{\"name\":\"projects/167984947943\",\"parent\":\"folders/306329820365\",\"projectId\":\"j1-gc-integration-dev-v3\",\"state\":\"ACTIVE\",\"displayName\":\"j1-gc-integration-dev-v3\",\"createTime\":\"2021-05-31T15:15:56.125Z\",\"updateTime\":\"2024-02-27T16:17:58.322524Z\",\"etag\":\"W/\\\"8f56735f6f3d890d\\\"\",\"labels\":{\"firebase\":\"enabled\"}}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:36 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:35.016Z", + "time": 1720, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1720 + } + }, + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1164, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1164, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:36 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:36.782Z", + "time": 185, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 185 + } + }, + { + "_id": "45fdb1f844a17b6d6ec371710879c815", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "dataflow.googleapis.com" + } + ], + "headersSize": 1305, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://dataflow.googleapis.com/v1b3/projects/j1-gc-integration-dev-v3/jobs" + }, + "response": { + "bodySize": 739, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 739, + "text": "{\"jobs\":[{\"id\":\"2024-05-02_05_25_02-17305733872805367843\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-job3\",\"type\":\"JOB_TYPE_BATCH\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-05-02T13:28:54.170517Z\",\"createTime\":\"2024-05-02T12:25:06.098727Z\",\"location\":\"us-central1\",\"jobMetadata\":{\"sdkVersion\":{\"version\":\"2.55.1\",\"versionDisplayName\":\"Apache Beam SDK for Java\",\"sdkSupportStatus\":\"SUPPORTED\"}},\"startTime\":\"2024-05-02T12:25:06.098727Z\"},{\"id\":\"2024-04-29_06_54_21-11420805499054724452\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job2\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T14:01:46.604199Z\",\"createTime\":\"2024-04-29T13:54:22.225266Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:54:22.225266Z\"},{\"id\":\"2024-04-29_06_40_47-701576625030004708\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job1\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T13:46:31.741443Z\",\"createTime\":\"2024-04-29T13:40:48.311738Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:40:48.311738Z\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:38 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:36.977Z", + "time": 1837, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1837 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-job_3299039843/recording.har b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-job_3299039843/recording.har new file mode 100644 index 00000000..32de1426 --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-job_3299039843/recording.har @@ -0,0 +1,486 @@ +{ + "log": { + "_recordingName": "fetch-project-has-google-cloud-dataflow-job", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1228, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1228, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:42 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:42.622Z", + "time": 192, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 192 + } + }, + { + "_id": "23a717153d5a0e7db6eb50c807895f67", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "cloudresourcemanager.googleapis.com" + } + ], + "headersSize": 1322, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://cloudresourcemanager.googleapis.com/v3/projects/j1-gc-integration-dev-v3" + }, + "response": { + "bodySize": 401, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 401, + "text": "{\"name\":\"projects/167984947943\",\"parent\":\"folders/306329820365\",\"projectId\":\"j1-gc-integration-dev-v3\",\"state\":\"ACTIVE\",\"displayName\":\"j1-gc-integration-dev-v3\",\"createTime\":\"2021-05-31T15:15:56.125Z\",\"updateTime\":\"2024-02-27T16:17:58.322524Z\",\"etag\":\"W/\\\"8f56735f6f3d890d\\\"\",\"labels\":{\"firebase\":\"enabled\"}}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:44 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:42.823Z", + "time": 1595, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1595 + } + }, + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1180, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1180, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:44 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:44.430Z", + "time": 217, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 217 + } + }, + { + "_id": "45fdb1f844a17b6d6ec371710879c815", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "dataflow.googleapis.com" + } + ], + "headersSize": 1305, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://dataflow.googleapis.com/v1b3/projects/j1-gc-integration-dev-v3/jobs" + }, + "response": { + "bodySize": 705, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 705, + "text": "{\"jobs\":[{\"id\":\"2024-05-02_05_25_02-17305733872805367843\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-job3\",\"type\":\"JOB_TYPE_BATCH\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-05-02T13:28:54.170517Z\",\"createTime\":\"2024-05-02T12:25:06.098727Z\",\"location\":\"us-central1\",\"jobMetadata\":{\"sdkVersion\":{\"version\":\"2.55.1\",\"versionDisplayName\":\"Apache Beam SDK for Java\",\"sdkSupportStatus\":\"SUPPORTED\"}},\"startTime\":\"2024-05-02T12:25:06.098727Z\"},{\"id\":\"2024-04-29_06_54_21-11420805499054724452\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job2\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T14:01:46.604199Z\",\"createTime\":\"2024-04-29T13:54:22.225266Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:54:22.225266Z\"},{\"id\":\"2024-04-29_06_40_47-701576625030004708\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job1\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T13:46:31.741443Z\",\"createTime\":\"2024-04-29T13:40:48.311738Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:40:48.311738Z\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:46 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:44.654Z", + "time": 1812, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1812 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow_982301565/recording.har b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow_982301565/recording.har new file mode 100644 index 00000000..1d03772c --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow_982301565/recording.har @@ -0,0 +1,250 @@ +{ + "log": { + "_recordingName": "fetch-project-has-google-cloud-dataflow", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1266, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1266, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:40 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:40.852Z", + "time": 184, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 184 + } + }, + { + "_id": "23a717153d5a0e7db6eb50c807895f67", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "cloudresourcemanager.googleapis.com" + } + ], + "headersSize": 1322, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://cloudresourcemanager.googleapis.com/v3/projects/j1-gc-integration-dev-v3" + }, + "response": { + "bodySize": 394, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 394, + "text": "{\"name\":\"projects/167984947943\",\"parent\":\"folders/306329820365\",\"projectId\":\"j1-gc-integration-dev-v3\",\"state\":\"ACTIVE\",\"displayName\":\"j1-gc-integration-dev-v3\",\"createTime\":\"2021-05-31T15:15:56.125Z\",\"updateTime\":\"2024-02-27T16:17:58.322524Z\",\"etag\":\"W/\\\"8f56735f6f3d890d\\\"\",\"labels\":{\"firebase\":\"enabled\"}}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:42 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:41.041Z", + "time": 1557, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1557 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__recordings__/fetchGoogleCloudDataFlowJob_1170190852/recording.har b/src/steps/data-flow/__recordings__/fetchGoogleCloudDataFlowJob_1170190852/recording.har new file mode 100644 index 00000000..c2657659 --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetchGoogleCloudDataFlowJob_1170190852/recording.har @@ -0,0 +1,250 @@ +{ + "log": { + "_recordingName": "fetchGoogleCloudDataFlowJob", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1207, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1207, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:06:43 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:06:43.025Z", + "time": 360, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 360 + } + }, + { + "_id": "ec06e7e115b06887935fb55a273b3303", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "dataflow.googleapis.com" + } + ], + "headersSize": 1305, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://dataflow.googleapis.com/v1b3/projects/j1-gc-integration-dev-v3/jobs" + }, + "response": { + "bodySize": 647, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 647, + "text": "{\"jobs\":[{\"id\":\"2024-05-02_05_25_02-17305733872805367843\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-job3\",\"type\":\"JOB_TYPE_BATCH\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-05-02T13:28:54.170517Z\",\"createTime\":\"2024-05-02T12:25:06.098727Z\",\"location\":\"us-central1\",\"jobMetadata\":{\"sdkVersion\":{\"version\":\"2.55.1\",\"versionDisplayName\":\"Apache Beam SDK for Java\",\"sdkSupportStatus\":\"SUPPORTED\"}},\"startTime\":\"2024-05-02T12:25:06.098727Z\"},{\"id\":\"2024-04-29_06_54_21-11420805499054724452\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job2\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T14:01:46.604199Z\",\"createTime\":\"2024-04-29T13:54:22.225266Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:54:22.225266Z\"},{\"id\":\"2024-04-29_06_40_47-701576625030004708\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job1\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T13:46:31.741443Z\",\"createTime\":\"2024-04-29T13:40:48.311738Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:40:48.311738Z\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:06:45 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:06:43.402Z", + "time": 1846, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1846 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__snapshots__/index.test.ts.snap b/src/steps/data-flow/__snapshots__/index.test.ts.snap new file mode 100644 index 00000000..b66c9349 --- /dev/null +++ b/src/steps/data-flow/__snapshots__/index.test.ts.snap @@ -0,0 +1,155 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`#fetchGoogleCloudDataFlowDataStore should collect data 1`] = ` +{ + "collectedEntities": [], + "collectedRelationships": [], + "encounteredTypes": [], + "numCollectedEntities": 0, + "numCollectedRelationships": 0, +} +`; + +exports[`#fetchGoogleCloudDataFlowJob should collect data 1`] = ` +{ + "collectedEntities": [ + { + "_class": [ + "Workflow", + ], + "_key": "2024-05-02_05_25_02-17305733872805367843", + "_rawData": [ + { + "name": "default", + "rawData": { + "createTime": "2024-05-02T12:25:06.098727Z", + "currentState": "JOB_STATE_FAILED", + "currentStateTime": "2024-05-02T13:28:54.170517Z", + "id": "2024-05-02_05_25_02-17305733872805367843", + "jobMetadata": { + "sdkVersion": { + "sdkSupportStatus": "SUPPORTED", + "version": "2.55.1", + "versionDisplayName": "Apache Beam SDK for Java", + }, + }, + "location": "us-central1", + "name": "test-job3", + "projectId": "j1-gc-integration-dev-v3", + "startTime": "2024-05-02T12:25:06.098727Z", + "type": "JOB_TYPE_BATCH", + }, + }, + ], + "_type": "google_cloud_dataflow_job", + "createdOn": undefined, + "description": undefined, + "displayName": "test-job3", + "id": "2024-05-02_05_25_02-17305733872805367843", + "name": "test-job3", + }, + { + "_class": [ + "Workflow", + ], + "_key": "2024-04-29_06_54_21-11420805499054724452", + "_rawData": [ + { + "name": "default", + "rawData": { + "createTime": "2024-04-29T13:54:22.225266Z", + "currentState": "JOB_STATE_FAILED", + "currentStateTime": "2024-04-29T14:01:46.604199Z", + "id": "2024-04-29_06_54_21-11420805499054724452", + "location": "us-central1", + "name": "test-metron-job2", + "projectId": "j1-gc-integration-dev-v3", + "startTime": "2024-04-29T13:54:22.225266Z", + }, + }, + ], + "_type": "google_cloud_dataflow_job", + "createdOn": undefined, + "description": undefined, + "displayName": "test-metron-job2", + "id": "2024-04-29_06_54_21-11420805499054724452", + "name": "test-metron-job2", + }, + { + "_class": [ + "Workflow", + ], + "_key": "2024-04-29_06_40_47-701576625030004708", + "_rawData": [ + { + "name": "default", + "rawData": { + "createTime": "2024-04-29T13:40:48.311738Z", + "currentState": "JOB_STATE_FAILED", + "currentStateTime": "2024-04-29T13:46:31.741443Z", + "id": "2024-04-29_06_40_47-701576625030004708", + "location": "us-central1", + "name": "test-metron-job1", + "projectId": "j1-gc-integration-dev-v3", + "startTime": "2024-04-29T13:40:48.311738Z", + }, + }, + ], + "_type": "google_cloud_dataflow_job", + "createdOn": undefined, + "description": undefined, + "displayName": "test-metron-job1", + "id": "2024-04-29_06_40_47-701576625030004708", + "name": "test-metron-job1", + }, + ], + "collectedRelationships": [], + "encounteredTypes": [ + "google_cloud_dataflow_job", + ], + "numCollectedEntities": 3, + "numCollectedRelationships": 0, +} +`; + +exports[`#fetchGoogleCloudDataFlowService should collect data 1`] = ` +{ + "collectedEntities": [ + { + "_class": [ + "Service", + ], + "_key": "google_cloud_dataflow:j1-gc-integration-dev-v3", + "_rawData": [], + "_type": "google_cloud_dataflow", + "category": [ + "network", + "security", + ], + "createdOn": undefined, + "displayName": "Google Cloud Data Flow", + "function": [ + "workflow", + "networking", + ], + "name": "Google Cloud Data Flow", + }, + ], + "collectedRelationships": [], + "encounteredTypes": [ + "google_cloud_dataflow", + ], + "numCollectedEntities": 1, + "numCollectedRelationships": 0, +} +`; + +exports[`#fetchGoogleCloudDataFlowSnapshot should collect data 1`] = ` +{ + "collectedEntities": [], + "collectedRelationships": [], + "encounteredTypes": [], + "numCollectedEntities": 0, + "numCollectedRelationships": 0, +} +`; diff --git a/src/steps/data-flow/index.ts b/src/steps/data-flow/index.ts index 4d5f8d52..b1aa7096 100644 --- a/src/steps/data-flow/index.ts +++ b/src/steps/data-flow/index.ts @@ -30,6 +30,8 @@ import { STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_CLASS, + STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, } from './constants'; import { createGoogleCloudDataFlowEntity, @@ -40,6 +42,7 @@ import { import { PROJECT_ENTITY_TYPE, STEP_RESOURCE_MANAGER_PROJECT } from '../resource-manager'; import { getProjectEntity } from '../../utils/project'; import { dataflow_v1b3 } from 'googleapis'; +import { ENTITY_TYPE_PUBSUB_TOPIC } from '../pub-sub/constants'; export async function fetchGoogleCloudDataFlowDataStore( context: IntegrationStepContext, @@ -374,5 +377,22 @@ export const dataFlowSteps: GoogleCloudIntegrationStep[] = [ executionHandler: fetchGoogleCloudDataFlowSnapshot, apis: ['dataflow.googleapis.com'], }, + // { + // id: STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + // ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + // name: 'Google Cloud Dataflow Snapshot uses Google pubsub', + // entities: [], + // relationships: [ + // { + // _class: RelationshipClass.HAS, + // _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + // sourceType: GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, + // targetType: ENTITY_TYPE_PUBSUB_TOPIC, + // }, + // ], + // dependsOn: [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT], + // executionHandler: buildCloudDataflow, + // apis: ['dataflow.googleapis.com'], + // }, ]; diff --git a/src/steps/data-flow/relationship.test.ts b/src/steps/data-flow/relationship.test.ts index 5b31b32a..6acb53cf 100644 --- a/src/steps/data-flow/relationship.test.ts +++ b/src/steps/data-flow/relationship.test.ts @@ -3,7 +3,7 @@ import { Recording, StepTestConfig, } from '@jupiterone/integration-sdk-testing'; - import { invocationConfig } from '../../..'; + import { invocationConfig } from '../..'; import { integrationConfig } from '../../../test/config'; import { getMatchRequestsBy, From 761fff44308ab2d1ff8944524b131ce5eeb8f5b0 Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Mon, 13 May 2024 11:21:06 +0530 Subject: [PATCH 4/6] added more details in converter --- src/steps/data-flow/converters.ts | 27 +++++++++++++++++++++++++-- src/steps/data-flow/index.ts | 4 ++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/steps/data-flow/converters.ts b/src/steps/data-flow/converters.ts index 67c29ffb..3a01478c 100644 --- a/src/steps/data-flow/converters.ts +++ b/src/steps/data-flow/converters.ts @@ -24,7 +24,6 @@ export function createGoogleCloudDataFlowEntity(serviceObj) { export function createGoogleCloudDataFlowJobEntity( data: dataflow_v1b3.Schema$Job, - projectId: string, ) { return createGoogleCloudIntegrationEntity(data, { entityData: { @@ -37,6 +36,23 @@ export function createGoogleCloudDataFlowJobEntity( name: data.name, displayName: data.name as string, description: data.environment as string, + projectId: data.projectId, + type: data.type, + stepLocation: data.stepsLocation, + currentState: data.currentState, + currentStateTime: data.currentStateTime, + requestedState: data.requestedState, + createTime: data.createTime, + replaceJobId: data.replaceJobId, + clientRequestId: data.clientRequestId, + location: data.location, + version: data.jobMetadata?.sdkVersion?.version, + startTime: data.startTime, + createFromSnapshotId: data.createdFromSnapshotId, + satisfiesPzs: data.satisfiesPzs, + satisfiesPzi: data.satisfiesPzi, + maxNumWorkers: data.runtimeUpdatableParams?.maxNumWorkers, + minNumWorkers: data.runtimeUpdatableParams?.minNumWorkers }, }, }); @@ -75,7 +91,14 @@ export function createGoogleCloudDataFlowSnapshotEntity( name: data.diskSizeBytes, classification: data.diskSizeBytes, encrypted: false, - jobId: jobId + sourceJobId: data.sourceJobId, + jobId: jobId, + creationTime: data.creationTime, + ttl: data.ttl, + state: data.state, + description: data.description, + region: data.region, + diskSizeBytes: data.diskSizeBytes, }, }, }); diff --git a/src/steps/data-flow/index.ts b/src/steps/data-flow/index.ts index b1aa7096..1777c36f 100644 --- a/src/steps/data-flow/index.ts +++ b/src/steps/data-flow/index.ts @@ -75,7 +75,7 @@ export async function fetchGoogleCloudDataFlowJob( const client = new dataFlowClient({ config }, logger); await client.iterateGoogleCloudDataFlowJob(async (dataflow) => { - await jobState.addEntity(createGoogleCloudDataFlowJobEntity(dataflow, client.projectId)); + await jobState.addEntity(createGoogleCloudDataFlowJobEntity(dataflow)); }); } @@ -113,7 +113,7 @@ export async function fetchGoogleCloudDataFlowSnapshot( } } }, - ); + ); } From e2107db9c41a3a5cc72acc517ce95dd308a7dde6 Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Wed, 22 May 2024 18:25:28 +0530 Subject: [PATCH 5/6] added relationships --- src/getStepStartStates.ts | 3 + src/steps/data-flow/client.ts | 38 +++++---- src/steps/data-flow/constants.ts | 13 +++- src/steps/data-flow/index.ts | 128 ++++++++++++++++++++++++++++--- 4 files changed, 154 insertions(+), 28 deletions(-) diff --git a/src/getStepStartStates.ts b/src/getStepStartStates.ts index 097ceac1..80f39698 100644 --- a/src/getStepStartStates.ts +++ b/src/getStepStartStates.ts @@ -211,6 +211,7 @@ import { STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, // STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + STEP_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE, STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE } from './steps/data-flow/constants'; @@ -353,6 +354,7 @@ function getDefaultStepStartStates(params: { // [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC]: { disabled: false }, [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW]: { disabled: false }, [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE]: { disabled: false }, [STEP_COMPUTE_DISKS]: { disabled: false }, [STEP_COMPUTE_REGION_DISKS]: { disabled: false }, [STEP_COMPUTE_IMAGES]: { disabled: false }, @@ -678,6 +680,7 @@ async function getStepStartStatesUsingServiceEnablements(params: { // [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC]: createStepStartState(ServiceUsageName.DATA_FLOW), [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW]: createStepStartState(ServiceUsageName.DATA_FLOW), [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE]: createStepStartState(ServiceUsageName.DATA_FLOW), [STEP_COMPUTE_DISKS]: createStepStartState(ServiceUsageName.COMPUTE), [STEP_COMPUTE_REGION_DISKS]: createStepStartState(ServiceUsageName.COMPUTE), [STEP_COMPUTE_IMAGES]: createStepStartState(ServiceUsageName.COMPUTE), diff --git a/src/steps/data-flow/client.ts b/src/steps/data-flow/client.ts index 5c55da8e..2f11fffc 100644 --- a/src/steps/data-flow/client.ts +++ b/src/steps/data-flow/client.ts @@ -3,8 +3,8 @@ import { Client } from '../../google-cloud/client'; import { DataFlowPermissions, STEP_GOOGLE_CLOUD_DATAFLOW, - } from './constants'; +import { googleCloudRegions } from '../../google-cloud/regions' export class dataFlowClient extends Client { private client = google.dataflow({ version: 'v1b3', retry: false }); @@ -14,21 +14,25 @@ export class dataFlowClient extends Client { ) { const auth = await this.getAuthenticatedServiceClient(); - await this.iterateApi( - async (nextPageToken) => { - return this.client.projects.jobs.list({ - projectId: this.projectId, - auth, - pageToken: nextPageToken, - }); - }, - async (data: dataflow_v1b3.Schema$ListJobsResponse) => { - for (const job of data.jobs || []) { - await callback(job); - } - }, - STEP_GOOGLE_CLOUD_DATAFLOW, - DataFlowPermissions.STEP_GOOGLE_CLOUD_DATAFLOW_JOB, - ); + // Iterate over each region + for (const region of googleCloudRegions) { + await this.iterateApi( + async (nextPageToken) => { + return this.client.projects.locations.jobs.list({ + projectId: this.projectId, + location: region.name, + auth, + pageToken: nextPageToken, + }); + }, + async (data: dataflow_v1b3.Schema$ListJobsResponse) => { + for (const job of data.jobs || []) { + await callback(job); + } + }, + STEP_GOOGLE_CLOUD_DATAFLOW, + DataFlowPermissions.STEP_GOOGLE_CLOUD_DATAFLOW_JOB, + ); + } } } diff --git a/src/steps/data-flow/constants.ts b/src/steps/data-flow/constants.ts index 3c736888..07d083a4 100644 --- a/src/steps/data-flow/constants.ts +++ b/src/steps/data-flow/constants.ts @@ -39,6 +39,11 @@ export const STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT = export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT = 'google_cloud_dataflow_job_uses_google_cloud_dataflow_snapshot'; +export const STEP_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE = + 'fetch-google-cloud-dataflow-uses-google-spanner-instance'; +export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE = + 'google_cloud_dataflow_uses_google_spanner_instance'; + export const STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC = 'fetch-google-cloud-dataflow-job-uses-google-pubsub-topic'; export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC = @@ -54,7 +59,8 @@ export const IngestionSources = { GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB: 'google-cloud-project-has-google-cloud-dataflow-job', GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE: 'google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore', GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT: 'google-cloud-dataflow-job-has-google-cloud-dataflow-snapshot', - GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC: 'google-cloud-dataflow-snapshot-uses-google-pubsub-topic' + GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC: 'google-cloud-dataflow-snapshot-uses-google-pubsub-topic', + GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE: 'google-cloud-dataflow-uses-google-spanner-instance' }; export const DataflowIngestionConfig = { @@ -107,6 +113,11 @@ export const DataflowIngestionConfig = { title: 'Google Cloud Dataflow Snapshot Uses Google Pub/Sub Topic', description: '', defaultsToDisabled: false, + }, + [IngestionSources.GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE]: { + title: 'Google Cloud Dataflow Uses Google Spanner Instance', + description: '', + defaultsToDisabled: false, } }; diff --git a/src/steps/data-flow/index.ts b/src/steps/data-flow/index.ts index 1777c36f..3d0cca20 100644 --- a/src/steps/data-flow/index.ts +++ b/src/steps/data-flow/index.ts @@ -32,6 +32,10 @@ import { GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_CLASS, STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + STEP_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE, + RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE } from './constants'; import { createGoogleCloudDataFlowEntity, @@ -42,7 +46,8 @@ import { import { PROJECT_ENTITY_TYPE, STEP_RESOURCE_MANAGER_PROJECT } from '../resource-manager'; import { getProjectEntity } from '../../utils/project'; import { dataflow_v1b3 } from 'googleapis'; -import { ENTITY_TYPE_PUBSUB_TOPIC } from '../pub-sub/constants'; +import { ENTITY_TYPE_PUBSUB_TOPIC, STEP_PUBSUB_TOPICS } from '../pub-sub/constants'; +import { ENTITY_TYPE_SPANNER_INSTANCE, STEP_SPANNER_INSTANCES } from '../spanner/constants'; export async function fetchGoogleCloudDataFlowDataStore( context: IntegrationStepContext, @@ -191,6 +196,7 @@ export async function buildProjectHasDataflowJobRelationship( }, ); } + export async function buildDataflowUsesDataflowDatastoreRelationship( executionContext: IntegrationStepContext, ) { @@ -224,6 +230,74 @@ export async function buildDataflowUsesDataflowDatastoreRelationship( ); } +export async function buildCloudDataflowSpannerInstanceRelation( + executionContext: IntegrationStepContext, +) { + const { jobState } = executionContext; + await jobState.iterateEntities( + + { _type: ENTITY_TYPE_SPANNER_INSTANCE }, + async (cloudSpannerInstanceEntity) => { + + const dataflowJobKey = cloudSpannerInstanceEntity.name as string + + const hasDataflowJobKey = jobState.hasKey(dataflowJobKey); + + if (!hasDataflowJobKey) { + throw new IntegrationMissingKeyError( + `Cannot build Relationship. + Error: Missing Key. + dataflowJobKey : ${dataflowJobKey}`, + ); + } + + await jobState.addRelationship( + createDirectRelationship({ + _class: RelationshipClass.USES, + fromKey: dataflowJobKey, + fromType: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, + toKey: cloudSpannerInstanceEntity._key, + toType: ENTITY_TYPE_SPANNER_INSTANCE, + }), + ); + }, + ); +} + +export async function buildCloudDataflowSnapshotPubsubTopicRelation( + executionContext: IntegrationStepContext, +) { + const { jobState } = executionContext; + await jobState.iterateEntities( + + { _type: GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE }, + async (snapshotEntity) => { + + const pubsubKey = snapshotEntity.pubsubName as string + + const haspubsubKey = jobState.hasKey(pubsubKey); + + if (!haspubsubKey) { + throw new IntegrationMissingKeyError( + `Cannot build Relationship. + Error: Missing Key. + haspubsubKey : ${haspubsubKey}`, + ); + } + + await jobState.addRelationship( + createDirectRelationship({ + _class: RelationshipClass.USES, + fromKey: snapshotEntity._key, + fromType: GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, + toKey: pubsubKey, + toType: ENTITY_TYPE_PUBSUB_TOPIC, + }), + ); + }, + ); +} + export const dataFlowSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_GOOGLE_CLOUD_DATAFLOW, @@ -377,21 +451,55 @@ export const dataFlowSteps: GoogleCloudIntegrationStep[] = [ executionHandler: fetchGoogleCloudDataFlowSnapshot, apis: ['dataflow.googleapis.com'], }, + { + id: STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + name: 'Google Cloud Dataflow Snapshot uses Google pubsub', + entities: [], + relationships: [ + { + _class: RelationshipClass.HAS, + _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + sourceType: GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, + targetType: ENTITY_TYPE_PUBSUB_TOPIC, + }, + ], + dependsOn: [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, STEP_PUBSUB_TOPICS], + executionHandler: buildCloudDataflowSnapshotPubsubTopicRelation, + apis: ['dataflow.googleapis.com'], + }, + { + id: STEP_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE, + ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE, + name: 'Google Cloud Dataflow uses Google Spanner Instance', + entities: [], + relationships: [ + { + _class: RelationshipClass.USES, + _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE, + sourceType: GOOGLE_CLOUD_DATAFLOW_TYPE, + targetType: ENTITY_TYPE_SPANNER_INSTANCE, + }, + ], + dependsOn: [STEP_SPANNER_INSTANCES, STEP_GOOGLE_CLOUD_DATAFLOW], + executionHandler: buildCloudDataflowSpannerInstanceRelation, + apis: ['dataflow.googleapis.com'], + }, // { - // id: STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, - // ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, - // name: 'Google Cloud Dataflow Snapshot uses Google pubsub', + // id: STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + // ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + // name: 'Google Cloud Dataflow Has Google Cloud DataFlow SNapshot', // entities: [], // relationships: [ // { - // _class: RelationshipClass.HAS, - // _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, - // sourceType: GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, - // targetType: ENTITY_TYPE_PUBSUB_TOPIC, + // _class: RelationshipClass.USES, + // _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + // sourceType: GOOGLE_CLOUD_DATAFLOW_TYPE, + // targetType: ENTITY_TYPE_SPANNER_INSTANCE, // }, // ], - // dependsOn: [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT], - // executionHandler: buildCloudDataflow, + // dependsOn: [STEP_SPANNER_INSTANCES, STEP_GOOGLE_CLOUD_DATAFLOW], + // executionHandler: buildCloudDataflowSnapshoteRelation, // apis: ['dataflow.googleapis.com'], // }, ]; From 2d5e348c0f4a365c920a9b220db4486a56f20612 Mon Sep 17 00:00:00 2001 From: sarthak-metron Date: Tue, 11 Jun 2024 10:42:07 +0530 Subject: [PATCH 6/6] updated the code as data is coming is mock data --- src/steps/data-flow/client.ts | 32 ++++++++++++- src/steps/data-flow/constants.ts | 11 +++-- src/steps/data-flow/converters.ts | 8 ++-- src/steps/data-flow/index.ts | 79 +++++++++++++++---------------- 4 files changed, 80 insertions(+), 50 deletions(-) diff --git a/src/steps/data-flow/client.ts b/src/steps/data-flow/client.ts index 2f11fffc..fbdb31e7 100644 --- a/src/steps/data-flow/client.ts +++ b/src/steps/data-flow/client.ts @@ -15,12 +15,12 @@ export class dataFlowClient extends Client { const auth = await this.getAuthenticatedServiceClient(); // Iterate over each region - for (const region of googleCloudRegions) { + //for (const region of googleCloudRegions) { await this.iterateApi( async (nextPageToken) => { return this.client.projects.locations.jobs.list({ projectId: this.projectId, - location: region.name, + location: 'us-east1', auth, pageToken: nextPageToken, }); @@ -33,6 +33,34 @@ export class dataFlowClient extends Client { STEP_GOOGLE_CLOUD_DATAFLOW, DataFlowPermissions.STEP_GOOGLE_CLOUD_DATAFLOW_JOB, ); + //} + } + + async iterateGoogleCloudDataFlowSnapshot( + callback: (data: dataflow_v1b3.Schema$Snapshot) => Promise, + ) { + const auth = await this.getAuthenticatedServiceClient(); + + // Iterate over each region + for (const region of googleCloudRegions) { + await this.iterateApi( + async () => { + const req=await this.client.projects.locations.snapshots.list({ + projectId: this.projectId, + location: region.name, + auth, + }); + console.log(req+" .........................") + return req + }, + async (data: dataflow_v1b3.Schema$ListSnapshotsResponse) => { + for (const snapshot of data.snapshots || []) { + await callback(snapshot); + } + }, + STEP_GOOGLE_CLOUD_DATAFLOW, + DataFlowPermissions.STEP_GOOGLE_CLOUD_DATAFLOW_JOB, + ); } } } diff --git a/src/steps/data-flow/constants.ts b/src/steps/data-flow/constants.ts index 07d083a4..881d5061 100644 --- a/src/steps/data-flow/constants.ts +++ b/src/steps/data-flow/constants.ts @@ -1,9 +1,9 @@ export const STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE = 'fetch-google-cloud-dataflow-datastore'; -export const GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS = 'Datastore'; +export const GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS = ['DataStore']; export const GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE = 'google_cloud_dataflow_datastore'; export const STEP_GOOGLE_CLOUD_DATAFLOW = 'fetch-google-cloud-dataflow'; -export const GOOGLE_CLOUD_DATAFLOW_CLASS = 'Service'; +export const GOOGLE_CLOUD_DATAFLOW_CLASS = ['Service']; export const GOOGLE_CLOUD_DATAFLOW_TYPE = 'google_cloud_dataflow'; export const STEP_GOOGLE_CLOUD_DATAFLOW_JOB = 'fetch-google-cloud-dataflow-job'; @@ -11,13 +11,13 @@ export const GOOGLE_CLOUD_DATAFLOW_JOB_CLASS = ['Workflow']; export const GOOGLE_CLOUD_DATAFLOW_JOB_TYPE = 'google_cloud_dataflow_job'; export const STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT = 'fetch-google-cloud-dataflow-snapshot'; -export const GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_CLASS = 'Database, DataStore, Image, Backup'; +export const GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_CLASS = ['Database', 'DataStore', 'Image', 'Backup']; export const GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE = 'google_cloud_dataflow_snapshot'; export const STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE = 'fetch-project-has-google-cloud-dataflow-datastore'; export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE = - 'google_cloud_project_has_google_cloud_dataflow_datastore'; + 'google_cloud_project_has_dataflow_datastore'; export const STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW = 'fetch-project-has-google-cloud-dataflow'; @@ -32,7 +32,7 @@ export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JO export const STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE = 'fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore'; export const RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE = - 'google_cloud_dataflow_job_uses_google_cloud_dataflow_datastore'; + 'google_cloud_dataflow_job_uses_datastore'; export const STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT = 'fetch-google-cloud-dataflow-job-has-google-cloud-dataflow-snapshot'; @@ -123,4 +123,5 @@ export const DataflowIngestionConfig = { export const DataFlowPermissions = { STEP_GOOGLE_CLOUD_DATAFLOW_JOB: ['dataflow.jobs.list'], + STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT: ['dataflow.snapshots.list'] }; diff --git a/src/steps/data-flow/converters.ts b/src/steps/data-flow/converters.ts index 3a01478c..38e8bc6f 100644 --- a/src/steps/data-flow/converters.ts +++ b/src/steps/data-flow/converters.ts @@ -35,7 +35,7 @@ export function createGoogleCloudDataFlowJobEntity( id: data.id as string, name: data.name, displayName: data.name as string, - description: data.environment as string, + description: data.jobMetadata?.pubsubDetails as string[], projectId: data.projectId, type: data.type, stepLocation: data.stepsLocation, @@ -52,7 +52,8 @@ export function createGoogleCloudDataFlowJobEntity( satisfiesPzs: data.satisfiesPzs, satisfiesPzi: data.satisfiesPzi, maxNumWorkers: data.runtimeUpdatableParams?.maxNumWorkers, - minNumWorkers: data.runtimeUpdatableParams?.minNumWorkers + minNumWorkers: data.runtimeUpdatableParams?.minNumWorkers, + datastoreDetails:data.jobMetadata?.datastoreDetails as string[], }, }, }); @@ -65,12 +66,13 @@ export function createGoogleCloudDataFlowDataStoreEntity( entityData: { source: data, assign: { - _key: data.namespace as string, + _key: (data.namespace+"/"+data.projectId) as string, _type: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, _class: GOOGLE_CLOUD_DATAFLOW_DATASTORE_CLASS, projectId: data.projectId as string, name: data.namespace, encrypted: false, + classification: 'true' }, }, }); diff --git a/src/steps/data-flow/index.ts b/src/steps/data-flow/index.ts index 3d0cca20..610a0f6e 100644 --- a/src/steps/data-flow/index.ts +++ b/src/steps/data-flow/index.ts @@ -1,6 +1,5 @@ import { createDirectRelationship, - getRawData, IntegrationMissingKeyError, RelationshipClass, } from '@jupiterone/integration-sdk-core'; @@ -32,20 +31,17 @@ import { GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_CLASS, STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, - STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, - RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, STEP_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE, - RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE + RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_USES_GOOGLE_SPANNER_INSTANCE, + GOOGLE_CLOUD_DATAFLOW_JOB_CLASS } from './constants'; import { createGoogleCloudDataFlowEntity, createGoogleCloudDataFlowDataStoreEntity, createGoogleCloudDataFlowJobEntity, - createGoogleCloudDataFlowSnapshotEntity, } from './converters'; import { PROJECT_ENTITY_TYPE, STEP_RESOURCE_MANAGER_PROJECT } from '../resource-manager'; import { getProjectEntity } from '../../utils/project'; -import { dataflow_v1b3 } from 'googleapis'; import { ENTITY_TYPE_PUBSUB_TOPIC, STEP_PUBSUB_TOPICS } from '../pub-sub/constants'; import { ENTITY_TYPE_SPANNER_INSTANCE, STEP_SPANNER_INSTANCES } from '../spanner/constants'; @@ -58,13 +54,16 @@ export async function fetchGoogleCloudDataFlowDataStore( await jobState.iterateEntities( { _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE }, - async (dataflowEntity) => { - const dataflow = getRawData(dataflowEntity); + async (dataflowEntity) => { - if (dataflow?.jobMetadata?.datastoreDetails) { - await jobState.addEntity(createGoogleCloudDataFlowDataStoreEntity(dataflow)); + const dataflowDataStore = dataflowEntity.datastoreDetails as any; + for(const datastore in dataflowDataStore){ + if (datastore) { + const dataflow=dataflowEntity.datastoreDetails[datastore] + await jobState.addEntity(createGoogleCloudDataFlowDataStoreEntity(dataflow as any)); } - }, + } + } ); } @@ -101,24 +100,16 @@ export async function fetchGoogleCloudDataFlowSnapshot( context: IntegrationStepContext, ): Promise { const { - jobState, instance: { config }, logger, } = context; const client = new dataFlowClient({ config }, logger); - await jobState.iterateEntities( - { _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE }, - async (dataflowEntity) => { - const dataflow = getRawData(dataflowEntity); - - if (dataflow?.snapshots) { - for (const snapshot of dataflow.snapshots) { - await jobState.addEntity(createGoogleCloudDataFlowSnapshotEntity(snapshot, client.projectId)); - } - } - }, - ); + console.log(";;;;;;;;") + await client.iterateGoogleCloudDataFlowSnapshot(async (snapshot) => { + console.log(snapshot+"''''''''") + } +) } @@ -133,13 +124,16 @@ export async function buildProjectHasDataflowDatastoreRelationship( await jobState.iterateEntities( { _type: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE }, - async (dataflowJob) => { + async (DataStore) => { + const projectId=projectEntity.projectId as string + + if(DataStore.projectId === projectId) await jobState.addRelationship( createDirectRelationship({ _class: RelationshipClass.HAS, fromKey: projectEntity._key as string, fromType: PROJECT_ENTITY_TYPE, - toKey: dataflowJob._key as string, + toKey: DataStore._key as string, toType: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, }), ); @@ -203,30 +197,35 @@ export async function buildDataflowUsesDataflowDatastoreRelationship( const { jobState } = executionContext; await jobState.iterateEntities( - { _type: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE }, - async (datastoreEntity) => { - const dataflowJobKey = datastoreEntity.jobId as string + { _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE }, + async (dataflowEntity) => { + const dataflowDataStore = dataflowEntity.datastoreDetails as any; + for(const datastore in dataflowDataStore){ - const hasDataflowJobKey = jobState.hasKey(dataflowJobKey); + const dataflow=dataflowEntity.datastoreDetails[datastore] + const dataStoreKey = (dataflow.namespace+"/"+dataflow.projectId) as string - if (!hasDataflowJobKey) { + const hasDataStoreKey = jobState.hasKey(dataStoreKey); + + if (!hasDataStoreKey) { throw new IntegrationMissingKeyError( `Cannot build Relationship. Error: Missing Key. - dataflowJobKey : ${dataflowJobKey}`, + dataflowJobKey : ${dataStoreKey}`, ); } - + else{ await jobState.addRelationship( createDirectRelationship({ - _class: RelationshipClass.HAS, - fromKey: dataflowJobKey, + _class: RelationshipClass.USES, + fromKey: dataflowEntity._key as string, fromType: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, - toKey: datastoreEntity._key, + toKey: dataStoreKey as string, toType: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, }), ); - }, + }} + } ); } @@ -323,7 +322,7 @@ export const dataFlowSteps: GoogleCloudIntegrationStep[] = [ { resourceName: 'Google Cloud Dataflow Job', _type: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, - _class: ['Workflow'], + _class: GOOGLE_CLOUD_DATAFLOW_JOB_CLASS, }, ], relationships: [], @@ -421,7 +420,7 @@ export const dataFlowSteps: GoogleCloudIntegrationStep[] = [ entities: [], relationships: [ { - _class: RelationshipClass.HAS, + _class: RelationshipClass.USES, _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, sourceType: GOOGLE_CLOUD_DATAFLOW_JOB_TYPE, targetType: GOOGLE_CLOUD_DATAFLOW_DATASTORE_TYPE, @@ -447,7 +446,7 @@ export const dataFlowSteps: GoogleCloudIntegrationStep[] = [ }, ], relationships: [], - dependsOn: [STEP_GOOGLE_CLOUD_DATAFLOW_JOB], + dependsOn: [], executionHandler: fetchGoogleCloudDataFlowSnapshot, apis: ['dataflow.googleapis.com'], },