diff --git a/src/getStepStartStates.ts b/src/getStepStartStates.ts index 5db07da4..097ceac1 100644 --- a/src/getStepStartStates.ts +++ b/src/getStepStartStates.ts @@ -201,7 +201,18 @@ import { WebSecurityScannerSteps } from './steps/web-security-scanner/constants' import { IntegrationConfig } from './types'; import { isMasterOrganizationInstance } from './utils/isMasterOrganizationInstance'; import { isSingleProjectInstance } from './utils/isSingleProjectInstance'; -import { STEP_GOOGLE_CLOUD_DATAFLOW_JOB } from './steps/data-flow/constants'; +import { + STEP_GOOGLE_CLOUD_DATAFLOW_JOB, + STEP_GOOGLE_CLOUD_DATAFLOW, + STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, + STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE, + // STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW, + STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE +} from './steps/data-flow/constants'; function makeStepStartStates( stepIds: string[], @@ -244,9 +255,8 @@ export default async function getStepStartStates( logger.publishEvent({ name: 'integration_config', - description: `Starting Google Cloud integration with service account (email=${ - config.serviceAccountKeyConfig.client_email - }, configureOrganizationProjects=${!!config.configureOrganizationProjects})`, + description: `Starting Google Cloud integration with service account (email=${config.serviceAccountKeyConfig.client_email + }, configureOrganizationProjects=${!!config.configureOrganizationProjects})`, }); const masterOrgInstance = isMasterOrganizationInstance(config); @@ -334,7 +344,15 @@ function getDefaultStepStartStates(params: { [STEP_AUDIT_CONFIG_IAM_POLICY]: { disabled: !!config.configureOrganizationProjects, }, - [STEP_GOOGLE_CLOUD_DATAFLOW_JOB]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: { disabled: false }, + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: { disabled: false }, + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { disabled: false }, + // [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC]: { disabled: false }, + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW]: { disabled: false }, + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: { disabled: false }, [STEP_COMPUTE_DISKS]: { disabled: false }, [STEP_COMPUTE_REGION_DISKS]: { disabled: false }, [STEP_COMPUTE_IMAGES]: { disabled: false }, @@ -651,6 +669,15 @@ async function getStepStartStatesUsingServiceEnablements(params: { ? { disabled: true } : createStepStartState(ServiceUsageName.RESOURCE_MANAGER), [STEP_GOOGLE_CLOUD_DATAFLOW_JOB]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_JOB]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB_HAS_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_DATAFLOW_JOB_USES_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: createStepStartState(ServiceUsageName.DATA_FLOW), + // [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW]: createStepStartState(ServiceUsageName.DATA_FLOW), + [STEP_GOOGLE_CLOUD_PROJECT_HAS_GOOGLE_CLOUD_DATAFLOW_DATASTORE]: createStepStartState(ServiceUsageName.DATA_FLOW), [STEP_COMPUTE_DISKS]: createStepStartState(ServiceUsageName.COMPUTE), [STEP_COMPUTE_REGION_DISKS]: createStepStartState(ServiceUsageName.COMPUTE), [STEP_COMPUTE_IMAGES]: createStepStartState(ServiceUsageName.COMPUTE), diff --git a/src/steps/data-flow/__recordings__/fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore_3496021278/recording.har b/src/steps/data-flow/__recordings__/fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore_3496021278/recording.har new file mode 100644 index 00000000..7869f432 --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore_3496021278/recording.har @@ -0,0 +1,250 @@ +{ + "log": { + "_recordingName": "fetch-google-cloud-dataflow-job-uses-google-cloud-dataflow-datastore", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1222, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1222, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:39 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:38.864Z", + "time": 187, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 187 + } + }, + { + "_id": "45fdb1f844a17b6d6ec371710879c815", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "dataflow.googleapis.com" + } + ], + "headersSize": 1305, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://dataflow.googleapis.com/v1b3/projects/j1-gc-integration-dev-v3/jobs" + }, + "response": { + "bodySize": 763, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 763, + "text": "{\"jobs\":[{\"id\":\"2024-05-02_05_25_02-17305733872805367843\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-job3\",\"type\":\"JOB_TYPE_BATCH\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-05-02T13:28:54.170517Z\",\"createTime\":\"2024-05-02T12:25:06.098727Z\",\"location\":\"us-central1\",\"jobMetadata\":{\"sdkVersion\":{\"version\":\"2.55.1\",\"versionDisplayName\":\"Apache Beam SDK for Java\",\"sdkSupportStatus\":\"SUPPORTED\"}},\"startTime\":\"2024-05-02T12:25:06.098727Z\"},{\"id\":\"2024-04-29_06_54_21-11420805499054724452\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job2\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T14:01:46.604199Z\",\"createTime\":\"2024-04-29T13:54:22.225266Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:54:22.225266Z\"},{\"id\":\"2024-04-29_06_40_47-701576625030004708\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job1\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T13:46:31.741443Z\",\"createTime\":\"2024-04-29T13:40:48.311738Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:40:48.311738Z\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:40 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:39.060Z", + "time": 1774, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1774 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-datastore_1189976537/recording.har b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-datastore_1189976537/recording.har new file mode 100644 index 00000000..f7a91e1c --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-datastore_1189976537/recording.har @@ -0,0 +1,486 @@ +{ + "log": { + "_recordingName": "fetch-project-has-google-cloud-dataflow-datastore", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1231, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1231, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:34 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:34.765Z", + "time": 234, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 234 + } + }, + { + "_id": "23a717153d5a0e7db6eb50c807895f67", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "cloudresourcemanager.googleapis.com" + } + ], + "headersSize": 1322, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://cloudresourcemanager.googleapis.com/v3/projects/j1-gc-integration-dev-v3" + }, + "response": { + "bodySize": 336, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 336, + "text": "{\"name\":\"projects/167984947943\",\"parent\":\"folders/306329820365\",\"projectId\":\"j1-gc-integration-dev-v3\",\"state\":\"ACTIVE\",\"displayName\":\"j1-gc-integration-dev-v3\",\"createTime\":\"2021-05-31T15:15:56.125Z\",\"updateTime\":\"2024-02-27T16:17:58.322524Z\",\"etag\":\"W/\\\"8f56735f6f3d890d\\\"\",\"labels\":{\"firebase\":\"enabled\"}}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:36 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:35.016Z", + "time": 1720, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1720 + } + }, + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1164, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1164, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:36 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:36.782Z", + "time": 185, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 185 + } + }, + { + "_id": "45fdb1f844a17b6d6ec371710879c815", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "dataflow.googleapis.com" + } + ], + "headersSize": 1305, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://dataflow.googleapis.com/v1b3/projects/j1-gc-integration-dev-v3/jobs" + }, + "response": { + "bodySize": 739, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 739, + "text": "{\"jobs\":[{\"id\":\"2024-05-02_05_25_02-17305733872805367843\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-job3\",\"type\":\"JOB_TYPE_BATCH\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-05-02T13:28:54.170517Z\",\"createTime\":\"2024-05-02T12:25:06.098727Z\",\"location\":\"us-central1\",\"jobMetadata\":{\"sdkVersion\":{\"version\":\"2.55.1\",\"versionDisplayName\":\"Apache Beam SDK for Java\",\"sdkSupportStatus\":\"SUPPORTED\"}},\"startTime\":\"2024-05-02T12:25:06.098727Z\"},{\"id\":\"2024-04-29_06_54_21-11420805499054724452\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job2\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T14:01:46.604199Z\",\"createTime\":\"2024-04-29T13:54:22.225266Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:54:22.225266Z\"},{\"id\":\"2024-04-29_06_40_47-701576625030004708\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job1\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T13:46:31.741443Z\",\"createTime\":\"2024-04-29T13:40:48.311738Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:40:48.311738Z\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:38 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:36.977Z", + "time": 1837, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1837 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-job_3299039843/recording.har b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-job_3299039843/recording.har new file mode 100644 index 00000000..32de1426 --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow-job_3299039843/recording.har @@ -0,0 +1,486 @@ +{ + "log": { + "_recordingName": "fetch-project-has-google-cloud-dataflow-job", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1228, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1228, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:42 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:42.622Z", + "time": 192, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 192 + } + }, + { + "_id": "23a717153d5a0e7db6eb50c807895f67", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "cloudresourcemanager.googleapis.com" + } + ], + "headersSize": 1322, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://cloudresourcemanager.googleapis.com/v3/projects/j1-gc-integration-dev-v3" + }, + "response": { + "bodySize": 401, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 401, + "text": "{\"name\":\"projects/167984947943\",\"parent\":\"folders/306329820365\",\"projectId\":\"j1-gc-integration-dev-v3\",\"state\":\"ACTIVE\",\"displayName\":\"j1-gc-integration-dev-v3\",\"createTime\":\"2021-05-31T15:15:56.125Z\",\"updateTime\":\"2024-02-27T16:17:58.322524Z\",\"etag\":\"W/\\\"8f56735f6f3d890d\\\"\",\"labels\":{\"firebase\":\"enabled\"}}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:44 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:42.823Z", + "time": 1595, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1595 + } + }, + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1180, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1180, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:44 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:44.430Z", + "time": 217, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 217 + } + }, + { + "_id": "45fdb1f844a17b6d6ec371710879c815", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "dataflow.googleapis.com" + } + ], + "headersSize": 1305, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://dataflow.googleapis.com/v1b3/projects/j1-gc-integration-dev-v3/jobs" + }, + "response": { + "bodySize": 705, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 705, + "text": "{\"jobs\":[{\"id\":\"2024-05-02_05_25_02-17305733872805367843\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-job3\",\"type\":\"JOB_TYPE_BATCH\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-05-02T13:28:54.170517Z\",\"createTime\":\"2024-05-02T12:25:06.098727Z\",\"location\":\"us-central1\",\"jobMetadata\":{\"sdkVersion\":{\"version\":\"2.55.1\",\"versionDisplayName\":\"Apache Beam SDK for Java\",\"sdkSupportStatus\":\"SUPPORTED\"}},\"startTime\":\"2024-05-02T12:25:06.098727Z\"},{\"id\":\"2024-04-29_06_54_21-11420805499054724452\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job2\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T14:01:46.604199Z\",\"createTime\":\"2024-04-29T13:54:22.225266Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:54:22.225266Z\"},{\"id\":\"2024-04-29_06_40_47-701576625030004708\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job1\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T13:46:31.741443Z\",\"createTime\":\"2024-04-29T13:40:48.311738Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:40:48.311738Z\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:46 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:44.654Z", + "time": 1812, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1812 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow_982301565/recording.har b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow_982301565/recording.har new file mode 100644 index 00000000..1d03772c --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetch-project-has-google-cloud-dataflow_982301565/recording.har @@ -0,0 +1,250 @@ +{ + "log": { + "_recordingName": "fetch-project-has-google-cloud-dataflow", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1266, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1266, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:40 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:40.852Z", + "time": 184, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 184 + } + }, + { + "_id": "23a717153d5a0e7db6eb50c807895f67", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "cloudresourcemanager.googleapis.com" + } + ], + "headersSize": 1322, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://cloudresourcemanager.googleapis.com/v3/projects/j1-gc-integration-dev-v3" + }, + "response": { + "bodySize": 394, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 394, + "text": "{\"name\":\"projects/167984947943\",\"parent\":\"folders/306329820365\",\"projectId\":\"j1-gc-integration-dev-v3\",\"state\":\"ACTIVE\",\"displayName\":\"j1-gc-integration-dev-v3\",\"createTime\":\"2021-05-31T15:15:56.125Z\",\"updateTime\":\"2024-02-27T16:17:58.322524Z\",\"etag\":\"W/\\\"8f56735f6f3d890d\\\"\",\"labels\":{\"firebase\":\"enabled\"}}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:39:42 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:39:41.041Z", + "time": 1557, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1557 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__recordings__/fetchGoogleCloudDataFlowJob_1170190852/recording.har b/src/steps/data-flow/__recordings__/fetchGoogleCloudDataFlowJob_1170190852/recording.har new file mode 100644 index 00000000..c2657659 --- /dev/null +++ b/src/steps/data-flow/__recordings__/fetchGoogleCloudDataFlowJob_1170190852/recording.har @@ -0,0 +1,250 @@ +{ + "log": { + "_recordingName": "fetchGoogleCloudDataFlowJob", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "acea721c8193b51ced888cae721cc423", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 727, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-type", + "value": "application/x-www-form-urlencoded" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/9.4.1" + }, + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-length", + "value": "727" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "www.googleapis.com" + } + ], + "headersSize": 293, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/x-www-form-urlencoded", + "params": [], + "text": "[REDACTED]" + }, + "queryString": [], + "url": "https://www.googleapis.com/oauth2/v4/token" + }, + "response": { + "bodySize": 1207, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 1207, + "text": "{\"access_token\":\"[REDACTED]\",\"expires_in\":9999,\"token_type\":\"Bearer\"}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:06:43 GMT" + }, + { + "name": "server", + "value": "scaffolding on HTTPServer2" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 390, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:06:43.025Z", + "time": 360, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 360 + } + }, + { + "_id": "ec06e7e115b06887935fb55a273b3303", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "x-goog-api-client", + "value": "gdcl/7.0.1 gl-node/18.17.0" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "google-api-nodejs-client/7.0.1 (gzip)" + }, + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "name": "host", + "value": "dataflow.googleapis.com" + } + ], + "headersSize": 1305, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://dataflow.googleapis.com/v1b3/projects/j1-gc-integration-dev-v3/jobs" + }, + "response": { + "bodySize": 647, + "content": { + "encoding": "utf-8", + "mimeType": "application/json; charset=UTF-8", + "size": 647, + "text": "{\"jobs\":[{\"id\":\"2024-05-02_05_25_02-17305733872805367843\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-job3\",\"type\":\"JOB_TYPE_BATCH\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-05-02T13:28:54.170517Z\",\"createTime\":\"2024-05-02T12:25:06.098727Z\",\"location\":\"us-central1\",\"jobMetadata\":{\"sdkVersion\":{\"version\":\"2.55.1\",\"versionDisplayName\":\"Apache Beam SDK for Java\",\"sdkSupportStatus\":\"SUPPORTED\"}},\"startTime\":\"2024-05-02T12:25:06.098727Z\"},{\"id\":\"2024-04-29_06_54_21-11420805499054724452\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job2\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T14:01:46.604199Z\",\"createTime\":\"2024-04-29T13:54:22.225266Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:54:22.225266Z\"},{\"id\":\"2024-04-29_06_40_47-701576625030004708\",\"projectId\":\"j1-gc-integration-dev-v3\",\"name\":\"test-metron-job1\",\"currentState\":\"JOB_STATE_FAILED\",\"currentStateTime\":\"2024-04-29T13:46:31.741443Z\",\"createTime\":\"2024-04-29T13:40:48.311738Z\",\"location\":\"us-central1\",\"startTime\":\"2024-04-29T13:40:48.311738Z\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json; charset=UTF-8" + }, + { + "name": "vary", + "value": "Origin, X-Origin, Referer" + }, + { + "name": "date", + "value": "Tue, 07 May 2024 12:06:45 GMT" + }, + { + "name": "server", + "value": "ESF" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "x-xss-protection", + "value": "0" + }, + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 367, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-07T12:06:43.402Z", + "time": 1846, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1846 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/data-flow/__snapshots__/index.test.ts.snap b/src/steps/data-flow/__snapshots__/index.test.ts.snap new file mode 100644 index 00000000..b66c9349 --- /dev/null +++ b/src/steps/data-flow/__snapshots__/index.test.ts.snap @@ -0,0 +1,155 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`#fetchGoogleCloudDataFlowDataStore should collect data 1`] = ` +{ + "collectedEntities": [], + "collectedRelationships": [], + "encounteredTypes": [], + "numCollectedEntities": 0, + "numCollectedRelationships": 0, +} +`; + +exports[`#fetchGoogleCloudDataFlowJob should collect data 1`] = ` +{ + "collectedEntities": [ + { + "_class": [ + "Workflow", + ], + "_key": "2024-05-02_05_25_02-17305733872805367843", + "_rawData": [ + { + "name": "default", + "rawData": { + "createTime": "2024-05-02T12:25:06.098727Z", + "currentState": "JOB_STATE_FAILED", + "currentStateTime": "2024-05-02T13:28:54.170517Z", + "id": "2024-05-02_05_25_02-17305733872805367843", + "jobMetadata": { + "sdkVersion": { + "sdkSupportStatus": "SUPPORTED", + "version": "2.55.1", + "versionDisplayName": "Apache Beam SDK for Java", + }, + }, + "location": "us-central1", + "name": "test-job3", + "projectId": "j1-gc-integration-dev-v3", + "startTime": "2024-05-02T12:25:06.098727Z", + "type": "JOB_TYPE_BATCH", + }, + }, + ], + "_type": "google_cloud_dataflow_job", + "createdOn": undefined, + "description": undefined, + "displayName": "test-job3", + "id": "2024-05-02_05_25_02-17305733872805367843", + "name": "test-job3", + }, + { + "_class": [ + "Workflow", + ], + "_key": "2024-04-29_06_54_21-11420805499054724452", + "_rawData": [ + { + "name": "default", + "rawData": { + "createTime": "2024-04-29T13:54:22.225266Z", + "currentState": "JOB_STATE_FAILED", + "currentStateTime": "2024-04-29T14:01:46.604199Z", + "id": "2024-04-29_06_54_21-11420805499054724452", + "location": "us-central1", + "name": "test-metron-job2", + "projectId": "j1-gc-integration-dev-v3", + "startTime": "2024-04-29T13:54:22.225266Z", + }, + }, + ], + "_type": "google_cloud_dataflow_job", + "createdOn": undefined, + "description": undefined, + "displayName": "test-metron-job2", + "id": "2024-04-29_06_54_21-11420805499054724452", + "name": "test-metron-job2", + }, + { + "_class": [ + "Workflow", + ], + "_key": "2024-04-29_06_40_47-701576625030004708", + "_rawData": [ + { + "name": "default", + "rawData": { + "createTime": "2024-04-29T13:40:48.311738Z", + "currentState": "JOB_STATE_FAILED", + "currentStateTime": "2024-04-29T13:46:31.741443Z", + "id": "2024-04-29_06_40_47-701576625030004708", + "location": "us-central1", + "name": "test-metron-job1", + "projectId": "j1-gc-integration-dev-v3", + "startTime": "2024-04-29T13:40:48.311738Z", + }, + }, + ], + "_type": "google_cloud_dataflow_job", + "createdOn": undefined, + "description": undefined, + "displayName": "test-metron-job1", + "id": "2024-04-29_06_40_47-701576625030004708", + "name": "test-metron-job1", + }, + ], + "collectedRelationships": [], + "encounteredTypes": [ + "google_cloud_dataflow_job", + ], + "numCollectedEntities": 3, + "numCollectedRelationships": 0, +} +`; + +exports[`#fetchGoogleCloudDataFlowService should collect data 1`] = ` +{ + "collectedEntities": [ + { + "_class": [ + "Service", + ], + "_key": "google_cloud_dataflow:j1-gc-integration-dev-v3", + "_rawData": [], + "_type": "google_cloud_dataflow", + "category": [ + "network", + "security", + ], + "createdOn": undefined, + "displayName": "Google Cloud Data Flow", + "function": [ + "workflow", + "networking", + ], + "name": "Google Cloud Data Flow", + }, + ], + "collectedRelationships": [], + "encounteredTypes": [ + "google_cloud_dataflow", + ], + "numCollectedEntities": 1, + "numCollectedRelationships": 0, +} +`; + +exports[`#fetchGoogleCloudDataFlowSnapshot should collect data 1`] = ` +{ + "collectedEntities": [], + "collectedRelationships": [], + "encounteredTypes": [], + "numCollectedEntities": 0, + "numCollectedRelationships": 0, +} +`; diff --git a/src/steps/data-flow/index.ts b/src/steps/data-flow/index.ts index 4d5f8d52..b1aa7096 100644 --- a/src/steps/data-flow/index.ts +++ b/src/steps/data-flow/index.ts @@ -30,6 +30,8 @@ import { STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT, GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_CLASS, + STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, } from './constants'; import { createGoogleCloudDataFlowEntity, @@ -40,6 +42,7 @@ import { import { PROJECT_ENTITY_TYPE, STEP_RESOURCE_MANAGER_PROJECT } from '../resource-manager'; import { getProjectEntity } from '../../utils/project'; import { dataflow_v1b3 } from 'googleapis'; +import { ENTITY_TYPE_PUBSUB_TOPIC } from '../pub-sub/constants'; export async function fetchGoogleCloudDataFlowDataStore( context: IntegrationStepContext, @@ -374,5 +377,22 @@ export const dataFlowSteps: GoogleCloudIntegrationStep[] = [ executionHandler: fetchGoogleCloudDataFlowSnapshot, apis: ['dataflow.googleapis.com'], }, + // { + // id: STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + // ingestionSourceId: IngestionSources.GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + // name: 'Google Cloud Dataflow Snapshot uses Google pubsub', + // entities: [], + // relationships: [ + // { + // _class: RelationshipClass.HAS, + // _type: RELATIONSHIP_TYPE_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_USES_GOOGLE_PUBSUB_TOPIC, + // sourceType: GOOGLE_CLOUD_DATAFLOW_SNAPSHOT_TYPE, + // targetType: ENTITY_TYPE_PUBSUB_TOPIC, + // }, + // ], + // dependsOn: [STEP_GOOGLE_CLOUD_DATAFLOW_SNAPSHOT], + // executionHandler: buildCloudDataflow, + // apis: ['dataflow.googleapis.com'], + // }, ]; diff --git a/src/steps/data-flow/relationship.test.ts b/src/steps/data-flow/relationship.test.ts index 5b31b32a..6acb53cf 100644 --- a/src/steps/data-flow/relationship.test.ts +++ b/src/steps/data-flow/relationship.test.ts @@ -3,7 +3,7 @@ import { Recording, StepTestConfig, } from '@jupiterone/integration-sdk-testing'; - import { invocationConfig } from '../../..'; + import { invocationConfig } from '../..'; import { integrationConfig } from '../../../test/config'; import { getMatchRequestsBy,