From 7239461191fedbe18e6a9c4c2d35f3f8dc4957aa Mon Sep 17 00:00:00 2001 From: Aryan Agrawal Date: Mon, 11 Dec 2023 01:16:23 +0530 Subject: [PATCH 1/2] Added Support for CloudTrace Resource --- example_config | 3 ++ src/gcp_scanner/client/client_factory.py | 2 + src/gcp_scanner/client/cloud_trace_client.py | 39 ++++++++++++++ .../crawler/cloud_trace_crawler.py | 52 +++++++++++++++++++ src/gcp_scanner/crawler/crawler_factory.py | 2 + src/gcp_scanner/scanner.py | 1 + src/gcp_scanner/test_unit.py | 28 ++++++++++ test/bootstrap/cloudtrace.sh | 4 ++ test/cloud_traces | 1 + 9 files changed, 132 insertions(+) create mode 100644 src/gcp_scanner/client/cloud_trace_client.py create mode 100644 src/gcp_scanner/crawler/cloud_trace_crawler.py create mode 100644 test/bootstrap/cloudtrace.sh create mode 100644 test/cloud_traces diff --git a/example_config b/example_config index 0bc35b7a..b91b3521 100644 --- a/example_config +++ b/example_config @@ -55,6 +55,9 @@ "cloud_functions": { "fetch": true }, + "cloud_traces":{ + "fetch": true + }, "bigtable_instances": { "fetch": true }, diff --git a/src/gcp_scanner/client/client_factory.py b/src/gcp_scanner/client/client_factory.py index ac7e90a6..b17cb449 100644 --- a/src/gcp_scanner/client/client_factory.py +++ b/src/gcp_scanner/client/client_factory.py @@ -19,6 +19,7 @@ from gcp_scanner.client.bigtable_client import BigTableClient from gcp_scanner.client.cloud_functions_client import CloudFunctionsClient from gcp_scanner.client.cloud_resource_manager_client import CloudResourceManagerClient +from gcp_scanner.client.cloud_trace_client import CloudTraceClient from gcp_scanner.client.compute_client import ComputeClient from gcp_scanner.client.datastore_client import DatastoreClient from gcp_scanner.client.dns_client import DNSClient @@ -46,6 +47,7 @@ class ClientFactory: "cloudfunctions": CloudFunctionsClient, "cloudkms": CloudKMSClient, "cloudresourcemanager": CloudResourceManagerClient, + "cloudtrace": CloudTraceClient, "compute": ComputeClient, "datastore": DatastoreClient, "domains": DomainsClient, diff --git a/src/gcp_scanner/client/cloud_trace_client.py b/src/gcp_scanner/client/cloud_trace_client.py new file mode 100644 index 00000000..5e65e876 --- /dev/null +++ b/src/gcp_scanner/client/cloud_trace_client.py @@ -0,0 +1,39 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from googleapiclient import discovery +from httplib2 import Credentials + +from .interface_client import IClient + + +class CloudTraceClient(IClient): + """CloudTraceClient class.""" + + def get_service(self, credentials: Credentials) -> discovery.Resource: + """Get discovery service for cloud trace resource. + + Args: + credentials: An google.oauth2.credentials.Credentials object. + + Returns: + An object of discovery.Resource + """ + return discovery.build( + 'cloudtrace', + 'v1', + credentials=credentials, + cache_discovery=False, + ) diff --git a/src/gcp_scanner/crawler/cloud_trace_crawler.py b/src/gcp_scanner/crawler/cloud_trace_crawler.py new file mode 100644 index 00000000..a705f9cc --- /dev/null +++ b/src/gcp_scanner/crawler/cloud_trace_crawler.py @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +from typing import Dict, Any, Union + +from googleapiclient import discovery + +from gcp_scanner.crawler.interface_crawler import ICrawler + + +class CloudTraceCrawler(ICrawler): + '''Handle crawling of Cloud Trace data.''' + + def crawl(self, project_id: str, service: discovery.Resource, + config: Dict[str, Union[bool, str]] = None) -> Dict[str, Any]: + '''Retrieve a list of Cloud Trace available in the project. + + Args: + project_id: A name of a project to query info about. + service: A resource object for interacting with the Cloud Trace API. + config: Configuration options for the crawler (Optional). + + Returns: + A list of resource objects representing the crawled data. + ''' + + logging.info("Retrieving CloudTrace") + trace_list = list() + try: + request = service.projects().traces().list(projectId=project_id) + while request is not None: + response = request.execute() + trace_list.extend(response.get("functions", [])) + request = service.projects().traces().list_next( + previous_request=request, previous_response=response) + except Exception: + logging.info("Failed to retrieve CloudTrace for project %s", project_id) + logging.info(sys.exc_info()) + return trace_list \ No newline at end of file diff --git a/src/gcp_scanner/crawler/crawler_factory.py b/src/gcp_scanner/crawler/crawler_factory.py index e8e97478..8bd4f03f 100644 --- a/src/gcp_scanner/crawler/crawler_factory.py +++ b/src/gcp_scanner/crawler/crawler_factory.py @@ -18,6 +18,7 @@ from gcp_scanner.crawler.bigquery_crawler import BigQueryCrawler from gcp_scanner.crawler.bigtable_instances_crawler import BigTableInstancesCrawler from gcp_scanner.crawler.cloud_functions_crawler import CloudFunctionsCrawler +from gcp_scanner.crawler.cloud_trace_crawler import CloudTraceCrawler from gcp_scanner.crawler.cloud_resource_manager_iam_policy_crawler import CloudResourceManagerIAMPolicyCrawler from gcp_scanner.crawler.cloud_resource_manager_project_info_crawler import CloudResourceManagerProjectInfoCrawler from gcp_scanner.crawler.cloud_resource_manager_project_list_crawler import CloudResourceManagerProjectListCrawler @@ -50,6 +51,7 @@ "bigtable_instances": BigTableInstancesCrawler, "bq": BigQueryCrawler, "cloud_functions": CloudFunctionsCrawler, + "cloud_traces": CloudTraceCrawler, "compute_disks": ComputeDisksCrawler, "compute_images": ComputeImagesCrawler, "compute_instances": ComputeInstancesCrawler, diff --git a/src/gcp_scanner/scanner.py b/src/gcp_scanner/scanner.py index 7db10319..3514e622 100644 --- a/src/gcp_scanner/scanner.py +++ b/src/gcp_scanner/scanner.py @@ -95,6 +95,7 @@ 'bigtable_instances': 'bigtableadmin', 'bq': 'bigquery', 'cloud_functions': 'cloudfunctions', + 'cloud_traces': 'cloudtrace', 'compute_disks': 'compute', 'compute_images': 'compute', 'compute_instances': 'compute', diff --git a/src/gcp_scanner/test_unit.py b/src/gcp_scanner/test_unit.py index f2401419..ca62298f 100644 --- a/src/gcp_scanner/test_unit.py +++ b/src/gcp_scanner/test_unit.py @@ -38,6 +38,7 @@ from .client.bigtable_client import BigTableClient from .client.client_factory import ClientFactory from .client.cloud_functions_client import CloudFunctionsClient +from .client.cloud_trace_client import CloudTraceClient from .client.cloud_resource_manager_client import CloudResourceManagerClient from .client.compute_client import ComputeClient from .client.datastore_client import DatastoreClient @@ -59,6 +60,7 @@ from .crawler.bigquery_crawler import BigQueryCrawler from .crawler.bigtable_instances_crawler import BigTableInstancesCrawler from .crawler.cloud_functions_crawler import CloudFunctionsCrawler +from .crawler.cloud_trace_crawler import CloudTraceCrawler from .crawler.cloud_resource_manager_iam_policy_crawler import CloudResourceManagerIAMPolicyCrawler from .crawler.cloud_resource_manager_project_info_crawler import CloudResourceManagerProjectInfoCrawler from .crawler.cloud_resource_manager_project_list_crawler import CloudResourceManagerProjectListCrawler @@ -628,6 +630,22 @@ def test_cloud_functions(self): ) ) + def test_cloud_traces(self): + """Test Cloudtrace list""" + self.assertTrue( + verify( + CrawlerFactory.create_crawler( + "cloud_traces", + ).crawl( + PROJECT_NAME, + ClientFactory.get_client("cloudtrace").get_service( + self.credentials + ), + ), + "cloud_traces", + ) + ) + def test_bigtable_instances(self): """Test BigTable Instances.""" self.assertTrue( @@ -883,6 +901,11 @@ def test_get_client_cloudfunctions(self): client = ClientFactory.get_client("cloudfunctions") self.assertIsInstance(client, CloudFunctionsClient) + def test_get_client_cloudtrace(self): + """Test get_client method with 'cloudtrace' name.""" + client = ClientFactory.get_client("cloudtrace") + self.assertIsInstance(client, CloudTraceClient) + def test_get_client_bigtable(self): """Test get_client method with 'bigtableadmin' name.""" client = ClientFactory.get_client("bigtableadmin") @@ -969,6 +992,11 @@ def test_create_crawler_cloud_functions(self): crawler = CrawlerFactory.create_crawler("cloud_functions") self.assertIsInstance(crawler, CloudFunctionsCrawler) + def test_create_crawler_cloud_traces(self): + """Test create_crawler method with 'cloud_traces' name.""" + crawler = CrawlerFactory.create_crawler("cloud_traces") + self.assertIsInstance(crawler, CloudTraceCrawler) + def test_create_crawler_bigtable_instances(self): """Test create_crawler method with 'app_services' name.""" crawler = CrawlerFactory.create_crawler("bigtable_instances") diff --git a/test/bootstrap/cloudtrace.sh b/test/bootstrap/cloudtrace.sh new file mode 100644 index 00000000..91616e73 --- /dev/null +++ b/test/bootstrap/cloudtrace.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Enable the CloudTrace API +gcloud services enable cloudtrace.googleapis.com \ No newline at end of file diff --git a/test/cloud_traces b/test/cloud_traces new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/test/cloud_traces @@ -0,0 +1 @@ +[] \ No newline at end of file From 2f065a11ad47610f235a08f2743417cd4a62b65f Mon Sep 17 00:00:00 2001 From: Aryan Agrawal Date: Sat, 30 Dec 2023 00:33:44 +0530 Subject: [PATCH 2/2] Test: Added tests --- src/gcp_scanner/crawler/cloud_trace_crawler.py | 2 +- test/cloud_traces | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/gcp_scanner/crawler/cloud_trace_crawler.py b/src/gcp_scanner/crawler/cloud_trace_crawler.py index a705f9cc..65c05b08 100644 --- a/src/gcp_scanner/crawler/cloud_trace_crawler.py +++ b/src/gcp_scanner/crawler/cloud_trace_crawler.py @@ -43,7 +43,7 @@ def crawl(self, project_id: str, service: discovery.Resource, request = service.projects().traces().list(projectId=project_id) while request is not None: response = request.execute() - trace_list.extend(response.get("functions", [])) + trace_list.extend(response.get("traces", [])) request = service.projects().traces().list_next( previous_request=request, previous_response=response) except Exception: diff --git a/test/cloud_traces b/test/cloud_traces index 0637a088..074caff8 100644 --- a/test/cloud_traces +++ b/test/cloud_traces @@ -1 +1,6 @@ -[] \ No newline at end of file +[ + { +CHECK "projectId": "cloud-trace-api", +CHECK "traceId": "1a6942a2b3cb71dc20e3dbb7dc07c21f" + } +] \ No newline at end of file