Merge pull request #261 from sudiptob2/feat/260/cloud-domains-crawler

Feat/260/cloud domains crawler

example_config

@@ -92,5 +92,8 @@
   },
   "datastore_kinds": {
     "fetch": true
+  },
+  "registered_domains": {
+    "fetch": true
   }
 }

src/gcp_scanner/client/client_factory.py

@@ -22,6 +22,7 @@
 from gcp_scanner.client.compute_client import ComputeClient
 from gcp_scanner.client.datastore_client import DatastoreClient
 from gcp_scanner.client.dns_client import DNSClient
+from gcp_scanner.client.domains_client import DomainsClient
 from gcp_scanner.client.filestore_client import FilestoreClient
 from gcp_scanner.client.firestore_client import FirestoreClient
 from gcp_scanner.client.iam_client import IAMClient
@@ -47,6 +48,7 @@ class ClientFactory:
     "cloudresourcemanager": CloudResourceManagerClient,
     "compute": ComputeClient,
     "datastore": DatastoreClient,
+    "domains": DomainsClient,
     "dns": DNSClient,
     "firestore": FirestoreClient,
     "file": FilestoreClient,

src/gcp_scanner/client/domains_client.py

@@ -0,0 +1,38 @@
+#  Copyright 2023 Google LLC
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from googleapiclient import discovery
+from httplib2 import Credentials
+
+from .interface_client import IClient
+
+
+class DomainsClient(IClient):
+  """Cloud DomainsClient class."""
+
+  def get_service(self, credentials: Credentials) -> discovery.Resource:
+    """Get discovery service for cloud domains resource.
+
+    Args:
+      credentials: An google.oauth2.credentials.Credentials object.
+
+    Returns:
+      An object of discovery.Resource
+    """
+    return discovery.build(
+      "domains",
+      "v1",
+      credentials=credentials,
+      cache_discovery=False,
+    )

src/gcp_scanner/crawler/crawler_factory.py

@@ -31,6 +31,7 @@
 from gcp_scanner.crawler.datastore_crawler import DatastoreCrawler
 from gcp_scanner.crawler.dns_managed_zones_crawler import DNSManagedZonesCrawler
 from gcp_scanner.crawler.dns_policies_crawler import DNSPoliciesCrawler
+from gcp_scanner.crawler.domains_crawler import DomainsCrawler
 from gcp_scanner.crawler.endpoints_crawler import EndpointsCrawler
 from gcp_scanner.crawler.filestore_instances_crawler import FilestoreInstancesCrawler
 from gcp_scanner.crawler.firestore_collections_crawler import FirestoreCollectionsCrawler
@@ -66,6 +67,7 @@
   "project_info": CloudResourceManagerProjectInfoCrawler,
   "project_list": CloudResourceManagerProjectListCrawler,
   "pubsub_subs": PubSubSubscriptionsCrawler,
+  "registered_domains": DomainsCrawler,
   "services": ServiceUsageCrawler,
   "service_accounts": ServiceAccountsCrawler,
   "sourcerepos": CloudSourceRepoCrawler,

src/gcp_scanner/crawler/domains_crawler.py

@@ -0,0 +1,55 @@
+#  Copyright 2023 Google LLC
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+import logging
+import sys
+from typing import List, Dict, Any, Union
+
+from googleapiclient import discovery
+
+from gcp_scanner.crawler.interface_crawler import ICrawler
+
+
+class DomainsCrawler(ICrawler):
+  """Handle crawling of cloud domains data."""
+
+  def crawl(self, project_name: str, service: discovery.Resource,
+            config: Dict[str, Union[bool, str]] = None) -> List[Dict[str, Any]]:
+    """Retrieve a list of cloud domains available in the project.
+
+     Args:
+         project_name: The name of the project to query information about.
+         service: A resource object for interacting with the GCP API.
+         config: Configuration options for the crawler (Optional).
+
+     Returns:
+         A list of resource objects representing the crawled data.
+     """
+    logging.info("Retrieving list of registered cloud domains names")
+    registered_domains_list = list()
+    parent = f"projects/{project_name}/locations/-"
+    try:
+      request = service.projects().locations().registrations().list(parent=parent)
+      while request is not None:
+        response = request.execute()
+        if response.get("registrations", None) is not None:
+          registered_domains_list.extend([registration['name'] for registration in response.get("registrations")])
+        request = service.projects().locations().registrations().list_next(
+          previous_request=request,
+          previous_response=response,
+        )
+    except Exception:
+      logging.info("Failed to enumerate cloud domains in the %s", project_name)
+      logging.info(sys.exc_info())
+
+    return registered_domains_list

src/gcp_scanner/scanner.py

@@ -82,6 +82,7 @@
   'managed_zones': 'dns',
   'project_info': 'cloudresourcemanager',
   'pubsub_subs': 'pubsub',
+  'registered_domains': 'domains',
   'services': 'serviceusage',
   'service_accounts': 'iam',
   'sourcerepos': 'sourcerepo',

src/gcp_scanner/test_acceptance.py

@@ -22,7 +22,7 @@
 
 from . import scanner
 
-RESOURCE_COUNT = 31
+RESOURCE_COUNT = 32
 RESULTS_JSON_COUNT = 1
 PROJECT_INFO_COUNT = 5
 IAM_POLICY_COUNT = 12
@@ -35,7 +35,7 @@
 FIREWALL_RULES_COUNT = 4
 APP_SERVICES_COUNT = 2
 STORAGE_BUCKETS_COUNT = 5
-MANAGED_ZONES_COUNT = 1
+MANAGED_ZONES_COUNT = 2
 GKE_CLUSTERS_COUNT = 0
 GKE_IMAGES_COUNT = 4
 SQL_INSTANCES_COUNT = 1
@@ -47,7 +47,7 @@
 CLOUD_FUNCTIONS = 1
 ENDPOINTS_COUNT = 0
 KMS_COUNT = 1
-SERVICES_COUNT = 39
+SERVICES_COUNT = 40
 SERVICE_ACCOUNTS_COUNT = 2
 
 

src/gcp_scanner/test_unit.py

@@ -42,6 +42,7 @@
 from .client.compute_client import ComputeClient
 from .client.datastore_client import DatastoreClient
 from .client.dns_client import DNSClient
+from .client.domains_client import DomainsClient
 from .client.filestore_client import FilestoreClient
 from .client.firestore_client import FirestoreClient
 from .client.iam_client import IAMClient
@@ -72,6 +73,7 @@
 from .crawler.datastore_crawler import DatastoreCrawler
 from .crawler.dns_managed_zones_crawler import DNSManagedZonesCrawler
 from .crawler.dns_policies_crawler import DNSPoliciesCrawler
+from .crawler.domains_crawler import DomainsCrawler
 from .crawler.endpoints_crawler import EndpointsCrawler
 from .crawler.filestore_instances_crawler import FilestoreInstancesCrawler
 from .crawler.firestore_collections_crawler import FirestoreCollectionsCrawler
@@ -820,7 +822,20 @@ def test_datastore_kinds(self):
           ClientFactory.get_client("datastore").get_service(self.credentials),
         ),
         "datastore_kinds",
-        False,
+      )
+    )
+
+  def test_cloud_domains(self):
+    """Test Cloud Domains."""
+    self.assertTrue(
+      verify(
+        CrawlerFactory.create_crawler(
+          "registered_domains",
+        ).crawl(
+          PROJECT_NAME,
+          ClientFactory.get_client("domains").get_service(self.credentials),
+        ),
+        "registered_domains",
       )
     )
 
@@ -923,6 +938,11 @@ def test_get_client_datastore(self):
     client = ClientFactory.get_client("datastore")
     self.assertIsInstance(client, DatastoreClient)
 
+  def test_get_client_domains(self):
+    """Test get_client method with 'domains' name."""
+    client = ClientFactory.get_client("domains")
+    self.assertIsInstance(client, DomainsClient)
+
   def test_get_client_invalid(self):
     """Test get_client method with invalid name."""
     with self.assertLogs(level=logging.ERROR) as log:
@@ -1079,6 +1099,11 @@ def test_create_crawler_datastore_kinds(self):
     crawler = CrawlerFactory.create_crawler("datastore_kinds")
     self.assertIsInstance(crawler, DatastoreCrawler)
 
+  def test_create_crawler_registered_domains(self):
+    """Test create_crawler method with 'registered_domains' name."""
+    crawler = CrawlerFactory.create_crawler("registered_domains")
+    self.assertIsInstance(crawler, DomainsCrawler)
+
   def test_create_crawler_invalid(self):
     """Test create_crawler method with invalid name."""
     with self.assertLogs(level=logging.ERROR) as log:

test/datastore_kinds

@@ -1,8 +1,8 @@
 {
   "kinds": [
-    "Person",
-    "Pet",
-    "Toy",
+    CHECK "Person",
+    CHECK "Pet",
+    CHECK "Toy",
     "__Stat_Kind_IsRootEntity__",
     "__Stat_Kind__",
     "__Stat_PropertyName_Kind__",

test/registered_domains

@@ -0,0 +1,3 @@
+[
+CHECK   "projects/test-gcp-scanner-2/locations/global/registrations/gcpscanner.com"
+]

test/services

@@ -466,7 +466,7 @@ CHECK    services/cloudbuild.googleapis.com",
     "parent": "projects/413204024550"
   },
   {
-CHECK    services/clouddebugger.googleapis.com",
+    services/clouddebugger.googleapis.com",  //ignore for the time being
     "config": {
       "name": "clouddebugger.googleapis.com",
       "title": "Cloud Debugger API",