From 4cdfe791516d70dec06534f2beb5253d90cfbaa8 Mon Sep 17 00:00:00 2001 From: oharan2 Date: Sun, 2 Jun 2024 17:33:23 +0300 Subject: [PATCH 1/5] Add datetime to pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 2bb722a7..2395b469 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ dependencies = [ "vspk==5.3.2", "wait_for", "websocket_client", + "datetime", ] [project.optional-dependencies] From 19b920a02f3b50f390cdc1be9f0a8c15134f83e9 Mon Sep 17 00:00:00 2001 From: oharan2 Date: Sun, 2 Jun 2024 17:43:15 +0300 Subject: [PATCH 2/5] Add support for collecting deletable OCPs from AWS --- wrapanapi/const.py | 1 + wrapanapi/systems/ec2.py | 89 ++++++++++++++++++++++++++++++++-------- 2 files changed, 73 insertions(+), 17 deletions(-) diff --git a/wrapanapi/const.py b/wrapanapi/const.py index bc3875b4..92ae09b6 100644 --- a/wrapanapi/const.py +++ b/wrapanapi/const.py @@ -3,3 +3,4 @@ """ CACHED_PROPERTY_TTL = 1 +EC2_INSTANCE = "ec2:instance" diff --git a/wrapanapi/systems/ec2.py b/wrapanapi/systems/ec2.py index 35c4db09..f5d9d93c 100644 --- a/wrapanapi/systems/ec2.py +++ b/wrapanapi/systems/ec2.py @@ -1,6 +1,7 @@ import base64 import os import re +from datetime import datetime import boto3 from boto3 import client as boto3client @@ -8,6 +9,7 @@ from botocore.config import Config from botocore.exceptions import ClientError +from wrapanapi.const import EC2_INSTANCE from wrapanapi.entities import ( Instance, Network, @@ -113,10 +115,17 @@ def __init__(self, system, raw=None, **kwargs): self._api = self.system.ec2_connection + def __repr__(self): + return f"{type(self)} Id: {self.id}, Name: {self.name}" + + @property + def id(self): + return self.raw.id + @property def name(self): tag_value = self.get_tag_value("Name") - return getattr(self.raw, "name", None) or tag_value if tag_value else self.raw.id + return getattr(self.raw, "name", None) or tag_value if tag_value else self.id def _get_state(self): self.refresh() @@ -487,13 +496,26 @@ class ResourceExplorerResource: This class represents a resource returned by Resource Explorer. """ - def __init__(self, arn, region, resource_type, service, properties=[]): + def __init__(self, arn, region, resource_type, service, properties, **kwargs): self.arn = arn self.region = region self.resource_type = resource_type self.service = service self.properties = properties + if self.resource_type == EC2_INSTANCE: + if kwargs: + system = kwargs.get("system") + if system: + kwargs["raw"] = system.ec2_resource.Instance(self.id) + kwargs["uuid"] = self.id + # When Calling the raw ec2_instance with non-matching AWS Client Region, + # the API call fails with a 'ClientError' (InvalidInstanceID.NotFound) + self.ec2_instance = EC2Instance(**kwargs) + + def __repr__(self): + return f"{type(self)} Id: {self.id}, Type: {self.resource_type}" + def get_tag_value(self, key) -> str: """ Returns a tag value for a given tag key. @@ -531,7 +553,7 @@ def id(self) -> str: This part is used as id in aws cli. """ if self.arn: - return self.arn.split(":")[-1] + return self.arn.split(":")[-1].split("/")[-1] return None @property @@ -545,6 +567,23 @@ def name(self) -> str: name = self.id return name + @property + def date_modified(self) -> datetime: + """ + Returns the time reference of the "LastReportedAt" tag, by the latest value. + If the tag doesn't exist, returns the current time. + Used for filtering resources by the latest modified time recorded. + + Example: + datetime.datetime(2019, 3, 13, 14, 45, 33, tzinfo=tzutc()) + """ + modified_date = None + if self.properties: + modified_date = max(self.properties, key=lambda x: x == "LastReportedAt").get( + "LastReportedAt" + ) + return modified_date + class EC2System(System, VmMixin, TemplateMixin, StackMixin, NetworkMixin): """EC2 Management System, powered by boto @@ -667,9 +706,10 @@ def _get_instances(self, **kwargs): instances = list() for reservation in reservations: for instance in reservation.get("Instances"): + instance_id = instance.get("InstanceId") instances.append( EC2Instance( - system=self, raw=self.ec2_resource.Instance(instance.get("InstanceId")) + system=self, raw=self.ec2_resource.Instance(instance_id), uuid=instance_id ) ) return instances @@ -1814,28 +1854,43 @@ def list_resources(self, query="", view="") -> list[ResourceExplorerResource]: Args: query: keywords and filters for resources; default is "" (all) view: arn of the view to use for the query; default is "" (default view) - Return: a list of resources satisfying the query Examples: Use query "tag.key:kubernetes.io/cluster/*" to list OCP resources """ + resource_list = [] + kwargs = {"system": self} args = {"QueryString": query} if view: args["ViewArn"] = view - list = [] + paginator = self.resource_explorer_connection.get_paginator("search") page_iterator = paginator.paginate(**args) - for page in page_iterator: - resources = page.get("Resources") - for r in resources: - resource = ResourceExplorerResource( - arn=r.get("Arn"), - region=r.get("Region"), - service=r.get("Service"), - properties=r.get("Properties"), - resource_type=r.get("ResourceType"), + + try: + for page in page_iterator: + resources = page.get("Resources") + for r in resources: + resource = ResourceExplorerResource( + arn=r.get("Arn"), + region=r.get("Region"), + service=r.get("Service"), + properties=r.get("Properties"), + resource_type=r.get("ResourceType"), + **kwargs, + ) + resource_list.append(resource) + except ClientError as error: + if error.response["Error"]["Code"] == "UnauthorizedException": + # Raised when the client uses a region that is not defined as an + # Index Aggregator in the Resource Explorer. Official docs: + # https://docs.aws.amazon.com/resource-explorer/latest/userguide/manage-aggregator-region.html + self.logger.info( + f"Region {self._region_name} is not an Index Aggregator. " + f"Cannot fetch resources with current client." ) - list.append(resource) - return list + else: + raise error + return resource_list From ade36a5d8416c1dccf4477ca4ca429ef4f13e64b Mon Sep 17 00:00:00 2001 From: oharan2 Date: Tue, 11 Jun 2024 13:31:24 +0300 Subject: [PATCH 3/5] Update ID extraction from ARNs, fix some logic --- wrapanapi/systems/ec2.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/wrapanapi/systems/ec2.py b/wrapanapi/systems/ec2.py index f5d9d93c..9d45f462 100644 --- a/wrapanapi/systems/ec2.py +++ b/wrapanapi/systems/ec2.py @@ -503,15 +503,13 @@ def __init__(self, arn, region, resource_type, service, properties, **kwargs): self.service = service self.properties = properties - if self.resource_type == EC2_INSTANCE: - if kwargs: - system = kwargs.get("system") - if system: - kwargs["raw"] = system.ec2_resource.Instance(self.id) - kwargs["uuid"] = self.id - # When Calling the raw ec2_instance with non-matching AWS Client Region, - # the API call fails with a 'ClientError' (InvalidInstanceID.NotFound) - self.ec2_instance = EC2Instance(**kwargs) + if self.resource_type == EC2_INSTANCE and kwargs: + if system := kwargs.get("system"): + kwargs["raw"] = system.ec2_resource.Instance(self.id) + kwargs["uuid"] = self.id + # When Calling the raw ec2_instance with non-matching AWS Client Region, + # the API call fails with a 'ClientError' (InvalidInstanceID.NotFound) + self.ec2_instance = EC2Instance(**kwargs) def __repr__(self): return f"{type(self)} Id: {self.id}, Type: {self.resource_type}" @@ -552,8 +550,14 @@ def id(self) -> str: Returns the last part of the arn. This part is used as id in aws cli. """ + # According to the docs: + # https://docs.aws.amazon.com/quicksight/latest/APIReference/qs-arn-format.html + # ARNs can end in either scheme: {ARN_VALUE}:resource-id, {ARN_VALUE}/resource-id if self.arn: - return self.arn.split(":")[-1].split("/")[-1] + arn = self.arn.split(":")[-1] + if "/" in arn: + arn = arn.split("/")[-1] + return arn return None @property From 6f7a1bde77b718983a2caf341db99167268c81f8 Mon Sep 17 00:00:00 2001 From: oharan2 Date: Thu, 13 Jun 2024 11:46:50 +0300 Subject: [PATCH 4/5] Remove datetime built-in package from pyproject.toml --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2395b469..2bb722a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,6 @@ dependencies = [ "vspk==5.3.2", "wait_for", "websocket_client", - "datetime", ] [project.optional-dependencies] From 76252251d36f3ccf3e69cc5e5c0e868b95f8be0b Mon Sep 17 00:00:00 2001 From: oharan2 Date: Thu, 13 Jun 2024 11:47:46 +0300 Subject: [PATCH 5/5] Promote id property func for the usage of __repr__ --- wrapanapi/systems/ec2.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/wrapanapi/systems/ec2.py b/wrapanapi/systems/ec2.py index 9d45f462..1fef67e0 100644 --- a/wrapanapi/systems/ec2.py +++ b/wrapanapi/systems/ec2.py @@ -511,6 +511,22 @@ def __init__(self, arn, region, resource_type, service, properties, **kwargs): # the API call fails with a 'ClientError' (InvalidInstanceID.NotFound) self.ec2_instance = EC2Instance(**kwargs) + @property + def id(self) -> str: + """ + Returns the last part of the arn. + This part is used as id in aws cli. + """ + # According to the docs: + # https://docs.aws.amazon.com/quicksight/latest/APIReference/qs-arn-format.html + # ARNs can end in either scheme: {ARN_VALUE}:resource-id, {ARN_VALUE}/resource-id + if self.arn: + arn = self.arn.split(":")[-1] + if "/" in arn: + arn = arn.split("/")[-1] + return arn + return None + def __repr__(self): return f"{type(self)} Id: {self.id}, Type: {self.resource_type}" @@ -544,22 +560,6 @@ def get_tags(self, regex="") -> list[dict]: list.append(tag) return list - @property - def id(self) -> str: - """ - Returns the last part of the arn. - This part is used as id in aws cli. - """ - # According to the docs: - # https://docs.aws.amazon.com/quicksight/latest/APIReference/qs-arn-format.html - # ARNs can end in either scheme: {ARN_VALUE}:resource-id, {ARN_VALUE}/resource-id - if self.arn: - arn = self.arn.split(":")[-1] - if "/" in arn: - arn = arn.split("/")[-1] - return arn - return None - @property def name(self) -> str: """