From 4bf47ad6140b8035999f351f25886132efd12011 Mon Sep 17 00:00:00 2001 From: Anderson Nogueira Date: Mon, 6 Oct 2025 23:04:45 +0200 Subject: [PATCH 1/3] Consolidate Lambda code and clean up YAML formatting - Remove separate cloud/aws-functions/email_running_instances.py file - Keep Lambda code inline in IaC/StagingStack.yml for simpler deployment - Remove AWS CLI example command from template header - Fix YAML formatting: proper comment spacing and remove trailing whitespaces --- IaC/StagingStack.yml | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/IaC/StagingStack.yml b/IaC/StagingStack.yml index f50b68f43d..86a5f4d48f 100644 --- a/IaC/StagingStack.yml +++ b/IaC/StagingStack.yml @@ -1,4 +1,3 @@ -# aws cloudformation --region us-east-2 create-stack --template-body file://IaC/StagingStack.yml --capabilities CAPABILITY_NAMED_IAM --stack-name pmm-staging --tags Key=iit-billing-tag,Value=pmm-staging --parameters ParameterKey=LambdaName,ParameterValue=email_running_instances ParameterKey=SShortName,ParameterValue=pmm-staging --- AWSTemplateFormatVersion: 2010-09-09 Parameters: @@ -21,7 +20,7 @@ Parameters: ConstraintDescription: must begin with a letter and must contain only lowercase letters, numbers, periods (.), and dashes (-). Resources: - SVPC: # separate virtual network for Staging instances + SVPC: # separate virtual network for Staging instances Type: AWS::EC2::VPC Properties: CidrBlock: 10.178.0.0/22 @@ -34,7 +33,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SInternetGateway: # Internet Gateway for Staging VPC + SInternetGateway: # Internet Gateway for Staging VPC Type: AWS::EC2::InternetGateway Properties: Tags: @@ -43,13 +42,13 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SVPCGatewayAttachment: # Attach Gateway to VPC + SVPCGatewayAttachment: # Attach Gateway to VPC Type: AWS::EC2::VPCGatewayAttachment Properties: VpcId: !Ref SVPC InternetGatewayId: !Ref SInternetGateway - SSubnetA: # create subnet in AZ + SSubnetA: # create subnet in AZ Type: AWS::EC2::Subnet Properties: VpcId: !Ref SVPC @@ -64,7 +63,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SSubnetB: # create subnet in AZ + SSubnetB: # create subnet in AZ Type: AWS::EC2::Subnet Properties: VpcId: !Ref SVPC @@ -79,7 +78,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SSubnetC: # create subnet in AZ + SSubnetC: # create subnet in AZ Type: AWS::EC2::Subnet Properties: VpcId: !Ref SVPC @@ -94,7 +93,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SRouteTable: # create route table for VPC + SRouteTable: # create route table for VPC Type: AWS::EC2::RouteTable Properties: VpcId: !Ref SVPC @@ -104,32 +103,32 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SInternetRoute: # add default route + SInternetRoute: # add default route Type: AWS::EC2::Route Properties: DestinationCidrBlock: 0.0.0.0/0 GatewayId: !Ref SInternetGateway RouteTableId: !Ref SRouteTable - SSubnetARouteTable: # add subnet route + SSubnetARouteTable: # add subnet route Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref SRouteTable SubnetId: !Ref SSubnetA - SSubnetBRouteTable: # add subnet route + SSubnetBRouteTable: # add subnet route Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref SRouteTable SubnetId: !Ref SSubnetB - SSubnetCRouteTable: # add subnet route + SSubnetCRouteTable: # add subnet route Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref SRouteTable SubnetId: !Ref SSubnetC - SSSHSecurityGroup: # allow ssh access to staging instances + SSSHSecurityGroup: # allow ssh access to staging instances Type: AWS::EC2::SecurityGroup Properties: GroupName: SSH @@ -144,7 +143,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SNOMADSecurityGroup: # allow nomad access to staging instances + SNOMADSecurityGroup: # allow nomad access to staging instances Type: AWS::EC2::SecurityGroup Properties: GroupName: NOMAD @@ -159,7 +158,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SHTTPSecurityGroup: # allow http and https access + SHTTPSecurityGroup: # allow http and https access Type: AWS::EC2::SecurityGroup Properties: GroupName: HTTP @@ -178,7 +177,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SInstancesRole: # separate IAM role for Staging instances + SInstancesRole: # separate IAM role for Staging instances Type: "AWS::IAM::Role" Properties: RoleName: !Join ["-", [!Ref SShortName, "slave"]] @@ -230,7 +229,7 @@ Resources: - !Ref SInstancesRole InstanceProfileName: !Join ["-", [!Ref SShortName, "slave"]] - SInstancesUser: # create standalone user for PMM jenkins jobs + SInstancesUser: # create standalone user for PMM jenkins jobs Type: AWS::IAM::User Properties: UserName: !Join ["-", [!Ref SShortName, "slave"]] @@ -328,7 +327,7 @@ Resources: Properties: UserName: !Ref SInstancesUser - LambdaExecutionRole: # create Role for lambda function + LambdaExecutionRole: # create Role for lambda function Type: AWS::IAM::Role Properties: RoleName: !Ref LambdaName @@ -359,7 +358,7 @@ Resources: - logs:CreateLogGroup - logs:PutLogEvents - LambdaFunction: # create lambda function which email staging owner at night + LambdaFunction: # create lambda function which email staging owner at night Type: AWS::Lambda::Function Properties: Description: email owner about running stagging instances @@ -484,7 +483,7 @@ Resources: return 'successful finish' - ScheduledRule: # cron rule + ScheduledRule: # cron rule Type: AWS::Events::Rule Properties: Name: !Ref LambdaName @@ -495,7 +494,7 @@ Resources: - Arn: !GetAtt LambdaFunction.Arn Id: !Ref LambdaName - PermissionForEventsToInvokeLambda: # create permitions to run function from ScheduledRule + PermissionForEventsToInvokeLambda: # create permitions to run function from ScheduledRule Type: AWS::Lambda::Permission Properties: FunctionName: !Ref LambdaFunction From 53e49a263872b4e8c3d85553c06bf7e2809beab5 Mon Sep 17 00:00:00 2001 From: Anderson Nogueira Date: Mon, 6 Oct 2025 23:07:53 +0200 Subject: [PATCH 2/3] Fix Lambda function for accurate cost calculation and richer emails - Fix email_running_instances Lambda: accurate cost calculation for spot/on-demand instances - Calculate time-weighted costs based on actual spot price history - Add richer email format with instance type, lifecycle, and region/AZ info - Fix Python 3 compatibility issues with filter() function - Clean up YAML formatting: fix comment spacing and remove trailing whitespaces --- IaC/StagingStack.yml | 88 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 19 deletions(-) diff --git a/IaC/StagingStack.yml b/IaC/StagingStack.yml index 86a5f4d48f..600eaf4f53 100644 --- a/IaC/StagingStack.yml +++ b/IaC/StagingStack.yml @@ -381,7 +381,7 @@ Resources: import collections def lambda_handler(event, context): - fullReportEmail = ['alexander.tymchuk@percona.com', 'talha.rizwan@percona.com'] + fullReportEmail = ['alexander.tymchuk@percona.com', 'talha.rizwan@percona.com', 'anderson.nogueira@percona.com', 'alex.miroshnychenko@percona.com'] region = 'us-east-2' session = boto3.Session(region_name=region) resources = session.resource('ec2') @@ -396,21 +396,21 @@ Resources: emails = collections.defaultdict(list) for instance in instances: # get instance Owner - ownerFilter = filter(lambda x: 'owner' == x['Key'], instance.tags) + ownerFilter = list(filter(lambda x: 'owner' == x['Key'], instance.tags)) if len(ownerFilter) >= 1: owner = ownerFilter[0]['Value'] + '@percona.com' else: owner = 'unknown' # get instance allowed days - allowedDaysFilter = filter(lambda x: 'stop-after-days' == x['Key'], instance.tags) - if len(allowedDaysFilter) >= 1 and allowedDaysFilter[0]['Value'] > 0: + allowedDaysFilter = list(filter(lambda x: 'stop-after-days' == x['Key'], instance.tags)) + if len(allowedDaysFilter) >= 1 and int(allowedDaysFilter[0]['Value']) > 0: allowedDays = allowedDaysFilter[0]['Value'] + ' days' else: allowedDays = 'unlimited' # get instance Name - nameFilter = filter(lambda x: 'Name' == x['Key'], instance.tags) + nameFilter = list(filter(lambda x: 'Name' == x['Key'], instance.tags)) if len(nameFilter) >= 1: name = nameFilter[0]['Value'] else: @@ -420,25 +420,75 @@ Resources: current_time = datetime.datetime.now(instance.launch_time.tzinfo) uptime = current_time - instance.launch_time - # get price - priceHistory = ec2.describe_spot_price_history( - InstanceTypes=[instance.instance_type], - StartTime=instance.launch_time, - EndTime=current_time, - AvailabilityZone=instance.placement['AvailabilityZone'] - ) - totalCost = 0.0 - for price in priceHistory['SpotPriceHistory']: - totalCost += float(price['SpotPrice']) + # get price - calculate time-weighted cost + is_spot = instance.instance_lifecycle == 'spot' + + if is_spot: + # Get spot price history for Linux/UNIX only + priceHistory = ec2.describe_spot_price_history( + InstanceTypes=[instance.instance_type], + StartTime=instance.launch_time, + EndTime=current_time, + AvailabilityZone=instance.placement['AvailabilityZone'], + ProductDescriptions=['Linux/UNIX'] + ) + + # Calculate time-weighted cost + totalCost = 0.0 + sorted_prices = sorted(priceHistory['SpotPriceHistory'], + key=lambda x: x['Timestamp']) + + for i, entry in enumerate(sorted_prices): + period_start = entry['Timestamp'] + price = float(entry['SpotPrice']) + + # Determine when this price period ended + if i < len(sorted_prices) - 1: + period_end = sorted_prices[i + 1]['Timestamp'] + else: + period_end = current_time + + # Only count time after instance launched + if period_end <= instance.launch_time: + continue + if period_start < instance.launch_time: + period_start = instance.launch_time + + # Calculate hours at this price + hours = (period_end - period_start).total_seconds() / 3600 + totalCost += hours * price + else: + # On-demand pricing (simplified - use current spot price as estimate) + priceHistory = ec2.describe_spot_price_history( + InstanceTypes=[instance.instance_type], + AvailabilityZone=instance.placement['AvailabilityZone'], + ProductDescriptions=['Linux/UNIX'], + MaxResults=1 + ) + if priceHistory['SpotPriceHistory']: + price = float(priceHistory['SpotPriceHistory'][0]['SpotPrice']) + else: + price = 0.05 # fallback + hours = uptime.total_seconds() / 3600 + totalCost = hours * price * 2 # On-demand is ~2x spot price + costStr = '%0.2f USD' % totalCost # prepare table for email if uptime.total_seconds() > 5*60*60: strUptime = re.match('^[^:]+:[^:]+', str(uptime)).group(0) - emails[owner].append('' + name + '' + owner + '' + strUptime + '' + allowedDays + '' + costStr + '') + lifecycle = 'Spot' if is_spot else 'On-Demand' + instance_details = instance.instance_type + ' (' + lifecycle + ')' + region_az = region + '/' + instance.placement['AvailabilityZone'] + + row = ('' + name + '' + instance_details + '' + + region_az + '' + owner + '' + strUptime + + '' + allowedDays + '' + costStr + '') + + emails[owner].append(row) for email in fullReportEmail: if owner != email: - emails[email].append('' + name + '' + owner + '' + strUptime + '' + allowedDays + '' + costStr + '') + emails[email].append(row) else: print('Skip: ' + name) @@ -451,8 +501,8 @@ Resources:

A friendly reminder - please don't forget to shutdown the following instances:

- - +
NameOwnerUptimeExpiryTotal Cost
+ %s
NameTypeRegion/AZOwnerUptimeExpiryTotal Cost

From b414b5a51ba0ad382978062586b6aaa8937be0b7 Mon Sep 17 00:00:00 2001 From: Anderson Nogueira Date: Tue, 7 Oct 2025 00:01:24 +0200 Subject: [PATCH 3/3] Improve Lambda pricing accuracy with AWS Pricing API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add AWS Pricing API integration for accurate on-demand pricing - Remove inaccurate spot × 2 fallback estimation - Add pricing:GetProducts IAM permission to Lambda role - Keep describe_spot_price_history for spot instances (API doesn't provide spot prices) - Limit region support to US and Europe only - Fix YAML formatting: remove trailing whitespaces and extra spaces before comments - Update email recipient list --- IaC/StagingStack.yml | 114 ++++++++++++++++++++++++++++++------------- 1 file changed, 81 insertions(+), 33 deletions(-) diff --git a/IaC/StagingStack.yml b/IaC/StagingStack.yml index 600eaf4f53..d6c02aa147 100644 --- a/IaC/StagingStack.yml +++ b/IaC/StagingStack.yml @@ -20,7 +20,7 @@ Parameters: ConstraintDescription: must begin with a letter and must contain only lowercase letters, numbers, periods (.), and dashes (-). Resources: - SVPC: # separate virtual network for Staging instances + SVPC: # separate virtual network for Staging instances Type: AWS::EC2::VPC Properties: CidrBlock: 10.178.0.0/22 @@ -33,7 +33,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SInternetGateway: # Internet Gateway for Staging VPC + SInternetGateway: # Internet Gateway for Staging VPC Type: AWS::EC2::InternetGateway Properties: Tags: @@ -42,13 +42,13 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SVPCGatewayAttachment: # Attach Gateway to VPC + SVPCGatewayAttachment: # Attach Gateway to VPC Type: AWS::EC2::VPCGatewayAttachment Properties: VpcId: !Ref SVPC InternetGatewayId: !Ref SInternetGateway - SSubnetA: # create subnet in AZ + SSubnetA: # create subnet in AZ Type: AWS::EC2::Subnet Properties: VpcId: !Ref SVPC @@ -63,7 +63,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SSubnetB: # create subnet in AZ + SSubnetB: # create subnet in AZ Type: AWS::EC2::Subnet Properties: VpcId: !Ref SVPC @@ -78,7 +78,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SSubnetC: # create subnet in AZ + SSubnetC: # create subnet in AZ Type: AWS::EC2::Subnet Properties: VpcId: !Ref SVPC @@ -93,7 +93,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SRouteTable: # create route table for VPC + SRouteTable: # create route table for VPC Type: AWS::EC2::RouteTable Properties: VpcId: !Ref SVPC @@ -103,32 +103,32 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SInternetRoute: # add default route + SInternetRoute: # add default route Type: AWS::EC2::Route Properties: DestinationCidrBlock: 0.0.0.0/0 GatewayId: !Ref SInternetGateway RouteTableId: !Ref SRouteTable - SSubnetARouteTable: # add subnet route + SSubnetARouteTable: # add subnet route Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref SRouteTable SubnetId: !Ref SSubnetA - SSubnetBRouteTable: # add subnet route + SSubnetBRouteTable: # add subnet route Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref SRouteTable SubnetId: !Ref SSubnetB - SSubnetCRouteTable: # add subnet route + SSubnetCRouteTable: # add subnet route Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref SRouteTable SubnetId: !Ref SSubnetC - SSSHSecurityGroup: # allow ssh access to staging instances + SSSHSecurityGroup: # allow ssh access to staging instances Type: AWS::EC2::SecurityGroup Properties: GroupName: SSH @@ -143,7 +143,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SNOMADSecurityGroup: # allow nomad access to staging instances + SNOMADSecurityGroup: # allow nomad access to staging instances Type: AWS::EC2::SecurityGroup Properties: GroupName: NOMAD @@ -158,7 +158,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SHTTPSecurityGroup: # allow http and https access + SHTTPSecurityGroup: # allow http and https access Type: AWS::EC2::SecurityGroup Properties: GroupName: HTTP @@ -177,7 +177,7 @@ Resources: - Key: iit-billing-tag Value: !Ref SShortName - SInstancesRole: # separate IAM role for Staging instances + SInstancesRole: # separate IAM role for Staging instances Type: "AWS::IAM::Role" Properties: RoleName: !Join ["-", [!Ref SShortName, "slave"]] @@ -229,7 +229,7 @@ Resources: - !Ref SInstancesRole InstanceProfileName: !Join ["-", [!Ref SShortName, "slave"]] - SInstancesUser: # create standalone user for PMM jenkins jobs + SInstancesUser: # create standalone user for PMM jenkins jobs Type: AWS::IAM::User Properties: UserName: !Join ["-", [!Ref SShortName, "slave"]] @@ -327,7 +327,7 @@ Resources: Properties: UserName: !Ref SInstancesUser - LambdaExecutionRole: # create Role for lambda function + LambdaExecutionRole: # create Role for lambda function Type: AWS::IAM::Role Properties: RoleName: !Ref LambdaName @@ -351,6 +351,7 @@ Resources: - ec2:DescribeSpotPriceHistory - ec2:DescribeInstances - ses:SendEmail + - pricing:GetProducts - Effect: Allow Resource: "arn:aws:logs:*:*:*" Action: @@ -358,7 +359,7 @@ Resources: - logs:CreateLogGroup - logs:PutLogEvents - LambdaFunction: # create lambda function which email staging owner at night + LambdaFunction: # create lambda function which email staging owner at night Type: AWS::Lambda::Function Properties: Description: email owner about running stagging instances @@ -379,9 +380,60 @@ Resources: import boto3 import datetime import collections + import json + from functools import lru_cache + + # Region to location mapping for pricing API + REGION_LOCATION_MAP = { + 'us-east-1': 'US East (N. Virginia)', + 'us-east-2': 'US East (Ohio)', + 'us-west-1': 'US West (N. California)', + 'us-west-2': 'US West (Oregon)', + 'eu-west-1': 'EU (Ireland)', + 'eu-central-1': 'EU (Frankfurt)', + 'eu-west-2': 'EU (London)', + 'eu-west-3': 'EU (Paris)', + 'eu-north-1': 'EU (Stockholm)' + } + + @lru_cache(maxsize=128) + def get_on_demand_price(instance_type, region='us-east-2'): + """Get accurate on-demand price from AWS Pricing API""" + try: + # Pricing API is only available in us-east-1 + pricing_client = boto3.client('pricing', region_name='us-east-1') + location = REGION_LOCATION_MAP.get(region, 'US East (Ohio)') + + response = pricing_client.get_products( + ServiceCode='AmazonEC2', + Filters=[ + {'Type': 'TERM_MATCH', 'Field': 'instanceType', 'Value': instance_type}, + {'Type': 'TERM_MATCH', 'Field': 'location', 'Value': location}, + {'Type': 'TERM_MATCH', 'Field': 'tenancy', 'Value': 'Shared'}, + {'Type': 'TERM_MATCH', 'Field': 'operatingSystem', 'Value': 'Linux'}, + {'Type': 'TERM_MATCH', 'Field': 'preInstalledSw', 'Value': 'NA'}, + {'Type': 'TERM_MATCH', 'Field': 'capacitystatus', 'Value': 'Used'} + ], + MaxResults=1 + ) + + if response['PriceList']: + price_item = json.loads(response['PriceList'][0]) + on_demand = price_item['terms']['OnDemand'] + + for term in on_demand.values(): + for price_dimension in term['priceDimensions'].values(): + price = float(price_dimension['pricePerUnit']['USD']) + if price > 0: + return price + except Exception as e: + print(f"Error: Could not get on-demand price for {instance_type} in {region}: {e}") + raise e # Re-raise to ensure we know if pricing fails + + raise ValueError(f"No on-demand price found for {instance_type} in {region}") def lambda_handler(event, context): - fullReportEmail = ['alexander.tymchuk@percona.com', 'talha.rizwan@percona.com', 'anderson.nogueira@percona.com', 'alex.miroshnychenko@percona.com'] + fullReportEmail = ['alexander.tymchuk@percona.com', 'talha.rizwan@percona.com', 'anderson.nogueira@percona.com'] region = 'us-east-2' session = boto3.Session(region_name=region) resources = session.resource('ec2') @@ -458,19 +510,15 @@ Resources: hours = (period_end - period_start).total_seconds() / 3600 totalCost += hours * price else: - # On-demand pricing (simplified - use current spot price as estimate) - priceHistory = ec2.describe_spot_price_history( - InstanceTypes=[instance.instance_type], - AvailabilityZone=instance.placement['AvailabilityZone'], - ProductDescriptions=['Linux/UNIX'], - MaxResults=1 - ) - if priceHistory['SpotPriceHistory']: - price = float(priceHistory['SpotPriceHistory'][0]['SpotPrice']) - else: - price = 0.05 # fallback + # On-demand pricing using AWS Pricing API hours = uptime.total_seconds() / 3600 - totalCost = hours * price * 2 # On-demand is ~2x spot price + try: + on_demand_price = get_on_demand_price(instance.instance_type, region) + totalCost = hours * on_demand_price + except Exception as e: + print(f"Failed to get on-demand price for {instance.instance_type}: {e}") + # Skip this instance if we can't get pricing + continue costStr = '%0.2f USD' % totalCost @@ -533,7 +581,7 @@ Resources: return 'successful finish' - ScheduledRule: # cron rule + ScheduledRule: # cron rule Type: AWS::Events::Rule Properties: Name: !Ref LambdaName @@ -544,7 +592,7 @@ Resources: - Arn: !GetAtt LambdaFunction.Arn Id: !Ref LambdaName - PermissionForEventsToInvokeLambda: # create permitions to run function from ScheduledRule + PermissionForEventsToInvokeLambda: # create permitions to run function from ScheduledRule Type: AWS::Lambda::Permission Properties: FunctionName: !Ref LambdaFunction