Skip to content

Commit 7e04ebb

Browse files
oumkaleoumkale
authored
Adding aws-az-experiment, Disable availability zones for load-balancer (#36)
* adding aws-az-experiment Signed-off-by: Oum Kale <[email protected]> * updating az experiment envs Signed-off-by: Oum Kale <[email protected]> * initialise the chaosdetails using types Signed-off-by: Oum Kale <[email protected]>
1 parent 8a43be3 commit 7e04ebb

File tree

19 files changed

+481
-4
lines changed

19 files changed

+481
-4
lines changed

‎bin/experiment/experiment.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/usr/bin/env python3
22

3+
import experiments.aws_az.aws_az_chaos.experiment.aws_az_chaos as aws_az_chaos
34
import experiments.generic.pod_delete.experiment.pod_delete as pod_delete
45
import argparse
56
import logging
@@ -23,6 +24,8 @@ def main():
2324
# invoke the corresponding experiment based on the the (-name) flag
2425
if args.name == "pod-delete":
2526
pod_delete.PodDelete(clients)
27+
elif args.name == "aws-az-chaos":
28+
aws_az_chaos.AwsAzExperiment(clients)
2629
else:
2730
logging.error("Unsupported -name %s, please provide the correct value of -name args", args.name)
2831
return

‎chaosLib/litmus/aws_az_chaos/__init__.py

Whitespace-only changes.

‎chaosLib/litmus/aws_az_chaos/lib/__init__.py

Whitespace-only changes.

‎chaosLib/litmus/aws_az_chaos/lib/aws_az_chaos.py

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
2+
import pkg.utils.common.common as common
3+
import pkg.types.types as types
4+
import pkg.events.events as events
5+
import logging
6+
from datetime import datetime
7+
import pkg.maths.maths as maths
8+
9+
#PrepareAWSAZExperiment contains the prepration steps before chaos injection
10+
def PrepareAWSAZExperiment(experimentsDetails , resultDetails, eventsDetails, chaosDetails, clients, statusAws):
11+
12+
# Waiting for the ramp time before chaos injection
13+
if experimentsDetails.RampTime != 0 :
14+
logging.info("[Ramp]: Waiting for the %s ramp time before injecting chaos",experimentsDetails.RampTime)
15+
common.WaitForDuration(experimentsDetails.RampTime)
16+
17+
# mode for chaos injection
18+
if experimentsDetails.Sequence.lower() == "serial":
19+
err = injectChaosInSerialMode(experimentsDetails, chaosDetails, eventsDetails, resultDetails, clients, statusAws)
20+
if err != None:
21+
return err
22+
elif experimentsDetails.Sequence.lower() == "parallel":
23+
err = injectChaosInParallelMode(experimentsDetails, chaosDetails, eventsDetails, resultDetails, clients, statusAws)
24+
if err != None:
25+
return err
26+
else:
27+
return ValueError("{} sequence is not supported".format(experimentsDetails.Sequence))
28+
29+
# Waiting for the ramp time after chaos injection
30+
if experimentsDetails.RampTime != 0 :
31+
logging.info("[Ramp]: Waiting for the %s ramp time after injecting chaos",experimentsDetails.RampTime)
32+
common.WaitForDuration(experimentsDetails.RampTime)
33+
34+
return None
35+
36+
# injectChaosInSerialMode disable the target available zone from loadbalancer in serial mode(one by one)
37+
def injectChaosInSerialMode(experimentsDetails , chaosDetails , eventsDetails , resultDetails, clients, statusAws):
38+
39+
#ChaosStartTimeStamp contains the start timestamp, when the chaos injection begin
40+
ChaosStartTimeStamp = datetime.now()
41+
duration = (datetime.now() - ChaosStartTimeStamp).seconds
42+
43+
while duration < experimentsDetails.ChaosDuration:
44+
45+
# Get the target available zones for the chaos execution
46+
targetZones = experimentsDetails.LoadBalancerZones.split(",")
47+
48+
logging.info("[Info]: Target available zone list, %s", targetZones)
49+
50+
if experimentsDetails.EngineName != "" :
51+
msg = "Injecting " + experimentsDetails.ExperimentName + " chaos on available zone"
52+
types.SetEngineEventAttributes(eventsDetails, types.ChaosInject, msg, "Normal", chaosDetails)
53+
events.GenerateEvents(eventsDetails, chaosDetails, "ChaosEngine", clients)
54+
55+
# Detaching the target zones from loa balancer
56+
for azone in targetZones:
57+
58+
logging.info("[Info]: Detaching the following zone(s), Zone Name : %s", azone)
59+
err = statusAws.detachAZfromLB(experimentsDetails, azone)
60+
if err != None:
61+
return err
62+
if chaosDetails.Randomness:
63+
err = common.RandomInterval(experimentsDetails.ChaosInterval)
64+
if err != None:
65+
return err
66+
else:
67+
#Waiting for the chaos interval after chaos injection
68+
if experimentsDetails.ChaosInterval != "":
69+
logging.info("[Wait]: Wait for the chaos interval %s",(experimentsDetails.ChaosInterval))
70+
waitTime = maths.atoi(experimentsDetails.ChaosInterval)
71+
common.WaitForDuration(waitTime)
72+
73+
# Attaching the target available zone after the chaos injection
74+
logging.info("[Status]: Attach the available zone back to load balancer")
75+
err = statusAws.attachAZtoLB(experimentsDetails, azone)
76+
if err != None:
77+
return err
78+
79+
#Verify the status of available zone after the chaos injection
80+
logging.info("[Status]: Checking AWS load balancer's AZ status")
81+
err = statusAws.CheckAWSStatus(experimentsDetails)
82+
if err != None:
83+
return err
84+
85+
duration = (datetime.now() - ChaosStartTimeStamp).seconds
86+
87+
logging.info("[Completion]: %s chaos is done",(experimentsDetails.ExperimentName))
88+
89+
return None
90+
91+
# injectChaosInParallelMode disable the target available zone from loadbalancer in parallel mode (all at once)
92+
def injectChaosInParallelMode(experimentsDetails , chaosDetails , eventsDetails , resultDetails, clients, statusAws):
93+
94+
95+
#ChaosStartTimeStamp contains the start timestamp, when the chaos injection begin
96+
ChaosStartTimeStamp = datetime.now()
97+
duration = (datetime.now() - ChaosStartTimeStamp).seconds
98+
99+
while duration < experimentsDetails.ChaosDuration:
100+
101+
# Get the target available zone details for the chaos execution
102+
targetZones = experimentsDetails.LoadBalancerZones.split(",")
103+
logging.info("[Info]: Target available zone list, %s", targetZones)
104+
105+
if experimentsDetails.EngineName != "" :
106+
msg = "Injecting " + experimentsDetails.ExperimentName + " chaos on available zone"
107+
types.SetEngineEventAttributes(eventsDetails, types.ChaosInject, msg, "Normal", chaosDetails)
108+
events.GenerateEvents(eventsDetails, chaosDetails, "ChaosEngine",clients)
109+
110+
# Detaching the target zones from loa balancer
111+
for azone in targetZones:
112+
logging.info("[Info]: Detaching the following zone(s), Zone Name %s", azone)
113+
err = statusAws.detachAZfromLB(experimentsDetails, azone)
114+
if err != None:
115+
return err
116+
117+
if chaosDetails.Randomness:
118+
err = common.RandomInterval(experimentsDetails.ChaosInterval)
119+
if err != None:
120+
return err
121+
else:
122+
#Waiting for the chaos interval after chaos injection
123+
if experimentsDetails.ChaosInterval != "" :
124+
logging.info("[Wait]: Wait for the chaos interval %s", experimentsDetails.ChaosInterval)
125+
waitTime = maths.atoi(experimentsDetails.ChaosInterval)
126+
common.WaitForDuration(waitTime)
127+
128+
# Attaching the target available zone after the chaos injection
129+
logging.info("[Status]: Attach the available zone back to load balancer")
130+
for azone in targetZones:
131+
err = statusAws.attachAZtoLB(experimentsDetails, azone)
132+
if err != None:
133+
return err
134+
135+
#Verify the status of available zone after the chaos injection
136+
logging.info("[Status]: Checking AWS load balancer's AZ status")
137+
err = statusAws.CheckAWSStatus(experimentsDetails)
138+
if err != None:
139+
return err
140+
141+
duration = (datetime.now() - ChaosStartTimeStamp).seconds
142+
143+
logging.info("[Completion]: %s chaos is done",(experimentsDetails.ExperimentName))
144+
145+
return None

‎experiments/aws_az/__init__.py

Whitespace-only changes.

‎experiments/aws_az/aws_az_chaos/__init__.py

Whitespace-only changes.

‎experiments/aws_az/aws_az_chaos/experiment/__init__.py

Whitespace-only changes.

‎experiments/aws_az/aws_az_chaos/experiment/aws_az_chaos.py

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
import pkg.types.types as types
2+
import pkg.aws_az.types.types as experimentDetails
3+
import pkg.aws_az.environment.environment as experimentEnv
4+
import pkg.events.events as events
5+
import logging
6+
import chaosLib.litmus.aws_az_chaos.lib.aws_az_chaos as litmusLIB
7+
import pkg.result.chaosresult as chaosResults
8+
import pkg.utils.common.common as common
9+
import pkg.aws_status.status as awsStatus
10+
11+
# AwsAzExperiment contains steps to inject chaos
12+
def AwsAzExperiment(clients):
13+
14+
# Initialising expermentDetails, resultDetails, eventsDetails, chaosDetails, status and result objects
15+
experimentsDetails = experimentDetails.ExperimentDetails()
16+
resultDetails = types.ResultDetails()
17+
eventsDetails = types.EventDetails()
18+
chaosDetails = types.ChaosDetails()
19+
result = chaosResults.ChaosResults()
20+
statusAws = awsStatus.AWS_AZ()
21+
22+
#Fetching all the ENV passed from the runner pod
23+
experimentEnv.GetENV(experimentsDetails)
24+
25+
logging.info("[PreReq]: Initialise Chaos Variables for the %s experiment", experimentsDetails.ExperimentName)
26+
27+
# Intialise the chaos attributes
28+
experimentEnv.InitialiseChaosVariables(chaosDetails, experimentsDetails)
29+
30+
# Intialise Chaos Result Parameters
31+
types.SetResultAttributes(resultDetails, chaosDetails)
32+
33+
#Updating the chaos result in the beginning of experiment
34+
logging.info("[PreReq]: Updating the chaos result of %s experiment (SOT)",(experimentsDetails.ExperimentName))
35+
err = result.ChaosResult(chaosDetails, resultDetails, "SOT", clients)
36+
if err != None:
37+
logging.error("Unable to Create the Chaos Result, err: %s",(err))
38+
failStep = "Updating the chaos result of aws-az-chaos experiment (SOT)"
39+
result.RecordAfterFailure(chaosDetails, resultDetails, failStep, eventsDetails, clients)
40+
return
41+
42+
# Set the chaos result uid
43+
result.SetResultUID(resultDetails, chaosDetails, clients)
44+
45+
# generating the event in chaosresult to marked the verdict as awaited
46+
msg = "Experiment " + experimentsDetails.ExperimentName + ", Result Awaited"
47+
types.SetResultEventAttributes(eventsDetails, types.AwaitedVerdict, msg, "Normal", resultDetails)
48+
events.GenerateEvents(eventsDetails, chaosDetails, "ChaosResult", clients)
49+
50+
# DISPLAY THE LOADBALANCER INFORMATION
51+
logging.info("[Info]: The application information is as follows LoadBalancer Name=%s, LoadBalancer Zones=%s, Ramp Time=%s",experimentsDetails.LoadBalancerName,experimentsDetails.LoadBalancerZones,experimentsDetails.RampTime)
52+
53+
# Calling AbortWatcher, it will continuously watch for the abort signal and generate the required and result
54+
common.AbortWatcher(experimentsDetails.ExperimentName, resultDetails, chaosDetails, eventsDetails, clients)
55+
56+
# PRE-CHAOS APPLICATION STATUS CHECK
57+
logging.info("[Status]: Verify that the AUT (Application Under Test) is running (pre-chaos)")
58+
err = statusAws.CheckAWSStatus(experimentsDetails)
59+
if err != None:
60+
logging.error("Target available zone status check failed, err: %s", err)
61+
failStep = "Verify that the AUT (Application Under Test) is running (pre-chaos)"
62+
result.RecordAfterFailure(chaosDetails, resultDetails, failStep, eventsDetails, clients)
63+
return
64+
65+
if experimentsDetails.EngineName != "":
66+
# marking AUT as running, as we already checked the status of application under test
67+
msg = "AUT: Running"
68+
# generating the for the pre-chaos check
69+
types.SetEngineEventAttributes(eventsDetails, types.PreChaosCheck, msg, "Normal", chaosDetails)
70+
events.GenerateEvents(eventsDetails, chaosDetails, "ChaosEngine", clients)
71+
72+
# Including the litmus lib for aws-az-chaos
73+
if experimentsDetails.ChaosLib == "litmus" :
74+
err = litmusLIB.PrepareAWSAZExperiment(experimentsDetails, resultDetails, eventsDetails, chaosDetails, clients, statusAws)
75+
if err != None:
76+
logging.error("Chaos injection failed, err: %s",(err))
77+
failStep = "failed in chaos injection phase"
78+
result.RecordAfterFailure(chaosDetails, resultDetails, failStep, eventsDetails, clients)
79+
return
80+
81+
else:
82+
logging.info("[Invalid]: Please Provide the correct LIB")
83+
failStep = "no match found for specified lib"
84+
result.RecordAfterFailure(chaosDetails, resultDetails, failStep, eventsDetails, clients)
85+
return
86+
87+
logging.info("[Confirmation]: %s chaos has been injected successfully", experimentsDetails.ExperimentName)
88+
resultDetails.Verdict = "Pass"
89+
90+
# POST-CHAOS APPLICATION STATUS CHECK
91+
logging.info("[Status]: Verify that the AUT (Application Under Test) is running (post-chaos)")
92+
err = statusAws.CheckAWSStatus(experimentsDetails)
93+
if err != None:
94+
logging.error("Target aws instance status check failed, err: %s", err)
95+
failStep = "Verify that the AUT (Application Under Test) is running (post-chaos)"
96+
result.RecordAfterFailure(chaosDetails, resultDetails, failStep, eventsDetails, clients)
97+
return
98+
99+
100+
if experimentsDetails.EngineName != "" :
101+
# marking AUT as running, as we already checked the status of application under test
102+
msg = "AUT: Running"
103+
104+
# generating post chaos event
105+
types.SetEngineEventAttributes(eventsDetails, types.PostChaosCheck, msg, "Normal", chaosDetails)
106+
events.GenerateEvents(eventsDetails, chaosDetails, "ChaosEngine", clients)
107+
108+
109+
#Updating the chaosResult in the end of experiment
110+
logging.info("[The End]: Updating the chaos result of %s experiment (EOT)", experimentsDetails.ExperimentName)
111+
err = result.ChaosResult(chaosDetails, resultDetails, "EOT", clients)
112+
if err != None:
113+
logging.error("Unable to Update the Chaos Result, err: %s", err)
114+
return
115+
116+
# generating the event in chaosresult to marked the verdict as pass/fail
117+
msg = "Experiment " + experimentsDetails.ExperimentName + ", Result " + resultDetails.Verdict
118+
reason = types.PassVerdict
119+
eventType = "Normal"
120+
if resultDetails.Verdict != "Pass":
121+
reason = types.FailVerdict
122+
eventType = "Warning"
123+
124+
types.SetResultEventAttributes(eventsDetails, reason, msg, eventType, resultDetails)
125+
events.GenerateEvents(eventsDetails, chaosDetails, "ChaosResult", clients)
126+
if experimentsDetails.EngineName != "":
127+
msg = experimentsDetails.ExperimentName + " experiment has been " + resultDetails.Verdict + "ed"
128+
types.SetEngineEventAttributes(eventsDetails, types.Summary, msg, "Normal", chaosDetails)
129+
events.GenerateEvents(eventsDetails, chaosDetails, "ChaosEngine", clients)

‎experiments/aws_az/aws_az_chaos/test/test.yml

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
---
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
name: litmus-experiment
6+
spec:
7+
replicas: 1
8+
selector:
9+
matchLabels:
10+
app: litmus-experiment
11+
template:
12+
metadata:
13+
labels:
14+
app: litmus-experiment
15+
spec:
16+
serviceAccountName: aws_az_chaos-sa
17+
containers:
18+
- name: gotest
19+
image: busybox
20+
command:
21+
- sleep
22+
- "3600"
23+
env:
24+
- name: LOAD_BALANCER_NAME
25+
value: ''
26+
27+
- name: LOAD_BALANCER_ZONES
28+
value: ''
29+
30+
- name: LOAD_BALANCERNAME_ARN
31+
value: ''
32+
33+
- name: AWS_SHARED_CREDENTIALS_FILE
34+
value: ""
35+
36+
- name: TOTAL_CHAOS_DURATION
37+
value: ''
38+
39+
- name: AWS_REGIONS
40+
value: ""
41+
42+
## Period to wait before injection of chaos in sec
43+
- name: RAMP_TIME
44+
value: ''
45+
46+
## env var that describes the library used to execute the chaos
47+
## default: litmus. Supported values: litmus, powerfulseal, chaoskube
48+
- name: LIB
49+
value: ''
50+
51+
# provide the chaos namespace
52+
- name: CHAOS_NAMESPACE
53+
value: ''
54+
55+
- name: POD_NAME
56+
valueFrom:
57+
fieldRef:
58+
fieldPath: metadata.name
59+
60+
- name: CHAOS_SERVICE_ACCOUNT
61+
valueFrom:
62+
fieldRef:
63+
fieldPath: spec.serviceAccountName
64+

‎pkg/aws_az/__init__.py

Whitespace-only changes.

‎pkg/aws_az/environment/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)