Merge pull request #52 from crytic/test-artifacts

add artifact writer

network-configs/plan/testing.yaml

@@ -5,8 +5,8 @@ participants:
       cl_client_image: sigp/lighthouse:latest
       el_min_cpu: 1000
       el_max_cpu: 1000
-      el_min_mem: 2048
-      el_max_mem: 2048
+      el_min_mem: 1024
+      el_max_mem: 1024
       bn_min_cpu: 1000
       bn_max_cpu: 1000
       bn_min_mem: 2048
@@ -22,8 +22,8 @@ participants:
       cl_client_image: consensys/teku:23.12.0
       el_min_cpu: 1000
       el_max_cpu: 1000
-      el_min_mem: 2048
-      el_max_mem: 2048
+      el_min_mem: 1024
+      el_max_mem: 1024
       bn_min_cpu: 1000
       bn_max_cpu: 1000
       bn_min_mem: 2048
@@ -35,8 +35,8 @@ participants:
       cl_client_image: chainsafe/lodestar:v1.12.1
       el_min_cpu: 1000
       el_max_cpu: 1000
-      el_min_mem: 2048
-      el_max_mem: 2048
+      el_min_mem: 1024
+      el_max_mem: 1024
       bn_min_cpu: 1000
       bn_max_cpu: 1000
       bn_min_mem: 2048
@@ -52,8 +52,8 @@ participants:
       cl_client_image: sigp/lighthouse:latest
       el_min_cpu: 1000
       el_max_cpu: 1000
-      el_min_mem: 2048
-      el_max_mem: 2048
+      el_min_mem: 1024
+      el_max_mem: 1024
       bn_min_cpu: 1000
       bn_max_cpu: 1000
       bn_min_mem: 2048
@@ -69,8 +69,8 @@ participants:
       cl_client_image: prysmaticlabs/prysm-beacon-chain:latest,prysmaticlabs/prysm-validator:latest
       el_min_cpu: 1000
       el_max_cpu: 1000
-      el_min_mem: 2048
-      el_max_mem: 2048
+      el_min_mem: 1024
+      el_max_mem: 1024
       bn_min_cpu: 1000
       bn_max_cpu: 1000
       bn_min_mem: 2048

pkg/artifacts/artifacts.go

@@ -0,0 +1,74 @@
+package artifacts
+
+import (
+	chaosMesh "attacknet/cmd/pkg/chaos-mesh"
+	"attacknet/cmd/pkg/health"
+	healthTypes "attacknet/cmd/pkg/health/types"
+	"attacknet/cmd/pkg/types"
+	"errors"
+	"fmt"
+	"github.com/kurtosis-tech/stacktrace"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+	"os"
+	path2 "path"
+	"time"
+)
+
+type TestArtifact struct {
+	TestDescription    string                         `yaml:"test_description"`
+	ContainersTargeted []string                       `yaml:"fault_injection_targets"`
+	TestPassed         bool                           `yaml:"test_passed"`
+	HealthResult       *healthTypes.HealthCheckResult `yaml:"health_check_results"`
+}
+
+func BuildTestArtifact(
+	healthResults *healthTypes.HealthCheckResult,
+	podsUnderTest []*chaosMesh.PodUnderTest,
+	test types.SuiteTest,
+) *TestArtifact {
+
+	var containersTargeted []string
+	for _, p := range podsUnderTest {
+		containersTargeted = append(containersTargeted, p.GetName())
+	}
+
+	testPassed := health.AllChecksPassed(healthResults)
+
+	return &TestArtifact{
+		test.TestName,
+		containersTargeted,
+		testPassed,
+		healthResults,
+	}
+}
+
+func SerializeTestArtifacts(artifacts []*TestArtifact) error {
+	artifactFilename := fmt.Sprintf("results-%d.yaml", time.Now().UnixMilli())
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	path := path2.Join(cwd, "artifacts")
+
+	if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
+		err := os.Mkdir(path, os.ModePerm)
+		if err != nil {
+			log.Println(err)
+		}
+	}
+
+	artifactPath := path2.Join(path, artifactFilename)
+	bs, err := yaml.Marshal(artifacts)
+	if err != nil {
+		return stacktrace.Propagate(err, "could not marshal test artifacts")
+	}
+
+	err = os.WriteFile(artifactPath, bs, 0600)
+	if err != nil {
+		return stacktrace.Propagate(err, "could not write artifacts to %s", artifactPath)
+	}
+	log.Infof("Wrote test artifact to %s", artifactPath)
+	return nil
+}

pkg/artifacts/types.go

@@ -0,0 +1 @@
+package artifacts

pkg/health/checker.go

@@ -33,16 +33,18 @@ func BuildHealthChecker(cfg *confTypes.ConfigParsed, kubeClient *kubernetes.Kube
 	return &CheckOrchestrator{checkerImpl: checkerImpl, gracePeriod: healthCheckConfig.GracePeriod}, nil
 }
 
-func (hc *CheckOrchestrator) RunChecks(ctx context.Context) ([]*types.CheckResult, error) {
+func (hc *CheckOrchestrator) RunChecks(ctx context.Context) (*types.HealthCheckResult, error) {
 	start := time.Now()
 	latestAllowable := start.Add(hc.gracePeriod)
 	log.Infof("Allowing up to %.0f seconds for health checks to pass on all nodes", hc.gracePeriod.Seconds())
 
+	lastHealthCheckResult := &types.HealthCheckResult{}
 	for {
-		results, err := hc.checkerImpl.RunAllChecks(ctx)
+		results, err := hc.checkerImpl.RunAllChecks(ctx, lastHealthCheckResult)
 		if err != nil {
 			return nil, err
 		}
+		lastHealthCheckResult = results
 		if AllChecksPassed(results) {
 			timeToPass := time.Since(start).Seconds()
 			pctGraceUsed := timeToPass / hc.gracePeriod.Seconds() * 100
@@ -52,19 +54,27 @@ func (hc *CheckOrchestrator) RunChecks(ctx context.Context) ([]*types.CheckResul
 
 		if time.Now().After(latestAllowable) {
 			log.Warn("Grace period elapsed and a health check is still failing")
-			return results, stacktrace.NewError("tests failed")
+			return results, nil
 		} else {
 			log.Warn("Health checks failed but still in grace period")
 			time.Sleep(1 * time.Second)
 		}
 	}
 }
 
-func AllChecksPassed(checks []*types.CheckResult) bool {
-	for _, r := range checks {
-		if len(r.PodsFailing) != 0 {
-			return false
-		}
+func AllChecksPassed(checks *types.HealthCheckResult) bool {
+	if len(checks.LatestElBlockResult.FailingClientsReportedBlock) > 0 {
+		return false
+	}
+	if len(checks.LatestElBlockResult.FailingClientsReportedHash) > 0 {
+		return false
 	}
+	if len(checks.FinalizedElBlockResult.FailingClientsReportedBlock) > 0 {
+		return false
+	}
+	if len(checks.FinalizedElBlockResult.FailingClientsReportedHash) > 0 {
+		return false
+	}
+
 	return true
 }

pkg/health/ethereum/consensus.go

@@ -16,7 +16,7 @@ type ClientForkChoice struct {
 	BlockHash   string
 }
 
-func getExecNetworkConsensus(ctx context.Context, nodeClients []*ExecRpcClient, blockType string) ([]*ClientForkChoice, error) {
+func getExecNetworkConsensus(ctx context.Context, nodeClients []*ExecClientRPC, blockType string) ([]*ClientForkChoice, error) {
 	clientForkVotes := make([]*ClientForkChoice, len(nodeClients))
 	for i, client := range nodeClients {
 		choice, err := client.GetLatestBlockBy(ctx, blockType)