Skip to content

Commit

Permalink
feat(RELEASE-1201): get pipelineRun logs from staging cluster
Browse files Browse the repository at this point in the history 
Signed-off-by: Jing Qi <[email protected]>

Update the error handling in the loop of running pipelineRun.
Store the pipelineRun log when pipelineRun failed.
Store release CR in AfterAll part.

feat: update createReleasePlan func

Signed-off-by: Jing Qi <[email protected]>
  • Loading branch information
@jinqi7
jinqi7 committed Oct 18, 2024
1 parent c69d596 commit 9a2d3aa
Show file tree
Hide file tree
Showing 8 changed files with 274 additions and 106 deletions.
39 changes: 39 additions & 0 deletions pkg/clients/release/releases.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ package release
import (
"context"
"fmt"
"strings"
"time"

"github.com/konflux-ci/e2e-tests/pkg/constants"
"github.com/konflux-ci/e2e-tests/pkg/logs"
"github.com/konflux-ci/e2e-tests/pkg/utils/tekton"
releaseApi "github.com/konflux-ci/release-service/api/v1alpha1"
. "github.com/onsi/ginkgo/v2"
Expand All @@ -17,6 +19,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"knative.dev/pkg/apis"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"
)

// CreateRelease creates a new Release using the given parameters.
Expand Down Expand Up @@ -101,6 +104,42 @@ func (r *ReleaseController) GetReleases(namespace string) (*releaseApi.ReleaseLi
return releaseList, err
}

// StoreRelease stores a given Release as an artifact.
func (r *ReleaseController) StoreRelease(release *releaseApi.Release) error {
artifacts := make(map[string][]byte)

releaseConditionStatus, err := r.GetReleaseConditionStatusMessages(release.Name, release.Namespace)
if err != nil {
return err
}
artifacts["release-condition-status-"+release.Name+".log"] = []byte(strings.Join(releaseConditionStatus, "\n"))

releaseYaml, err := yaml.Marshal(release)
if err != nil {
return err
}
artifacts["component-"+release.Name+".yaml"] = releaseYaml

if err := logs.StoreArtifacts(artifacts); err != nil {
return err
}

return nil
}

// Get the message from the status of a release. Useful for debugging purposes.
func (r *ReleaseController) GetReleaseConditionStatusMessages(name, namespace string) (messages []string, err error) {
release, err := r.GetRelease(name, "", namespace)
if err != nil {
return messages, fmt.Errorf("error getting Release: %v", err)
}
for _, condition := range release.Status.Conditions {
messages = append(messages, fmt.Sprintf("condition.Type: %s, condition.Status: %s, condition.Reason: %s\n",
condition.Type, condition.Status, condition.Reason))
}
return
}

// GetFirstReleaseInNamespace returns the first Release from list of releases in the given namespace.
func (r *ReleaseController) GetFirstReleaseInNamespace(namespace string) (*releaseApi.Release, error) {
releaseList, err := r.GetReleases(namespace)
Expand Down
57 changes: 28 additions & 29 deletions tests/release/pipelines/fbc_release.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/konflux-ci/e2e-tests/pkg/framework"
"github.com/konflux-ci/e2e-tests/pkg/utils"
"github.com/konflux-ci/e2e-tests/pkg/utils/tekton"
pipeline "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
releasecommon "github.com/konflux-ci/e2e-tests/tests/release"
releaseapi "github.com/konflux-ci/release-service/api/v1alpha1"
tektonutils "github.com/konflux-ci/release-service/tekton/utils"
Expand All @@ -32,6 +33,7 @@ const (

var snapshot *appservice.Snapshot
var releaseCR *releaseapi.Release
var pipelineRun *pipeline.PipelineRun
var err error
var devWorkspace = utils.GetEnv(constants.RELEASE_DEV_WORKSPACE_ENV, constants.DevReleaseTeam)
var managedWorkspace = utils.GetEnv(constants.RELEASE_MANAGED_WORKSPACE_ENV, constants.ManagedReleaseTeam)
Expand Down Expand Up @@ -64,8 +66,6 @@ var _ = framework.ReleasePipelinesSuiteDescribe("FBC e2e-tests", Label("release-
var fbcPreGAECPolicyName = "fbc-prega-policy-" + util.GenerateRandomString(4)
var sampleImage = "quay.io/hacbs-release-tests/fbc-sample-repo@sha256:857814679c1deec5bc5d6d8064832b4c0af40dcb07dad57c48f23e5ba6926aed"

AfterEach(framework.ReportFailure(&devFw))

Describe("with FBC happy path", Label("fbcHappyPath"), func() {
BeforeAll(func() {
devFw = releasecommon.NewFramework(devWorkspace)
Expand All @@ -91,6 +91,13 @@ var _ = framework.ReleasePipelinesSuiteDescribe("FBC e2e-tests", Label("release-
AfterAll(func() {
devFw = releasecommon.NewFramework(devWorkspace)
managedFw = releasecommon.NewFramework(managedWorkspace)
// store pipelineRun and Release CR
if err = managedFw.AsKubeDeveloper.TektonController.StorePipelineRun(pipelineRun.Name, pipelineRun); err != nil {
GinkgoWriter.Printf("failed to store PipelineRun %s:%s: %s\n", pipelineRun.GetNamespace(), pipelineRun.GetName(), err.Error())
}
if err = devFw.AsKubeDeveloper.ReleaseController.StoreRelease(releaseCR); err != nil {
GinkgoWriter.Printf("failed to store Release %s:%s: %s\n", releaseCR.GetNamespace(), releaseCR.GetName(), err.Error())
}
Expect(devFw.AsKubeDeveloper.HasController.DeleteApplication(fbcApplicationName, devNamespace, false)).NotTo(HaveOccurred())
Expect(managedFw.AsKubeDeveloper.TektonController.DeleteEnterpriseContractPolicy(fbcEnterpriseContractPolicyName, managedNamespace, false)).NotTo(HaveOccurred())
Expect(managedFw.AsKubeDeveloper.ReleaseController.DeleteReleasePlanAdmission(fbcReleasePlanAdmissionName, managedNamespace, false)).NotTo(HaveOccurred())
Expand Down Expand Up @@ -132,6 +139,13 @@ var _ = framework.ReleasePipelinesSuiteDescribe("FBC e2e-tests", Label("release-
AfterAll(func() {
devFw = releasecommon.NewFramework(devWorkspace)
managedFw = releasecommon.NewFramework(managedWorkspace)
// store pipelineRun and Release CR
if err = managedFw.AsKubeDeveloper.TektonController.StorePipelineRun(pipelineRun.Name, pipelineRun); err != nil {
GinkgoWriter.Printf("failed to store PipelineRun %s:%s: %s\n", pipelineRun.GetNamespace(), pipelineRun.GetName(), err.Error())
}
if err = devFw.AsKubeDeveloper.ReleaseController.StoreRelease(releaseCR); err != nil {
GinkgoWriter.Printf("failed to store Release %s:%s: %s\n", releaseCR.GetNamespace(), releaseCR.GetName(), err.Error())
}
Expect(devFw.AsKubeDeveloper.HasController.DeleteApplication(fbcHotfixAppName, devNamespace, false)).NotTo(HaveOccurred())
Expect(managedFw.AsKubeDeveloper.TektonController.DeleteEnterpriseContractPolicy(fbcHotfixECPolicyName, managedNamespace, false)).NotTo(HaveOccurred())
Expect(managedFw.AsKubeDeveloper.ReleaseController.DeleteReleasePlanAdmission(fbcHotfixRPAName, managedNamespace, false)).NotTo(HaveOccurred())
Expand Down Expand Up @@ -172,11 +186,16 @@ var _ = framework.ReleasePipelinesSuiteDescribe("FBC e2e-tests", Label("release-
AfterAll(func() {
devFw = releasecommon.NewFramework(devWorkspace)
managedFw = releasecommon.NewFramework(managedWorkspace)
if !CurrentSpecReport().Failed() {
Expect(devFw.AsKubeDeveloper.HasController.DeleteApplication(fbcPreGAAppName, devNamespace, false)).NotTo(HaveOccurred())
Expect(managedFw.AsKubeDeveloper.TektonController.DeleteEnterpriseContractPolicy(fbcPreGAECPolicyName, managedNamespace, false)).NotTo(HaveOccurred())
Expect(managedFw.AsKubeDeveloper.ReleaseController.DeleteReleasePlanAdmission(fbcPreGARPAName, managedNamespace, false)).NotTo(HaveOccurred())
// store pipelineRun and Release CR
if err = managedFw.AsKubeDeveloper.TektonController.StorePipelineRun(pipelineRun.Name, pipelineRun); err != nil {
GinkgoWriter.Printf("failed to store PipelineRun %s:%s: %s\n", pipelineRun.GetNamespace(), pipelineRun.GetName(), err.Error())
}
if err = devFw.AsKubeDeveloper.ReleaseController.StoreRelease(releaseCR); err != nil {
GinkgoWriter.Printf("failed to store Release %s:%s: %s\n", releaseCR.GetNamespace(), releaseCR.GetName(), err.Error())
}
Expect(devFw.AsKubeDeveloper.HasController.DeleteApplication(fbcPreGAAppName, devNamespace, false)).NotTo(HaveOccurred())
Expect(managedFw.AsKubeDeveloper.TektonController.DeleteEnterpriseContractPolicy(fbcPreGAECPolicyName, managedNamespace, false)).NotTo(HaveOccurred())
Expect(managedFw.AsKubeDeveloper.ReleaseController.DeleteReleasePlanAdmission(fbcPreGARPAName, managedNamespace, false)).NotTo(HaveOccurred())
})

var _ = Describe("FBC pre-GA post-release verification", func() {
Expand Down Expand Up @@ -206,7 +225,7 @@ func assertReleasePipelineRunSucceeded(devFw, managedFw *framework.Framework, de
}, 5*time.Minute, releasecommon.DefaultInterval).Should(Succeed(), "timed out when waiting for Release being created")

Eventually(func() error {
pipelineRun, err := managedFw.AsKubeAdmin.ReleaseController.GetPipelineRunInNamespace(managedNamespace, releaseCR.GetName(), releaseCR.GetNamespace())
pipelineRun, err = managedFw.AsKubeAdmin.ReleaseController.GetPipelineRunInNamespace(managedNamespace, releaseCR.GetName(), releaseCR.GetNamespace())
if err != nil {
return fmt.Errorf("PipelineRun has not been created yet for release %s/%s", releaseCR.GetNamespace(), releaseCR.GetName())
}
Expand Down Expand Up @@ -242,28 +261,8 @@ func assertReleaseCRSucceeded(devFw *framework.Framework, devNamespace, managedN
if err != nil {
return err
}
GinkgoWriter.Printf("releaseCR: %s", releaseCR.Name)
conditions := releaseCR.Status.Conditions
GinkgoWriter.Printf("len of conditions: %d", len(conditions))
if len(conditions) > 0 {
for _, c := range conditions {
GinkgoWriter.Printf("type of c: %s", c.Type)
if c.Type == "Released" {
GinkgoWriter.Printf("status of c: %s", c.Status)
if c.Status == "True" {
GinkgoWriter.Println("Release CR is released")
return nil
} else if c.Status == "False" {
GinkgoWriter.Println("Release CR failed")
Expect(string(c.Status)).To(Equal("True"), fmt.Sprintf("Release %s failed", releaseCR.Name))
return nil
} else {
return fmt.Errorf("release %s/%s is not marked as finished yet", releaseCR.GetNamespace(), releaseCR.GetName())
}
}
}
}
return nil
err = releasecommon.CheckReleaseStatus(releaseCR)
return err
}, releasecommon.ReleaseCreationTimeout, releasecommon.DefaultInterval).Should(Succeed())
}

Expand Down
66 changes: 51 additions & 15 deletions tests/release/pipelines/multiarch_advisories.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,25 @@ import (
"os"
"time"

ecp "github.com/enterprise-contract/enterprise-contract-controller/api/v1alpha1"
appservice "github.com/konflux-ci/application-api/api/v1alpha1"
corev1 "k8s.io/api/core/v1"
ecp "github.com/enterprise-contract/enterprise-contract-controller/api/v1alpha1"
"github.com/konflux-ci/e2e-tests/pkg/constants"
"github.com/konflux-ci/e2e-tests/pkg/framework"
"github.com/konflux-ci/e2e-tests/pkg/utils"
"github.com/konflux-ci/e2e-tests/pkg/utils/tekton"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"knative.dev/pkg/apis"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
pipeline "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
releasecommon "github.com/konflux-ci/e2e-tests/tests/release"
releaseapi "github.com/konflux-ci/release-service/api/v1alpha1"
tektonutils "github.com/konflux-ci/release-service/tekton/utils"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"knative.dev/pkg/apis"
)

const (
Expand Down Expand Up @@ -56,8 +59,7 @@ var _ = framework.ReleasePipelinesSuiteDescribe("e2e tests for multi arch test f

var snapshotPush *appservice.Snapshot
var releaseCR *releaseapi.Release

AfterEach(framework.ReportFailure(&devFw))
var pipelineRun *pipeline.PipelineRun

Describe("Multi-arch happy path", Label("MultiArchAdvisories"), func() {
BeforeAll(func() {
Expand Down Expand Up @@ -131,6 +133,16 @@ var _ = framework.ReleasePipelinesSuiteDescribe("e2e tests for multi arch test f
GinkgoWriter.Println("snapshotPush.Name: %s", snapshotPush.GetName())
})

AfterAll(func() {
// store pipelineRun and Release CR
if err = managedFw.AsKubeDeveloper.TektonController.StorePipelineRun(pipelineRun.Name, pipelineRun); err != nil {
GinkgoWriter.Printf("failed to store PipelineRun %s:%s: %s\n", pipelineRun.GetNamespace(), pipelineRun.GetName(), err.Error())
}
if err = devFw.AsKubeDeveloper.ReleaseController.StoreRelease(releaseCR); err != nil {
GinkgoWriter.Printf("failed to store Release %s:%s: %s\n", releaseCR.GetNamespace(), releaseCR.GetName(), err.Error())
}
})

var _ = Describe("Post-release verification", func() {

It("verifies the multiarch release pipelinerun is running and succeeds", func() {
Expand All @@ -143,7 +155,7 @@ var _ = framework.ReleasePipelinesSuiteDescribe("e2e tests for multi arch test f
}, 10*time.Minute, releasecommon.DefaultInterval).Should(Succeed())

Eventually(func() error {
pipelineRun, err := managedFw.AsKubeAdmin.ReleaseController.GetPipelineRunInNamespace(managedNamespace, releaseCR.GetName(), releaseCR.GetNamespace())
pipelineRun, err = managedFw.AsKubeAdmin.ReleaseController.GetPipelineRunInNamespace(managedNamespace, releaseCR.GetName(), releaseCR.GetNamespace())
if err != nil {
return fmt.Errorf("PipelineRun has not been created yet for release %s/%s", releaseCR.GetNamespace(), releaseCR.GetName())
}
Expand All @@ -158,11 +170,15 @@ var _ = framework.ReleasePipelinesSuiteDescribe("e2e tests for multi arch test f
if pipelineRun.GetStatusCondition().GetCondition(apis.ConditionSucceeded).IsTrue() {
return nil
} else {
var prLogs string
prLogs := ""
if prLogs, err = tekton.GetFailedPipelineRunLogs(managedFw.AsKubeAdmin.ReleaseController.KubeRest(), managedFw.AsKubeAdmin.ReleaseController.KubeInterface(), pipelineRun); err != nil {
return fmt.Errorf("failed to get PLR logs: %+v", err)
GinkgoWriter.Printf("failed to get PLR logs: %+v", err)
Expect(err).ShouldNot(HaveOccurred())
return nil
}
return fmt.Errorf("%s", prLogs)
GinkgoWriter.Printf("logs: %s", prLogs)
Expect(prLogs).To(Equal(""), fmt.Sprintf("PipelineRun %s failed", pipelineRun.Name))
return nil
}
}, releasecommon.BuildPipelineRunCompletionTimeout, releasecommon.DefaultInterval).Should(Succeed(), fmt.Sprintf("timed out when waiting for the release PipelineRun to be finished for the release %s/%s", releaseCR.GetName(), releaseCR.GetNamespace()))
})
Expand All @@ -173,12 +189,32 @@ var _ = framework.ReleasePipelinesSuiteDescribe("e2e tests for multi arch test f
if err != nil {
return err
}
GinkgoWriter.Println("Release CR: ", releaseCR.Name)
if !releaseCR.IsReleased() {
return fmt.Errorf("release %s/%s is not marked as finished yet", releaseCR.GetNamespace(), releaseCR.GetName())
GinkgoWriter.Println("releaseCR: %s", releaseCR.Name)
conditions := releaseCR.Status.Conditions
GinkgoWriter.Println("len of conditions: %d", len(conditions))
if len(conditions) > 0 {
for _, c := range conditions {
GinkgoWriter.Println("type of c: %s", c.Type)
if c.Type == "Released" {
GinkgoWriter.Println("status of c: %s", c.Status)
if c.Status == "True" {
GinkgoWriter.Println("Release CR is released")
return nil
} else if c.Status == "False" && c.Reason == "Progressing" {
return fmt.Errorf("release %s/%s is in progressing", releaseCR.GetNamespace(), releaseCR.GetName())
} else {
GinkgoWriter.Println("Release CR failed/skipped")
Expect(string(c.Status)).To(Equal("True"), fmt.Sprintf("Release %s failed/skipped", releaseCR.Name))
return nil
}
}
}
}
return nil
}, 10*time.Minute, releasecommon.DefaultInterval).Should(Succeed(), fmt.Sprintf("timed out when waiting for the Release CR %s/%s completed", releaseCR.GetName(), releaseCR.GetNamespace()))
if err = devFw.AsKubeDeveloper.ReleaseController.StoreRelease(releaseCR); err != nil {
GinkgoWriter.Printf("failed to store Release %s:%s: %s\n", releaseCR.GetNamespace(), releaseCR.GetName(), err.Error())
}
})
})
})
Expand Down
Loading

0 comments on commit 9a2d3aa

Please sign in to comment.