Skip to content

Commit

Permalink
Restructure Conformance API tests to match Spec
Browse files Browse the repository at this point in the history 
Fixes knative#6006 by making TestCustomResourceLimits to be a conformance
test and moving e2e parts out.

For knative#6226 fixes it for TestContainerErrorMsg and
TestContainerExitingMsg, though could be other tests checking errors
incorrectly.

Addresses knative#6264 though continuing work is needed.

Don't mix E2E and conformance:
Strip E2E portions of conformance out into tests in the test/e2e
directory. Commonality goes into test/scenarios or test/v1 depending
on specificity of the common functionality.

Rename to more align with runtime test naming (Must/Should).
  • Loading branch information
@coryrc
coryrc committed Feb 23, 2020
1 parent 9cae6c4 commit 0c07394
Show file tree
Hide file tree
Showing 10 changed files with 587 additions and 275 deletions.
5 changes: 5 additions & 0 deletions test/conformance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ const (
PizzaPlanetText2 = "Re-energize yourself with a slice of pepperoni!"
HelloWorldText = "Hello World! How about some tasty noodles?"

// The Failing image will always exit with an exit code of 5
ExitCodeReason = "ExitCode5"
// ... and will print "Crashed..." before it exits
ErrorLog = "Crashed..."

ConcurrentRequests = 50
// We expect to see 100% of requests succeed for traffic sent directly to revisions.
// This might be a bad assumption.
Expand Down
236 changes: 45 additions & 191 deletions test/conformance/api/v1/errorcondition_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,213 +19,67 @@ limitations under the License.
package v1

import (
"fmt"
"strings"
"errors"
"testing"

"github.com/google/go-containerregistry/pkg/v1/remote/transport"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ptest "knative.dev/pkg/test"
v1 "knative.dev/serving/pkg/apis/serving/v1"
serviceresourcenames "knative.dev/serving/pkg/reconciler/service/resources/names"
"knative.dev/pkg/apis"
"knative.dev/pkg/test/logging"
"knative.dev/serving/test"
v1test "knative.dev/serving/test/v1"

rtesting "knative.dev/serving/pkg/testing/v1"
)

const (
containerMissing = "ContainerMissing"
"knative.dev/serving/test/scenarios"
v1 "knative.dev/serving/test/v1"
)

// TestContainerErrorMsg is to validate the error condition defined at
// TestMustErrorContainerError is to validate the error condition defined at
// https://github.com/knative/serving/blob/master/docs/spec/errors.md
// for the container image missing scenario.
func TestContainerErrorMsg(t *testing.T) {
t.Parallel()
if strings.HasSuffix(strings.Split(ptest.Flags.DockerRepo, "/")[0], ".local") {
t.Skip("Skipping for local docker repo")
}
clients := test.Setup(t)

names := test.ResourceNames{
Service: test.ObjectNameForTest(t),
Image: test.InvalidHelloWorld,
}

defer test.TearDown(clients, names)
test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })

// Specify an invalid image path
// A valid DockerRepo is still needed, otherwise will get UNAUTHORIZED instead of container missing error
t.Logf("Creating a new Service %s", names.Service)
svc, err := createService(t, clients, names, 2)
if err != nil {
t.Fatalf("Failed to create Service: %v", err)
}

names.Config = serviceresourcenames.Configuration(svc)
names.Route = serviceresourcenames.Route(svc)

manifestUnknown := string(transport.ManifestUnknownErrorCode)
t.Log("When the imagepath is invalid, the Configuration should have error status.")

// Wait for ServiceState becomes NotReady. It also waits for the creation of Configuration.
if err := v1test.WaitForServiceState(clients.ServingClient, names.Service, v1test.IsServiceNotReady, "ServiceIsNotReady"); err != nil {
t.Fatalf("The Service %s was unexpected state: %v", names.Service, err)
}

// Checking for "Container image not present in repository" scenario defined in error condition spec
err = v1test.WaitForConfigurationState(clients.ServingClient, names.Config, func(r *v1.Configuration) (bool, error) {
cond := r.Status.GetCondition(v1.ConfigurationConditionReady)
if cond != nil && !cond.IsUnknown() {
if strings.Contains(cond.Message, manifestUnknown) && cond.IsFalse() {
func TestMustErrorOnContainerError(legacy *testing.T) {
t, cancel := logging.NewTLogger(legacy)
defer cancel()

scenarios.ContainerError(t,
func(ts *logging.TLogger, cond *apis.Condition) (bool, error) {
// API Spec does not have constraints on the Message content
if cond.Message != "" {
return true, nil
}
t.Logf("Reason: %s ; Message: %s ; Status %s", cond.Reason, cond.Message, cond.Status)
return true, fmt.Errorf("The configuration %s was not marked with expected error condition (Reason=%q, Message=%q, Status=%q), but with (Reason=%q, Message=%q, Status=%q)",
names.Config, containerMissing, manifestUnknown, "False", cond.Reason, cond.Message, cond.Status)
}
return false, nil
}, "ContainerImageNotPresent")

if err != nil {
t.Fatalf("Failed to validate configuration state: %s", err)
}

revisionName, err := getRevisionFromConfiguration(clients, names.Config)
if err != nil {
t.Fatalf("Failed to get revision from configuration %s: %v", names.Config, err)
}

t.Log("When the imagepath is invalid, the revision should have error status.")
err = v1test.WaitForRevisionState(clients.ServingClient, revisionName, func(r *v1.Revision) (bool, error) {
cond := r.Status.GetCondition(v1.RevisionConditionReady)
if cond != nil {
if cond.Reason == containerMissing && strings.Contains(cond.Message, manifestUnknown) {
ts.Fatal("The configuration was not marked with expected error condition",
"wantMessage", "!\"\"", "wantStatus", "False")
return true, errors.New("Shouldn't get here")
},
func(ts *logging.TLogger, cond *apis.Condition) (bool, error) {
// API Spec does not have constraints on the Message content
if cond.Reason == v1.ContainerMissing && cond.Message != "" {
return true, nil
}
return true, fmt.Errorf("The revision %s was not marked with expected error condition (Reason=%q, Message=%q), but with (Reason=%q, Message=%q)",
revisionName, containerMissing, manifestUnknown, cond.Reason, cond.Message)
}
return false, nil
}, "ImagePathInvalid")

if err != nil {
t.Fatalf("Failed to validate revision state: %s", err)
}

t.Log("Checking to ensure Route is in desired state")
err = v1test.CheckRouteState(clients.ServingClient, names.Route, v1test.IsRouteNotReady)
if err != nil {
t.Fatalf("the Route %s was not desired state: %v", names.Route, err)
}
ts.Fatal("The revision was not marked with expected error condition",
"wantReason", v1.ContainerMissing, "wantMessage", "!\"\"")
return true, errors.New("Shouldn't get here")
})
}

// TestContainerExitingMsg is to validate the error condition defined at
// TestMustErrorContainerExiting is to validate the error condition defined at
// https://github.com/knative/serving/blob/master/docs/spec/errors.md
// for the container crashing scenario.
func TestContainerExitingMsg(t *testing.T) {
t.Parallel()
const (
// The given image will always exit with an exit code of 5
exitCodeReason = "ExitCode5"
// ... and will print "Crashed..." before it exits
errorLog = "Crashed..."
)

tests := []struct {
Name string
ReadinessProbe *corev1.Probe
}{{
Name: "http",
ReadinessProbe: &corev1.Probe{
Handler: corev1.Handler{
HTTPGet: &corev1.HTTPGetAction{},
},
},
}, {
Name: "tcp",
ReadinessProbe: &corev1.Probe{
Handler: corev1.Handler{
TCPSocket: &corev1.TCPSocketAction{},
},
},
}}

for _, tt := range tests {
tt := tt
t.Run(tt.Name, func(t *testing.T) {
t.Parallel()
clients := test.Setup(t)

names := test.ResourceNames{
Config: test.ObjectNameForTest(t),
Image: test.Failing,
}

defer test.TearDown(clients, names)
test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })

t.Logf("Creating a new Configuration %s", names.Config)

if _, err := v1test.CreateConfiguration(t, clients, names, rtesting.WithConfigReadinessProbe(tt.ReadinessProbe)); err != nil {
t.Fatalf("Failed to create configuration %s: %v", names.Config, err)
}

t.Log("When the containers keep crashing, the Configuration should have error status.")

err := v1test.WaitForConfigurationState(clients.ServingClient, names.Config, func(r *v1.Configuration) (bool, error) {
cond := r.Status.GetCondition(v1.ConfigurationConditionReady)
if cond != nil && !cond.IsUnknown() {
if strings.Contains(cond.Message, errorLog) && cond.IsFalse() {
return true, nil
}
t.Logf("Reason: %s ; Message: %s ; Status: %s", cond.Reason, cond.Message, cond.Status)
return true, fmt.Errorf("The configuration %s was not marked with expected error condition (Reason=%q, Message=%q, Status=%q), but with (Reason=%q, Message=%q, Status=%q)",
names.Config, containerMissing, errorLog, "False", cond.Reason, cond.Message, cond.Status)
}
return false, nil
}, "ConfigContainersCrashing")

if err != nil {
t.Fatalf("Failed to validate configuration state: %s", err)
}

revisionName, err := getRevisionFromConfiguration(clients, names.Config)
if err != nil {
t.Fatalf("Failed to get revision from configuration %s: %v", names.Config, err)
func TestMustErrorOnContainerExiting(legacy *testing.T) {
t, cancel := logging.NewTLogger(legacy)
defer cancel()
scenarios.ContainerExiting(t,
func(ts *logging.TLogger, cond *apis.Condition) (bool, error) {
// API Spec does not have constraints on the Message content
if cond.Message != "" {
return true, nil
}

t.Log("When the containers keep crashing, the revision should have error status.")
err = v1test.WaitForRevisionState(clients.ServingClient, revisionName, func(r *v1.Revision) (bool, error) {
cond := r.Status.GetCondition(v1.RevisionConditionReady)
if cond != nil {
if cond.Reason == exitCodeReason && strings.Contains(cond.Message, errorLog) {
return true, nil
}
return true, fmt.Errorf("The revision %s was not marked with expected error condition (Reason=%q, Message=%q), but with (Reason=%q, Message=%q)",
revisionName, exitCodeReason, errorLog, cond.Reason, cond.Message)
}
return false, nil
}, "RevisionContainersCrashing")

if err != nil {
t.Fatalf("Failed to validate revision state: %s", err)
ts.Fatal("The configuration was not marked with expected error condition.",
"wantMessage", "!\"\"", "wantStatus", "False")
return true, errors.New("Shouldn't get here")
},
func(ts *logging.TLogger, cond *apis.Condition) (bool, error) {
// API Spec does not have constraints on the Message content
if cond.Reason == test.ExitCodeReason && cond.Message != "" {
return true, nil
}
ts.Fatal("The revision was not marked with expected error condition.",
"wantReason", test.ExitCodeReason, "wantMessage", "!\"\"")
return true, errors.New("Shouldn't get here")
})
}
}

// Get revision name from configuration.
func getRevisionFromConfiguration(clients *test.Clients, configName string) (string, error) {
config, err := clients.ServingClient.Configs.Get(configName, metav1.GetOptions{})
if err != nil {
return "", err
}
if config.Status.LatestCreatedRevisionName != "" {
return config.Status.LatestCreatedRevisionName, nil
}
return "", fmt.Errorf("No valid revision name found in configuration %s", configName)
}
Loading

0 comments on commit 0c07394

Please sign in to comment.