diff --git a/test/conformance.go b/test/conformance.go index 4ede42a2db53..c83f7dd35e09 100644 --- a/test/conformance.go +++ b/test/conformance.go @@ -45,6 +45,11 @@ const ( PizzaPlanetText2 = "Re-energize yourself with a slice of pepperoni!" HelloWorldText = "Hello World! How about some tasty noodles?" + // The Failing image will always exit with an exit code of 5 + ExitCodeReason = "ExitCode5" + // ... and will print "Crashed..." before it exits + ErrorLog = "Crashed..." + ConcurrentRequests = 50 // We expect to see 100% of requests succeed for traffic sent directly to revisions. // This might be a bad assumption. diff --git a/test/conformance/api/v1/errorcondition_test.go b/test/conformance/api/v1/errorcondition_test.go index a030c4442868..04ba9145f8e3 100644 --- a/test/conformance/api/v1/errorcondition_test.go +++ b/test/conformance/api/v1/errorcondition_test.go @@ -19,213 +19,67 @@ limitations under the License. package v1 import ( - "fmt" - "strings" + "errors" "testing" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ptest "knative.dev/pkg/test" - v1 "knative.dev/serving/pkg/apis/serving/v1" - serviceresourcenames "knative.dev/serving/pkg/reconciler/service/resources/names" + "knative.dev/pkg/apis" + "knative.dev/pkg/test/logging" "knative.dev/serving/test" - v1test "knative.dev/serving/test/v1" - - rtesting "knative.dev/serving/pkg/testing/v1" -) - -const ( - containerMissing = "ContainerMissing" + "knative.dev/serving/test/scenarios" + v1 "knative.dev/serving/test/v1" ) -// TestContainerErrorMsg is to validate the error condition defined at +// TestMustErrorContainerError is to validate the error condition defined at // https://github.com/knative/serving/blob/master/docs/spec/errors.md // for the container image missing scenario. -func TestContainerErrorMsg(t *testing.T) { - t.Parallel() - if strings.HasSuffix(strings.Split(ptest.Flags.DockerRepo, "/")[0], ".local") { - t.Skip("Skipping for local docker repo") - } - clients := test.Setup(t) - - names := test.ResourceNames{ - Service: test.ObjectNameForTest(t), - Image: test.InvalidHelloWorld, - } - - defer test.TearDown(clients, names) - test.CleanupOnInterrupt(func() { test.TearDown(clients, names) }) - - // Specify an invalid image path - // A valid DockerRepo is still needed, otherwise will get UNAUTHORIZED instead of container missing error - t.Logf("Creating a new Service %s", names.Service) - svc, err := createService(t, clients, names, 2) - if err != nil { - t.Fatalf("Failed to create Service: %v", err) - } - - names.Config = serviceresourcenames.Configuration(svc) - names.Route = serviceresourcenames.Route(svc) - - manifestUnknown := string(transport.ManifestUnknownErrorCode) - t.Log("When the imagepath is invalid, the Configuration should have error status.") - - // Wait for ServiceState becomes NotReady. It also waits for the creation of Configuration. - if err := v1test.WaitForServiceState(clients.ServingClient, names.Service, v1test.IsServiceNotReady, "ServiceIsNotReady"); err != nil { - t.Fatalf("The Service %s was unexpected state: %v", names.Service, err) - } - - // Checking for "Container image not present in repository" scenario defined in error condition spec - err = v1test.WaitForConfigurationState(clients.ServingClient, names.Config, func(r *v1.Configuration) (bool, error) { - cond := r.Status.GetCondition(v1.ConfigurationConditionReady) - if cond != nil && !cond.IsUnknown() { - if strings.Contains(cond.Message, manifestUnknown) && cond.IsFalse() { +func TestMustErrorOnContainerError(legacy *testing.T) { + t, cancel := logging.NewTLogger(legacy) + defer cancel() + + scenarios.ContainerError(t, + func(ts *logging.TLogger, cond *apis.Condition) (bool, error) { + // API Spec does not have constraints on the Message content + if cond.Message != "" { return true, nil } - t.Logf("Reason: %s ; Message: %s ; Status %s", cond.Reason, cond.Message, cond.Status) - return true, fmt.Errorf("The configuration %s was not marked with expected error condition (Reason=%q, Message=%q, Status=%q), but with (Reason=%q, Message=%q, Status=%q)", - names.Config, containerMissing, manifestUnknown, "False", cond.Reason, cond.Message, cond.Status) - } - return false, nil - }, "ContainerImageNotPresent") - - if err != nil { - t.Fatalf("Failed to validate configuration state: %s", err) - } - - revisionName, err := getRevisionFromConfiguration(clients, names.Config) - if err != nil { - t.Fatalf("Failed to get revision from configuration %s: %v", names.Config, err) - } - - t.Log("When the imagepath is invalid, the revision should have error status.") - err = v1test.WaitForRevisionState(clients.ServingClient, revisionName, func(r *v1.Revision) (bool, error) { - cond := r.Status.GetCondition(v1.RevisionConditionReady) - if cond != nil { - if cond.Reason == containerMissing && strings.Contains(cond.Message, manifestUnknown) { + ts.Fatal("The configuration was not marked with expected error condition", + "wantMessage", "!\"\"", "wantStatus", "False") + return true, errors.New("Shouldn't get here") + }, + func(ts *logging.TLogger, cond *apis.Condition) (bool, error) { + // API Spec does not have constraints on the Message content + if cond.Reason == v1.ContainerMissing && cond.Message != "" { return true, nil } - return true, fmt.Errorf("The revision %s was not marked with expected error condition (Reason=%q, Message=%q), but with (Reason=%q, Message=%q)", - revisionName, containerMissing, manifestUnknown, cond.Reason, cond.Message) - } - return false, nil - }, "ImagePathInvalid") - - if err != nil { - t.Fatalf("Failed to validate revision state: %s", err) - } - - t.Log("Checking to ensure Route is in desired state") - err = v1test.CheckRouteState(clients.ServingClient, names.Route, v1test.IsRouteNotReady) - if err != nil { - t.Fatalf("the Route %s was not desired state: %v", names.Route, err) - } + ts.Fatal("The revision was not marked with expected error condition", + "wantReason", v1.ContainerMissing, "wantMessage", "!\"\"") + return true, errors.New("Shouldn't get here") + }) } -// TestContainerExitingMsg is to validate the error condition defined at +// TestMustErrorContainerExiting is to validate the error condition defined at // https://github.com/knative/serving/blob/master/docs/spec/errors.md // for the container crashing scenario. -func TestContainerExitingMsg(t *testing.T) { - t.Parallel() - const ( - // The given image will always exit with an exit code of 5 - exitCodeReason = "ExitCode5" - // ... and will print "Crashed..." before it exits - errorLog = "Crashed..." - ) - - tests := []struct { - Name string - ReadinessProbe *corev1.Probe - }{{ - Name: "http", - ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{}, - }, - }, - }, { - Name: "tcp", - ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{ - TCPSocket: &corev1.TCPSocketAction{}, - }, - }, - }} - - for _, tt := range tests { - tt := tt - t.Run(tt.Name, func(t *testing.T) { - t.Parallel() - clients := test.Setup(t) - - names := test.ResourceNames{ - Config: test.ObjectNameForTest(t), - Image: test.Failing, - } - - defer test.TearDown(clients, names) - test.CleanupOnInterrupt(func() { test.TearDown(clients, names) }) - - t.Logf("Creating a new Configuration %s", names.Config) - - if _, err := v1test.CreateConfiguration(t, clients, names, rtesting.WithConfigReadinessProbe(tt.ReadinessProbe)); err != nil { - t.Fatalf("Failed to create configuration %s: %v", names.Config, err) - } - - t.Log("When the containers keep crashing, the Configuration should have error status.") - - err := v1test.WaitForConfigurationState(clients.ServingClient, names.Config, func(r *v1.Configuration) (bool, error) { - cond := r.Status.GetCondition(v1.ConfigurationConditionReady) - if cond != nil && !cond.IsUnknown() { - if strings.Contains(cond.Message, errorLog) && cond.IsFalse() { - return true, nil - } - t.Logf("Reason: %s ; Message: %s ; Status: %s", cond.Reason, cond.Message, cond.Status) - return true, fmt.Errorf("The configuration %s was not marked with expected error condition (Reason=%q, Message=%q, Status=%q), but with (Reason=%q, Message=%q, Status=%q)", - names.Config, containerMissing, errorLog, "False", cond.Reason, cond.Message, cond.Status) - } - return false, nil - }, "ConfigContainersCrashing") - - if err != nil { - t.Fatalf("Failed to validate configuration state: %s", err) - } - - revisionName, err := getRevisionFromConfiguration(clients, names.Config) - if err != nil { - t.Fatalf("Failed to get revision from configuration %s: %v", names.Config, err) +func TestMustErrorOnContainerExiting(legacy *testing.T) { + t, cancel := logging.NewTLogger(legacy) + defer cancel() + scenarios.ContainerExiting(t, + func(ts *logging.TLogger, cond *apis.Condition) (bool, error) { + // API Spec does not have constraints on the Message content + if cond.Message != "" { + return true, nil } - - t.Log("When the containers keep crashing, the revision should have error status.") - err = v1test.WaitForRevisionState(clients.ServingClient, revisionName, func(r *v1.Revision) (bool, error) { - cond := r.Status.GetCondition(v1.RevisionConditionReady) - if cond != nil { - if cond.Reason == exitCodeReason && strings.Contains(cond.Message, errorLog) { - return true, nil - } - return true, fmt.Errorf("The revision %s was not marked with expected error condition (Reason=%q, Message=%q), but with (Reason=%q, Message=%q)", - revisionName, exitCodeReason, errorLog, cond.Reason, cond.Message) - } - return false, nil - }, "RevisionContainersCrashing") - - if err != nil { - t.Fatalf("Failed to validate revision state: %s", err) + ts.Fatal("The configuration was not marked with expected error condition.", + "wantMessage", "!\"\"", "wantStatus", "False") + return true, errors.New("Shouldn't get here") + }, + func(ts *logging.TLogger, cond *apis.Condition) (bool, error) { + // API Spec does not have constraints on the Message content + if cond.Reason == test.ExitCodeReason && cond.Message != "" { + return true, nil } + ts.Fatal("The revision was not marked with expected error condition.", + "wantReason", test.ExitCodeReason, "wantMessage", "!\"\"") + return true, errors.New("Shouldn't get here") }) - } -} - -// Get revision name from configuration. -func getRevisionFromConfiguration(clients *test.Clients, configName string) (string, error) { - config, err := clients.ServingClient.Configs.Get(configName, metav1.GetOptions{}) - if err != nil { - return "", err - } - if config.Status.LatestCreatedRevisionName != "" { - return config.Status.LatestCreatedRevisionName, nil - } - return "", fmt.Errorf("No valid revision name found in configuration %s", configName) } diff --git a/test/conformance/api/v1/resources_test.go b/test/conformance/api/v1/resources_test.go index 5631b2698232..143b15ccd502 100644 --- a/test/conformance/api/v1/resources_test.go +++ b/test/conformance/api/v1/resources_test.go @@ -19,99 +19,55 @@ limitations under the License. package v1 import ( - "fmt" - "net/http" - "net/url" "testing" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - pkgTest "knative.dev/pkg/test" - "knative.dev/pkg/test/spoof" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/pkg/test/logging" "knative.dev/serving/test" v1test "knative.dev/serving/test/v1" rtesting "knative.dev/serving/pkg/testing/v1" ) -func TestCustomResourcesLimits(t *testing.T) { - t.Parallel() - clients := test.Setup(t) +var resourceLimit resource.Quantity - t.Log("Creating a new Route and Configuration") - withResources := rtesting.WithResourceRequirements(corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("350Mi"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("350Mi"), - }, - }) - - names := test.ResourceNames{ - Service: test.ObjectNameForTest(t), - Image: test.Autoscale, - } - - test.CleanupOnInterrupt(func() { test.TearDown(clients, names) }) - defer test.TearDown(clients, names) - - objects, err := v1test.CreateServiceReady(t, clients, &names, withResources) - if err != nil { - t.Fatalf("Failed to create initial Service %v: %v", names.Service, err) - } - endpoint := objects.Route.Status.URL.URL() - - _, err = pkgTest.WaitForEndpointState( - clients.KubeClient, - t.Logf, - endpoint, - v1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK)), - "ResourceTestServesText", - test.ServingFlags.ResolvableDomain) - if err != nil { - t.Fatalf("Error probing %s: %v", endpoint, err) - } - - sendPostRequest := func(resolvableDomain bool, url *url.URL) (*spoof.Response, error) { - t.Logf("Request %s", url) - client, err := pkgTest.NewSpoofingClient(clients.KubeClient, t.Logf, url.Hostname(), resolvableDomain) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPost, url.String(), nil) - if err != nil { - return nil, err - } - return client.Do(req) - } +func init() { + resourceLimit = resource.MustParse(test.ContainerMemoryLimit) +} - pokeCowForMB := func(mb int) error { - u, _ := url.Parse(endpoint.String()) - q := u.Query() - q.Set("bloat", fmt.Sprintf("%d", mb)) - u.RawQuery = q.Encode() - response, err := sendPostRequest(test.ServingFlags.ResolvableDomain, u) - if err != nil { - return err +func TestMustSetCustomResourcesLimits(legacy *testing.T) { + t, cancel := logging.NewTLogger(legacy) + defer cancel() + + clients, names, objects, cancel, err := v1test.CreateBasicServiceTest(t, + test.Autoscale, + rtesting.WithResourceRequirements(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resourceLimit, + }, + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resourceLimit, + }, + })) + defer cancel() + t.FatalIfErr(err, "Failed to create initial Service", "name", names.Service) + + t.Run("API", func(t *logging.TLogger) { + svc, err := clients.ServingClient.Revisions.Get(objects.Revision.Status.ServiceName, metav1.GetOptions{}) + t.FatalIfErr(err, "Failed requesting information about Revision") + + // TODO: need to not panic if any nil pointers/missing keys + resources := svc.Spec.Containers[0].Resources + limit := resources.Limits["memory"] + request := resources.Requests["memory"] + + if limit.Cmp(resourceLimit) != 0 { + t.Error("Memory limit did not match", "want", resourceLimit, "got", limit) } - if response.StatusCode != http.StatusOK { - return fmt.Errorf("StatusCode = %d, want %d", response.StatusCode, http.StatusOK) + if request.Cmp(resourceLimit) != 0 { + t.Error("Memory request did not match", "want", resourceLimit, "got", request) } - return nil - } - - t.Log("Querying the application to see if the memory limits are enforced.") - if err := pokeCowForMB(100); err != nil { - t.Fatalf("Didn't get a response from bloating cow with %d MBs of Memory: %v", 100, err) - } - - if err := pokeCowForMB(200); err != nil { - t.Fatalf("Didn't get a response from bloating cow with %d MBs of Memory: %v", 200, err) - } - - if err := pokeCowForMB(500); err == nil { - t.Fatalf("We shouldn't have got a response from bloating cow with %d MBs of Memory: %v", 500, err) - } + }) } diff --git a/test/conformance/api/v1/revision_timeout_test.go b/test/conformance/api/v1/revision_timeout_test.go index 62c1752f212d..88d8fa049fe0 100644 --- a/test/conformance/api/v1/revision_timeout_test.go +++ b/test/conformance/api/v1/revision_timeout_test.go @@ -41,7 +41,7 @@ import ( // createService creates a service in namespace with the name names.Service // that uses the image specified by names.Image -func createService(t *testing.T, clients *test.Clients, names test.ResourceNames, revisionTimeoutSeconds int64) (*v1.Service, error) { +func createService(t pkgTest.T, clients *test.Clients, names test.ResourceNames, revisionTimeoutSeconds int64) (*v1.Service, error) { service := v1test.Service(names, WithRevisionTimeoutSeconds(revisionTimeoutSeconds)) v1test.LogResourceObject(t, v1test.ResourceObjects{Service: service}) return clients.ServingClient.Services.Create(service) @@ -62,7 +62,7 @@ func updateServiceWithTimeout(clients *test.Clients, names test.ResourceNames, r } // sendRequests send a request to "endpoint", returns error if unexpected response code, nil otherwise. -func sendRequest(t *testing.T, kubeClient *pkgTest.KubeClient, endpoint *url.URL, +func sendRequest(t pkgTest.TLegacy, kubeClient *pkgTest.KubeClient, endpoint *url.URL, initialSleep, sleep time.Duration, expectedResponseCode int) error { client, err := pkgTest.NewSpoofingClient(kubeClient, t.Logf, endpoint.Hostname(), test.ServingFlags.ResolvableDomain) if err != nil { diff --git a/test/e2e/errorcondition_test.go b/test/e2e/errorcondition_test.go new file mode 100644 index 000000000000..8448a06bdfd8 --- /dev/null +++ b/test/e2e/errorcondition_test.go @@ -0,0 +1,87 @@ +// +build e2e + +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "knative.dev/pkg/apis" + "knative.dev/pkg/test/logging" + "knative.dev/serving/test" + "knative.dev/serving/test/scenarios" + v1 "knative.dev/serving/test/v1" +) + +func TestMustErrorOnContainerError(legacy *testing.T) { + t, cancel := logging.NewTLogger(legacy) + defer cancel() + + manifestUnknown := string(transport.ManifestUnknownErrorCode) + + scenarios.ContainerError(t, + func(ts *logging.TLogger, cond *apis.Condition) (bool, error) { + // API Spec does not have constraints on the Message, + // but we want to confirm it failed for the correct reason + if strings.Contains(cond.Message, manifestUnknown) { + return true, nil + } + ts.Fatal("The configuration was not marked with expected error condition", + "wantMessage", manifestUnknown, "wantStatus", "False") + return true, errors.New("Shouldn't get here") + }, + func(ts *logging.TLogger, cond *apis.Condition) (bool, error) { + // API Spec does not have constraints on the Message, + // but we want to confirm it failed for the correct reason + if cond.Reason == v1.ContainerMissing && strings.Contains(cond.Message, manifestUnknown) { + return true, nil + } + ts.Fatal("The revision was not marked with expected error condition", + "wantReason", v1.ContainerMissing, "wantMessage", "!\"\"") + return true, errors.New("Shouldn't get here") + }) +} + +func TestMustErrorOnContainerExiting(legacy *testing.T) { + t, cancel := logging.NewTLogger(legacy) + defer cancel() + scenarios.ContainerExiting(t, + func(ts *logging.TLogger, cond *apis.Condition) (bool, error) { + // API Spec does not have constraints on the Message, + // but we want to confirm it failed for the correct reason + if strings.Contains(cond.Message, test.ErrorLog) { + return true, nil + } + ts.Fatal("The configuration was not marked with expected error condition.", + "wantMessage", test.ErrorLog, "wantStatus", "False") + return true, errors.New("Shouldn't get here") + }, + func(ts *logging.TLogger, cond *apis.Condition) (bool, error) { + // API Spec does not have constraints on the Message, + // but we want to confirm it failed for the correct reason + if cond.Reason == test.ExitCodeReason && strings.Contains(cond.Message, test.ErrorLog) { + return true, nil + } + ts.Fatal("The revision was not marked with expected error condition.", + "wantReason", test.ExitCodeReason, "wantMessage", test.ErrorLog) + return true, errors.New("Shouldn't get here") + }) +} diff --git a/test/e2e/resources_test.go b/test/e2e/resources_test.go new file mode 100644 index 000000000000..a8ec5bec8421 --- /dev/null +++ b/test/e2e/resources_test.go @@ -0,0 +1,122 @@ +// +build e2e + +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "net/http" + "net/url" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/klog" + pkgTest "knative.dev/pkg/test" + "knative.dev/pkg/test/logging" + "knative.dev/pkg/test/spoof" + "knative.dev/serving/test" + v1test "knative.dev/serving/test/v1" + + rtesting "knative.dev/serving/pkg/testing/v1" +) + +var resourceLimit resource.Quantity + +func init() { + resourceLimit = resource.MustParse(test.ContainerMemoryLimit) +} + +func TestMustSetCustomResourcesLimits(legacy *testing.T) { + t, cancel := logging.NewTLogger(legacy) + defer cancel() + + clients, names, objects, cancel, err := v1test.CreateBasicServiceTest(t, + test.Autoscale, + rtesting.WithResourceRequirements(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resourceLimit, + }, + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resourceLimit, + }, + })) + defer cancel() + t.FatalIfErr(err, "Failed to create initial Service", "name", names.Service) + + // This is in e2e, not conformance, because k8s does not require implementations to terminate + // See https://github.com/knative/serving/pull/6014#issuecomment-553714724 + endpoint := objects.Route.Status.URL.URL() + _, err = pkgTest.WaitForEndpointState( + clients.KubeClient, + t.Logf, + endpoint, + v1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK)), + "ResourceTestServesText", + test.ServingFlags.ResolvableDomain) + t.FatalIfErr(err, "Error probing", "URL", endpoint) + + sendPostRequest := func(resolvableDomain bool, url *url.URL) (*spoof.Response, error) { + client, err := pkgTest.NewSpoofingClient(clients.KubeClient, klog.V(4).Infof, url.Hostname(), resolvableDomain) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, url.String(), nil) + if err != nil { + return nil, err + } + return client.Do(req) + } + + bloatAndCheck := func(mb int, wantSuccess bool) { + expect := "failure" + if wantSuccess { + expect = "success" + } + t.V(2).Info("Bloating", "MB increase", mb, "want", expect) + u, _ := url.Parse(endpoint.String()) + q := u.Query() + q.Set("bloat", fmt.Sprintf("%d", mb)) + u.RawQuery = q.Encode() + response, err := sendPostRequest(test.ServingFlags.ResolvableDomain, u) + if err != nil { + t.V(5).Info("Received error from sendPostRequest (may be expected)", "error", err) + if wantSuccess { + t.Error("Didn't get a response from bloating RAM", "MB", mb) + } + } else if response.StatusCode == http.StatusOK { + if !wantSuccess { + t.Error("We shouldn't have got a response from bloating RAM", "MB", mb) + } + } else if response.StatusCode == http.StatusBadRequest { + t.Error("Test Issue: Received BadRequest from test app, which probably means the test & test image are not cooperating with each other.") + } else { + // Accept all other StatusCode as failure; different systems could return 404, 502, etc on failure + t.V(5).Info("Received non-OK http code from sendPostRequest; interpreting as failure of bloat", "StatusCode", response.StatusCode) + if wantSuccess { + t.Error("Didn't get a good response from bloating RAM", "MB", mb) + } + } + } + + t.V(1).Info("Querying the application to see if the memory limits are enforced.") + bloatAndCheck(100, true) + bloatAndCheck(200, true) + bloatAndCheck(500, false) +} diff --git a/test/scenarios/container_failures.go b/test/scenarios/container_failures.go new file mode 100644 index 000000000000..63814f264ff1 --- /dev/null +++ b/test/scenarios/container_failures.go @@ -0,0 +1,198 @@ +// +build e2e + +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scenarios + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/pkg/apis" + pkgTest "knative.dev/pkg/test" + "knative.dev/pkg/test/logging" + v1 "knative.dev/serving/pkg/apis/serving/v1" + serviceresourcenames "knative.dev/serving/pkg/reconciler/service/resources/names" + rtesting "knative.dev/serving/pkg/testing/v1" + "knative.dev/serving/test" + v1test "knative.dev/serving/test/v1" + + . "knative.dev/serving/pkg/testing/v1" +) + +func ContainerError(t *logging.TLogger, falseConfigurationValidator func(*logging.TLogger, *apis.Condition) (bool, error), revisionValidator func(*logging.TLogger, *apis.Condition) (bool, error)) { + if strings.HasSuffix(strings.Split(pkgTest.Flags.DockerRepo, "/")[0], ".local") { + t.V(0).Info("Skipping for local docker repo") + t.SkipNow() + } + t.Parallel() + clients := test.Setup(t) + + names := test.ResourceNames{ + Service: test.ObjectNameForTest(t), + Image: test.InvalidHelloWorld, + } + + defer test.TearDown(clients, names) + test.CleanupOnInterrupt(func() { test.TearDown(clients, names) }) + + // Specify an invalid image path + // A valid DockerRepo is still needed, otherwise will get UNAUTHORIZED instead of container missing error + t.V(2).Info("Creating a new Service", "service", names.Service) + svc, err := v1test.CreateService(t, clients, names, WithRevisionTimeoutSeconds(2)) + t.FatalIfErr(err, "Failed to create Service") + + names.Config = serviceresourcenames.Configuration(svc) + names.Route = serviceresourcenames.Route(svc) + + t.Run("API", func(ts *logging.TLogger) { + ts.V(1).Info("When the imagepath is invalid, the Configuration should have error status.") + ts.V(8).Info("Wait for ServiceState becomes NotReady. It also waits for the creation of Configuration.") + err = v1test.WaitForServiceState(clients.ServingClient, names.Service, v1test.IsServiceNotReady, "ServiceIsNotReady") + ts.FatalIfErr(err, "The Service was unexpected state", + "service", names.Service) + + ts.V(8).Info("Checking for 'Container image not present in repository' scenario defined in error condition spec.") + err = v1test.WaitForConfigurationState(clients.ServingClient, names.Config, func(r *v1.Configuration) (bool, error) { + cond := r.Status.GetCondition(v1.ConfigurationConditionReady) + ts.WithValues("configuration", names.Config, "condition", cond) + v1test.ValidateCondition(ts, cond) + if cond != nil && !cond.IsUnknown() { + if cond.IsFalse() { + return falseConfigurationValidator(ts, cond) + } else { + ts.Fatal("Configuration should not have become ready") + } + } + return false, nil + }, "ContainerImageNotPresent") + + ts.FatalIfErr(err, "Failed to validate configuration state") + + revisionName, err := getRevisionFromConfiguration(clients, names.Config) + ts.FatalIfErr(err, "Failed to get revision from configuration", "configuration", names.Config) + + ts.V(1).Info("When the imagepath is invalid, the revision should have error status.") + err = v1test.WaitForRevisionState(clients.ServingClient, revisionName, func(r *v1.Revision) (bool, error) { + cond := r.Status.GetCondition(v1.RevisionConditionReady) + ts := ts.WithValues("revision", revisionName, "condition", cond) + v1test.ValidateCondition(ts, cond) + if cond != nil { + return revisionValidator(ts, cond) + } + return false, nil + }, "ImagePathInvalid") + + ts.FatalIfErr(err, "Failed to validate revision state") + + ts.V(1).Info("Checking to ensure Route is in desired state") + err = v1test.CheckRouteState(clients.ServingClient, names.Route, v1test.IsRouteNotReady) + ts.FatalIfErr(err, "The Route was not desired state", "route", names.Route) + }) +} + +func ContainerExiting(t *logging.TLogger, falseConfigurationValidator func(*logging.TLogger, *apis.Condition) (bool, error), revisionValidator func(*logging.TLogger, *apis.Condition) (bool, error)) { + t.Parallel() + tests := []struct { + Name string + ReadinessProbe *corev1.Probe + }{{ + Name: "http", + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{}, + }, + }, + }, { + Name: "tcp", + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + TCPSocket: &corev1.TCPSocketAction{}, + }, + }, + }} + + for _, tt := range tests { + tt := tt + t.Run(tt.Name, func(t *logging.TLogger) { + t.Parallel() + clients := test.Setup(t) + + names := test.ResourceNames{ + Config: test.ObjectNameForTest(t), + Image: test.Failing, + } + + defer test.TearDown(clients, names) + test.CleanupOnInterrupt(func() { test.TearDown(clients, names) }) + + t.Run("API", func(ts *logging.TLogger) { + ts.V(2).Info("Creating a new Configuration", "configuration", names.Config) + + _, err := v1test.CreateConfiguration(t, clients, names, rtesting.WithConfigReadinessProbe(tt.ReadinessProbe)) + ts.FatalIfErr(err, "Failed to create Configuration", "configuration", names.Config) + + ts.V(1).Info("When the containers keep crashing, the Configuration should have error status.") + + err = v1test.WaitForConfigurationState(clients.ServingClient, names.Config, func(r *v1.Configuration) (bool, error) { + cond := r.Status.GetCondition(v1.ConfigurationConditionReady) + ts := ts.WithValues("configuration", names.Config, "condition", cond) + v1test.ValidateCondition(ts, cond) + if cond != nil && !cond.IsUnknown() { + if cond.IsFalse() { + return falseConfigurationValidator(ts, cond) + } else { + ts.Fatal("Configuration should not have become ready") + } + } + return false, nil + }, "ConfigContainersCrashing") + + ts.FatalIfErr(err, "Failed to validate configuration state") + + revisionName, err := getRevisionFromConfiguration(clients, names.Config) + ts.FatalIfErr(err, "Failed to get revision from configuration", "configuration", names.Config) + + ts.V(1).Info("When the containers keep crashing, the revision should have error status.") + err = v1test.WaitForRevisionState(clients.ServingClient, revisionName, func(r *v1.Revision) (bool, error) { + cond := r.Status.GetCondition(v1.RevisionConditionReady) + ts := ts.WithValues("revision", revisionName, "condition", cond) + v1test.ValidateCondition(ts, cond) + if cond != nil { + return revisionValidator(ts, cond) + } + return false, nil + }, "RevisionContainersCrashing") + + ts.FatalIfErr(err, "Failed to validate revision state") + }) + }) + } +} + +// Get revision name from configuration. +func getRevisionFromConfiguration(clients *test.Clients, configName string) (string, error) { + config, err := clients.ServingClient.Configs.Get(configName, metav1.GetOptions{}) + if err != nil { + return "", err + } + if config.Status.LatestCreatedRevisionName != "" { + return config.Status.LatestCreatedRevisionName, nil + } + return "", logging.Error("No valid revision name found", "configuration", configName) +} diff --git a/test/v1/condition.go b/test/v1/condition.go new file mode 100644 index 000000000000..2dccfb04c280 --- /dev/null +++ b/test/v1/condition.go @@ -0,0 +1,48 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "regexp" + + corev1 "k8s.io/api/core/v1" + "knative.dev/pkg/apis" + "knative.dev/pkg/test/logging" +) + +var camelCaseRegex = regexp.MustCompile(`^[[:upper:]].*`) +var camelCaseSingleWordRegex = regexp.MustCompile(`^[[:upper:]][^[\s]]+$`) + +func ValidateCondition(t *logging.TLogger, c *apis.Condition) { + if c == nil { + return + } + if c.Type == "" { + t.Error("A Condition.Type must not be an empty string") + } else if !camelCaseRegex.MatchString(string(c.Type)) { + t.Error("A Condition.Type must be CamelCase, so must start with an upper-case letter") + } + if c.Status != corev1.ConditionTrue && c.Status != corev1.ConditionFalse && c.Status != corev1.ConditionUnknown { + t.Error("A Condition.Status must be True, False, or Unknown") + } + if c.Reason != "" && !camelCaseRegex.MatchString(c.Reason) { + t.Error("A Condition.Reason, if given, must be a single-word CamelCase") + } + if c.Severity != apis.ConditionSeverityError && c.Severity != apis.ConditionSeverityWarning && c.Severity != apis.ConditionSeverityInfo { + t.Error("A Condition.Status must be '', Warning, or Info") + } +} diff --git a/test/v1/constants.go b/test/v1/constants.go new file mode 100644 index 000000000000..86cb4f980b7f --- /dev/null +++ b/test/v1/constants.go @@ -0,0 +1,22 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +const ( + // TODO: shouldn't this be exported by the code so we can reuse it? It was in v1alpha1 + ContainerMissing = "ContainerMissing" +) diff --git a/test/v1/service.go b/test/v1/service.go index 11d3796c97f2..86665fffb961 100644 --- a/test/v1/service.go +++ b/test/v1/service.go @@ -124,7 +124,7 @@ func CreateServiceReady(t pkgTest.T, clients *test.Clients, names *test.Resource t.Log("Getting latest objects Created by Service") resources, err := GetResourceObjects(clients, *names) if err == nil { - t.Log("Successfully created Service", names.Service) + t.Log("Successfully created Service", "name", names.Service) } return resources, err } @@ -255,3 +255,23 @@ func IsServiceReady(s *v1.Service) (bool, error) { func IsServiceNotReady(s *v1.Service) (bool, error) { return s.Generation == s.Status.ObservedGeneration && !s.Status.IsReady(), nil } + +// CreateBasicServiceTest combines several steps common to many tests: +// test.Setup, define ResourceNames with auto-named Service + given image, arrange cleanup, +// and CreateServiceReady. +// Accepts multiple test.ServiceOption which get given to CreateServiceReady +func CreateBasicServiceTest(t *logging.TLogger, imageName string, options ...rtesting.ServiceOption) (*test.Clients, *test.ResourceNames, *ResourceObjects, func(), error) { + t.Parallel() + clients := test.Setup(t) + + names := &test.ResourceNames{ + Service: test.ObjectNameForTest(t), + Image: imageName, + } + + cleanup := func() { test.TearDown(clients, *names) } + test.CleanupOnInterrupt(cleanup) + + objects, err := CreateServiceReady(t, clients, names, options...) + return clients, names, objects, cleanup, err +}