Fix TestCustomResourceLimits to be a conformance test

Include changes from knative#6006 work

test/conformance/api/v1/resources_test.go

@@ -26,25 +26,35 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog"
 	pkgTest "knative.dev/pkg/test"
+	"knative.dev/pkg/test/logging"
 	"knative.dev/pkg/test/spoof"
 	"knative.dev/serving/test"
 	v1test "knative.dev/serving/test/v1"
 
 	rtesting "knative.dev/serving/pkg/testing/v1"
 )
 
-func TestCustomResourcesLimits(t *testing.T) {
+var resourceLimit resource.Quantity
+
+func init() {
+	resourceLimit = resource.MustParse("350Mi")
+}
+
+func TestCustomResourcesLimits(legacy *testing.T) {
+	t := logging.NewTLogger(legacy)
+	defer t.CleanUp()
 	t.Parallel()
 	clients := test.Setup(t)
 
-	t.Log("Creating a new Route and Configuration")
 	withResources := rtesting.WithResourceRequirements(corev1.ResourceRequirements{
 		Limits: corev1.ResourceList{
-			corev1.ResourceMemory: resource.MustParse("350Mi"),
+			corev1.ResourceMemory: resourceLimit,
 		},
 		Requests: corev1.ResourceList{
-			corev1.ResourceMemory: resource.MustParse("350Mi"),
+			corev1.ResourceMemory: resourceLimit,
 		},
 	})
 
@@ -57,61 +67,85 @@ func TestCustomResourcesLimits(t *testing.T) {
 	defer test.TearDown(clients, names)
 
 	objects, err := v1test.CreateServiceReady(t, clients, &names, withResources)
-	if err != nil {
-		t.Fatalf("Failed to create initial Service %v: %v", names.Service, err)
-	}
-	endpoint := objects.Route.Status.URL.URL()
-
-	_, err = pkgTest.WaitForEndpointState(
-		clients.KubeClient,
-		t.Logf,
-		endpoint,
-		v1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK)),
-		"ResourceTestServesText",
-		test.ServingFlags.ResolvableDomain)
-	if err != nil {
-		t.Fatalf("Error probing %s: %v", endpoint, err)
-	}
+	t.FatalIfErr(err, "Failed to create initial Service", "name", names.Service)
 
-	sendPostRequest := func(resolvableDomain bool, url *url.URL) (*spoof.Response, error) {
-		t.Logf("Request %s", url)
-		client, err := pkgTest.NewSpoofingClient(clients.KubeClient, t.Logf, url.Hostname(), resolvableDomain)
-		if err != nil {
-			return nil, err
-		}
+	t.Run("API", func(t *logging.TLogger) {
+		svc, err := clients.ServingClient.Revisions.Get(objects.Revision.Status.ServiceName, metav1.GetOptions{})
+		t.FatalIfErr(err, "Failed requesting information about Revision")
 
-		req, err := http.NewRequest(http.MethodPost, url.String(), nil)
-		if err != nil {
-			return nil, err
-		}
-		return client.Do(req)
-	}
+		// TODO: need to not panic if any nil pointers/missing keys
+		resources := svc.Spec.Containers[0].Resources
+		limit := resources.Limits["memory"]
+		request := resources.Requests["memory"]
 
-	pokeCowForMB := func(mb int) error {
-		u, _ := url.Parse(endpoint.String())
-		q := u.Query()
-		q.Set("bloat", fmt.Sprintf("%d", mb))
-		u.RawQuery = q.Encode()
-		response, err := sendPostRequest(test.ServingFlags.ResolvableDomain, u)
-		if err != nil {
-			return err
+		if limit.Cmp(resourceLimit) != 0 {
+			t.Error("Memory limit did not match", "want", resourceLimit, "got", limit)
 		}
-		if response.StatusCode != http.StatusOK {
-			return fmt.Errorf("StatusCode = %d, want %d", response.StatusCode, http.StatusOK)
+		if request.Cmp(resourceLimit) != 0 {
+			t.Error("Memory request did not match", "want", resourceLimit, "got", request)
 		}
-		return nil
-	}
+	})
 
-	t.Log("Querying the application to see if the memory limits are enforced.")
-	if err := pokeCowForMB(100); err != nil {
-		t.Fatalf("Didn't get a response from bloating cow with %d MBs of Memory: %v", 100, err)
-	}
+	// This is e2e, not Runtime, because k8s does not require implementations to terminate
+	// See https://github.com/knative/serving/pull/6014#issuecomment-553714724
+	t.Run("e2e", func(t *logging.TLogger) {
+		endpoint := objects.Route.Status.URL.URL()
+		_, err = pkgTest.WaitForEndpointState(
+			clients.KubeClient,
+			t.Logf,
+			endpoint,
+			v1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK)),
+			"ResourceTestServesText",
+			test.ServingFlags.ResolvableDomain)
+		t.FatalIfErr(err, "Error probing", "URL", endpoint)
+
+		sendPostRequest := func(resolvableDomain bool, url *url.URL) (*spoof.Response, error) {
+			client, err := pkgTest.NewSpoofingClient(clients.KubeClient, klog.V(4).Infof, url.Hostname(), resolvableDomain)
+			if err != nil {
+				return nil, err
+			}
+
+			req, err := http.NewRequest(http.MethodPost, url.String(), nil)
+			if err != nil {
+				return nil, err
+			}
+			return client.Do(req)
+		}
 
-	if err := pokeCowForMB(200); err != nil {
-		t.Fatalf("Didn't get a response from bloating cow with %d MBs of Memory: %v", 200, err)
-	}
+		bloatAndCheck := func(mb int, wantSuccess bool) {
+			expect := "failure"
+			if wantSuccess {
+				expect = "success"
+			}
+			t.V(2).Info("Bloating", "MB increase", mb, "want", expect)
+			u, _ := url.Parse(endpoint.String())
+			q := u.Query()
+			q.Set("bloat", fmt.Sprintf("%d", mb))
+			u.RawQuery = q.Encode()
+			response, err := sendPostRequest(test.ServingFlags.ResolvableDomain, u)
+			if err != nil {
+				t.V(5).Info("Received error from sendPostRequest (may be expected)", "error", err)
+				if wantSuccess {
+					t.Error("Didn't get a response from bloating RAM", "MB", mb)
+				}
+			} else if response.StatusCode == http.StatusOK {
+				if !wantSuccess {
+					t.Error("We shouldn't have got a response from bloating RAM", "MB", mb)
+				}
+			} else if response.StatusCode == http.StatusBadRequest {
+				t.Error("Test Issue: Received BadRequest from test app, which probably means the test & test image are not cooperating with each other.")
+			} else {
+				// Accept all other StatusCode as failure; different systems could return 404, 502, etc on failure
+				t.V(5).Info("Received non-OK http code from sendPostRequest; interpreting as failure of bloat", "StatusCode", response.StatusCode)
+				if wantSuccess {
+					t.Error("Didn't get a good response from bloating RAM", "MB", mb)
+				}
+			}
+		}
 
-	if err := pokeCowForMB(500); err == nil {
-		t.Fatalf("We shouldn't have got a response from bloating cow with %d MBs of Memory: %v", 500, err)
-	}
+		t.V(1).Info("Querying the application to see if the memory limits are enforced.")
+		bloatAndCheck(100, true)
+		bloatAndCheck(200, true)
+		bloatAndCheck(500, false)
+	})
 }