Experiment: log error/success on teardown and look at private _outcome

Author: sergiitk

framework/test_cases/base_testcase.py

@@ -14,15 +14,33 @@
 """Base test case used for xds test suites."""
 import inspect
 import traceback
-from typing import Optional, Union
-import unittest
+from typing import Optional, TypeVar, Union
+import unittest.case
 
 from absl import logging
 from absl.testing import absltest
+from typing_extensions import override
+
+# Aliases
+_OutcomeType: TypeVar = unittest.case._Outcome  # noqa
 
 
 class BaseTestCase(absltest.TestCase):
-    # @override
+    @override
+    def tearDown(self) -> None:
+        if not (outcome := self._get_outcome()):
+            return
+        if outcome.success:
+            logging.info(
+                "----- PSM Test Case PASSED: %s -----\n", self.test_name
+            )
+            return
+
+        test_errors = [error for test, error in outcome.errors if test is self]
+        logging.error("----- PSM Test Case FAILED: %s -----", self.test_name)
+        self._log_test_errors(test_errors)
+
+    @override
     def run(self, result: Optional[unittest.TestResult] = None) -> None:
         super().run(result)
         # TODO(sergiitk): should this method be returning result? See
@@ -40,17 +58,18 @@ def run(self, result: Optional[unittest.TestResult] = None) -> None:
         )
         # Assume one test case will only have one status.
         if test_errors or test_failures:
-            total_errors = len(test_errors) + len(test_failures)
-            logging.error(
-                "----- PSM Test Case FAILED: %s -----", self.test_name
-            )
-            self._log_test_errors(test_errors, is_unexpected=True)
-            self._log_test_errors(test_failures)
-            logging.info(
-                "----- PSM Test Case %s Error Count: %s -----",
-                self.test_name,
-                total_errors,
-            )
+            pass
+            # total_errors = len(test_errors) + len(test_failures)
+            # logging.error(
+            #     "----- PSM Test Case FAILED: %s -----", self.test_name
+            # )
+            # self._log_test_errors(test_errors, is_unexpected=True)
+            # self._log_test_errors(test_failures)
+            # logging.info(
+            #     "----- PSM Test Case %s Error Count: %s -----",
+            #     self.test_name,
+            #     total_errors,
+            # )
         elif test_unexpected_successes:
             logging.error(
                 "----- PSM Test Case UNEXPECTEDLY SUCCEEDED: %s -----\n",
@@ -69,6 +88,9 @@ def run(self, result: Optional[unittest.TestResult] = None) -> None:
                 self.test_name,
             )
 
+    def _get_outcome(self) -> Optional[_OutcomeType]:
+        return getattr(self, "_outcome", None)
+
     @property
     def test_name(self) -> str:
         """Pretty test name (details in the description).

framework/xds_k8s_testcase.py

@@ -752,6 +752,8 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner:
         raise NotImplementedError
 
     def tearDown(self):
+        super().tearDown()
+
         logger.info("----- TestMethod %s teardown -----", self.test_name)
         logger.debug("Getting pods restart times")
         client_restarts: int = 0

tests/bootstrap_generator_test.py

@@ -123,6 +123,7 @@ def tearDownClass(cls):
         super().tearDownClass()
 
     def tearDown(self):
+        super().tearDown()
         logger.info("----- TestMethod %s teardown -----", self.test_name)
         retryer = retryers.constant_retryer(
             wait_fixed=_timedelta(seconds=10),
@@ -133,7 +134,6 @@ def tearDown(self):
             retryer(self._cleanup)
         except retryers.RetryError:
             logger.exception("Got error during teardown")
-        super().tearDown()
 
     def _cleanup(self):
         self.client_runner.cleanup(force=self.force_cleanup)
@@ -168,6 +168,7 @@ class BootstrapGeneratorServerTest(
     test_server: XdsTestServer
 
     def tearDown(self):
+        super().tearDown()
         logger.info("----- TestMethod %s teardown -----", self.test_name)
         retryer = retryers.constant_retryer(
             wait_fixed=_timedelta(seconds=10),
@@ -178,7 +179,6 @@ def tearDown(self):
             retryer(self._cleanup)
         except retryers.RetryError:
             logger.exception("Got error during teardown")
-        super().tearDown()
 
     def _cleanup(self):
         self.client_runner.cleanup(force=self.force_cleanup)