Fix test resource leakage and add timeouts to prevent hangs

Fixed critical test isolation issue where DeploymentTest resources were
being reused by ReplicaSetTest, causing infinite loops and CI timeouts.

Changes:
- ReplicaSetTest: Changed label from 'tier: backend' to 'tier: backend-rs'
  to prevent Deployment controller from adopting the ReplicaSet
- DeploymentTest: Added tearDown() method to ensure cleanup of Deployment
  and HPA resources after each test, preventing resource leakage
- Both tests: Added 60-second timeouts to all wait loops (scaling, deletion)
  to prevent infinite hangs and provide clear error messages

Root cause: DeploymentTest created a Deployment with selector 'tier: backend'
which adopted the ReplicaSet from ReplicaSetTest (also labeled 'tier: backend'),
causing the Deployment controller to scale the ReplicaSet to 0. ReplicaSetTest
then waited indefinitely for 2 replicas that would never appear.

This fix ensures tests are properly isolated and will fail fast with descriptive
errors rather than hanging for 25+ minutes.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

Author: cuppett

tests/DeploymentTest.php

@@ -10,6 +10,42 @@
 
 class DeploymentTest extends TestCase
 {
+    protected function tearDown(): void
+    {
+        // Clean up deployment and HPA if they exist
+        try {
+            $dep = $this->cluster->getDeploymentByName('mariadb');
+            if ($dep->exists()) {
+                $dep->delete();
+
+                $timeout = 30;
+                $start = time();
+                while ($dep->exists() && (time() - $start < $timeout)) {
+                    sleep(1);
+                }
+            }
+        } catch (\Exception $e) {
+            // Deployment doesn't exist, that's fine
+        }
+
+        try {
+            $hpa = $this->cluster->getHorizontalPodAutoscalerByName('deploy-mariadb');
+            if ($hpa->exists()) {
+                $hpa->delete();
+
+                $timeout = 30;
+                $start = time();
+                while ($hpa->exists() && (time() - $start < $timeout)) {
+                    sleep(1);
+                }
+            }
+        } catch (\Exception $e) {
+            // HPA doesn't exist, that's fine
+        }
+
+        parent::tearDown();
+    }
+
     public function test_deployment_build()
     {
         $mariadb = $this->createMariadbContainer();
@@ -233,15 +269,32 @@ public function runDeletionTests()
         $this->assertTrue($dep->delete());
         $this->assertTrue($hpa->delete());
 
+        $timeout = 60; // 60 second timeout
+        $start = time();
+
         while ($hpa->exists()) {
+            if (time() - $start > $timeout) {
+                $this->fail('Timeout waiting for HPA to be deleted');
+            }
             sleep(1);
         }
 
+        $start = time();
         while ($dep->exists()) {
+            if (time() - $start > $timeout) {
+                $this->fail('Timeout waiting for Deployment to be deleted');
+            }
             sleep(1);
         }
 
+        $start = time();
         while ($dep->getPods()->count() > 0) {
+            if (time() - $start > $timeout) {
+                $this->fail(sprintf(
+                    'Timeout waiting for Deployment pods to be deleted. Remaining: %d',
+                    $dep->getPods()->count()
+                ));
+            }
             sleep(1);
         }
 
@@ -277,7 +330,17 @@ public function runScalingTests()
 
         $scaler = $dep->scale(2);
 
+        $timeout = 60; // 60 second timeout
+        $start = time();
+
         while ($dep->getReadyReplicasCount() < 2 || $scaler->getReplicas() < 2) {
+            if (time() - $start > $timeout) {
+                $this->fail(sprintf(
+                    'Timeout waiting for deployment to scale to 2. Current state: ready=%d, scaler=%d',
+                    $dep->getReadyReplicasCount(),
+                    $scaler->getReplicas()
+                ));
+            }
             $scaler->refresh();
             $dep->refresh();
             sleep(1);

tests/ReplicaSetTest.php

@@ -27,7 +27,7 @@ public function test_replica_set_build()
 
         $this->assertEquals('apps/v1', $rs->getApiVersion());
         $this->assertEquals('mariadb-rs', $rs->getName());
-        $this->assertEquals(['tier' => 'backend'], $rs->getLabels());
+        $this->assertEquals(['tier' => 'backend-rs'], $rs->getLabels());
         $this->assertEquals(['mariadb/annotation' => 'yes'], $rs->getAnnotations());
         $this->assertEquals(3, $rs->getReplicas());
         $this->assertEquals($pod->getName(), $rs->getTemplate()->getName());
@@ -47,7 +47,7 @@ public function test_replica_set_from_yaml()
 
         $this->assertEquals('apps/v1', $rs->getApiVersion());
         $this->assertEquals('mariadb-rs', $rs->getName());
-        $this->assertEquals(['tier' => 'backend'], $rs->getLabels());
+        $this->assertEquals(['tier' => 'backend-rs'], $rs->getLabels());
         $this->assertEquals(['mariadb/annotation' => 'yes'], $rs->getAnnotations());
         $this->assertEquals(3, $rs->getReplicas());
         $this->assertEquals($pod->getName(), $rs->getTemplate()->getName());
@@ -85,7 +85,7 @@ public function runCreationTests()
 
         $rs = $this->cluster->replicaSet()
             ->setName('mariadb-rs')
-            ->setLabels(['tier' => 'backend'])
+            ->setLabels(['tier' => 'backend-rs'])
             ->setAnnotations(['mariadb/annotation' => 'yes'])
             ->setSelectors(['matchLabels' => ['app' => 'mariadb-rs']])
             ->setReplicas(1)
@@ -103,7 +103,7 @@ public function runCreationTests()
 
         $this->assertEquals('apps/v1', $rs->getApiVersion());
         $this->assertEquals('mariadb-rs', $rs->getName());
-        $this->assertEquals(['tier' => 'backend'], $rs->getLabels());
+        $this->assertEquals(['tier' => 'backend-rs'], $rs->getLabels());
         $this->assertEquals(['mariadb/annotation' => 'yes'], $rs->getAnnotations());
         $this->assertEquals(1, $rs->getReplicas());
         $this->assertEquals($pod->getName(), $rs->getTemplate()->getName());
@@ -170,7 +170,7 @@ public function runGetTests()
 
         $this->assertEquals('apps/v1', $rs->getApiVersion());
         $this->assertEquals('mariadb-rs', $rs->getName());
-        $this->assertEquals(['tier' => 'backend'], $rs->getLabels());
+        $this->assertEquals(['tier' => 'backend-rs'], $rs->getLabels());
         $this->assertEquals(['mariadb/annotation' => 'yes'], $rs->getAnnotations());
         $this->assertEquals(1, $rs->getReplicas());
 
@@ -187,7 +187,18 @@ public function runScalingTests()
 
         $this->assertTrue($rs->isSynced());
 
+        $timeout = 60; // 60 second timeout
+        $start = time();
+
         while ($rs->getReadyReplicasCount() < 2 || $scaler->getReplicas() < 2) {
+            if (time() - $start > $timeout) {
+                $this->fail(sprintf(
+                    'Timeout waiting for replicas to scale to 2. Current state: ready=%d, scaler=%d',
+                    $rs->getReadyReplicasCount(),
+                    $scaler->getReplicas()
+                ));
+            }
+
             $scaler->refresh();
             $rs->refresh();
 
@@ -212,7 +223,7 @@ public function runUpdateTests()
 
         $this->assertEquals('apps/v1', $rs->getApiVersion());
         $this->assertEquals('mariadb-rs', $rs->getName());
-        $this->assertEquals(['tier' => 'backend'], $rs->getLabels());
+        $this->assertEquals(['tier' => 'backend-rs'], $rs->getLabels());
         $this->assertEquals([], $rs->getAnnotations());
         $this->assertEquals(2, $rs->getReplicas());
 

tests/yaml/replicaset.yaml

@@ -3,7 +3,7 @@ kind: ReplicaSet
 metadata:
   name: mariadb-rs
   labels:
-    tier: backend
+    tier: backend-rs
   annotations:
     mariadb/annotation: "yes"
 spec: