Improve Quarkus testing feedback loop

Some background: Quarkus is restarted whenever a test resource is
restricted to the class or different from the ones from other tests and
also when the test profile is different.

There is a clever orderer in Quarkus that tries to order things cleverly
to minimize the number of restarts but... ITs and Failsafe are not
honoring the JUnit test orderer so you lose this benefit and you end up
with Quarkus being restarted a lot more often than strictly necessary.

So the first fix here is to consider the Quarkus tests as Surefire
tests, not failsafe tests.
If more practical for you, they can be isolated in a specific Surefire
execution but we should keep them executed with Surefire.

Second issue is that @WithKubernetesTestServer is restricted to a given
test clean, meaning every time you have this annotation, the server will
have to be restarted.
I will start discussions to see if we should change that but in the
meantime I added an annotation that has a less aggressive behavior.

Unfortunately, you are also affected by a Quarkus bug, for which I will
push a fix soon and that we will hopefully backport to the next 3.17.

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/s3/S3FeatureIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/s3/S3FeatureTest.java

@@ -40,7 +40,7 @@
 import org.jboss.sbomer.service.feature.sbom.k8s.model.GenerationRequest;
 import org.jboss.sbomer.service.feature.sbom.k8s.model.SbomGenerationStatus;
 import org.jboss.sbomer.service.feature.sbom.model.SbomGenerationRequest;
-import org.jboss.sbomer.service.test.integ.feature.s3.S3FeatureIT.S3ClientConfig;
+import org.jboss.sbomer.service.test.integ.feature.s3.S3FeatureTest.S3ClientConfig;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -59,7 +59,7 @@
 
 @QuarkusTest
 @TestProfile(S3ClientConfig.class)
-class S3FeatureIT {
+class S3FeatureTest {
 
     @Inject
     S3StorageHandler storageHandler;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/s3/S3StorageHandlerTestIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/s3/S3StorageHandlerTest.java

@@ -38,7 +38,7 @@
 
 @QuarkusTest
 @TestProfile(TestUmbProfile.class)
-public class S3StorageHandlerTestIT {
+public class S3StorageHandlerTest {
 
     @Inject
     S3StorageHandler storageHandler;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/RequestEventRepositoryIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/RequestEventRepositoryTest.java

@@ -19,7 +19,6 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.jboss.sbomer.core.config.request.ErrataAdvisoryRequestConfig;
 import org.jboss.sbomer.core.config.request.PncBuildRequestConfig;
@@ -30,6 +29,7 @@
 import org.jboss.sbomer.service.feature.sbom.model.RequestEvent;
 import org.jboss.sbomer.service.feature.sbom.service.RequestEventRepository;
 import org.jboss.sbomer.service.test.utils.QuarkusTransactionalTest;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Order;
@@ -39,16 +39,15 @@
 import com.fasterxml.jackson.databind.JsonNode;
 
 import io.quarkus.test.junit.TestProfile;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import jakarta.inject.Inject;
 import lombok.extern.slf4j.Slf4j;
 
 @QuarkusTransactionalTest
-@WithKubernetesTestServer
+@WithSharedKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
 @Slf4j
-class RequestEventRepositoryIT {
+class RequestEventRepositoryTest {
 
     @Inject
     RequestEventRepository repository;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/SBOMResourceRSQIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/SBOMResourceRSQTest.java

@@ -33,6 +33,7 @@
 import org.jboss.sbomer.service.feature.sbom.model.Sbom;
 import org.jboss.sbomer.service.feature.sbom.model.SbomGenerationRequest;
 import org.jboss.sbomer.service.feature.sbom.service.SbomService;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
@@ -43,14 +44,13 @@
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.mockito.InjectSpy;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import io.restassured.http.ContentType;
 import io.restassured.path.json.JsonPath;
 
 @QuarkusTest
-@WithKubernetesTestServer
+@WithSharedKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
-class SBOMResourceRSQIT {
+class SBOMResourceRSQTest {
 
     @InjectSpy
     SbomService sbomService;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/SBOMServiceTestIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/SBOMServiceTest.java

@@ -33,6 +33,7 @@
 import org.jboss.sbomer.service.feature.sbom.service.SbomRepository;
 import org.jboss.sbomer.service.feature.sbom.service.SbomService;
 import org.jboss.sbomer.service.rest.QueryParameters;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
@@ -42,15 +43,13 @@
 import io.quarkus.test.junit.QuarkusMock;
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.TestProfile;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import jakarta.inject.Inject;
 import lombok.extern.slf4j.Slf4j;
 
 @QuarkusTest
-@WithKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
 @Slf4j
-class SBOMServiceTestIT {
+class SBOMServiceTest {
 
     @Inject
     SbomService sbomService;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/SbomGenerationRequestRepositoryIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/SbomGenerationRequestRepositoryTest.java

@@ -50,22 +50,22 @@
 import org.jboss.sbomer.service.feature.sbom.service.SbomService;
 import org.jboss.sbomer.service.rest.QueryParameters;
 import org.jboss.sbomer.service.test.utils.QuarkusTransactionalTest;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.Test;
 
 import io.quarkus.panache.common.Parameters;
 import io.quarkus.test.junit.TestProfile;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import jakarta.annotation.PostConstruct;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 import jakarta.validation.Validator;
 
 @ApplicationScoped
 @QuarkusTransactionalTest
-@WithKubernetesTestServer
+@WithSharedKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
-class SbomGenerationRequestRepositoryIT {
+class SbomGenerationRequestRepositoryTest {
 
     @Inject
     Validator validator;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/SbomRepositoryIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/SbomRepositoryTest.java

@@ -42,6 +42,7 @@
 import org.jboss.sbomer.service.feature.sbom.service.SbomRepository;
 import org.jboss.sbomer.service.rest.QueryParameters;
 import org.jboss.sbomer.service.test.utils.QuarkusTransactionalTest;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.Test;
 
@@ -50,15 +51,14 @@
 
 import io.quarkus.logging.Log;
 import io.quarkus.test.junit.TestProfile;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import jakarta.inject.Inject;
 import jakarta.validation.ConstraintViolation;
 import jakarta.validation.Validator;
 
 @QuarkusTransactionalTest
-@WithKubernetesTestServer
+@WithSharedKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
-class SbomRepositoryIT {
+class SbomRepositoryTest {
 
     @Inject
     Validator validator;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/generation/SyftImageResourceIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/generation/SyftImageResourceTest.java

@@ -4,20 +4,20 @@
 
 import org.hamcrest.CoreMatchers;
 import org.jboss.sbomer.service.feature.sbom.service.SbomService;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.mockito.InjectSpy;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import io.restassured.http.ContentType;
 
 @QuarkusTest
-@WithKubernetesTestServer
+@WithSharedKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
-class SyftImageResourceIT {
+class SyftImageResourceTest {
 
     @InjectSpy
     SbomService sbomService;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/k8s/reconciler/GenerationPhaseGenerationRequestReconcilerIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/k8s/reconciler/GenerationPhaseGenerationRequestReconcilerTest.java

@@ -17,6 +17,7 @@
  */
 package org.jboss.sbomer.service.test.integ.feature.sbom.k8s.reconciler;
 
+import static org.junit.Assert.assertSame;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -43,6 +44,7 @@
 import org.jboss.sbomer.service.feature.sbom.k8s.model.SbomGenerationStatus;
 import org.jboss.sbomer.service.feature.sbom.k8s.reconciler.BuildController;
 import org.jboss.sbomer.service.feature.sbom.k8s.resources.Labels;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
@@ -63,16 +65,16 @@
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.kubernetes.client.KubernetesTestServer;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import jakarta.inject.Inject;
+import okhttp3.mockwebserver.RecordedRequest;
 
 /**
  * Class responsible for testing reconciliation workflow for the generation phase.
  */
 @QuarkusTest
-@WithKubernetesTestServer
+@WithSharedKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
-class GenerationPhaseGenerationRequestReconcilerIT {
+class GenerationPhaseGenerationRequestReconcilerTest {
 
     @InjectMock
     GenerationRequestControllerConfig controllerConfig;
@@ -433,6 +435,8 @@ void testFailedWithCleanup(@TempDir Path tempDir) throws Exception {
 
     @Test
     void testFailedWithoutCleanup(@TempDir Path tempDir) throws Exception {
+        RecordedRequest lastRequestPriorToExecution = mockServer.getLastRequest();
+
         when(controllerConfig.cleanup()).thenReturn(false);
         when(controllerConfig.sbomDir()).thenReturn(tempDir.toAbsolutePath().toString());
 
@@ -449,7 +453,9 @@ void testFailedWithoutCleanup(@TempDir Path tempDir) throws Exception {
                 .reconcile(request, mockContext(Collections.emptySet()));
 
         assertTrue(updateControl.isNoUpdate());
-        assertNull(mockServer.getLastRequest());
+        // there is no proper way to reset the last request at the beginning of the method
+        // so let's just compare it's the same as before and no requests have been executed.
+        assertSame(lastRequestPriorToExecution, mockServer.getLastRequest());
         assertTrue(Files.exists(tempDir));
         assertTrue(Files.exists(filePath));
         assertTrue(Files.exists(newDirPath));

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/k8s/reconciler/GenerationRequestReconcilerIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/k8s/reconciler/GenerationRequestReconcilerTest.java

@@ -40,7 +40,7 @@
 
 @QuarkusTest
 @Disabled("This doesn't work, because we are not starting Tekton properly, we need to think about different ways of running integration tests")
-class GenerationRequestReconcilerIT {
+class GenerationRequestReconcilerTest {
     @Inject
     Operator operator;
 

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/k8s/reconciler/InitializationPhaseGenerationRequestReconcilerIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/k8s/reconciler/InitializationPhaseGenerationRequestReconcilerTest.java

@@ -64,7 +64,7 @@
  */
 @QuarkusTest
 @TestProfile(TestUmbProfile.class)
-class InitializationPhaseGenerationRequestReconcilerIT {
+class InitializationPhaseGenerationRequestReconcilerTest {
 
     @Inject
     BuildController controller;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/messaging/AmqpMessageConsumerIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/messaging/AmqpMessageConsumerTest.java

@@ -33,7 +33,7 @@
 
 @QuarkusTest
 @TestProfile(TestUmbProfile.class)
-class AmqpMessageConsumerIT {
+class AmqpMessageConsumerTest {
     @Inject
     AmqpMessageConsumer consumer;
 

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/messaging/GenerationFinishedMessageBodyIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/messaging/GenerationFinishedMessageBodyTest.java

@@ -39,18 +39,18 @@
 import org.jboss.sbomer.service.feature.sbom.features.umb.producer.model.Sbom.BomFormat;
 import org.jboss.sbomer.service.feature.sbom.features.umb.producer.model.Sbom.OperationGenerationRequest;
 import org.jboss.sbomer.service.feature.sbom.features.umb.producer.model.Sbom.PncBuildGenerationRequest;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.Test;
 
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.TestProfile;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import jakarta.inject.Inject;
 
 @QuarkusTest
-@WithKubernetesTestServer
+@WithSharedKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
-class GenerationFinishedMessageBodyIT {
+class GenerationFinishedMessageBodyTest {
 
     @Inject
     GenerationFinishedMessageBodyValidator validator;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/messaging/PncBuildIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/messaging/PncBuildTest.java

@@ -51,6 +51,7 @@
 import org.jboss.sbomer.service.feature.sbom.service.SbomGenerationRequestRepository;
 import org.jboss.sbomer.service.test.PncWireMock;
 import org.jboss.sbomer.service.test.utils.AmqpMessageHelper;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
@@ -63,19 +64,18 @@
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.mockito.InjectSpy;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import io.smallrye.reactive.messaging.memory.InMemoryConnector;
 import io.smallrye.reactive.messaging.memory.InMemorySource;
 import io.vertx.core.json.JsonObject;
 import jakarta.inject.Inject;
 import lombok.extern.slf4j.Slf4j;
 
 @QuarkusTest
+@WithSharedKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
 @WithTestResource(PncWireMock.class)
 @Slf4j
-@WithKubernetesTestServer
-class PncBuildIT {
+class PncBuildTest {
 
     @Inject
     AmqpMessageConsumer consumer;

service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/umb/producer/NotificationServiceIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/feature/sbom/umb/producer/NotificationServiceTest.java

@@ -57,7 +57,7 @@
 
 @QuarkusTest
 @TestProfile(TestUmbProfile.class)
-class NotificationServiceIT {
+class NotificationServiceTest {
 
     @InjectMock
     FeatureFlags featureFlags;
@@ -67,7 +67,7 @@ static Path sbomPath(String fileName) {
     }
 
     private Sbom createOperationSBOM() throws IOException {
-        Bom bom = SbomUtils.fromPath(NotificationServiceIT.sbomPath("complete_operation_sbom.json"));
+        Bom bom = SbomUtils.fromPath(NotificationServiceTest.sbomPath("complete_operation_sbom.json"));
 
         SbomGenerationRequest generationRequest = SbomGenerationRequest.builder()
                 .withId("AABB")
@@ -86,7 +86,7 @@ private Sbom createOperationSBOM() throws IOException {
     }
 
     private Sbom createSBOM() throws IOException {
-        Bom bom = SbomUtils.fromPath(NotificationServiceIT.sbomPath("sbom_with_errata.json"));
+        Bom bom = SbomUtils.fromPath(NotificationServiceTest.sbomPath("sbom_with_errata.json"));
 
         SbomGenerationRequest generationRequest = SbomGenerationRequest.builder()
                 .withId("AABB")
@@ -105,7 +105,7 @@ private Sbom createSBOM() throws IOException {
     }
 
     private Sbom createMinimalRpmManifest() throws IOException {
-        Bom bom = SbomUtils.fromPath(NotificationServiceIT.sbomPath("minimal-rpm.json"));
+        Bom bom = SbomUtils.fromPath(NotificationServiceTest.sbomPath("minimal-rpm.json"));
 
         SbomGenerationRequest generationRequest = SbomGenerationRequest.builder()
                 .withId("AABB")

service/src/test/java/org/jboss/sbomer/service/test/integ/rest/ErrorResourcesIT.java → service/src/test/java/org/jboss/sbomer/service/test/integ/rest/ErrorResourcesTest.java

@@ -22,27 +22,26 @@
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.Matchers;
 import org.jboss.sbomer.service.feature.sbom.service.SbomService;
+import org.jboss.sbomer.service.test.utils.WithSharedKubernetesTestServer;
 import org.jboss.sbomer.service.test.utils.umb.TestUmbProfile;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.mockito.InjectSpy;
-import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 import io.restassured.RestAssured;
 import jakarta.ws.rs.ForbiddenException;
-import jakarta.ws.rs.NotAcceptableException;
 import jakarta.ws.rs.NotAllowedException;
 import jakarta.ws.rs.NotAuthorizedException;
 
 /**
  * Ensures that we can handle errors properly.
  */
 @QuarkusTest
-@WithKubernetesTestServer
+@WithSharedKubernetesTestServer
 @TestProfile(TestUmbProfile.class)
-class ErrorResourcesIT {
+class ErrorResourcesTest {
 
     @InjectSpy
     SbomService sbomService;