From af800a68d6fe588422af5639030310051b968bd6 Mon Sep 17 00:00:00 2001 From: Ryan Baxter Date: Thu, 21 Nov 2024 14:56:19 -0500 Subject: [PATCH] Add support for the configuration watcher to shut down the application to refresh the application See #1772 --- ...loud-kubernetes-configuration-watcher.adoc | 11 +- .../KubernetesDiscoveryProperties.java | 2 +- .../watcher/BusRefreshTrigger.java | 22 +++- ...urationWatcherConfigurationProperties.java | 26 ++++ .../watcher/HttpRefreshTrigger.java | 13 +- .../RefreshTriggerAutoConfiguration.java | 4 +- ...edConfigMapWatcherChangeDetectorTests.java | 46 +++++-- ...asedSecretsWatcherChangeDetectorTests.java | 48 ++++++-- ...asedConfigMapWatchChangeDetectorTests.java | 112 +++++++++++++----- ...pBasedSecretsWatchChangeDetectorTests.java | 85 +++++++++---- .../watcher/ActuatorRefreshIT.java | 26 ++++ .../configuration/watcher/TestUtil.java | 33 ++++++ 12 files changed, 343 insertions(+), 85 deletions(-) diff --git a/docs/modules/ROOT/pages/spring-cloud-kubernetes-configuration-watcher.adoc b/docs/modules/ROOT/pages/spring-cloud-kubernetes-configuration-watcher.adoc index c93649c095..30b6f17f49 100644 --- a/docs/modules/ROOT/pages/spring-cloud-kubernetes-configuration-watcher.adoc +++ b/docs/modules/ROOT/pages/spring-cloud-kubernetes-configuration-watcher.adoc @@ -193,6 +193,9 @@ change to a ConfigMap or Secret occurs then the HTTP implementation will use the instances of the application which match the name of the ConfigMap or Secret and send an HTTP POST request to the application's actuator `/refresh` endpoint. By default, it will send the post request to `/actuator/refresh` using the port registered in the discovery client. +You can also configure the configuration watcher to call the instances `shutdown` actuator endpoint. To do this you can set +`spring.cloud.kubernetes.configuration.watcher.refresh-strategy=shutdown`. + ### Non-Default Management Port and Actuator Path If the application is using a non-default actuator path and/or using a different port for the management endpoints, the Kubernetes service for the application @@ -224,7 +227,13 @@ Another way you can choose to configure the actuator path and/or management port ## Messaging Implementation The messaging implementation can be enabled by setting profile to either `bus-amqp` (RabbitMQ) or `bus-kafka` (Kafka) when the Spring Cloud Kubernetes Configuration Watcher -application is deployed to Kubernetes. +application is deployed to Kubernetes. By default, when using the messaging implementation the configuration watcher will send a `RefreshRemoteApplicationEvent` using +Spring Cloud Bus to all application instances. This will cause the application instances to refresh the application's configuration properties without +restarting the instance. + +You can also configure the configuration to shut down the application instances in order to refresh the application's configuration properties. +When the application shuts down, Kubernetes will restart the application instance and the new configuration properties will be loaded. To use +this strategy set `spring.cloud.kubernetes.configuration.watcher.refresh-strategy=shutdown`. ## Configuring RabbitMQ diff --git a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryProperties.java b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryProperties.java index a07f367ae4..1bee49cc20 100644 --- a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryProperties.java +++ b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryProperties.java @@ -27,7 +27,7 @@ /** * @param enabled if kubernetes discovery is enabled - * @param allNamespaces if discover is enabled for all namespaces + * @param allNamespaces if discovery is enabled for all namespaces * @param namespaces If set and allNamespaces is false, then only the services and * endpoints matching these namespaces will be fetched from the Kubernetes API server. * @param waitCacheReady wait for the discovery cache (service and endpoints) to be fully diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusRefreshTrigger.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusRefreshTrigger.java index 411fa68404..b88dec6a5c 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusRefreshTrigger.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusRefreshTrigger.java @@ -21,8 +21,12 @@ import org.springframework.cloud.bus.event.PathDestinationFactory; import org.springframework.cloud.bus.event.RefreshRemoteApplicationEvent; +import org.springframework.cloud.bus.event.RemoteApplicationEvent; +import org.springframework.cloud.bus.event.ShutdownRemoteApplicationEvent; import org.springframework.context.ApplicationEventPublisher; +import static org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties.RefreshStrategy.SHUTDOWN; + /** * An event publisher for an 'event bus' type of application. * @@ -34,16 +38,28 @@ final class BusRefreshTrigger implements RefreshTrigger { private final String busId; - BusRefreshTrigger(ApplicationEventPublisher applicationEventPublisher, String busId) { + private final ConfigurationWatcherConfigurationProperties watcherConfigurationProperties; + + BusRefreshTrigger(ApplicationEventPublisher applicationEventPublisher, String busId, + ConfigurationWatcherConfigurationProperties watcherConfigurationProperties) { this.applicationEventPublisher = applicationEventPublisher; this.busId = busId; + this.watcherConfigurationProperties = watcherConfigurationProperties; } @Override public Mono triggerRefresh(KubernetesObject configMap, String appName) { - applicationEventPublisher.publishEvent(new RefreshRemoteApplicationEvent(configMap, busId, - new PathDestinationFactory().getDestination(appName))); + applicationEventPublisher.publishEvent(createRefreshApplicationEvent(configMap, appName)); return Mono.empty(); } + private RemoteApplicationEvent createRefreshApplicationEvent(KubernetesObject configMap, String appName) { + if (watcherConfigurationProperties.getRefreshStrategy() == SHUTDOWN) { + return new ShutdownRemoteApplicationEvent(configMap, busId, + new PathDestinationFactory().getDestination(appName)); + } + return new RefreshRemoteApplicationEvent(configMap, busId, + new PathDestinationFactory().getDestination(appName)); + } + } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java index 47e9474d5b..964a13cf72 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java @@ -70,6 +70,8 @@ public class ConfigurationWatcherConfigurationProperties { @DurationUnit(ChronoUnit.MILLIS) private Duration refreshDelay = Duration.ofMillis(120000); + private RefreshStrategy refreshStrategy = RefreshStrategy.REFRESH; + private int threadPoolSize = 1; private String actuatorPath = "/actuator"; @@ -115,4 +117,28 @@ public void setThreadPoolSize(int threadPoolSize) { this.threadPoolSize = threadPoolSize; } + public RefreshStrategy getRefreshStrategy() { + return refreshStrategy; + } + + public void setRefreshStrategy(RefreshStrategy refreshStrategy) { + this.refreshStrategy = refreshStrategy; + } + + public enum RefreshStrategy { + + /** + * Call the Actuator refresh endpoint or send a refresh event over Spring Cloud + * Bus. + */ + REFRESH, + + /** + * Call the Actuator shutdown endpoint or send a shutdown event over Spring Cloud + * Bus. + */ + SHUTDOWN + + } + } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpRefreshTrigger.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpRefreshTrigger.java index 29ef23d3dc..91ba617f05 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpRefreshTrigger.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpRefreshTrigger.java @@ -31,6 +31,8 @@ import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.util.UriComponentsBuilder; +import static org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties.RefreshStrategy.SHUTDOWN; + /** * @author wind57 */ @@ -91,7 +93,7 @@ private URI getActuatorUri(ServiceInstance si, String actuatorPath, int actuator } else { int port = actuatorPort < 0 ? si.getPort() : actuatorPort; - actuatorUriBuilder = actuatorUriBuilder.path(actuatorPath + "/refresh").port(port); + actuatorUriBuilder = actuatorUriBuilder.path(actuatorPath + getRefreshStrategyEndpoint()).port(port); } return actuatorUriBuilder.build().toUri(); @@ -99,7 +101,7 @@ private URI getActuatorUri(ServiceInstance si, String actuatorPath, int actuator private void setActuatorUriFromAnnotation(UriComponentsBuilder actuatorUriBuilder, String metadataUri) { URI annotationUri = URI.create(metadataUri); - actuatorUriBuilder.path(annotationUri.getPath() + "/refresh"); + actuatorUriBuilder.path(annotationUri.getPath() + getRefreshStrategyEndpoint()); // The URI may not contain a host so if that is the case the port in the URI will // be -1. The authority of the URI will be : for example :9090, we just need @@ -114,4 +116,11 @@ private void setActuatorUriFromAnnotation(UriComponentsBuilder actuatorUriBuilde } } + private String getRefreshStrategyEndpoint() { + if (k8SConfigurationProperties.getRefreshStrategy() == SHUTDOWN) { + return "/shutdown"; + } + return "/refresh"; + } + } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/RefreshTriggerAutoConfiguration.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/RefreshTriggerAutoConfiguration.java index 40c6477f50..38341310fe 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/RefreshTriggerAutoConfiguration.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/RefreshTriggerAutoConfiguration.java @@ -41,8 +41,8 @@ class RefreshTriggerAutoConfiguration { @ConditionalOnMissingBean @Profile({ AMQP, KAFKA }) BusRefreshTrigger busRefreshTrigger(ApplicationEventPublisher applicationEventPublisher, - BusProperties busProperties) { - return new BusRefreshTrigger(applicationEventPublisher, busProperties.getId()); + BusProperties busProperties, ConfigurationWatcherConfigurationProperties properties) { + return new BusRefreshTrigger(applicationEventPublisher, busProperties.getId(), properties); } @Bean diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetectorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetectorTests.java index d15ef54318..338bc9761c 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetectorTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetectorTests.java @@ -28,6 +28,8 @@ import org.springframework.cloud.bus.BusProperties; import org.springframework.cloud.bus.event.RefreshRemoteApplicationEvent; +import org.springframework.cloud.bus.event.RemoteApplicationEvent; +import org.springframework.cloud.bus.event.ShutdownRemoteApplicationEvent; import org.springframework.cloud.kubernetes.client.config.KubernetesClientConfigMapPropertySourceLocator; import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; import org.springframework.cloud.kubernetes.commons.config.reload.ConfigReloadProperties; @@ -39,6 +41,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.verify; import static org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider.NAMESPACE_PROPERTY; +import static org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties.RefreshStrategy; /** * @author Ryan Baxter @@ -68,31 +71,52 @@ class BusEventBasedConfigMapWatcherChangeDetectorTests { private BusProperties busProperties; + private MockEnvironment mockEnvironment; + @BeforeEach void setup() { - MockEnvironment mockEnvironment = new MockEnvironment(); + mockEnvironment = new MockEnvironment(); mockEnvironment.setProperty(NAMESPACE_PROPERTY, "default"); - ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); busProperties = new BusProperties(); - changeDetector = new BusEventBasedConfigMapWatcherChangeDetector(coreV1Api, mockEnvironment, - ConfigReloadProperties.DEFAULT, UPDATE_STRATEGY, configMapPropertySourceLocator, - new KubernetesNamespaceProvider(mockEnvironment), configurationWatcherConfigurationProperties, - threadPoolTaskExecutor, new BusRefreshTrigger(applicationEventPublisher, busProperties.getId())); } @Test void triggerRefreshWithConfigMap() { - V1ObjectMeta objectMeta = new V1ObjectMeta(); - objectMeta.setName("foo"); - V1ConfigMap configMap = new V1ConfigMap(); - configMap.setMetadata(objectMeta); - changeDetector.triggerRefresh(configMap, configMap.getMetadata().getName()); ArgumentCaptor argumentCaptor = ArgumentCaptor .forClass(RefreshRemoteApplicationEvent.class); + triggerRefreshWithConfigMap(RefreshStrategy.REFRESH, argumentCaptor); + } + + @Test + void triggerRefreshWithConfigMapUsingShutdown() { + ArgumentCaptor argumentCaptor = ArgumentCaptor + .forClass(ShutdownRemoteApplicationEvent.class); + triggerRefreshWithConfigMap(RefreshStrategy.SHUTDOWN, argumentCaptor); + } + + void triggerRefreshWithConfigMap(RefreshStrategy strategy, + ArgumentCaptor argumentCaptor) { + V1ObjectMeta objectMeta = new V1ObjectMeta(); + objectMeta.setName("foo"); + V1ConfigMap configMap = getV1ConfigMap(objectMeta, strategy); verify(applicationEventPublisher).publishEvent(argumentCaptor.capture()); assertThat(argumentCaptor.getValue().getSource()).isEqualTo(configMap); assertThat(argumentCaptor.getValue().getOriginService()).isEqualTo(busProperties.getId()); assertThat(argumentCaptor.getValue().getDestinationService()).isEqualTo("foo:**"); } + private V1ConfigMap getV1ConfigMap(V1ObjectMeta objectMeta, RefreshStrategy refreshStrategy) { + V1ConfigMap configMap = new V1ConfigMap(); + configMap.setMetadata(objectMeta); + ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); + configurationWatcherConfigurationProperties.setRefreshStrategy(refreshStrategy); + BusEventBasedConfigMapWatcherChangeDetector changeDetector = new BusEventBasedConfigMapWatcherChangeDetector( + coreV1Api, mockEnvironment, ConfigReloadProperties.DEFAULT, UPDATE_STRATEGY, + configMapPropertySourceLocator, new KubernetesNamespaceProvider(mockEnvironment), + configurationWatcherConfigurationProperties, threadPoolTaskExecutor, new BusRefreshTrigger( + applicationEventPublisher, busProperties.getId(), configurationWatcherConfigurationProperties)); + changeDetector.triggerRefresh(configMap, configMap.getMetadata().getName()); + return configMap; + } + } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetectorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetectorTests.java index b79a068b32..3ad8529678 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetectorTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetectorTests.java @@ -28,6 +28,7 @@ import org.springframework.cloud.bus.BusProperties; import org.springframework.cloud.bus.event.RefreshRemoteApplicationEvent; +import org.springframework.cloud.bus.event.RemoteApplicationEvent; import org.springframework.cloud.kubernetes.client.config.KubernetesClientSecretsPropertySourceLocator; import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; import org.springframework.cloud.kubernetes.commons.config.reload.ConfigReloadProperties; @@ -39,6 +40,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.verify; import static org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider.NAMESPACE_PROPERTY; +import static org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties.RefreshStrategy; /** * @author Ryan Baxter @@ -64,35 +66,55 @@ class BusEventBasedSecretsWatcherChangeDetectorTests { @Mock private ApplicationEventPublisher applicationEventPublisher; - private BusEventBasedSecretsWatcherChangeDetector changeDetector; - private BusProperties busProperties; + private MockEnvironment mockEnvironment; + @BeforeEach void setup() { - MockEnvironment mockEnvironment = new MockEnvironment(); + mockEnvironment = new MockEnvironment(); mockEnvironment.setProperty(NAMESPACE_PROPERTY, "default"); - ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); busProperties = new BusProperties(); - changeDetector = new BusEventBasedSecretsWatcherChangeDetector(coreV1Api, mockEnvironment, - ConfigReloadProperties.DEFAULT, UPDATE_STRATEGY, secretsPropertySourceLocator, - new KubernetesNamespaceProvider(mockEnvironment), configurationWatcherConfigurationProperties, - threadPoolTaskExecutor, new BusRefreshTrigger(applicationEventPublisher, busProperties.getId())); } @Test void triggerRefreshWithSecret() { - V1ObjectMeta objectMeta = new V1ObjectMeta(); - objectMeta.setName("foo"); - V1Secret secret = new V1Secret(); - secret.setMetadata(objectMeta); - changeDetector.triggerRefresh(secret, secret.getMetadata().getName()); ArgumentCaptor argumentCaptor = ArgumentCaptor .forClass(RefreshRemoteApplicationEvent.class); + triggerRefreshWithSecret(ConfigurationWatcherConfigurationProperties.RefreshStrategy.REFRESH, argumentCaptor); + } + + @Test + void triggerRefreshWithSecretWithShutdown() { + ArgumentCaptor argumentCaptor = ArgumentCaptor + .forClass(RefreshRemoteApplicationEvent.class); + triggerRefreshWithSecret(ConfigurationWatcherConfigurationProperties.RefreshStrategy.REFRESH, argumentCaptor); + } + + void triggerRefreshWithSecret(RefreshStrategy strategy, + ArgumentCaptor argumentCaptor) { + V1ObjectMeta objectMeta = new V1ObjectMeta(); + objectMeta.setName("foo"); + V1Secret secret = getV1Secret(objectMeta, strategy); verify(applicationEventPublisher).publishEvent(argumentCaptor.capture()); assertThat(argumentCaptor.getValue().getSource()).isEqualTo(secret); assertThat(argumentCaptor.getValue().getOriginService()).isEqualTo(busProperties.getId()); assertThat(argumentCaptor.getValue().getDestinationService()).isEqualTo("foo:**"); } + private V1Secret getV1Secret(V1ObjectMeta objectMeta, + ConfigurationWatcherConfigurationProperties.RefreshStrategy refreshStrategy) { + V1Secret secret = new V1Secret(); + secret.setMetadata(objectMeta); + ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); + configurationWatcherConfigurationProperties.setRefreshStrategy(refreshStrategy); + BusEventBasedSecretsWatcherChangeDetector changeDetector = new BusEventBasedSecretsWatcherChangeDetector( + coreV1Api, mockEnvironment, ConfigReloadProperties.DEFAULT, UPDATE_STRATEGY, + secretsPropertySourceLocator, new KubernetesNamespaceProvider(mockEnvironment), + configurationWatcherConfigurationProperties, threadPoolTaskExecutor, new BusRefreshTrigger( + applicationEventPublisher, busProperties.getId(), configurationWatcherConfigurationProperties)); + changeDetector.triggerRefresh(secret, secret.getMetadata().getName()); + return secret; + } + } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetectorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetectorTests.java index 7c6a3c6b5a..e0b237992a 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetectorTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetectorTests.java @@ -30,6 +30,7 @@ import io.kubernetes.client.openapi.models.V1EndpointAddress; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.util.ClientBuilder; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -58,6 +59,7 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import static org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider.NAMESPACE_PROPERTY; +import static org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties.RefreshStrategy; /** * @author Ryan Baxter @@ -83,9 +85,11 @@ class HttpBasedConfigMapWatchChangeDetectorTests { @Mock private KubernetesInformerReactiveDiscoveryClient reactiveDiscoveryClient; - private HttpBasedConfigMapWatchChangeDetector changeDetector; + private MockEnvironment mockEnvironment; - private ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties; + private WebClient webClient; + + private ConfigurationUpdateStrategy strategy; @BeforeAll static void beforeAll() { @@ -105,58 +109,112 @@ static void teardown() { @BeforeEach void setup() { - - MockEnvironment mockEnvironment = new MockEnvironment(); + mockEnvironment = new MockEnvironment(); mockEnvironment.setProperty(NAMESPACE_PROPERTY, "default"); - configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); - WebClient webClient = WebClient.builder().build(); - - ConfigurationUpdateStrategy strategy = new ConfigurationUpdateStrategy("refresh", () -> { + webClient = WebClient.builder().build(); + strategy = new ConfigurationUpdateStrategy("refresh", () -> { }); + } - changeDetector = new HttpBasedConfigMapWatchChangeDetector(coreV1Api, mockEnvironment, - ConfigReloadProperties.DEFAULT, strategy, configMapPropertySourceLocator, - new KubernetesNamespaceProvider(mockEnvironment), configurationWatcherConfigurationProperties, - threadPoolTaskExecutor, new HttpRefreshTrigger(reactiveDiscoveryClient, - configurationWatcherConfigurationProperties, webClient)); + @Test + void triggerConfigMapRefreshUsingRefresh() { + triggerConfigMapRefresh("/actuator/refresh", RefreshStrategy.REFRESH); } @Test - void triggerConfigMapRefresh() { + void triggerConfigMapRefreshUsingShutdown() { + triggerConfigMapRefresh("/actuator/shutdown", RefreshStrategy.SHUTDOWN); + } + + void triggerConfigMapRefresh(String actuatorPath, RefreshStrategy refreshStrategy) { stubReactiveCall(); V1ConfigMap configMap = new V1ConfigMap(); V1ObjectMeta objectMeta = new V1ObjectMeta(); objectMeta.setName("foo"); configMap.setMetadata(objectMeta); WireMock.configureFor("localhost", WIRE_MOCK_SERVER.port()); - WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/actuator/refresh")) - .willReturn(WireMock.aResponse().withStatus(200))); + WireMock + .stubFor(WireMock.post(WireMock.urlEqualTo(actuatorPath)).willReturn(WireMock.aResponse().withStatus(200))); + + ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); + configurationWatcherConfigurationProperties.setRefreshStrategy(refreshStrategy); + HttpBasedConfigMapWatchChangeDetector changeDetector = new HttpBasedConfigMapWatchChangeDetector(coreV1Api, + mockEnvironment, ConfigReloadProperties.DEFAULT, strategy, configMapPropertySourceLocator, + new KubernetesNamespaceProvider(mockEnvironment), configurationWatcherConfigurationProperties, + threadPoolTaskExecutor, new HttpRefreshTrigger(reactiveDiscoveryClient, + configurationWatcherConfigurationProperties, webClient)); StepVerifier.create(changeDetector.triggerRefresh(configMap, configMap.getMetadata().getName())) .verifyComplete(); - WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/refresh"))); + WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo(actuatorPath))); + } + + @Test + void triggerConfigMapRefreshWithPropertiesBasedActuatorPathUsingRefresh() { + triggerConfigMapRefreshWithPropertiesBasedActuatorPath("/refresh", RefreshStrategy.REFRESH); } @Test - void triggerConfigMapRefreshWithPropertiesBasedActuatorPath() { + void triggerConfigMapRefreshWithPropertiesBasedActuatorPathUsingShutdown() { + triggerConfigMapRefreshWithPropertiesBasedActuatorPath("/shutdown", RefreshStrategy.SHUTDOWN); + } + + void triggerConfigMapRefreshWithPropertiesBasedActuatorPath(String endpoint, RefreshStrategy refreshStrategy) { stubReactiveCall(); + ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); + configurationWatcherConfigurationProperties.setRefreshStrategy(refreshStrategy); configurationWatcherConfigurationProperties.setActuatorPath("/my/custom/actuator"); V1ConfigMap configMap = new V1ConfigMap(); V1ObjectMeta objectMeta = new V1ObjectMeta(); objectMeta.setName("foo"); configMap.setMetadata(objectMeta); WireMock.configureFor("localhost", WIRE_MOCK_SERVER.port()); - WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/my/custom/actuator/refresh")) + WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/my/custom/actuator" + endpoint)) .willReturn(WireMock.aResponse().withStatus(200))); + HttpBasedConfigMapWatchChangeDetector changeDetector = new HttpBasedConfigMapWatchChangeDetector(coreV1Api, + mockEnvironment, ConfigReloadProperties.DEFAULT, strategy, configMapPropertySourceLocator, + new KubernetesNamespaceProvider(mockEnvironment), configurationWatcherConfigurationProperties, + threadPoolTaskExecutor, new HttpRefreshTrigger(reactiveDiscoveryClient, + configurationWatcherConfigurationProperties, webClient)); StepVerifier.create(changeDetector.triggerRefresh(configMap, configMap.getMetadata().getName())) .verifyComplete(); - WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/my/custom/actuator/refresh"))); + WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/my/custom/actuator" + endpoint))); + } + + @Test + void triggerConfigMapRefreshWithAnnotationActuatorPathUsingRefresh() { + triggerConfigMapRefreshWithAnnotationActuatorPath("/refresh", RefreshStrategy.REFRESH); } @Test - void triggerConfigMapRefreshWithAnnotationActuatorPath() { + void triggerConfigMapRefreshWithAnnotationActuatorPathUsingShutdown() { + triggerConfigMapRefreshWithAnnotationActuatorPath("/shutdown", RefreshStrategy.SHUTDOWN); + } + + void triggerConfigMapRefreshWithAnnotationActuatorPath(String endpoint, RefreshStrategy refreshStrategy) { int port = WIRE_MOCK_SERVER.port(); WireMock.configureFor("localhost", port); + List instances = getServiceInstances(port); + when(reactiveDiscoveryClient.getInstances(eq("foo"))).thenReturn(Flux.fromIterable(instances)); + V1ConfigMap configMap = new V1ConfigMap(); + V1ObjectMeta objectMeta = new V1ObjectMeta(); + objectMeta.setName("foo"); + configMap.setMetadata(objectMeta); + WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/my/custom/actuator" + endpoint)) + .willReturn(WireMock.aResponse().withStatus(200))); + ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); + configurationWatcherConfigurationProperties.setRefreshStrategy(refreshStrategy); + HttpBasedConfigMapWatchChangeDetector changeDetector = new HttpBasedConfigMapWatchChangeDetector(coreV1Api, + mockEnvironment, ConfigReloadProperties.DEFAULT, strategy, configMapPropertySourceLocator, + new KubernetesNamespaceProvider(mockEnvironment), configurationWatcherConfigurationProperties, + threadPoolTaskExecutor, new HttpRefreshTrigger(reactiveDiscoveryClient, + configurationWatcherConfigurationProperties, webClient)); + StepVerifier.create(changeDetector.triggerRefresh(configMap, configMap.getMetadata().getName())) + .verifyComplete(); + WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/my/custom/actuator" + endpoint))); + } + + private static @NotNull List getServiceInstances(int port) { Map metadata = new HashMap<>(); metadata.put(ConfigurationWatcherConfigurationProperties.ANNOTATION_KEY, "http://:" + port + "/my/custom/actuator"); @@ -169,20 +227,10 @@ void triggerConfigMapRefreshWithAnnotationActuatorPath() { DefaultKubernetesServiceInstance fooServiceInstance = new DefaultKubernetesServiceInstance("foo", "foo", fooEndpointAddress.getIp(), fooEndpointPort.getPort(), metadata, false); instances.add(fooServiceInstance); - when(reactiveDiscoveryClient.getInstances(eq("foo"))).thenReturn(Flux.fromIterable(instances)); - V1ConfigMap configMap = new V1ConfigMap(); - V1ObjectMeta objectMeta = new V1ObjectMeta(); - objectMeta.setName("foo"); - configMap.setMetadata(objectMeta); - WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/my/custom/actuator/refresh")) - .willReturn(WireMock.aResponse().withStatus(200))); - StepVerifier.create(changeDetector.triggerRefresh(configMap, configMap.getMetadata().getName())) - .verifyComplete(); - WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/my/custom/actuator/refresh"))); + return instances; } private void stubReactiveCall() { - V1EndpointAddress fooEndpointAddress = new V1EndpointAddress(); fooEndpointAddress.setIp("127.0.0.1"); fooEndpointAddress.setHostname("localhost"); diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetectorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetectorTests.java index 87fae0c7a0..df8f6c7c79 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetectorTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetectorTests.java @@ -52,12 +52,14 @@ import org.springframework.cloud.kubernetes.commons.discovery.DefaultKubernetesServiceInstance; import org.springframework.mock.env.MockEnvironment; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.util.StringUtils; import org.springframework.web.reactive.function.client.WebClient; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import static org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider.NAMESPACE_PROPERTY; +import static org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties.RefreshStrategy; /** * @author Ryan Baxter @@ -86,21 +88,15 @@ class HttpBasedSecretsWatchChangeDetectorTests { @Mock private KubernetesInformerReactiveDiscoveryClient reactiveDiscoveryClient; - private HttpBasedSecretsWatchChangeDetector changeDetector; + private MockEnvironment mockEnvironment; - private ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties; + private WebClient webClient; @BeforeEach void setup() { - MockEnvironment mockEnvironment = new MockEnvironment(); + mockEnvironment = new MockEnvironment(); mockEnvironment.setProperty(NAMESPACE_PROPERTY, "default"); - configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); - WebClient webClient = WebClient.builder().build(); - changeDetector = new HttpBasedSecretsWatchChangeDetector(coreV1Api, mockEnvironment, - ConfigReloadProperties.DEFAULT, updateStrategy, secretsPropertySourceLocator, - new KubernetesNamespaceProvider(mockEnvironment), configurationWatcherConfigurationProperties, - threadPoolTaskExecutor, new HttpRefreshTrigger(reactiveDiscoveryClient, - configurationWatcherConfigurationProperties, webClient)); + webClient = WebClient.builder().build(); } @BeforeAll @@ -120,36 +116,83 @@ static void teardown() { } @Test - void triggerSecretRefresh() { + void triggerSecretRefreshUsingRefresh() { + triggerSecretRefresh("/refresh", RefreshStrategy.REFRESH); + } + + @Test + void triggerSecretRefreshUsingShutdown() { + triggerSecretRefresh("/shutdown", RefreshStrategy.SHUTDOWN); + } + + void triggerSecretRefresh(String endpoint, RefreshStrategy refreshStrategy) { stubReactiveCall(); V1Secret secret = new V1Secret(); V1ObjectMeta objectMeta = new V1ObjectMeta(); objectMeta.setName("foo"); secret.setMetadata(objectMeta); WireMock.configureFor("localhost", WIRE_MOCK_SERVER.port()); - WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/actuator/refresh")) + WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/actuator" + endpoint)) .willReturn(WireMock.aResponse().withStatus(200))); + HttpBasedSecretsWatchChangeDetector changeDetector = getHttpBasedSecretsWatchChangeDetector(refreshStrategy); StepVerifier.create(changeDetector.triggerRefresh(secret, secret.getMetadata().getName())).verifyComplete(); - WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/refresh"))); + WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator" + endpoint))); + } + + private HttpBasedSecretsWatchChangeDetector getHttpBasedSecretsWatchChangeDetector( + RefreshStrategy refreshStrategy) { + return getHttpBasedSecretsWatchChangeDetector(null, refreshStrategy); + } + + private HttpBasedSecretsWatchChangeDetector getHttpBasedSecretsWatchChangeDetector(String actuatorPath, + RefreshStrategy refreshStrategy) { + ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); + if (StringUtils.hasText(actuatorPath)) { + configurationWatcherConfigurationProperties.setActuatorPath(actuatorPath); + } + configurationWatcherConfigurationProperties.setRefreshStrategy(refreshStrategy); + return new HttpBasedSecretsWatchChangeDetector(coreV1Api, mockEnvironment, ConfigReloadProperties.DEFAULT, + updateStrategy, secretsPropertySourceLocator, new KubernetesNamespaceProvider(mockEnvironment), + configurationWatcherConfigurationProperties, threadPoolTaskExecutor, new HttpRefreshTrigger( + reactiveDiscoveryClient, configurationWatcherConfigurationProperties, webClient)); } @Test - void triggerSecretRefreshWithPropertiesBasedActuatorPath() { + void triggerSecretRefreshWithPropertiesBasedActuatorPathUsingRefresh() { + triggerSecretRefreshWithPropertiesBasedActuatorPath("/refresh", RefreshStrategy.REFRESH); + } + + @Test + void triggerSecretRefreshWithPropertiesBasedActuatorPathUsingShutdown() { + triggerSecretRefreshWithPropertiesBasedActuatorPath("/shutdown", RefreshStrategy.SHUTDOWN); + } + + void triggerSecretRefreshWithPropertiesBasedActuatorPath(String endpoint, RefreshStrategy refreshStrategy) { stubReactiveCall(); - configurationWatcherConfigurationProperties.setActuatorPath("/my/custom/actuator"); V1Secret secret = new V1Secret(); V1ObjectMeta objectMeta = new V1ObjectMeta(); objectMeta.setName("foo"); secret.setMetadata(objectMeta); WireMock.configureFor("localhost", WIRE_MOCK_SERVER.port()); - WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/my/custom/actuator/refresh")) + WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/my/custom/actuator" + endpoint)) .willReturn(WireMock.aResponse().withStatus(200))); + HttpBasedSecretsWatchChangeDetector changeDetector = getHttpBasedSecretsWatchChangeDetector( + "/my/custom/actuator", refreshStrategy); StepVerifier.create(changeDetector.triggerRefresh(secret, secret.getMetadata().getName())).verifyComplete(); - WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/my/custom/actuator/refresh"))); + WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/my/custom/actuator" + endpoint))); } @Test - void triggerSecretRefreshWithAnnotationActuatorPath() { + void triggerSecretRefreshWithAnnotationActuatorPathUsingRefresh() { + triggerSecretRefreshWithAnnotationActuatorPath("/refresh", RefreshStrategy.REFRESH); + } + + @Test + void triggerSecretRefreshWithAnnotationActuatorPathUsingShutdown() { + triggerSecretRefreshWithAnnotationActuatorPath("/shutdown", RefreshStrategy.SHUTDOWN); + } + + void triggerSecretRefreshWithAnnotationActuatorPath(String endpoint, RefreshStrategy refreshStrategy) { WireMock.configureFor("localhost", WIRE_MOCK_SERVER.port()); Map metadata = new HashMap<>(); metadata.put(ConfigurationWatcherConfigurationProperties.ANNOTATION_KEY, @@ -168,10 +211,12 @@ void triggerSecretRefreshWithAnnotationActuatorPath() { V1ObjectMeta objectMeta = new V1ObjectMeta(); objectMeta.setName("foo"); secret.setMetadata(objectMeta); - WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/my/custom/actuator/refresh")) + WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/my/custom/actuator" + endpoint)) .willReturn(WireMock.aResponse().withStatus(200))); + HttpBasedSecretsWatchChangeDetector changeDetector = getHttpBasedSecretsWatchChangeDetector( + "/my/custom/actuator", refreshStrategy); StepVerifier.create(changeDetector.triggerRefresh(secret, secret.getMetadata().getName())).verifyComplete(); - WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/my/custom/actuator/refresh"))); + WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/my/custom/actuator" + endpoint))); } private void stubReactiveCall() { diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshIT.java index 26f7301c1b..52ed7f2727 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshIT.java @@ -126,6 +126,32 @@ void testActuatorRefresh() { } + void testActuatorShutdown() { + TestUtil.patchForShutdownRefresh(SPRING_CLOUD_K8S_CONFIG_WATCHER_APP_NAME, NAMESPACE, DOCKER_IMAGE); + WireMock.configureFor(WIREMOCK_HOST, WIREMOCK_PORT); + await().timeout(Duration.ofSeconds(60)) + .ignoreException(SocketTimeoutException.class) + .until(() -> WireMock + .stubFor(WireMock.post(WireMock.urlEqualTo("/actuator/shutdown")) + .willReturn(WireMock.aResponse().withBody("{}").withStatus(200))) + .getResponse() + .wasConfigured()); + + createConfigMap(); + + // Wait a bit before we verify + await().atMost(Duration.ofSeconds(30)) + .until(() -> !WireMock.findAll(WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/shutdown"))) + .isEmpty()); + WireMock.verify(WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/shutdown"))); + + deleteConfigMap(); + + // the other test + testActuatorRefreshReloadDisabled(); + + } + /* * same test as above, but reload is disabled. */ diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/TestUtil.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/TestUtil.java index b9d77740a8..6a21ff68e4 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/TestUtil.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/TestUtil.java @@ -61,8 +61,41 @@ private TestUtil() { } """; + private static final String USE_SHUTDOWN = """ + { + "spec": { + "template": { + "spec": { + "containers": [{ + "name": "spring-cloud-kubernetes-configuration-watcher", + "image": "image_name_here", + "env": [ + { + "name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS_CONFIG_RELOAD", + "value": "DEBUG" + }, + { + "name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CONFIGURATION_WATCHER", + "value": "DEBUG" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_CONFIGURATION_WATCHER_REFRESH_STRATEGY", + "value": "shutdown" + } + ] + }] + } + } + } + } + """; + static void patchForDisabledReload(String deploymentName, String namespace, String imageName) { patchWithReplace(imageName, deploymentName, namespace, BODY_ONE, POD_LABELS); } + static void patchForShutdownRefresh(String deploymentName, String namespace, String imageName) { + patchWithReplace(imageName, deploymentName, namespace, USE_SHUTDOWN, POD_LABELS); + } + }