Cluster Has Inconsistent View of Existing Docker Containers

build.gradle

@@ -19,7 +19,7 @@ apply from: "https://raw.githubusercontent.com/gocd/gocd-plugin-gradle-task-help
 
 gocdPlugin {
   id = 'cd.go.contrib.elastic-agent.docker'
-  pluginVersion = '3.1.0'
+  pluginVersion = '3.1.1'
   goCdVersion = '19.3.0'
   name = 'Docker Elastic Agent Plugin'
   description = 'Docker Based Elastic Agent Plugins for GoCD'

src/main/java/cd/go/contrib/elasticagents/docker/DockerContainers.java

@@ -31,7 +31,11 @@
 
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 import static cd.go.contrib.elasticagents.docker.DockerPlugin.LOG;
@@ -40,11 +44,19 @@
 public class DockerContainers implements AgentInstances<DockerContainer> {
     private final Map<String, DockerContainer> instances = new ConcurrentHashMap<>();
     private List<JobIdentifier> jobsWaitingForAgentCreation = new ArrayList<>();
-    private boolean refreshed;
+    private AtomicBoolean refreshed = new AtomicBoolean(false);
+    private final int FORCE_REFRESH_TIMEOUT_MINUTES = 60;
+
     public Clock clock = Clock.DEFAULT;
 
     final Semaphore semaphore = new Semaphore(0, true);
 
+    private final ScheduledExecutorService timerService = Executors.newSingleThreadScheduledExecutor();
+
+    public DockerContainers() {
+        scheduleForceRefresh();
+    }
+
     @Override
     public DockerContainer create(CreateAgentRequest request, PluginRequest pluginRequest, ConsoleLogAppender consoleLogAppender) throws Exception {
         LOG.info(String.format("[Create Agent] Processing create agent request for %s", request.jobIdentifier()));
@@ -77,12 +89,20 @@ public DockerContainer create(CreateAgentRequest request, PluginRequest pluginRe
         }
     }
 
+    private void scheduleForceRefresh() {
+        timerService.scheduleAtFixedRate(this::forceNextRefresh, 0, FORCE_REFRESH_TIMEOUT_MINUTES, TimeUnit.MINUTES);
+    }
+
     private void doWithLockOnSemaphore(Runnable runnable) {
         synchronized (semaphore) {
             runnable.run();
         }
     }
 
+    protected void forceNextRefresh() {
+        refreshed.set(false);
+    }
+
     @Override
     public void terminate(String agentId, PluginSettings settings) throws Exception {
         DockerContainer instance = instances.get(agentId);
@@ -125,24 +145,22 @@ public Agents instancesCreatedAfterTimeout(PluginSettings settings, Agents agent
             if (instance == null) {
                 continue;
             }
-
             if (clock.now().isAfter(instance.createdAt().plus(settings.getAutoRegisterPeriod()))) {
                 oldAgents.add(agent);
             }
         }
-
         return new Agents(oldAgents);
     }
 
     @Override
     public void refreshAll(ClusterProfileProperties clusterProfileProperties) throws Exception {
-        if (!refreshed) {
+        if (!refreshed.get()){
             DockerClient docker = docker(clusterProfileProperties);
             List<Container> containers = docker.listContainers(DockerClient.ListContainersParam.withLabel(Constants.CREATED_BY_LABEL_KEY, Constants.PLUGIN_ID));
             for (Container container : containers) {
                 register(DockerContainer.fromContainerInfo(docker.inspectContainer(container.id())));
             }
-            refreshed = true;
+            refreshed.set(true);
         }
     }
 

src/test/java/cd/go/contrib/elasticagents/docker/DockerContainersTest.java

@@ -231,4 +231,20 @@ public void shouldGetAgentStatusReportUsingDockerContainer() throws Exception {
         assertThat(agentStatusReport.getElasticAgentId(), is(container.name()));
         assertThat(agentStatusReport.getJobIdentifier(), is(request.jobIdentifier()));
     }
+
+    @Test
+    public void shouldForceRefreshAgentInstancesAfterTimeout() throws Exception {
+
+        DockerContainers dockerContainers = new DockerContainers();
+        ClusterProfileProperties profileProperties = createClusterProfiles();
+        dockerContainers.refreshAll(profileProperties);
+
+        DockerContainer container = DockerContainer.create(request, clusterProfile, docker, consoleLogAppender);
+        containers.add(container.name());
+
+        dockerContainers.forceNextRefresh();
+
+        dockerContainers.refreshAll(profileProperties);
+        assertEquals(dockerContainers.find(container.name()), container);
+    }
 }