Arachne #61 Add config option to specify docker pull policy (#401)

OHDSI · Oct 9, 2024 · 50f5673 · 50f5673
1 parent cc5549e
commit 50f5673
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -11,12 +11,21 @@ ARACHNE Execution Engine is able to use local or Docker Image/tar ball pre-built
 
 ### Important configuration options
 
-docker.image.default - default docker image to be used for execution. If specified, this enables docker execution support. 
-docker.image.filter - a regex used to scan for docker images to be listed. Specifying enables docker execution support and image scanning.
-runtimeservice.dist.archive - Default image to use for running executions
-runtime.local - true/false, enables using local R runtime.
+`docker.image.default` - default docker image to be used for execution. If specified, this enables docker execution support.
 
-docker.enable - outdated option. This is only kept for backward compatibility with implementations that don't support dybamic mode detection. 
+`docker.image.filter` - a regex used to scan for docker images to be listed. Specifying enables docker execution support and image scanning.
+
+`docker.image.pull` - policy that defines when docker images should be pulled. The possible values are as follows:  
+    `NEVER`   Never pull any images. If image does not exist in local repository, fail analysis.  
+    `MISSING` If image exists in local repository, use it. Do not check for updated image.  
+    `ALWAYS`  Always attempt to pull. If pull failed but image exists in local repository, proceed with that image.   
+    `FORCE`   Always attempt to pull. If pull fails, analysis will fail as well, even if local image exists.  
+
+`runtimeservice.dist.archive` - Default image to use for running executions
+
+`runtime.local` - true/false, enables using local R runtime.
+
+`docker.enable` - outdated option. This is only kept for backward compatibility with implementations that don't support dynamic mode detection. 
 
 ### Sample options for creating a container for running locally
 

diff --git a/...e/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/DockerPullPolicy.java b/...e/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/DockerPullPolicy.java
@@ -0,0 +1,12 @@
+package com.odysseusinc.arachne.executionengine.execution.r;
+
+public enum DockerPullPolicy {
+    /** Never pull any images. If image does not exist in local registry, fail analysis. */
+    NEVER,
+    /** If image exists in local repository, use it. Do not check for updated image. Pull if image is missing.*/
+    MISSING,
+    /** Always attempt to pull. If pull failed but image exists in local repository, proceed with that image. */
+    ALWAYS,
+    /** Always attempt to pull. If pull fails, analysis will fail as well, even if local image exists. */
+    FORCE,
+}
diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/DockerService.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/DockerService.java
@@ -3,6 +3,8 @@
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.command.CreateContainerResponse;
 import com.github.dockerjava.api.command.PullImageResultCallback;
+import com.github.dockerjava.api.exception.DockerException;
+import com.github.dockerjava.api.exception.NotFoundException;
 import com.github.dockerjava.api.model.Bind;
 import com.github.dockerjava.api.model.HostConfig;
 import com.github.dockerjava.api.model.Image;
@@ -27,6 +29,7 @@
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.PostConstruct;
 import java.io.File;
 import java.net.URI;
 import java.time.Instant;
@@ -40,6 +43,9 @@
 import java.util.stream.Stream;
 
 import static com.github.dockerjava.api.model.ResponseItem.ProgressDetail;
+import static com.odysseusinc.arachne.executionengine.execution.r.DockerPullPolicy.ALWAYS;
+import static com.odysseusinc.arachne.executionengine.execution.r.DockerPullPolicy.FORCE;
+import static com.odysseusinc.arachne.executionengine.execution.r.DockerPullPolicy.NEVER;
 
 @Service
 @Slf4j
@@ -66,14 +72,14 @@ public class DockerService extends RService implements AutoCloseable {
     @Value("${docker.image.filter:#{null}}")
     private String filterRegex;
 
-    private final boolean pull;
+    @Value("${docker.image.pull:#{T(com.odysseusinc.arachne.executionengine.execution.r.DockerPullPolicy).MISSING}}")
+    private DockerPullPolicy pull;
 
     @Autowired
     @Qualifier("analysisTaskExecutor")
     private ThreadPoolTaskExecutor executor;
 
     public DockerService(DockerRegistryProperties properties) {
-        pull = properties.getUrl() != null;
         DefaultDockerClientConfig.Builder builder = DefaultDockerClientConfig.createDefaultConfigBuilder()
                 .withDockerTlsVerify(certPath != null)
                 .withDockerCertPath(certPath)
@@ -90,7 +96,11 @@ public DockerService(DockerRegistryProperties properties) {
                 .maxConnections(50)
                 .build();
         client = DockerClientImpl.getInstance(config, dockerHttpClient);
-        log.info("Initialized Docker interface [{}]", host);
+    }
+
+    @PostConstruct
+    public void init() {
+        log.info("Initialized Docker interface [{}], pull policy: {}", host, pull);
     }
 
     @Override
@@ -113,13 +123,35 @@ protected Overseer analyze(
 
         CompletableFuture<String> init = CompletableFuture.supplyAsync(
                 () -> {
-                    log.info("Execution [{}] use Docker image [{}], pull: {}", id, image, pull);
-                    if (pull) {
+                    log.info("Execution [{}] use Docker image [{}], pull policy: {}", id, image, pull);
+                    if (pull == FORCE || pull == ALWAYS) {
+                        log.info("Execution [{}] force pull image image {}", id, image);
+                        callback.accept(Stage.INITIALIZE, "Force pull image [" + image + "]\r\n");
+                        try {
+                            pullImage(callback, id, image, stdout, client);
+                            callback.accept(Stage.INITIALIZE, "Pull complete, creating container\r\n");
+                        } catch (DockerException e) {
+                            if (pull == ALWAYS && imageExists(image)) {
+                                log.warn("Execution [{}] pull failed: {}, execution will proceed with image found locally", id, e.getMessage());
+                                callback.accept(Stage.INITIALIZE, "Execution [" + id + "] pull failed: " + e.getMessage() + ", proceed with local image");
+                            } else {
+                                log.error("Execution [{}] pull failed: ", id, e);
+                                callback.accept(Stage.INITIALIZE, ExceptionUtils.getStackTrace(e));
+                                throw e;
+                            }
+                        }
+                    } else if (imageExists(image)) {
+                        log.info("Execution [{}] image found, will proceed without pull", id);
+                        callback.accept(Stage.INITIALIZE, "Image found, proceed to execution\r\n");
+                    } else if (pull == NEVER) {
+                        log.info("Execution [{}] image not found, aborting", id);
+                        callback.accept(Stage.INITIALIZE, "Image image not found, aborting\r\n");
+                        throw new NotFoundException("Image [" + image + "] not found");
+                    } else {
+                        log.info("Execution [{}] image not found, proceed with pull", id);
+                        callback.accept(Stage.INITIALIZE, "Image [" + image + "] not found, proceed to pull\r\n");
                         pullImage(callback, id, image, stdout, client);
                         callback.accept(Stage.INITIALIZE, "Pull complete, creating container\r\n");
-                    } else {
-                        log.info("Execution [{}] pull skipped,  no docker registry configured", id);
-                        callback.accept(Stage.INITIALIZE, "Pull skipped, no docker registry configured\r\n");
                     }
                     log.info("Execution [{}] creating container with image [{}]", id, image);
                     String containerId = createContainer(analysisDir, env, image, script);
@@ -136,6 +168,14 @@ protected Overseer analyze(
         return new DockerOverseer(id, client, started, runtimeTimeOutSec, stdout, init, updateInterval, sendCallback, imageTagOrId, killTimeoutSec);
     }
 
+    private boolean imageExists(String image) {
+        try {
+            return client.inspectImageCmd(image).exec() != null;
+        } catch (NotFoundException e) {
+            return false;
+        }
+    }
+
     private void pullImage(BiConsumer<String, String> callback, Long id, String image, StringBuffer stdout, DockerClient client) {
         try {
             callback.accept(Stage.INITIALIZE, "Pulling image [" + image + "]\r\n");