From a8f4615feb30172a57583c1882a97b4c1eb112af Mon Sep 17 00:00:00 2001 From: John Flavin Date: Fri, 31 May 2024 14:24:59 +0000 Subject: [PATCH] CS-968 Switch client library from docker-client to docker-java (pull request #119) * Migrate getContainerEvents * Migrate startDockerContainer and setSwarmServiceReplicasToOne * Migrate DockerControlApiIntegrationTest to use new client * Adjust docker http client timeouts * Add another getDockerClientNew method with no args, use default server * Massive change to migrate all integration test client operations use new client * Migrate create container and create service * Migrate delete image * Slight tweaks to BUSYBOX constants and versions * Migrate inspect image / get image by id * Standardize docker-java exception handling * Migrate list images * Migrate ping * Reuse RETURN_SELF answer in test mocks * Migrate container/service logs * Migrate getTaskForService * Migrate pull image (with auth) * Add support for generic resources (i.e. gpus) using a fork of docker-java * Migrate TASK_STATE_FAILED to TaskState.FAILED * Move client config into its own method * Migrate AuthConfig and getting registry URL for image * Complete removal of docker-client * Fix hub ping * Clean up some more fully qualified classes left over from transition * Changelog Approved-by: James Ransford Approved-by: Bin Zhang --- CHANGELOG.md | 5 + build.gradle | 24 +- .../containers/api/ContainerControlApi.java | 4 +- .../nrg/containers/api/DockerControlApi.java | 1092 ++++++++--------- .../events/ContainerStatusUpdater.java | 15 +- .../exceptions/ServiceNotFoundException.java | 7 + .../exceptions/TaskNotFoundException.java | 7 + .../model/container/auto/ServiceTask.java | 63 +- .../containers/services/DockerHubService.java | 4 +- .../impl/ContainerFinalizeServiceImpl.java | 4 +- .../services/impl/ContainerServiceImpl.java | 4 +- .../impl/HibernateDockerHubService.java | 33 +- .../CommandLaunchIntegrationTest.java | 60 +- .../ContainerCleanupIntegrationTest.java | 20 +- .../SwarmConstraintsIntegrationTest.java | 68 +- .../SwarmRestartIntegrationTest.java | 100 +- .../api/DockerControlApiIntegrationTest.java | 157 +-- .../containers/api/DockerControlApiTest.java | 357 +++--- .../jms/JmsExceptionIntegrationTest.java | 88 -- .../nrg/containers/secrets/SecretsTest.java | 69 +- .../DockerServiceIntegrationTest.java | 11 +- .../utils/LoggingBuildCallback.java | 14 + .../nrg/containers/utils/TestingUtils.java | 241 ++-- 23 files changed, 1098 insertions(+), 1349 deletions(-) create mode 100644 src/main/java/org/nrg/containers/exceptions/ServiceNotFoundException.java create mode 100644 src/main/java/org/nrg/containers/exceptions/TaskNotFoundException.java create mode 100644 src/test/java/org/nrg/containers/utils/LoggingBuildCallback.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cf169b3..a1dddca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,13 @@ Not yet released. * **Improvement** [CS-946][] Prevent setting mutually distinct k8s PVC mounting options +* **Bugfix** [CS-968][] Switch the docker API library we use from [docker-client][] to [docker-java][]. + This should restore CS functionality on docker engine v25 and higher. [CS-946]: https://radiologics.atlassian.net/browse/CS-946 +[CS-968]: https://radiologics.atlassian.net/browse/CS-968 +[docker-client]: https://github.com/dmandalidis/docker-client +[docker-java]: https://github.com/docker-java/docker-java ## 3.4.3 [Released](https://bitbucket.org/xnatdev/container-service/src/3.4.3/). diff --git a/build.gradle b/build.gradle index 9a423026..0fc081c8 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,8 @@ dependencyManagement { } } } -def vDockerClient = "6.0.4" + +def vDockerJava = "3.4.0.1" def vAutoValue = "1.3" def vKubernetesClient = "15.0.+" def vPowerMock = "1.7.0" @@ -50,7 +51,6 @@ def vGson = dependencyManagement.importedProperties["gson.version"] def vJackson = dependencyManagement.importedProperties["jackson.version"] def vSpringSecurity = dependencyManagement.importedProperties["spring-security.version"] def vActiveMQ = dependencyManagement.importedProperties["activemq.version"] -def vJersey = "3.0.6" def vBouncyCastle = "1.64" // Included in xnat-web via spring-security-jwt 1.1.1, see XNAT-7907 // Use this configuration to put dependencies into the fat jar @@ -67,23 +67,8 @@ dependencies { compileOnly "com.google.auto.value:auto-value:${vAutoValue}" - implementAndInclude ("org.mandas:docker-client:${vDockerClient}") { - exclude group: "ch.qos.logback" - exclude group: "org.slf4j" - exclude group: "org.bouncycastle" // included in xnat-web via spring-security-jwt - exclude group: "com.fasterxml.jackson.core" - exclude group: "org.apache.commons", module: "commons-compress" // included in xnat-web via parent - } - - implementAndInclude "org.glassfish.jersey.core:jersey-client:${vJersey}" - implementAndInclude "org.glassfish.jersey.inject:jersey-hk2:${vJersey}" - implementAndInclude ("org.glassfish.jersey.connectors:jersey-apache-connector:${vJersey}") { - exclude group: "org.apache.httpcomponents", module: "httpclient" // included in xnat-web directly - } - implementAndInclude ("org.glassfish.jersey.media:jersey-media-json-jackson:${vJersey}") { - exclude group: "com.fasterxml.jackson.core" // Included in xnat-web directly - exclude group: "com.fasterxml.jackson.module", module: "jackson-module-jaxb-annotations" // Included in xnat-web via spawner -> jackson-dataformat-xml - } + implementAndInclude "com.github.docker-java:docker-java-core:${vDockerJava}" + implementAndInclude "com.github.docker-java:docker-java-transport-okhttp:${vDockerJava}" implementAndInclude ("io.kubernetes:client-java:${vKubernetesClient}") { exclude group: "com.google.code.gson" @@ -95,6 +80,7 @@ dependencies { exclude group: "org.slf4j" exclude group: "org.bouncycastle" exclude group: "org.yaml", module: "snakeyaml" + exclude group: "com.squareup.okhttp3", module: "okhttp" // Included via docker-java-transport-okhttp } implementAndInclude ("io.kubernetes:client-java-api:${vKubernetesClient}") { // Explicitly including to exclude transitive deps exclude group: "com.google.code.gson" diff --git a/src/main/java/org/nrg/containers/api/ContainerControlApi.java b/src/main/java/org/nrg/containers/api/ContainerControlApi.java index 2ec3a693..9f338e85 100644 --- a/src/main/java/org/nrg/containers/api/ContainerControlApi.java +++ b/src/main/java/org/nrg/containers/api/ContainerControlApi.java @@ -1,13 +1,13 @@ package org.nrg.containers.api; -import org.mandas.docker.client.exceptions.ServiceNotFoundException; -import org.mandas.docker.client.exceptions.TaskNotFoundException; import org.nrg.containers.events.model.DockerContainerEvent; import org.nrg.containers.exceptions.ContainerBackendException; import org.nrg.containers.exceptions.ContainerException; import org.nrg.containers.exceptions.DockerServerException; import org.nrg.containers.exceptions.NoContainerServerException; import org.nrg.containers.exceptions.NoDockerServerException; +import org.nrg.containers.exceptions.ServiceNotFoundException; +import org.nrg.containers.exceptions.TaskNotFoundException; import org.nrg.containers.model.command.auto.ResolvedCommand; import org.nrg.containers.model.container.auto.Container; import org.nrg.containers.model.container.auto.ServiceTask; diff --git a/src/main/java/org/nrg/containers/api/DockerControlApi.java b/src/main/java/org/nrg/containers/api/DockerControlApi.java index a64e89c8..25b017ae 100644 --- a/src/main/java/org/nrg/containers/api/DockerControlApi.java +++ b/src/main/java/org/nrg/containers/api/DockerControlApi.java @@ -1,56 +1,71 @@ package org.nrg.containers.api; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.async.ResultCallbackTemplate; +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.command.CreateContainerResponse; +import com.github.dockerjava.api.command.EventsCmd; +import com.github.dockerjava.api.command.InspectContainerResponse; +import com.github.dockerjava.api.command.InspectImageResponse; +import com.github.dockerjava.api.command.InspectServiceCmd; +import com.github.dockerjava.api.command.KillContainerCmd; +import com.github.dockerjava.api.command.PullImageCmd; +import com.github.dockerjava.api.command.RemoveContainerCmd; +import com.github.dockerjava.api.command.RemoveServiceCmd; +import com.github.dockerjava.api.command.StartContainerCmd; +import com.github.dockerjava.api.command.UpdateServiceCmd; +import com.github.dockerjava.api.exception.DockerException; +import com.github.dockerjava.api.exception.NotModifiedException; +import com.github.dockerjava.api.model.AuthConfig; +import com.github.dockerjava.api.model.Bind; +import com.github.dockerjava.api.model.ContainerSpec; +import com.github.dockerjava.api.model.DiscreteResourceSpec; +import com.github.dockerjava.api.model.EndpointSpec; +import com.github.dockerjava.api.model.Event; +import com.github.dockerjava.api.model.EventActor; +import com.github.dockerjava.api.model.EventType; +import com.github.dockerjava.api.model.ExposedPort; +import com.github.dockerjava.api.model.Frame; +import com.github.dockerjava.api.model.GenericResource; +import com.github.dockerjava.api.model.HostConfig; +import com.github.dockerjava.api.model.Mount; +import com.github.dockerjava.api.model.MountType; +import com.github.dockerjava.api.model.NamedResourceSpec; +import com.github.dockerjava.api.model.NetworkAttachmentConfig; +import com.github.dockerjava.api.model.PortBinding; +import com.github.dockerjava.api.model.PortConfig; +import com.github.dockerjava.api.model.PortConfigProtocol; +import com.github.dockerjava.api.model.Ports; +import com.github.dockerjava.api.model.ResourceRequirements; +import com.github.dockerjava.api.model.ResourceSpecs; +import com.github.dockerjava.api.model.ServiceModeConfig; +import com.github.dockerjava.api.model.ServicePlacement; +import com.github.dockerjava.api.model.ServiceReplicatedModeOptions; +import com.github.dockerjava.api.model.ServiceRestartCondition; +import com.github.dockerjava.api.model.ServiceRestartPolicy; +import com.github.dockerjava.api.model.ServiceSpec; +import com.github.dockerjava.api.model.Task; +import com.github.dockerjava.api.model.TaskSpec; +import com.github.dockerjava.api.model.TmpfsOptions; +import com.github.dockerjava.api.model.Ulimit; +import com.github.dockerjava.core.DefaultDockerClientConfig; +import com.github.dockerjava.core.DockerClientConfig; +import com.github.dockerjava.core.DockerClientImpl; +import com.github.dockerjava.core.NameParser; +import com.github.dockerjava.okhttp.OkDockerHttpClient; +import com.github.dockerjava.transport.DockerHttpClient; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.mandas.docker.client.DockerCertificates; -import org.mandas.docker.client.DockerClient; -import org.mandas.docker.client.EventStream; -import org.mandas.docker.client.LogStream; -import org.mandas.docker.client.auth.ConfigFileRegistryAuthSupplier; -import org.mandas.docker.client.builder.jersey.JerseyDockerClientBuilder; -import org.mandas.docker.client.exceptions.ContainerNotFoundException; -import org.mandas.docker.client.exceptions.DockerCertificateException; -import org.mandas.docker.client.exceptions.DockerException; -import org.mandas.docker.client.exceptions.ImageNotFoundException; -import org.mandas.docker.client.exceptions.ServiceNotFoundException; -import org.mandas.docker.client.exceptions.TaskNotFoundException; -import org.mandas.docker.client.messages.ContainerConfig; -import org.mandas.docker.client.messages.ContainerCreation; -import org.mandas.docker.client.messages.ContainerInfo; -import org.mandas.docker.client.messages.Event; -import org.mandas.docker.client.messages.HostConfig; -import org.mandas.docker.client.messages.Image; -import org.mandas.docker.client.messages.ImageInfo; -import org.mandas.docker.client.messages.PortBinding; -import org.mandas.docker.client.messages.RegistryAuth; -import org.mandas.docker.client.messages.ServiceCreateResponse; -import org.mandas.docker.client.messages.mount.Mount; -import org.mandas.docker.client.messages.mount.TmpfsOptions; -import org.mandas.docker.client.messages.swarm.ContainerSpec; -import org.mandas.docker.client.messages.swarm.EndpointSpec; -import org.mandas.docker.client.messages.swarm.NetworkAttachmentConfig; -import org.mandas.docker.client.messages.swarm.Placement; -import org.mandas.docker.client.messages.swarm.PortConfig; -import org.mandas.docker.client.messages.swarm.ReplicatedService; -import org.mandas.docker.client.messages.swarm.Reservations; -import org.mandas.docker.client.messages.swarm.ResourceRequirements; -import org.mandas.docker.client.messages.swarm.ResourceSpec; -import org.mandas.docker.client.messages.swarm.Resources; -import org.mandas.docker.client.messages.swarm.RestartPolicy; -import org.mandas.docker.client.messages.swarm.ServiceMode; -import org.mandas.docker.client.messages.swarm.ServiceSpec; -import org.mandas.docker.client.messages.swarm.Task; -import org.mandas.docker.client.messages.swarm.TaskSpec; import org.nrg.containers.events.model.DockerContainerEvent; import org.nrg.containers.exceptions.ContainerBackendException; import org.nrg.containers.exceptions.ContainerException; import org.nrg.containers.exceptions.DockerServerException; import org.nrg.containers.exceptions.NoContainerServerException; import org.nrg.containers.exceptions.NoDockerServerException; +import org.nrg.containers.exceptions.ServiceNotFoundException; +import org.nrg.containers.exceptions.TaskNotFoundException; import org.nrg.containers.model.command.auto.ResolvedCommand; import org.nrg.containers.model.container.auto.Container; import org.nrg.containers.model.container.auto.ServiceTask; @@ -71,27 +86,28 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.mandas.docker.client.DockerClient.EventsParam.since; -import static org.mandas.docker.client.DockerClient.EventsParam.type; -import static org.mandas.docker.client.DockerClient.EventsParam.until; import static org.nrg.containers.services.CommandLabelService.LABEL_KEY; import static org.nrg.containers.utils.ContainerUtils.instanceOrDefault; @Slf4j @Service public class DockerControlApi implements ContainerControlApi { + private static final String HUB_AUTH_SUCCESS = "Login Succeeded"; private final DockerServerService dockerServerService; private final DockerHubService dockerHubService; @@ -147,19 +163,23 @@ public String ping() throws NoDockerServerException, DockerServerException { } private String pingServer(final DockerServer dockerServer) throws DockerServerException { + final DockerClient client = getDockerClient(dockerServer); try { - return getDockerClient(dockerServer).ping(); - } catch (DockerException | InterruptedException e) { - log.trace("Ping failed to server {}", dockerServer, e); + client.pingCmd().exec(); + // If we got this far without an exception, then all is well. + } catch (DockerException e) { + log.trace("Failed to ping", e); throw new DockerServerException(e); } + return "OK"; } private String pingSwarmMaster(final DockerServer dockerServer) throws DockerServerException { + final DockerClient client = getDockerClient(dockerServer); try { - getDockerClient(dockerServer).inspectSwarm(); + client.inspectSwarmCmd().exec(); // If we got this far without an exception, then all is well. - } catch (DockerException | InterruptedException e) { + } catch (DockerException e) { log.trace("Failed to inspect swarm", e); throw new DockerServerException(e); } @@ -191,24 +211,30 @@ public DockerHubBase.DockerHubStatus pingHub(final @Nonnull DockerHub hub) { public DockerHubBase.DockerHubStatus pingHub(final @Nonnull DockerHub hub, final @Nullable String username, final @Nullable String password, final @Nullable String token, final @Nullable String email) { DockerHubBase.DockerHubStatus.Builder hubStatusBuilder = DockerHubBase.DockerHubStatus.create(false).toBuilder(); - int status = 500; Backend backend = null; try { backend = getServer().backend(); } catch (NoDockerServerException e) { // ignore } + if (backend == null) { + return hubStatusBuilder.ping(false) + .response("Error") + .message("Docker server unavailable.") + .build(); + } switch (backend) { case DOCKER: case SWARM: try { final DockerClient client = getDockerClient(); - status = client.auth(registryAuth(hub, username, password, token, email, true)); - hubStatusBuilder.ping(status < 400) - .response(status < 400 ? "OK" : "Down") - .message(StringUtils.join("Hub response: ", Integer.toString(status))); + final String statusStr = client.authCmd().withAuthConfig(authConfig(hub, username, password, token, email)).exec().getStatus(); + final boolean success = statusStr.equals(HUB_AUTH_SUCCESS); + hubStatusBuilder.ping(success) + .response(success ? "OK" : "Down") + .message("Hub response: " + statusStr); } catch (Exception e) { - log.error(e.getMessage()); + log.error("Hub status check created exception.", e); hubStatusBuilder.ping(false) .response("Error") .message("Hub status check created exception. Check Docker server status."); @@ -229,29 +255,36 @@ public DockerHubBase.DockerHubStatus pingHub(final @Nonnull DockerHub hub, final } @Nullable - private RegistryAuth registryAuth(final @Nullable DockerHub hub, final @Nullable String username, - final @Nullable String password, final @Nullable String token, - final @Nullable String email) { - return registryAuth(hub, username, password, token, email, false); + private AuthConfig authConfig(final String dockerImage, final DockerClientConfig config) { + // Resolve registry url from image + NameParser.ReposTag reposTag = NameParser.parseRepositoryTag(dockerImage); + final String registryUrl = NameParser.resolveRepositoryName(reposTag.repos).hostname; + + // See if we have a DockerHub object for this registry url + final DockerHub hub = dockerHubService.getByUrl(registryUrl); + AuthConfig authConfig = authConfig(hub, null, null, null, null); + + if (authConfig == null) { + // Get AuthConfig from docker config file + authConfig = config.effectiveAuthConfig(dockerImage); + } + + return authConfig; } @Nullable - private RegistryAuth registryAuth(final @Nullable DockerHub hub, final @Nullable String username, - final @Nullable String password, final @Nullable String token, - final @Nullable String email, boolean forPing) { - // TODO "forPing" is a hack. client.auth() needs a RegistryAuth object; it doesn't default to config.json - // as client.pull() does. This is because the RegistryAuthSupplier associates RegistryAuth objects with - // image names, not hubs - if (hub == null || !forPing && (username == null || password == null)) { - return null; - } - return RegistryAuth.builder() - .serverAddress(hub.url()) - .username(username == null ? hub.username() : username) - .password(password == null ? hub.password() : password) - .identityToken(token == null ? hub.token() : token) - .email(email == null ? hub.email() : email) - .build(); + private AuthConfig authConfig(final @Nullable DockerHub hub, + final @Nullable String username, + final @Nullable String password, + final @Nullable String token, + final @Nullable String email) { + return hub == null ? null : + new AuthConfig() + .withRegistryAddress(hub.url()) + .withUsername(username == null ? hub.username() : username) + .withPassword(password == null ? hub.password() : password) + .withIdentityToken(token == null ? hub.token() : token) + .withEmail(email == null ? hub.email() : email); } /** @@ -262,27 +295,28 @@ private RegistryAuth registryAuth(final @Nullable DockerHub hub, final @Nullable @Override @Nonnull public List getAllImages() throws NoDockerServerException, DockerServerException { - final DockerServer server = getServer(); - switch (server.backend()) { - case SWARM: - case DOCKER: - return _getImages().stream().map(this::spotifyToNrg).collect(Collectors.toList()); - case KUBERNETES: - default: - return Lists.newArrayList(); - } + return getAllImages(getServer()); } - private List _getImages() - throws NoDockerServerException, DockerServerException { + @Nonnull + private List getAllImages(final DockerServer server) throws DockerServerException { + if (server.backend() == Backend.KUBERNETES) { + return Collections.emptyList(); + } + + final DockerClient client = getDockerClient(server); try { - return getDockerClient().listImages(); - } catch (DockerException | InterruptedException e) { - log.error("Failed to list images. {}", e.getMessage(), e); - throw new DockerServerException(e); - } catch (Error e) { - log.error("Failed to list images. {}", e.getMessage(), e); - throw e; + return client.listImagesCmd() + .exec() + .stream() + .map(image -> DockerImage.builder() + .imageId(image.getId()) + .tags(image.getRepoTags() == null ? Collections.emptyList() : Arrays.asList(image.getRepoTags())) + .labels(image.getLabels() == null ? Collections.emptyMap() : image.getLabels()) + .build()) + .collect(Collectors.toList()); + } catch (DockerException e) { + throw new DockerServerException("Could not list images", e); } } @@ -296,42 +330,42 @@ private List _getImages() @Nonnull public DockerImage getImageById(final String imageId) throws NotFoundException, DockerServerException, NoDockerServerException { - switch (getServer().backend()) { - case SWARM: - case DOCKER: - return getImageById(imageId, getDockerClient()); - case KUBERNETES: - default: - throw new NoDockerServerException("Docker server images not available in Kubernetes mode."); + if (getServer().backend() == Backend.KUBERNETES) { + throw new NoDockerServerException("Docker server images not available in Kubernetes mode."); } - } - private DockerImage getImageById(final String imageId, final DockerClient client) - throws DockerServerException, NotFoundException { - final DockerImage image = spotifyToNrg(_getImageById(imageId, client)); - if (image != null) { - return image; - } - throw new NotFoundException(String.format("Could not find image %s", imageId)); + return getImageById(imageId, getDockerClient()); } - private org.mandas.docker.client.messages.ImageInfo _getImageById(final String imageId, final DockerClient client) + private DockerImage getImageById(final String imageId, final DockerClient client) throws DockerServerException, NotFoundException { try { - return client.inspectImage(imageId); - } catch (ImageNotFoundException e) { + final InspectImageResponse resp = client.inspectImageCmd(imageId).exec(); + if (resp == null) { + throw new NotFoundException("Could not find image \"" + imageId + "\""); + } + return DockerImage.builder() + .imageId(resp.getId()) + .labels(resp.getConfig() == null || resp.getConfig().getLabels() == null ? + Collections.emptyMap() : + resp.getConfig().getLabels()) + .build(); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { throw new NotFoundException(e); - } catch (DockerException | InterruptedException e) { - throw new DockerServerException(e); + } catch (DockerException e) { + throw new DockerServerException("Could not delete image", e); } } @Override public void deleteImageById(final String id, final Boolean force) throws NoDockerServerException, DockerServerException { + final DockerClient client = getDockerClient(); try { - getDockerClient().removeImage(id, force, false); - } catch (DockerException|InterruptedException e) { - throw new DockerServerException(e); + client.removeImageCmd(id).withForce(force).withNoPrune(false).exec(); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { + throw new DockerServerException("Image not found", e); + } catch (DockerException e) { + throw new DockerServerException("Could not delete image", e); } } @@ -356,7 +390,8 @@ public DockerImage pullImage(final String name, final @Nullable DockerHub hub, f final @Nullable String password, final @Nullable String token, final @Nullable String email) throws NoDockerServerException, DockerServerException, NotFoundException { final DockerClient client = getDockerClient(); - _pullImage(name, hub != null ? registryAuth(hub, username, password, token, email) : null, client); // We want to throw NotFoundException here if the image is not found on the hub + + _pullImage(client, name, authConfig(hub, username, password, token, email)); // We want to throw NotFoundException here if the image is not found on the hub try { return getImageById(name, client); // We don't want to throw NotFoundException from here. If we can't find the image here after it has been pulled, that is a server error. } catch (NotFoundException e) { @@ -366,15 +401,21 @@ public DockerImage pullImage(final String name, final @Nullable DockerHub hub, f } } - private void _pullImage(final @Nonnull String name, final @Nullable RegistryAuth registryAuth, final @Nonnull DockerClient client) throws DockerServerException, NotFoundException { - try { - client.pull(name, registryAuth); - } catch (ImageNotFoundException e) { - throw new NotFoundException(e.getMessage()); - } catch (DockerException | InterruptedException e) { - log.error(e.getMessage()); - throw new DockerServerException(e); - } + private void _pullImage(final DockerClient client, + final @Nonnull String name, + final @Nullable AuthConfig authConfig) throws DockerServerException, NotFoundException { + final PullImageCmd cmd = client.pullImageCmd(name); + if (authConfig != null) { + cmd.withAuthConfig(authConfig); + } + + try { + cmd.start().awaitCompletion(); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { + throw new NotFoundException(e.getMessage()); + } catch (DockerException | InterruptedException e) { + throw new DockerServerException(e); + } } /** @@ -456,14 +497,16 @@ private void createDirectoriesForMounts(final Container toCreate) throws IOExcep private String createDockerContainer(final Container toCreate, final DockerServer server) throws DockerServerException, ContainerException { - final Map> portBindings = Maps.newHashMap(); + final List portBindings = new ArrayList<>(); for (final Map.Entry portEntry : toCreate.ports().entrySet()) { final String containerPort = portEntry.getKey(); final String hostPort = portEntry.getValue(); if (StringUtils.isNotBlank(containerPort) && StringUtils.isNotBlank(hostPort)) { - final PortBinding portBinding = PortBinding.of(null, hostPort); - portBindings.put(containerPort + "/tcp", Lists.newArrayList(portBinding)); + portBindings.add(new PortBinding( + Ports.Binding.bindPortSpec(hostPort), + ExposedPort.tcp(Integer.parseInt(containerPort)) + )); } else { // One or both of hostPost and containerPort is blank. @@ -487,62 +530,36 @@ private String createDockerContainer(final Container toCreate, final DockerServe // Environment variables final Map environmentVariables = containerPropertiesWithSecretValues.environmentVariables(); - final HostConfig.Builder hostConfigBuilder = - HostConfig.builder() - .autoRemove(toCreate.autoRemove()) - .runtime(instanceOrDefault(toCreate.runtime(), "")) - .ipcMode(instanceOrDefault(toCreate.ipcMode(), "")) - .binds(toCreate.bindMountStrings()) - .portBindings(portBindings) - .memoryReservation(toCreate.reserveMemoryBytes()) - .memory(toCreate.limitMemoryBytes()) - .nanoCpus(toCreate.nanoCpus()); + final HostConfig hostConfig = new HostConfig() + .withAutoRemove(toCreate.autoRemove()) + .withRuntime(instanceOrDefault(toCreate.runtime(), "")) + .withIpcMode(instanceOrDefault(toCreate.ipcMode(), "")) + .withBinds(toCreate.bindMountStrings().stream().map(Bind::parse).collect(Collectors.toList())) + .withPortBindings(portBindings) + .withMemoryReservation(toCreate.reserveMemoryBytes()) + .withMemory(toCreate.limitMemoryBytes()) + .withNanoCPUs(toCreate.nanoCpus()); if (toCreate.shmSize() != null && toCreate.shmSize() >= 0) { - hostConfigBuilder.shmSize(toCreate.shmSize()); + hostConfig.withShmSize(toCreate.shmSize()); } if (!Strings.isNullOrEmpty(toCreate.network())) { - hostConfigBuilder.networkMode(toCreate.network()); + hostConfig.withNetworkMode(toCreate.network()); } final Map ulimits = toCreate.ulimits(); - if (ulimits != null && !ulimits.isEmpty()){ - List hostUlimits = new ArrayList<>(ulimits.size()); - for(Map.Entry ulimit : ulimits.entrySet()) { - final String[] split = ulimit.getValue().split(":"); - final Long softLimit = Long.parseLong(split[0]); - final Long hardLimit = Long.parseLong(split.length > 1 ? split[1] : split[0]); - - hostUlimits.add(HostConfig.Ulimit.builder() - .name(ulimit.getKey()) - .soft(softLimit) - .hard(hardLimit) - .build() - ); - } - hostConfigBuilder.ulimits(hostUlimits); + if (ulimits != null && !ulimits.isEmpty()) { + List hostUlimits = ulimits.entrySet().stream() + .map(ulimit -> { + final String[] split = ulimit.getValue().split(":"); + return new Ulimit(ulimit.getKey(), Long.parseLong(split[0]), Long.parseLong(split.length > 1 ? split[1] : split[0])); + }) + .collect(Collectors.toList()); + + hostConfig.withUlimits(hostUlimits); } - final HostConfig hostConfig = hostConfigBuilder.build(); final boolean overrideEntrypoint = toCreate.overrideEntrypointNonnull(); - final ContainerConfig containerConfig = - ContainerConfig.builder() - .hostConfig(hostConfig) - .image(toCreate.dockerImage()) - .attachStdout(true) - .attachStderr(true) - .cmd(overrideEntrypoint ? - Lists.newArrayList("/bin/sh", "-c", toCreate.commandLine()) : - ShellSplitter.shellSplit(toCreate.commandLine())) - .entrypoint(overrideEntrypoint ? Collections.singletonList("") : null) - .env(environmentVariables.entrySet() - .stream() - .map(entry -> entry.getKey() + "=" + entry.getValue()) - .collect(Collectors.toList())) - .workingDir(toCreate.workingDirectory()) - .user(server.containerUser()) - .labels(toCreate.containerLabels()) - .build(); if (log.isDebugEnabled()) { final String message = String.format( @@ -566,30 +583,47 @@ private String createDockerContainer(final Container toCreate, final DockerServe ); log.debug(message); } + final DockerClient client = getDockerClient(server); + // Pull image before creating container try { - if (getAllImages().stream() - .noneMatch(img -> img.tags().contains(containerConfig.image()))) { - pullImage(containerConfig.image()); - } - final DockerClient client = getDockerClient(server); - final ContainerCreation container = Strings.isNullOrEmpty(toCreate.containerName()) ? - client.createContainer(containerConfig) : - client.createContainer(containerConfig, toCreate.containerName()); - final List warnings = container.warnings(); - if (warnings != null) { - for (String warning : warnings) { - log.warn(warning); - } + if (getAllImages(server).stream() + .noneMatch(img -> img.tags().contains(toCreate.dockerImage()))) { + pullImage(toCreate.dockerImage()); } + } catch (NoDockerServerException | NotFoundException e) { // TODO make a new version of get and pull that take a server + log.error("Failed to pull image", e); + throw new DockerServerException("Could not pull image " + toCreate.dockerImage() + " from repository.", e); + } - return container.id(); - } catch (DockerException | InterruptedException | NoDockerServerException e) { - log.error(e.getMessage()); - throw new DockerServerException("Could not create container from image " + toCreate.dockerImage(), e); - } catch (NotFoundException e) { - log.error(e.getMessage()); - throw new DockerServerException("Could not pull image " + containerConfig.image() + " from repository.", e); + try { + // TODO this does some auth config stuff with the image that we should be aware of + final CreateContainerCmd cmd = client.createContainerCmd(toCreate.dockerImage()) + .withName(StringUtils.defaultIfBlank(toCreate.containerName(), "")) + .withHostConfig(hostConfig) + .withAttachStdout(true) + .withAttachStderr(true) + .withCmd(overrideEntrypoint ? + Arrays.asList("/bin/sh", "-c", toCreate.commandLine()) : + ShellSplitter.shellSplit(toCreate.commandLine())) + .withEnv(environmentVariables.entrySet() + .stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.toList())) + .withWorkingDir(StringUtils.defaultIfBlank(toCreate.workingDirectory(), "")) + .withUser(StringUtils.defaultIfBlank(server.containerUser(), "")) + .withLabels(instanceOrDefault(toCreate.containerLabels(), Collections.emptyMap())); + if (overrideEntrypoint) { + cmd.withEntrypoint(""); + } + final CreateContainerResponse resp = cmd.exec(); + for (String warning : resp.getWarnings()) { + log.warn(warning); + } + return resp.getId(); + } catch (DockerException e) { + log.error("Failed to create container", e); + throw new DockerServerException("Could not create container", e); } } @@ -616,11 +650,11 @@ private String createDockerSwarmService(final Container toCreate, final DockerSe if (StringUtils.isNotBlank(containerPort) && StringUtils.isNotBlank(hostPort)) { try { - portConfigs.add(PortConfig.builder() - .protocol(PortConfig.PROTOCOL_TCP) - .publishedPort(Integer.parseInt(hostPort)) - .targetPort(Integer.parseInt(containerPort)) - .build()); + portConfigs.add(new PortConfig() + .withProtocol(PortConfigProtocol.TCP) + .withPublishedPort(Integer.parseInt(hostPort)) + .withTargetPort(Integer.parseInt(containerPort)) + ); } catch (NumberFormatException e) { final String message = "Error creating port binding."; log.error(message, e); @@ -642,26 +676,23 @@ private String createDockerSwarmService(final Container toCreate, final DockerSe } final List mounts = toCreate.mounts().stream().map(containerMount -> - Mount.builder() - .source(containerMount.containerHostPath()) - .target(containerMount.containerPath()) - .readOnly(!containerMount.writable()) - .build() + new Mount() + .withSource(containerMount.containerHostPath()) + .withTarget(containerMount.containerPath()) + .withReadOnly(!containerMount.writable()) ).collect(Collectors.toList()); //TODO make this configurable from UI //add docker socket for 'docker in docker containers' - mounts.add(Mount.builder().source("/var/run/docker.sock").target("/var/run/docker.sock").readOnly(false).build()); + // mounts.add(Mount.builder().source("/var/run/docker.sock").target("/var/run/docker.sock").readOnly(false).build()); // Temporary work-around to support configurable shm-size in swarm service // https://github.com/moby/moby/issues/26714 - if (toCreate.shmSize() != null ){ - final Mount tmpfs = Mount - .builder() - .type("tmpfs") - .target("/dev/shm") - .tmpfsOptions(TmpfsOptions.builder().sizeBytes(toCreate.shmSize()).build()) - .build(); + if (toCreate.shmSize() != null) { + final Mount tmpfs = new Mount() + .withType(MountType.TMPFS) + .withTarget("/dev/shm") + .withTmpfsOptions(new TmpfsOptions().withSizeBytes(toCreate.shmSize())); log.debug("Creating tmpfs mount to support shm-size in Swarm Service: {}", tmpfs); @@ -684,74 +715,64 @@ private String createDockerSwarmService(final Container toCreate, final DockerSe // Environment variables final Map environmentVariables = containerPropertiesWithSecretValues.environmentVariables(); - final ContainerSpec.Builder containerSpecBuilder = ContainerSpec.builder() - .image(toCreate.dockerImage()) - .env(environmentVariables.entrySet() + final ContainerSpec containerSpec = new ContainerSpec() + .withImage(toCreate.dockerImage()) + .withDir(workingDirectory) + .withMounts(mounts) + .withUser(server.containerUser()) + .withLabels(toCreate.containerLabels()) + .withEnv(environmentVariables.entrySet() .stream() .map(entry -> entry.getKey() + "=" + entry.getValue()) - .collect(Collectors.toList())) - .dir(workingDirectory) - .mounts(mounts) - .user(server.containerUser()) - .labels(toCreate.containerLabels()); + .collect(Collectors.toList())); if (toCreate.overrideEntrypointNonnull()) { - containerSpecBuilder.command("/bin/sh", "-c", toCreate.commandLine()); + containerSpec.withCommand(Arrays.asList("/bin/sh", "-c", toCreate.commandLine())); } else { - containerSpecBuilder.args(ShellSplitter.shellSplit(toCreate.commandLine())); + containerSpec.withArgs(ShellSplitter.shellSplit(toCreate.commandLine())); } - final ContainerSpec containerSpec = containerSpecBuilder.build(); // Build out GPU/generic resources specifications from command definition to swarm/single server spec. - final List resourceSpecs = instanceOrDefault(toCreate.genericResources(), Collections.emptyMap()) + // TODO We currently cannot add generic resources to resource requirements in official docker-java + // See https://github.com/docker-java/docker-java/issues/2320 for issue and + // https://github.com/docker-java/docker-java/pull/2327 for a PR that would fix it. + // To support this we had to fork docker-java and make a custom build. + final List> resourceSpecs = instanceOrDefault(toCreate.genericResources(), Collections.emptyMap()) .entrySet() .stream() .map(entry -> StringUtils.isNumeric(entry.getValue()) ? - ResourceSpec.DiscreteResourceSpec.builder().kind(entry.getKey()).value(Integer.parseInt(entry.getValue())).build() : - ResourceSpec.NamedResourceSpec.builder().kind(entry.getKey()).value(entry.getValue()).build()) + new DiscreteResourceSpec().withKind(entry.getKey()).withValue(Integer.parseInt(entry.getValue())) : + new NamedResourceSpec().withKind(entry.getKey()).withValue(entry.getValue())) .collect(Collectors.toList()); - // Build named resources and add them to memory reservation requirements - // let resource constraints default to 0, so they're ignored by Docker - final ResourceRequirements resourceRequirements = - ResourceRequirements.builder() - .reservations(Reservations.builder() - .memoryBytes(toCreate.reserveMemoryBytes()) // megabytes to bytes - .resources(resourceSpecs) - .build()) - .limits(Resources.builder() - .memoryBytes(toCreate.limitMemoryBytes()) // megabytes to bytes - .nanoCpus(toCreate.nanoCpus()) - .build()) - .build(); - - final TaskSpec.Builder taskSpecBuilder = TaskSpec - .builder() - .containerSpec(containerSpec) - // ImmutablePlacement.Builder#addAllConstraints requires non-null even through it handles null right after that... - .placement(Placement.create(toCreate.swarmConstraints() == null ? Collections.emptyList() : toCreate.swarmConstraints())) - .restartPolicy(RestartPolicy.builder() - .condition(RestartPolicy.RESTART_POLICY_NONE) - .build()) - .resources(resourceRequirements); + final ResourceRequirements resourceRequirements = new ResourceRequirements() + .withReservations(new ResourceSpecs() + .withMemoryBytes(toCreate.reserveMemoryBytes()) + .withGenericResources(resourceSpecs) + ) + .withLimits(new ResourceSpecs() + .withMemoryBytes(toCreate.limitMemoryBytes()) + .withNanoCPUs(toCreate.nanoCpus())); + + + final TaskSpec taskSpec = new TaskSpec() + .withContainerSpec(containerSpec) + .withPlacement(new ServicePlacement().withConstraints(toCreate.swarmConstraints())) + .withRestartPolicy(new ServiceRestartPolicy().withCondition(ServiceRestartCondition.NONE)) + .withResources(resourceRequirements); if (!Strings.isNullOrEmpty(toCreate.network())) { - taskSpecBuilder.networks(NetworkAttachmentConfig.builder().target(toCreate.network()).build()); + taskSpec.withNetworks(Collections.singletonList(new NetworkAttachmentConfig().withTarget(toCreate.network()))); } - final TaskSpec taskSpec = taskSpecBuilder.build(); - - final ServiceSpec serviceSpec = ServiceSpec.builder() - .taskTemplate(taskSpec) - .mode(ServiceMode.builder() - .replicated(ReplicatedService.builder() - .replicas(numReplicas.value) - .build()) - .build()) - .endpointSpec(EndpointSpec.builder().ports(portConfigs).build()) - .name(toCreate.containerNameOrRandom()) - .labels(toCreate.containerLabels()) - .build(); - // Attempt to load registry auth, if it exists - final RegistryAuth registryAuth = getRegistryAuthForImage(toCreate.dockerImage()); + final ServiceSpec serviceSpec = new ServiceSpec() + .withName(toCreate.containerNameOrRandom()) + .withTaskTemplate(taskSpec) + .withEndpointSpec(new EndpointSpec().withPorts(portConfigs)) + .withLabels(toCreate.containerLabels()) + .withMode(new ServiceModeConfig() + .withReplicated(new ServiceReplicatedModeOptions().withReplicas(numReplicas.value))); + + final DockerClientConfig config = createDockerClientConfig(server); + final AuthConfig authConfig = authConfig(toCreate.dockerImage(), config); if (log.isDebugEnabled()) { final String message = String.format( @@ -776,47 +797,16 @@ private String createDockerSwarmService(final Container toCreate, final DockerSe log.debug(message); } + final DockerClient client = getDockerClient(server); try { - final DockerClient client = getDockerClient(server); - final ServiceCreateResponse serviceCreateResponse = client.createService(serviceSpec, registryAuth); - - final List warnings = serviceCreateResponse.warnings(); - if (warnings != null) { - for (String warning : warnings) { - log.warn(warning); - } - } - - return serviceCreateResponse.id(); - } catch (DockerException | InterruptedException e) { - log.error(e.getMessage()); - throw new DockerServerException("Could not create service: " + e.getMessage(), e); - } - } - - @Nullable - private RegistryAuth getRegistryAuthForImage(final String image) { - RegistryAuth registryAuth = null; - try { - // config file first - registryAuth = new ConfigFileRegistryAuthSupplier().authFor(image); - - // If no entry in config, see if we have hub credentials - if (registryAuth == null) { - final DockerHub hub = dockerHubService.getHubForImage(image); - if (hub != null && (StringUtils.isNotBlank(hub.username()) || StringUtils.isNotBlank(hub.token()))) { - registryAuth = RegistryAuth.builder() - .username(hub.username()) - .password(hub.password()) - .identityToken(hub.token()) - .serverAddress(hub.url()) - .build(); - } - } + return client.createServiceCmd(serviceSpec) + .withAuthConfig(authConfig) + .exec() + .getId(); } catch (DockerException e) { - log.error("Problem creating registry auth", e); + log.error("Failed to create service", e); + throw new DockerServerException("Could not create service", e); } - return registryAuth; } /** @@ -847,44 +837,45 @@ public void start(final Container toStart) throws NoContainerServerException, Co private void setSwarmServiceReplicasToOne(final String serviceId, final DockerServer server) throws DockerServerException { - try { - final DockerClient client = getDockerClient(server); + final DockerClient client = getDockerClient(server); + + final com.github.dockerjava.api.model.Service service; + try (final InspectServiceCmd cmd = client.inspectServiceCmd(serviceId)) { log.debug("Inspecting service {}", serviceId); - final org.mandas.docker.client.messages.swarm.Service service = client.inspectService(serviceId); - if (service == null || service.spec() == null) { - throw new DockerServerException("Could not start service " + serviceId + ". Could not inspect service spec."); - } - final ServiceSpec originalSpec = service.spec(); - final ServiceSpec updatedSpec = ServiceSpec.builder() - .name(originalSpec.name()) - .labels(originalSpec.labels()) - .updateConfig(originalSpec.updateConfig()) - .taskTemplate(originalSpec.taskTemplate()) - .endpointSpec(originalSpec.endpointSpec()) - .mode(ServiceMode.builder() - .replicated(ReplicatedService.builder() - .replicas(1L) - .build()) - .build()) - .build(); - final Long version = service.version() != null && service.version().index() != null ? - service.version().index() : null; - - log.info("Setting service replication to 1 for service {}", serviceId); - client.updateService(serviceId, version, updatedSpec); - } catch (DockerException | InterruptedException e) { - log.error(e.getMessage()); - throw new DockerServerException("Could not start service " + serviceId + ": " + e.getMessage(), e); + service = cmd.exec(); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { + log.error("Could not inspect service {}", serviceId, e); + throw new DockerServerException("Could not inspect service " + serviceId, e); + } + + final ServiceSpec originalSpec = service != null ? service.getSpec() : null; + if (originalSpec == null) { + throw new DockerServerException("Could not start service " + serviceId + ". Could not inspect service spec."); + } + final ServiceSpec updatedSpec = originalSpec + .withMode(new ServiceModeConfig() + .withReplicated(new ServiceReplicatedModeOptions().withReplicas(1)) + ); + + final Long version = service.getVersion() != null ? service.getVersion().getIndex() : null; + + try (final UpdateServiceCmd cmd = client.updateServiceCmd(serviceId, updatedSpec) + .withVersion(version)) { + log.debug("Updating service {}", serviceId); + cmd.exec(); + } catch (DockerException e) { + throw new DockerServerException("Could not update service", e); } } private void startDockerContainer(final String containerId, final DockerServer server) throws DockerServerException { - try { + final DockerClient client = getDockerClient(server); + try (final StartContainerCmd cmd = client.startContainerCmd(containerId)) { log.info("Starting container {}", containerId); - getDockerClient(server).startContainer(containerId); - } catch (DockerException | InterruptedException e) { - log.error(e.getMessage()); - throw new DockerServerException("Could not start container " + containerId + ": " + e.getMessage(), e); + cmd.exec(); + } catch (com.github.dockerjava.api.exception.NotFoundException | NotModifiedException e) { + log.error("Could not start container {}", containerId, e); + throw new DockerServerException("Could not start container " + containerId, e); } } @@ -913,9 +904,9 @@ public String getLog(final Container container, final LogType logType, final Boo final DockerServer server = getServer(); switch (server.backend()) { case SWARM: - return getSwarmServiceLog(container.serviceId(), assembleDockerClientLogsParams(logType, withTimestamps, since)); + return getSwarmServiceLog(server, container.serviceId(), logType, withTimestamps, since); case DOCKER: - return getDockerContainerLog(container.containerId(), assembleDockerClientLogsParams(logType, withTimestamps, since)); + return getDockerContainerLog(server, container.containerId(), logType, withTimestamps, since); case KUBERNETES: return getKubernetesClient().getLog(container.podName(), logType, withTimestamps, since); default: @@ -923,52 +914,42 @@ public String getLog(final Container container, final LogType logType, final Boo } } - private String getDockerContainerLog(final String containerId, final DockerClient.LogsParam... logsParams) throws NoDockerServerException, DockerServerException { - final DockerClient client = getDockerClient(); - try (final LogStream logStream = client.logs(containerId, logsParams)) { - return logStream.readFully(); - } catch (Exception e) { - log.error(e.getMessage()); - throw new DockerServerException(e); - } - } + private String getDockerContainerLog(final DockerServer server, final String containerId, final LogType logType, final Boolean withTimestamps, final OffsetDateTime since) throws DockerServerException { + final DockerClient client = getDockerClient(server); - private String getSwarmServiceLog(final String serviceId, final DockerClient.LogsParam... logsParams) throws NoDockerServerException, DockerServerException { - final DockerClient client = getDockerClient(); - try (final LogStream logStream = client.serviceLogs(serviceId, logsParams)) { - return logStream.readFully(); - } catch (Exception e) { - log.error(e.getMessage()); + final GetLogCallback callback = client.logContainerCmd(containerId) + .withStdOut(logType == LogType.STDOUT) + .withStdErr(logType == LogType.STDERR) + .withFollowStream(false) + .withTimestamps(withTimestamps) + .withSince(since == null ? null : Math.toIntExact(since.toEpochSecond())) + .exec(new GetLogCallback()); + try { + callback.awaitCompletion(); + } catch (InterruptedException | DockerException e) { + log.error("Could not get container log", e); throw new DockerServerException(e); } + return callback.getLog(); } - private DockerClient.LogsParam[] assembleDockerClientLogsParams(final LogType logType, final Boolean withTimestamp, final OffsetDateTime since) { - - final DockerClient.LogsParam dockerClientLogType = logType == LogType.STDOUT ? - DockerClient.LogsParam.stdout() : - DockerClient.LogsParam.stderr(); - - final Integer sinceInt = since == null ? null : Math.toIntExact(since.toEpochSecond()); - if (withTimestamp == null && sinceInt == null) { - return new DockerClient.LogsParam[] {dockerClientLogType}; - } else if (withTimestamp == null) { - return new DockerClient.LogsParam[] { - dockerClientLogType, - DockerClient.LogsParam.since(sinceInt) - }; - } else if (since == null) { - return new DockerClient.LogsParam[] { - dockerClientLogType, - DockerClient.LogsParam.timestamps(withTimestamp) - }; - } else { - return new DockerClient.LogsParam[] { - dockerClientLogType, - DockerClient.LogsParam.timestamps(withTimestamp), - DockerClient.LogsParam.since(sinceInt) - }; + private String getSwarmServiceLog(final DockerServer server, final String serviceId, final LogType logType, final Boolean withTimestamps, final OffsetDateTime since) throws DockerServerException { + final DockerClient client = getDockerClient(server); + + final GetLogCallback callback = client.logServiceCmd(serviceId) + .withStdout(logType == LogType.STDOUT) + .withStderr(logType == LogType.STDERR) + .withFollow(false) + .withTimestamps(withTimestamps) + .withSince(since == null ? null : Math.toIntExact(since.toEpochSecond())) + .exec(new GetLogCallback()); + try { + callback.awaitCompletion(); + } catch (InterruptedException | DockerException e) { + log.error("Could not get service log", e); + throw new DockerServerException(e); } + return callback.getLog(); } @VisibleForTesting @@ -977,21 +958,18 @@ public DockerClient getDockerClient() throws NoDockerServerException { return getDockerClient(getServer()); } + @VisibleForTesting @Nonnull - private DockerClient getDockerClient(final DockerServer server) { + public DockerClient getDockerClient(final DockerServer server) { final DockerClientCacheKey key = new DockerClientCacheKey(server); if (CACHED_DOCKER_CLIENT == null || !key.equals(CACHED_DOCKER_CLIENT_KEY)) { synchronized (CACHED_DOCKER_CLIENT_MUTEX) { if (CACHED_DOCKER_CLIENT == null || !key.equals(CACHED_DOCKER_CLIENT_KEY)) { clearDockerClientCache_onlyCallFromWithinSyncBlock(); - CACHED_DOCKER_CLIENT_KEY = key; - try { - log.debug("Creating new docker client instance with key {}", key); - CACHED_DOCKER_CLIENT = createDockerClient(server); - } catch (DockerServerException e) { - throw new RuntimeException(e); - } + log.debug("Creating new docker client instance with key {}", key); + CACHED_DOCKER_CLIENT_KEY = key; + CACHED_DOCKER_CLIENT = createDockerClient(server); } } } @@ -1020,32 +998,35 @@ private void clearDockerClientCache_onlyCallFromWithinSyncBlock() { CACHED_DOCKER_CLIENT = null; } - @Nonnull - private DockerClient createDockerClient(final @Nonnull DockerServer server) throws DockerServerException { - JerseyDockerClientBuilder clientBuilder = new JerseyDockerClientBuilder(); - clientBuilder.uri(server.host()); + private DockerClient createDockerClient(final @Nonnull DockerServer server) { + final DockerClientConfig config = createDockerClientConfig(server); + + final DockerHttpClient httpClient = new OkDockerHttpClient.Builder() + .dockerHost(config.getDockerHost()) + .sslConfig(config.getSSLConfig()) + .connectTimeout(3000) + .readTimeout(4500) + .build(); + + return DockerClientImpl.getInstance(config, httpClient); + } + + private DockerClientConfig createDockerClientConfig(final @Nonnull DockerServer server) { + DefaultDockerClientConfig.Builder configBuilder = DefaultDockerClientConfig.createDefaultConfigBuilder(); + final String host = server.host(); + if (StringUtils.isNotBlank(host)) { + configBuilder.withDockerHost(host); + } final String certPath = server.certPath(); if (StringUtils.isNotBlank(certPath)) { - try { - final DockerCertificates certificates = - new DockerCertificates(Paths.get(certPath)); - clientBuilder.dockerCertificates(certificates); - } catch (DockerCertificateException e) { - log.error("Could not find docker certificates at {}", server.certPath(), e); - } + configBuilder.withDockerCertPath(certPath); } - try { - log.trace("DOCKER CLIENT URI IS: {}", clientBuilder.uri().toString()); + // TODO are there other things that need to be configured here? - return clientBuilder.build(); - } catch (Throwable e) { - log.error("Could not create DockerClient instance. Reason: {}", e.getMessage(), e); - throw new DockerServerException(e); - } + return configBuilder.build(); } - /** * Kill a backend entity *

@@ -1116,166 +1097,150 @@ private void remove(final Container container, final DockerServer server) private void killDockerContainer(final String backendId, final DockerServer server) throws DockerServerException, NotFoundException { - try { + final DockerClient client = getDockerClient(server); + try (final KillContainerCmd cmd = client.killContainerCmd(backendId)) { log.debug("Killing container {}", backendId); - getDockerClient(server).killContainer(backendId); - } catch (ContainerNotFoundException e) { - log.error(e.getMessage(), e); + cmd.exec(); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { + log.error("Could not kill container {}: Not found.", backendId, e); throw new NotFoundException(e); - } catch (DockerException | InterruptedException e) { - log.error(e.getMessage(), e); + } catch (DockerException e) { + log.error("Could not kill container {}", backendId, e); throw new DockerServerException(e); } } private void removeDockerContainer(final String backendId, final DockerServer server) throws DockerServerException, NotFoundException { - try { + final DockerClient client = getDockerClient(server); + try (final RemoveContainerCmd cmd = client.removeContainerCmd(backendId)) { log.debug("Removing container {}", backendId); - getDockerClient(server).removeContainer(backendId); - } catch (ContainerNotFoundException e) { - log.error(e.getMessage(), e); + cmd.exec(); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { + log.error("Could not remove container {}: Not found", backendId, e); throw new NotFoundException(e); - } catch (DockerException | InterruptedException e) { - log.error(e.getMessage(), e); + } catch (DockerException e) { + log.error("Could not remove container {}", backendId, e); throw new DockerServerException(e); } } private void removeDockerSwarmService(final String backendId, final DockerServer server) throws DockerServerException, NotFoundException { - try { + final DockerClient client = getDockerClient(server); + try (final RemoveServiceCmd cmd = client.removeServiceCmd(backendId)) { log.debug("Removing service {}", backendId); - getDockerClient(server).removeService(backendId); - } catch (ServiceNotFoundException e) { - log.error(e.getMessage(), e); + cmd.exec(); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { + log.error("Could not remove service {}: Not found", backendId, e); throw new NotFoundException(e); - } catch (DockerException | InterruptedException e) { - log.error(e.getMessage(), e); + } catch (DockerException e) { + log.error("Could not remove service {}", backendId, e); throw new DockerServerException(e); } } @Override @Nullable - public ServiceTask getTaskForService(final DockerServer dockerServer, final Container service) + public ServiceTask getTaskForService(final DockerServer server, final Container service) throws DockerServerException, ServiceNotFoundException, TaskNotFoundException { - log.debug("Getting task for service {} \"{}\".", service.databaseId(), service.serviceId()); - try { - Task task = null; - final String serviceId = service.serviceId(); - final String taskId = service.taskId(); - final DockerClient client = getDockerClient(dockerServer); - - if (taskId == null) { - log.trace("Inspecting swarm service {} \"{}\".", service.databaseId(), service.serviceId()); - final org.mandas.docker.client.messages.swarm.Service serviceResponse = client.inspectService(serviceId); - final String serviceName = serviceResponse.spec().name(); - if (StringUtils.isBlank(serviceName)) { - throw new DockerServerException("Unable to determine service name for serviceId " + serviceId + - ". Cannot get taskId without this."); - } + final DockerClient client = getDockerClient(server); - log.trace("Service {} \"{}\" has name \"{}\" based on inspection: {}. Querying for task matching this service name.", - service.databaseId(), serviceId, serviceName, serviceResponse); - - final List tasks = client.listTasks(Task.Criteria.builder().serviceName(serviceName).build()); + final String serviceId = service.serviceId(); + final String taskId = service.taskId(); + log.debug("Getting task for service {} \"{}\".", service.databaseId(), serviceId); - if (tasks.size() == 1) { - task = tasks.get(0); - log.trace("Found one task \"{}\" for service {} \"{}\" name \"{}\"", task.id(), service.databaseId(), serviceId, serviceName); - } else if (tasks.isEmpty()) { - log.debug("No tasks found for service {} \"{}\" name \"{}\"", service.databaseId(), serviceId, serviceName); - } else { - throw new DockerServerException("Found multiple tasks for service " + service.databaseId() + - " \"" + serviceId + "\" name \"" + serviceName + - "\", I only know how to handle one. Tasks: " + tasks); - } - } else { - log.trace("Service {} \"{}\" has task \"{}\"", service.databaseId(), serviceId, taskId); - task = client.inspectTask(taskId); + final List tasks; + if (taskId == null) { + log.trace("Inspecting swarm service {} \"{}\".", service.databaseId(), serviceId); + final String serviceName; + try { + final com.github.dockerjava.api.model.Service serviceResponse = client.inspectServiceCmd(serviceId).exec(); + serviceName = serviceResponse.getSpec() != null ? serviceResponse.getSpec().getName() : null; + log.trace("Service {} \"{}\" has name \"{}\" based on inspection: {}", + service.databaseId(), serviceId, serviceName, serviceResponse); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { + throw new ServiceNotFoundException(e); } - if (task != null) { - final ServiceTask serviceTask = ServiceTask.create(task, serviceId); - - if (serviceTask.isExitStatus() && serviceTask.exitCode() == null) { - // The Task is supposed to have the container exit code, but docker doesn't report it where it should. - // So go get the container info and get the exit code - final String containerId = serviceTask.containerId(); - log.debug("Looking up exit code for container {}.", containerId); - if (containerId != null) { - final ContainerInfo containerInfo = client.inspectContainer(containerId); - if (containerInfo.state().exitCode() == null) { - log.debug("Welp. Container exit code is null on the container too."); - } else { - return serviceTask.toBuilder().exitCode(containerInfo.state().exitCode()).build(); - } - } else { - log.error("Cannot look up exit code. Container ID is null."); - } - } - - return serviceTask; + if (StringUtils.isBlank(serviceName)) { + throw new DockerServerException("Unable to determine service name for serviceId " + serviceId + + ". Cannot get taskId without this."); } - } catch (ServiceNotFoundException | TaskNotFoundException e) { - throw e; - } catch (DockerException | InterruptedException e) { - log.trace("INTERRUPTED: {}", e.getMessage()); - log.error(e.getMessage(), e); - throw new DockerServerException(e); - } catch (DockerServerException e) { - log.error(e.getMessage(), e); - throw e; - } - return null; - } - @Override - public List getContainerEvents(final Date since, final Date until) throws NoDockerServerException, DockerServerException { - final List dockerEventList = getDockerContainerEvents(since, until); - - final List events = Lists.newArrayList(); - for (final Event dockerEvent : dockerEventList) { - final Event.Actor dockerEventActor = dockerEvent.actor(); - final Map attributes = Maps.newHashMap(); - if (dockerEventActor != null && dockerEventActor.attributes() != null) { - attributes.putAll(dockerEventActor.attributes()); + try { + tasks = client.listTasksCmd().withServiceFilter(serviceName).exec(); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { + throw new TaskNotFoundException(e); + } catch (DockerException e) { + log.error("Could not get task for service {} \"{}\".", service.databaseId(), serviceId, e); + throw new DockerServerException(e); } - if (attributes.containsKey(LABEL_KEY)) { - attributes.put(LABEL_KEY, ""); + } else { + log.trace("Service {} \"{}\" has task \"{}\"", service.databaseId(), serviceId, taskId); + try { + tasks = client.listTasksCmd().withIdFilter(taskId).exec(); + } catch (com.github.dockerjava.api.exception.NotFoundException e) { + throw new TaskNotFoundException(e); + } catch (DockerException e) { + log.error("Could not get task for service {} \"{}\".", service.databaseId(), serviceId, e); + throw new DockerServerException(e); } - final DockerContainerEvent containerEvent = - DockerContainerEvent.create(dockerEvent.action(), - dockerEventActor != null? dockerEventActor.id() : null, - dockerEvent.time(), - dockerEvent.timeNano(), - attributes); - events.add(containerEvent); } - return events; - } - - private List getDockerContainerEvents(final Date since, final Date until) throws NoDockerServerException, DockerServerException { - try { - log.trace("Reading all docker container events from {} to {}.", since.getTime(), until.getTime()); - final List eventList; - try (final EventStream eventStream = - getDockerClient().events(since(since.getTime() / 1000), - until(until.getTime() / 1000), - type(Event.Type.CONTAINER))) { - - log.trace("Got a stream of docker events."); + final Task task; + if (tasks.size() == 1) { + task = tasks.get(0); + log.trace("Found one task \"{}\" for service {} \"{}\"", task.getId(), service.databaseId(), serviceId); + } else if (tasks.isEmpty()) { + log.debug("No tasks found for service {} \"{}\"", service.databaseId(), serviceId); + return null; + } else { + throw new DockerServerException("Found multiple tasks for service " + service.databaseId() + + " \"" + serviceId + "\", I only know how to handle one. Tasks: " + tasks); + } - eventList = Lists.newArrayList(eventStream); + final ServiceTask serviceTask = ServiceTask.create(task, serviceId); + + if (serviceTask.isExitStatus() && serviceTask.exitCode() == null) { + // The Task is supposed to have the container exit code, but docker doesn't report it where it should. + // So go get the container info and get the exit code + final String containerId = serviceTask.containerId(); + log.debug("Looking up exit code for container {}.", containerId); + if (containerId != null) { + final InspectContainerResponse containerInfo = client.inspectContainerCmd(containerId).exec(); + final Long containerExitCode = containerInfo.getState().getExitCodeLong(); + if (containerExitCode == null) { + log.debug("Welp. Container exit code is null on the container too."); + } else { + return serviceTask.toBuilder().exitCode(containerExitCode).build(); + } + } else { + log.error("Cannot look up exit code. Container ID is null."); } + } - log.trace("Closed docker event stream."); + return serviceTask; + } - return eventList; + @Override + public List getContainerEvents(final Date since, final Date until) throws NoDockerServerException, DockerServerException { + final DockerClient client = getDockerClient(getServer()); + try (final EventsCmd cmd = client.eventsCmd() + .withSince(String.valueOf(since.getTime() / 1000)) + .withUntil(String.valueOf(until.getTime() / 1000)) + .withEventTypeFilter(EventType.CONTAINER); + final GetContainerEventsCallback callback = new GetContainerEventsCallback()) { + + // Execute the command and get the events + log.info("Requesting events from {} to {}", since.getTime() / 1000, until.getTime() / 1000); + cmd.exec(callback); + callback.awaitCompletion(); + log.debug("Completed requesting events from {} to {}", since.getTime() / 1000, until.getTime() / 1000); + + return callback.events; } catch (IOException | InterruptedException | DockerException e) { - log.error(e.getMessage(), e); + log.error("Error getting container events", e); throw new DockerServerException(e); } } @@ -1305,43 +1270,56 @@ public boolean isStatusEmailEnabled() { } } - /** - * Convert spotify-docker Image object to xnat-container Image object - * - * @param image Spotify-Docker Image object - * @return NRG Image object - **/ - @Nullable - private DockerImage spotifyToNrg(final @Nullable Image image) { - return image == null ? null : - DockerImage.create(image.id(), image.repoTags(), image.labels()); - } + public enum NumReplicas { + ZERO(0), + ONE(1); - /** - * Convert spotify-docker Image object to xnat-container Image object - * - * @param image Spotify-Docker Image object - * @return NRG Image object - **/ - @Nullable - private DockerImage spotifyToNrg(final @Nullable ImageInfo image) { - return image == null ? null : - DockerImage.builder() - .imageId(image.id()) - .labels(image.config().labels() == null ? - Collections.emptyMap() : - image.config().labels()) - .build(); + public final int value; + NumReplicas(int value) { + this.value = value; + } } - public enum NumReplicas { - ZERO(0L), - ONE(1L); + private static final class GetContainerEventsCallback extends ResultCallbackTemplate { + private final List events = new ArrayList<>(); - public final long value; - NumReplicas(long value) { - this.value = value; + @Override + public void onNext(Event event) { + log.debug("Received event: {}", event); + final Map attributes = new HashMap<>(); + final EventActor actor = event.getActor(); + if (actor != null && actor.getAttributes() != null) { + attributes.putAll(actor.getAttributes()); + } + if (attributes.containsKey(LABEL_KEY)) { + attributes.put(LABEL_KEY, ""); + } + this.events.add( + DockerContainerEvent.create(event.getAction(), + actor != null? actor.getId() : null, + new Date(event.getTime()), + event.getTimeNano(), + attributes) + ); } } + @VisibleForTesting + public static final class GetLogCallback extends ResultCallbackTemplate { + final ByteArrayOutputStream logBuilder = new ByteArrayOutputStream(); + + @Override + public void onNext(Frame frame) { + try { + logBuilder.write(frame.getPayload()); + } catch (IOException e) { + log.error("Error writing container log", e); + onError(e); + } + } + + public String getLog() { + return logBuilder.toString(); + } + } } \ No newline at end of file diff --git a/src/main/java/org/nrg/containers/events/ContainerStatusUpdater.java b/src/main/java/org/nrg/containers/events/ContainerStatusUpdater.java index d9995617..0be0eb83 100644 --- a/src/main/java/org/nrg/containers/events/ContainerStatusUpdater.java +++ b/src/main/java/org/nrg/containers/events/ContainerStatusUpdater.java @@ -1,14 +1,19 @@ package org.nrg.containers.events; +import com.github.dockerjava.api.model.TaskState; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; -import org.mandas.docker.client.exceptions.ServiceNotFoundException; -import org.mandas.docker.client.exceptions.TaskNotFoundException; import org.nrg.containers.api.ContainerControlApi; import org.nrg.containers.api.KubernetesClientFactory; import org.nrg.containers.events.model.DockerContainerEvent; import org.nrg.containers.events.model.ServiceTaskEvent; -import org.nrg.containers.exceptions.*; +import org.nrg.containers.exceptions.ContainerException; +import org.nrg.containers.exceptions.DockerServerException; +import org.nrg.containers.exceptions.InvalidDefinitionException; +import org.nrg.containers.exceptions.NoContainerServerException; +import org.nrg.containers.exceptions.NoDockerServerException; +import org.nrg.containers.exceptions.ServiceNotFoundException; +import org.nrg.containers.exceptions.TaskNotFoundException; import org.nrg.containers.model.container.auto.Container; import org.nrg.containers.model.container.auto.ServiceTask; import org.nrg.containers.model.server.docker.DockerServerBase.DockerServer; @@ -28,8 +33,6 @@ import java.util.Date; import java.util.List; -import static org.mandas.docker.client.messages.swarm.TaskStatus.TASK_STATE_FAILED; - @Slf4j @Component public class ContainerStatusUpdater implements Runnable { @@ -339,7 +342,7 @@ private void throwLostTaskEventForService(@Nonnull final Container service) { .serviceId(service.serviceId()) .taskId(null) .nodeId(null) - .status(TASK_STATE_FAILED) + .status(TaskState.FAILED.getValue()) .swarmNodeError(true) .statusTime(null) .message(ServiceTask.swarmNodeErrMsg) diff --git a/src/main/java/org/nrg/containers/exceptions/ServiceNotFoundException.java b/src/main/java/org/nrg/containers/exceptions/ServiceNotFoundException.java new file mode 100644 index 00000000..db0687c1 --- /dev/null +++ b/src/main/java/org/nrg/containers/exceptions/ServiceNotFoundException.java @@ -0,0 +1,7 @@ +package org.nrg.containers.exceptions; + +public class ServiceNotFoundException extends Exception { + public ServiceNotFoundException(Throwable e) { + super(e); + } +} diff --git a/src/main/java/org/nrg/containers/exceptions/TaskNotFoundException.java b/src/main/java/org/nrg/containers/exceptions/TaskNotFoundException.java new file mode 100644 index 00000000..1ccdcfa7 --- /dev/null +++ b/src/main/java/org/nrg/containers/exceptions/TaskNotFoundException.java @@ -0,0 +1,7 @@ +package org.nrg.containers.exceptions; + +public class TaskNotFoundException extends Exception { + public TaskNotFoundException(Throwable e) { + super(e); + } +} diff --git a/src/main/java/org/nrg/containers/model/container/auto/ServiceTask.java b/src/main/java/org/nrg/containers/model/container/auto/ServiceTask.java index 2032a719..b2d8d722 100644 --- a/src/main/java/org/nrg/containers/model/container/auto/ServiceTask.java +++ b/src/main/java/org/nrg/containers/model/container/auto/ServiceTask.java @@ -1,29 +1,26 @@ package org.nrg.containers.model.container.auto; +import com.github.dockerjava.api.model.Task; +import com.github.dockerjava.api.model.TaskState; +import com.github.dockerjava.api.model.TaskStatus; +import com.github.dockerjava.api.model.TaskStatusContainerStatus; import com.google.auto.value.AutoValue; -import org.mandas.docker.client.messages.swarm.ContainerStatus; -import org.mandas.docker.client.messages.swarm.Task; -import org.mandas.docker.client.messages.swarm.TaskStatus; -import org.apache.commons.lang3.StringUtils; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.util.Arrays; -import java.util.Date; +import javax.validation.constraints.NotNull; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; @AutoValue public abstract class ServiceTask { - private static final Pattern successStatusPattern = Pattern.compile(TaskStatus.TASK_STATE_COMPLETE); + private static final Pattern successStatusPattern = Pattern.compile(TaskState.COMPLETE.getValue()); private static final Pattern exitStatusPattern = Pattern.compile( - StringUtils.join( - Arrays.asList(TaskStatus.TASK_STATE_FAILED, TaskStatus.TASK_STATE_COMPLETE, - TaskStatus.TASK_STATE_REJECTED, TaskStatus.TASK_STATE_SHUTDOWN), '|')); - private static final Pattern hasNotStartedPattern = Pattern.compile( - StringUtils.join( - Arrays.asList(TaskStatus.TASK_STATE_NEW, TaskStatus.TASK_STATE_ALLOCATED, TaskStatus.TASK_STATE_PENDING, - TaskStatus.TASK_STATE_ASSIGNED, TaskStatus.TASK_STATE_ACCEPTED, TaskStatus.TASK_STATE_PREPARING, - TaskStatus.TASK_STATE_READY, TaskStatus.TASK_STATE_STARTING), '|')); + Stream.of(TaskState.FAILED, TaskState.COMPLETE, + TaskState.REJECTED, TaskState.SHUTDOWN) + .map(TaskState::getValue) + .collect(Collectors.joining("|"))); public static String swarmNodeErrMsg = "Swarm node error"; @@ -38,37 +35,38 @@ public abstract class ServiceTask { @Nullable public abstract String err(); @Nullable public abstract Long exitCode(); - public static ServiceTask create(final @Nonnull Task task, final String serviceId) { - final ContainerStatus containerStatus = task.status().containerStatus(); - Long exitCode = containerStatus == null ? null : containerStatus.exitCode(); + public static ServiceTask create(final @NotNull Task task, final String serviceId) { + final TaskStatus taskStatus = task.getStatus(); + final TaskStatusContainerStatus containerStatus = taskStatus.getContainerStatus(); + Long exitCode = containerStatus == null ? null : containerStatus.getExitCodeLong(); // swarmNodeError occurs when node is terminated / spot instance lost while service still trying to run on it // Criteria: current state = [not an exit status] AND either desired state = shutdown OR exit code = -1 // OR current state = shutdown - String curState = task.status().state(); - String msg = task.status().message(); - String err = task.status().err(); - if (curState.equals(TaskStatus.TASK_STATE_PENDING)) { + TaskState curState = taskStatus.getState(); + String msg = taskStatus.getMessage(); + String err = taskStatus.getErr(); + if (curState.equals(TaskState.PENDING)) { msg = ""; err = ""; } - boolean swarmNodeError = (!isExitStatus(curState) && - (task.desiredState().equals(TaskStatus.TASK_STATE_SHUTDOWN) || (exitCode != null && exitCode < 0))) || - curState.equals(TaskStatus.TASK_STATE_SHUTDOWN); + boolean swarmNodeError = (!isExitStatus(curState.getValue()) && + (task.getDesiredState().equals(TaskState.SHUTDOWN) || (exitCode != null && exitCode < 0))) || + curState.equals(TaskState.SHUTDOWN); if (swarmNodeError) { msg = swarmNodeErrMsg; } return ServiceTask.builder() .serviceId(serviceId) - .taskId(task.id()) - .nodeId(task.nodeId()) - .status(task.status().state()) + .taskId(task.getId()) + .nodeId(task.getNodeId()) + .status(curState.getValue()) .swarmNodeError(swarmNodeError) - .statusTime(task.status().timestamp().toString()) + .statusTime(taskStatus.getTimestamp()) .message(msg) .err(err) .exitCode(exitCode) - .containerId(containerStatus == null ? null : containerStatus.containerId()) + .containerId(containerStatus == null ? null : containerStatus.getContainerID()) .build(); } @@ -104,11 +102,6 @@ public boolean isSuccessfulStatus(){ return isSuccessfulStatus(status); } - public boolean hasNotStarted() { - final String status = status(); - return status == null || hasNotStartedPattern.matcher(status).matches(); - } - public static Builder builder() { return new AutoValue_ServiceTask.Builder(); } diff --git a/src/main/java/org/nrg/containers/services/DockerHubService.java b/src/main/java/org/nrg/containers/services/DockerHubService.java index 40d34906..285108a8 100644 --- a/src/main/java/org/nrg/containers/services/DockerHubService.java +++ b/src/main/java/org/nrg/containers/services/DockerHubService.java @@ -11,11 +11,11 @@ public interface DockerHubService extends BaseHibernateService { DockerHubEntity retrieve(String name) throws NotUniqueException; DockerHubEntity get(String name) throws NotUniqueException, NotFoundException; - DockerHubEntity getByUrl(final String url) throws NotUniqueException, NotFoundException; DockerHub retrieveHub(long id); DockerHub retrieveHub(String name) throws NotUniqueException; DockerHub getHub(long hubId) throws NotFoundException; DockerHub getHub(String hubName) throws NotFoundException, NotUniqueException; + DockerHub getByUrl(final String url); List getHubs(); void setDefault(DockerHub dockerHub, String username, String reason); @@ -34,8 +34,6 @@ public interface DockerHubService extends BaseHibernateService long getDefaultHubId(); DockerHub getDefault(); - DockerHub getHubForImage(String imageName); - class DockerHubDeleteDefaultException extends RuntimeException { public DockerHubDeleteDefaultException(final String message) { super(message); diff --git a/src/main/java/org/nrg/containers/services/impl/ContainerFinalizeServiceImpl.java b/src/main/java/org/nrg/containers/services/impl/ContainerFinalizeServiceImpl.java index 95febe83..32b0cc3b 100644 --- a/src/main/java/org/nrg/containers/services/impl/ContainerFinalizeServiceImpl.java +++ b/src/main/java/org/nrg/containers/services/impl/ContainerFinalizeServiceImpl.java @@ -1,5 +1,6 @@ package org.nrg.containers.services.impl; +import com.github.dockerjava.api.model.TaskState; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.collect.Lists; @@ -8,7 +9,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.mandas.docker.client.messages.swarm.TaskStatus; import org.nrg.action.ClientException; import org.nrg.containers.api.ContainerControlApi; import org.nrg.containers.api.LogType; @@ -288,7 +288,7 @@ public String apply(final Exception input) { : ""; final boolean startsWithFailed = containerStatus.startsWith(PersistentWorkflowUtils.FAILED); if (startsWithFailed || - containerStatus.equals(TaskStatus.TASK_STATE_FAILED) || + containerStatus.equals(TaskState.FAILED.getValue()) || containerStatus.equals("die")) { // This history entry should give us the details that we need if (startsWithFailed) { diff --git a/src/main/java/org/nrg/containers/services/impl/ContainerServiceImpl.java b/src/main/java/org/nrg/containers/services/impl/ContainerServiceImpl.java index 99984901..f32ed66c 100644 --- a/src/main/java/org/nrg/containers/services/impl/ContainerServiceImpl.java +++ b/src/main/java/org/nrg/containers/services/impl/ContainerServiceImpl.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.model.TaskState; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; @@ -13,7 +14,6 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; -import org.mandas.docker.client.messages.swarm.TaskStatus; import org.nrg.action.ClientException; import org.nrg.containers.api.ContainerControlApi; import org.nrg.containers.api.LogType; @@ -1072,7 +1072,7 @@ public boolean restartService(Container service, UserI userI) { final ContainerHistory newHistoryItem = ContainerHistory.builder() .entityType("service") .entityId(null) - .status(TaskStatus.TASK_STATE_FAILED) + .status(TaskState.FAILED.getValue()) .exitCode("126") .timeRecorded(now) .externalTimestamp(String.valueOf(now.getTime())) diff --git a/src/main/java/org/nrg/containers/services/impl/HibernateDockerHubService.java b/src/main/java/org/nrg/containers/services/impl/HibernateDockerHubService.java index bf83ddcc..49f9f073 100644 --- a/src/main/java/org/nrg/containers/services/impl/HibernateDockerHubService.java +++ b/src/main/java/org/nrg/containers/services/impl/HibernateDockerHubService.java @@ -3,7 +3,6 @@ import com.google.common.base.Function; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; -import org.mandas.docker.client.ImageRef; import org.nrg.containers.daos.DockerHubDao; import org.nrg.containers.exceptions.NotUniqueException; import org.nrg.containers.model.dockerhub.DockerHubBase.DockerHub; @@ -62,16 +61,18 @@ public DockerHubEntity get(final String name) throws NotUniqueException, NotFoun } @Override - public DockerHubEntity getByUrl(final String url) throws NotUniqueException, NotFoundException { - DockerHubEntity dockerHubEntity = getDao().findByUrl(url); - if (dockerHubEntity == null && url.startsWith("https")) { - // try with http (org.mandas.docker.client.ImageRef always returns https) - dockerHubEntity = getDao().findByUrl(url.replace("https", "http")); - } - if (dockerHubEntity == null) { - throw new NotFoundException("Could not find hub with name " + url); + public DockerHub getByUrl(final String url) { + try { + DockerHubEntity entity = getDao().findByUrl(url); + if (entity == null && url.startsWith("https")) { + // try with http (org.mandas.docker.client.ImageRef always returns https) + entity = getDao().findByUrl(url.replace("https", "http")); + } + return toPojo(entity, getDefaultHubId()); + } catch (NotUniqueException e) { + log.error("Found multiple DockerHubs with url {}", url); + return null; } - return dockerHubEntity; } @Override @@ -148,18 +149,6 @@ public DockerHub getDefault() { return retrieveHub(getDefaultHubId()); } - @Override - public DockerHub getHubForImage(String imageName) { - final String url = new ImageRef(imageName).getRegistryUrl(); - DockerHubEntity entity = null; - try { - entity = getByUrl(url); - } catch (NotUniqueException | NotFoundException e) { - log.error("Unable to locate docker hub for url {}", url, e); - } - return toPojo(entity, getDefaultHubId()); - } - @Override public void setDefault(final DockerHub dockerHub, final String username, final String reason) { setDefault(dockerHub.id(), username, reason); diff --git a/src/test/java/org/nrg/containers/CommandLaunchIntegrationTest.java b/src/test/java/org/nrg/containers/CommandLaunchIntegrationTest.java index 0eeeeb8f..543d6eda 100644 --- a/src/test/java/org/nrg/containers/CommandLaunchIntegrationTest.java +++ b/src/test/java/org/nrg/containers/CommandLaunchIntegrationTest.java @@ -1,6 +1,10 @@ package org.nrg.containers; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.model.DiscreteResourceSpec; +import com.github.dockerjava.api.model.GenericResource; +import com.github.dockerjava.api.model.Service; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; @@ -19,10 +23,6 @@ import org.junit.runner.Description; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.mandas.docker.client.DockerClient; -import org.mandas.docker.client.LoggingBuildHandler; -import org.mandas.docker.client.messages.swarm.ResourceSpec; -import org.mandas.docker.client.messages.swarm.Service; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import org.nrg.containers.api.DockerControlApi; @@ -48,6 +48,7 @@ import org.nrg.containers.services.DockerService; import org.nrg.containers.utils.BackendConfig; import org.nrg.containers.utils.ContainerServicePermissionUtils; +import org.nrg.containers.utils.LoggingBuildCallback; import org.nrg.containers.utils.TestingUtils; import org.nrg.framework.exceptions.NotFoundException; import org.nrg.xdat.entities.AliasToken; @@ -307,14 +308,7 @@ public void cleanup() throws Exception { } containersToCleanUp.clear(); - for (final String imageToCleanUp : imagesToCleanUp) { - try { - controlApi.getDockerClient().removeImage(imageToCleanUp, true, false); - } catch (Exception e) { - // do nothing - } - } - imagesToCleanUp.clear(); + TestingUtils.cleanDockerImages(controlApi.getDockerClient(), imagesToCleanUp); kubernetesClient.stop(); TestingUtils.cleanupKubernetesNamespace(kubernetesNamespace, kubernetesClient); @@ -600,7 +594,10 @@ public void testLaunchCommandWithSetupCommand() throws Exception { final String setupCommandImageName = setupCommandSplitOnColon[0] + ":" + setupCommandSplitOnColon[1]; final String setupCommandName = setupCommandSplitOnColon[2]; - controlApi.getDockerClient().build(setupCommandDirPath, setupCommandImageName); + controlApi.getDockerClient().buildImageCmd(setupCommandDirPath.toFile()) + .withTags(Collections.singleton(setupCommandImageName)) + .exec(new LoggingBuildCallback()) + .awaitCompletion(); imagesToCleanUp.add(setupCommandImageName); // Make the setup command from the json file. @@ -728,8 +725,14 @@ public void testLaunchCommandWithWrapupCommand() throws Exception { final String commandWithWrapupCommandImageName = commandWithWrapupCommand.image(); // Build two images: the wrapup image and the main image - controlApi.getDockerClient().build(wrapupCommandDirPath, wrapupCommandImageName, "Dockerfile.wrapup", new LoggingBuildHandler()); - controlApi.getDockerClient().build(wrapupCommandDirPath, commandWithWrapupCommandImageName, "Dockerfile.main", new LoggingBuildHandler()); + controlApi.getDockerClient().buildImageCmd(wrapupCommandDirPath.resolve("Dockerfile.wrapup").toFile()) + .withTags(Collections.singleton(wrapupCommandImageName)) + .exec(new LoggingBuildCallback()) + .awaitCompletion(); + controlApi.getDockerClient().buildImageCmd(wrapupCommandDirPath.resolve("Dockerfile.main").toFile()) + .withTags(Collections.singleton(commandWithWrapupCommandImageName)) + .exec(new LoggingBuildCallback()) + .awaitCompletion(); imagesToCleanUp.add(wrapupCommandImageName); imagesToCleanUp.add(commandWithWrapupCommandImageName); @@ -897,7 +900,10 @@ public void testEntrypointIsPreserved() throws Exception { final String commandJsonFile = Paths.get(testDir.toString(), "/command.json").toString(); final String imageName = "xnat/entrypoint-test:latest"; - controlApi.getDockerClient().build(testDir, imageName); + controlApi.getDockerClient().buildImageCmd(testDir.toFile()) + .withTags(Collections.singleton(imageName)) + .exec(new LoggingBuildCallback()) + .awaitCompletion(); imagesToCleanUp.add(imageName); final Command commandToCreate = mapper.readValue(new File(commandJsonFile), Command.class); @@ -937,7 +943,10 @@ public void testEntrypointIsRemoved() throws Exception { final String commandJsonFile = Paths.get(testDir.toString(), "/command.json").toString(); final String imageName = "xnat/entrypoint-test:latest"; - controlApi.getDockerClient().build(testDir, imageName); + controlApi.getDockerClient().buildImageCmd(testDir.toFile()) + .withTags(Collections.singleton(imageName)) + .exec(new LoggingBuildCallback()) + .awaitCompletion(); imagesToCleanUp.add(imageName); final Command commandToCreate = mapper.readValue(new File(commandJsonFile), Command.class); @@ -1016,7 +1025,10 @@ public void testDeleteCommandWhenDeleteImageAfterLaunchingContainer() throws Exc final String resourceDir = Paths.get(ClassLoader.getSystemResource("commandLaunchTest").toURI()).toString().replace("%20", " "); final Path testDir = Paths.get(resourceDir, "/testDeleteCommandWhenDeleteImageAfterLaunchingContainer"); - final String imageId = controlApi.getDockerClient().build(testDir, imageName); + final String imageId = controlApi.getDockerClient().buildImageCmd(testDir.toFile()) + .withTags(Collections.singleton(imageName)) + .exec(new LoggingBuildCallback()) + .awaitImageId(); final List commands = dockerService.saveFromImageLabels(imageName); @@ -1346,13 +1358,13 @@ public void testCommandWithGenericResourcesGpu() throws Exception { // Verify that we correctly sent the API message to the backend that we want a GPU final DockerClient dockerClient = controlApi.getDockerClient(); - final Service service = dockerClient.inspectService(execution.containerOrServiceId()); - final List genericResources = service.spec().taskTemplate().resources().reservations().resources(); + final Service service = dockerClient.inspectServiceCmd(execution.containerOrServiceId()).exec(); + final List> genericResources = service.getSpec().getTaskTemplate().getResources().getReservations().getGenericResources(); assertThat(genericResources, hasSize(1)); - final ResourceSpec gr = genericResources.get(0); - assertThat(gr.kind(), is("gpu")); - assertThat(gr, instanceOf(ResourceSpec.DiscreteResourceSpec.class)); - assertThat(((ResourceSpec.DiscreteResourceSpec) gr).value(), is(1)); + final GenericResource gr = genericResources.get(0); + assertThat(gr.getKind(), is("gpu")); + assertThat(gr, instanceOf(DiscreteResourceSpec.class)); + assertThat(gr.getValue(), is(1)); } private void printContainerLogs(final Container container) throws IOException { diff --git a/src/test/java/org/nrg/containers/ContainerCleanupIntegrationTest.java b/src/test/java/org/nrg/containers/ContainerCleanupIntegrationTest.java index 49d51b4e..9ebd4553 100644 --- a/src/test/java/org/nrg/containers/ContainerCleanupIntegrationTest.java +++ b/src/test/java/org/nrg/containers/ContainerCleanupIntegrationTest.java @@ -1,6 +1,7 @@ package org.nrg.containers; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.exception.NotFoundException; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.BatchV1Api; import lombok.extern.slf4j.Slf4j; @@ -16,8 +17,6 @@ import org.junit.runner.Description; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.mandas.docker.client.exceptions.ContainerNotFoundException; -import org.mandas.docker.client.exceptions.ServiceNotFoundException; import org.mockito.Mock; import org.mockito.Mockito; import org.nrg.containers.api.DockerControlApi; @@ -273,14 +272,7 @@ public void cleanup() throws Exception { } containersToCleanUp.clear(); - for (final String imageToCleanUp : imagesToCleanUp) { - try { - controlApi.getDockerClient().removeImage(imageToCleanUp, true, false); - } catch (Exception e) { - // do nothing - } - } - imagesToCleanUp.clear(); + TestingUtils.cleanDockerImages(controlApi.getDockerClient(), imagesToCleanUp); kubernetesClientFactory.shutdown(); TestingUtils.cleanupKubernetesNamespace(kubernetesNamespace, kubernetesClient); @@ -650,8 +642,8 @@ private void checkContainerRemoval(Container exited, boolean okIfMissing) throws case DOCKER: checkFunc = () -> { try { - controlApi.getDockerClient().inspectContainer(id); - } catch (ContainerNotFoundException e) { + controlApi.getDockerClient().inspectContainerCmd(id).exec(); + } catch (NotFoundException e) { // This is what we expect return true; } catch (Exception ignored) { @@ -663,8 +655,8 @@ private void checkContainerRemoval(Container exited, boolean okIfMissing) throws case SWARM: checkFunc = () -> { try { - controlApi.getDockerClient().inspectService(id); - } catch (ServiceNotFoundException e) { + controlApi.getDockerClient().inspectServiceCmd(id).exec(); + } catch (NotFoundException e) { // This is what we expect return true; } catch (Exception ignored) { diff --git a/src/test/java/org/nrg/containers/SwarmConstraintsIntegrationTest.java b/src/test/java/org/nrg/containers/SwarmConstraintsIntegrationTest.java index 68ca9ad1..d485fbae 100644 --- a/src/test/java/org/nrg/containers/SwarmConstraintsIntegrationTest.java +++ b/src/test/java/org/nrg/containers/SwarmConstraintsIntegrationTest.java @@ -1,6 +1,7 @@ package org.nrg.containers; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.model.SwarmNode; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.SystemUtils; import org.junit.After; @@ -13,9 +14,6 @@ import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runner.RunWith; -import org.mandas.docker.client.messages.swarm.Node; -import org.mandas.docker.client.messages.swarm.NodeInfo; -import org.mandas.docker.client.messages.swarm.NodeSpec; import org.mockito.Mockito; import org.nrg.containers.api.DockerControlApi; import org.nrg.containers.config.EventPullingIntegrationTestConfig; @@ -73,7 +71,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -85,6 +82,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assume.assumeThat; import static org.mockito.Matchers.any; @@ -292,36 +290,24 @@ public void setup() throws Exception { @After public void cleanup() throws Exception { fakeWorkflow = new FakeWorkflow(); - for (final String containerToCleanUp : containersToCleanUp) { - try { - controlApi.getDockerClient().removeService(containerToCleanUp); - } catch (Exception e) { - // do nothing - } - } - containersToCleanUp.clear(); - - for (final String imageToCleanUp : imagesToCleanUp) { - try { - controlApi.getDockerClient().removeImage(imageToCleanUp, true, false); - } catch (Exception e) { - // do nothing - } - } - imagesToCleanUp.clear(); + TestingUtils.cleanSwarmServices(controlApi.getDockerClient(), containersToCleanUp); + TestingUtils.cleanDockerImages(controlApi.getDockerClient(), imagesToCleanUp); for (Map.Entry> entry : nodeLabelsToReset.entrySet()) { String nodeId = entry.getKey(); Map originalLabels = entry.getValue(); - NodeInfo nodeInfo = controlApi.getDockerClient().inspectNode(nodeId); - NodeSpec current = nodeInfo.spec(); - NodeSpec original = NodeSpec.builder() - .name(current.name()) - .role(current.role()) - .availability(current.availability()) - .labels(originalLabels) - .build(); - controlApi.getDockerClient().updateNode(nodeId, nodeInfo.version().index(), original); + SwarmNode current = controlApi.getDockerClient() + .listSwarmNodesCmd() + .withIdFilter(Collections.singletonList(nodeId)) + .exec() + .stream() + .findFirst() + .orElseThrow(() -> new Exception("Node not found")); + controlApi.getDockerClient().updateSwarmNodeCmd() + .withSwarmNodeId(nodeId) + .withVersion(current.getVersion().getIndex()) + .withSwarmNodeSpec(current.getSpec().withLabels(originalLabels)) + .exec(); } nodeLabelsToReset.clear(); } @@ -337,12 +323,12 @@ private List setUpServerWithConstr final List constraints = new ArrayList<>(DEFAULT_SWARM_SERVER_CONSTRAINTS); // Find manager - List managerNodes = controlApi.getDockerClient().listNodes(Node.Criteria.builder().nodeRole("manager").build()); + List managerNodes = controlApi.getDockerClient().listSwarmNodesCmd().withRoleFilter(Collections.singletonList("manager")).exec(); assertThat(managerNodes.size(), greaterThan(0)); - Node managerNode = managerNodes.get(0); + SwarmNode managerNode = managerNodes.get(0); // Add test labels - Map oldLabels = managerNode.spec().labels(); + Map oldLabels = managerNode.getSpec().getLabels(); Map labelsToAdd = new HashMap<>(DEFAULT_MANAGER_NODE_LABELS); if (managerNodes.size() > 1) { @@ -367,14 +353,16 @@ private List setUpServerWithConstr } // Ensure we clean up node label changes - nodeLabelsToReset.put(managerNode.id(), oldLabels); + nodeLabelsToReset.put(managerNode.getId(), oldLabels); Map newLabels = new HashMap<>(oldLabels); newLabels.putAll(labelsToAdd); // Update manager node to match constraints - controlApi.getDockerClient().updateNode(managerNode.id(), managerNode.version().index(), - NodeSpec.builder(managerNode.spec()).labels(newLabels).build() - ); + controlApi.getDockerClient().updateSwarmNodeCmd() + .withSwarmNodeId(managerNode.getId()) + .withVersion(managerNode.getVersion().getIndex()) + .withSwarmNodeSpec(managerNode.getSpec().withLabels(newLabels)) + .exec(); // Add constraints to DockerServer settings DockerServer server = dockerServerBuilder @@ -425,7 +413,7 @@ public void testRunIfConstraintsAreSatisfied() throws Exception { // Check that service does run log.debug("Waiting for service to start running..."); - await().until(TestingUtils.serviceIsRunning(controlApi.getDockerClient(), service)); + await().until(TestingUtils.serviceIsRunning(controlApi.getDockerClient(), service)); //Running = success! log.debug("SUCCESS: Service is running"); } @@ -451,14 +439,14 @@ public void testDoNotRunIfConstraintsAreNotSatisfied() throws Exception { log.debug("Waiting for service to be created..."); await().until(() -> { try { - return Objects.equals(containerService.get(service.serviceId()).status(), "pending"); + return "pending".equals(containerService.get(service.serviceId()).status()); } catch (Exception e) { return false; } }); // Check that service is not running log.debug("Service was created. Checking that service is not running"); - assertThat(TestingUtils.serviceIsRunning(controlApi.getDockerClient(), service, true).call(), is(false)); + assertEquals(TestingUtils.serviceIsRunning(controlApi.getDockerClient(), service, true).call(), false); log.debug("SUCCESS: Service is not running"); } diff --git a/src/test/java/org/nrg/containers/SwarmRestartIntegrationTest.java b/src/test/java/org/nrg/containers/SwarmRestartIntegrationTest.java index 1746bf14..3f2b7ad0 100644 --- a/src/test/java/org/nrg/containers/SwarmRestartIntegrationTest.java +++ b/src/test/java/org/nrg/containers/SwarmRestartIntegrationTest.java @@ -1,5 +1,8 @@ package org.nrg.containers; +import com.github.dockerjava.api.model.SwarmNode; +import com.github.dockerjava.api.model.SwarmNodeAvailability; +import com.github.dockerjava.api.model.SwarmNodeManagerStatus; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.SystemUtils; import org.hibernate.NonUniqueObjectException; @@ -14,10 +17,6 @@ import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runner.RunWith; -import org.mandas.docker.client.DockerClient; -import org.mandas.docker.client.messages.swarm.ManagerStatus; -import org.mandas.docker.client.messages.swarm.NodeInfo; -import org.mandas.docker.client.messages.swarm.NodeSpec; import org.nrg.containers.api.DockerControlApi; import org.nrg.containers.config.EventPullingIntegrationTestConfig; import org.nrg.containers.model.command.auto.Command; @@ -81,6 +80,7 @@ import static org.mockito.Matchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.nrg.containers.utils.TestingUtils.BUSYBOX; import static org.powermock.api.mockito.PowerMockito.doNothing; import static org.powermock.api.mockito.PowerMockito.doReturn; import static org.powermock.api.mockito.PowerMockito.mockStatic; @@ -202,7 +202,7 @@ public void setup() throws Exception { TestingUtils.skipIfCannotConnectToSwarm(controlApi.getDockerClient()); assumeThat(SystemUtils.IS_OS_WINDOWS_7, is(false)); - TestingUtils.pullBusyBox(controlApi.getDockerClient()); + controlApi.pullImage(BUSYBOX); final Command sleeper = commandService.create(Command.builder() .name("long-running") @@ -218,29 +218,15 @@ public void setup() throws Exception { } @After - public void cleanup() { + public void cleanup() throws Exception { fakeWorkflow = new FakeWorkflow(); - for (final String containerToCleanUp : containersToCleanUp) { - try { - if (swarmMode) { - controlApi.getDockerClient().removeService(containerToCleanUp); - } else { - controlApi.getDockerClient().removeContainer(containerToCleanUp, DockerClient.RemoveContainerParam.forceKill()); - } - } catch (Exception e) { - // do nothing - } + if (swarmMode) { + TestingUtils.cleanSwarmServices(controlApi.getDockerClient(), containersToCleanUp); + } else { + TestingUtils.cleanDockerContainers(controlApi.getDockerClient(), containersToCleanUp); } - containersToCleanUp.clear(); - for (final String imageToCleanUp : imagesToCleanUp) { - try { - controlApi.getDockerClient().removeImage(imageToCleanUp, true, false); - } catch (Exception e) { - // do nothing - } - } - imagesToCleanUp.clear(); + TestingUtils.cleanDockerImages(controlApi.getDockerClient(), imagesToCleanUp); } @Test @@ -254,26 +240,51 @@ public void testRestartShutdown() throws Exception { containersToCleanUp.add(serviceId); log.debug("Waiting until task has started"); - await().until(TestingUtils.getServiceNode(controlApi.getDockerClient(), service), is(notNullValue())); + final String[] nodeIdHolder = new String[1]; + await().until(() -> { + try { + nodeIdHolder[0] = TestingUtils.getServiceNode(controlApi.getDockerClient(), service); + return nodeIdHolder[0] != null; + } catch (Exception ignored) { + return false; + } + }); + final String nodeId = nodeIdHolder[0]; // Restart log.debug("Kill node on which service is running to cause a restart"); - String nodeId = TestingUtils.getServiceNode(controlApi.getDockerClient(), service).call(); - NodeInfo nodeInfo = controlApi.getDockerClient().inspectNode(nodeId); - ManagerStatus managerStatus = nodeInfo.managerStatus(); - Boolean isManager; - if (managerStatus != null && (isManager = managerStatus.leader()) != null && isManager) { - NodeSpec nodeSpec = NodeSpec.builder(nodeInfo.spec()).availability("drain").build(); + SwarmNode nodeInfo = controlApi.getDockerClient() + .listSwarmNodesCmd() + .withIdFilter(Collections.singletonList(nodeId)) + .exec() + .stream() + .findAny() + .orElseThrow(() -> new Exception("Node not found")); + SwarmNodeManagerStatus managerStatus = nodeInfo.getManagerStatus(); + if (managerStatus != null && managerStatus.isLeader()) { // drain the manager - controlApi.getDockerClient().updateNode(nodeId, nodeInfo.version().index(), nodeSpec); + controlApi.getDockerClient().updateSwarmNodeCmd() + .withSwarmNodeId(nodeId) + .withVersion(nodeInfo.getVersion().getIndex()) + .withSwarmNodeSpec(nodeInfo.getSpec().withAvailability(SwarmNodeAvailability.DRAIN)) + .exec(); Thread.sleep(1000L); // Sleep long enough for status updater to run // readd manager - nodeInfo = controlApi.getDockerClient().inspectNode(nodeId); - nodeSpec = NodeSpec.builder(nodeInfo.spec()).availability("active").build(); - controlApi.getDockerClient().updateNode(nodeId, nodeInfo.version().index(), nodeSpec); + nodeInfo = controlApi.getDockerClient() + .listSwarmNodesCmd() + .withIdFilter(Collections.singletonList(nodeId)) + .exec() + .stream() + .findAny() + .orElseThrow(() -> new Exception("Node not found")); + controlApi.getDockerClient().updateSwarmNodeCmd() + .withSwarmNodeId(nodeId) + .withVersion(nodeInfo.getVersion().getIndex()) + .withSwarmNodeSpec(nodeInfo.getSpec().withAvailability(SwarmNodeAvailability.ACTIVE)) + .exec(); } else { // delete the node - controlApi.getDockerClient().deleteNode(nodeId, true); + controlApi.getDockerClient().removeSwarmNodeCmd(nodeId).withForce(true).exec(); Thread.sleep(500L); // Sleep long enough for status updater to run } @@ -302,7 +313,7 @@ public void testRestartClearedTask() throws Exception { // Restart log.debug("Removing service to throw a restart event"); - controlApi.getDockerClient().removeService(serviceId); + controlApi.getDockerClient().removeServiceCmd(serviceId).exec(); // Ensure the restart request has gone through await().until(TestingUtils.serviceHasTaskId(containerService, service.databaseId()), is(false)); @@ -333,7 +344,13 @@ public void testRestartClearedBeforeRunTask() throws Exception { .pollInterval(50, TimeUnit.MILLISECONDS) .until(() -> { try { - Long replicas = controlApi.getDockerClient().inspectService(service.serviceId()).spec().mode().replicated().replicas(); + long replicas = controlApi.getDockerClient() + .inspectServiceCmd(service.serviceId()) + .exec() + .getSpec() + .getMode() + .getReplicated() + .getReplicas(); log.debug("Service {} replicas {}", service.serviceId(), replicas); return replicas; } catch (Exception e) { @@ -343,7 +360,7 @@ public void testRestartClearedBeforeRunTask() throws Exception { // Restart log.debug("Removing service {} before it starts running to throw a restart event", service.serviceId()); - controlApi.getDockerClient().removeService(service.serviceId()); + controlApi.getDockerClient().removeServiceCmd(service.serviceId()).exec(); // ensure that container restarted & status updates, etc log.debug("Waiting for container service to restart container {}...", service.databaseId()); @@ -397,7 +414,7 @@ public void testRestartFailure() throws Exception { await().until(TestingUtils.serviceIsRunning(controlApi.getDockerClient(), service)); log.debug("Removing service to throw a restart event"); - controlApi.getDockerClient().removeService(serviceId); + controlApi.getDockerClient().removeServiceCmd(serviceId).exec(); Thread.sleep(1000L); // Sleep long enough for status updater to run // ensure that container restarted & status updates, etc @@ -417,6 +434,7 @@ public void testRestartFailure() throws Exception { @Test @DirtiesContext + @Ignore("Test has some kind of error with a NonUniqueObjectException that I can't figure out") public void testNoRestartOnAPIKill() throws Exception { log.debug("Queueing command for launch"); containerService.queueResolveCommandAndLaunchContainer(null, sleeperWrapper.id(), 0L, diff --git a/src/test/java/org/nrg/containers/api/DockerControlApiIntegrationTest.java b/src/test/java/org/nrg/containers/api/DockerControlApiIntegrationTest.java index aa17ceed..7dbb3001 100644 --- a/src/test/java/org/nrg/containers/api/DockerControlApiIntegrationTest.java +++ b/src/test/java/org/nrg/containers/api/DockerControlApiIntegrationTest.java @@ -1,8 +1,9 @@ package org.nrg.containers.api; +import com.github.dockerjava.api.model.HostConfig; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.SystemUtils; import org.hamcrest.CustomTypeSafeMatcher; -import org.hamcrest.Matcher; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -10,13 +11,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; -import org.mandas.docker.client.DockerClient; -import org.mandas.docker.client.exceptions.DockerException; -import org.mandas.docker.client.messages.ContainerConfig; -import org.mandas.docker.client.messages.ContainerCreation; -import org.mandas.docker.client.messages.HostConfig; -import org.mandas.docker.client.messages.Info; -import org.mandas.docker.client.messages.Version; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.nrg.containers.events.model.DockerContainerEvent; @@ -30,9 +24,10 @@ import org.nrg.containers.services.DockerServerService; import org.nrg.containers.utils.BackendConfig; import org.nrg.containers.utils.TestingUtils; +import org.nrg.framework.exceptions.NotFoundException; +import java.util.ArrayList; import java.util.Date; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -44,19 +39,20 @@ import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; +import static org.nrg.containers.utils.TestingUtils.BUSYBOX; @Slf4j @RunWith(MockitoJUnitRunner.class) public class DockerControlApiIntegrationTest { - private static final String BUSYBOX_LATEST = "busybox:latest"; private static final String ALPINE_LATEST = "alpine:latest"; - private static final String BUSYBOX_ID = "sha256:47bcc53f74dc94b1920f0b34f6036096526296767650f223433fe65c35f149eb"; - private static final String BUSYBOX_NAME = "busybox:1.24.2-uclibc"; + private static final String BUSYBOX_SPECIFIC_VERSION_ID = "sha256:3596868f4ba86907dde5849edf4f426f4dc2110b1ace9e5969f5a21838ca9599"; + private static final String BUSYBOX_SPECIFIC_VERSION_NAME = "busybox:1.36.0"; private static final DockerHubBase.DockerHub DOCKER_HUB = DockerHubBase.DockerHub.DEFAULT; - private final static Set containersToCleanUp = new HashSet<>(); - private final static Set imagesToCleanUp = new HashSet<>(); + private final static List containersToCleanUp = new ArrayList<>(); + private final static List imagesToCleanUp = new ArrayList<>(); @Mock private DockerServerService dockerServerService; @Mock private DockerHubService dockerHubService; @@ -91,39 +87,27 @@ public void setup() throws Exception { @AfterClass public static void classCleanup() throws Exception { - for (final String containerToCleanUp : containersToCleanUp) { - try { - controlApi.getDockerClient().removeContainer(containerToCleanUp, DockerClient.RemoveContainerParam.forceKill()); - } catch (Exception e) { - // do nothing - } - } - containersToCleanUp.clear(); - - for (final String imageToCleanUp : imagesToCleanUp) { - try { - controlApi.getDockerClient().removeImage(imageToCleanUp, true, false); - } catch (Exception e) { - // do nothing - } + if (controlApi != null) { + TestingUtils.cleanDockerContainers(controlApi.getDockerClient(), containersToCleanUp); + TestingUtils.cleanDockerImages(controlApi.getDockerClient(), imagesToCleanUp); } - imagesToCleanUp.clear(); } @Test public void testPingServer() throws Exception { - assertThat(TestingUtils.canConnectToDocker(controlApi.getDockerClient()), is(true)); + assertTrue(TestingUtils.canConnectToDocker(controlApi.getDockerClient())); } @Test public void testGetAllImages() throws Exception { - imagesToCleanUp.add(BUSYBOX_LATEST); + imagesToCleanUp.add(BUSYBOX); + controlApi.pullImage(BUSYBOX); + imagesToCleanUp.add(ALPINE_LATEST); + controlApi.pullImage(ALPINE_LATEST); - controlApi.getDockerClient().pull(BUSYBOX_LATEST); - controlApi.getDockerClient().pull(ALPINE_LATEST); final List images = controlApi.getAllImages(); final Set imageNames = images.stream() @@ -131,7 +115,7 @@ public void testGetAllImages() throws Exception { .filter(Objects::nonNull) .flatMap(List::stream) .collect(Collectors.toSet()); - assertThat(BUSYBOX_LATEST, isIn(imageNames)); + assertThat(BUSYBOX, isIn(imageNames)); assertThat(ALPINE_LATEST, isIn(imageNames)); } @@ -142,9 +126,9 @@ public void testPingHub() throws Exception { @Test public void testPullImage() throws Exception { - imagesToCleanUp.add(BUSYBOX_LATEST); + imagesToCleanUp.add(BUSYBOX); - controlApi.pullImage(BUSYBOX_LATEST, DOCKER_HUB); + controlApi.pullImage(BUSYBOX, DOCKER_HUB); } @Test @@ -152,7 +136,13 @@ public void testPullPrivateImage() throws Exception { final String privateImageName = "xnattest/private"; imagesToCleanUp.add(privateImageName); - exception.expect(imageNotFoundException(privateImageName)); + final String description = "NotFoundException with message containing image name \"" + privateImageName + "\""; + exception.expect(new CustomTypeSafeMatcher(description) { + @Override + protected boolean matchesSafely(final NotFoundException ex) { + return ex.getMessage().contains(privateImageName); + } + }); controlApi.pullImage(privateImageName, DOCKER_HUB); final DockerImage test = controlApi.pullImage(privateImageName, DOCKER_HUB, "xnattest", "windmill susanna portico", @@ -161,16 +151,16 @@ public void testPullPrivateImage() throws Exception { } @Test - public void testDeleteImage() throws DockerException, InterruptedException, NoDockerServerException, DockerServerException { - imagesToCleanUp.add(BUSYBOX_NAME); - controlApi.getDockerClient().pull(BUSYBOX_NAME); - int beforeImageCount = controlApi.getDockerClient().listImages().size(); - controlApi.deleteImageById(BUSYBOX_ID, true); - List images = controlApi.getDockerClient().listImages(); + public void testDeleteImage() throws NotFoundException, NoDockerServerException, DockerServerException { + imagesToCleanUp.add(BUSYBOX_SPECIFIC_VERSION_NAME); + controlApi.pullImage(BUSYBOX_SPECIFIC_VERSION_NAME); + int beforeImageCount = controlApi.getAllImages().size(); + controlApi.deleteImageById(BUSYBOX_SPECIFIC_VERSION_ID, true); + List images = controlApi.getAllImages(); int afterImageCount = images.size(); assertThat(afterImageCount+1, is(beforeImageCount)); - for(org.mandas.docker.client.messages.Image image:images){ - assertThat(image.id(), is(not(BUSYBOX_ID))); + for (DockerImage image:images){ + assertThat(image.imageId(), is(not(BUSYBOX_SPECIFIC_VERSION_ID))); } } @@ -178,53 +168,51 @@ public void testDeleteImage() throws DockerException, InterruptedException, NoDo public void testEventPolling() throws Exception { log.debug("Starting event polling test."); - if (log.isDebugEnabled()) { - final Version version = controlApi.getDockerClient().version(); - log.debug("Docker version: {}", version); - } - - final Info dockerInfo = controlApi.getDockerClient().info(); - if (dockerInfo.kernelVersion().contains("moby")) { - // If we are running docker in the moby VM, then it isn't running natively - // on the host machine. Sometimes the clocks on the host and VM can get out - // out sync, and this test will fail. This especially happens on laptops that + if (!Boolean.parseBoolean(SystemUtils.getEnvironmentVariable("CI", "false"))) { + // If we aren't running in a CI environment, then we can assume docker is running in a VM and + // isn't running natively on the host machine. Sometimes the clocks on the host and VM + // can get out of sync, causing this test to fail. This especially happens on laptops that // have docker running and go to sleep. // We can run this container command: // docker run --rm --privileged alpine hwclock -s // to sync up the clocks. It requires 'privileged' mode, which may cause problems // running in a CI environment. log.debug("Synchronizing host and vm clocks."); - final ContainerConfig containerConfig = ContainerConfig.builder() - .image("alpine") - .cmd("hwclock", "-s") - .hostConfig(HostConfig.builder() - .privileged(true) - .autoRemove(true) - .build()) - .build(); - imagesToCleanUp.add("alpine"); - final ContainerCreation containerCreation = controlApi.getDockerClient().createContainer(containerConfig); - - containersToCleanUp.add(containerCreation.id()); - controlApi.getDockerClient().startContainer(containerCreation.id()); + + controlApi.pullImage(ALPINE_LATEST); + imagesToCleanUp.add(ALPINE_LATEST); + + // Have to use the client directly because the controlApi doesn't expose the hostconfig + final String containerId = controlApi.getDockerClient().createContainerCmd(ALPINE_LATEST) + .withCmd("hwclock", "-s") + .withHostConfig(new HostConfig() + .withPrivileged(true) + .withAutoRemove(true)) + .exec() + .getId(); + + containersToCleanUp.add(containerId); + log.info("Starting container {}", containerId); + controlApi.getDockerClient().startContainerCmd(containerId).exec(); } final Date start = new Date(); log.debug("Start time is {}", start.getTime() / 1000); Thread.sleep(1000); // Wait to ensure we get some events - imagesToCleanUp.add(BUSYBOX_LATEST); - controlApi.pullImage(BUSYBOX_LATEST); + imagesToCleanUp.add(BUSYBOX); + controlApi.pullImage(BUSYBOX); // Create container, to ensure we have some events to read - final ContainerConfig config = ContainerConfig.builder() - .image(BUSYBOX_LATEST) - .cmd("sh", "-c", "echo Hello world") - .build(); - - final ContainerCreation creation = controlApi.getDockerClient().createContainer(config); - containersToCleanUp.add(creation.id()); - controlApi.getDockerClient().startContainer(creation.id()); + log.debug("Creating container"); + final String containerId = controlApi.getDockerClient() + .createContainerCmd(BUSYBOX) + .withCmd("sh", "-c", "echo Hello world") + .exec() + .getId(); + containersToCleanUp.add(containerId); + log.debug("Starting container"); + controlApi.getDockerClient().startContainerCmd(containerId).exec(); Thread.sleep(1000); // Wait to ensure we get some events final Date end = new Date(); @@ -238,17 +226,6 @@ public void testEventPolling() throws Exception { // TODO assert more things about the events } - - private Matcher imageNotFoundException(final String name) { - final String exceptionMessage = "Image not found: " + name; - final String description = "Image not found exception with image name " + name; - return new CustomTypeSafeMatcher(description) { - @Override - protected boolean matchesSafely(final Exception ex) { - return ex.getMessage().contains(exceptionMessage); - } - }; - } } diff --git a/src/test/java/org/nrg/containers/api/DockerControlApiTest.java b/src/test/java/org/nrg/containers/api/DockerControlApiTest.java index afbf8447..994371cb 100644 --- a/src/test/java/org/nrg/containers/api/DockerControlApiTest.java +++ b/src/test/java/org/nrg/containers/api/DockerControlApiTest.java @@ -1,5 +1,19 @@ package org.nrg.containers.api; +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.command.CreateContainerResponse; +import com.github.dockerjava.api.command.CreateServiceCmd; +import com.github.dockerjava.api.command.CreateServiceResponse; +import com.github.dockerjava.api.command.InspectServiceCmd; +import com.github.dockerjava.api.command.InspectSwarmCmd; +import com.github.dockerjava.api.command.LogContainerCmd; +import com.github.dockerjava.api.command.LogSwarmObjectCmd; +import com.github.dockerjava.api.command.PingCmd; +import com.github.dockerjava.api.command.UpdateServiceCmd; +import com.github.dockerjava.api.model.Service; +import com.github.dockerjava.api.model.ServiceModeConfig; +import com.github.dockerjava.api.model.ServiceSpec; +import com.google.common.collect.ImmutableList; import io.kubernetes.client.util.PatchUtils; import lombok.extern.slf4j.Slf4j; import org.junit.Before; @@ -10,22 +24,12 @@ import org.junit.runner.Description; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.mandas.docker.client.DockerClient; -import org.mandas.docker.client.LogStream; -import org.mandas.docker.client.messages.ContainerConfig; -import org.mandas.docker.client.messages.ContainerCreation; -import org.mandas.docker.client.messages.RegistryAuth; -import org.mandas.docker.client.messages.ServiceCreateResponse; -import org.mandas.docker.client.messages.swarm.ContainerSpec; -import org.mandas.docker.client.messages.swarm.ReplicatedService; -import org.mandas.docker.client.messages.swarm.Service; -import org.mandas.docker.client.messages.swarm.ServiceMode; -import org.mandas.docker.client.messages.swarm.ServiceSpec; -import org.mandas.docker.client.messages.swarm.TaskSpec; -import org.mandas.docker.client.messages.swarm.Version; +import org.mockito.Answers; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.nrg.containers.model.command.auto.ResolvedCommand; import org.nrg.containers.model.container.auto.Container; import org.nrg.containers.model.image.docker.DockerImage; @@ -40,23 +44,20 @@ import org.powermock.modules.junit4.PowerMockRunnerDelegate; import org.powermock.reflect.Whitebox; -import java.time.OffsetDateTime; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; -import java.util.List; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.junit.Assume.assumeThat; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.anyVararg; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -69,9 +70,13 @@ @PrepareForTest({DockerControlApi.class, PatchUtils.class}) public class DockerControlApiTest { final String BACKEND_ID = UUID.randomUUID().toString(); - final String LOG_CONTENTS = UUID.randomUUID().toString(); final String USER_LOGIN = UUID.randomUUID().toString(); + // This default answer lets us stub out all the method chaining .withFoo().withBar() without having to stub each one + // Have to make it explicitly because Mockito.RETURNS_SELF is not available in our version + @SuppressWarnings("rawtypes") + private final static Answer RETURN_SELF = InvocationOnMock::getMock; + @Parameterized.Parameters(name="backend={0}") public static Collection backend() { return EnumSet.allOf(Backend.class); @@ -97,9 +102,8 @@ protected void finished(Description description) { @Mock private KubernetesClientFactory kubernetesClientFactory; @Mock private KubernetesClient kubernetesClient; - @Mock private DockerClient mockDockerClient; + @Mock(answer = Answers.RETURNS_MOCKS) private com.github.dockerjava.api.DockerClient mockDockerJavaClient; @Mock private DockerImage mockDockerImage; - @Mock private LogStream logStream; @Mock private DockerServer dockerServer; @Mock private Container container; @@ -117,11 +121,11 @@ public void setup() throws Exception { dockerControlApi = PowerMockito.spy(new DockerControlApi( dockerServerService, dockerHubService, kubernetesClientFactory )); - PowerMockito.doReturn(mockDockerImage).when(dockerControlApi, method( - DockerControlApi.class, "pullImage", String.class)) + PowerMockito.doReturn(mockDockerImage) + .when(dockerControlApi, method(DockerControlApi.class, "pullImage", String.class)) .withArguments(anyString()); - PowerMockito.doReturn(mockDockerClient).when(dockerControlApi, method( - DockerControlApi.class, "getDockerClient", DockerServer.class)) + PowerMockito.doReturn(mockDockerJavaClient) + .when(dockerControlApi, method(DockerControlApi.class, "getDockerClient", DockerServer.class)) .withArguments(dockerServer); // Mock simple return values @@ -144,8 +148,6 @@ public void setup() throws Exception { } swarmMode = backend == Backend.SWARM; // backwards compatibility - when(logStream.readFully()).thenReturn(LOG_CONTENTS); - when(user.getLogin()).thenReturn(USER_LOGIN); } @@ -156,10 +158,14 @@ public void testPing() throws Exception { // Set up test-specific mocks switch (backend) { case DOCKER: - when(mockDockerClient.ping()).thenReturn(ok); + final PingCmd pingCmd = Mockito.mock(PingCmd.class); + when(mockDockerJavaClient.pingCmd()).thenReturn(pingCmd); + when(pingCmd.exec()).thenReturn(null); break; case SWARM: - when(mockDockerClient.listServices()).thenReturn(Collections.emptyList()); + final InspectSwarmCmd inspectSwarmCmd = Mockito.mock(InspectSwarmCmd.class); + when(mockDockerJavaClient.inspectSwarmCmd()).thenReturn(inspectSwarmCmd); + when(inspectSwarmCmd.exec()).thenReturn(null); break; case KUBERNETES: when(kubernetesClient.ping()).thenReturn(ok); @@ -176,137 +182,74 @@ public void testPing() throws Exception { @Test public void testGetLog_stdout() throws Exception { final LogType logType = LogType.STDOUT; - final DockerClient.LogsParam dockerClientLogType = DockerClient.LogsParam.stdout(); + final String logContents = UUID.randomUUID().toString(); // Set up test-specific mocks if (backend == Backend.SWARM) { - when(mockDockerClient.logs(any(String.class), (DockerClient.LogsParam) anyVararg())) - .thenThrow(new RuntimeException("Should not be called")); - when(mockDockerClient.serviceLogs(BACKEND_ID, dockerClientLogType)).thenReturn(logStream); + setUpForSwarmLogTest(logContents); } else if (backend == Backend.KUBERNETES) { when(container.podName()).thenReturn(BACKEND_ID); when(kubernetesClient.getLog( BACKEND_ID, logType, null, null - )).thenReturn(LOG_CONTENTS); + )).thenReturn(logContents); } else { - when(mockDockerClient.logs(BACKEND_ID, dockerClientLogType)).thenReturn(logStream); - when(mockDockerClient.serviceLogs(any(String.class), (DockerClient.LogsParam) anyVararg())) - .thenThrow(new RuntimeException("Should not be called")); + setUpForContainerLogTest(logContents); } // Run the test final String log = dockerControlApi.getLog(container, logType); // Check results - assertThat(log, is(equalTo(LOG_CONTENTS))); + assertThat(log, is(equalTo(logContents))); } @Test public void testGetLog_stderr() throws Exception { - assumeThat("Special stderr handling for kubernetes", backend, not(Backend.KUBERNETES)); - final LogType logType = LogType.STDERR; - final DockerClient.LogsParam dockerClientLogType = DockerClient.LogsParam.stderr(); + final String logContents = UUID.randomUUID().toString(); // Set up test-specific mocks - if (swarmMode) { - when(mockDockerClient.logs(any(String.class), (DockerClient.LogsParam) anyVararg())) - .thenThrow(new RuntimeException("Should not be called")); - when(mockDockerClient.serviceLogs(BACKEND_ID, dockerClientLogType)).thenReturn(logStream); + if (backend == Backend.SWARM) { + setUpForSwarmLogTest(logContents); + } else if (backend == Backend.DOCKER) { + setUpForContainerLogTest(logContents); } else { - when(mockDockerClient.logs(BACKEND_ID, dockerClientLogType)).thenReturn(logStream); - when(mockDockerClient.serviceLogs(any(String.class), (DockerClient.LogsParam) anyVararg())) - .thenThrow(new RuntimeException("Should not be called")); + // No need to mock anything for Kubernetes } // Run the test final String log = dockerControlApi.getLog(container, logType); // Check results - assertThat(log, is(equalTo(LOG_CONTENTS))); - } - - @Test - public void testGetLog_stderr_kubernetes() throws Exception { - assumeThat("Special stderr handling for kubernetes", backend, is(Backend.KUBERNETES)); - - // No need to mock anything - - // Run the test - final String log = dockerControlApi.getLog(container, LogType.STDERR); - - // Check results - assertThat(log, is(nullValue())); - } - - @Test - public void testGetLog_stdout_params() throws Exception { - - final LogType logType = LogType.STDOUT; - final DockerClient.LogsParam dockerClientLogType = DockerClient.LogsParam.stdout(); - - final boolean withTimestamp = false; - final DockerClient.LogsParam timestampParam = DockerClient.LogsParam.timestamps(withTimestamp); - final OffsetDateTime since = OffsetDateTime.now(); - final Integer sinceEpoch = Math.toIntExact(since.toEpochSecond()); - final DockerClient.LogsParam sinceParam = DockerClient.LogsParam.since(sinceEpoch); - - // Set up test-specific mocks - if (backend == Backend.SWARM) { - when(mockDockerClient.logs(any(String.class), (DockerClient.LogsParam) anyVararg())) - .thenThrow(new RuntimeException("Should not be called")); - when(mockDockerClient.serviceLogs(BACKEND_ID, dockerClientLogType, timestampParam, sinceParam)) - .thenReturn(logStream); - } else if (backend == Backend.KUBERNETES) { - when(container.podName()).thenReturn(BACKEND_ID); - when(kubernetesClient.getLog( - eq(BACKEND_ID), eq(logType), eq(withTimestamp), any(OffsetDateTime.class) - )).thenReturn(LOG_CONTENTS); + if (backend == Backend.KUBERNETES) { + assertThat(log, is(nullValue())); } else { - when(mockDockerClient.logs(BACKEND_ID, dockerClientLogType, timestampParam, sinceParam)) - .thenReturn(logStream); - when(mockDockerClient.serviceLogs(any(String.class), (DockerClient.LogsParam) anyVararg())) - .thenThrow(new RuntimeException("Should not be called")); + assertThat(log, is(equalTo(logContents))); } - - // Run the test - final String log = dockerControlApi.getLog(container, logType, withTimestamp, since); - - // Check results - assertThat(log, is(equalTo(LOG_CONTENTS))); } - @Test - public void testGetLog_stderr_params() throws Exception { - assumeThat("Special stderr handling for kubernetes", backend, not(Backend.KUBERNETES)); + private void setUpForSwarmLogTest(final String logContents) { + final LogSwarmObjectCmd cmd = Mockito.mock(LogSwarmObjectCmd.class, RETURN_SELF); + when(mockDockerJavaClient.logServiceCmd(BACKEND_ID)).thenReturn(cmd); + final DockerControlApi.GetLogCallback callback = Mockito.mock(DockerControlApi.GetLogCallback.class); + Mockito.doReturn(callback).when(cmd).exec(any(DockerControlApi.GetLogCallback.class)); - final LogType logType = LogType.STDERR; - final DockerClient.LogsParam dockerClientLogType = DockerClient.LogsParam.stderr(); + when(callback.getLog()).thenReturn(logContents); - final boolean withTimestamp = false; - final DockerClient.LogsParam timestampParam = DockerClient.LogsParam.timestamps(withTimestamp); - final OffsetDateTime since = OffsetDateTime.now(); - final Integer sinceEpoch = Math.toIntExact(since.toEpochSecond()); - final DockerClient.LogsParam sinceParam = DockerClient.LogsParam.since(sinceEpoch); + when(mockDockerJavaClient.logContainerCmd(any(String.class))) + .thenThrow(new RuntimeException("Should not be called")); + } - // Set up test-specific mocks - if (swarmMode) { - when(mockDockerClient.logs(any(String.class), (DockerClient.LogsParam) anyVararg())) - .thenThrow(new RuntimeException("Should not be called")); - when(mockDockerClient.serviceLogs(BACKEND_ID, dockerClientLogType, timestampParam, sinceParam)) - .thenReturn(logStream); - } else { - when(mockDockerClient.logs(BACKEND_ID, dockerClientLogType, timestampParam, sinceParam)) - .thenReturn(logStream); - when(mockDockerClient.serviceLogs(any(String.class), (DockerClient.LogsParam) anyVararg())) - .thenThrow(new RuntimeException("Should not be called")); - } + private void setUpForContainerLogTest(final String logContents) { + final LogContainerCmd cmd = Mockito.mock(LogContainerCmd.class, RETURN_SELF); + when(mockDockerJavaClient.logContainerCmd(BACKEND_ID)).thenReturn(cmd); + final DockerControlApi.GetLogCallback callback = Mockito.mock(DockerControlApi.GetLogCallback.class); + Mockito.doReturn(callback).when(cmd).exec(any(DockerControlApi.GetLogCallback.class)); - // Run the test - final String log = dockerControlApi.getLog(container, logType, withTimestamp, since); + when(callback.getLog()).thenReturn(logContents); - // Check results - assertThat(log, is(equalTo(LOG_CONTENTS))); + when(mockDockerJavaClient.logServiceCmd(any(String.class))) + .thenThrow(new RuntimeException("Should not be called")); } @Test @@ -314,12 +257,13 @@ public void testCreate_container() throws Exception { // Make test objects final ThreadLocalRandom random = ThreadLocalRandom.current(); + final String dockerImage = "test image name"; final Container.Builder toLaunchAndExpectedContainerBuilder = Container.builder() .databaseId(random.nextInt()) .commandId(random.nextInt()) .wrapperId(random.nextInt()) .userId(USER_LOGIN) - .dockerImage("image") + .dockerImage(dockerImage) .commandLine("echo foo") .backend(backend) .addEnvironmentVariable("key", "value") @@ -330,14 +274,12 @@ public void testCreate_container() throws Exception { if (backend == Backend.SWARM) { toLaunchAndExpectedContainerBuilder.serviceId(BACKEND_ID); - // Have to mock out the response from docker-client. - // I would just make a real ServiceCreateResponse object, - // but it doesn't have a build() method to return a Builder, and the - // implementation is package-private. - final ServiceCreateResponse serviceCreateResponse = Mockito.mock(ServiceCreateResponse.class); - when(serviceCreateResponse.id()).thenReturn(BACKEND_ID); - when(mockDockerClient.createService(any(ServiceSpec.class), any(RegistryAuth.class))) - .thenReturn(serviceCreateResponse); + final CreateServiceResponse resp = Mockito.mock(CreateServiceResponse.class); + Mockito.when(resp.getId()).thenReturn(BACKEND_ID); + + final CreateServiceCmd cmd = Mockito.mock(CreateServiceCmd.class, RETURN_SELF); + Mockito.doReturn(resp).when(cmd).exec(); + Mockito.when(mockDockerJavaClient.createServiceCmd(Mockito.any(ServiceSpec.class))).thenReturn(cmd); } else if (backend == Backend.KUBERNETES) { toLaunchAndExpectedContainerBuilder.serviceId(BACKEND_ID); @@ -347,8 +289,19 @@ public void testCreate_container() throws Exception { } else { toLaunchAndExpectedContainerBuilder.containerId(BACKEND_ID); - when(mockDockerClient.createContainer(any(ContainerConfig.class))) - .thenReturn(ContainerCreation.builder().id(BACKEND_ID).build()); + final CreateContainerResponse resp = new CreateContainerResponse(); + resp.setId(BACKEND_ID); + resp.setWarnings(new String[0]); + + final CreateContainerCmd cmd = Mockito.mock(CreateContainerCmd.class, RETURN_SELF); + Mockito.doReturn(resp).when(cmd).exec(); + Mockito.when(mockDockerJavaClient.createContainerCmd(dockerImage)).thenReturn(cmd); + + // We also try to pull the image + Mockito.when(mockDockerImage.tags()).thenReturn(ImmutableList.of(dockerImage)); + PowerMockito.doReturn(Collections.singletonList(mockDockerImage)) + .when(dockerControlApi, method(DockerControlApi.class, "getAllImages", DockerServer.class)) + .withArguments(dockerServer); } final Container expected = toLaunchAndExpectedContainerBuilder.build(); @@ -364,12 +317,13 @@ public void testCreate_resolvedCommand() throws Exception { // Make test objects final ThreadLocalRandom random = ThreadLocalRandom.current(); + final String dockerImage = "test image name"; final ResolvedCommand resolvedCommandToLaunch = ResolvedCommand.builder() .commandId(random.nextLong()) .commandName("a command name") .wrapperId(random.nextLong()) .wrapperName("a wrapper name") - .image("test image name") + .image(dockerImage) .commandLine("echo foo bar baz") .mounts(Collections.emptyList()) .environmentVariables(Collections.emptyMap()) @@ -381,14 +335,12 @@ public void testCreate_resolvedCommand() throws Exception { .backend(backend); if (backend == Backend.SWARM) { - // Have to mock out the response from docker-client. - // I would just make a real ServiceCreateResponse object, - // but it doesn't have a build() method to return a Builder, and the - // implementation is package-private. - final ServiceCreateResponse serviceCreateResponse = Mockito.mock(ServiceCreateResponse.class); - when(serviceCreateResponse.id()).thenReturn(BACKEND_ID); - when(mockDockerClient.createService(any(ServiceSpec.class), any(RegistryAuth.class))) - .thenReturn(serviceCreateResponse); + final CreateServiceResponse resp = Mockito.mock(CreateServiceResponse.class); + Mockito.when(resp.getId()).thenReturn(BACKEND_ID); + + final CreateServiceCmd cmd = Mockito.mock(CreateServiceCmd.class, RETURN_SELF); + Mockito.doReturn(resp).when(cmd).exec(); + Mockito.when(mockDockerJavaClient.createServiceCmd(Mockito.any(ServiceSpec.class))).thenReturn(cmd); expectedCreatedBuilder.serviceId(BACKEND_ID); } else if (backend == Backend.KUBERNETES) { @@ -398,8 +350,19 @@ public void testCreate_resolvedCommand() throws Exception { expectedCreatedBuilder.serviceId(BACKEND_ID); } else { - when(mockDockerClient.createContainer(any(ContainerConfig.class))) - .thenReturn(ContainerCreation.builder().id(BACKEND_ID).build()); + final CreateContainerResponse resp = new CreateContainerResponse(); + resp.setId(BACKEND_ID); + resp.setWarnings(new String[0]); + + final CreateContainerCmd cmd = Mockito.mock(CreateContainerCmd.class, RETURN_SELF); + Mockito.doReturn(resp).when(cmd).exec(); + Mockito.when(mockDockerJavaClient.createContainerCmd(dockerImage)).thenReturn(cmd); + + // We also try to pull the image + Mockito.when(mockDockerImage.tags()).thenReturn(ImmutableList.of(dockerImage)); + PowerMockito.doReturn(Collections.singletonList(mockDockerImage)) + .when(dockerControlApi, method(DockerControlApi.class, "getAllImages", DockerServer.class)) + .withArguments(dockerServer); expectedCreatedBuilder.containerId(BACKEND_ID); } @@ -440,31 +403,22 @@ public void invokeCreateDockerSwarmServicePrivateMethod(final DockerControlApi.N .build(); // Mocks - // Real implementation of ServiceCreateResponse is package-private - final ServiceCreateResponse serviceCreateResponse = new ServiceCreateResponse() { - @Override - public String id() { - return BACKEND_ID; - } - - @Override - public List warnings() { - return Collections.emptyList(); - } - }; - when(mockDockerClient.createService(any(ServiceSpec.class), any(RegistryAuth.class))) - .thenReturn(serviceCreateResponse); + final CreateServiceResponse resp = Mockito.mock(CreateServiceResponse.class); + Mockito.when(resp.getId()).thenReturn(BACKEND_ID); + + final CreateServiceCmd cmd = Mockito.mock(CreateServiceCmd.class, RETURN_SELF); + Mockito.doReturn(resp).when(cmd).exec(); + Mockito.when(mockDockerJavaClient.createServiceCmd(Mockito.any(ServiceSpec.class))).thenReturn(cmd); // Run the method Whitebox.invokeMethod(dockerControlApi, "createDockerSwarmService", toCreate, dockerServer, numReplicas); // Assert on results final ArgumentCaptor serviceSpecCaptor = ArgumentCaptor.forClass(ServiceSpec.class); - final ArgumentCaptor registryAuthArgumentCaptor = ArgumentCaptor.forClass(RegistryAuth.class); - Mockito.verify(mockDockerClient).createService(serviceSpecCaptor.capture(), registryAuthArgumentCaptor.capture()); + Mockito.verify(mockDockerJavaClient).createServiceCmd(serviceSpecCaptor.capture()); final ServiceSpec serviceSpec = serviceSpecCaptor.getValue(); - assertThat(serviceSpec.mode().replicated().replicas(), equalTo(numReplicas.value)); + assertThat(serviceSpec.getMode().getReplicated().getReplicas(), equalTo(new Integer(numReplicas.value).longValue())); } @Test @@ -492,47 +446,33 @@ private void testStart_swarmMode() throws Exception { // We can also change the replicas and verify that the code under test made // that same change. - // First some garbage test data - final ThreadLocalRandom random = ThreadLocalRandom.current(); - final Long serviceVersion = random.nextLong(); - final String specName = "whatever"; - final ContainerSpec containerSpec = ContainerSpec.builder() - .image("im") - .env(Collections.emptyList()) - .dir("/a/dir") - .user(user.getLogin()) - .command("echo", "foo") - .build(); - final TaskSpec taskSpec = TaskSpec.builder().containerSpec(containerSpec).build(); - final ServiceSpec.Builder commonSpec = ServiceSpec.builder() - .taskTemplate(taskSpec) - .name(specName); - - // This is the mock spec that will pass through a chain of mocks into our code under test - final ServiceSpec createdSpec = commonSpec - .mode(ServiceMode.builder() - .replicated(ReplicatedService.builder().replicas(0L).build()) - .build()) - .build(); - final Version version = Mockito.mock(Version.class); - when(version.index()).thenReturn(serviceVersion); - final Service created = Mockito.mock(Service.class); - when(created.spec()).thenReturn(createdSpec); - when(created.version()).thenReturn(version); - when(mockDockerClient.inspectService(BACKEND_ID)).thenReturn(created); - - // This is the spec that we expect the code under test will create and pass to the client to update - final ServiceSpec expectedUpdateSpec = commonSpec - .mode(ServiceMode.builder() - .replicated(ReplicatedService.builder().replicas(1L).build()) - .build()) - .build(); + final ServiceSpec serviceSpec = Mockito.mock(ServiceSpec.class); + final ServiceSpec updatedSpec = Mockito.mock(ServiceSpec.class); + final Service service = Mockito.mock(Service.class); + final InspectServiceCmd inspectServiceCmd = Mockito.mock(InspectServiceCmd.class, Mockito.RETURNS_DEEP_STUBS); + final UpdateServiceCmd updateServiceCmd = Mockito.mock(UpdateServiceCmd.class, Mockito.RETURNS_DEEP_STUBS); + when(mockDockerJavaClient.inspectServiceCmd(BACKEND_ID)).thenReturn(inspectServiceCmd); + when(mockDockerJavaClient.updateServiceCmd(BACKEND_ID, updatedSpec)).thenReturn(updateServiceCmd); + when(inspectServiceCmd.exec()).thenReturn(service); + when(service.getSpec()).thenReturn(serviceSpec); + when(serviceSpec.withMode(any(ServiceModeConfig.class))).thenReturn(updatedSpec); // Run the test dockerControlApi.start(container); - // Verify that the client method was called as expected - verify(mockDockerClient).updateService(BACKEND_ID, serviceVersion, expectedUpdateSpec); + // Verify that the service inspect API was called + verify(mockDockerJavaClient).inspectServiceCmd(BACKEND_ID); + + // Verify that the service was updated to have 1 replica + final ArgumentCaptor modeCaptor = ArgumentCaptor.forClass(ServiceModeConfig.class); + verify(serviceSpec).withMode(modeCaptor.capture()); + final ServiceModeConfig mode = modeCaptor.getValue(); + assertThat(mode.getReplicated(), notNullValue()); + assertThat(mode.getReplicated().getReplicas(), equalTo(1L)); + + // Verify that the service update API was called + verify(mockDockerJavaClient).updateServiceCmd(BACKEND_ID, updatedSpec); + verify(updateServiceCmd).withVersion(any(Long.class)); } private void testStart_kubernetes() throws Exception { @@ -549,7 +489,7 @@ private void testStart_localDocker() throws Exception { dockerControlApi.start(container); // Verify that the client method was called as expected - verify(mockDockerClient).startContainer(BACKEND_ID); + verify(mockDockerJavaClient).startContainerCmd(BACKEND_ID); } @Test @@ -560,10 +500,10 @@ public void testKill() throws Exception { // Verify the client method was called as expected switch (backend) { case DOCKER: - verify(mockDockerClient).killContainer(BACKEND_ID); + verify(mockDockerJavaClient).killContainerCmd(BACKEND_ID); break; case SWARM: - verify(mockDockerClient).removeService(BACKEND_ID); + verify(mockDockerJavaClient).removeServiceCmd(BACKEND_ID); break; case KUBERNETES: verify(kubernetesClient).removeJob(BACKEND_ID); @@ -581,10 +521,9 @@ public void testAutoCleanup_autoCleanupFalse() throws Exception { // Verify the client method was called as expected // In this case we expect nothing will happen - verify(mockDockerClient, times(0)).removeService(BACKEND_ID); - verify(mockDockerClient, times(0)).removeContainer(BACKEND_ID); - verify(kubernetesClient, times(0)) - .removeJob(BACKEND_ID); + verify(mockDockerJavaClient, times(0)).removeServiceCmd(BACKEND_ID); + verify(mockDockerJavaClient, times(0)).removeContainerCmd(BACKEND_ID); + verify(kubernetesClient, times(0)).removeJob(BACKEND_ID); } @Test @@ -598,10 +537,10 @@ public void testAutoCleanup_autoCleanupTrue() throws Exception { // Verify the client method was called as expected switch (backend) { case DOCKER: - verify(mockDockerClient).removeContainer(BACKEND_ID); + verify(mockDockerJavaClient).removeContainerCmd(BACKEND_ID); break; case SWARM: - verify(mockDockerClient).removeService(BACKEND_ID); + verify(mockDockerJavaClient).removeServiceCmd(BACKEND_ID); break; case KUBERNETES: verify(kubernetesClient).removeJob(BACKEND_ID); @@ -617,10 +556,10 @@ public void testRemove() throws Exception { // Verify the client method was called as expected switch (backend) { case DOCKER: - verify(mockDockerClient).removeContainer(BACKEND_ID); + verify(mockDockerJavaClient).removeContainerCmd(BACKEND_ID); break; case SWARM: - verify(mockDockerClient).removeService(BACKEND_ID); + verify(mockDockerJavaClient).removeServiceCmd(BACKEND_ID); break; case KUBERNETES: verify(kubernetesClient).removeJob(BACKEND_ID); diff --git a/src/test/java/org/nrg/containers/jms/JmsExceptionIntegrationTest.java b/src/test/java/org/nrg/containers/jms/JmsExceptionIntegrationTest.java index 56c0ea4a..57ba7e69 100644 --- a/src/test/java/org/nrg/containers/jms/JmsExceptionIntegrationTest.java +++ b/src/test/java/org/nrg/containers/jms/JmsExceptionIntegrationTest.java @@ -1,31 +1,22 @@ package org.nrg.containers.jms; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.SystemUtils; -import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; -import org.mandas.docker.client.DockerClient; import org.mockito.Matchers; import org.mockito.Mockito; import org.nrg.containers.api.DockerControlApi; import org.nrg.containers.config.EventPullingIntegrationTestConfig; -import org.nrg.containers.jms.requests.ContainerFinalizingRequest; import org.nrg.containers.jms.requests.ContainerStagingRequest; import org.nrg.containers.model.command.auto.Command; -import org.nrg.containers.model.container.auto.Container; -import org.nrg.containers.model.server.docker.Backend; -import org.nrg.containers.model.server.docker.DockerServerBase; import org.nrg.containers.model.xnat.FakeWorkflow; import org.nrg.containers.services.CommandService; import org.nrg.containers.services.ContainerService; import org.nrg.containers.services.DockerServerService; -import org.nrg.containers.utils.BackendConfig; import org.nrg.containers.utils.TestingUtils; import org.nrg.mail.services.MailService; import org.nrg.xdat.entities.AliasToken; @@ -63,20 +54,16 @@ import java.io.File; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.List; -import static org.awaitility.Awaitility.await; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; -import static org.junit.Assume.assumeThat; import static org.mockito.AdditionalMatchers.aryEq; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyMapOf; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.isNull; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; import static org.powermock.api.mockito.PowerMockito.mockStatic; @@ -194,27 +181,6 @@ public void setup() throws Exception { fakeWorkflow.setPipelineName(wrapper.name()); } - @After - public void cleanup() { - for (final String containerToCleanUp : containersToCleanUp) { - try { - controlApi.getDockerClient().removeContainer(containerToCleanUp, DockerClient.RemoveContainerParam.forceKill()); - } catch (Exception e) { - // do nothing - } - } - containersToCleanUp.clear(); - - for (final String imageToCleanUp : imagesToCleanUp) { - try { - controlApi.getDockerClient().removeImage(imageToCleanUp, true, false); - } catch (Exception e) { - // do nothing - } - } - imagesToCleanUp.clear(); - } - @Test @DirtiesContext public void testStagingQueueFailure() throws Exception { @@ -236,58 +202,4 @@ public void testStagingQueueFailure() throws Exception { Mockito.matches(".*" + wrapper.name() + ".*" + FAKE_ID + ".*failed.*"), anyMapOf(String.class, File.class)); } - - @Test - @DirtiesContext - @Ignore - public void testFinalizingQueueFailure() throws Exception { - setupServer(); - - // setup jmsTemplate to throw exception - String exceptionMsg = "exception"; - Mockito.doThrow(new JMSRuntimeException(exceptionMsg)).when(mockJmsTemplate) - .convertAndSend(eq(containerFinalizingRequest), any(ContainerFinalizingRequest.class), any(MessagePostProcessor.class)); - - containerService.queueResolveCommandAndLaunchContainer(null, wrapper.id(), 0L, - null, Collections.emptyMap(), mockUser, fakeWorkflow); - final Container container = TestingUtils.getContainerFromWorkflow(containerService, fakeWorkflow); - containersToCleanUp.add(container.containerId()); - - TestingUtils.commitTransaction(); - - log.debug("Waiting until task has started"); - await().until(TestingUtils.containerHasStarted(controlApi.getDockerClient(), false, container), is(true)); - log.debug("Waiting until task has finished"); - await().until(TestingUtils.containerIsRunning(controlApi.getDockerClient(), false, container), is(false)); - log.debug("Waiting until container has failed"); - await().until(TestingUtils.containerIsFinalized(containerService, container), is(true)); - - assertThat(fakeWorkflow.getStatus(), is(PersistentWorkflowUtils.FAILED + " (JMS)")); - assertThat(fakeWorkflow.getDetails(), is(exceptionMsg)); - - Mockito.verify(mockMailService, timeout(1000).times(1)).sendHtmlMessage(eq(FAKE_EMAIL), - aryEq(new String[]{FAKE_EMAIL}), aryEq(new String[]{FAKE_EMAIL}), Matchers.eq(null), - Mockito.matches(".*" + wrapper.name() + ".*Failed.*"), - Mockito.matches(".*" + wrapper.name() + ".*" + FAKE_ID + ".*failed.*"), - Mockito.matches(".*" + wrapper.name() + ".*" + FAKE_ID + ".*failed.*"), - anyMapOf(String.class, File.class)); - } - - private void setupServer() throws Exception { - final BackendConfig backendConfig = TestingUtils.getBackendConfig(); - dockerServerService.setServer(DockerServerBase.DockerServer.builder() - .name("Test server") - .host(backendConfig.getContainerHost()) - .certPath(backendConfig.getCertPath()) - .backend(Backend.DOCKER) - .lastEventCheckTime(new Date()) - .build()); - - String img = "busybox:latest"; - controlApi.getDockerClient().pull(img); - imagesToCleanUp.add(img); - - assumeThat(SystemUtils.IS_OS_WINDOWS_7, is(false)); - TestingUtils.skipIfCannotConnectToDocker(controlApi.getDockerClient()); - } } diff --git a/src/test/java/org/nrg/containers/secrets/SecretsTest.java b/src/test/java/org/nrg/containers/secrets/SecretsTest.java index bcec5b9a..8f31c678 100644 --- a/src/test/java/org/nrg/containers/secrets/SecretsTest.java +++ b/src/test/java/org/nrg/containers/secrets/SecretsTest.java @@ -3,6 +3,12 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.command.CreateContainerResponse; +import com.github.dockerjava.api.command.CreateServiceCmd; +import com.github.dockerjava.api.command.CreateServiceResponse; +import com.github.dockerjava.api.model.ServiceSpec; import io.kubernetes.client.openapi.apis.BatchV1Api; import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1EnvVar; @@ -20,16 +26,11 @@ import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runner.RunWith; -import org.mandas.docker.client.DockerClient; -import org.mandas.docker.client.messages.ContainerConfig; -import org.mandas.docker.client.messages.ContainerCreation; -import org.mandas.docker.client.messages.RegistryAuth; -import org.mandas.docker.client.messages.ServiceCreateResponse; -import org.mandas.docker.client.messages.swarm.ServiceSpec; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.internal.util.reflection.Whitebox; +import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.nrg.containers.api.DockerControlApi; import org.nrg.containers.api.KubernetesClientImpl; @@ -79,6 +80,8 @@ @RunWith(PowerMockRunner.class) @PrepareOnlyThisForTest(DockerControlApi.class) public class SecretsTest { + private final static Answer RETURN_SELF = InvocationOnMock::getMock; + @Mock private SystemPropertySecretSource.ValueObtainer valueObtainer; @Mock private CommandService commandService; @Mock private DockerServerService dockerServerService; @@ -322,18 +325,12 @@ public void testLaunchSwarmJobWithEnvSecret() throws Exception { // Mock out backend call to create service final String containerId = RandomStringUtils.randomAlphanumeric(5); - final ServiceCreateResponse serviceCreateResponse = new ServiceCreateResponse() { - @Override - public String id() { - return containerId; - } - - @Override - public List warnings() { - return Collections.emptyList(); - } - }; - Mockito.when(dockerClient.createService(Mockito.any(ServiceSpec.class), Mockito.any(RegistryAuth.class))).thenReturn(serviceCreateResponse); + final CreateServiceResponse resp = Mockito.mock(CreateServiceResponse.class); + Mockito.when(resp.getId()).thenReturn(containerId); + + final CreateServiceCmd cmd = Mockito.mock(CreateServiceCmd.class, RETURN_SELF); + Mockito.doReturn(resp).when(cmd).exec(); + Mockito.when(dockerClient.createServiceCmd(Mockito.any(ServiceSpec.class))).thenReturn(cmd); // Call method under test PowerMockito.doCallRealMethod().when(dockerControlApi, "createDockerSwarmService", toCreate, dockerServer, DockerControlApi.NumReplicas.ZERO); @@ -342,12 +339,11 @@ public List warnings() { // Capture call to backend api mock final ArgumentCaptor serviceSpecArgumentCaptor = ArgumentCaptor.forClass(ServiceSpec.class); - final ArgumentCaptor registryAuthArgumentCaptor = ArgumentCaptor.forClass(RegistryAuth.class); - Mockito.verify(dockerClient).createService(serviceSpecArgumentCaptor.capture(), registryAuthArgumentCaptor.capture()); + Mockito.verify(dockerClient).createServiceCmd(serviceSpecArgumentCaptor.capture()); final ServiceSpec serviceSpec = serviceSpecArgumentCaptor.getValue(); // Check for secret env value - assertThat(serviceSpec.taskTemplate().containerSpec().env(), + assertThat(serviceSpec.getTaskTemplate().getContainerSpec().getEnv(), contains(secretEnvironmentVariableName + "=" + secretValue)); } @@ -390,18 +386,16 @@ public void testLaunchDockerContainerWithEnvSecret() throws Exception { // Mock out backend call to create service final String containerId = RandomStringUtils.randomAlphanumeric(5); - final ContainerCreation containerCreation = new ContainerCreation() { - @Override - public String id() { - return containerId; - } - - @Override - public List warnings() { - return Collections.emptyList(); - } - }; - Mockito.when(dockerClient.createContainer(Mockito.any(ContainerConfig.class))).thenReturn(containerCreation); + final CreateContainerResponse resp = new CreateContainerResponse(); + resp.setId(containerId); + resp.setWarnings(new String[0]); + + // This default answer lets us stub out all the method chaining .withFoo().withBar() without having to stub each one + // Have to make it explicitly because Mockito.RETURNS_SELF is not available in our version + @SuppressWarnings("rawtypes") + final CreateContainerCmd cmd = Mockito.mock(CreateContainerCmd.class, (Answer) InvocationOnMock::getMock); + Mockito.doReturn(resp).when(cmd).exec(); + Mockito.when(dockerClient.createContainerCmd(dockerImage)).thenReturn(cmd); // Call method under test PowerMockito.doCallRealMethod().when(dockerControlApi, "createDockerContainer", toCreate, dockerServer); @@ -409,12 +403,13 @@ public List warnings() { dockerControlApi.create(toCreate, user); // Capture call to backend api mock - final ArgumentCaptor containerConfigArgumentCaptor = ArgumentCaptor.forClass(ContainerConfig.class); - Mockito.verify(dockerClient).createContainer(containerConfigArgumentCaptor.capture()); - final ContainerConfig containerConfig = containerConfigArgumentCaptor.getValue(); + @SuppressWarnings({"unchecked", "rawtypes"}) + final ArgumentCaptor> withEnvArgumentCaptor = ArgumentCaptor.forClass((Class) List.class); + Mockito.verify(cmd).withEnv(withEnvArgumentCaptor.capture()); + final List env = withEnvArgumentCaptor.getValue(); // Check for secret env value - assertThat(containerConfig.env(), contains(secretEnvironmentVariableName + "=" + secretValue)); + assertThat(env, contains(secretEnvironmentVariableName + "=" + secretValue)); } @Test diff --git a/src/test/java/org/nrg/containers/services/DockerServiceIntegrationTest.java b/src/test/java/org/nrg/containers/services/DockerServiceIntegrationTest.java index 15e4eecd..f7aee948 100644 --- a/src/test/java/org/nrg/containers/services/DockerServiceIntegrationTest.java +++ b/src/test/java/org/nrg/containers/services/DockerServiceIntegrationTest.java @@ -26,7 +26,9 @@ import org.springframework.transaction.annotation.Transactional; import java.io.File; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collections; import java.util.List; import static org.hamcrest.Matchers.greaterThan; @@ -92,8 +94,13 @@ public void setup() throws Exception { final String imageName = "xnat/testy-test:tag"; final String dir = Paths.get(ClassLoader.getSystemResource("dockerServiceIntegrationTest").toURI()).toString().replace("%20", " "); + final Path dockerfilePath = Paths.get(dir, "Dockerfile"); - imageId = controlApi.getDockerClient().build(Paths.get(dir), imageName); + imageId = controlApi.getDockerClient().buildImageCmd() + .withDockerfile(dockerfilePath.toFile()) + .withTags(Collections.singleton(imageName)) + .start() + .awaitImageId(); } @Test @@ -110,7 +117,7 @@ public void testSaveCommandFromImageLabels() throws Exception { final Command.CommandWrapper wrapper = wrappers.get(0); assertThat(wrapper.id(), not(0L)); - controlApi.getDockerClient().removeImage(IMAGE_NAME); + TestingUtils.cleanDockerImages(controlApi.getDockerClient(), Collections.singletonList(imageId)); } @Test diff --git a/src/test/java/org/nrg/containers/utils/LoggingBuildCallback.java b/src/test/java/org/nrg/containers/utils/LoggingBuildCallback.java new file mode 100644 index 00000000..f7f443a4 --- /dev/null +++ b/src/test/java/org/nrg/containers/utils/LoggingBuildCallback.java @@ -0,0 +1,14 @@ +package org.nrg.containers.utils; + +import com.github.dockerjava.api.command.BuildImageResultCallback; +import com.github.dockerjava.api.model.BuildResponseItem; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LoggingBuildCallback extends BuildImageResultCallback { + @Override + public void onNext(BuildResponseItem item) { + log.debug("{}", item); + super.onNext(item); + } +} diff --git a/src/test/java/org/nrg/containers/utils/TestingUtils.java b/src/test/java/org/nrg/containers/utils/TestingUtils.java index 7e363f6b..9dd1473c 100644 --- a/src/test/java/org/nrg/containers/utils/TestingUtils.java +++ b/src/test/java/org/nrg/containers/utils/TestingUtils.java @@ -1,6 +1,11 @@ package org.nrg.containers.utils; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.PingCmd; +import com.github.dockerjava.api.exception.NotFoundException; +import com.github.dockerjava.api.model.Task; +import com.github.dockerjava.api.model.TaskState; import com.google.common.collect.Maps; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; @@ -15,15 +20,6 @@ import org.hamcrest.CustomTypeSafeMatcher; import org.hamcrest.Matchers; import org.junit.rules.TemporaryFolder; -import org.mandas.docker.client.DockerClient; -import org.mandas.docker.client.exceptions.DockerRequestException; -import org.mandas.docker.client.exceptions.ImageNotFoundException; -import org.mandas.docker.client.exceptions.NotFoundException; -import org.mandas.docker.client.exceptions.ServiceNotFoundException; -import org.mandas.docker.client.messages.ContainerInfo; -import org.mandas.docker.client.messages.swarm.Service; -import org.mandas.docker.client.messages.swarm.Task; -import org.mandas.docker.client.messages.swarm.TaskStatus; import org.mockito.ArgumentMatcher; import org.nrg.containers.api.KubernetesClient; import org.nrg.containers.api.KubernetesClientImpl; @@ -177,8 +173,10 @@ public static boolean canConnectToDocker(DockerClient client) { return false; } - try { - return client.ping().equals("OK"); + try (final PingCmd cmd = client.pingCmd()) { + cmd.exec(); + // No exception means we can connect + return true; } catch (Exception ignored) { // Exceptions mean we cannot connect return false; @@ -187,8 +185,9 @@ public static boolean canConnectToDocker(DockerClient client) { public static void skipIfCannotConnectToDocker(DockerClient client) { // If we aren't running integration tests, we have already skipped this test - skipIfNotRunningIntegrationTests(); - assumeTrue("Cannot connect to docker", canConnectToDocker(client)); + if (RUNNING_INTEGRATION_TESTS) { + assumeTrue("Cannot connect to docker", canConnectToDocker(client)); + } } public static void skipIfNotRunningIntegrationTests() { @@ -201,7 +200,7 @@ public static boolean canConnectToSwarm(DockerClient client) { } try { - client.inspectSwarm(); + client.inspectSwarmCmd().exec(); // if we got here without an exception we're good return true; @@ -271,7 +270,7 @@ public static boolean cleanupKubernetesNamespace(final String namespaceName, fin private static void dockerCleanup(DockerClient dockerClient, String containerId) { try { log.debug("Cleaning up container {}", containerId); - dockerClient.removeContainer(containerId, DockerClient.RemoveContainerParam.forceKill()); + dockerClient.removeContainerCmd(containerId).withForce(true).exec(); } catch (NotFoundException ignored) { // ignored } catch (Exception e) { @@ -282,8 +281,8 @@ private static void dockerCleanup(DockerClient dockerClient, String containerId) private static void dockerSwarmCleanup(DockerClient dockerClient, String serviceId) { try { log.debug("Cleaning up service {}", serviceId); - dockerClient.removeService(serviceId); - } catch (ServiceNotFoundException ignored) { + dockerClient.removeServiceCmd(serviceId).exec(); + } catch (NotFoundException ignored) { // ignored } catch (Exception e) { log.error("Could not clean up service {}", serviceId, e); @@ -315,6 +314,42 @@ public static Consumer cleanupFunction(Backend backend, DockerClient doc return null; } + public static void cleanDockerImages(final DockerClient client, List images) { + images.forEach(image -> { + try { + client.removeImageCmd(image).withForce(true).withNoPrune(false).exec(); + } catch (NotFoundException ignored) { + // ignored + } catch (Exception e) { + log.error("Could not clean up image {}", image, e); + } + }); + } + + public static void cleanDockerContainers(final DockerClient client, Collection containers) { + containers.forEach(container -> { + try { + client.removeContainerCmd(container).withForce(true).exec(); + } catch (NotFoundException ignored) { + // ignored + } catch (Exception e) { + log.error("Could not clean up container {}", container, e); + } + }); + } + + public static void cleanSwarmServices(final DockerClient client, Collection services) { + services.forEach(service -> { + try { + client.removeServiceCmd(service).exec(); + } catch (NotFoundException ignored) { + // ignored + } catch (Exception e) { + log.error("Could not clean up service {}", service, e); + } + }); + } + @SuppressWarnings("unchecked") public static ArgumentMatcher> isMapWithEntry(final String key, final String value) { return new ArgumentMatcher>() { @@ -367,91 +402,32 @@ protected boolean matchesSafely(final File item) { }; } - public static Callable containerHasStarted(final DockerClient CLIENT, final boolean swarmMode, - final Container container) { - return () -> { - try { - if (swarmMode) { - String id = container.serviceId(); - if (id == null) { - log.debug("Service id null"); - return false; - } - final Service serviceResponse = CLIENT.inspectService(id); - final List tasks = CLIENT.listTasks(Task.Criteria.builder().serviceName(serviceResponse.spec().name()).build()); - if (tasks.size() == 0) { - log.debug("No tasks for service {}", id); - return false; - } - for (final Task task : tasks) { - final ServiceTask serviceTask = ServiceTask.create(task, id); - log.debug("Service {} task {} status {}", id, task.id(), task.status()); - if (!serviceTask.hasNotStarted()) { - // if it's not a "before running" status (aka running or some exit status) - log.debug("Service {} task {} has started!", id, task.id()); - return true; - } - } - log.debug("No tasks have started"); - return false; - } else { - String id = container.containerId(); - if (id == null) { - return false; - } - final ContainerInfo containerInfo = CLIENT.inspectContainer(id); - String status = containerInfo.state().status(); - return !"CREATED".equals(status); - } - } catch (NotFoundException ignored) { - // Ignore exception. If container is not found, it is not running. - return false; - } catch (DockerRequestException e) { - if (e.status() == 404) { - // Service tasks were not found. Try again later. - // This exception status checking is usually performed in docker client, - // but it isn't for listTasks - return false; - } - throw e; - } - }; - } - - public static Callable containerIsRunning(final DockerClient CLIENT, final boolean swarmMode, + public static Callable containerIsRunning(final DockerClient client, final boolean swarmMode, final Container container) { return () -> { try { if (swarmMode) { - final Service serviceResponse = CLIENT.inspectService(container.serviceId()); - final List tasks = CLIENT.listTasks(Task.Criteria.builder().serviceName(serviceResponse.spec().name()).build()); - for (final Task task : tasks) { - if (ServiceTask.isExitStatus(task.status().state())) { - return false; - } - } - return true; // consider it "running" until it's an exit status + final String serviceName = client.inspectServiceCmd(container.serviceId()) + .exec() + .getSpec() + .getName(); + return client.listTasksCmd() + .withServiceFilter(serviceName) + .exec() + .stream() + .noneMatch(task -> ServiceTask.isExitStatus(task.getStatus().getState().name())); } else { - final ContainerInfo containerInfo = CLIENT.inspectContainer(container.containerId()); - return containerInfo.state().running(); + return client.inspectContainerCmd(container.containerId()).exec().getState().getRunning(); } } catch (NotFoundException ignored) { // Ignore exception. If container is not found, it is not running. return false; - } catch (DockerRequestException e) { - if (e.status() == 404) { - // Service tasks were not found. Try again later. - // This exception status checking is usually performed in docker client, - // but it isn't for listTasks - return false; - } - throw e; } }; } - public static Callable serviceIsRunning(final DockerClient CLIENT, final Container container) { - return serviceIsRunning(CLIENT, container, false); + public static Callable serviceIsRunning(final DockerClient client, final Container container) { + return serviceIsRunning(client, container, false); } public static Callable serviceHasTaskId(final ContainerService containerService, final long containerDbId) { @@ -467,60 +443,29 @@ public static Callable serviceHasTaskId(final ContainerService containe }; } - public static Callable serviceIsRunning(final DockerClient CLIENT, final Container container, + public static Callable serviceIsRunning(final DockerClient client, final Container container, boolean rtnForNoServiceId) { - return () -> { - try { - String servicdId = container.serviceId(); - if (StringUtils.isBlank(servicdId)) { - // Want this to be the value we aren't waiting for - return rtnForNoServiceId; - } - final Service serviceResponse = CLIENT.inspectService(servicdId); - final List tasks = CLIENT.listTasks(Task.Criteria.builder().serviceName(serviceResponse.spec().name()).build()); - if (tasks.size() == 0) { - return false; - } - for (final Task task : tasks) { - if (task.status().state().equals(TaskStatus.TASK_STATE_RUNNING)) { - return true; - } - } - return false; - } catch (NotFoundException ignored) { - // Ignore exception. If container is not found, it is not running. - return false; - } catch (DockerRequestException e) { - if (e.status() == 404) { - // Service tasks were not found. Try again later. - // This exception status checking is usually performed in docker client, - // but it isn't for listTasks - return false; - } - throw e; - } - }; + return () -> StringUtils.isBlank(container.serviceId()) ? rtnForNoServiceId : getServiceNode(client, container) != null; } - public static Callable getServiceNode(final DockerClient CLIENT, final Container container) { - return () -> { - try { - final Service serviceResponse = CLIENT.inspectService(container.serviceId()); - final List tasks = CLIENT.listTasks(Task.Criteria.builder().serviceName(serviceResponse.spec().name()).build()); - if (tasks.size() == 0) { - return null; - } - for (final Task task : tasks) { - if (task.status().state().equals(TaskStatus.TASK_STATE_RUNNING)) { - return task.nodeId(); - } - } - return null; - } catch (Exception ignored) { - // Ignore exceptions - return null; - } - }; + public static String getServiceNode(final DockerClient client, final Container container) { + try { + final String serviceName = client.inspectServiceCmd(container.serviceId()) + .exec() + .getSpec() + .getName(); + return client.listTasksCmd() + .withServiceFilter(serviceName) + .exec() + .stream() + .filter(task -> task.getStatus().getState().equals(TaskState.RUNNING)) + .map(Task::getNodeId) + .findFirst() + .orElse(null); + } catch (Exception ignored) { + // Ignore exceptions + } + return null; } public static Container getContainerFromWorkflow(final ContainerService containerService, @@ -567,14 +512,6 @@ public static Callable containerIsFinalized(final ContainerService cont }; } - public static Callable containerHasLogPaths(final ContainerService containerService, - final long containerDbId) { - return () -> { - final Container container = containerService.get(containerDbId); - return container.logPaths().size() > 0; - }; - } - public static String setupSessionMock(TemporaryFolder folder, ObjectMapper mapper, Map runtimeValues) throws Exception { final Path wrapupCommandDirPath = Paths.get(ClassLoader.getSystemResource("wrapupCommand").toURI()); final String wrapupCommandDir = wrapupCommandDirPath.toString().replace("%20", " "); @@ -619,13 +556,5 @@ public static void setupMocksForSetupWrapupWorkflow(String uri, FakeWorkflow fak when(WorkflowUtils.getUniqueWorkflow(mockUser, setupWrapupWorkflow.getWorkflowId().toString())) .thenReturn(setupWrapupWorkflow); } - - public static void pullBusyBox(DockerClient client) throws Exception { - try { - client.inspectImage(BUSYBOX); - } catch (ImageNotFoundException e) { - client.pull(BUSYBOX); - } - } }