Skip to content

Commit

Permalink
Merged in rad-174-dev-29Jan19 (pull request NrgXnat#7)
Browse files Browse the repository at this point in the history 
Rad 174 dev 29Jan19
  • Loading branch information
RadiologicsA committed Jan 29, 2019
2 parents ef936f7 + 226568c commit 050e09e
Show file tree
Hide file tree
Showing 12 changed files with 326 additions and 233 deletions.
6 changes: 3 additions & 3 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ buildscript {
}

group 'org.nrg'
version '2.0.0.15-radiologics'
version '2.0.0.16-radiologics'

apply plugin: 'java'
apply plugin: 'idea'
Expand All @@ -42,8 +42,8 @@ apply plugin: "com.github.zhurlik.swagger"
sourceCompatibility = 1.7
targetCompatibility = 1.7

def vXnat = '1.7.5-SNAPSHOT'
def vXnatDev = '1.7.5-SNAPSHOT'
def vXnat = '1.7.5'
def vXnatDev = '1.7.5'
def vMockito = '1.10.19'
def vJavassist = '3.21.0-GA'
def vAwaitility = '2.0.0'
Expand Down
168 changes: 91 additions & 77 deletions src/main/java/org/nrg/containers/events/DockerStatusUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,13 @@
import org.nrg.containers.services.DockerServerService;
import org.nrg.containers.utils.ContainerUtils;
import org.nrg.framework.exceptions.NotFoundException;
import org.nrg.framework.task.XnatTask;
import org.nrg.framework.task.services.XnatTaskService;
import org.nrg.xdat.turbine.utils.AdminUtils;
import org.nrg.xft.event.persist.PersistentWorkflowUtils;
import org.nrg.xft.schema.XFTManager;
import org.nrg.xnat.services.XnatAppInfo;
import org.nrg.xnat.task.AbstractXnatTask;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

Expand All @@ -26,12 +30,13 @@

@Slf4j
@Component
public class DockerStatusUpdater implements Runnable {
public class DockerStatusUpdater implements Runnable {

private ContainerControlApi controlApi;
private DockerServerService dockerServerService;
private ContainerService containerService;

final XnatAppInfo xnatAppInfo;

private boolean haveLoggedDockerConnectFailure = false;
private boolean haveLoggedNoServerInDb = false;
private boolean haveLoggedXftInitFailure = false;
Expand All @@ -40,88 +45,95 @@ public class DockerStatusUpdater implements Runnable {
@SuppressWarnings("SpringJavaAutowiringInspection")
public DockerStatusUpdater(final ContainerControlApi controlApi,
final DockerServerService dockerServerService,
final ContainerService containerService) {
final ContainerService containerService,
final XnatAppInfo xnatAppInfo) {
this.controlApi = controlApi;
this.dockerServerService = dockerServerService;
this.containerService = containerService;
this.xnatAppInfo=xnatAppInfo;
}

@Override
public void run() {
log.trace("-----------------------------------------------------------------------------");
log.trace("Attempting to update status with docker.");

final String skipMessage = "Skipping attempt to update status.";

if (!XFTManager.isInitialized()) {
if (!haveLoggedXftInitFailure) {
log.info("XFT is not initialized. " + skipMessage);
haveLoggedXftInitFailure = true;
}
return;
}

// Since XFT is up, we should be able to connect to the database and read the docker server
DockerServer dockerServer = null;
try {
dockerServer = dockerServerService.getServer();
} catch (NotFoundException e) {
log.error("Docker server not found");
}
if (dockerServer == null) {
log.trace("Docker server is null");
if (!haveLoggedNoServerInDb) {
log.info("No docker server has been defined (or enabled) in the database. " + skipMessage);
haveLoggedNoServerInDb = true;
haveLoggedXftInitFailure = false;
}
log.trace("haveLoggedNoServerInDb "+ haveLoggedNoServerInDb + " about to return ");
return;
}

if (!controlApi.canConnect()) {
log.info("Cannot connect to docker server " + dockerServer.name() + ". " + skipMessage);
if (!haveLoggedDockerConnectFailure) {
log.info("Cannot ping docker server " + dockerServer.name() + ". " + skipMessage);
haveLoggedDockerConnectFailure = true;
haveLoggedXftInitFailure = false;
haveLoggedNoServerInDb = false;
}
log.trace("haveLoggedDockerConnectFailure: " + haveLoggedDockerConnectFailure + " about to return");
return;
}

// Now we should be able to check the status
final UpdateReport updateReport = dockerServer.swarmMode() ? updateServices(dockerServer) : updateContainers(dockerServer);
if (updateReport.successful == null) {
// This means some, but not all, of the services didn't update properly. Which ones?
for (final UpdateReportEntry entry : updateReport.updateReports) {
if (!entry.successful) {
log.error("Could not update status for {}. Message: {}", entry.id, entry.message);
} else {
log.debug("Updated successfully for {}.", entry.id);
}
}

// Reset failure flags
haveLoggedDockerConnectFailure = false;
haveLoggedXftInitFailure = false;
haveLoggedNoServerInDb = false;
} else if (updateReport.successful) {
if (updateReport.updateReports.size() > 0) {
log.debug("Updated status successfully.");
}
// Reset failure flags
haveLoggedDockerConnectFailure = false;
haveLoggedXftInitFailure = false;
haveLoggedNoServerInDb = false;
} else {
log.info("Did not update status successfully.");
}
log.trace("-----------------------------------------------------------------------------");
log.trace("DOCKERSTATUSUPDATER: RUN COMPLETE");
log.trace("-----------------------------------------------------------------------------");

if(!xnatAppInfo.isPrimaryNode()) {
log.trace("Not the Primary node: skipping update status with docker."); return;
}
log.trace("-----------------------------------------------------------------------------");

log.trace("Primary node: Attempting to update status with docker.");

final String skipMessage = "Skipping attempt to update status.";

if (!XFTManager.isInitialized()) {
if (!haveLoggedXftInitFailure) {
log.info("XFT is not initialized. " + skipMessage);
haveLoggedXftInitFailure = true;
}
return;
}

// Since XFT is up, we should be able to connect to the database and read the docker server
DockerServer dockerServer = null;
try {
dockerServer = dockerServerService.getServer();
} catch (NotFoundException e) {
log.error("Docker server not found");
}
if (dockerServer == null) {
log.trace("Docker server is null");
if (!haveLoggedNoServerInDb) {
log.info("No docker server has been defined (or enabled) in the database. " + skipMessage);
haveLoggedNoServerInDb = true;
haveLoggedXftInitFailure = false;
}
log.trace("haveLoggedNoServerInDb "+ haveLoggedNoServerInDb + " about to return ");
return;
}

if (!controlApi.canConnect()) {
log.info("Cannot connect to docker server " + dockerServer.name() + ". " + skipMessage);
if (!haveLoggedDockerConnectFailure) {
log.info("Cannot ping docker server " + dockerServer.name() + ". " + skipMessage);
haveLoggedDockerConnectFailure = true;
haveLoggedXftInitFailure = false;
haveLoggedNoServerInDb = false;
}
log.trace("haveLoggedDockerConnectFailure: " + haveLoggedDockerConnectFailure + " about to return");
return;
}


// Now we should be able to check the status
final UpdateReport updateReport = dockerServer.swarmMode() ? updateServices(dockerServer) : updateContainers(dockerServer);
if (updateReport.successful == null) {
// This means some, but not all, of the services didn't update properly. Which ones?
for (final UpdateReportEntry entry : updateReport.updateReports) {
if (!entry.successful) {
log.error("Could not update status for {}. Message: {}", entry.id, entry.message);
} else {
log.debug("Updated successfully for {}.", entry.id);
}
}

// Reset failure flags
haveLoggedDockerConnectFailure = false;
haveLoggedXftInitFailure = false;
haveLoggedNoServerInDb = false;
} else if (updateReport.successful) {
if (updateReport.updateReports.size() > 0) {
log.debug("Updated status successfully.");
}
// Reset failure flags
haveLoggedDockerConnectFailure = false;
haveLoggedXftInitFailure = false;
haveLoggedNoServerInDb = false;
} else {
log.info("Did not update status successfully.");
}
log.trace("-----------------------------------------------------------------------------");
log.trace("DOCKERSTATUSUPDATER: RUN COMPLETE");
log.trace("-----------------------------------------------------------------------------");

}

@Nonnull
Expand Down Expand Up @@ -243,4 +255,6 @@ public static UpdateReportEntry failure(final String id,
return updateReportEntry;
}
}


}
26 changes: 19 additions & 7 deletions ...ava/org/nrg/containers/initialization/tasks/ResetContainersInOrphanedFinalizingState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import org.nrg.containers.services.ContainerService;
import org.nrg.xnat.initialization.tasks.AbstractInitializingTask;
import org.nrg.xnat.initialization.tasks.InitializingTaskException;
import org.nrg.xnat.services.XnatAppInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

Expand All @@ -19,8 +20,9 @@ public class ResetContainersInOrphanedFinalizingState extends AbstractInitializi


@Autowired
public ResetContainersInOrphanedFinalizingState(final ContainerService containerService) {
public ResetContainersInOrphanedFinalizingState(final ContainerService containerService, final XnatAppInfo appInfo) {
this.containerService = containerService;
this._appInfo=appInfo;
}

@Override
Expand All @@ -30,12 +32,22 @@ public String getTaskName() {

@Override
protected void callImpl() throws InitializingTaskException {
log.debug("Checking if any containers exist in orphaned Finalizing state in database. If they do, resetting them to Waiting/Failed");
//MR: 10/30/2018 - If this is the first time the DockerStatusUpdater is running
//Look for all containers which are in Finalizing state
//These are probably in "hung" state
//Change the state of these to Waiting
containerService.resetFinalizingStatusToWaitingOrFailed();
if(!_appInfo.isPrimaryNode()) {
log.debug("This is not the primary node. Skippping Reset Containers In Orphaned Finalizing State");
return;
}
log.debug("This is the primary node. Checking if any containers exist in orphaned Finalizing state in database. If they do, resetting them to Waiting/Failed");
//MR: 10/30/2018 - If this is the first time the DockerStatusUpdater is running
//Look for all containers which are in Finalizing state
//These are probably in "hung" state
//Change the state of these to Waiting
containerService.resetFinalizingStatusToWaitingOrFailed();
log.debug("Reset Complete Orphaned Finalizing states to Waiting/Failed State");

}

private final XnatAppInfo _appInfo;



}
46 changes: 28 additions & 18 deletions src/main/java/org/nrg/containers/model/container/auto/Container.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
package org.nrg.containers.model.container.auto;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.auto.value.AutoValue;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;

import javax.annotation.Nullable;

import org.nrg.containers.events.model.ContainerEvent;
import org.nrg.containers.events.model.DockerContainerEvent;
import org.nrg.containers.model.command.auto.ResolvedCommand;
Expand All @@ -21,13 +19,22 @@
import org.nrg.containers.model.container.entity.ContainerEntityMount;
import org.nrg.containers.model.container.entity.ContainerEntityOutput;
import org.nrg.containers.model.container.entity.ContainerMountFilesEntity;
import org.nrg.containers.utils.JsonDateSerializer;
import org.nrg.containers.utils.JsonStringToDateSerializer;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.google.auto.value.AutoValue;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

import javax.annotation.Nullable;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;

@AutoValue
public abstract class Container {
Expand All @@ -36,6 +43,7 @@ public abstract class Container {
@JsonProperty("id") public abstract long databaseId();
@JsonProperty("command-id") public abstract long commandId();
@Nullable @JsonProperty("status") public abstract String status();
@JsonSerialize(using=JsonDateSerializer.class)
@Nullable @JsonProperty("status-time") public abstract Date statusTime();
@JsonProperty("wrapper-id") public abstract long wrapperId();
@Nullable @JsonProperty("container-id") public abstract String containerId();
Expand Down Expand Up @@ -735,7 +743,9 @@ public static abstract class ContainerHistory {
@JsonProperty("status") public abstract String status();
@JsonProperty("entity-type") public abstract String entityType();
@Nullable @JsonProperty("entity-id") public abstract String entityId();
@JsonSerialize(using=JsonDateSerializer.class)
@JsonProperty("time-recorded") public abstract Date timeRecorded();
@JsonSerialize(using=JsonStringToDateSerializer.class)
@Nullable @JsonProperty("external-timestamp") public abstract String externalTimestamp();
@Nullable @JsonProperty("message") public abstract String message();
@Nullable @JsonProperty("exitCode") public abstract String exitCode();
Expand Down Expand Up @@ -792,7 +802,7 @@ public static ContainerHistory fromSystem(final String status,
.entityType("system")
.entityId(null)
.timeRecorded(new Date())
.externalTimestamp(null)
.externalTimestamp(Long.toString(System.nanoTime()))
.build();
}

Expand All @@ -802,7 +812,7 @@ public static ContainerHistory fromUserAction(final String status, final String
.entityType("user")
.entityId(username)
.timeRecorded(new Date())
.externalTimestamp(null)
.externalTimestamp(Long.toString(System.nanoTime()))
.message(null)
.build();
}
Expand Down
Loading

0 comments on commit 050e09e

Please sign in to comment.