From df9d06be7dc53e9ce6eb365ce7cb1eaec1958d21 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Mon, 7 Oct 2024 10:21:34 -0400 Subject: [PATCH 1/8] Add command for processing source files, and configuration for boxctron jobs --- .../unc/lib/boxc/migration/cdm/CLIMain.java | 3 +- .../cdm/ProcessSourceFilesCommand.java | 97 +++++++++++++++++++ .../migration/cdm/model/BxcEnvironment.java | 84 ++++++++++++++++ .../cdm/ProcessSourceFilesCommandIT.java | 81 ++++++++++++++++ 4 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java create mode 100644 src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/CLIMain.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/CLIMain.java index 0539e722..ccacaa48 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/CLIMain.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/CLIMain.java @@ -43,7 +43,8 @@ PermissionsCommand.class, ExportObjectsCommand.class, ListProjectsCommand.class, - ArchiveProjectsCommand.class + ArchiveProjectsCommand.class, + ProcessSourceFilesCommand.class }) public class CLIMain implements Callable { @Option(names = { "-w", "--work-dir" }, diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java new file mode 100644 index 00000000..7b6c9f08 --- /dev/null +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java @@ -0,0 +1,97 @@ +package edu.unc.lib.boxc.migration.cdm; + +import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException; +import edu.unc.lib.boxc.migration.cdm.jobs.VelocicroptorRemoteJob; +import edu.unc.lib.boxc.migration.cdm.model.MigrationProject; +import edu.unc.lib.boxc.migration.cdm.options.ProcessSourceFilesOptions; +import edu.unc.lib.boxc.migration.cdm.services.CdmIndexService; +import edu.unc.lib.boxc.migration.cdm.services.ChompbConfigService; +import edu.unc.lib.boxc.migration.cdm.services.MigrationProjectFactory; +import edu.unc.lib.boxc.migration.cdm.services.SourceFileService; +import edu.unc.lib.boxc.migration.cdm.services.SourceFilesToRemoteService; +import edu.unc.lib.boxc.migration.cdm.util.SshClientService; +import org.slf4j.Logger; +import picocli.CommandLine; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.Callable; + +import static edu.unc.lib.boxc.migration.cdm.util.CLIConstants.outputLogger; +import static org.slf4j.LoggerFactory.getLogger; + +/** + * @author bbpennel + */ +@CommandLine.Command(name = "process_source_files", + description = { + "Perform a processing job on the source files mapped in this project."}) +public class ProcessSourceFilesCommand implements Callable { + private static final Logger log = getLogger(ProcessSourceFilesCommand.class); + + @CommandLine.ParentCommand + private CLIMain parentCommand; + private VelocicroptorRemoteJob velocicroptorRemoteJob; + + private MigrationProject project; + + @CommandLine.Mixin + private ProcessSourceFilesOptions options; + + @Override + public Integer call() throws Exception { + long start = System.nanoTime(); + try { + validateActionName(options.getActionName()); + initialize(); + velocicroptorRemoteJob.run(options); + outputLogger.info("Completed {} job to process source files for {} in {}s", + options.getActionName(), project.getProjectName(), (System.nanoTime() - start) / 1e9); + return 0; + } catch (MigrationException | IllegalArgumentException e) { + outputLogger.info("Source file processing command failed: {}", e.getMessage()); + log.warn("Source file processing command failed", e); + return 1; + } catch (Exception e) { + log.error("Source file processing command failed", e); + outputLogger.info("Source file processing command failed: {}", e.getMessage(), e); + return 1; + } + } + + private void validateActionName(String actionName) { + if (!actionName.equals("velocicroptor")) { + throw new IllegalArgumentException("Invalid action name provided: " + actionName); + } + } + + private void initialize() throws IOException { + Path currentPath = parentCommand.getWorkingDirectory(); + project = MigrationProjectFactory.loadMigrationProject(currentPath); + var config = parentCommand.getChompbConfig(); + var boxcEnv = config.getBxcEnvironments().get(project.getProjectProperties().getBxcEnvironmentId()); + var sshClientService = new SshClientService(); + sshClientService.setSshHost(boxcEnv.getBoxctronHost()); + sshClientService.setSshPort(boxcEnv.getBoxctronPort()); + sshClientService.setSshKeyPath(boxcEnv.getBoxctronKeyPath()); + sshClientService.setSshUsername(boxcEnv.getBoxctronSshUser()); + sshClientService.initialize(); + var cdmIndexService = new CdmIndexService(); + cdmIndexService.setProject(project); + var sourceFileService = new SourceFileService(); + sourceFileService.setProject(project); + sourceFileService.setIndexService(cdmIndexService); + var sourceFilesToRemoteService = new SourceFilesToRemoteService(); + sourceFilesToRemoteService.setSourceFileService(sourceFileService); + sourceFilesToRemoteService.setSshClientService(sshClientService); + velocicroptorRemoteJob = new VelocicroptorRemoteJob(); + velocicroptorRemoteJob.setProject(project); + velocicroptorRemoteJob.setSshClientService(sshClientService); + velocicroptorRemoteJob.setOutputServer(boxcEnv.getBoxctronOutputServer()); + velocicroptorRemoteJob.setOutputPath(boxcEnv.getBoxctronOutputBasePath()); + velocicroptorRemoteJob.setRemoteProjectsPath(boxcEnv.getBoxctronRemoteProjectsPath()); + velocicroptorRemoteJob.setAdminEmail(boxcEnv.getBoxctronAdminEmail()); + velocicroptorRemoteJob.setRemoteJobScriptPath(boxcEnv.getBoxctronRemoteJobScriptPath()); + velocicroptorRemoteJob.setSourceFilesToRemoteService(sourceFilesToRemoteService); + } +} diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java index 5d626cf2..cd9f9ecb 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java @@ -1,5 +1,7 @@ package edu.unc.lib.boxc.migration.cdm.model; +import java.nio.file.Path; + /** * Configuration information for a Box-c environment * @@ -9,6 +11,16 @@ public class BxcEnvironment { private String httpBaseUrl; private String solrServerUrl; + private String boxctronHost; + private int boxctronPort; + private String boxctronSshUser; + private Path boxctronKeyPath; + private Path boxctronRemoteProjectsPath; + private String boxctronAdminEmail; + private String boxctronOutputServer; + private Path boxctronOutputBasePath; + private Path boxctronRemoteJobScriptPath; + public String getHttpBaseUrl() { return httpBaseUrl; } @@ -24,4 +36,76 @@ public String getSolrServerUrl() { public void setSolrServerUrl(String solrServerUrl) { this.solrServerUrl = solrServerUrl; } + + public String getBoxctronHost() { + return boxctronHost; + } + + public void setBoxctronHost(String boxctronHost) { + this.boxctronHost = boxctronHost; + } + + public int getBoxctronPort() { + return boxctronPort; + } + + public void setBoxctronPort(int boxctronPort) { + this.boxctronPort = boxctronPort; + } + + public String getBoxctronSshUser() { + return boxctronSshUser; + } + + public void setBoxctronSshUser(String boxctronSshUser) { + this.boxctronSshUser = boxctronSshUser; + } + + public Path getBoxctronKeyPath() { + return boxctronKeyPath; + } + + public void setBoxctronKeyPath(Path boxctronKeyPath) { + this.boxctronKeyPath = boxctronKeyPath; + } + + public Path getBoxctronRemoteProjectsPath() { + return boxctronRemoteProjectsPath; + } + + public void setBoxctronRemoteProjectsPath(Path boxctronRemoteProjectsPath) { + this.boxctronRemoteProjectsPath = boxctronRemoteProjectsPath; + } + + public String getBoxctronAdminEmail() { + return boxctronAdminEmail; + } + + public void setBoxctronAdminEmail(String boxctronAdminEmail) { + this.boxctronAdminEmail = boxctronAdminEmail; + } + + public String getBoxctronOutputServer() { + return boxctronOutputServer; + } + + public void setBoxctronOutputServer(String boxctronOutputServer) { + this.boxctronOutputServer = boxctronOutputServer; + } + + public Path getBoxctronOutputBasePath() { + return boxctronOutputBasePath; + } + + public void setBoxctronOutputBasePath(Path boxctronOutputBasePath) { + this.boxctronOutputBasePath = boxctronOutputBasePath; + } + + public Path getBoxctronRemoteJobScriptPath() { + return boxctronRemoteJobScriptPath; + } + + public void setBoxctronRemoteJobScriptPath(Path boxctronRemoteJobScriptPath) { + this.boxctronRemoteJobScriptPath = boxctronRemoteJobScriptPath; + } } diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java new file mode 100644 index 00000000..1f864b04 --- /dev/null +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java @@ -0,0 +1,81 @@ +package edu.unc.lib.boxc.migration.cdm; + +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.unc.lib.boxc.migration.cdm.services.ChompbConfigService; +import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper; +import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper; +import edu.unc.lib.boxc.migration.cdm.test.TestSshServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * @author bbpennel + */ +public class ProcessSourceFilesCommandIT extends AbstractCommandIT { + private TestSshServer testSshServer; + private static final Path CLIENT_KEY_PATH = Paths.get("src/test/resources/test_client_key"); + + @BeforeEach + public void setup() throws Exception { + initProjectAndHelper(); + testSshServer = new TestSshServer(); + testSshServer.setClientKeyPath(CLIENT_KEY_PATH); + testSshServer.startServer(); + setupChompbConfig(); + } + + @AfterEach + public void tearDown() throws Exception { + testSshServer.stopServer(); + } + + @Override + protected void setupChompbConfig() throws IOException { + var configPath = tmpFolder.resolve("config.json"); + var config = new ChompbConfigService.ChompbConfig(); + config.setCdmEnvironments(CdmEnvironmentHelper.getTestMapping()); + var bxcEnvs = BxcEnvironmentHelper.getTestMapping(); + config.setBxcEnvironments(bxcEnvs); + var bxcEnv = bxcEnvs.get(BxcEnvironmentHelper.DEFAULT_ENV_ID); + bxcEnv.setBoxctronHost("127.0.0.1"); + bxcEnv.setBoxctronPort(42222); + bxcEnv.setBoxctronSshUser("testuser"); + bxcEnv.setBoxctronKeyPath(CLIENT_KEY_PATH); + bxcEnv.setBoxctronAdminEmail("chompb@example.com"); + bxcEnv.setBoxctronOutputServer("chompb.example.com"); + bxcEnv.setBoxctronOutputBasePath(tmpFolder); + bxcEnv.setBoxctronRemoteJobScriptPath(tmpFolder.resolve("remote_job.sh")); + bxcEnv.setBoxctronRemoteProjectsPath(tmpFolder.resolve("remote_projects")); + ObjectMapper mapper = new ObjectMapper(); + mapper.writeValue(Files.newOutputStream(configPath), config); + chompbConfigPath = configPath.toString(); + } + + @Test + public void testProcessSourceFilesInvalidAction() { + String[] args = new String[] { + "-w", project.getProjectPath().toString(), + "--env-config", chompbConfigPath, + "process_source_files", + "-a", "garbo"}; + executeExpectFailure(args); + assertOutputContains("Invalid action name provided: garbo"); + } + + @Test + public void testProcessSourceFilesValid() { + String[] args = new String[] { + "-w", project.getProjectPath().toString(), + "--env-config", chompbConfigPath, + "process_source_files", + "-a", "velocicroptor"}; + executeExpectSuccess(args); + assertOutputContains("Completed velocicroptor job to process source files"); + } +} From ab6cac37e28d53fe338e7952f1cf9c76a568f219 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Mon, 7 Oct 2024 11:11:33 -0400 Subject: [PATCH 2/8] Separate ssh host for transfer and running scripts --- .../cdm/ProcessSourceFilesCommand.java | 25 ++++++++++++------- .../migration/cdm/model/BxcEnvironment.java | 19 ++++++++++---- .../cdm/ProcessSourceFilesCommandIT.java | 3 ++- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java index 7b6c9f08..53140d72 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java @@ -5,7 +5,6 @@ import edu.unc.lib.boxc.migration.cdm.model.MigrationProject; import edu.unc.lib.boxc.migration.cdm.options.ProcessSourceFilesOptions; import edu.unc.lib.boxc.migration.cdm.services.CdmIndexService; -import edu.unc.lib.boxc.migration.cdm.services.ChompbConfigService; import edu.unc.lib.boxc.migration.cdm.services.MigrationProjectFactory; import edu.unc.lib.boxc.migration.cdm.services.SourceFileService; import edu.unc.lib.boxc.migration.cdm.services.SourceFilesToRemoteService; @@ -70,12 +69,20 @@ private void initialize() throws IOException { project = MigrationProjectFactory.loadMigrationProject(currentPath); var config = parentCommand.getChompbConfig(); var boxcEnv = config.getBxcEnvironments().get(project.getProjectProperties().getBxcEnvironmentId()); - var sshClientService = new SshClientService(); - sshClientService.setSshHost(boxcEnv.getBoxctronHost()); - sshClientService.setSshPort(boxcEnv.getBoxctronPort()); - sshClientService.setSshKeyPath(boxcEnv.getBoxctronKeyPath()); - sshClientService.setSshUsername(boxcEnv.getBoxctronSshUser()); - sshClientService.initialize(); + // Separate service for executing scripts on the remote server + var sshClientScriptService = new SshClientService(); + sshClientScriptService.setSshHost(boxcEnv.getBoxctronScriptHost()); + sshClientScriptService.setSshPort(boxcEnv.getBoxctronPort()); + sshClientScriptService.setSshKeyPath(boxcEnv.getBoxctronKeyPath()); + sshClientScriptService.setSshUsername(boxcEnv.getBoxctronSshUser()); + sshClientScriptService.initialize(); + // Separate service for transferring files to the remote server + var sshClientTransferService = new SshClientService(); + sshClientTransferService.setSshHost(boxcEnv.getBoxctronTransferHost()); + sshClientTransferService.setSshPort(boxcEnv.getBoxctronPort()); + sshClientTransferService.setSshKeyPath(boxcEnv.getBoxctronKeyPath()); + sshClientTransferService.setSshUsername(boxcEnv.getBoxctronSshUser()); + sshClientTransferService.initialize(); var cdmIndexService = new CdmIndexService(); cdmIndexService.setProject(project); var sourceFileService = new SourceFileService(); @@ -83,10 +90,10 @@ private void initialize() throws IOException { sourceFileService.setIndexService(cdmIndexService); var sourceFilesToRemoteService = new SourceFilesToRemoteService(); sourceFilesToRemoteService.setSourceFileService(sourceFileService); - sourceFilesToRemoteService.setSshClientService(sshClientService); + sourceFilesToRemoteService.setSshClientService(sshClientTransferService); velocicroptorRemoteJob = new VelocicroptorRemoteJob(); velocicroptorRemoteJob.setProject(project); - velocicroptorRemoteJob.setSshClientService(sshClientService); + velocicroptorRemoteJob.setSshClientService(sshClientScriptService); velocicroptorRemoteJob.setOutputServer(boxcEnv.getBoxctronOutputServer()); velocicroptorRemoteJob.setOutputPath(boxcEnv.getBoxctronOutputBasePath()); velocicroptorRemoteJob.setRemoteProjectsPath(boxcEnv.getBoxctronRemoteProjectsPath()); diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java index cd9f9ecb..a31d6a86 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java @@ -11,7 +11,8 @@ public class BxcEnvironment { private String httpBaseUrl; private String solrServerUrl; - private String boxctronHost; + private String boxctronScriptHost; + private String boxctronTransferHost; private int boxctronPort; private String boxctronSshUser; private Path boxctronKeyPath; @@ -37,12 +38,20 @@ public void setSolrServerUrl(String solrServerUrl) { this.solrServerUrl = solrServerUrl; } - public String getBoxctronHost() { - return boxctronHost; + public String getBoxctronScriptHost() { + return boxctronScriptHost; } - public void setBoxctronHost(String boxctronHost) { - this.boxctronHost = boxctronHost; + public void setBoxctronScriptHost(String boxctronScriptHost) { + this.boxctronScriptHost = boxctronScriptHost; + } + + public String getBoxctronTransferHost() { + return boxctronTransferHost; + } + + public void setBoxctronTransferHost(String boxctronTransferHost) { + this.boxctronTransferHost = boxctronTransferHost; } public int getBoxctronPort() { diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java index 1f864b04..c3e4ea64 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java @@ -43,7 +43,8 @@ protected void setupChompbConfig() throws IOException { var bxcEnvs = BxcEnvironmentHelper.getTestMapping(); config.setBxcEnvironments(bxcEnvs); var bxcEnv = bxcEnvs.get(BxcEnvironmentHelper.DEFAULT_ENV_ID); - bxcEnv.setBoxctronHost("127.0.0.1"); + bxcEnv.setBoxctronScriptHost("127.0.0.1"); + bxcEnv.setBoxctronTransferHost("127.0.0.1"); bxcEnv.setBoxctronPort(42222); bxcEnv.setBoxctronSshUser("testuser"); bxcEnv.setBoxctronKeyPath(CLIENT_KEY_PATH); From cde7bf2244283a44f35f80066469c91f0556b33d Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Mon, 7 Oct 2024 11:26:38 -0400 Subject: [PATCH 3/8] Change script path to script directory path so it can be used for multiple jobs, and have job specify the specific filename --- .../migration/cdm/ProcessSourceFilesCommand.java | 2 +- .../migration/cdm/jobs/VelocicroptorRemoteJob.java | 10 ++++++---- .../boxc/migration/cdm/model/BxcEnvironment.java | 10 +++++----- .../migration/cdm/ProcessSourceFilesCommandIT.java | 2 +- .../cdm/jobs/VelocicroptorRemoteJobTest.java | 13 ++++--------- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java index 53140d72..433403e4 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java @@ -98,7 +98,7 @@ private void initialize() throws IOException { velocicroptorRemoteJob.setOutputPath(boxcEnv.getBoxctronOutputBasePath()); velocicroptorRemoteJob.setRemoteProjectsPath(boxcEnv.getBoxctronRemoteProjectsPath()); velocicroptorRemoteJob.setAdminEmail(boxcEnv.getBoxctronAdminEmail()); - velocicroptorRemoteJob.setRemoteJobScriptPath(boxcEnv.getBoxctronRemoteJobScriptPath()); + velocicroptorRemoteJob.setRemoteJobScriptsPath(boxcEnv.getBoxctronRemoteJobScriptsPath()); velocicroptorRemoteJob.setSourceFilesToRemoteService(sourceFilesToRemoteService); } } diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java index 3fd06cc3..66ee9b3c 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java @@ -25,12 +25,13 @@ public class VelocicroptorRemoteJob { private static final String JOB_ID_PATTERN_FORMAT = "ddMMyyyyHHmmssSSS"; private static final DateTimeFormatter JOB_ID_FORMATTER = DateTimeFormatter.ofPattern(JOB_ID_PATTERN_FORMAT) .withZone(ZoneId.systemDefault()); + private static final String JOB_FILENAME = "velocicroptor_job.sh"; private SshClientService sshClientService; private MigrationProject project; private SourceFilesToRemoteService sourceFilesToRemoteService; private Path remoteProjectsPath; - private Path remoteJobScriptPath; + private Path remoteJobScriptsPath; private String adminEmail; private String outputServer; private Path outputPath; @@ -61,7 +62,8 @@ public String run(ProcessSourceFilesOptions options) { String configJson = mapper.writeValueAsString(config); // Trigger remote job, passing config as argument - sshClientService.executeRemoteCommand("sbatch " + remoteJobScriptPath.toString() + " '" + configJson + "'"); + var scriptPath = remoteJobScriptsPath.resolve(JOB_FILENAME).toAbsolutePath(); + sshClientService.executeRemoteCommand("sbatch " + scriptPath + " '" + configJson + "'"); } catch (IOException e) { throw new MigrationException(e); } @@ -100,8 +102,8 @@ public void setRemoteProjectsPath(Path remoteProjectsPath) { this.remoteProjectsPath = remoteProjectsPath; } - public void setRemoteJobScriptPath(Path remoteJobScriptPath) { - this.remoteJobScriptPath = remoteJobScriptPath; + public void setRemoteJobScriptsPath(Path remoteJobScriptsPath) { + this.remoteJobScriptsPath = remoteJobScriptsPath; } public void setAdminEmail(String adminEmail) { diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java index a31d6a86..f52b4356 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java @@ -20,7 +20,7 @@ public class BxcEnvironment { private String boxctronAdminEmail; private String boxctronOutputServer; private Path boxctronOutputBasePath; - private Path boxctronRemoteJobScriptPath; + private Path boxctronRemoteJobScriptsPath; public String getHttpBaseUrl() { return httpBaseUrl; @@ -110,11 +110,11 @@ public void setBoxctronOutputBasePath(Path boxctronOutputBasePath) { this.boxctronOutputBasePath = boxctronOutputBasePath; } - public Path getBoxctronRemoteJobScriptPath() { - return boxctronRemoteJobScriptPath; + public Path getBoxctronRemoteJobScriptsPath() { + return boxctronRemoteJobScriptsPath; } - public void setBoxctronRemoteJobScriptPath(Path boxctronRemoteJobScriptPath) { - this.boxctronRemoteJobScriptPath = boxctronRemoteJobScriptPath; + public void setBoxctronRemoteJobScriptsPath(Path boxctronRemoteJobScriptsPath) { + this.boxctronRemoteJobScriptsPath = boxctronRemoteJobScriptsPath; } } diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java index c3e4ea64..c50a0a53 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java @@ -51,7 +51,7 @@ protected void setupChompbConfig() throws IOException { bxcEnv.setBoxctronAdminEmail("chompb@example.com"); bxcEnv.setBoxctronOutputServer("chompb.example.com"); bxcEnv.setBoxctronOutputBasePath(tmpFolder); - bxcEnv.setBoxctronRemoteJobScriptPath(tmpFolder.resolve("remote_job.sh")); + bxcEnv.setBoxctronRemoteJobScriptsPath(tmpFolder.resolve("scripts")); bxcEnv.setBoxctronRemoteProjectsPath(tmpFolder.resolve("remote_projects")); ObjectMapper mapper = new ObjectMapper(); mapper.writeValue(Files.newOutputStream(configPath), config); diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJobTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJobTest.java index fd0627e4..bf0825c5 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJobTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJobTest.java @@ -8,7 +8,6 @@ import edu.unc.lib.boxc.migration.cdm.services.SourceFilesToRemoteService; import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper; import edu.unc.lib.boxc.migration.cdm.util.SshClientService; -import org.apache.solr.client.solrj.SolrQuery; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -29,7 +28,6 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.openMocks; /** @@ -48,7 +46,7 @@ public class VelocicroptorRemoteJobTest { @Mock private SourceFilesToRemoteService sourceFilesToRemoteService; private Path remoteProjectsPath; - private Path remoteJobScriptPath; + private Path remoteJobScriptsPath; private Path projectPath; private Path outputPath; @Captor @@ -66,7 +64,7 @@ public void setup() throws IOException { tmpFolder, PROJECT_NAME, null, USERNAME, null, BxcEnvironmentHelper.DEFAULT_ENV_ID); remoteProjectsPath = tmpFolder.resolve("remote_projects"); - remoteJobScriptPath = tmpFolder.resolve("remote_job_script.sh"); + remoteJobScriptsPath = tmpFolder.resolve("scripts"); outputPath = projectPath.resolve(VelocicroptorRemoteJob.RESULTS_REL_PATH); @@ -75,7 +73,7 @@ public void setup() throws IOException { job.setProject(project); job.setSourceFilesToRemoteService(sourceFilesToRemoteService); job.setRemoteProjectsPath(remoteProjectsPath); - job.setRemoteJobScriptPath(remoteJobScriptPath); + job.setRemoteJobScriptsPath(remoteJobScriptsPath); job.setAdminEmail(ADMIN_EMAIL); job.setOutputServer(OUTPUT_SERVER); job.setOutputPath(outputPath); @@ -86,9 +84,6 @@ void tearDown() throws Exception { closeable.close(); } - // Successful - // IOException - @Test public void runSuccessfulTest() throws Exception { var options = new ProcessSourceFilesOptions(); @@ -107,7 +102,7 @@ public void runSuccessfulTest() throws Exception { var arguments = remoteArgumentCaptor.getValue().split(" ", 3); assertEquals("sbatch", arguments[0]); - assertEquals(remoteJobScriptPath.toString(), arguments[1]); + assertEquals(remoteJobScriptsPath.resolve("velocicroptor_job.sh").toString(), arguments[1]); // Parse the config (with outer quotes trimmed off) ObjectMapper mapper = new ObjectMapper(); From 6e9e1471fd126bd1a424ed36b5007acd51f4d784 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Fri, 11 Oct 2024 11:35:07 -0400 Subject: [PATCH 4/8] Change user and key to being options exclusively rather than coming from config. Setup a default email address --- .../cdm/ProcessSourceFilesCommand.java | 27 ++++++++++++++----- .../migration/cdm/model/BxcEnvironment.java | 18 ------------- .../options/ProcessSourceFilesOptions.java | 15 +++++++++++ .../migration/cdm/AbstractOutputTest.java | 24 +++++++++++++++++ .../cdm/ProcessSourceFilesCommandIT.java | 23 ++++++++++++++-- 5 files changed, 81 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java index 433403e4..35dbee4c 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommand.java @@ -2,6 +2,7 @@ import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException; import edu.unc.lib.boxc.migration.cdm.jobs.VelocicroptorRemoteJob; +import edu.unc.lib.boxc.migration.cdm.model.BxcEnvironment; import edu.unc.lib.boxc.migration.cdm.model.MigrationProject; import edu.unc.lib.boxc.migration.cdm.options.ProcessSourceFilesOptions; import edu.unc.lib.boxc.migration.cdm.services.CdmIndexService; @@ -9,6 +10,7 @@ import edu.unc.lib.boxc.migration.cdm.services.SourceFileService; import edu.unc.lib.boxc.migration.cdm.services.SourceFilesToRemoteService; import edu.unc.lib.boxc.migration.cdm.util.SshClientService; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import picocli.CommandLine; @@ -27,12 +29,14 @@ "Perform a processing job on the source files mapped in this project."}) public class ProcessSourceFilesCommand implements Callable { private static final Logger log = getLogger(ProcessSourceFilesCommand.class); + private static final String DEFAULT_EMAIL_DOMAIN = "@ad.unc.edu"; @CommandLine.ParentCommand private CLIMain parentCommand; private VelocicroptorRemoteJob velocicroptorRemoteJob; private MigrationProject project; + private BxcEnvironment boxcEnv; @CommandLine.Mixin private ProcessSourceFilesOptions options; @@ -42,6 +46,8 @@ public Integer call() throws Exception { long start = System.nanoTime(); try { validateActionName(options.getActionName()); + loadProjectEnvironment(); + setDefaultOptions(); initialize(); velocicroptorRemoteJob.run(options); outputLogger.info("Completed {} job to process source files for {} in {}s", @@ -64,24 +70,33 @@ private void validateActionName(String actionName) { } } - private void initialize() throws IOException { + private void setDefaultOptions() { + if (options.getEmailAddress() == null) { + options.setEmailAddress(options.getUsername() + DEFAULT_EMAIL_DOMAIN); + } + } + + private void loadProjectEnvironment() throws IOException { Path currentPath = parentCommand.getWorkingDirectory(); project = MigrationProjectFactory.loadMigrationProject(currentPath); var config = parentCommand.getChompbConfig(); - var boxcEnv = config.getBxcEnvironments().get(project.getProjectProperties().getBxcEnvironmentId()); + boxcEnv = config.getBxcEnvironments().get(project.getProjectProperties().getBxcEnvironmentId()); + } + + private void initialize() throws IOException { // Separate service for executing scripts on the remote server var sshClientScriptService = new SshClientService(); sshClientScriptService.setSshHost(boxcEnv.getBoxctronScriptHost()); sshClientScriptService.setSshPort(boxcEnv.getBoxctronPort()); - sshClientScriptService.setSshKeyPath(boxcEnv.getBoxctronKeyPath()); - sshClientScriptService.setSshUsername(boxcEnv.getBoxctronSshUser()); + sshClientScriptService.setSshKeyPath(options.getSshKeyPath()); + sshClientScriptService.setSshUsername(options.getUsername()); sshClientScriptService.initialize(); // Separate service for transferring files to the remote server var sshClientTransferService = new SshClientService(); sshClientTransferService.setSshHost(boxcEnv.getBoxctronTransferHost()); sshClientTransferService.setSshPort(boxcEnv.getBoxctronPort()); - sshClientTransferService.setSshKeyPath(boxcEnv.getBoxctronKeyPath()); - sshClientTransferService.setSshUsername(boxcEnv.getBoxctronSshUser()); + sshClientTransferService.setSshKeyPath(options.getSshKeyPath()); + sshClientTransferService.setSshUsername(options.getUsername()); sshClientTransferService.initialize(); var cdmIndexService = new CdmIndexService(); cdmIndexService.setProject(project); diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java index f52b4356..68d18640 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java @@ -14,8 +14,6 @@ public class BxcEnvironment { private String boxctronScriptHost; private String boxctronTransferHost; private int boxctronPort; - private String boxctronSshUser; - private Path boxctronKeyPath; private Path boxctronRemoteProjectsPath; private String boxctronAdminEmail; private String boxctronOutputServer; @@ -62,22 +60,6 @@ public void setBoxctronPort(int boxctronPort) { this.boxctronPort = boxctronPort; } - public String getBoxctronSshUser() { - return boxctronSshUser; - } - - public void setBoxctronSshUser(String boxctronSshUser) { - this.boxctronSshUser = boxctronSshUser; - } - - public Path getBoxctronKeyPath() { - return boxctronKeyPath; - } - - public void setBoxctronKeyPath(Path boxctronKeyPath) { - this.boxctronKeyPath = boxctronKeyPath; - } - public Path getBoxctronRemoteProjectsPath() { return boxctronRemoteProjectsPath; } diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/options/ProcessSourceFilesOptions.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/options/ProcessSourceFilesOptions.java index eb450ac9..a6472f5e 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/options/ProcessSourceFilesOptions.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/options/ProcessSourceFilesOptions.java @@ -2,6 +2,8 @@ import picocli.CommandLine; +import java.nio.file.Path; + /** * Options for job to process source files * @author bbpennel @@ -16,6 +18,11 @@ public class ProcessSourceFilesOptions { defaultValue = "${sys:user.name}") private String username; + @CommandLine.Option(names = {"-k", "--ssh-key"}, + description = "Path to the ssh key to use for the remote server.", + required = true) + private Path sshKeyPath; + @CommandLine.Option(names = {"-e", "--email"}, description = "Email of the user that started this job") private String emailAddress; @@ -36,6 +43,14 @@ public void setUsername(String username) { this.username = username; } + public Path getSshKeyPath() { + return sshKeyPath; + } + + public void setSshKeyPath(Path sshKeyPath) { + this.sshKeyPath = sshKeyPath; + } + public String getEmailAddress() { return emailAddress; } diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/AbstractOutputTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/AbstractOutputTest.java index 51a62285..a564f54b 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/AbstractOutputTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/AbstractOutputTest.java @@ -31,6 +31,9 @@ public class AbstractOutputTest { protected final ByteArrayOutputStream out = new ByteArrayOutputStream(); protected String output; + protected final PrintStream originalErr = System.err; + protected ByteArrayOutputStream err; + @AfterEach public void resetOut() { System.setOut(originalOut); @@ -45,8 +48,26 @@ public void setupOutput() throws Exception { output = null; } + /** + * Setup capture of error output to a stream. + * Note: This should be called before the command is executed, + * and resetError should be called before making any assertions. + */ + protected void setupError() { + err = new ByteArrayOutputStream(); + err.reset(); + System.setErr(new PrintStream(err)); + } + + protected void resetError() { + System.setErr(originalErr); + } + protected void resetOutput() { out.reset(); + if (err != null) { + err.reset(); + } output = null; } @@ -100,6 +121,9 @@ protected String getOutput() { if (output == null) { output = out.toString(); } + if (err != null) { + output += err.toString(); + } return output; } } diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java index c50a0a53..d1cb62ac 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/ProcessSourceFilesCommandIT.java @@ -46,8 +46,6 @@ protected void setupChompbConfig() throws IOException { bxcEnv.setBoxctronScriptHost("127.0.0.1"); bxcEnv.setBoxctronTransferHost("127.0.0.1"); bxcEnv.setBoxctronPort(42222); - bxcEnv.setBoxctronSshUser("testuser"); - bxcEnv.setBoxctronKeyPath(CLIENT_KEY_PATH); bxcEnv.setBoxctronAdminEmail("chompb@example.com"); bxcEnv.setBoxctronOutputServer("chompb.example.com"); bxcEnv.setBoxctronOutputBasePath(tmpFolder); @@ -64,17 +62,38 @@ public void testProcessSourceFilesInvalidAction() { "-w", project.getProjectPath().toString(), "--env-config", chompbConfigPath, "process_source_files", + "-u", "testuser", + "-k", CLIENT_KEY_PATH.toString(), "-a", "garbo"}; executeExpectFailure(args); assertOutputContains("Invalid action name provided: garbo"); } + @Test + public void testProcessSourceFilesNoKey() { + String[] args = new String[] { + "-w", project.getProjectPath().toString(), + "--env-config", chompbConfigPath, + "process_source_files", + "-u", "testuser", + "-a", "velocicroptor"}; + try { + setupError(); + executeExpectFailure(args); + } finally { + resetError(); + } + assertOutputContains("Missing required option"); + } + @Test public void testProcessSourceFilesValid() { String[] args = new String[] { "-w", project.getProjectPath().toString(), "--env-config", chompbConfigPath, "process_source_files", + "-u", "testuser", + "-k", CLIENT_KEY_PATH.toString(), "-a", "velocicroptor"}; executeExpectSuccess(args); assertOutputContains("Completed velocicroptor job to process source files"); From ca91200544f7149f91154267dac796544fddada7 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Thu, 17 Oct 2024 13:40:54 -0400 Subject: [PATCH 5/8] Throw error if command fails based on exit status. Add logging and cleanup --- .../services/SourceFilesToRemoteService.java | 19 +++++++++++++------ .../migration/cdm/util/SshClientService.java | 9 +++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java index 8ab1628b..9a705055 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java @@ -2,6 +2,7 @@ import edu.unc.lib.boxc.migration.cdm.model.SourceFilesInfo; import edu.unc.lib.boxc.migration.cdm.util.SshClientService; +import org.slf4j.Logger; import java.io.IOException; import java.nio.file.Path; @@ -11,11 +12,14 @@ import java.util.concurrent.ConcurrentLinkedDeque; import java.util.stream.Collectors; +import static org.slf4j.LoggerFactory.getLogger; + /** * Service for transferring source files from the local server to a remote destination. * @author bbpennel */ public class SourceFilesToRemoteService { + private static final Logger log = getLogger(SourceFilesToRemoteService.class); private SourceFileService sourceFileService; private SshClientService sshClientService; private int concurrentTransfers = 5; @@ -30,13 +34,13 @@ public void transferFiles(Path destinationPath) throws IOException { var sourceMappings = sourceFileService.loadMappings(); final Path destinationBasePath = destinationPath.toAbsolutePath(); // Get all the source paths as a thread safe queue - var sourcePaths = sourceMappings.getMappings().stream() - .map(SourceFilesInfo.SourceFileMapping::getFirstSourcePath) - .collect(Collectors.toList()); - var pathsDeque = new ConcurrentLinkedDeque(sourcePaths); + var pathsDeque = sourceMappings.getMappings().stream() + .map(SourceFilesInfo.SourceFileMapping::getFirstSourcePath) + .collect(Collectors.toCollection(ConcurrentLinkedDeque::new)); // For tracking if a parent directory has already been created Set createdParentsSet = ConcurrentHashMap.newKeySet(); // Create the remote destination directory + log.info("Creating remote destination directory {}", destinationBasePath); sshClientService.executeRemoteCommand("mkdir -p " + destinationBasePath); createdParentsSet.add(destinationBasePath.toString()); @@ -61,7 +65,9 @@ public void transferFiles(Path destinationPath) throws IOException { private Thread createTransferThread(ConcurrentLinkedDeque pathsDeque, Path destinationBasePath, Set createdParentsSet) { - var thread = new Thread(() -> { + // Create the parent path if we haven't already done so + // Upload the file to the appropriate path on the remote server + return new Thread(() -> { Path nextPath; while ((nextPath = pathsDeque.poll()) != null) { final Path sourcePath = nextPath; @@ -72,6 +78,7 @@ private Thread createTransferThread(ConcurrentLinkedDeque pathsDeque, // Create the parent path if we haven't already done so synchronized (createdParentsSet) { if (!createdParentsSet.contains(destParentPath.toString())) { + log.debug("Creating missing parent directory {}", destParentPath); createdParentsSet.add(destParentPath.toString()); sshClientService.executeRemoteCommand("mkdir -p " + destPath.getParent()); } @@ -79,6 +86,7 @@ private Thread createTransferThread(ConcurrentLinkedDeque pathsDeque, // Upload the file to the appropriate path on the remote server sshClientService.executeScpBlock(sshClient, (scpClient) -> { try { + log.info("Transferring file {} to {}", sourcePath, destPath); scpClient.upload(sourcePath.toString(), destPath.toString()); } catch (IOException e) { throw new RuntimeException("Failed to transfer file " + sourcePath, e); @@ -87,7 +95,6 @@ private Thread createTransferThread(ConcurrentLinkedDeque pathsDeque, }); } }); - return thread; } public void setSourceFileService(SourceFileService sourceFileService) { diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/util/SshClientService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/util/SshClientService.java index 40ad921f..bb347246 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/util/SshClientService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/util/SshClientService.java @@ -12,6 +12,7 @@ import org.apache.sshd.scp.client.ScpClient; import org.apache.sshd.scp.client.ScpClientCreator; import org.apache.sshd.common.util.security.SecurityUtils; +import org.slf4j.Logger; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -25,12 +26,14 @@ import java.util.function.Consumer; import static java.util.Collections.singletonList; +import static org.slf4j.LoggerFactory.getLogger; /** * Service for executing remote commands and transfers * @author bbpennel */ public class SshClientService { + private static final Logger log = getLogger(SshClientService.class); private static final int SSH_TIMEOUT_SECONDS = 10; private String sshHost; @@ -97,6 +100,12 @@ public String executeRemoteCommand(ClientSession clientSession, String command) channel.open().verify(5, TimeUnit.SECONDS); channel.waitFor(EnumSet.of(ClientChannelEvent.CLOSED), 5000); + if (channel.getExitStatus() != 0) { + throw new MigrationException("Remote command \"" + command + "\" failed with exit status " + + channel.getExitStatus() + ": " + responseStream); + } + log.debug("Command executed on remote server: {}", command); + return responseStream.toString(); } catch (Exception e) { throw new MigrationException("Failed to execute remote command", e); From 65cee10a9da3673e6570bd469f1f28082855e242 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Thu, 17 Oct 2024 13:42:51 -0400 Subject: [PATCH 6/8] Add umask for directory creation --- .../migration/cdm/services/SourceFilesToRemoteService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java index 9a705055..7b94d0a8 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java @@ -41,7 +41,7 @@ public void transferFiles(Path destinationPath) throws IOException { Set createdParentsSet = ConcurrentHashMap.newKeySet(); // Create the remote destination directory log.info("Creating remote destination directory {}", destinationBasePath); - sshClientService.executeRemoteCommand("mkdir -p " + destinationBasePath); + sshClientService.executeRemoteCommand("umask 002 && mkdir -p " + destinationBasePath); createdParentsSet.add(destinationBasePath.toString()); var threads = new ArrayList(concurrentTransfers); @@ -80,7 +80,7 @@ private Thread createTransferThread(ConcurrentLinkedDeque pathsDeque, if (!createdParentsSet.contains(destParentPath.toString())) { log.debug("Creating missing parent directory {}", destParentPath); createdParentsSet.add(destParentPath.toString()); - sshClientService.executeRemoteCommand("mkdir -p " + destPath.getParent()); + sshClientService.executeRemoteCommand("umask 002 && mkdir -p " + destPath.getParent()); } } // Upload the file to the appropriate path on the remote server From 6d8a15527c2136717168c153298f645efa70693d Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Fri, 18 Oct 2024 14:52:09 -0400 Subject: [PATCH 7/8] Add environment to config for job. Extend timeout for starting job and handling of responses. Add some logging to help track what's happening --- .../migration/cdm/jobs/VelocicroptorRemoteJob.java | 10 +++++++++- .../cdm/services/SourceFilesToRemoteService.java | 4 ++-- .../lib/boxc/migration/cdm/util/SshClientService.java | 6 ++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java index 66ee9b3c..5a00f1f8 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java @@ -6,6 +6,7 @@ import edu.unc.lib.boxc.migration.cdm.options.ProcessSourceFilesOptions; import edu.unc.lib.boxc.migration.cdm.services.SourceFilesToRemoteService; import edu.unc.lib.boxc.migration.cdm.util.SshClientService; +import org.slf4j.Logger; import java.io.IOException; import java.nio.file.Files; @@ -16,11 +17,14 @@ import java.util.HashMap; import java.util.Map; +import static org.slf4j.LoggerFactory.getLogger; + /** * Job which prepares and executes a remote velocicroptor job to crop color bars from images * @author bbpennel */ public class VelocicroptorRemoteJob { + private static final Logger log = getLogger(VelocicroptorRemoteJob.class); protected static final String RESULTS_REL_PATH = "processing/results/velocicroptor"; private static final String JOB_ID_PATTERN_FORMAT = "ddMMyyyyHHmmssSSS"; private static final DateTimeFormatter JOB_ID_FORMATTER = DateTimeFormatter.ofPattern(JOB_ID_PATTERN_FORMAT) @@ -63,7 +67,10 @@ public String run(ProcessSourceFilesOptions options) { // Trigger remote job, passing config as argument var scriptPath = remoteJobScriptsPath.resolve(JOB_FILENAME).toAbsolutePath(); - sshClientService.executeRemoteCommand("sbatch " + scriptPath + " '" + configJson + "'"); + var sbatchCommand = "sbatch " + scriptPath + " '" + configJson + "'"; + log.info("Executing remote job with command: {}", sbatchCommand); + var response = sshClientService.executeRemoteCommand("sbatch " + scriptPath + " '" + configJson + "'"); + log.info("Job submitted with response: {}", response); } catch (IOException e) { throw new MigrationException(e); } @@ -75,6 +82,7 @@ private Map createJobConfig(ProcessSourceFilesOptions options, I config.put("job_id", jobId); config.put("job_name", options.getActionName()); config.put("chompb_proj_name", project.getProjectName()); + config.put("environment", project.getProjectProperties().getBxcEnvironmentId()); config.put("admin_address", adminEmail); // User that initiated the job config.put("username", options.getUsername()); diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java index 7b94d0a8..9a705055 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFilesToRemoteService.java @@ -41,7 +41,7 @@ public void transferFiles(Path destinationPath) throws IOException { Set createdParentsSet = ConcurrentHashMap.newKeySet(); // Create the remote destination directory log.info("Creating remote destination directory {}", destinationBasePath); - sshClientService.executeRemoteCommand("umask 002 && mkdir -p " + destinationBasePath); + sshClientService.executeRemoteCommand("mkdir -p " + destinationBasePath); createdParentsSet.add(destinationBasePath.toString()); var threads = new ArrayList(concurrentTransfers); @@ -80,7 +80,7 @@ private Thread createTransferThread(ConcurrentLinkedDeque pathsDeque, if (!createdParentsSet.contains(destParentPath.toString())) { log.debug("Creating missing parent directory {}", destParentPath); createdParentsSet.add(destParentPath.toString()); - sshClientService.executeRemoteCommand("umask 002 && mkdir -p " + destPath.getParent()); + sshClientService.executeRemoteCommand("mkdir -p " + destPath.getParent()); } } // Upload the file to the appropriate path on the remote server diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/util/SshClientService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/util/SshClientService.java index bb347246..537763f8 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/util/SshClientService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/util/SshClientService.java @@ -99,8 +99,10 @@ public String executeRemoteCommand(ClientSession clientSession, String command) channel.setErr(responseStream); channel.open().verify(5, TimeUnit.SECONDS); - channel.waitFor(EnumSet.of(ClientChannelEvent.CLOSED), 5000); - if (channel.getExitStatus() != 0) { + channel.waitFor(EnumSet.of(ClientChannelEvent.CLOSED), 10000); + log.info("Channel is {}", channel); + log.info("Channel exist status is {}", channel.getExitStatus()); + if (channel.getExitStatus() == null || channel.getExitStatus() != 0) { throw new MigrationException("Remote command \"" + command + "\" failed with exit status " + channel.getExitStatus() + ": " + responseStream); } From 9d75fa55d0b5c7cd6b4242ec69725b9d4236dce3 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Fri, 18 Oct 2024 15:11:23 -0400 Subject: [PATCH 8/8] Update test ssh server to respond to sbatch commands, to get IT test working --- .../migration/cdm/test/TestSshServer.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/test/TestSshServer.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/test/TestSshServer.java index 034c0835..0075d5f6 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/test/TestSshServer.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/test/TestSshServer.java @@ -3,14 +3,18 @@ import org.apache.sshd.common.util.io.resource.PathResource; import org.apache.sshd.common.util.security.SecurityUtils; import org.apache.sshd.scp.server.ScpCommandFactory; +import org.apache.sshd.server.Environment; +import org.apache.sshd.server.ExitCallback; import org.apache.sshd.server.SshServer; import org.apache.sshd.server.channel.ChannelSession; import org.apache.sshd.server.command.Command; import org.apache.sshd.server.command.CommandFactory; +import org.apache.sshd.server.command.CommandLifecycle; import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; import org.apache.sshd.server.shell.ProcessShellFactory; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.security.GeneralSecurityException; @@ -77,9 +81,47 @@ public DelegatingCommandFactory() { public Command createCommand(ChannelSession channel, String command) throws IOException { if (command.startsWith("scp")) { return scpCommandFactory.createCommand(channel, command); + } else if (command.startsWith("sbatch")) { + return new SbatchCommand(); } else { return shellCommandFactory.createCommand(channel, command); } } } + + public class SbatchCommand implements Command, CommandLifecycle { + private OutputStream out; + + public SbatchCommand() { + } + + @Override + public void setInputStream(java.io.InputStream in) { } + + @Override + public void setOutputStream(OutputStream out) { + this.out = out; + } + + @Override + public void setErrorStream(java.io.OutputStream err) { } + + @Override + public void start(ChannelSession channel, Environment env) throws IOException { + // Simulate successful sbatch job submission + out.write(("Submitted batch job 123456\n").getBytes()); + out.flush(); + out.close(); + } + + @Override + public void destroy(ChannelSession channel) throws Exception { + } + + @Override + public void setExitCallback(ExitCallback callback) { + // Exit callback handling (success) + callback.onExit(0); + } + } }