From f2f44b91196a8ab46c95cba67d3d230c5641298f Mon Sep 17 00:00:00 2001 From: ErmiasG Date: Tue, 7 May 2024 11:19:54 +0200 Subject: [PATCH] [HWORKS-859] run notebook conversion and git commands in kube job for ee (#1526) --- hopsworks-IT/src/test/ruby/spec/git_spec.rb | 1 + .../src/test/ruby/spec/spec_helper.rb | 2 +- .../api/featurestore/code/CodeBeanParam.java | 10 +- .../api/featurestore/code/CodeBuilder.java | 14 +- .../api/featurestore/code/CodeResource.java | 14 +- .../hops/hopsworks/api/git/GitResource.java | 18 +- .../hopsworks/api/jupyter/JupyterService.java | 27 +- .../hopsworks/common/dao/git/GitPaths.java | 13 +- .../dao/git/GitRepositoryRemotesFacade.java | 7 + .../common/dataset/DatasetController.java | 8 +- .../featurestore/code/CodeController.java | 11 +- .../git/AsynchronousGitCommandExecutor.java | 126 +-------- .../hopsworks/common/git/CommandExecutor.java | 29 ++ .../hopsworks/common/git/GitController.java | 36 ++- .../common/git/GitExecutionController.java | 74 +---- .../hopsworks/common/git/GitJWTManager.java | 6 + .../common/git/GitTimeoutCommandsMonitor.java | 26 +- .../common/git/LocalCommandExecutor.java | 257 ++++++++++++++++++ .../git/util/GitCommandOperationUtil.java | 63 +---- .../common/jobs/spark/SparkController.java | 3 +- .../common/jupyter/JupyterController.java | 100 +------ .../jupyter/LocalNoteBookConverter.java | 117 ++++++++ .../common/jupyter/NoteBookConverter.java | 37 +++ .../common/jupyter/NotebookConversion.java | 39 +++ .../common/system/job/SystemJobStatus.java | 52 ++++ .../hops/hopsworks/common/util/Settings.java | 2 +- 26 files changed, 667 insertions(+), 425 deletions(-) create mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/git/CommandExecutor.java create mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/git/LocalCommandExecutor.java create mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/LocalNoteBookConverter.java create mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/NoteBookConverter.java create mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/NotebookConversion.java create mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/system/job/SystemJobStatus.java diff --git a/hopsworks-IT/src/test/ruby/spec/git_spec.rb b/hopsworks-IT/src/test/ruby/spec/git_spec.rb index 90c0f91e16..7bfc0913c0 100644 --- a/hopsworks-IT/src/test/ruby/spec/git_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/git_spec.rb @@ -369,6 +369,7 @@ create_branch(@project[:id], repository_id, "test_branch") checkout_branch(@project[:id], repository_id, "test_branch") make_commit_in_repo(@project, repository_id) + #TODO fix this test it is trying to pull a branch that does not exist git_pull(@project[:id], repository_id, remote_name="origin", branch_name="test_branch") wait_for_git_operation_completed(@project[:id], repository_id, json_body[:id], "Success") end diff --git a/hopsworks-IT/src/test/ruby/spec/spec_helper.rb b/hopsworks-IT/src/test/ruby/spec/spec_helper.rb index 81d6fe0f97..cf599ed176 100644 --- a/hopsworks-IT/src/test/ruby/spec/spec_helper.rb +++ b/hopsworks-IT/src/test/ruby/spec/spec_helper.rb @@ -145,7 +145,7 @@ Airborne.configure do |config| config.base_url = "https://#{ENV['WEB_HOST']}:#{ENV['WEB_PORT']}" - config.timeout = 120 + config.timeout = 240 end def try_start_all_services diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBeanParam.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBeanParam.java index 23f29c4194..8e4d45f7ea 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBeanParam.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBeanParam.java @@ -17,7 +17,7 @@ package io.hops.hopsworks.api.featurestore.code; import com.google.common.base.Strings; -import io.hops.hopsworks.common.jupyter.JupyterController; +import io.hops.hopsworks.common.jupyter.NotebookConversion; import io.swagger.annotations.ApiParam; import javax.ws.rs.QueryParam; @@ -45,13 +45,13 @@ public class CodeBeanParam { @QueryParam("format") @ApiParam(value = "ex. format=html", allowableValues = "format=html") - private JupyterController.NotebookConversion format; + private NotebookConversion format; public CodeBeanParam( @QueryParam("sort_by") String sortBy, @QueryParam("filter_by") Set filter, @QueryParam("fields") String fields, - @QueryParam("format") JupyterController.NotebookConversion format) { + @QueryParam("format") NotebookConversion format) { this.sortBy = sortBy; this.sortBySet = getSortBy(sortBy); this.filterSet = filter; @@ -89,7 +89,7 @@ public Set getFieldSet() { return fieldSet; } - public JupyterController.NotebookConversion getFormat() { - return format == null ? JupyterController.NotebookConversion.HTML : format; + public NotebookConversion getFormat() { + return format == null ? NotebookConversion.HTML : format; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBuilder.java index 46cf4699c1..a9ad585d95 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBuilder.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBuilder.java @@ -20,7 +20,7 @@ import io.hops.hopsworks.common.dao.AbstractFacade; import io.hops.hopsworks.common.featurestore.code.CodeController; import io.hops.hopsworks.common.featurestore.code.FeaturestoreCodeFacade; -import io.hops.hopsworks.common.jupyter.JupyterController; +import io.hops.hopsworks.common.jupyter.NotebookConversion; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; @@ -104,7 +104,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user, Featuregroup featuregroup, FeaturestoreCode featurestoreCode, - JupyterController.NotebookConversion format) throws FeaturestoreException, ServiceException { + NotebookConversion format) throws FeaturestoreException, ServiceException { Path fullCodePath = new Path(codeController.getCodeDirFullPath(project, featuregroup), featurestoreCode.getFileName()); CodeDTO dto = new CodeDTO(); @@ -133,7 +133,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user, TrainingDataset trainingDataset, FeaturestoreCode featurestoreCode, - JupyterController.NotebookConversion format) throws FeaturestoreException, ServiceException { + NotebookConversion format) throws FeaturestoreException, ServiceException { Path fullCodePath = new Path(codeController.getCodeDirFullPath(project, trainingDataset), featurestoreCode.getFileName()); CodeDTO dto = new CodeDTO(); @@ -156,7 +156,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user, Featurestore featurestore, Featuregroup featuregroup, - JupyterController.NotebookConversion format) + NotebookConversion format) throws ServiceException, FeaturestoreException { CodeDTO dto = new CodeDTO(); dto.setHref(uri(uriInfo, project, featurestore, featuregroup)); @@ -180,7 +180,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user, Featurestore featurestore, TrainingDataset trainingDataset, - JupyterController.NotebookConversion format) + NotebookConversion format) throws ServiceException, FeaturestoreException { CodeDTO dto = new CodeDTO(); dto.setHref(uri(uriInfo, project, featurestore, trainingDataset)); @@ -205,7 +205,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user, Featuregroup featuregroup, Integer codeId, - JupyterController.NotebookConversion format) + NotebookConversion format) throws FeaturestoreException, ServiceException { FeaturestoreCode featurestoreCode = codeFacade.findFeaturestoreCodeById(featuregroup, codeId) .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CODE_NOT_FOUND, Level.FINE)); @@ -216,7 +216,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user, TrainingDataset trainingDataset, Integer codeId, - JupyterController.NotebookConversion format) + NotebookConversion format) throws FeaturestoreException, ServiceException { FeaturestoreCode featurestoreCode = codeFacade.findFeaturestoreCodeById(trainingDataset, codeId) .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CODE_NOT_FOUND, Level.FINE)); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java index 7df12f43f3..2d2fec4524 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java @@ -16,9 +16,9 @@ package io.hops.hopsworks.api.featurestore.code; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; @@ -26,7 +26,7 @@ import io.hops.hopsworks.common.featurestore.code.CodeController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; -import io.hops.hopsworks.common.jupyter.JupyterController; +import io.hops.hopsworks.common.jupyter.NotebookConversion; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.HopsSecurityException; @@ -49,15 +49,15 @@ import javax.enterprise.context.RequestScoped; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; -import javax.ws.rs.POST; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.Path; -import javax.ws.rs.Produces; import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import javax.ws.rs.core.Context; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; @@ -196,12 +196,12 @@ public Response post(@Context UriInfo uriInfo, codeDTO.getFeatureGroupCommitId(), codeDTO.getApplicationId(), featuregroup, entityId, databricksNotebook, databricksArchive, type); dto = codeBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.CODE), - project, user, featuregroup, featurestoreCode, JupyterController.NotebookConversion.HTML); + project, user, featuregroup, featurestoreCode, NotebookConversion.HTML); } else { FeaturestoreCode featurestoreCode = codeController.registerCode(project, user, codeDTO.getCommitTime(), codeDTO.getApplicationId(), trainingDataset, entityId, databricksNotebook, databricksArchive, type); dto = codeBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.CODE), - project, user, trainingDataset, featurestoreCode, JupyterController.NotebookConversion.HTML); + project, user, trainingDataset, featurestoreCode, NotebookConversion.HTML); } return Response.ok().entity(dto).build(); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java index 24386ea930..e2aa8c04cb 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java @@ -16,9 +16,9 @@ package io.hops.hopsworks.api.git; import com.google.common.base.Strings; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.git.branch.BranchBuilder; import io.hops.hopsworks.api.git.branch.BranchDTO; import io.hops.hopsworks.api.git.execution.ExecutionBeanParam; @@ -33,22 +33,21 @@ import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.common.git.BranchCommits; import io.hops.hopsworks.common.git.CloneCommandConfiguration; +import io.hops.hopsworks.common.git.GitBranchAction; +import io.hops.hopsworks.common.git.GitCommitDTO; import io.hops.hopsworks.common.git.GitController; import io.hops.hopsworks.common.git.GitRemotesAction; -import io.hops.hopsworks.common.git.GitBranchAction; import io.hops.hopsworks.common.git.GitRepositoryAction; -import io.hops.hopsworks.common.git.GitCommitDTO; -import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator; import io.hops.hopsworks.common.git.RepositoryActionCommandConfiguration; +import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator; import io.hops.hopsworks.exceptions.DatasetException; -import io.hops.hopsworks.exceptions.UserException; -import io.hops.hopsworks.persistence.entity.git.GitOpExecution; -import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.exceptions.GitOpException; import io.hops.hopsworks.exceptions.HopsSecurityException; import io.hops.hopsworks.jwt.annotation.JWTRequired; +import io.hops.hopsworks.persistence.entity.git.GitOpExecution; import io.hops.hopsworks.persistence.entity.git.GitRepository; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; @@ -165,9 +164,10 @@ public Response gitRepository(@PathParam("repositoryId") Integer repositoryId, @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response clone(CloneCommandConfiguration commandDTO, @Context SecurityContext sc, + @Context HttpServletRequest req, @Context UriInfo uriInfo, - @BeanParam ExecutionBeanParam executionBeanParam) - throws GitOpException, HopsSecurityException, IllegalArgumentException, UserException, DatasetException { + @BeanParam ExecutionBeanParam executionBeanParam) throws GitOpException, HopsSecurityException, + IllegalArgumentException, DatasetException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); GitOpExecution execution = gitController.clone(commandDTO, project, hopsworksUser); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java index 7638a59f3d..084c507a8c 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java @@ -47,25 +47,27 @@ import io.hops.hopsworks.common.dao.jupyter.JupyterSettingsFacade; import io.hops.hopsworks.common.dao.jupyter.config.JupyterDTO; import io.hops.hopsworks.common.dao.jupyter.config.JupyterFacade; -import io.hops.hopsworks.common.jupyter.NotebookDTO; -import io.hops.hopsworks.common.jupyter.JupyterManager; -import io.hops.hopsworks.common.jupyter.RemoteFSDriverType; import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.common.hdfs.HdfsUsersController; import io.hops.hopsworks.common.hdfs.Utils; import io.hops.hopsworks.common.jobs.spark.SparkController; import io.hops.hopsworks.common.jupyter.JupyterController; import io.hops.hopsworks.common.jupyter.JupyterJWTManager; +import io.hops.hopsworks.common.jupyter.JupyterManager; +import io.hops.hopsworks.common.jupyter.NotebookConversion; +import io.hops.hopsworks.common.jupyter.NotebookDTO; +import io.hops.hopsworks.common.jupyter.RemoteFSDriverType; import io.hops.hopsworks.common.livy.LivyController; import io.hops.hopsworks.common.livy.LivyMsg; +import io.hops.hopsworks.common.system.job.SystemJobStatus; import io.hops.hopsworks.common.user.UsersController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.HopsSecurityException; -import io.hops.hopsworks.exceptions.ProjectException; -import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.exceptions.JobException; import io.hops.hopsworks.exceptions.OpenSearchException; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration; import io.hops.hopsworks.persistence.entity.jupyter.JupyterMode; @@ -88,6 +90,7 @@ import javax.validation.constraints.Min; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; @@ -95,7 +98,6 @@ import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; -import javax.ws.rs.DefaultValue; import javax.ws.rs.core.Context; import javax.ws.rs.core.GenericEntity; import javax.ws.rs.core.MediaType; @@ -397,15 +399,18 @@ public Response stopNotebookServer(@Context HttpServletRequest req, public Response convertIPythonNotebook(@PathParam("path") String path, @Context HttpServletRequest req, @Context SecurityContext sc) throws ServiceException { - String ipynbPath = Utils.getProjectPath(this.project.getName()) + "/" + path; + if (path.startsWith("/")) { + path = path.replaceFirst("/", ""); + } + String ipynbPath = Utils.getProjectPath(this.project.getName()) + path; int extensionIndex = ipynbPath.lastIndexOf(".ipynb"); StringBuilder pathBuilder = new StringBuilder(ipynbPath.substring(0, extensionIndex)).append(".py"); String pyAppPath = pathBuilder.toString(); Users user = jWTHelper.getUserPrincipal(sc); - JupyterController.NotebookConversion conversionType = jupyterController - .getNotebookConversionType(ipynbPath, user, this.project); - jupyterController.convertIPythonNotebook(project, user, ipynbPath, pyAppPath, conversionType); - return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).build(); + NotebookConversion conversionType = jupyterController.getNotebookConversionType(ipynbPath, user, this.project); + SystemJobStatus status = + jupyterController.convertIPythonNotebook(project, user, ipynbPath, pyAppPath, conversionType); + return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(status).build(); } @POST diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitPaths.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitPaths.java index 4297043da3..398ca86125 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitPaths.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitPaths.java @@ -15,8 +15,6 @@ */ package io.hops.hopsworks.common.dao.git; -import java.io.File; - public class GitPaths { private String secret; private String gitPath; @@ -28,11 +26,12 @@ public class GitPaths { public GitPaths(String privateDir, String secretConfig) { this.gitPath = privateDir + secretConfig; - this.logDirPath = gitPath + File.separator + "git_logs"; - this.confDirPath = gitPath + File.separator + "conf"; - this.certificatesDirPath = gitPath + File.separator + "certificates"; - this.runDirPath = gitPath + File.separator + "run"; - this.tokenPath = gitPath + File.separator + "token"; + String gitPathWithSlash = this.gitPath.endsWith("/") ? this.gitPath : this.gitPath + "/"; + this.logDirPath = gitPathWithSlash + "git_logs"; + this.confDirPath = gitPathWithSlash + "conf"; + this.certificatesDirPath = gitPathWithSlash + "certificates"; + this.runDirPath = gitPathWithSlash + "run"; + this.tokenPath = gitPathWithSlash + "token"; this.secret = secretConfig; } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitRepositoryRemotesFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitRepositoryRemotesFacade.java index bb86f87bd4..b3408b3033 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitRepositoryRemotesFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitRepositoryRemotesFacade.java @@ -72,4 +72,11 @@ public void updateRepositoryRemotes(List remotes, GitReposi save(remote); } } + + public GitRepositoryRemote create(GitRepository repository, String name, String url) { + GitRepositoryRemote gitRepositoryRemote = new GitRepositoryRemote(repository, name, url); + em.persist(gitRepositoryRemote); + em.flush(); + return gitRepositoryRemote; + } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java index c2a0286267..d1e975c732 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java @@ -60,8 +60,10 @@ import io.hops.hopsworks.common.hdfs.command.HdfsCommandExecutionController; import io.hops.hopsworks.common.hdfs.inode.InodeController; import io.hops.hopsworks.common.jupyter.JupyterController; +import io.hops.hopsworks.common.jupyter.NotebookConversion; import io.hops.hopsworks.common.provenance.core.HopsFSProvenanceController; import io.hops.hopsworks.common.provenance.core.dto.ProvTypeDTO; +import io.hops.hopsworks.common.system.job.SystemJobStatus; import io.hops.hopsworks.common.util.HopsUtils; import io.hops.hopsworks.common.util.ProjectUtils; import io.hops.hopsworks.common.util.Settings; @@ -625,10 +627,10 @@ public FilePreviewDTO filePreview(Project project, Users user, Path fullPath, Fi throw new DatasetException(RESTCodes.DatasetErrorCode.IMAGE_SIZE_INVALID, Level.FINE); } } else if(fileExtension.equalsIgnoreCase("ipynb")) { - String html = jupyterController.convertIPythonNotebook(project, user, fullPath.toString(), "''", - JupyterController.NotebookConversion.HTML); + SystemJobStatus status = jupyterController.convertIPythonNotebook(project, user, fullPath.toString(), "''", + NotebookConversion.HTML); filePreviewDTO = new FilePreviewDTO(Settings.FILE_PREVIEW_HTML_TYPE, fileExtension.toLowerCase(), - html); + status.getLog()); } else { try (DataInputStream dis = new DataInputStream(is)) { int sizeThreshold = Settings.FILE_PREVIEW_TXT_SIZE_BYTES; //in bytes diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/code/CodeController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/code/CodeController.java index 0672b5e9a9..5b66d93384 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/code/CodeController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/code/CodeController.java @@ -25,6 +25,8 @@ import io.hops.hopsworks.common.hdfs.HdfsUsersController; import io.hops.hopsworks.common.hdfs.Utils; import io.hops.hopsworks.common.jupyter.JupyterController; +import io.hops.hopsworks.common.jupyter.NotebookConversion; +import io.hops.hopsworks.common.system.job.SystemJobStatus; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.ServiceException; @@ -91,7 +93,7 @@ public CodeContentFormat getContentFormat(String path) throws FeaturestoreExcept } public String readContent(Project project, Users user, String path, CodeContentFormat contentFormat, - JupyterController.NotebookConversion format) + NotebookConversion format) throws FeaturestoreException, ServiceException { //returns empty contents in the case of jar file switch (contentFormat) { @@ -114,9 +116,10 @@ public String readContent(Project project, Users user, String path, CodeContentF } private String readNotebookContent(Project project, Users user, String path, - JupyterController.NotebookConversion format) throws ServiceException { - if (format == JupyterController.NotebookConversion.HTML) { - return jupyterController.convertIPythonNotebook(project, user, path, "", format); + NotebookConversion format) throws ServiceException { + if (format == NotebookConversion.HTML) { + SystemJobStatus status = jupyterController.convertIPythonNotebook(project, user, path, "", format); + return status.getLog(); } //returns empty contents in the case of not supported notebookConversion return null; diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/AsynchronousGitCommandExecutor.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/AsynchronousGitCommandExecutor.java index 0d528955b4..e3ff4f8e1c 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/AsynchronousGitCommandExecutor.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/AsynchronousGitCommandExecutor.java @@ -15,139 +15,39 @@ */ package io.hops.hopsworks.common.git; -import com.google.common.base.Strings; -import io.hops.hopsworks.common.dao.git.GitRepositoryFacade; -import io.hops.hopsworks.common.git.util.GitCommandOperationUtil; -import io.hops.hopsworks.common.util.OSProcessExecutor; -import io.hops.hopsworks.common.util.ProcessDescriptor; -import io.hops.hopsworks.common.util.ProcessResult; -import io.hops.hopsworks.common.util.Settings; -import io.hops.hopsworks.common.util.templates.git.GitContainerLaunchScriptArgumentsTemplate; +import io.hops.hopsworks.exceptions.GitOpException; +import io.hops.hopsworks.exceptions.HopsSecurityException; import io.hops.hopsworks.persistence.entity.git.GitOpExecution; -import io.hops.hopsworks.common.dao.git.GitOpExecutionFacade; -import io.hops.hopsworks.common.dao.git.GitPaths; - -import io.hops.hopsworks.persistence.entity.git.GitRepository; -import io.hops.hopsworks.persistence.entity.git.config.GitOpExecutionState; +import io.hops.hopsworks.restutils.RESTCodes; import javax.ejb.Asynchronous; -import javax.ejb.EJB; import javax.ejb.LocalBean; import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.Optional; -import java.util.concurrent.TimeUnit; +import javax.inject.Inject; import java.util.logging.Level; -import java.util.logging.Logger; @Stateless @LocalBean public class AsynchronousGitCommandExecutor { - private static final Logger LOGGER = Logger.getLogger(AsynchronousGitCommandExecutor.class.getName()); - @EJB - private Settings settings; - @EJB - private OSProcessExecutor osProcessExecutor; - @EJB - private GitOpExecutionFacade gitOpExecutionFacade; - @EJB - private GitCommandOperationUtil gitCommandOperationUtil; - @EJB - private GitRepositoryFacade gitRepositoryFacade; + @Inject + private CommandExecutor commandExecutor; @Asynchronous @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) - public void execute(GitOpExecution gitOpExecution, GitPaths gitPaths) { - int maxTries = 5; - String pid = ""; - String gitCommand = gitOpExecution.getGitCommandConfiguration().getCommandType().getGitCommand(); - String prog = settings.getSudoersDir() + "/git.sh"; - String commandArgumentsFile = gitPaths.getConfDirPath() + File.separator - + GitContainerLaunchScriptArgumentsTemplate.FILE_NAME; - - while (maxTries > 0 && Strings.isNullOrEmpty(pid)) { - try { - ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder() - .addCommand("/usr/bin/sudo") - .addCommand(prog) - .addCommand("start") - .addCommand(commandArgumentsFile) - .redirectErrorStream(true) - .setCurrentWorkingDirectory(new File(gitPaths.getGitPath())) - .setWaitTimeout(60L, TimeUnit.SECONDS) - .build(); - - String pidFile = gitPaths.getRunDirPath() + "/git.pid"; - ProcessResult processResult = osProcessExecutor.execute(processDescriptor); - if (processResult.getExitCode() != 0) { - String errorMsg = - "Could not start git service to execute command " + gitCommand + " . " + "Exit code: " + - processResult.getExitCode() + " Error: stdout: " + processResult.getStdout() + " stderr: " + - processResult.getStderr(); - LOGGER.log(Level.SEVERE, errorMsg); - throw new IOException(errorMsg); - } else { - pid = com.google.common.io.Files.readFirstLine(new File(pidFile), Charset.defaultCharset()); - //Get the updated repository - Optional optional = gitRepositoryFacade.findById(gitOpExecution.getRepository().getId()); - gitRepositoryFacade.updateRepositoryCid(optional.get(), pid); - //gitOpExecutionFacade.updateState(gitOpExecution, GitOpExecutionState.SUBMITTED); - } - } catch (Exception ex) { - LOGGER.log(Level.SEVERE, "Problem executing shell script to start git command service", ex); - maxTries--; - } - } - if (Strings.isNullOrEmpty(pid)) { - updateExecutionStateToFail(gitOpExecution); + public void execute(GitOpExecution gitOpExecution, BasicAuthSecrets authSecrets) throws GitOpException, + HopsSecurityException { + try { + commandExecutor.execute(gitOpExecution, authSecrets); + } catch (Exception ex) { + throw new GitOpException(RESTCodes.GitOpErrorCode.GIT_OPERATION_ERROR, Level.SEVERE, ex.getMessage()); } } @Asynchronous @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) public void cancelGitExecution(GitOpExecution execution, String message) { - if (execution.getState().isFinalState()) { - return; - } - try { - gitOpExecutionFacade.updateState(execution, GitOpExecutionState.CANCELLED, message); - GitRepository repository = execution.getRepository(); - int maxTries = 10; // wait time if the container is not launched - while (maxTries > 0 && org.opensearch.common.Strings.isNullOrEmpty(repository.getCid())) { - Optional optional = gitRepositoryFacade.findById(repository.getId()); - if (optional.isPresent()) { - repository = optional.get(); - } - try { - Thread.sleep(100); - } catch (InterruptedException e) { - LOGGER.log(Level.INFO, "Interrupted while waiting for the git container to start"); - } - maxTries--; - } - gitCommandOperationUtil.shutdownCommandService(repository, execution); - } catch (Exception e) { - LOGGER.log(Level.INFO, "Error when cancelling git execution with ID: " + execution.getId(), e); - } - } - - private void updateExecutionStateToFail(GitOpExecution gitOpExecution) { - // Get latest execution object - Optional optional = gitOpExecutionFacade.findByIdAndRepository(gitOpExecution.getRepository(), - gitOpExecution.getId()); - gitOpExecution = optional.get(); - if (gitOpExecution.getState() == GitOpExecutionState.CANCELLED) { - return; - } - gitCommandOperationUtil.cleanUp(gitOpExecution.getRepository().getProject(), - gitOpExecution.getUser(), gitOpExecution.getConfigSecret()); - gitOpExecutionFacade.updateState(gitOpExecution, GitOpExecutionState.FAILED, "Could not launch " + - "container to execute git command."); - gitRepositoryFacade.updateRepositoryCid(gitOpExecution.getRepository(), null); - LOGGER.log(Level.SEVERE, "Problem executing shell script to start git command service." ); + commandExecutor.cancelGitExecution(execution, message); } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/CommandExecutor.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/CommandExecutor.java new file mode 100644 index 0000000000..0087d4eaac --- /dev/null +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/CommandExecutor.java @@ -0,0 +1,29 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.common.git; + +import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; +import io.hops.hopsworks.exceptions.GitOpException; +import io.hops.hopsworks.persistence.entity.git.GitOpExecution; + +import java.io.IOException; + +public interface CommandExecutor { + void execute(GitOpExecution execution, BasicAuthSecrets authSecrets) throws GitOpException, IOException, + ServiceDiscoveryException; + void cancelGitExecution(GitOpExecution execution, String message); + void monitorCommands(GitOpExecution execution, String localMemberIp, boolean amIPrimary, long waitTime); +} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitController.java index fc06d90dbd..c06fbd686c 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitController.java @@ -16,10 +16,9 @@ package io.hops.hopsworks.common.git; import com.google.common.base.Strings; - -import io.hops.hopsworks.common.dao.git.GitRepositoryRemotesFacade; -import io.hops.hopsworks.common.dao.git.GitRepositoryFacade; import io.hops.hopsworks.common.dao.git.GitCommitsFacade; +import io.hops.hopsworks.common.dao.git.GitRepositoryFacade; +import io.hops.hopsworks.common.dao.git.GitRepositoryRemotesFacade; import io.hops.hopsworks.common.dataset.DatasetController; import io.hops.hopsworks.common.git.util.Constants; import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator; @@ -29,12 +28,10 @@ import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.GitOpException; import io.hops.hopsworks.exceptions.HopsSecurityException; - import io.hops.hopsworks.persistence.entity.git.CommitterSignature; +import io.hops.hopsworks.persistence.entity.git.GitCommit; import io.hops.hopsworks.persistence.entity.git.GitOpExecution; import io.hops.hopsworks.persistence.entity.git.GitRepository; -import io.hops.hopsworks.persistence.entity.git.GitCommit; -import io.hops.hopsworks.persistence.entity.git.GitRepositoryRemote; import io.hops.hopsworks.persistence.entity.git.config.GitCommandConfiguration; import io.hops.hopsworks.persistence.entity.git.config.GitCommandType; import io.hops.hopsworks.persistence.entity.project.Project; @@ -56,7 +53,6 @@ @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) public class GitController { private static final Logger LOGGER = Logger.getLogger(GitController.class.getName()); - private static final String REPO_NAME_DELIMETER = "_"; @EJB private DistributedFsService dfsService; @@ -100,15 +96,15 @@ public GitOpExecution clone(CloneCommandConfiguration cloneConfigurationDTO, Pro GitRepository repository = gitRepositoryFacade.create(project, cloneConfigurationDTO.getProvider(), hopsworksUser, repositoryName, fullRepoDirPath); //Create the default remote - gitRepositoryRemotesFacade.save(new GitRepositoryRemote(repository, Constants.REPOSITORY_DEFAULT_REMOTE_NAME, - cloneConfigurationDTO.getUrl())); + gitRepositoryRemotesFacade.create(repository, Constants.REPOSITORY_DEFAULT_REMOTE_NAME, + cloneConfigurationDTO.getUrl()); GitCommandConfiguration configuration = new GitCommandConfigurationBuilder().setCommandType(GitCommandType.CLONE) .setUrl(cloneConfigurationDTO.getUrl()) .setProvider(cloneConfigurationDTO.getProvider()) .setPath(fullRepoDirPath) .setBranchName(cloneConfigurationDTO.getBranch()) .build(); - return executionController.createExecution(configuration, project, hopsworksUser, repository); + return executionController.createExecution(configuration, hopsworksUser, repository); } public GitOpExecution executeRepositoryAction(RepositoryActionCommandConfiguration configurationDTO, @@ -143,7 +139,7 @@ public GitOpExecution commit(CommitCommandConfiguration commitConfigurationDTO, .setCommitter(new CommitterSignature(userFullName, hopsworksUser.getEmail())) .setPath(repository.getRepositoryPath()) .build(); - return executionController.createExecution(commandConfiguration, project, hopsworksUser, repository); + return executionController.createExecution(commandConfiguration, hopsworksUser, repository); } public GitOpExecution executeBranchAction(GitBranchAction action, Project project, Users hopsworksUser, @@ -162,7 +158,7 @@ public GitOpExecution executeBranchAction(GitBranchAction action, Project projec } builder.setCommandType(GitCommandType.CREATE_BRANCH); builder.setCheckout(action == GitBranchAction.CREATE_CHECKOUT); - return executionController.createExecution(builder.build(), project, hopsworksUser, repository); + return executionController.createExecution(builder.build(), hopsworksUser, repository); case DELETE: if (Strings.isNullOrEmpty(branchName)) { throw new GitOpException(RESTCodes.GitOpErrorCode.INVALID_BRANCH_NAME, Level.WARNING, "Branch name is empty" + @@ -170,7 +166,7 @@ public GitOpExecution executeBranchAction(GitBranchAction action, Project projec } builder.setCommandType(GitCommandType.DELETE_BRANCH); builder.setDeleteOnRemote(false); - return executionController.createExecution(builder.build(), project, hopsworksUser, repository); + return executionController.createExecution(builder.build(), hopsworksUser, repository); case CHECKOUT: case CHECKOUT_FORCE: if (Strings.isNullOrEmpty(branchName) && Strings.isNullOrEmpty(commit)) { @@ -183,7 +179,7 @@ public GitOpExecution executeBranchAction(GitBranchAction action, Project projec builder.setCommandType(GitCommandType.CHECKOUT); builder.setCommit(commit); builder.setForce(action == GitBranchAction.CHECKOUT_FORCE); - return executionController.createExecution(builder.build(), project, hopsworksUser, repository); + return executionController.createExecution(builder.build(), hopsworksUser, repository); default: throw new IllegalArgumentException(RESTCodes.GitOpErrorCode.INVALID_BRANCH_ACTION.getMessage()); } @@ -206,10 +202,10 @@ public GitOpExecution addOrDeleteRemote(GitRemotesAction action, Project project } builder.setCommandType(GitCommandType.ADD_REMOTE); builder.setRemoteUrl(remoteUrl); - return executionController.createExecution(builder.build(), project, hopsworksUser, repository); + return executionController.createExecution(builder.build(), hopsworksUser, repository); case DELETE: builder.setCommandType(GitCommandType.DELETE_REMOTE); - return executionController.createExecution(builder.build(), project, hopsworksUser, repository); + return executionController.createExecution(builder.build(), hopsworksUser, repository); default: throw new IllegalArgumentException(RESTCodes.GitOpErrorCode.INVALID_REMOTES_ACTION.getMessage()); } @@ -228,7 +224,7 @@ public GitOpExecution push(PushCommandConfiguration configurationDTO, Project pr .setForce(configurationDTO.isForce()) .setPath(repository.getRepositoryPath()) .build(); - return executionController.createExecution(pushCommandConfiguration, project, hopsworksUser, repository); + return executionController.createExecution(pushCommandConfiguration, hopsworksUser, repository); } public GitOpExecution pull(PullCommandConfiguration configDTO, Project project, Users hopsworksUser, @@ -245,7 +241,7 @@ public GitOpExecution pull(PullCommandConfiguration configDTO, Project project, .setPath(repository.getRepositoryPath()) .setCommitter(new CommitterSignature(userFullName, hopsworksUser.getEmail())) .build(); - return executionController.createExecution(pullCommandConfiguration, project, hopsworksUser, repository); + return executionController.createExecution(pullCommandConfiguration, hopsworksUser, repository); } public GitOpExecution status(Project project, Users hopsworksUser, Integer repositoryId) @@ -256,7 +252,7 @@ public GitOpExecution status(Project project, Users hopsworksUser, Integer repos .setCommandType(GitCommandType.STATUS) .setPath(repository.getRepositoryPath()) .build(); - return executionController.createExecution(statusCommandConfig, project, hopsworksUser, repository); + return executionController.createExecution(statusCommandConfig, hopsworksUser, repository); } public GitOpExecution fileCheckout(Project project, Users hopsworksUser, Integer repositoryId, @@ -271,7 +267,7 @@ public GitOpExecution fileCheckout(Project project, Users hopsworksUser, Integer .setPath(repository.getRepositoryPath()) .setFiles(filePaths) .build(); - return executionController.createExecution(fileCheckoutConfiguration, project, hopsworksUser, repository); + return executionController.createExecution(fileCheckoutConfiguration, hopsworksUser, repository); } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitExecutionController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitExecutionController.java index 883cfbbf44..3b9a9f0028 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitExecutionController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitExecutionController.java @@ -16,19 +16,12 @@ package io.hops.hopsworks.common.git; import com.google.common.base.Strings; -import io.hops.hopsworks.common.dao.git.GitRepositoryRemotesFacade; +import io.hops.hopsworks.common.dao.git.GitCommitsFacade; import io.hops.hopsworks.common.dao.git.GitOpExecutionFacade; import io.hops.hopsworks.common.dao.git.GitRepositoryFacade; -import io.hops.hopsworks.common.dao.git.GitCommitsFacade; -import io.hops.hopsworks.common.dao.git.GitPaths; +import io.hops.hopsworks.common.dao.git.GitRepositoryRemotesFacade; import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator; import io.hops.hopsworks.common.git.util.GitCommandOperationUtil; -import io.hops.hopsworks.common.git.util.GitContainerArgumentsWriter; -import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; -import io.hops.hopsworks.common.hdfs.DistributedFsService; -import io.hops.hopsworks.common.hdfs.HdfsUsersController; -import io.hops.hopsworks.common.security.CertificateMaterializer; -import io.hops.hopsworks.common.util.HopsUtils; import io.hops.hopsworks.common.util.PayaraClusterManager; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.GitOpException; @@ -47,7 +40,6 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; -import java.io.IOException; import java.util.Optional; import java.util.concurrent.ThreadLocalRandom; import java.util.logging.Level; @@ -60,12 +52,6 @@ public class GitExecutionController { @EJB private Settings settings; @EJB - private CertificateMaterializer certificateMaterializer; - @EJB - private GitJWTManager gitJWTManager; - @EJB - private DistributedFsService dfsService; - @EJB private GitOpExecutionFacade gitOpExecutionFacade; @EJB private AsynchronousGitCommandExecutor gitCommandExecutor; @@ -76,33 +62,26 @@ public class GitExecutionController { @EJB private GitCommitsFacade gitCommitsFacade; @EJB - private HdfsUsersController hdfsUsersController; - @EJB private GitRepositoryRemotesFacade gitRepositoryRemotesFacade; @EJB private GitCommandConfigurationValidator commandConfigurationValidator; @EJB - private GitContainerArgumentsWriter argumentsWriter; - @EJB private PayaraClusterManager payaraClusterManager; /** * initializes the execution of all git commands * * @param gitCommandConfiguration - * @param project * @param hopsworksUser * @param repository * @return * @throws HopsSecurityException * @throws GitOpException */ - public GitOpExecution createExecution(GitCommandConfiguration gitCommandConfiguration, Project project, - Users hopsworksUser, GitRepository repository) - throws HopsSecurityException, GitOpException { + public GitOpExecution createExecution(GitCommandConfiguration gitCommandConfiguration, Users hopsworksUser, + GitRepository repository) throws HopsSecurityException, GitOpException { commandConfigurationValidator.verifyReadOnly(gitCommandConfiguration.getCommandType(), repository); gitCommandConfiguration.setReadOnly(settings.getEnableGitReadOnlyRepositories()); - String hdfsUsername = hdfsUsersController.getHdfsUserName(project, hopsworksUser); //set the provider to validate secrets for some commands gitCommandConfiguration.setProvider(repository.getGitProvider()); BasicAuthSecrets authSecrets = gitCommandOperationUtil.getAuthenticationSecrets(hopsworksUser, @@ -111,48 +90,11 @@ public GitOpExecution createExecution(GitCommandConfiguration gitCommandConfigur String configSecret = DigestUtils.sha256Hex(Integer.toString(ThreadLocalRandom.current().nextInt())); lockRepository(repository.getId()); GitOpExecution gitOpExecution = null; - DistributedFileSystemOps udfso = null; String localMemberIp = payaraClusterManager.getLocalIp(); - try { - udfso = dfsService.getDfsOps(hdfsUsername); - GitPaths gitPaths = prepareCommandExecution(project, hopsworksUser, udfso, configSecret); - gitOpExecution = gitOpExecutionFacade.create(gitCommandConfiguration, hopsworksUser, repository, configSecret, - localMemberIp); - argumentsWriter.createArgumentFile(gitOpExecution, gitPaths, authSecrets); - gitCommandExecutor.execute(gitOpExecution, gitPaths); - return gitOpExecution; - } catch (Exception ex) { - gitRepositoryFacade.updateRepositoryCid(repository, null); - gitCommandOperationUtil.cleanUp(project, hopsworksUser, configSecret); - if (ex instanceof IOException) { - throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_MATERIALIZATION_ERROR, Level.SEVERE, - ex.getMessage(), null, ex); - } - throw new GitOpException(RESTCodes.GitOpErrorCode.GIT_OPERATION_ERROR, Level.SEVERE, ex.getMessage()); - } finally { - if (udfso != null) { - dfsService.closeDfsClient(udfso); - } - } - } - - /** - * Generate the git paths, materialize user certificates, and jwt - * - * @param project - * @param hopsworksUser - * @param dfso - * @throws GitOpException - * @throws IOException - */ - public GitPaths prepareCommandExecution(Project project, Users hopsworksUser, DistributedFileSystemOps dfso, - String configSecret) throws GitOpException, IOException { - GitPaths gitPaths = new GitPaths(settings.getStagingDir() + settings.PRIVATE_DIRS, configSecret); - gitCommandOperationUtil.generatePaths(gitPaths); - HopsUtils.materializeCertificatesForUserCustomDir(project.getName(), hopsworksUser.getUsername(), - settings.getHdfsTmpCertDir(), dfso, certificateMaterializer, settings, gitPaths.getCertificatesDirPath()); - gitJWTManager.materializeJWT(hopsworksUser, gitPaths.getTokenPath()); - return gitPaths; + gitOpExecution = gitOpExecutionFacade.create(gitCommandConfiguration, hopsworksUser, repository, configSecret, + localMemberIp); + gitCommandExecutor.execute(gitOpExecution, authSecrets); // this is async so we cant do cleanup here + return gitOpExecution; } private synchronized void lockRepository(Integer repositoryId) throws GitOpException { diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitJWTManager.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitJWTManager.java index 5653436ecb..605d6f8c38 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitJWTManager.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitJWTManager.java @@ -67,6 +67,12 @@ public void materializeJWT(Users user, String tokenPath) throws GitOpException { } } + @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) + public String createGitContainerJWT(Users user) throws GitOpException { + LocalDateTime expirationDate = LocalDateTime.now().plus(settings.getGitJwtExpMs(), ChronoUnit.MILLIS); + return createTokenForGitContainer(user, expirationDate); + } + private String createTokenForGitContainer(Users user, LocalDateTime expirationDate) throws GitOpException { String[] userRoles = userUtilities.getUserRoles(user).toArray(new String[1]); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitTimeoutCommandsMonitor.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitTimeoutCommandsMonitor.java index f2d00db81f..75ae3891dc 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitTimeoutCommandsMonitor.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/GitTimeoutCommandsMonitor.java @@ -17,12 +17,9 @@ import io.hops.hopsworks.common.dao.git.GitOpExecutionFacade; import io.hops.hopsworks.common.dao.git.GitRepositoryFacade; -import io.hops.hopsworks.common.git.util.GitCommandOperationUtil; import io.hops.hopsworks.common.util.PayaraClusterManager; -import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.persistence.entity.git.GitOpExecution; import io.hops.hopsworks.persistence.entity.git.GitRepository; -import io.hops.hopsworks.persistence.entity.git.config.GitOpExecutionState; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -35,6 +32,7 @@ import javax.ejb.Timer; import javax.ejb.TimerConfig; import javax.ejb.TimerService; +import javax.inject.Inject; import java.util.Collection; import java.util.Optional; import java.util.logging.Level; @@ -46,7 +44,7 @@ public class GitTimeoutCommandsMonitor { private static final Logger LOGGER = Logger.getLogger(GitTimeoutCommandsMonitor.class.getName()); //Time to wait to kill the container after it has reached timeout. 90 seconds - private static final Integer BONUS_TIME = 1000 * 90; + private static final long BONUS_TIME = 1000 * 90; //Time to wait after the repository is locked and the execution is created. 30s; private static final Integer WAIT_TIME_BEFORE_EXECUTION_OBJECT_CREATION = 3000; @@ -55,11 +53,9 @@ public class GitTimeoutCommandsMonitor { @EJB private GitRepositoryFacade gitRepositoryFacade; @EJB - private GitCommandOperationUtil gitCommandOperationUtil; - @EJB - private Settings settings; - @EJB private PayaraClusterManager payaraClusterManager; + @Inject + private CommandExecutor commandExecutor; @Resource private TimerService timerService; @@ -86,23 +82,15 @@ public void destroy() { public void gitCommandMonitor(Timer timer) { //Should run on all LOGGER.log(Level.FINE, "Running GitTimeoutCommandsMonitor"); + boolean amIPrimary = payaraClusterManager.amIThePrimary(); Collection repositories = gitRepositoryFacade.findAllWithOngoingOperations(); for (GitRepository repository : repositories) { Optional optional = gitOpExecutionFacade.findRunningInRepository(repository); if (optional.isPresent()) { GitOpExecution execution = optional.get(); //we need all GitOpExecution to check for repository with a pid but no execution. So check host here - if (execution.getHostname().equals(localMemberIp)) { - long timeElapsed = System.currentTimeMillis() - execution.getExecutionStart(); - if (timeElapsed > (settings.getGitJwtExpMs() + BONUS_TIME)) { - //kill this container - LOGGER.log(Level.INFO, "Killing git execution with Id + [{0}] with state {1}", - new Object[]{execution.getId(), execution.getState().toString()}); - gitOpExecutionFacade.updateState(execution, GitOpExecutionState.TIMEDOUT, "Timeout"); - gitCommandOperationUtil.shutdownCommandService(repository, execution); - } - } - } else if (payaraClusterManager.amIThePrimary()) { + commandExecutor.monitorCommands(execution, localMemberIp, amIPrimary, BONUS_TIME); + } else if (amIPrimary && repository.getCid() != null) { //A repository with a pid but no execution object try { long executionStart = Long.parseLong(repository.getCid()); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/LocalCommandExecutor.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/LocalCommandExecutor.java new file mode 100644 index 0000000000..b08e0c5c48 --- /dev/null +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/LocalCommandExecutor.java @@ -0,0 +1,257 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.common.git; + +import com.google.common.base.Strings; +import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; +import io.hops.hopsworks.common.dao.git.GitOpExecutionFacade; +import io.hops.hopsworks.common.dao.git.GitPaths; +import io.hops.hopsworks.common.dao.git.GitRepositoryFacade; +import io.hops.hopsworks.common.git.util.GitCommandOperationUtil; +import io.hops.hopsworks.common.git.util.GitContainerArgumentsWriter; +import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; +import io.hops.hopsworks.common.hdfs.DistributedFsService; +import io.hops.hopsworks.common.hdfs.HdfsUsersController; +import io.hops.hopsworks.common.integrations.LocalhostStereotype; +import io.hops.hopsworks.common.security.CertificateMaterializer; +import io.hops.hopsworks.common.util.HopsUtils; +import io.hops.hopsworks.common.util.OSProcessExecutor; +import io.hops.hopsworks.common.util.ProcessDescriptor; +import io.hops.hopsworks.common.util.ProcessResult; +import io.hops.hopsworks.common.util.Settings; +import io.hops.hopsworks.common.util.templates.git.GitContainerLaunchScriptArgumentsTemplate; +import io.hops.hopsworks.exceptions.GitOpException; +import io.hops.hopsworks.persistence.entity.git.GitOpExecution; +import io.hops.hopsworks.persistence.entity.git.GitRepository; +import io.hops.hopsworks.persistence.entity.git.config.GitOpExecutionState; + +import javax.ejb.EJB; +import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Stateless +@LocalhostStereotype +@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) +public class LocalCommandExecutor implements CommandExecutor { + private static final Logger LOGGER = Logger.getLogger(LocalCommandExecutor.class.getName()); + @EJB + private Settings settings; + @EJB + private OSProcessExecutor osProcessExecutor; + @EJB + private GitOpExecutionFacade gitOpExecutionFacade; + @EJB + private GitCommandOperationUtil gitCommandOperationUtil; + @EJB + private GitRepositoryFacade gitRepositoryFacade; + @EJB + private HdfsUsersController hdfsUsersController; + @EJB + private GitContainerArgumentsWriter argumentsWriter; + @EJB + private CertificateMaterializer certificateMaterializer; + @EJB + private GitJWTManager gitJWTManager; + @EJB + private DistributedFsService dfsService; + + @Override + public void execute(GitOpExecution gitOpExecution, BasicAuthSecrets authSecrets) throws GitOpException, IOException, + ServiceDiscoveryException { + int maxTries = 5; + String pid = ""; + String gitCommand = gitOpExecution.getGitCommandConfiguration().getCommandType().getGitCommand(); + String prog = settings.getSudoersDir() + "/git.sh"; + + GitPaths gitPaths = prepareCommandExecution(gitOpExecution); + argumentsWriter.createArgumentFile(gitOpExecution, gitPaths, authSecrets); + String commandArgumentsFile = gitPaths.getConfDirPath() + File.separator + + GitContainerLaunchScriptArgumentsTemplate.FILE_NAME; + + while (maxTries > 0 && Strings.isNullOrEmpty(pid)) { + try { + ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder() + .addCommand("/usr/bin/sudo") + .addCommand(prog) + .addCommand("start") + .addCommand(commandArgumentsFile) + .redirectErrorStream(true) + .setCurrentWorkingDirectory(new File(gitPaths.getGitPath())) + .setWaitTimeout(60L, TimeUnit.SECONDS) + .build(); + + String pidFile = gitPaths.getRunDirPath() + "/git.pid"; + ProcessResult processResult = osProcessExecutor.execute(processDescriptor); + if (processResult.getExitCode() != 0) { + String errorMsg = + "Could not start git service to execute command " + gitCommand + " . " + "Exit code: " + + processResult.getExitCode() + " Error: stdout: " + processResult.getStdout() + " stderr: " + + processResult.getStderr(); + LOGGER.log(Level.SEVERE, errorMsg); + throw new IOException(errorMsg); + } else { + pid = com.google.common.io.Files.readFirstLine(new File(pidFile), Charset.defaultCharset()); + //Get the updated repository + Optional optional = gitRepositoryFacade.findById(gitOpExecution.getRepository().getId()); + gitRepositoryFacade.updateRepositoryCid(optional.get(), pid); + //gitOpExecutionFacade.updateState(gitOpExecution, GitOpExecutionState.SUBMITTED); + } + } catch (Exception ex) { + LOGGER.log(Level.SEVERE, "Problem executing shell script to start git command service", ex); + maxTries--; + } + } + if (Strings.isNullOrEmpty(pid)) { + updateExecutionStateToFail(gitOpExecution); + } + } + + @Override + public void cancelGitExecution(GitOpExecution execution, String message) { + if (execution.getState().isFinalState()) { + return; + } + try { + gitOpExecutionFacade.updateState(execution, GitOpExecutionState.CANCELLED, message); + GitRepository repository = execution.getRepository(); + int maxTries = 10; // wait time if the container is not launched + while (maxTries > 0 && Strings.isNullOrEmpty(repository.getCid())) { + Optional optional = gitRepositoryFacade.findById(repository.getId()); + if (optional.isPresent()) { + repository = optional.get(); + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + LOGGER.log(Level.INFO, "Interrupted while waiting for the git container to start"); + } + maxTries--; + } + shutdownCommandService(repository, execution); + } catch (Exception e) { + LOGGER.log(Level.INFO, "Error when cancelling git execution with ID: " + execution.getId(), e); + } + } + + /** + * Generate the git paths, materialize user certificates, and jwt + * + * @throws GitOpException + * @throws IOException + */ + private GitPaths prepareCommandExecution(GitOpExecution gitOpExecution) throws GitOpException, IOException { + GitPaths gitPaths = + new GitPaths(settings.getStagingDir() + Settings.PRIVATE_DIRS, gitOpExecution.getConfigSecret()); + gitCommandOperationUtil.generatePaths(gitPaths); + DistributedFileSystemOps udfso = null; + String hdfsUsername = hdfsUsersController.getHdfsUserName(gitOpExecution.getRepository().getProject(), + gitOpExecution.getUser()); + try { + udfso = dfsService.getDfsOps(hdfsUsername); + HopsUtils.materializeCertificatesForUserCustomDir(gitOpExecution.getRepository().getProject().getName(), + gitOpExecution.getUser().getUsername(), settings.getHdfsTmpCertDir(), udfso, certificateMaterializer, settings, + gitPaths.getCertificatesDirPath()); + gitJWTManager.materializeJWT(gitOpExecution.getUser(), gitPaths.getTokenPath()); + } finally { + if (udfso != null) { + dfsService.closeDfsClient(udfso); + } + } + return gitPaths; + } + + private void shutdownCommandService(GitRepository repository, GitOpExecution execution) { + String cid = repository.getCid(); + try { + gitRepositoryFacade.updateRepositoryCid(repository, null); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Failed to update repository pid", e); + } + killGitContainer(execution, cid); + gitCommandOperationUtil.cleanUp(repository.getProject(), execution.getUser(), execution.getConfigSecret()); + } + + @Override + public void monitorCommands(GitOpExecution execution, String localMemberIp, boolean amIPrimary, long waitTime) { + if (execution.getHostname().equals(localMemberIp)) { + long timeElapsed = System.currentTimeMillis() - execution.getExecutionStart(); + if (timeElapsed > (settings.getGitJwtExpMs() + waitTime)) { + //kill this container + LOGGER.log(Level.INFO, "Killing git execution with Id + [{0}] with state {1}", + new Object[]{execution.getId(), execution.getState().toString()}); + gitOpExecutionFacade.updateState(execution, GitOpExecutionState.TIMEDOUT, "Timeout"); + shutdownCommandService(execution.getRepository(), execution); + } + } + } + + private void killGitContainer(GitOpExecution execution, String containerId) { + if (Strings.isNullOrEmpty(containerId)) { + return; + } + String gitHomePath = gitCommandOperationUtil.getGitHome(execution.getConfigSecret()); + String hdfsUsername = hdfsUsersController.getHdfsUserName(execution.getRepository().getProject(), + execution.getUser()); + String prog = settings.getSudoersDir() + "/git.sh"; + int exitValue = 0; + ProcessDescriptor.Builder pdBuilder = new ProcessDescriptor.Builder() + .addCommand("/usr/bin/sudo") + .addCommand(prog) + .addCommand("kill") + .addCommand(gitHomePath) + .addCommand(containerId) + .addCommand(hdfsUsername) + .redirectErrorStream(true) + .setWaitTimeout(10L, TimeUnit.SECONDS); + try { + ProcessResult processResult = osProcessExecutor.execute(pdBuilder.build()); + LOGGER.log(Level.FINE, processResult.getStdout()); + exitValue = processResult.getExitCode(); + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, + "Failed to shutdown git container executing command for user " + hdfsUsername, ex); + } + if (exitValue != 0) { + LOGGER.log(Level.SEVERE, + "Exited with " + exitValue + "Failed to shutdown git container executing command for user " + + hdfsUsername); + } + } + + private void updateExecutionStateToFail(GitOpExecution gitOpExecution) { + // Get latest execution object + Optional optional = gitOpExecutionFacade.findByIdAndRepository(gitOpExecution.getRepository(), + gitOpExecution.getId()); + gitOpExecution = optional.get(); + if (gitOpExecution.getState() == GitOpExecutionState.CANCELLED) { + return; + } + gitCommandOperationUtil.cleanUp(gitOpExecution.getRepository().getProject(), + gitOpExecution.getUser(), gitOpExecution.getConfigSecret()); + gitOpExecutionFacade.updateState(gitOpExecution, GitOpExecutionState.FAILED, "Could not launch " + + "container to execute git command."); + gitRepositoryFacade.updateRepositoryCid(gitOpExecution.getRepository(), null); + LOGGER.log(Level.SEVERE, "Problem executing shell script to start git command service."); + } +} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/util/GitCommandOperationUtil.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/util/GitCommandOperationUtil.java index 8bfc1e1c08..f6b3586e83 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/util/GitCommandOperationUtil.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/git/util/GitCommandOperationUtil.java @@ -15,17 +15,11 @@ */ package io.hops.hopsworks.common.git.util; -import io.hops.hopsworks.common.dao.git.GitOpExecutionFacade; import io.hops.hopsworks.common.dao.git.GitPaths; -import io.hops.hopsworks.common.dao.git.GitRepositoryFacade; import io.hops.hopsworks.common.git.BasicAuthSecrets; -import io.hops.hopsworks.common.hdfs.HdfsUsersController; import io.hops.hopsworks.common.security.CertificateMaterializer; import io.hops.hopsworks.common.security.secrets.SecretsController; import io.hops.hopsworks.common.util.HopsUtils; -import io.hops.hopsworks.common.util.OSProcessExecutor; -import io.hops.hopsworks.common.util.ProcessDescriptor; -import io.hops.hopsworks.common.util.ProcessResult; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.GitOpException; import io.hops.hopsworks.exceptions.UserException; @@ -41,7 +35,6 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.opensearch.common.Strings; import javax.ejb.EJB; import javax.ejb.Stateless; @@ -56,7 +49,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -72,20 +64,13 @@ public class GitCommandOperationUtil { // Will work if the repository name will not contain this pattern private static final String LOG_ATTRIBUTES_SEPERATOR = "--s--"; - @EJB - private GitRepositoryFacade gitRepositoryFacade; - @EJB - private GitOpExecutionFacade executionFacade; @EJB private CertificateMaterializer certificateMaterializer; @EJB private Settings settings; @EJB - private OSProcessExecutor osProcessExecutor; - @EJB private SecretsController secretsController; - @EJB - private HdfsUsersController hdfsUsersController; + public void cleanUp(Project project, Users user, String configSecret) { String gitHomePath = getGitHome(configSecret); @@ -236,51 +221,7 @@ public List convertToRemote(GitRepository repository, Strin return remotes; } - public void shutdownCommandService(GitRepository repository, GitOpExecution execution) { - String cid = repository.getCid(); - try { - gitRepositoryFacade.updateRepositoryCid(repository, null); - } catch (Exception e) { - LOGGER.log(Level.SEVERE, "Failed to update repository pid", e); - } - killGitContainer(execution, cid); - cleanUp(repository.getProject(), execution.getUser(), getGitHome(execution.getConfigSecret())); - } - - public void killGitContainer(GitOpExecution execution, String containerId) { - if (Strings.isNullOrEmpty(containerId)) { - return; - } - String gitHomePath = getGitHome(execution.getConfigSecret()); - String hdfsUsername = hdfsUsersController.getHdfsUserName(execution.getRepository().getProject(), - execution.getUser()); - String prog = settings.getSudoersDir() + "/git.sh"; - int exitValue = 0; - ProcessDescriptor.Builder pdBuilder = new ProcessDescriptor.Builder() - .addCommand("/usr/bin/sudo") - .addCommand(prog) - .addCommand("kill") - .addCommand(gitHomePath) - .addCommand(containerId) - .addCommand(hdfsUsername) - .redirectErrorStream(true) - .setWaitTimeout(10L, TimeUnit.SECONDS); - try { - ProcessResult processResult = osProcessExecutor.execute(pdBuilder.build()); - LOGGER.log(Level.FINE, processResult.getStdout()); - exitValue = processResult.getExitCode(); - } catch (IOException ex) { - LOGGER.log(Level.SEVERE, - "Failed to shutdown git container executing command for user " + hdfsUsername, ex); - } - if (exitValue != 0) { - LOGGER.log(Level.SEVERE, - "Exited with " + exitValue + "Failed to shutdown git container executing command for user " - + hdfsUsername); - } - } - public String getGitHome(String secret) { - return settings.getStagingDir() + settings.PRIVATE_DIRS + secret; + return settings.getStagingDir() + Settings.PRIVATE_DIRS + secret; } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/jobs/spark/SparkController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jobs/spark/SparkController.java index 42fec9ab68..51985d1a30 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/jobs/spark/SparkController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jobs/spark/SparkController.java @@ -48,6 +48,7 @@ import io.hops.hopsworks.common.hosts.ServiceDiscoveryController; import io.hops.hopsworks.common.jobs.AsynchronousJobExecutor; import io.hops.hopsworks.common.jupyter.JupyterController; +import io.hops.hopsworks.common.jupyter.NotebookConversion; import io.hops.hopsworks.common.kafka.KafkaBrokers; import io.hops.hopsworks.common.serving.ServingConfig; import io.hops.hopsworks.common.util.HopsUtils; @@ -145,7 +146,7 @@ public Execution startJob(final Jobs job, String args, final Users user) String pyAppPath = HopsUtils.prepJupyterNotebookConversion(exec, username, dfs); sparkConfig.setAppPath(pyAppPath); jupyterController.convertIPythonNotebook(job.getProject(), user, appPath, pyAppPath, - JupyterController.NotebookConversion.PY); + NotebookConversion.PY); } submitter.startExecution(sparkjob, args); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/JupyterController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/JupyterController.java index 14684ea2d4..f2ca18c512 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/JupyterController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/JupyterController.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; import com.google.common.base.Strings; -import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; import io.hops.hopsworks.common.dao.jupyter.JupyterSettingsFacade; import io.hops.hopsworks.common.dao.jupyter.config.JupyterFacade; import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; @@ -29,11 +28,8 @@ import io.hops.hopsworks.common.livy.LivyController; import io.hops.hopsworks.common.livy.LivyMsg; import io.hops.hopsworks.common.security.CertificateMaterializer; +import io.hops.hopsworks.common.system.job.SystemJobStatus; import io.hops.hopsworks.common.util.HopsUtils; -import io.hops.hopsworks.common.util.OSProcessExecutor; -import io.hops.hopsworks.common.util.ProcessDescriptor; -import io.hops.hopsworks.common.util.ProcessResult; -import io.hops.hopsworks.common.util.ProjectUtils; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.persistence.entity.jupyter.JupyterProject; @@ -41,7 +37,6 @@ import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.restutils.RESTCodes; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; @@ -64,9 +59,6 @@ import java.util.Date; import java.util.List; import java.util.Optional; -import java.util.Arrays; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -79,8 +71,6 @@ public class JupyterController { @EJB private DistributedFsService dfs; @EJB - private OSProcessExecutor osProcessExecutor; - @EJB private Settings settings; @EJB private LivyController livyController; @@ -99,9 +89,9 @@ public class JupyterController { @EJB private JupyterJWTManager jupyterJWTManager; @EJB - private ProjectUtils projectUtils; - @EJB private XAttrsController xAttrsController; + @Inject + private NoteBookConverter noteBookConverter; private ObjectMapper objectMapper; @@ -112,61 +102,9 @@ public void init() { } @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) - public String convertIPythonNotebook(Project project, Users user, String notebookPath, String pyPath, - NotebookConversion notebookConversion) throws ServiceException { - - File baseDir = new File(settings.getStagingDir() + settings.CONVERSION_DIR); - if(!baseDir.exists()){ - baseDir.mkdir(); - } - File conversionDir = new File(baseDir, DigestUtils. - sha256Hex(Integer.toString(ThreadLocalRandom.current().nextInt()))); - conversionDir.mkdir(); - - String hdfsUser = hdfsUsersController.getHdfsUserName(project, user); - try{ - String prog = settings.getSudoersDir() + "/convert-ipython-notebook.sh"; - ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder() - .addCommand("/usr/bin/sudo") - .addCommand(prog) - .addCommand(notebookPath) - .addCommand(hdfsUser) - .addCommand(settings.getAnacondaProjectDir()) - .addCommand(pyPath) - .addCommand(conversionDir.getAbsolutePath()) - .addCommand(notebookConversion.name()) - .addCommand(projectUtils.getFullDockerImageName(project, true)) - .setWaitTimeout(120L, TimeUnit.SECONDS) //on a TLS VM the timeout needs to be greater than 20s - .redirectErrorStream(true) - .build(); - - LOGGER.log(Level.FINE, processDescriptor.toString()); - certificateMaterializer. - materializeCertificatesLocalCustomDir(user.getUsername(), project.getName(), conversionDir.getAbsolutePath()); - ProcessResult processResult = osProcessExecutor.execute(processDescriptor); - if (!processResult.processExited() || processResult.getExitCode() != 0) { - LOGGER.log(Level.WARNING, "error code: " + processResult.getExitCode(), "Failed to convert " - + notebookPath + "\nstderr: " + processResult.getStderr() + "\nstdout: " + processResult.getStdout()); - throw new ServiceException(RESTCodes.ServiceErrorCode.IPYTHON_CONVERT_ERROR, Level.SEVERE, - "Failed to convert ipython notebook to " + notebookConversion + " file", "Failed to convert " + notebookPath - + "\nstderr: " + processResult.getStderr() - + "\nstdout: " + processResult.getStdout()); - } - String stdOut = processResult.getStdout(); - if(!Strings.isNullOrEmpty(stdOut) && notebookConversion.equals(NotebookConversion.HTML)) { - StringBuilder renderedNotebookSB = new StringBuilder(stdOut); - int startIndex = renderedNotebookSB.indexOf(" kernel = NotebookConversion.fromKernel(notebookKernel); + // display_name can be Python 3 or just Python. + String[] parts = notebookKernel.trim().split(" "); + Optional kernel = NotebookConversion.fromKernel(parts[0]); if(kernel.isPresent()) { - if(kernel.get() == NotebookConversion.PY_JOB || kernel.get() == NotebookConversion.PY) { + if(NotebookConversion.PY_JOB.equals(kernel.get()) || NotebookConversion.PY.equals(kernel.get())) { return kernel.get(); } else { throw new ServiceException(RESTCodes.ServiceErrorCode.IPYTHON_CONVERT_ERROR, Level.FINE, @@ -390,24 +330,4 @@ public NotebookConversion getNotebookConversionType(String notebookPath, Users u dfs.closeDfsClient(udfso); } } - - public enum NotebookConversion { - PY("PySpark"), - HTML("Html"), - PY_JOB("Python"); - - private String kernel; - NotebookConversion(String kernel) { - this.kernel = kernel; - } - - public static Optional fromKernel(String kernel) { - return Arrays.stream(NotebookConversion.values()).filter(k -> - k.getKernel().equalsIgnoreCase(kernel)).findFirst(); - } - - public String getKernel() { - return kernel; - } - } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/LocalNoteBookConverter.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/LocalNoteBookConverter.java new file mode 100644 index 0000000000..b67a8614c8 --- /dev/null +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/LocalNoteBookConverter.java @@ -0,0 +1,117 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.common.jupyter; + +import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; +import io.hops.hopsworks.common.hdfs.HdfsUsersController; +import io.hops.hopsworks.common.integrations.LocalhostStereotype; +import io.hops.hopsworks.common.security.CertificateMaterializer; +import io.hops.hopsworks.common.system.job.SystemJobStatus; +import io.hops.hopsworks.common.util.OSProcessExecutor; +import io.hops.hopsworks.common.util.ProcessDescriptor; +import io.hops.hopsworks.common.util.ProcessResult; +import io.hops.hopsworks.common.util.ProjectUtils; +import io.hops.hopsworks.common.util.Settings; +import io.hops.hopsworks.exceptions.ServiceException; +import io.hops.hopsworks.persistence.entity.project.Project; +import io.hops.hopsworks.persistence.entity.user.Users; +import io.hops.hopsworks.restutils.RESTCodes; +import org.apache.commons.codec.digest.DigestUtils; + +import javax.ejb.EJB; +import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; +import java.io.File; +import java.io.IOException; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Stateless +@LocalhostStereotype +@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) +public class LocalNoteBookConverter implements NoteBookConverter { + private static final Logger LOGGER = Logger.getLogger(LocalNoteBookConverter.class.getName()); + @EJB + private Settings settings; + @EJB + private ProjectUtils projectUtils; + @EJB + private CertificateMaterializer certificateMaterializer; + @EJB + private HdfsUsersController hdfsUsersController; + @EJB + private OSProcessExecutor osProcessExecutor; + + @Override + public SystemJobStatus convertIPythonNotebook(Project project, Users user, String notebookPath, String pyPath, + NotebookConversion notebookConversion) throws ServiceException { + + File baseDir = new File(settings.getStagingDir() + "/" + Settings.CONVERSION_DIR); + if(!baseDir.exists()){ + baseDir.mkdir(); + } + File conversionDir = new File(baseDir, DigestUtils. + sha256Hex(Integer.toString(ThreadLocalRandom.current().nextInt()))); + conversionDir.mkdir(); + + String hdfsUser = hdfsUsersController.getHdfsUserName(project, user); + try{ + String prog = settings.getSudoersDir() + "/convert-ipython-notebook.sh"; + ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder() + .addCommand("/usr/bin/sudo") + .addCommand(prog) + .addCommand(notebookPath) + .addCommand(hdfsUser) + .addCommand(settings.getAnacondaProjectDir()) + .addCommand(pyPath) + .addCommand(conversionDir.getAbsolutePath()) + .addCommand(notebookConversion.name()) + .addCommand(projectUtils.getFullDockerImageName(project, true)) + .setWaitTimeout(120L, TimeUnit.SECONDS) //on a TLS VM the timeout needs to be greater than 20s + .redirectErrorStream(true) + .build(); + + LOGGER.log(Level.FINE, processDescriptor.toString()); + certificateMaterializer. + materializeCertificatesLocalCustomDir(user.getUsername(), project.getName(), conversionDir.getAbsolutePath()); + ProcessResult processResult = osProcessExecutor.execute(processDescriptor); + if (!processResult.processExited() || processResult.getExitCode() != 0) { + LOGGER.log(Level.WARNING, "error code: " + processResult.getExitCode(), "Failed to convert " + + notebookPath + "\nstderr: " + processResult.getStderr() + "\nstdout: " + processResult.getStdout()); + throw new ServiceException(RESTCodes.ServiceErrorCode.IPYTHON_CONVERT_ERROR, Level.SEVERE, + "Failed to convert ipython notebook to " + notebookConversion + " file", "Failed to convert " + notebookPath + + "\nstderr: " + processResult.getStderr() + + "\nstdout: " + processResult.getStdout()); + } + String stdOut = processResult.getStdout(); + SystemJobStatus status = new SystemJobStatus(0); + status.setLog(stdOut); + if(notebookConversion.equals(NotebookConversion.HTML)) { + status.setLog(getHtmlFromLog(stdOut)); + } + return status; + } catch (IOException | ServiceDiscoveryException ex) { + throw new ServiceException(RESTCodes.ServiceErrorCode.IPYTHON_CONVERT_ERROR, Level.SEVERE, null, ex.getMessage(), + ex); + } finally { + certificateMaterializer.removeCertificatesLocalCustomDir(user.getUsername(), project.getName(), conversionDir. + getAbsolutePath()); + } + } +} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/NoteBookConverter.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/NoteBookConverter.java new file mode 100644 index 0000000000..ecfca5c3bc --- /dev/null +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/NoteBookConverter.java @@ -0,0 +1,37 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.common.jupyter; + +import com.google.common.base.Strings; +import io.hops.hopsworks.common.system.job.SystemJobStatus; +import io.hops.hopsworks.exceptions.ServiceException; +import io.hops.hopsworks.persistence.entity.project.Project; +import io.hops.hopsworks.persistence.entity.user.Users; + +public interface NoteBookConverter { + + default String getHtmlFromLog(String log) { + if(!Strings.isNullOrEmpty(log)) { + StringBuilder renderedNotebookSB = new StringBuilder(log); + int startIndex = renderedNotebookSB.indexOf(". + */ +package io.hops.hopsworks.common.jupyter; + +import java.util.Arrays; +import java.util.Optional; + +public enum NotebookConversion { + PY("PySpark"), + HTML("Html"), + PY_JOB("Python"); + + private String kernel; + NotebookConversion(String kernel) { + this.kernel = kernel; + } + + public static Optional fromKernel(String kernel) { + return Arrays.stream(NotebookConversion.values()).filter(k -> + k.getKernel().equalsIgnoreCase(kernel)).findFirst(); + } + + public String getKernel() { + return kernel; + } +} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/system/job/SystemJobStatus.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/system/job/SystemJobStatus.java new file mode 100644 index 0000000000..f8f71d91b8 --- /dev/null +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/system/job/SystemJobStatus.java @@ -0,0 +1,52 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.common.system.job; + +public class SystemJobStatus { + private int exitCode; + private String log; + + public SystemJobStatus() { + } + + public SystemJobStatus(int exitCode) { + this.exitCode = exitCode; + } + + public int getExitCode() { + return exitCode; + } + + public void setExitCode(int exitCode) { + this.exitCode = exitCode; + } + + public String getLog() { + return log; + } + + public void setLog(String log) { + this.log = log; + } + + @Override + public String toString() { + return "SystemJobStatus{" + + "exitCode=" + exitCode + + ", log='" + log + '\'' + + '}'; + } +} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java index a09b777494..70f27809aa 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java @@ -1079,7 +1079,7 @@ public synchronized int getSparkExecutorMinMemory() { private String SPARK_DIR = "/srv/hops/spark"; - public static final String CONVERSION_DIR = "/ipython_conversions/"; + public static final String CONVERSION_DIR = "ipython_conversions/"; public static final String SPARK_NUMBER_EXECUTORS_ENV = "spark.executor.instances";