[HWORKS-859] run notebook conversion and git commands in kube job for…

… ee (#1526)

hopsworks-IT/src/test/ruby/spec/git_spec.rb

@@ -369,6 +369,7 @@
         create_branch(@project[:id], repository_id, "test_branch")
         checkout_branch(@project[:id], repository_id, "test_branch")
         make_commit_in_repo(@project, repository_id)
+        #TODO fix this test it is trying to pull a branch that does not exist
         git_pull(@project[:id], repository_id, remote_name="origin", branch_name="test_branch")
         wait_for_git_operation_completed(@project[:id], repository_id, json_body[:id], "Success")
       end

hopsworks-IT/src/test/ruby/spec/spec_helper.rb

@@ -145,7 +145,7 @@
 
 Airborne.configure do |config|
   config.base_url = "https://#{ENV['WEB_HOST']}:#{ENV['WEB_PORT']}"
-  config.timeout = 120
+  config.timeout = 240
 end
 
 def try_start_all_services

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBeanParam.java

@@ -17,7 +17,7 @@
 package io.hops.hopsworks.api.featurestore.code;
 
 import com.google.common.base.Strings;
-import io.hops.hopsworks.common.jupyter.JupyterController;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
 import io.swagger.annotations.ApiParam;
 
 import javax.ws.rs.QueryParam;
@@ -45,13 +45,13 @@ public class CodeBeanParam {
 
   @QueryParam("format")
   @ApiParam(value = "ex. format=html", allowableValues = "format=html")
-  private JupyterController.NotebookConversion format;
+  private NotebookConversion format;
 
   public CodeBeanParam(
           @QueryParam("sort_by") String sortBy,
           @QueryParam("filter_by") Set<FilterBy> filter,
           @QueryParam("fields") String fields,
-          @QueryParam("format") JupyterController.NotebookConversion format) {
+          @QueryParam("format") NotebookConversion format) {
     this.sortBy = sortBy;
     this.sortBySet = getSortBy(sortBy);
     this.filterSet = filter;
@@ -89,7 +89,7 @@ public Set<String> getFieldSet() {
     return fieldSet;
   }
 
-  public JupyterController.NotebookConversion getFormat() {
-    return format == null ? JupyterController.NotebookConversion.HTML : format;
+  public NotebookConversion getFormat() {
+    return format == null ? NotebookConversion.HTML : format;
   }
 }

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBuilder.java

@@ -20,7 +20,7 @@
 import io.hops.hopsworks.common.dao.AbstractFacade;
 import io.hops.hopsworks.common.featurestore.code.CodeController;
 import io.hops.hopsworks.common.featurestore.code.FeaturestoreCodeFacade;
-import io.hops.hopsworks.common.jupyter.JupyterController;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
 import io.hops.hopsworks.exceptions.ServiceException;
 import io.hops.hopsworks.persistence.entity.featurestore.Featurestore;
@@ -104,7 +104,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
                        Project project, Users user,
                        Featuregroup featuregroup,
                        FeaturestoreCode featurestoreCode,
-                       JupyterController.NotebookConversion format) throws FeaturestoreException, ServiceException {
+                       NotebookConversion format) throws FeaturestoreException, ServiceException {
     Path fullCodePath = new Path(codeController.getCodeDirFullPath(project, featuregroup),
         featurestoreCode.getFileName());
     CodeDTO dto = new CodeDTO();
@@ -133,7 +133,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
                        Project project, Users user,
                        TrainingDataset trainingDataset,
                        FeaturestoreCode featurestoreCode,
-                       JupyterController.NotebookConversion format) throws FeaturestoreException, ServiceException {
+                       NotebookConversion format) throws FeaturestoreException, ServiceException {
     Path fullCodePath = new Path(codeController.getCodeDirFullPath(project, trainingDataset),
         featurestoreCode.getFileName());
     CodeDTO dto = new CodeDTO();
@@ -156,7 +156,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
 
   public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
                        Project project, Users user, Featurestore featurestore, Featuregroup featuregroup,
-                       JupyterController.NotebookConversion format)
+                       NotebookConversion format)
       throws ServiceException, FeaturestoreException {
     CodeDTO dto = new CodeDTO();
     dto.setHref(uri(uriInfo, project, featurestore, featuregroup));
@@ -180,7 +180,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
 
   public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
                        Project project, Users user, Featurestore featurestore, TrainingDataset trainingDataset,
-                       JupyterController.NotebookConversion format)
+                       NotebookConversion format)
       throws ServiceException, FeaturestoreException {
     CodeDTO dto = new CodeDTO();
     dto.setHref(uri(uriInfo, project, featurestore, trainingDataset));
@@ -205,7 +205,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
   public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
                        Project project, Users user, Featuregroup featuregroup,
                        Integer codeId,
-                       JupyterController.NotebookConversion format)
+                       NotebookConversion format)
       throws FeaturestoreException, ServiceException {
     FeaturestoreCode featurestoreCode = codeFacade.findFeaturestoreCodeById(featuregroup, codeId)
         .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CODE_NOT_FOUND, Level.FINE));
@@ -216,7 +216,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
   public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
                        Project project, Users user, TrainingDataset trainingDataset,
                        Integer codeId,
-                       JupyterController.NotebookConversion format)
+                       NotebookConversion format)
       throws FeaturestoreException, ServiceException {
     FeaturestoreCode featurestoreCode = codeFacade.findFeaturestoreCodeById(trainingDataset, codeId)
         .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CODE_NOT_FOUND, Level.FINE));

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java

@@ -16,17 +16,17 @@
 
 package io.hops.hopsworks.api.featurestore.code;
 
+import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.api.filter.AllowedProjectRoles;
 import io.hops.hopsworks.api.filter.Audience;
-import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.api.jwt.JWTHelper;
 import io.hops.hopsworks.api.util.Pagination;
 import io.hops.hopsworks.common.api.ResourceRequest;
 import io.hops.hopsworks.common.featurestore.code.CodeActions;
 import io.hops.hopsworks.common.featurestore.code.CodeController;
 import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController;
 import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController;
-import io.hops.hopsworks.common.jupyter.JupyterController;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
 import io.hops.hopsworks.exceptions.DatasetException;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
 import io.hops.hopsworks.exceptions.HopsSecurityException;
@@ -49,15 +49,15 @@
 import javax.enterprise.context.RequestScoped;
 import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.BeanParam;
-import javax.ws.rs.POST;
 import javax.ws.rs.GET;
+import javax.ws.rs.POST;
 import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
 import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Context;
 import javax.ws.rs.core.SecurityContext;
 import javax.ws.rs.core.UriInfo;
 
@@ -196,12 +196,12 @@ public Response post(@Context UriInfo uriInfo,
           codeDTO.getFeatureGroupCommitId(), codeDTO.getApplicationId(), featuregroup,
           entityId, databricksNotebook, databricksArchive, type);
       dto = codeBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.CODE),
-              project, user, featuregroup, featurestoreCode, JupyterController.NotebookConversion.HTML);
+              project, user, featuregroup, featurestoreCode, NotebookConversion.HTML);
     } else {
       FeaturestoreCode featurestoreCode = codeController.registerCode(project, user, codeDTO.getCommitTime(),
               codeDTO.getApplicationId(), trainingDataset, entityId, databricksNotebook, databricksArchive, type);
       dto = codeBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.CODE),
-              project, user, trainingDataset, featurestoreCode, JupyterController.NotebookConversion.HTML);
+              project, user, trainingDataset, featurestoreCode, NotebookConversion.HTML);
     }
 
     return Response.ok().entity(dto).build();

hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java

@@ -16,9 +16,9 @@
 package io.hops.hopsworks.api.git;
 
 import com.google.common.base.Strings;
+import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.api.filter.AllowedProjectRoles;
 import io.hops.hopsworks.api.filter.Audience;
-import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.api.git.branch.BranchBuilder;
 import io.hops.hopsworks.api.git.branch.BranchDTO;
 import io.hops.hopsworks.api.git.execution.ExecutionBeanParam;
@@ -33,22 +33,21 @@
 import io.hops.hopsworks.api.jwt.JWTHelper;
 import io.hops.hopsworks.api.util.Pagination;
 import io.hops.hopsworks.common.api.ResourceRequest;
+import io.hops.hopsworks.common.dao.project.ProjectFacade;
 import io.hops.hopsworks.common.git.BranchCommits;
 import io.hops.hopsworks.common.git.CloneCommandConfiguration;
+import io.hops.hopsworks.common.git.GitBranchAction;
+import io.hops.hopsworks.common.git.GitCommitDTO;
 import io.hops.hopsworks.common.git.GitController;
 import io.hops.hopsworks.common.git.GitRemotesAction;
-import io.hops.hopsworks.common.git.GitBranchAction;
 import io.hops.hopsworks.common.git.GitRepositoryAction;
-import io.hops.hopsworks.common.git.GitCommitDTO;
-import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator;
 import io.hops.hopsworks.common.git.RepositoryActionCommandConfiguration;
+import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator;
 import io.hops.hopsworks.exceptions.DatasetException;
-import io.hops.hopsworks.exceptions.UserException;
-import io.hops.hopsworks.persistence.entity.git.GitOpExecution;
-import io.hops.hopsworks.common.dao.project.ProjectFacade;
 import io.hops.hopsworks.exceptions.GitOpException;
 import io.hops.hopsworks.exceptions.HopsSecurityException;
 import io.hops.hopsworks.jwt.annotation.JWTRequired;
+import io.hops.hopsworks.persistence.entity.git.GitOpExecution;
 import io.hops.hopsworks.persistence.entity.git.GitRepository;
 import io.hops.hopsworks.persistence.entity.project.Project;
 import io.hops.hopsworks.persistence.entity.user.Users;
@@ -165,9 +164,10 @@ public Response gitRepository(@PathParam("repositoryId") Integer repositoryId,
   @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"})
   public Response clone(CloneCommandConfiguration commandDTO,
                         @Context SecurityContext sc,
+                        @Context HttpServletRequest req,
                         @Context UriInfo uriInfo,
-                        @BeanParam ExecutionBeanParam executionBeanParam)
-      throws GitOpException, HopsSecurityException, IllegalArgumentException, UserException, DatasetException {
+                        @BeanParam ExecutionBeanParam executionBeanParam) throws GitOpException, HopsSecurityException,
+      IllegalArgumentException, DatasetException {
     Users hopsworksUser = jWTHelper.getUserPrincipal(sc);
     GitOpExecution execution = gitController.clone(commandDTO, project, hopsworksUser);
     ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION);

hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java

@@ -47,25 +47,27 @@
 import io.hops.hopsworks.common.dao.jupyter.JupyterSettingsFacade;
 import io.hops.hopsworks.common.dao.jupyter.config.JupyterDTO;
 import io.hops.hopsworks.common.dao.jupyter.config.JupyterFacade;
-import io.hops.hopsworks.common.jupyter.NotebookDTO;
-import io.hops.hopsworks.common.jupyter.JupyterManager;
-import io.hops.hopsworks.common.jupyter.RemoteFSDriverType;
 import io.hops.hopsworks.common.dao.project.ProjectFacade;
 import io.hops.hopsworks.common.hdfs.HdfsUsersController;
 import io.hops.hopsworks.common.hdfs.Utils;
 import io.hops.hopsworks.common.jobs.spark.SparkController;
 import io.hops.hopsworks.common.jupyter.JupyterController;
 import io.hops.hopsworks.common.jupyter.JupyterJWTManager;
+import io.hops.hopsworks.common.jupyter.JupyterManager;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
+import io.hops.hopsworks.common.jupyter.NotebookDTO;
+import io.hops.hopsworks.common.jupyter.RemoteFSDriverType;
 import io.hops.hopsworks.common.livy.LivyController;
 import io.hops.hopsworks.common.livy.LivyMsg;
+import io.hops.hopsworks.common.system.job.SystemJobStatus;
 import io.hops.hopsworks.common.user.UsersController;
 import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.exceptions.GenericException;
 import io.hops.hopsworks.exceptions.HopsSecurityException;
-import io.hops.hopsworks.exceptions.ProjectException;
-import io.hops.hopsworks.exceptions.ServiceException;
 import io.hops.hopsworks.exceptions.JobException;
 import io.hops.hopsworks.exceptions.OpenSearchException;
+import io.hops.hopsworks.exceptions.ProjectException;
+import io.hops.hopsworks.exceptions.ServiceException;
 import io.hops.hopsworks.jwt.annotation.JWTRequired;
 import io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration;
 import io.hops.hopsworks.persistence.entity.jupyter.JupyterMode;
@@ -88,14 +90,14 @@
 import javax.validation.constraints.Min;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
-import javax.ws.rs.DefaultValue;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.GenericEntity;
 import javax.ws.rs.core.MediaType;
@@ -397,15 +399,18 @@ public Response stopNotebookServer(@Context HttpServletRequest req,
   public Response convertIPythonNotebook(@PathParam("path") String path,
                                          @Context HttpServletRequest req,
                                          @Context SecurityContext sc) throws ServiceException {
-    String ipynbPath = Utils.getProjectPath(this.project.getName()) + "/" + path;
+    if (path.startsWith("/")) {
+      path = path.replaceFirst("/", "");
+    }
+    String ipynbPath = Utils.getProjectPath(this.project.getName()) + path;
     int extensionIndex = ipynbPath.lastIndexOf(".ipynb");
     StringBuilder pathBuilder = new StringBuilder(ipynbPath.substring(0, extensionIndex)).append(".py");
     String pyAppPath = pathBuilder.toString();
     Users user = jWTHelper.getUserPrincipal(sc);
-    JupyterController.NotebookConversion conversionType = jupyterController
-        .getNotebookConversionType(ipynbPath, user, this.project);
-    jupyterController.convertIPythonNotebook(project, user, ipynbPath, pyAppPath, conversionType);
-    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).build();
+    NotebookConversion conversionType = jupyterController.getNotebookConversionType(ipynbPath, user, this.project);
+    SystemJobStatus status =
+      jupyterController.convertIPythonNotebook(project, user, ipynbPath, pyAppPath, conversionType);
+    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(status).build();
   }
 
   @POST

hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitPaths.java

@@ -15,8 +15,6 @@
  */
 package io.hops.hopsworks.common.dao.git;
 
-import java.io.File;
-
 public class GitPaths {
   private String secret;
   private String gitPath;
@@ -28,11 +26,12 @@ public class GitPaths {
 
   public GitPaths(String privateDir, String secretConfig) {
     this.gitPath = privateDir + secretConfig;
-    this.logDirPath = gitPath + File.separator + "git_logs";
-    this.confDirPath = gitPath + File.separator + "conf";
-    this.certificatesDirPath = gitPath + File.separator + "certificates";
-    this.runDirPath = gitPath + File.separator + "run";
-    this.tokenPath = gitPath + File.separator + "token";
+    String gitPathWithSlash = this.gitPath.endsWith("/") ? this.gitPath  : this.gitPath + "/";
+    this.logDirPath = gitPathWithSlash + "git_logs";
+    this.confDirPath = gitPathWithSlash + "conf";
+    this.certificatesDirPath = gitPathWithSlash + "certificates";
+    this.runDirPath = gitPathWithSlash + "run";
+    this.tokenPath = gitPathWithSlash + "token";
     this.secret = secretConfig;
   }
 

hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitRepositoryRemotesFacade.java

@@ -72,4 +72,11 @@ public void updateRepositoryRemotes(List<GitRepositoryRemote> remotes, GitReposi
       save(remote);
     }
   }
+
+  public GitRepositoryRemote create(GitRepository repository, String name, String url) {
+    GitRepositoryRemote gitRepositoryRemote = new GitRepositoryRemote(repository, name, url);
+    em.persist(gitRepositoryRemote);
+    em.flush();
+    return gitRepositoryRemote;
+  }
 }

hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java

@@ -60,8 +60,10 @@
 import io.hops.hopsworks.common.hdfs.command.HdfsCommandExecutionController;
 import io.hops.hopsworks.common.hdfs.inode.InodeController;
 import io.hops.hopsworks.common.jupyter.JupyterController;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
 import io.hops.hopsworks.common.provenance.core.HopsFSProvenanceController;
 import io.hops.hopsworks.common.provenance.core.dto.ProvTypeDTO;
+import io.hops.hopsworks.common.system.job.SystemJobStatus;
 import io.hops.hopsworks.common.util.HopsUtils;
 import io.hops.hopsworks.common.util.ProjectUtils;
 import io.hops.hopsworks.common.util.Settings;
@@ -625,10 +627,10 @@ public FilePreviewDTO filePreview(Project project, Users user, Path fullPath, Fi
           throw new DatasetException(RESTCodes.DatasetErrorCode.IMAGE_SIZE_INVALID, Level.FINE);
         }
       } else if(fileExtension.equalsIgnoreCase("ipynb")) {
-        String html = jupyterController.convertIPythonNotebook(project, user, fullPath.toString(), "''",
-            JupyterController.NotebookConversion.HTML);
+        SystemJobStatus status = jupyterController.convertIPythonNotebook(project, user, fullPath.toString(), "''",
+          NotebookConversion.HTML);
         filePreviewDTO = new FilePreviewDTO(Settings.FILE_PREVIEW_HTML_TYPE, fileExtension.toLowerCase(),
-            html);
+          status.getLog());
       } else {
         try (DataInputStream dis = new DataInputStream(is)) {
           int sizeThreshold = Settings.FILE_PREVIEW_TXT_SIZE_BYTES; //in bytes