[HWORKS-135] Models backend should store metadata in tables instead o…

…f opensearch (#1516) (#1473)
logicalclocks · Feb 9, 2024 · 9e5daab · 9e5daab
1 parent d4d9904
commit 9e5daab
Show file tree

Hide file tree

Showing 18 changed files with 1,232 additions and 441 deletions.
diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/FilterBy.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/FilterBy.java
@@ -16,35 +16,45 @@
 package io.hops.hopsworks.api.modelregistry.models;
 
 import io.hops.hopsworks.common.dao.AbstractFacade;
+import io.hops.hopsworks.common.models.version.ModelVersionFacade;
 
 public class FilterBy implements AbstractFacade.FilterBy {
 
-  private String param = null;
-  private String value = null;
+  private final ModelVersionFacade.Filters filter;
+  private final String param;
 
   public FilterBy(String param) {
-    if(param.contains(":")) {
-      String[] paramSplit = param.split(":");
-      this.param = paramSplit[0];
-      this.value = paramSplit[1];
+    if (param.contains(":")) {
+      this.filter = ModelVersionFacade.Filters.valueOf(param.substring(0, param.indexOf(':')).toUpperCase());
+      this.param = param.substring(param.indexOf(':') + 1);
+    } else {
+      this.filter = ModelVersionFacade.Filters.valueOf(param);
+      this.param = this.filter.getDefaultParam();
     }
   }
 
+  @Override
   public String getParam() {
     return param;
   }
 
+  @Override
+  public String getValue() {
+    return this.filter.getValue();
+  }
+
   @Override
   public String getSql() {
-    return null;
+    return this.filter.getSql();
   }
 
   @Override
   public String getField() {
-    return null;
+    return this.filter.getField();
   }
 
-  public String getValue() {
-    return value;
+  @Override
+  public String toString() {
+    return filter.toString();
   }
-}
+}
diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelExpansions.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelExpansions.java
@@ -17,6 +17,7 @@
 package io.hops.hopsworks.api.modelregistry.models;
 
 import io.hops.hopsworks.api.modelregistry.models.provenance.ModelTrainingDatasetResourceRequest;
+import io.hops.hopsworks.api.user.UserResourceRequest;
 import io.hops.hopsworks.common.api.Expansions;
 import io.hops.hopsworks.common.api.ResourceRequest;
 
@@ -35,6 +36,9 @@ public ModelExpansions(String queryParam) {
       case TRAININGDATASETS:
         resourceRequest = new ModelTrainingDatasetResourceRequest(name, queryParam);
         break;
+      case USERS:
+        resourceRequest = new UserResourceRequest(name, queryParam);
+        break;
       case MODELSCHEMA:
         resourceRequest = new ModelSchemaResourceRequest(name, queryParam);
         break;

diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelUtils.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelUtils.java
@@ -15,39 +15,28 @@
  */
 package io.hops.hopsworks.api.modelregistry.models;
 
-import com.google.common.base.Strings;
 import io.hops.hopsworks.api.modelregistry.models.dto.ModelDTO;
 import io.hops.hopsworks.common.dao.project.ProjectFacade;
 import io.hops.hopsworks.common.dataset.DatasetController;
 import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps;
 import io.hops.hopsworks.common.hdfs.DistributedFsService;
 import io.hops.hopsworks.common.hdfs.HdfsUsersController;
 import io.hops.hopsworks.common.hdfs.Utils;
-import io.hops.hopsworks.common.provenance.state.dto.ProvStateDTO;
-import io.hops.hopsworks.common.python.environment.EnvironmentController;
 import io.hops.hopsworks.common.util.AccessController;
 import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.exceptions.DatasetException;
 import io.hops.hopsworks.exceptions.GenericException;
-import io.hops.hopsworks.exceptions.JobException;
-import io.hops.hopsworks.exceptions.MetadataException;
-import io.hops.hopsworks.exceptions.ModelRegistryException;
+
 import io.hops.hopsworks.exceptions.ProjectException;
-import io.hops.hopsworks.exceptions.PythonException;
-import io.hops.hopsworks.exceptions.ServiceException;
 import io.hops.hopsworks.persistence.entity.dataset.Dataset;
 import io.hops.hopsworks.persistence.entity.project.Project;
 import io.hops.hopsworks.persistence.entity.user.Users;
 import io.hops.hopsworks.restutils.RESTCodes;
-import org.json.JSONObject;
 
 import javax.ejb.EJB;
 import javax.ejb.Stateless;
 import javax.ejb.TransactionAttribute;
 import javax.ejb.TransactionAttributeType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriBuilder;
-import javax.ws.rs.core.UriInfo;
 import java.util.logging.Level;
 
 @Stateless
@@ -64,12 +53,6 @@ public class ModelUtils {
   private HdfsUsersController hdfsUsersController;
   @EJB
   private DistributedFsService dfs;
-  @EJB
-  private ModelsController modelsController;
-  @EJB
-  private EnvironmentController environmentController;
-  @EJB
-  private ModelConverter modelConverter;
 
   public String getModelsDatasetPath(Project userProject, Project modelRegistryProject) {
     String modelsPath = Utils.getProjectPath(userProject.getName()) + Settings.HOPS_MODELS_DATASET + "/";
@@ -139,39 +122,10 @@ public ModelsController.Accessor getModelsAccessor(Users user, Project userProje
     }
   }
 
-  public Response createModel(UriInfo uriInfo, ModelsController.Accessor accessor, String mlId, ModelDTO modelDTO,
-                              String jobName, String kernelId)
-    throws DatasetException, MetadataException, JobException, ServiceException, PythonException,
-    ModelRegistryException {
-    String realName = accessor.user.getFname() + " " + accessor.user.getLname();
-    //Only attach program and environment if exporting inside Hopsworks
-    if (!Strings.isNullOrEmpty(jobName) || !Strings.isNullOrEmpty(kernelId)) {
-
-      modelDTO.setProgram(modelsController.versionProgram(accessor, jobName, kernelId,
-            modelDTO.getName(), modelDTO.getVersion()));
-      //Export environment to correct path here
-      modelDTO.setEnvironment(environmentController.exportEnv(accessor.experimentProject, accessor.user,
-          getModelFullPath(accessor.modelProject, modelDTO.getName(), modelDTO.getVersion()) +
-              "/" + Settings.ENVIRONMENT_FILE
-      ));
-    }
-
-    modelDTO.setModelRegistryId(accessor.modelProject.getId());
-
-    modelsController.attachModel(accessor.udfso, accessor.modelProject, realName, modelDTO);
-    UriBuilder builder = uriInfo.getAbsolutePathBuilder().path(mlId);
-    return Response.created(builder.build()).entity(modelDTO).build();
-  }
-
   public String getModelFullPath(Project modelRegistryProject, String modelName, Integer modelVersion) {
     return Utils.getProjectPath(modelRegistryProject.getName()) +
         Settings.HOPS_MODELS_DATASET + "/" + modelName + "/" + modelVersion;
   }
-
-  public ModelDTO convertProvenanceHitToModel(ProvStateDTO model) throws ModelRegistryException {
-    JSONObject summary = new JSONObject(model.getXattrs().get(ModelsBuilder.MODEL_SUMMARY_XATTR_NAME));
-    return modelConverter.unmarshalDescription(summary.toString());
-  }
 
   public String[] getModelNameAndVersion(String mlId) {
     int splitIndex = mlId.lastIndexOf("_");

diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBeanParam.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBeanParam.java
@@ -37,6 +37,14 @@ public class ModelsBeanParam {
   @BeanParam
   private ModelExpansionBeanParam expansions;
 
+  public ModelsBeanParam(
+    @QueryParam("sort_by") String sortBy,
+    @QueryParam("filter_by") Set<FilterBy> filter) {
+    this.sortBy = sortBy;
+    this.sortBySet = getSortBy(sortBy);
+    this.filter = filter;
+  }
+
   private Set<SortBy> getSortBy(String param) {
     if (param == null || param.isEmpty()) {
       return new LinkedHashSet<>();
@@ -52,10 +60,12 @@ private Set<SortBy> getSortBy(String param) {
     return sortBys;
   }
 
-  public ModelsBeanParam(@QueryParam("filter_by") Set<FilterBy> filter, @QueryParam("sort_by") String sortBy) {
-    this.filter = filter;
+  public String getSortBy() {
+    return sortBy;
+  }
+
+  public void setSortBy(String sortBy) {
     this.sortBy = sortBy;
-    sortBySet = getSortBy(sortBy);
   }
 
   public Set<FilterBy> getFilter() {
@@ -66,15 +76,6 @@ public void setFilter(Set<FilterBy> filter) {
     this.filter = filter;
   }
 
-
-  public String getSortBy() {
-    return sortBy;
-  }
-
-  public void setSortBy(String sortBy) {
-    this.sortBy = sortBy;
-  }
-
   public Set<SortBy> getSortBySet() {
     return sortBySet;
   }