[FSTORE-1130] Get schema from shared project (#1645) (#1442)

hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb

@@ -3754,6 +3754,44 @@
 
         get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"])
       end
+
+      # get subject
+
+      it 'data owner should be able to get fg schema' do
+        create_session(@user1[:email], @user1_params[:password])
+        fs = get_featurestore(@project1[:id])
+        json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"])
+        expect_status_details(201)
+        fg = JSON.parse(json_result, :symbolize_names => true)
+
+        create_session(@user_data_owner[:email], @user_data_owner_params[:password])
+        get_featurestore_subject_details(@project_read_only, fs["featurestoreId"], "#{fg[:name]}_#{fg[:version]}", 1)
+        expect_status_details(200)
+      end
+
+      it 'data scientist should be able to get fg schema' do
+        create_session(@user1[:email], @user1_params[:password])
+        fs = get_featurestore(@project1[:id])
+        json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"])
+        expect_status_details(201)
+        fg = JSON.parse(json_result, :symbolize_names => true)
+
+        create_session(@user_data_scientist[:email], @user_data_scientist_params[:password])
+        get_featurestore_subject_details(@project_read_only, fs["featurestoreId"], "#{fg[:name]}_#{fg[:version]}", 1)
+        expect_status_details(200)
+      end
+
+      it 'user should not be able to get fg schema from project that is not shared' do
+        create_session(@user2[:email], @user2_params[:password])
+        fs = get_featurestore(@project_read_only[:id])
+        json_result, _ = create_stream_featuregroup(@project_read_only[:id], fs["featurestoreId"])
+        expect_status_details(201)
+        fg = JSON.parse(json_result, :symbolize_names => true)
+
+        create_session(@user1[:email], @user1_params[:password])
+        get_featurestore_subject_details(@project1, fs["featurestoreId"], "#{fg[:name]}_#{fg[:version]}", 1)
+        expect_status_details(404)
+      end
     end
 
     context "shared on demand feature group permission" do

hopsworks-IT/src/test/ruby/spec/helpers/kafka_schema_helper.rb

@@ -84,4 +84,8 @@ def get_topic_subject_details(project, topic)
   def update_topic_subject_version(project, topic, subject, version)
     put "#{ENV['HOPSWORKS_API']}/project/#{project.id.to_s}/kafka/topics/#{topic}/subjects/#{subject}/versions/#{version}"
   end
+
+  def get_featurestore_subject_details(project, fs_id, subject, version)
+    get "#{ENV['HOPSWORKS_API']}/project/#{project.id.to_s}/featurestores/#{fs_id.to_s}/kafka/subjects/#{subject}/versions/#{version.to_s}"
+  end
 end

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java

@@ -27,6 +27,7 @@
 import io.hops.hopsworks.api.filter.AllowedProjectRoles;
 import io.hops.hopsworks.api.filter.Audience;
 import io.hops.hopsworks.api.filter.NoCacheResponse;
+import io.hops.hopsworks.api.kafka.KafkaResource;
 import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.common.api.ResourceRequest;
 import io.hops.hopsworks.common.dao.project.ProjectFacade;
@@ -38,6 +39,7 @@
 import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
 import io.hops.hopsworks.jwt.annotation.JWTRequired;
+import io.hops.hopsworks.persistence.entity.featurestore.Featurestore;
 import io.hops.hopsworks.persistence.entity.project.Project;
 import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope;
 import io.hops.hopsworks.restutils.RESTCodes;
@@ -100,6 +102,8 @@ public class FeaturestoreService {
   private TransformationFunctionResource transformationFunctionResource;
   @Inject
   private GreatExpectationResource greatExpectationResource;
+  @Inject
+  private KafkaResource kafkaResource;
 
   private Project project;
 
@@ -174,7 +178,8 @@ public Response getFeaturestore(@ApiParam(value = "Id of the featurestore", requ
     if (featurestoreId == null) {
       throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage());
     }
-    FeaturestoreDTO featurestoreDTO = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId);
+    FeaturestoreDTO featurestoreDTO = 
+      featurestoreController.getFeaturestoreDTOForProjectWithId(project, featurestoreId);
     GenericEntity<FeaturestoreDTO> featurestoreDTOGeneric =
       new GenericEntity<FeaturestoreDTO>(featurestoreDTO) {
       };
@@ -350,7 +355,7 @@ public TransformationFunctionResource transformationResource(
       throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage());
     }
     this.transformationFunctionResource.setFeaturestore(
-        featurestoreController.getFeaturestoreWithId(featurestoreId));
+        featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId));
     return transformationFunctionResource;
   }
 
@@ -369,7 +374,27 @@ public GreatExpectationResource greatExpectationResource(
       throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage());
     }
     this.greatExpectationResource.setFeaturestore(
-        featurestoreController.getFeaturestoreWithId(featurestoreId));
+        featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId));
     return greatExpectationResource;
   }
+
+  /**
+   * Kafka sub-resource
+   *
+   * @param featurestoreId id of the featurestore
+   * @return the feature store kafka resource
+   * @throws FeaturestoreException
+   */
+  @Logged(logLevel = LogLevel.OFF)
+  @Path("{featurestoreId}/kafka")
+  public KafkaResource kafkaResource(@PathParam("featurestoreId") Integer featurestoreId)
+      throws FeaturestoreException {
+    if (featurestoreId == null) {
+      throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage());
+    }
+    //This call verifies that the project have access to the featurestoreId provided
+    Featurestore featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId);
+    this.kafkaResource.setProject(featurestore.getProject());
+    return this.kafkaResource;
+  }
 }

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java

@@ -39,7 +39,6 @@
 import io.hops.hopsworks.api.jwt.JWTHelper;
 import io.hops.hopsworks.common.api.ResourceRequest;
 import io.hops.hopsworks.common.featurestore.FeaturestoreController;
-import io.hops.hopsworks.common.featurestore.FeaturestoreDTO;
 import io.hops.hopsworks.common.featurestore.OptionDTO;
 import io.hops.hopsworks.common.featurestore.app.FsJobManagerController;
 import io.hops.hopsworks.common.featurestore.featuregroup.ImportFgJobConf;
@@ -172,8 +171,7 @@ public void setProject(Project project) {
    */
   public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreException {
     //This call verifies that the project have access to the featurestoreId provided
-    FeaturestoreDTO featurestoreDTO = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId);
-    this.featurestore = featurestoreController.getFeaturestoreWithId(featurestoreDTO.getFeaturestoreId());
+    this.featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId);
   }
 
   /**

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewService.java

@@ -25,7 +25,6 @@
 import io.hops.hopsworks.api.featurestore.transformation.TransformationResource;
 import io.hops.hopsworks.api.provenance.FeatureViewProvenanceResource;
 import io.hops.hopsworks.common.featurestore.FeaturestoreController;
-import io.hops.hopsworks.common.featurestore.FeaturestoreDTO;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
 import io.hops.hopsworks.persistence.entity.featurestore.Featurestore;
 import io.hops.hopsworks.persistence.entity.project.Project;
@@ -170,9 +169,10 @@ public void setProject(Project project) {
     this.project = project;
   }
 
-  public void setFeaturestore(Integer id) throws FeaturestoreException {
-    FeaturestoreDTO featurestoreDTO = featurestoreController.getFeaturestoreForProjectWithId(project, id);
-    this.featurestore = featurestoreController.getFeaturestoreWithId(featurestoreDTO.getFeaturestoreId());
+  @Logged(logLevel = LogLevel.OFF)
+  public void setFeaturestore(Integer featurestoreId) throws FeaturestoreException {
+    //This call verifies that the project have access to the featurestoreId provided
+    this.featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId);
   }
 
   @Path("/{name: [a-z0-9_]*(?=[a-z])[a-z0-9_]+}/version/{version: [0-9]+}/provenance")

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java

@@ -24,7 +24,6 @@
 import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.api.jwt.JWTHelper;
 import io.hops.hopsworks.common.featurestore.FeaturestoreController;
-import io.hops.hopsworks.common.featurestore.FeaturestoreDTO;
 import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreController;
 import io.hops.hopsworks.common.featurestore.storageconnectors.connectionChecker.ConnectionChecker;
 import io.hops.hopsworks.common.featurestore.storageconnectors.connectionChecker.ConnectionCheckerDTO;
@@ -118,8 +117,7 @@ public void setProject(Project project) {
    */
   public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreException {
     //This call verifies that the project have access to the featurestoreId provided
-    FeaturestoreDTO featurestoreDTO = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId);
-    this.featurestore = featurestoreController.getFeaturestoreWithId(featurestoreDTO.getFeaturestoreId());
+    this.featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId);
   }
 
   /**

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetService.java

@@ -38,7 +38,6 @@
 import io.hops.hopsworks.common.api.ResourceRequest;
 import io.hops.hopsworks.common.dao.user.activity.ActivityFacade;
 import io.hops.hopsworks.common.featurestore.FeaturestoreController;
-import io.hops.hopsworks.common.featurestore.FeaturestoreDTO;
 import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController;
 import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO;
 import io.hops.hopsworks.common.featurestore.transformationFunction.TransformationFunctionAttachedDTO;
@@ -149,8 +148,7 @@ public void setProject(Project project) {
    */
   public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreException {
     //This call verifies that the project have access to the featurestoreId provided
-    FeaturestoreDTO featurestoreDTO = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId);
-    this.featurestore = featurestoreController.getFeaturestoreWithId(featurestoreDTO.getFeaturestoreId());
+    this.featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId);
   }
 
   /**

hopsworks-api/src/main/java/io/hops/hopsworks/api/kafka/KafkaResource.java

@@ -54,7 +54,6 @@
 import io.hops.hopsworks.common.dao.kafka.schemas.CompatibilityLevel;
 import io.hops.hopsworks.common.dao.kafka.schemas.SchemaRegistryError;
 import io.hops.hopsworks.common.dao.kafka.schemas.SubjectDTO;
-import io.hops.hopsworks.common.dao.project.ProjectFacade;
 import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController;
 import io.hops.hopsworks.common.featurestore.storageconnectors.kafka.FeatureStoreKafkaConnectorDTO;
 import io.hops.hopsworks.common.kafka.KafkaController;
@@ -100,8 +99,6 @@ public class KafkaResource {
 
   private static final Logger LOGGER = Logger.getLogger(KafkaResource.class.getName());
 
-  @EJB
-  private ProjectFacade projectFacade;
   @EJB
   private KafkaController kafkaController;
   @EJB
@@ -121,11 +118,10 @@ public class KafkaResource {
 
   public KafkaResource() {
   }
-  public void setProjectId(Integer projectId) {
-    this.project = this.projectFacade.find(projectId);
-  }
-  public Project getProject() {
-    return project;
+
+  @Logged(logLevel = LogLevel.OFF)
+  public void setProject(Project project) {
+    this.project = project;
   }
 
   @ApiOperation(value = "Retrieve Kafka broker endpoints.")

hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java

@@ -664,8 +664,9 @@ public Response client(@PathParam("projectId") Integer id, @Context HttpServletR
   }
 
   @Path("{projectId}/kafka")
-  public KafkaResource kafka(@PathParam("projectId") Integer id) {
-    this.kafka.setProjectId(id);
+  public KafkaResource kafka(@PathParam("projectId") Integer id) throws ProjectException {
+    Project project = projectController.findProjectById(id);
+    this.kafka.setProject(project);
     return this.kafka;
   }
 

hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java

@@ -211,12 +211,25 @@ public FeaturestoreDTO getFeaturestoreForProjectWithName(Project project, String
    * @return a DTO representation of the featurestore
    * @throws FeaturestoreException
    */
-  public FeaturestoreDTO getFeaturestoreForProjectWithId(Project project, Integer featurestoreId)
+  public FeaturestoreDTO getFeaturestoreDTOForProjectWithId(Project project, Integer featurestoreId)
+      throws FeaturestoreException {
+    Featurestore featurestore = getFeaturestoreForProjectWithId(project, featurestoreId);
+    return convertFeaturestoreToDTO(featurestore);
+  }
+
+  /**
+   * Gets a featurestore with a particular featurestoreId from the list of featurestores for this project
+   *
+   * @param project the project to look for the featurestore in
+   * @param featurestoreId the featurestoreId of the featurestore
+   * @return a featurestore Object
+   * @throws FeaturestoreException
+   */
+  public Featurestore getFeaturestoreForProjectWithId(Project project, Integer featurestoreId)
       throws FeaturestoreException {
     try {
       return getProjectFeaturestores(project).stream()
           .filter(fs -> fs.getId().equals(featurestoreId))
-          .map(this::convertFeaturestoreToDTO)
           .findAny()
           .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_NOT_FOUND,
               Level.FINE, "featurestoreId: " + featurestoreId + " , project: " + project.getName()));
@@ -230,21 +243,6 @@ public FeaturestoreDTO getFeaturestoreForProjectWithId(Project project, Integer
     }
   }
 
-  /**
-   * Retrieves a featurestore with a particular Id from the database
-   *
-   * @param id the id of the featurestore
-   * @return featurestore entity with the given id
-   */
-  public Featurestore getFeaturestoreWithId(Integer id) throws FeaturestoreException {
-    Featurestore featurestore = featurestoreFacade.findById(id);
-    if (featurestore == null) {
-      throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_NOT_FOUND,
-          Level.FINE, "featurestoreId: " + id);
-    }
-    return featurestore;
-  }
-
   /**
    * Creates a new featurestore in the database
    *