[HWORKS-697][APPEND] Exclude airflow user from some project sharing f…

…unctions (#1576) (#1416)

hopsworks-IT/src/test/ruby/spec/helpers/project_helper.rb

@@ -71,7 +71,7 @@ def add_spark_tour_files()
   end
 
   def create_project(projectName = nil, 
-                     services: ["JOBS","JUPYTER","HIVE","KAFKA","SERVING", "FEATURESTORE"],
+                     services: ["JOBS","JUPYTER","HIVE","KAFKA","SERVING", "FEATURESTORE", "AIRFLOW"],
                      validate_session: true)
     if validate_session
       with_valid_session

hopsworks-api/src/main/java/io/hops/hopsworks/api/opensearch/OpenSearchHitsBuilder.java

@@ -24,6 +24,7 @@
 import io.hops.hopsworks.common.opensearch.OpenSearchController;
 import io.hops.hopsworks.common.hdfs.HdfsUsersController;
 import io.hops.hopsworks.common.hdfs.inode.InodeController;
+import io.hops.hopsworks.common.util.ProjectUtils;
 import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.exceptions.OpenSearchException;
 import io.hops.hopsworks.exceptions.ServiceException;
@@ -69,6 +70,8 @@ public class OpenSearchHitsBuilder {
   private UserFacade userFacade;
   @EJB
   private HdfsUsersController hdfsUsersController;
+  @EJB
+  private ProjectUtils projectUtils;
 
   public OpenSearchHitDTO buildOpenSearchHits(String searchTerm, Users user)
       throws ServiceException, OpenSearchException {
@@ -298,7 +301,7 @@ private Boolean getBooleanValue(SearchHit hit, String name) {
 
   private List<String> getMembers(Project project) {
     List<String> members = new ArrayList<>();
-    for (ProjectTeam member: project.getProjectTeamCollection()) {
+    for (ProjectTeam member: projectUtils.getProjectTeamCollection(project)) {
       members.add(member.getUser().getFname() + " " + member.getUser().getLname());
     }
     return members;

hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java

@@ -63,6 +63,7 @@
 import io.hops.hopsworks.common.provenance.core.HopsFSProvenanceController;
 import io.hops.hopsworks.common.provenance.core.dto.ProvTypeDTO;
 import io.hops.hopsworks.common.util.HopsUtils;
+import io.hops.hopsworks.common.util.ProjectUtils;
 import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.exceptions.DatasetException;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
@@ -152,6 +153,8 @@ public class DatasetController {
   private OnlineFeaturestoreController onlineFeaturestoreController;
   @EJB
   private HdfsCommandExecutionController hdfsCommandExecutionController;
+  @EJB
+  private ProjectUtils projectUtils;
 
   /**
    * Create a new DataSet. This is, a folder right under the project home
@@ -804,9 +807,9 @@ private void addMembersToGroup(DatasetSharedWith datasetSharedWith) throws Datas
     DistributedFileSystemOps dfso = null;
     try {
       dfso = dfs.getDfsOps();
-      for (ProjectTeam teamMember : datasetSharedWith.getProject().getProjectTeamCollection()) {
-        hdfsUsersController.addNewMember(datasetSharedWith.getDataset(), datasetSharedWith.getPermission(), teamMember,
-          dfso);
+      for (ProjectTeam teamMember : projectUtils.getProjectTeamCollection(datasetSharedWith.getProject())) {
+        hdfsUsersController.addNewMember(datasetSharedWith.getDataset(), datasetSharedWith.getPermission(),
+            teamMember, dfso);
       }
     } catch (IOException e) {
       throw new DatasetException(RESTCodes.DatasetErrorCode.DATASET_OPERATION_ERROR, Level.FINE,
@@ -1278,7 +1281,7 @@ private void removeAllShareMembers(DatasetSharedWith datasetSharedWith) throws D
 
   private void removeAllShareMembers(DatasetSharedWith datasetSharedWith, DistributedFileSystemOps dfso)
     throws DatasetException {
-    for (ProjectTeam teamMember : datasetSharedWith.getProject().getProjectTeamCollection()) {
+    for (ProjectTeam teamMember : projectUtils.getProjectTeamCollection(datasetSharedWith.getProject())) {
       try {
         hdfsUsersController.removeMember(datasetSharedWith.getDataset(), datasetSharedWith.getPermission(),
           teamMember, dfso);

hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/acl/PermissionsFixer.java

@@ -26,6 +26,7 @@
 import io.hops.hopsworks.common.hdfs.HdfsUsersController;
 import io.hops.hopsworks.common.hdfs.Utils;
 import io.hops.hopsworks.common.hdfs.inode.InodeController;
+import io.hops.hopsworks.common.util.ProjectUtils;
 import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.persistence.entity.dataset.Dataset;
 import io.hops.hopsworks.persistence.entity.dataset.DatasetAccessPermission;
@@ -76,6 +77,9 @@ public class PermissionsFixer {
   private InodeController inodeController;
   @EJB
   private Settings settings;
+  @EJB
+  private ProjectUtils projectUtils;
+
   @Asynchronous
   public void fixPermissions() {
     fixPermissions(0, 0L);
@@ -158,7 +162,8 @@ private void fixPermission(Inode projectInode, Dataset dataset, HdfsGroups hdfsD
     testFsPermission(dataset, datasetInode, dfso);
     testAndFixPermissionForAllMembers(dataset.getProject(), dfso, hdfsDatasetGroup, hdfsDatasetAclGroup,
       datasetInode.getHdfsUser(), dataset.getPermission());
-    List<ProjectTeam> datasetTeamCollection = new ArrayList<>(dataset.getProject().getProjectTeamCollection());
+    List<ProjectTeam> datasetTeamCollection =
+        new ArrayList<>(projectUtils.getProjectTeamCollection(dataset.getProject()));
     for (DatasetSharedWith datasetSharedWith : dataset.getDatasetSharedWithCollection()) {
       if (dataset.isPublicDs() && !DatasetAccessPermission.READ_ONLY.equals(datasetSharedWith.getPermission())) {
         datasetSharedWith.setPermission(DatasetAccessPermission.READ_ONLY);
@@ -167,7 +172,7 @@ private void fixPermission(Inode projectInode, Dataset dataset, HdfsGroups hdfsD
       if (datasetSharedWith.getAccepted()) {
         testAndFixPermissionForAllMembers(datasetSharedWith.getProject(), dfso, hdfsDatasetGroup, hdfsDatasetAclGroup,
           null, datasetSharedWith.getPermission());
-        datasetTeamCollection.addAll(datasetSharedWith.getProject().getProjectTeamCollection());
+        datasetTeamCollection.addAll(projectUtils.getProjectTeamCollection(dataset.getProject()));
       }
     }
     testAndRemoveUsersFromGroup(datasetTeamCollection, hdfsDatasetGroup, hdfsDatasetAclGroup,
@@ -198,7 +203,7 @@ private void testFsPermission(Dataset dataset, Inode datasetInode, DistributedFi
   private void testAndFixPermissionForAllMembers(Project project, DistributedFileSystemOps dfso,
     HdfsGroups hdfsDatasetGroup, HdfsGroups hdfsDatasetAclGroup, HdfsUsers owner, DatasetAccessPermission permission)
     throws IOException {
-    for (ProjectTeam projectTeam : project.getProjectTeamCollection()) {
+    for (ProjectTeam projectTeam : projectUtils.getProjectTeamCollection(project)) {
       testAndFixPermission(projectTeam, dfso, hdfsDatasetGroup, hdfsDatasetAclGroup, owner, permission);
     }
   }

hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/online/OnlineFeaturestoreController.java

@@ -15,13 +15,13 @@
  */
 package io.hops.hopsworks.common.featurestore.online;
 
-import io.hops.hopsworks.common.dao.project.team.ProjectTeamFacade;
 import io.hops.hopsworks.common.dao.user.security.secrets.SecretsFacade;
 import io.hops.hopsworks.common.featurestore.FeaturestoreConstants;
 import io.hops.hopsworks.common.featurestore.OptionDTO;
 import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreConnectorFacade;
 import io.hops.hopsworks.common.featurestore.storageconnectors.StorageConnectorUtil;
 import io.hops.hopsworks.common.security.secrets.SecretsController;
+import io.hops.hopsworks.common.util.ProjectUtils;
 import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
 import io.hops.hopsworks.exceptions.UserException;
@@ -73,11 +73,11 @@ public class OnlineFeaturestoreController {
   @EJB
   private OnlineFeaturestoreFacade onlineFeaturestoreFacade;
   @EJB
-  private ProjectTeamFacade projectTeamFacade;
-  @EJB
   private FeaturestoreConnectorFacade featurestoreConnectorFacade;
   @EJB
   private StorageConnectorUtil storageConnectorUtil;
+  @EJB
+  private ProjectUtils projectUtils;
 
   /**
    * Sets up the online feature store database for a new project and creating a database-user for the project-owner
@@ -269,7 +269,7 @@ public void removeOnlineFeatureStore(Project project) throws FeaturestoreExcepti
       return;
     }
     try (Connection connection = onlineFeaturestoreFacade.establishAdminConnection()) {
-      for (ProjectTeam member : projectTeamFacade.findMembersByProject(project)) {
+      for (ProjectTeam member : projectUtils.getProjectTeamCollection(project)) {
         String dbUser = onlineDbUsername(project, member.getUser());
         try {
           secretsController.delete(member.getUser(), dbUser);
@@ -328,7 +328,7 @@ public void shareOnlineFeatureStore(Project project, Featurestore featurestore,
     }
 
     try (Connection connection = onlineFeaturestoreFacade.establishAdminConnection()) {
-      for (ProjectTeam member : projectTeamFacade.findMembersByProject(project)) {
+      for (ProjectTeam member : projectUtils.getProjectTeamCollection(project)) {
         shareOnlineFeatureStoreUser(project, member.getUser(), member.getTeamRole(), featureStoreDb, permission,
           connection);
       }
@@ -403,7 +403,7 @@ public void unshareOnlineFeatureStore(Project project, Featurestore featurestore
 
 
     try (Connection connection = onlineFeaturestoreFacade.establishAdminConnection()) {
-      for (ProjectTeam member : projectTeamFacade.findMembersByProject(project)) {
+      for (ProjectTeam member : projectUtils.getProjectTeamCollection(project)) {
         String dbUser = onlineDbUsername(project, member.getUser());
         onlineFeaturestoreFacade.revokeUserPrivileges(featureStoreDb, dbUser, connection);
       }

hopsworks-common/src/main/java/io/hops/hopsworks/common/hdfs/HdfsUsersController.java

@@ -41,6 +41,8 @@
 import io.hops.hopsworks.common.constants.auth.AllowedRoles;
 import io.hops.hopsworks.common.dao.hdfsUser.HdfsGroupsFacade;
 import io.hops.hopsworks.common.dao.hdfsUser.HdfsUsersFacade;
+import io.hops.hopsworks.common.util.ProjectUtils;
+import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.persistence.entity.dataset.PermissionTransition;
 import io.hops.hopsworks.persistence.entity.dataset.Dataset;
 import io.hops.hopsworks.persistence.entity.dataset.DatasetAccessPermission;
@@ -78,6 +80,10 @@ public class HdfsUsersController {
   private HdfsGroupsFacade hdfsGroupsFacade;
   @EJB
   private DistributedFsService dfsService;
+  @EJB
+  private Settings settings;
+  @EJB
+  private ProjectUtils projectUtils;
 
   /**
    * Creates a new group in HDFS with the name <code>projectName</code> if it
@@ -140,7 +146,7 @@ public void createDatasetGroupsAndSetPermissions(Users owner, Project project, D
     addToGroup(hdfsUsername, hdfsGroup.getName(), dfso);
 
     //add every member to the new ds group
-    addMembersToGroups(datasetGroup, datasetAclGroup, dfso, project.getProjectTeamCollection(),
+    addMembersToGroups(datasetGroup, datasetAclGroup, dfso, projectUtils.getProjectTeamCollection(project),
         dataset.getPermission());
   }
 
@@ -769,7 +775,7 @@ public void changePermission(Dataset ds, Project targetProject, PermissionTransi
 
   public void changePermission(Dataset ds, Project targetProject, PermissionTransition permissionTransition,
     DistributedFileSystemOps dfso) throws IOException {
-    for (ProjectTeam teamMember : targetProject.getProjectTeamCollection()) {
+    for (ProjectTeam teamMember : projectUtils.getProjectTeamCollection(targetProject)) {
       changePermission(ds, teamMember, permissionTransition, dfso);
     }
   }

hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java

@@ -1131,7 +1131,7 @@ public String[] forceCleanup(String projectName, Users user) {
         // Get Yarn applications
         List<ApplicationReport> projectApps = null;
         try {
-          Collection<ProjectTeam> team = project.getProjectTeamCollection();
+          Collection<ProjectTeam> team = projectUtils.getProjectTeamCollection(project);
           Set<String> hdfsUsers = new HashSet<>();
           for (ProjectTeam pt : team) {
             String hdfsUsername = hdfsUsersController.getHdfsUserName(project, pt.
@@ -1529,7 +1529,7 @@ private void waitForJobLogs(List<ApplicationReport> projectsApps, YarnClient cli
    * @param project Project to be deleted
    */
   private void removeCertificatesFromMaterializer(Project project) {
-    for (ProjectTeam team : project.getProjectTeamCollection()) {
+    for (ProjectTeam team : projectUtils.getProjectTeamCollection(project)) {
       certificateMaterializer.forceRemoveLocalMaterial(team.getUser().getUsername(), project.getName(), null, true);
       String remoteCertsDirectory = settings.getHdfsTmpCertDir() + Path.SEPARATOR +
         hdfsUsersController.getHdfsUserName(project, team.getUser());
@@ -1582,7 +1582,7 @@ public void cleanup(Project project, List<Future<?>> projectCreationFutures, Use
          * we can't know if the status in "NOT_START" because we should wait for it or because the
          * resourcemanager restarted.
          */
-        Collection<ProjectTeam> team = project.getProjectTeamCollection();
+        Collection<ProjectTeam> team = projectUtils.getProjectTeamCollection(project);
         Set<String> hdfsUsers = new HashSet<>();
         for (ProjectTeam pt : team) {
           String hdfsUsername = hdfsUsersController.getHdfsUserName(project, pt.getUser());

hopsworks-common/src/main/java/io/hops/hopsworks/common/security/CertificatesController.java

@@ -39,6 +39,7 @@
 package io.hops.hopsworks.common.security;
 
 import io.hops.hopsworks.common.dao.certificates.CertsFacade;
+import io.hops.hopsworks.common.util.ProjectUtils;
 import io.hops.hopsworks.persistence.entity.certificates.UserCerts;
 import io.hops.hopsworks.persistence.entity.project.Project;
 import io.hops.hopsworks.persistence.entity.project.team.ProjectTeam;
@@ -122,6 +123,8 @@ public class CertificatesController {
   private Instance<CertificateHandler> certificateHandlers;
   @EJB
   private CAProxy caProxy;
+  @EJB
+  private ProjectUtils projectUtils;
 
   private KeyPairGenerator keyPairGenerator = null;
   private CertificateFactory certificateFactory = null;
@@ -195,7 +198,7 @@ public void revokeProjectCertificates(Project project, Users owner)
       throws GenericException, HopsSecurityException, IOException {
     String projectName = project.getName();
 
-    Set<Users> users2deleteCertificates = Optional.ofNullable(project.getProjectTeamCollection())
+    Set<Users> users2deleteCertificates = Optional.ofNullable(projectUtils.getProjectTeamCollection(project))
             .map(Collection::stream).orElse(Stream.empty())
             .map(ProjectTeam::getUser).collect(Collectors.toSet());
     if (owner != null) {

hopsworks-common/src/main/java/io/hops/hopsworks/common/util/ProjectUtils.java

@@ -44,6 +44,7 @@
 import io.hops.hopsworks.exceptions.ProjectException;
 import io.hops.hopsworks.persistence.entity.jobs.configuration.DockerJobConfiguration;
 import io.hops.hopsworks.persistence.entity.project.Project;
+import io.hops.hopsworks.persistence.entity.project.team.ProjectTeam;
 import io.hops.hopsworks.persistence.entity.serving.DockerResourcesConfiguration;
 import io.hops.hopsworks.restutils.RESTCodes;
 import io.hops.hopsworks.servicediscovery.HopsworksService;
@@ -52,10 +53,14 @@
 import javax.ejb.Stateless;
 import javax.ejb.TransactionAttribute;
 import javax.ejb.TransactionAttributeType;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 @Stateless
 @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
@@ -260,4 +265,15 @@ public DockerResourcesConfiguration buildDockerResourceConfig(){
 
     return dockerResourceConfig;
   }
+
+  /**
+   * Get the project team members except some service users like airflow
+   * @param project
+   * @return
+   */
+  public Collection<ProjectTeam> getProjectTeamCollection(Project project) {
+    List<String> usersToFilter = Arrays.asList(settings.getAirflowUser());
+    return project.getProjectTeamCollection().stream()
+        .filter(m -> !usersToFilter.contains(m.getUser().getUsername())).collect(Collectors.toList());
+  }
 }