[HWORKS-912] remove StagingManager (#1451)

logicalclocks · Jan 11, 2024 · bb7a683 · bb7a683
1 parent 30b04fe
commit bb7a683
Show file tree

Hide file tree

Showing 4 changed files with 3 additions and 137 deletions.
diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/StagingManager.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/StagingManager.java
diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/UploadController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/UploadController.java
@@ -40,8 +40,6 @@
 public class UploadController {
   private static final Logger LOGGER = Logger.getLogger(UploadController.class.getName());
 
-  @EJB
-  private StagingManager stagingManager;
   @EJB
   private DistributedFsService dfs;
   @EJB
@@ -51,8 +49,7 @@ public UploadController() {
   }
 
   @VisibleForTesting
-  public UploadController(StagingManager stagingManager, DistributedFsService dfs, ResumableInfoStorage storage) {
-    this.stagingManager = stagingManager;
+  public UploadController(DistributedFsService dfs, ResumableInfoStorage storage) {
     this.dfs = dfs;
     this.storage = storage;
   }
@@ -257,9 +254,7 @@ private void createStagingDirIfNotExist(DistributedFileSystemOps dfsOps, Path st
   }
 
   private Path getTmpStagingDir(String hdfsPath, String filename) {
-    String baseDir = stagingManager.getStagingPath() + hdfsPath;
-    //get hdfs staging dir staging-dir/hdfs-path/filename.temp
-    return toTemp(baseDir, filename);
+    return toTemp(hdfsPath, filename);
   }
 
   private Path toTemp(String baseDir, String filename) {

diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java
@@ -143,7 +143,6 @@ public class Settings implements Serializable {
   private static final String VARIABLE_HOPSWORKS_USER = "hopsworks_user";
   private static final String VARIABLE_JUPYTER_GROUP = "jupyter_group";
   private static final String VARIABLE_STAGING_DIR = "staging_dir";
-  private static final String VARIABLE_UPLOAD_STAGING_DIR = "upload_staging_dir";
   private static final String VARIABLE_AIRFLOW_DIR = "airflow_dir";
   private static final String VARIABLE_AIRFLOW_USER = "airflow_user";
   private static final String VARIABLE_JUPYTER_DIR = "jupyter_dir";
@@ -641,7 +640,6 @@ private void populateCache() {
       SPARK_DIR = setDirVar(VARIABLE_SPARK_DIR, SPARK_DIR);
       FLINK_DIR = setDirVar(VARIABLE_FLINK_DIR, FLINK_DIR);
       STAGING_DIR = setVar(VARIABLE_STAGING_DIR, STAGING_DIR);
-      UPLOAD_STAGING_DIR = setVar(VARIABLE_UPLOAD_STAGING_DIR, UPLOAD_STAGING_DIR);
       HIVE_SUPERUSER = setStrVar(VARIABLE_HIVE_SUPERUSER, HIVE_SUPERUSER);
       HIVE_WAREHOUSE = setStrVar(VARIABLE_HIVE_WAREHOUSE, HIVE_WAREHOUSE);
       HIVE_SCRATCHDIR = setStrVar(VARIABLE_HIVE_SCRATCHDIR, HIVE_SCRATCHDIR);
@@ -1195,13 +1193,6 @@ public synchronized String getStagingDir() {
     return STAGING_DIR;
   }
 
-  private String UPLOAD_STAGING_DIR = DIR_ROOT;
-
-  public synchronized String getUploadStagingDir() {
-    checkCache();
-    return UPLOAD_STAGING_DIR;
-  }
-
   private final String FLINK_CONF_DIR = "conf";
   // Remember to change this in docker-images as well
   private String FLINK_DIR = "/srv/hops/flink";

diff --git a/hopsworks-common/src/test/io/hops/hopsworks/common/dataset/util/TestUpload.java b/hopsworks-common/src/test/io/hops/hopsworks/common/dataset/util/TestUpload.java
@@ -20,7 +20,6 @@
 import io.hops.hopsworks.common.hdfs.FsPermissions;
 import io.hops.hopsworks.common.upload.FlowInfo;
 import io.hops.hopsworks.common.upload.ResumableInfoStorage;
-import io.hops.hopsworks.common.upload.StagingManager;
 import io.hops.hopsworks.common.upload.UploadController;
 import io.hops.hopsworks.exceptions.DatasetException;
 import org.apache.commons.io.FileUtils;
@@ -72,22 +71,16 @@ public class TestUpload {
   private final Path destExistingFilePath = new Path(testUploadPath, "dest/random.txt");
   private final DistributedFsService dfs = mock(DistributedFsService.class);
   private final DistributedFileSystemOps distributedFileSystemOps = mock(DistributedFileSystemOps.class);
-  private final StagingManager stagingManager = mock(StagingManager.class);
 
   private UploadController uploadController;
 
   @Before
   public void setUp() throws IOException {
-    Mockito.doAnswer((Answer<String>) invocation -> {
-      LOGGER.log(LEVEL, "Get StagingPath: {0}.", "");
-      return "";
-    }).when(stagingManager).getStagingPath();
-
     ResumableInfoStorage storage = new ResumableInfoStorage();
     storage.initialize();
     setUpDistributedFileSystemOps();
     setUpDistributedFsService();
-    uploadController = new UploadController(stagingManager, dfs, storage);
+    uploadController = new UploadController(dfs, storage);
   }
 
   @After