From bb7a683e94e0c5d2134dc72bbfa70c6c5134dbbe Mon Sep 17 00:00:00 2001 From: ErmiasG Date: Fri, 12 Jan 2024 00:01:49 +0100 Subject: [PATCH] [HWORKS-912] remove StagingManager (#1451) --- .../common/upload/StagingManager.java | 113 ------------------ .../common/upload/UploadController.java | 9 +- .../hops/hopsworks/common/util/Settings.java | 9 -- .../common/dataset/util/TestUpload.java | 9 +- 4 files changed, 3 insertions(+), 137 deletions(-) delete mode 100755 hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/StagingManager.java diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/StagingManager.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/StagingManager.java deleted file mode 100755 index 75eaf673e9..0000000000 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/StagingManager.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Changes to this file committed after and not including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b - * are released under the following license: - * - * This file is part of Hopsworks - * Copyright (C) 2018, Logical Clocks AB. All rights reserved - * - * Hopsworks is free software: you can redistribute it and/or modify it under the terms of - * the GNU Affero General Public License as published by the Free Software Foundation, - * either version 3 of the License, or (at your option) any later version. - * - * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - * PURPOSE. See the GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License along with this program. - * If not, see . - * - * Changes to this file committed before and including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b - * are released under the following license: - * - * Copyright (C) 2013 - 2018, Logical Clocks AB and RISE SICS AB. All rights reserved - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this - * software and associated documentation files (the "Software"), to deal in the Software - * without restriction, including without limitation the rights to use, copy, modify, merge, - * publish, distribute, sublicense, and/or sell copies of the Software, and to permit - * persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or - * substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING - * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package io.hops.hopsworks.common.upload; - -import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; -import io.hops.hopsworks.common.hdfs.DistributedFsService; -import io.hops.hopsworks.common.util.Settings; -import org.apache.hadoop.fs.Path; - -import javax.annotation.PostConstruct; -import javax.ejb.DependsOn; -import javax.ejb.EJB; -import javax.ejb.Singleton; -import javax.ejb.TransactionAttribute; -import javax.ejb.TransactionAttributeType; -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Basically provides a temporary folder in which to stage uploaded files. - */ -@Singleton -@DependsOn("Settings") -@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) -public class StagingManager { - private static final Logger LOGGER = Logger.getLogger(StagingManager.class.getName()); - private Path hdfsStagingFolder; - - @EJB - private Settings settings; - @EJB - private DistributedFsService dfs; - - @PostConstruct - public void init() { - hdfsStagingFolder = new Path(settings.getUploadStagingDir()); - createStagingDir(hdfsStagingFolder); - } - - private void createStagingDir(Path stagingDir) { - DistributedFileSystemOps dfsOps = null; - try { - dfsOps = dfs.getDfsOps(); - if (!dfsOps.exists(stagingDir) && !isRootDir(stagingDir.toString())) { - boolean created = dfsOps.mkdirs(stagingDir, dfsOps.getParentPermission(stagingDir)); - if (!created) { - LOGGER.log(Level.WARNING, "Failed to create upload staging dir. Path: {0}", stagingDir.toString()); - } - } - } catch (IOException e) { - LOGGER.log(Level.WARNING, "Failed to create upload staging dir. Path: {0}", stagingDir.toString()); - } finally { - if (dfsOps != null) { - dfs.closeDfsClient(dfsOps); - } - } - } - - public String getStagingPath() { - // if hdfs staging dir is set to Projects return empty b/c upload path will contain /Projects - if (isRootDir(hdfsStagingFolder.toString())) { - return ""; - } - //If staging dir not created or changes recreate - if (hdfsStagingFolder == null || !hdfsStagingFolder.equals(new Path(settings.getUploadStagingDir()))) { - hdfsStagingFolder = new Path(settings.getUploadStagingDir()); - createStagingDir(hdfsStagingFolder); - } - return hdfsStagingFolder.toString(); - } - - private boolean isRootDir(String path) { - return path.equals(Settings.DIR_ROOT) || path.equals("/" + Settings.DIR_ROOT); - } -} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/UploadController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/UploadController.java index 01af3d5cd9..3e743a7f40 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/UploadController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/upload/UploadController.java @@ -40,8 +40,6 @@ public class UploadController { private static final Logger LOGGER = Logger.getLogger(UploadController.class.getName()); - @EJB - private StagingManager stagingManager; @EJB private DistributedFsService dfs; @EJB @@ -51,8 +49,7 @@ public UploadController() { } @VisibleForTesting - public UploadController(StagingManager stagingManager, DistributedFsService dfs, ResumableInfoStorage storage) { - this.stagingManager = stagingManager; + public UploadController(DistributedFsService dfs, ResumableInfoStorage storage) { this.dfs = dfs; this.storage = storage; } @@ -257,9 +254,7 @@ private void createStagingDirIfNotExist(DistributedFileSystemOps dfsOps, Path st } private Path getTmpStagingDir(String hdfsPath, String filename) { - String baseDir = stagingManager.getStagingPath() + hdfsPath; - //get hdfs staging dir staging-dir/hdfs-path/filename.temp - return toTemp(baseDir, filename); + return toTemp(hdfsPath, filename); } private Path toTemp(String baseDir, String filename) { diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java index f3176718e4..e5aed6e2bc 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java @@ -143,7 +143,6 @@ public class Settings implements Serializable { private static final String VARIABLE_HOPSWORKS_USER = "hopsworks_user"; private static final String VARIABLE_JUPYTER_GROUP = "jupyter_group"; private static final String VARIABLE_STAGING_DIR = "staging_dir"; - private static final String VARIABLE_UPLOAD_STAGING_DIR = "upload_staging_dir"; private static final String VARIABLE_AIRFLOW_DIR = "airflow_dir"; private static final String VARIABLE_AIRFLOW_USER = "airflow_user"; private static final String VARIABLE_JUPYTER_DIR = "jupyter_dir"; @@ -641,7 +640,6 @@ private void populateCache() { SPARK_DIR = setDirVar(VARIABLE_SPARK_DIR, SPARK_DIR); FLINK_DIR = setDirVar(VARIABLE_FLINK_DIR, FLINK_DIR); STAGING_DIR = setVar(VARIABLE_STAGING_DIR, STAGING_DIR); - UPLOAD_STAGING_DIR = setVar(VARIABLE_UPLOAD_STAGING_DIR, UPLOAD_STAGING_DIR); HIVE_SUPERUSER = setStrVar(VARIABLE_HIVE_SUPERUSER, HIVE_SUPERUSER); HIVE_WAREHOUSE = setStrVar(VARIABLE_HIVE_WAREHOUSE, HIVE_WAREHOUSE); HIVE_SCRATCHDIR = setStrVar(VARIABLE_HIVE_SCRATCHDIR, HIVE_SCRATCHDIR); @@ -1195,13 +1193,6 @@ public synchronized String getStagingDir() { return STAGING_DIR; } - private String UPLOAD_STAGING_DIR = DIR_ROOT; - - public synchronized String getUploadStagingDir() { - checkCache(); - return UPLOAD_STAGING_DIR; - } - private final String FLINK_CONF_DIR = "conf"; // Remember to change this in docker-images as well private String FLINK_DIR = "/srv/hops/flink"; diff --git a/hopsworks-common/src/test/io/hops/hopsworks/common/dataset/util/TestUpload.java b/hopsworks-common/src/test/io/hops/hopsworks/common/dataset/util/TestUpload.java index 59057fc819..009c484a5c 100644 --- a/hopsworks-common/src/test/io/hops/hopsworks/common/dataset/util/TestUpload.java +++ b/hopsworks-common/src/test/io/hops/hopsworks/common/dataset/util/TestUpload.java @@ -20,7 +20,6 @@ import io.hops.hopsworks.common.hdfs.FsPermissions; import io.hops.hopsworks.common.upload.FlowInfo; import io.hops.hopsworks.common.upload.ResumableInfoStorage; -import io.hops.hopsworks.common.upload.StagingManager; import io.hops.hopsworks.common.upload.UploadController; import io.hops.hopsworks.exceptions.DatasetException; import org.apache.commons.io.FileUtils; @@ -72,22 +71,16 @@ public class TestUpload { private final Path destExistingFilePath = new Path(testUploadPath, "dest/random.txt"); private final DistributedFsService dfs = mock(DistributedFsService.class); private final DistributedFileSystemOps distributedFileSystemOps = mock(DistributedFileSystemOps.class); - private final StagingManager stagingManager = mock(StagingManager.class); private UploadController uploadController; @Before public void setUp() throws IOException { - Mockito.doAnswer((Answer) invocation -> { - LOGGER.log(LEVEL, "Get StagingPath: {0}.", ""); - return ""; - }).when(stagingManager).getStagingPath(); - ResumableInfoStorage storage = new ResumableInfoStorage(); storage.initialize(); setUpDistributedFileSystemOps(); setUpDistributedFsService(); - uploadController = new UploadController(stagingManager, dfs, storage); + uploadController = new UploadController(dfs, storage); } @After