diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java index bf549356a2..e42ee83a15 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java @@ -58,7 +58,6 @@ import io.hops.hopsworks.common.util.templates.jupyter.SparkMagicConfigTemplateBuilder; import io.hops.hopsworks.exceptions.ApiKeyException; import io.hops.hopsworks.exceptions.JobException; -import io.hops.hopsworks.common.hive.HiveController; import io.hops.hopsworks.common.hosts.ServiceDiscoveryController; import io.hops.hopsworks.common.kafka.KafkaBrokers; import io.hops.hopsworks.persistence.entity.jobs.configuration.DockerJobConfiguration; @@ -117,8 +116,6 @@ public class JupyterConfigFilesGenerator { @EJB private KafkaBrokers kafkaBrokers; @EJB - private HiveController hiveController; - @EJB private JobController jobController; @EJB private HdfsUsersController hdfsUsersController; @@ -211,14 +208,12 @@ public void createJupyterKernelConfig(Writer out, Project project, JupyterSettin .setAnacondaHome(settings.getAnacondaProjectDir()) .setSecretDirectory(settings.getStagingDir() + Settings.PRIVATE_DIRS + js.getSecret()) .setProject(project) - .setHiveEndpoints(hiveController.getHiveServerInternalEndpoint()) .setLibHdfsOpts("-Xmx512m") .build(); - Map dataModel = new HashMap<>(1); dataModel.put("kernel", kernelTemplate); templateEngine.template(KernelTemplate.TEMPLATE_NAME, dataModel, out); - } catch (TemplateException | ServiceDiscoveryException ex) { + } catch (TemplateException ex) { throw new IOException(ex); } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java index d4dfdf1d1d..7310ee31cf 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java @@ -20,21 +20,17 @@ import io.hops.hopsworks.common.dao.user.activity.ActivityFacade; import io.hops.hopsworks.common.dataset.DatasetController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewFacade; -import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupFacade; -import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreFacade; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreConnectorFacade; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorDTO; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetFacade; import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; import io.hops.hopsworks.common.hdfs.DistributedFsService; import io.hops.hopsworks.common.hdfs.HdfsUsersController; import io.hops.hopsworks.common.hdfs.inode.InodeController; import io.hops.hopsworks.common.hive.HiveController; -import io.hops.hopsworks.common.hosts.ServiceDiscoveryController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.FeaturestoreException; @@ -49,8 +45,6 @@ import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.activity.ActivityFlag; import io.hops.hopsworks.restutils.RESTCodes; -import io.hops.hopsworks.servicediscovery.HopsworksService; -import io.hops.hopsworks.servicediscovery.tags.HiveTags; import org.apache.hadoop.fs.Path; import javax.ejb.EJB; @@ -78,10 +72,6 @@ public class FeaturestoreController { @EJB private Settings settings; @EJB - private OnlineFeaturestoreController onlineFeaturestoreController; - @EJB - private OnlineFeaturestoreFacade onlineFeaturestoreFacade; - @EJB private HiveController hiveController; @EJB private FeaturegroupFacade featuregroupFacade; @@ -92,8 +82,6 @@ public class FeaturestoreController { @EJB private FeaturestoreStorageConnectorController featurestoreStorageConnectorController; @EJB - private ServiceDiscoveryController serviceDiscoveryController; - @EJB private InodeController inodeController; @EJB private DatasetController datasetController; @@ -268,8 +256,6 @@ public Featurestore createProjectFeatureStore(Project project, Users user, Strin activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + trainingDatasetsFolder. getName(), project, project.getOwner(), ActivityFlag.SERVICE); - featurestoreStorageConnectorController - .createStorageConnector(user, project, featurestore, createOfflineJdbcConnector(featurestoreName)); activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + project.getName(), project, project.getOwner(), ActivityFlag.SERVICE); @@ -314,30 +300,6 @@ public FeaturestoreStorageConnectorDTO hopsfsTrainingDatasetConnector(Dataset ho return featurestoreHopsfsConnectorDTO; } - public FeaturestoreStorageConnectorDTO createOfflineJdbcConnector(String databaseName) - throws FeaturestoreException { - String hiveEndpoint = ""; - try { - hiveEndpoint = serviceDiscoveryController - .constructServiceFQDNWithPort(HopsworksService.HIVE.getNameWithTag(HiveTags.hiveserver2_tls)); - } catch (ServiceDiscoveryException e) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, - Level.SEVERE, "Could not create Hive connection string", e.getMessage(), e); - } - String connectionString = HiveController.HIVE_JDBC_PREFIX + hiveEndpoint + "/" + databaseName + - ";auth=noSasl;ssl=true;twoWay=true;"; - List arguments = FeaturestoreConstants.OFFLINE_JDBC_CONNECTOR_ARGS.stream() - .map(arg -> new OptionDTO(arg, null)) - .collect(Collectors.toList()); - FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO = new FeaturestoreJdbcConnectorDTO(); - featurestoreJdbcConnectorDTO.setStorageConnectorType(FeaturestoreConnectorType.JDBC); - featurestoreJdbcConnectorDTO.setName(databaseName); - featurestoreJdbcConnectorDTO.setDescription("JDBC connector for the Offline Feature Store"); - featurestoreJdbcConnectorDTO.setConnectionString(connectionString); - featurestoreJdbcConnectorDTO.setArguments(arguments); - return featurestoreJdbcConnectorDTO; - } - /** * Converts a featurestore entity to a Featurestore DTO, supplements the featurestore entity * with Hive metadata and remove foreign keys that are less interesting for users. @@ -352,17 +314,9 @@ public FeaturestoreDTO convertFeaturestoreToDTO(Featurestore featurestore) { featurestoreDTO.setFeaturestoreName(featureStoreName); featurestoreDTO.setOfflineFeaturestoreName(featureStoreName); - try { - featurestoreDTO.setHiveEndpoint(hiveController.getHiveServerInternalEndpoint()); - if (settings.isOnlineFeaturestore() && - onlineFeaturestoreController.checkIfDatabaseExists( - onlineFeaturestoreController.getOnlineFeaturestoreDbName(featurestore.getProject()))) { - featurestoreDTO.setMysqlServerEndpoint(onlineFeaturestoreFacade.getJdbcURL()); - featurestoreDTO.setOnlineFeaturestoreName(featurestore.getProject().getName()); - featurestoreDTO.setOnlineEnabled(true); - } - } catch (ServiceDiscoveryException ex) { - throw new RuntimeException(ex); + if (settings.isOnlineFeaturestore()) { + featurestoreDTO.setOnlineFeaturestoreName(featurestore.getProject().getName()); + featurestoreDTO.setOnlineEnabled(true); } // add counters diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreDTO.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreDTO.java index 910382b3b6..74b487aff1 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreDTO.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreDTO.java @@ -36,7 +36,6 @@ public class FeaturestoreDTO { private Integer projectId; private String onlineFeaturestoreName; private String offlineFeaturestoreName; - private String hiveEndpoint; private String mysqlServerEndpoint; private Boolean onlineEnabled = false; private Long numFeatureGroups; @@ -55,7 +54,6 @@ public FeaturestoreDTO(Featurestore featurestore) { this.featurestoreName = null; this.onlineFeaturestoreName = null; this.offlineFeaturestoreName = null; - this.hiveEndpoint = null; this.mysqlServerEndpoint = null; this.onlineEnabled = false; } @@ -103,24 +101,6 @@ public void setOfflineFeaturestoreName(String offlineFeaturestoreName) { this.offlineFeaturestoreName = offlineFeaturestoreName; } - @XmlElement - public String getHiveEndpoint() { - return hiveEndpoint; - } - - public void setHiveEndpoint(String hiveEndpoint) { - this.hiveEndpoint = hiveEndpoint; - } - - @XmlElement - public String getMysqlServerEndpoint() { - return mysqlServerEndpoint; - } - - public void setMysqlServerEndpoint(String mysqlServerEndpoint) { - this.mysqlServerEndpoint = mysqlServerEndpoint; - } - public void setOnlineFeaturestoreName(String onlineFeaturestoreName) { this.onlineFeaturestoreName = onlineFeaturestoreName; } @@ -176,7 +156,6 @@ public String toString() { ", projectId=" + projectId + ", onlineFeaturestoreName='" + onlineFeaturestoreName + '\'' + ", offlineFeaturestoreName='" + offlineFeaturestoreName + '\'' + - ", hiveEndpoint='" + hiveEndpoint + '\'' + ", mysqlServerEndpoint='" + mysqlServerEndpoint + '\'' + ", onlineEnabled=" + onlineEnabled + ", numFeatureGroups=" + numFeatureGroups + diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java index 86193ee778..eac4426ba3 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java @@ -16,23 +16,17 @@ package io.hops.hopsworks.common.featurestore.featuregroup.cached; -import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; -import com.logicalclocks.servicediscoverclient.service.Service; import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.feature.FeatureGroupFeatureDTO; import io.hops.hopsworks.common.hdfs.HdfsUsersController; -import io.hops.hopsworks.common.hosts.ServiceDiscoveryController; -import io.hops.hopsworks.common.security.CertificateMaterializer; +import io.hops.hopsworks.common.hive.HiveController; import io.hops.hopsworks.common.util.Settings; -import io.hops.hopsworks.exceptions.CryptoPasswordNotFoundException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.restutils.RESTCodes; -import io.hops.hopsworks.servicediscovery.HopsworksService; -import io.hops.hopsworks.servicediscovery.tags.HiveTags; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.TableType; @@ -48,13 +42,7 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.thrift.TConfiguration; import org.apache.thrift.TException; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.TSSLTransportFactory; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransport; import javax.annotation.PostConstruct; import javax.ejb.EJB; @@ -83,9 +71,7 @@ public class OfflineFeatureGroupController { @EJB private Settings settings; @EJB - private CertificateMaterializer certificateMaterializer; - @EJB - private ServiceDiscoveryController serviceDiscoveryController; + private HiveController hiveController; private Configuration metastoreConf; @@ -95,9 +81,6 @@ public void init() { metastoreConf.addResource(new Path(settings.getHiveConfPath())); } - private static final String COMMENT = "comment"; - private static final int CONNECTION_TIMEOUT = 600000; - public enum Formats { ORC("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", @@ -166,7 +149,7 @@ public void createHiveTable(Featurestore featurestore, String tableName, try { createTable(client, table, defaultConstraints); } finally { - finalizeMetastoreOperation(project, user, client); + hiveController.finalizeMetastoreOperation(project, user, client); } } @@ -193,7 +176,7 @@ public void alterHiveTableFeatures(Featurestore featurestore, String tableName, alterTable(client, table); addDefaultConstraints(client, defaultConstraints); } finally { - finalizeMetastoreOperation(project, user, client); + hiveController.finalizeMetastoreOperation(project, user, client); } } @@ -209,7 +192,7 @@ public List getSchema(Featurestore featurestore, String schema = getFields(client, dbName, tableName); defaultConstraints = getDefaultConstraints(client, "hive", dbName, tableName); } finally { - finalizeMetastoreOperation(project, user, client); + hiveController.finalizeMetastoreOperation(project, user, client); } // Setup a map of constraint values for easy access @@ -299,95 +282,27 @@ private List getFields(ThriftHiveMetastore.Client client, String db private ThriftHiveMetastore.Client getMetaStoreClient(Project project, Users user) throws FeaturestoreException { try { - return openMetastoreClient(project, user); + return hiveController.openUserMetastoreClient(project, user); } catch (ServiceException | IOException e) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ERROR_CREATING_HIVE_METASTORE_CLIENT, Level.SEVERE, "Error opening the Hive Metastore client: " + e.getMessage(), e.getMessage(), e); } } - private void finalizeMetastoreOperation(Project project, Users user, ThriftHiveMetastore.Client client) { - certificateMaterializer.removeCertificatesLocal(user.getUsername(), project.getName()); - if (client != null) { - try { - client.shutdown(); - } catch (TException e) { - LOGGER.log(Level.SEVERE, "Error closing Metastore connection", e); - } - } - } - public void dropFeatureGroup(String dbName, String tableName, Project project, Users user) throws FeaturestoreException, ServiceException, IOException { ThriftHiveMetastore.Client client = null; try { - client = openMetastoreClient(project, user); + client = hiveController.openUserMetastoreClient(project, user); client.drop_table(dbName, tableName, true); } catch (TException e) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_DELETE_FEATUREGROUP, Level.SEVERE, "Error dropping feature group in the Hive Metastore: " + e.getMessage(), e.getMessage(), e); } finally { - finalizeMetastoreOperation(project, user, client); + hiveController.finalizeMetastoreOperation(project, user, client); } } - // Here we can't use the HiveMetaStoreClient.java wrapper as we would need to export environment variables and so on - // instead we assemble directly the thirft client, which is what the HiveMetaStoreClient does behind the scenes. - private ThriftHiveMetastore.Client openMetastoreClient(Project project, Users user) - throws ServiceException, IOException { - String hdfsUsername = hdfsUsersController.getHdfsUserName(project, user); - ThriftHiveMetastore.Client client = null; - - try { - certificateMaterializer.materializeCertificatesLocal(user.getUsername(), project.getName()); - CertificateMaterializer.CryptoMaterial userMaterial = - certificateMaterializer.getUserMaterial(user.getUsername(), project.getName()); - - // read Password - String password = String.copyValueOf(userMaterial.getPassword()); - - // Get metastore service information from consul - Service metastoreService = serviceDiscoveryController - .getAnyAddressOfServiceWithDNS(HopsworksService.HIVE.getNameWithTag(HiveTags.metastore)); - - TTransport transport; - if (settings.getHopsRpcTls()) { - // Setup secure connection with the Hive metastore. - TSSLTransportFactory.TSSLTransportParameters params = - new TSSLTransportFactory.TSSLTransportParameters(); - params.setTrustStore(certificateMaterializer.getUserTransientTruststorePath(project, user), password); - params.setKeyStore(certificateMaterializer.getUserTransientKeystorePath(project, user), password); - - transport = TSSLTransportFactory.getClientSocket(metastoreService.getAddress(), - metastoreService.getPort(), CONNECTION_TIMEOUT, params); - } else { - transport = new TSocket(TConfiguration.DEFAULT, metastoreService.getAddress(), metastoreService.getPort(), - CONNECTION_TIMEOUT); - } - - TProtocol protocol = new TBinaryProtocol(transport); - client = new ThriftHiveMetastore.Client(protocol); - - // Open transport - if (!transport.isOpen()) { - transport.open(); - } - - // Set the UGI on the metastore side - client.set_ugi(hdfsUsername, new ArrayList<>()); - - if (settings.getHopsRpcTls()) { - // Send the certificate to the metastore so it can operate with the fs. - client.set_crypto(userMaterial.getKeyStore(), password, userMaterial.getTrustStore(), password, false); - } - } catch (CryptoPasswordNotFoundException | ServiceDiscoveryException | TException e) { - throw new ServiceException(RESTCodes.ServiceErrorCode.METASTORE_CONNECTION_ERROR, Level.SEVERE, - "Hive metastore connection error", e.getMessage(), e); - } - - return client; - } - private Table getEmptyTable(String databaseName, String tableName, String username, Formats format) { StorageDescriptor sd = new StorageDescriptor(); { diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/hive/HiveController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/hive/HiveController.java index e7d7c702f0..6c2109f1ff 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/hive/HiveController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/hive/HiveController.java @@ -41,7 +41,6 @@ import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; import com.logicalclocks.servicediscoverclient.service.Service; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.hops.hopsworks.common.dao.dataset.DatasetFacade; import io.hops.hopsworks.common.dao.user.activity.ActivityFacade; import io.hops.hopsworks.common.dataset.DatasetController; @@ -53,46 +52,54 @@ import io.hops.hopsworks.common.provenance.core.HopsFSProvenanceController; import io.hops.hopsworks.common.provenance.core.dto.ProvTypeDTO; import io.hops.hopsworks.common.security.BaseHadoopClientsService; +import io.hops.hopsworks.common.security.CertificateMaterializer; import io.hops.hopsworks.common.util.Settings; +import io.hops.hopsworks.exceptions.CryptoPasswordNotFoundException; import io.hops.hopsworks.exceptions.ProvenanceException; +import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.persistence.entity.dataset.Dataset; import io.hops.hopsworks.persistence.entity.dataset.DatasetAccessPermission; import io.hops.hopsworks.persistence.entity.dataset.DatasetType; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.hdfs.inode.Inode; -import io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers; import io.hops.hopsworks.persistence.entity.log.operation.OperationType; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.activity.ActivityFlag; +import io.hops.hopsworks.restutils.RESTCodes; import io.hops.hopsworks.servicediscovery.HopsworksService; import io.hops.hopsworks.servicediscovery.tags.HiveTags; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TSSLTransportFactory; +import org.apache.thrift.transport.TTransport; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import javax.ejb.EJB; import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.logging.Level; import java.util.logging.Logger; @Stateless(name = "HiveController") +@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) public class HiveController { public final static String HIVE_JDBC_PREFIX = "jdbc:hopshive://"; public final static String HIVE_DRIVER = "io.hops.hive.jdbc.HiveDriver"; + public final static int CONNECTION_TIMEOUT = 600000; @EJB private Settings settings; @EJB - private HdfsUsersController hdfsUsersBean; + private HdfsUsersController hdfsUsersController; @EJB private InodeController inodeController; @EJB @@ -107,47 +114,11 @@ public class HiveController { private HopsFSProvenanceController fsProvenanceCtrl; @EJB private ServiceDiscoveryController serviceDiscoveryController; + @EJB + private CertificateMaterializer certificateMaterializer; private final static Logger logger = Logger.getLogger(HiveController.class.getName()); - private Connection conn; - private String jdbcString = null; - - @PostConstruct - public void init() { - try { - // Load Hive JDBC Driver - Class.forName(HIVE_DRIVER); - } catch (ClassNotFoundException e) { - logger.log(Level.SEVERE, "Could not load the Hive driver: " + HIVE_DRIVER, e); - } - } - - private void initConnection() throws SQLException, ServiceDiscoveryException { - // Create connection url - String hiveEndpoint = getHiveServerInternalEndpoint(); - jdbcString = HIVE_JDBC_PREFIX + hiveEndpoint + "/default;" + - "auth=noSasl;ssl=true;twoWay=true;" + - "sslTrustStore=" + bhcs.getSuperTrustStorePath() + ";" + - "trustStorePassword=" + bhcs.getSuperTrustStorePassword() + ";" + - "sslKeyStore=" + bhcs.getSuperKeystorePath() + ";" + - "keyStorePassword=" + bhcs.getSuperKeystorePassword(); - - conn = DriverManager.getConnection(jdbcString); - } - - @PreDestroy - public void close() { - try { - if (conn != null && !conn.isClosed()) { - conn.close(); - } - } catch (SQLException e) { - logger.log(Level.WARNING, "Error closing Hive JDBC connection: " + - e); - } - } - /** * Creates a Hopsworks dataset of a Hive database * @@ -157,7 +128,6 @@ public void close() { * @param dbName name of the hive database * @throws IOException */ - @TransactionAttribute(TransactionAttributeType.NEVER) public void createDatasetDb(Project project, Users user, DistributedFileSystemOps dfso, String dbName, ProvTypeDTO metaStatus) throws IOException { createDatasetDb(project, user, dfso, dbName, DatasetType.HIVEDB, null, metaStatus); @@ -175,7 +145,6 @@ public void createDatasetDb(Project project, Users user, DistributedFileSystemOp * defaults to null. * @throws IOException */ - @TransactionAttribute(TransactionAttributeType.NEVER) public void createDatasetDb(Project project, Users user, DistributedFileSystemOps dfso, String dbName, DatasetType datasetType, Featurestore featurestore, ProvTypeDTO metaStatus) throws IOException { @@ -183,24 +152,23 @@ public void createDatasetDb(Project project, Users user, DistributedFileSystemOp throw new IllegalArgumentException("Invalid dataset type for hive database"); } - // Hive database names are case insensitive and lower case - Path dbPath = getDbPath(dbName); - Inode dbInode = inodeController.getInodeAtPath(dbPath.toString()); - // Persist Hive db as dataset in the Hopsworks database // Make the dataset editable by owners by default - Dataset dbDataset = new Dataset(project, dbInode.getInodePK().getName(), - DatasetAccessPermission.EDITABLE_BY_OWNERS); + Dataset dbDataset = new Dataset(project, getDbDirName(dbName), DatasetAccessPermission.EDITABLE_BY_OWNERS); dbDataset.setDsType(datasetType); dbDataset.setSearchable(true); dbDataset.setFeatureStore(featurestore); datasetFacade.persistDataset(dbDataset); + // Hive database names are case insensitive and lower case + Path dbPath = getDbPath(dbName); + try { // Assign database directory to the user and project group - hdfsUsersBean.createDatasetGroupsAndSetPermissions(user, project, dbDataset, dbPath, dfso); + hdfsUsersController.createDatasetGroupsAndSetPermissions(user, project, dbDataset, dbPath, dfso); fsProvenanceCtrl.updateHiveDatasetProvCore(project, dbPath.toString(), metaStatus, dfso); + Inode dbInode = inodeController.getInodeAtPath(dbPath.toString()); datasetController.logDataset(project, dbDataset, dbInode, OperationType.Add); activityFacade.persistActivity(ActivityFacade.NEW_DATA + dbDataset.getName(), project, user, ActivityFlag.DATASET); @@ -215,7 +183,7 @@ public void createDatasetDb(Project project, Users user, DistributedFileSystemOp break; } } catch (IOException | ProvenanceException e) { - logger.log(Level.SEVERE, "Cannot assign Hive database directory " + dbPath.toString() + + logger.log(Level.SEVERE, "Cannot assign Hive database directory " + dbPath + " to correct user/group. Trace: " + e); // Remove the database directory and cleanup the metadata @@ -223,64 +191,168 @@ public void createDatasetDb(Project project, Users user, DistributedFileSystemOp dfso.rm(dbPath, true); } catch (IOException rmEx) { // Nothing we can really do here - logger.log(Level.SEVERE, "Cannot delete Hive database directory: " + dbPath.toString() + " Trace: " + rmEx); + logger.log(Level.SEVERE, "Cannot delete Hive database directory: " + dbPath + " Trace: " + rmEx); } throw new IOException(e); } } - /** - * Creates the Hive Database - * - * @param dbName name of the database - * @param dbComment description of the database - * @throws SQLException - */ - @SuppressFBWarnings(justification = "Not called with user input", value = "SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE") - @TransactionAttribute(TransactionAttributeType.NEVER) - public void createDatabase(String dbName, String dbComment) - throws SQLException, ServiceDiscoveryException { - if (conn == null || conn.isClosed()) { - initConnection(); + public void dropDatabases(Project project, DistributedFileSystemOps dfso, boolean forceCleanup) + throws IOException, ServiceException { + // To avoid case sensitive bugs, check if the project has a Hive database + Dataset projectDs = datasetController + .getByProjectAndDsName(project, this.settings.getHiveWarehouse(), project.getName().toLowerCase() + ".db"); + Dataset featurestoreDs = datasetController.getByProjectAndDsName(project, this.settings.getHiveWarehouse(), + project.getName().toLowerCase() + FeaturestoreConstants.FEATURESTORE_HIVE_DB_SUFFIX + ".db"); + if ((projectDs != null && projectDs.getDsType() == DatasetType.HIVEDB) || forceCleanup) { + dropDatabase(project.getName(), project, dfso); + } + + if ((featurestoreDs != null && featurestoreDs.getDsType() == DatasetType.FEATURESTORE) || forceCleanup) { + dropDatabase(project.getName() + FeaturestoreConstants.FEATURESTORE_HIVE_DB_SUFFIX, project, dfso); } + } + + public void createDatabase(String dbName, String dbDescription) throws IOException, ServiceException { + // Create Hive database object + Database database = new Database(); + database.setName(dbName); + database.setDescription(dbDescription); - Statement stmt = null; + // Connect to metastore and create the database + ThriftHiveMetastore.Client client = null; try { - // Create database - stmt = conn.createStatement(); - // Project name cannot include any spacial character or space. - stmt.executeUpdate("create database " + dbName + " COMMENT '" + dbComment + "'"); + client = openSuperMetastoreClient(); + client.create_database(database); + } catch (TException e) { + throw new IOException(e); } finally { - if (stmt != null) { - stmt.close(); + finalizeMetastoreOperation(null, null, client); + } + } + + private void dropDatabase(String dbName, Project project, DistributedFileSystemOps dfso) + throws IOException, ServiceException { + // Connect to metastore and create the database + ThriftHiveMetastore.Client superClient = null; + ThriftHiveMetastore.Client projectOwnerClient = null; + + try { + projectOwnerClient = openUserMetastoreClient(project, project.getOwner()); + // Drop all hive tables as project owner + for (String tableName : projectOwnerClient.get_all_tables(dbName)) { + projectOwnerClient.drop_table(dbName, tableName, true); } + + Path dbPath = getDbPath(dbName); + // User the DFSo to delete the storage_connector_resources + dfso.rm(new Path(dbPath, FeaturestoreConstants.STORAGE_CONNECTOR_SUBDIR), true); + + // chown the database back to the users + dfso.setOwner(dbPath, settings.getHiveSuperUser(), settings.getHiveSuperUser()); + + // The database is now empty and it can be deleted by the super user + superClient = openSuperMetastoreClient(); + superClient.drop_database(dbName, true, true); + } catch (TException e) { + throw new IOException(e); + } finally { + finalizeMetastoreOperation(null, null, superClient); + finalizeMetastoreOperation(project, project.getOwner(), projectOwnerClient); } } - public void dropDatabases(Project project, DistributedFileSystemOps dfso, boolean forceCleanup) - throws IOException { - // To avoid case sensitive bugs, check if the project has a Hive database - Dataset projectDs = datasetController - .getByProjectAndDsName(project, this.settings.getHiveWarehouse(), project.getName().toLowerCase() + ".db"); - Dataset featurestoreDs = datasetController.getByProjectAndDsName(project, this.settings.getHiveWarehouse(), - project.getName().toLowerCase() + FeaturestoreConstants.FEATURESTORE_HIVE_DB_SUFFIX + ".db"); - if ((projectDs != null && projectDs.getDsType() == DatasetType.HIVEDB) - || forceCleanup) { - dropDatabase(project, dfso, project.getName()); + private ThriftHiveMetastore.Client openSuperMetastoreClient() throws ServiceException, IOException { + return openMetastoreClient(settings.getHopsworksUser(), + bhcs.getSuperKeystorePath(), + bhcs.getSuperKeystorePassword(), + bhcs.getSuperKeystore(), + bhcs.getSuperTrustStorePath(), + bhcs.getSuperTrustStorePassword(), + bhcs.getSuperTrustStore()); + } + + // Here we can't use the HiveMetaStoreClient.java wrapper as we would need to export environment variables and so on + // instead we assemble directly the thirft client, which is what the HiveMetaStoreClient does behind the scenes. + public ThriftHiveMetastore.Client openUserMetastoreClient(Project project, Users user) + throws ServiceException, IOException { + String hdfsUsername = hdfsUsersController.getHdfsUserName(project, user); + try { + certificateMaterializer.materializeCertificatesLocal(user.getUsername(), project.getName()); + CertificateMaterializer.CryptoMaterial userMaterial = + certificateMaterializer.getUserMaterial(user.getUsername(), project.getName()); + + // read Password + String password = String.copyValueOf(userMaterial.getPassword()); + + return openMetastoreClient(hdfsUsername, + certificateMaterializer.getUserTransientKeystorePath(project, user), + password, + userMaterial.getKeyStore(), + certificateMaterializer.getUserTransientTruststorePath(project, user), + password, + userMaterial.getTrustStore() + ); + } catch (CryptoPasswordNotFoundException e) { + throw new ServiceException(RESTCodes.ServiceErrorCode.METASTORE_CONNECTION_ERROR, Level.SEVERE, + "Hive metastore connection error", e.getMessage(), e); } + } + + private ThriftHiveMetastore.Client openMetastoreClient(String user, + String keyStorePath, + String keyStorePassword, + ByteBuffer keyStore, + String trustStorePath, + String trustStorePassword, + ByteBuffer trustStore) throws ServiceException { + ThriftHiveMetastore.Client client = null; + try { + // Get metastore service information from consul + Service metastoreService = serviceDiscoveryController + .getAnyAddressOfServiceWithDNS(HopsworksService.HIVE.getNameWithTag(HiveTags.metastore)); + + // Setup secure connection with the Hive metastore. + TSSLTransportFactory.TSSLTransportParameters params = + new TSSLTransportFactory.TSSLTransportParameters(); + params.setTrustStore(trustStorePath, trustStorePassword); + params.setKeyStore(keyStorePath, keyStorePassword); + + TTransport transport = TSSLTransportFactory.getClientSocket(metastoreService.getAddress(), + metastoreService.getPort(), CONNECTION_TIMEOUT, params); + + TProtocol protocol = new TBinaryProtocol(transport); + client = new ThriftHiveMetastore.Client(protocol); - if ((featurestoreDs != null && featurestoreDs.getDsType() == DatasetType.FEATURESTORE) - || forceCleanup) { - dropDatabase(project, dfso, project.getName() + FeaturestoreConstants.FEATURESTORE_HIVE_DB_SUFFIX); + // Open transport + if (!transport.isOpen()) { + transport.open(); + } + + // Set the UGI on the metastore side + client.set_ugi(user, new ArrayList<>()); + + // Send the certificate to the metastore so it can operate with the fs. + client.set_crypto(keyStore, keyStorePassword, trustStore, trustStorePassword, false); + } catch (ServiceDiscoveryException | TException e) { + throw new ServiceException(RESTCodes.ServiceErrorCode.METASTORE_CONNECTION_ERROR, Level.SEVERE, + "Hive metastore connection error", e.getMessage(), e); } + + return client; } - private void dropDatabase(Project project, DistributedFileSystemOps dfso, String dbName) throws IOException { - // Delete HopsFs db directory -- will automatically clean up all the related Hive's metadata - dfso.rm(getDbPath(dbName), true); - // Delete all the scratchdirs - for (HdfsUsers u : hdfsUsersBean.getAllProjectHdfsUsers(project.getName())) { - dfso.rm(new Path(settings.getHiveScratchdir(), u.getName()), true); + public void finalizeMetastoreOperation(Project project, Users user, ThriftHiveMetastore.Client client) { + if (project != null && user != null) { + certificateMaterializer.removeCertificatesLocal(user.getUsername(), project.getName()); + } + + if (client != null) { + try { + client.shutdown(); + } catch (TException e) { + logger.log(Level.SEVERE, "Error closing Metastore connection", e); + } } } @@ -288,12 +360,10 @@ public Path getDbPath(String dbName) { return new Path(settings.getHiveWarehouse(), dbName.toLowerCase() + ".db"); } - @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) - public String getHiveServerExternalEndpoint() throws ServiceDiscoveryException { - return getHiveServerEndpoint(HiveTags.hiveserver2_plain); + public String getDbDirName(String dbName) { + return dbName.toLowerCase() + ".db"; } - @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) public String getHiveServerInternalEndpoint() throws ServiceDiscoveryException { return getHiveServerEndpoint(HiveTags.hiveserver2_tls); } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/hive/HiveTableType.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/hive/HiveTableType.java deleted file mode 100644 index be1fb4ae56..0000000000 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/hive/HiveTableType.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * This file is part of Hopsworks - * Copyright (C) 2019, Logical Clocks AB. All rights reserved - * - * Hopsworks is free software: you can redistribute it and/or modify it under the terms of - * the GNU Affero General Public License as published by the Free Software Foundation, - * either version 3 of the License, or (at your option) any later version. - * - * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - * PURPOSE. See the GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License along with this program. - * If not, see . - */ - -package io.hops.hopsworks.common.hive; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Type of Hive Table - */ -public enum HiveTableType { - @JsonProperty("MANAGED_TABLE") - MANAGED_TABLE, - @JsonProperty("EXTERNAL_TABLE") - EXTERNAL_TABLE; -} \ No newline at end of file diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java index 3d18e90088..e18e786cfa 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java @@ -40,7 +40,6 @@ package io.hops.hopsworks.common.project; import com.google.common.annotations.VisibleForTesting; -import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; import io.hops.hopsworks.alert.AMClient; import io.hops.hopsworks.alert.exception.AlertManagerUnreachableException; import io.hops.hopsworks.alerting.api.alert.dto.PostableAlert; @@ -890,9 +889,9 @@ private void addServiceHive(Project project, Users user, DistributedFileSystemOp try { hiveController.createDatabase(project.getName(), "Project general-purpose Hive database"); hiveController.createDatasetDb(project, user, dfso, project.getName(), datasetProvCore); - } catch (SQLException | IOException | ServiceDiscoveryException ex) { + } catch (ServiceException | IOException ex) { throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_HIVEDB_CREATE_ERROR, Level.SEVERE, - "project: " + project.getName(), ex.getMessage(), ex); + "project: " + project.getName(), ex.getMessage(), ex); } } @@ -1014,7 +1013,7 @@ private void addServiceFeaturestore(Project project, Users user, datasetProvCore); //Register built-in transformation function. transformationFunctionController.registerBuiltInTransformationFunctions(user, project, featurestore); - } catch (SQLException | IOException | ServiceDiscoveryException ex) { + } catch (ServiceException | IOException ex) { LOGGER.log(Level.SEVERE, RESTCodes.FeaturestoreErrorCode.COULD_NOT_CREATE_FEATURESTORE.getMessage(), ex); throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_CREATE_FEATURESTORE, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex); @@ -1210,6 +1209,15 @@ public String[] forceCleanup(String projectName, Users user) { cleanupLogger.logError(ex.getMessage()); } + // Delete Hive database - will automatically cleanup all the Hive's metadata + try { + hiveController.dropDatabases(project, dfso, true); + cleanupLogger.logSuccess("Removed Hive db"); + } catch (Exception ex) { + cleanupLogger.logError("Error when removing hive db during project cleanup"); + cleanupLogger.logError(ex.getMessage()); + } + // Remove certificates try { certificatesController.revokeProjectCertificates(project); @@ -1253,16 +1261,7 @@ public String[] forceCleanup(String projectName, Users user) { cleanupLogger.logError("Error when changing ownership during project cleanup"); cleanupLogger.logError(ex.getMessage()); } - - // 16) Delete Hive database - will automatically cleanup all the Hive's metadata - try { - hiveController.dropDatabases(project, dfso, true); - cleanupLogger.logSuccess("Removed Hive db"); - } catch (Exception ex) { - cleanupLogger.logError("Error when removing hive db during project cleanup"); - cleanupLogger.logError(ex.getMessage()); - } - + // 17) Delete online featurestore database try { onlineFeaturestoreController.removeOnlineFeatureStore(project); @@ -1387,7 +1386,7 @@ public String[] forceCleanup(String projectName, Users user) { try { hiveController.dropDatabases(toDeleteProject, dfso, true); cleanupLogger.logSuccess("Dropped Hive database"); - } catch (IOException ex) { + } catch (IOException | ServiceException ex) { cleanupLogger.logError(ex.getMessage()); } @@ -1656,6 +1655,9 @@ private void removeProjectInt(Project project, List usersToClean, //remove kafka topics kafkaController.removeKafkaTopics(project); + //Delete Hive database - will automatically cleanup all the Hive's metadata + hiveController.dropDatabases(project, dfso, false); + // remove user certificate from local node // (they will be removed from db when the project folder is deleted) // projectCreationFutures will be null during project deletion. @@ -1703,9 +1705,6 @@ private void removeProjectInt(Project project, List usersToClean, //Delete online featurestore database onlineFeaturestoreController.removeOnlineFeatureStore(project); - //Delete Hive database - will automatically cleanup all the Hive's metadata - hiveController.dropDatabases(project, dfso, false); - try { //Delete OpenSearch template for this project removeOpenSearch(project); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/BaseHadoopClientsService.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/BaseHadoopClientsService.java index 21c882d20d..3e3d47818d 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/BaseHadoopClientsService.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/BaseHadoopClientsService.java @@ -43,6 +43,7 @@ import io.hops.hopsworks.common.util.Settings; import io.hops.security.HopsUtil; import io.hops.security.SuperuserKeystoresLoader; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.net.HopsSSLSocketFactory; @@ -53,6 +54,7 @@ import javax.ejb.Stateless; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Paths; import java.util.logging.Level; import java.util.logging.Logger; @@ -98,15 +100,25 @@ public void init() { public String getSuperKeystorePath() { return securityMaterial.getKeyStoreLocation().toString(); } + + public ByteBuffer getSuperKeystore() throws IOException { + byte[] keystore = FileUtils.readFileToByteArray(securityMaterial.getKeyStoreLocation().toFile()); + return ByteBuffer.wrap(keystore); + } public String getSuperKeystorePassword() { return materialPassword; } - + public String getSuperTrustStorePath() { return securityMaterial.getTrustStoreLocation().toString(); } - + + public ByteBuffer getSuperTrustStore() throws IOException { + byte[] trustStore = FileUtils.readFileToByteArray(securityMaterial.getTrustStoreLocation().toFile()); + return ByteBuffer.wrap(trustStore); + } + public String getSuperTrustStorePassword() { return materialPassword; } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/templates/jupyter/KernelTemplate.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/templates/jupyter/KernelTemplate.java index e76b38eb7c..74689967d6 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/templates/jupyter/KernelTemplate.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/templates/jupyter/KernelTemplate.java @@ -23,7 +23,6 @@ public class KernelTemplate extends JupyterTemplate { private final String anacondaHome; private final String hadoopVersion; private final String secretDirectory; - private final String hiveEndpoints; private final String libHdfsOpts; public KernelTemplate(KernelTemplateBuilder builder) { @@ -31,7 +30,6 @@ public KernelTemplate(KernelTemplateBuilder builder) { this.anacondaHome = builder.getAnacondaHome(); this.hadoopVersion = builder.getHadoopVersion(); this.secretDirectory = builder.getSecretDirectory(); - this.hiveEndpoints = builder.getHiveEndpoints(); this.libHdfsOpts = builder.getLibHdfsOpts(); } @@ -47,10 +45,6 @@ public String getSecretDirectory() { return secretDirectory; } - public String getHiveEndpoints() { - return hiveEndpoints; - } - public String getLibHdfsOpts() { return libHdfsOpts; } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/templates/jupyter/KernelTemplateBuilder.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/templates/jupyter/KernelTemplateBuilder.java index 99d20fb932..573b19f720 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/templates/jupyter/KernelTemplateBuilder.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/templates/jupyter/KernelTemplateBuilder.java @@ -26,7 +26,6 @@ public class KernelTemplateBuilder { private String anacondaHome; private String hadoopVersion; private String secretDirectory; - private String hiveEndpoints; private String libHdfsOpts; private KernelTemplateBuilder() {} @@ -88,11 +87,6 @@ public KernelTemplateBuilder setSecretDirectory(String secretDirectory) { public String getSecretDirectory() { return secretDirectory; } - - public KernelTemplateBuilder setHiveEndpoints(String hiveEndpoints) { - this.hiveEndpoints = hiveEndpoints; - return this; - } public String getLibHdfsOpts() { return libHdfsOpts; @@ -102,11 +96,7 @@ public KernelTemplateBuilder setLibHdfsOpts(String libHdfsOpts) { this.libHdfsOpts = libHdfsOpts; return this; } - - public String getHiveEndpoints() { - return hiveEndpoints; - } - + public KernelTemplate build() { return new KernelTemplate(this); }