From 192e1890e8ec1119050636825274c65ee1ae1ebe Mon Sep 17 00:00:00 2001 From: rkimmel-hv Date: Thu, 18 May 2023 10:30:36 -0400 Subject: [PATCH 1/4] BACKLOG-37664 POC to make pentaho server work with vfs metastore --- .../di/repository/pur/PurRepository.java | 36 +++++++++---------- .../pur/AbsSecurityProviderTest.java | 3 +- .../repository/pur/PurRepositoryUnitTest.java | 11 +++--- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java b/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java index b473e07c1f71..9e547d200c6f 100644 --- a/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java +++ b/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java @@ -36,17 +36,19 @@ import org.pentaho.di.core.exception.IdNotFoundException; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleFileException; +import org.pentaho.di.core.exception.KettlePluginException; import org.pentaho.di.core.exception.KettleSecurityException; import org.pentaho.di.core.extension.ExtensionPointHandler; import org.pentaho.di.core.extension.KettleExtensionPoint; import org.pentaho.di.core.logging.LogChannel; import org.pentaho.di.core.logging.LogChannelInterface; +import org.pentaho.di.core.osgi.api.MetastoreLocatorOsgi; +import org.pentaho.di.core.service.PluginServiceLoader; import org.pentaho.di.core.util.Utils; import org.pentaho.di.i18n.BaseMessages; import org.pentaho.di.imp.Import; import org.pentaho.di.job.JobMeta; import org.pentaho.di.laf.BasePropertyHandler; -import org.pentaho.di.metastore.MetaStoreConst; import org.pentaho.di.partition.PartitionSchema; import org.pentaho.di.repository.AbstractRepository; import org.pentaho.di.repository.IRepositoryExporter; @@ -113,6 +115,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.EnumMap; @@ -221,7 +224,7 @@ public class PurRepository extends AbstractRepository implements Repository, Rec private String connectMessage = null; - protected PurRepositoryMetaStore metaStore; + protected IMetaStore metaStore; private ConnectionManager connectionManager = ConnectionManager.getInstance(); @@ -237,12 +240,16 @@ protected enum RepositoryServers { private static final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); private static final ReadWriteLock sharedObjectsLock = new ReentrantReadWriteLock(); + private final MetastoreLocatorOsgi metastoreLocator; + // ~ Constructors ==================================================================================================== - public PurRepository() { + public PurRepository() throws KettlePluginException { super(); initSharedObjectAssemblerMap(); + Collection metastoreLocators = PluginServiceLoader.loadServices( MetastoreLocatorOsgi.class ); + metastoreLocator = metastoreLocators.stream().findFirst().get(); } // ~ Methods ========================================================================================================= @@ -290,7 +297,10 @@ public RootRef getRootRef() { purRepositoryServiceRegistry.registerService( ILockService.class, new UnifiedRepositoryLockService( pur ) ); purRepositoryServiceRegistry .registerService( IAclService.class, new UnifiedRepositoryConnectionAclService( pur ) ); - metaStore = new PurRepositoryMetaStore( this ); + metaStore = metastoreLocator.getExplicitMetastore( "VfsMetastoreProvider" ); + if ( metaStore == null ) { + metaStore = new PurRepositoryMetaStore( this ); + } try { metaStore.createNamespace( PentahoDefaults.NAMESPACE ); } catch ( MetaStoreException e ) { @@ -342,11 +352,9 @@ public RootRef getRootRef() { if ( log.isDetailed() ) { log.logDetailed( BaseMessages.getString( PKG, "PurRepositoryMetastore.Create.Message" ) ); } - metaStore = new PurRepositoryMetaStore( this ); - IMetaStore activeMetaStore = metaStore; - if ( activeMetaStore != null ) { - final IMetaStore connectedMetaStore = activeMetaStore; - connectionManager.setMetastoreSupplier( () -> connectedMetaStore ); + metaStore = metastoreLocator.getExplicitMetastore( "VfsMetastoreProvider" ); + if ( metaStore == null ) { + metaStore = new PurRepositoryMetaStore( this ); } // Create the default Pentaho namespace if it does not exist @@ -380,16 +388,6 @@ public RootRef getRootRef() { @Override public void disconnect() { connected = false; metaStore = null; - IMetaStore activeMetaStore = null; - try { - activeMetaStore = MetaStoreConst.openLocalPentahoMetaStore(); - } catch ( MetaStoreException e ) { - activeMetaStore = null; - } - if ( activeMetaStore != null ) { - final IMetaStore connectedMetaStore = activeMetaStore; - connectionManager.setMetastoreSupplier( () -> connectedMetaStore ); - } purRepositoryConnector.disconnect(); } diff --git a/plugins/pur/core/src/test/java/org/pentaho/di/repository/pur/AbsSecurityProviderTest.java b/plugins/pur/core/src/test/java/org/pentaho/di/repository/pur/AbsSecurityProviderTest.java index 01a2f840ea74..d0f75b9885a5 100644 --- a/plugins/pur/core/src/test/java/org/pentaho/di/repository/pur/AbsSecurityProviderTest.java +++ b/plugins/pur/core/src/test/java/org/pentaho/di/repository/pur/AbsSecurityProviderTest.java @@ -20,6 +20,7 @@ import org.junit.Before; import org.junit.Test; import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettlePluginException; import org.pentaho.di.repository.RepositoryOperation; import org.pentaho.di.repository.UserInfo; import org.pentaho.di.ui.repository.pur.services.IAbsSecurityProvider; @@ -32,7 +33,7 @@ public class AbsSecurityProviderTest { private AbsSecurityProvider provider; @Before - public void setUp() { + public void setUp() throws KettlePluginException { provider = new AbsSecurityProvider( new PurRepository(), new PurRepositoryMeta(), new UserInfo(), mock( ServiceManager.class ) ); provider = spy( provider ); diff --git a/plugins/pur/core/src/test/java/org/pentaho/di/repository/pur/PurRepositoryUnitTest.java b/plugins/pur/core/src/test/java/org/pentaho/di/repository/pur/PurRepositoryUnitTest.java index 8c248860148a..f91309e7f23e 100644 --- a/plugins/pur/core/src/test/java/org/pentaho/di/repository/pur/PurRepositoryUnitTest.java +++ b/plugins/pur/core/src/test/java/org/pentaho/di/repository/pur/PurRepositoryUnitTest.java @@ -32,6 +32,7 @@ import org.mockito.Mockito; import org.pentaho.di.core.Const; import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettlePluginException; import org.pentaho.di.core.extension.ExtensionPointInterface; import org.pentaho.di.core.extension.ExtensionPointPluginType; import org.pentaho.di.core.extension.KettleExtensionPoint; @@ -502,7 +503,7 @@ public void testSaveTransOrJob() throws KettleException { } @Test - public void testGetJobPathWithoutExtension() { + public void testGetJobPathWithoutExtension() throws KettlePluginException { PurRepository pur = new PurRepository(); RepositoryDirectoryInterface rdi = mock( RepositoryDirectoryInterface.class ); doReturn( mock( ObjectId.class ) ).when( rdi ).getObjectId(); @@ -512,7 +513,7 @@ public void testGetJobPathWithoutExtension() { } @Test - public void testGetJobPathWithExtension() { + public void testGetJobPathWithExtension() throws KettlePluginException { PurRepository pur = new PurRepository(); RepositoryDirectoryInterface rdi = mock( RepositoryDirectoryInterface.class ); doReturn( mock( ObjectId.class ) ).when( rdi ).getObjectId(); @@ -522,7 +523,7 @@ public void testGetJobPathWithExtension() { } @Test - public void testGetTransPathWithoutExtension() { + public void testGetTransPathWithoutExtension() throws KettlePluginException { PurRepository pur = new PurRepository(); RepositoryDirectoryInterface rdi = mock( RepositoryDirectoryInterface.class ); doReturn( mock( ObjectId.class ) ).when( rdi ).getObjectId(); @@ -532,7 +533,7 @@ public void testGetTransPathWithoutExtension() { } @Test - public void testGetTransPathWithExtension() { + public void testGetTransPathWithExtension() throws KettlePluginException { PurRepository pur = new PurRepository(); RepositoryDirectoryInterface rdi = mock( RepositoryDirectoryInterface.class ); doReturn( mock( ObjectId.class ) ).when( rdi ).getObjectId(); @@ -618,7 +619,7 @@ public void testCreateValidRepositoryDirectoryForNotRoot() throws KettleExceptio @Test @SuppressWarnings( {"squid:S1075", "squid:S3655"} ) - public void testGetURI() throws URISyntaxException { + public void testGetURI() throws URISyntaxException, KettlePluginException { PurRepository purRepo = new PurRepository(); PurRepositoryMeta repoMeta = new PurRepositoryMeta(); repoMeta.setRepositoryLocation( new PurRepositoryLocation( "http://localhost:8080/pentaho" ) ); From d0142b22e4196f98c607a6bbcf79f45618a54f18 Mon Sep 17 00:00:00 2001 From: rkimmel-hv Date: Thu, 18 May 2023 14:22:35 -0400 Subject: [PATCH 2/4] Add back setting connection manager metastore just in case --- .../org/pentaho/di/repository/pur/PurRepository.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java b/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java index 9e547d200c6f..9569644c3009 100644 --- a/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java +++ b/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java @@ -49,6 +49,7 @@ import org.pentaho.di.imp.Import; import org.pentaho.di.job.JobMeta; import org.pentaho.di.laf.BasePropertyHandler; +import org.pentaho.di.metastore.MetaStoreConst; import org.pentaho.di.partition.PartitionSchema; import org.pentaho.di.repository.AbstractRepository; import org.pentaho.di.repository.IRepositoryExporter; @@ -356,6 +357,8 @@ public RootRef getRootRef() { if ( metaStore == null ) { metaStore = new PurRepositoryMetaStore( this ); } + final IMetaStore connectedMetaStore = metaStore; + connectionManager.setMetastoreSupplier( () -> connectedMetaStore ); // Create the default Pentaho namespace if it does not exist try { @@ -388,6 +391,15 @@ public RootRef getRootRef() { @Override public void disconnect() { connected = false; metaStore = null; + try { + metaStore = MetaStoreConst.openLocalPentahoMetaStore(); + } catch ( MetaStoreException e ) { + metaStore = null; + } + if ( metaStore != null ) { + final IMetaStore connectedMetaStore = metaStore; + connectionManager.setMetastoreSupplier( () -> connectedMetaStore ); + } purRepositoryConnector.disconnect(); } From 54b9197bd01e74722d481b0c8e70f880170795c8 Mon Sep 17 00:00:00 2001 From: rkimmel-hv Date: Mon, 22 May 2023 10:00:58 -0400 Subject: [PATCH 3/4] BACKLOG-37664 use constant for metstore lookup --- .../java/org/pentaho/di/repository/pur/PurRepository.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java b/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java index 9569644c3009..b4d9b062dc71 100644 --- a/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java +++ b/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java @@ -150,6 +150,7 @@ public class PurRepository extends AbstractRepository implements Repository, Rec // Kettle property that when set to false disabled the lazy repository access public static final String LAZY_REPOSITORY = "KETTLE_LAZY_REPOSITORY"; + public static final String VFS_METASTORE_PROVIDER = "VfsMetastoreProvider"; private static Class PKG = PurRepository.class; @@ -298,7 +299,7 @@ public RootRef getRootRef() { purRepositoryServiceRegistry.registerService( ILockService.class, new UnifiedRepositoryLockService( pur ) ); purRepositoryServiceRegistry .registerService( IAclService.class, new UnifiedRepositoryConnectionAclService( pur ) ); - metaStore = metastoreLocator.getExplicitMetastore( "VfsMetastoreProvider" ); + metaStore = metastoreLocator.getExplicitMetastore( VFS_METASTORE_PROVIDER ); if ( metaStore == null ) { metaStore = new PurRepositoryMetaStore( this ); } @@ -353,7 +354,7 @@ public RootRef getRootRef() { if ( log.isDetailed() ) { log.logDetailed( BaseMessages.getString( PKG, "PurRepositoryMetastore.Create.Message" ) ); } - metaStore = metastoreLocator.getExplicitMetastore( "VfsMetastoreProvider" ); + metaStore = metastoreLocator.getExplicitMetastore( VFS_METASTORE_PROVIDER ); if ( metaStore == null ) { metaStore = new PurRepositoryMetaStore( this ); } From 718a3bf5044d8f203e1e53999f02c64a7d63e7d2 Mon Sep 17 00:00:00 2001 From: rkimmel-hv Date: Fri, 26 May 2023 11:44:28 -0400 Subject: [PATCH 4/4] Make sure disconnect doesnt crash spoon --- .../di/repository/pur/PurRepository.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java b/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java index b4d9b062dc71..49a2268026d3 100644 --- a/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java +++ b/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepository.java @@ -391,15 +391,17 @@ public RootRef getRootRef() { @Override public void disconnect() { connected = false; - metaStore = null; - try { - metaStore = MetaStoreConst.openLocalPentahoMetaStore(); - } catch ( MetaStoreException e ) { + if ( metaStore instanceof PurRepositoryMetaStore ) { metaStore = null; - } - if ( metaStore != null ) { - final IMetaStore connectedMetaStore = metaStore; - connectionManager.setMetastoreSupplier( () -> connectedMetaStore ); + try { + metaStore = MetaStoreConst.openLocalPentahoMetaStore(); + } catch ( MetaStoreException e ) { + metaStore = null; + } + if ( metaStore != null ) { + final IMetaStore connectedMetaStore = metaStore; + connectionManager.setMetastoreSupplier( () -> connectedMetaStore ); + } } purRepositoryConnector.disconnect();