diff --git a/pom.xml b/pom.xml index d95a761b..ce2ac285 100644 --- a/pom.xml +++ b/pom.xml @@ -6,13 +6,13 @@ org.springframework.boot spring-boot-starter-parent - 2.7.18 + 3.2.10 com.activeviam.tools mac - 3.0.0-SNAPSHOT + 3.1.0-SNAPSHOT Atoti Server Memory Analysis Cube @@ -23,12 +23,11 @@ - 11 + 21 - 6.0.10 - 5.1.13 + 6.1.3-1733969770-ef208661 - 9.0.31 + 10.1.30 com.activeviam.mac.app.MacSpringBootApp @@ -46,49 +45,41 @@ pom import + + jakarta.servlet + jakarta.servlet-api + 6.0.0 + + + org.springframework.boot + spring-boot-starter-json + 3.2.10 + - org.projectlombok - lombok - provided - - - - - com.activeviam.activepivot - activepivot-server-spring - - - - - - com.activeviam.tech - composer-intf - compile - - - com.activeviam.tech - composer-impl - compile + com.activeviam.springboot + atoti-server-starter + ${atoti-server.version} - - com.activeviam.tech - content-server-spring + com.activeviam.springboot + atoti-ui-starter + ${atoti-server.version} - com.activeviam.web - activeviam-web-spring + org.projectlombok + lombok + provided org.hibernate hibernate-core + 6.4.10.Final com.h2database @@ -97,14 +88,7 @@ org.hibernate hibernate-c3p0 - - - - - com.activeviam.activeui - activeui - ${atoti-ui.version} - runtime + 6.4.10.Final @@ -149,11 +133,6 @@ mockito-core test - - javax.servlet - javax.servlet-api - provided - junit junit @@ -170,6 +149,45 @@ 32.1.3-jre test + + com.activeviam.source + csv-source + ${atoti-server.version} + + + org.apache.commons + commons-compress + 1.27.1 + + + jakarta.persistence + jakarta.persistence-api + ${jakarta-persistence.version} + + + org.yaml + snakeyaml + 2.2 + + + jakarta.ws.rs + jakarta.ws.rs-api + ${jakarta-ws-rs.version} + + + org.springframework + spring-web + ${spring-framework.version} + + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.version} + + + org.springframework.boot + spring-boot-autoconfigure + diff --git a/src/main/java/com/activeviam/mac/app/MacSpringBootApp.java b/src/main/java/com/activeviam/mac/app/MacSpringBootApp.java index c544cf8a..95c2c7f3 100644 --- a/src/main/java/com/activeviam/mac/app/MacSpringBootApp.java +++ b/src/main/java/com/activeviam/mac/app/MacSpringBootApp.java @@ -8,9 +8,7 @@ package com.activeviam.mac.app; import com.activeviam.mac.cfg.impl.MacServerConfig; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import javax.servlet.MultipartConfigElement; +import jakarta.servlet.MultipartConfigElement; import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -36,11 +34,6 @@ @Import({MacServerConfig.class}) public class MacSpringBootApp { - /* Before anything else we statically initialize the Quartet FS Registry. */ - static { - Registry.setContributionProvider(new ClasspathContributionProvider()); - } - /** * Starts the Memory Analysis Cube application. * diff --git a/src/main/java/com/activeviam/mac/cfg/impl/ActivePivotWithDatastoreConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/ActivePivotWithDatastoreConfig.java new file mode 100644 index 00000000..2c2b51df --- /dev/null +++ b/src/main/java/com/activeviam/mac/cfg/impl/ActivePivotWithDatastoreConfig.java @@ -0,0 +1,48 @@ +package com.activeviam.mac.cfg.impl; + +import com.activeviam.activepivot.core.datastore.api.builder.ApplicationWithDatastore; +import com.activeviam.activepivot.core.datastore.api.builder.StartBuilding; +import com.activeviam.activepivot.core.intf.api.cube.IActivePivotManager; +import com.activeviam.activepivot.server.spring.api.config.IActivePivotBranchPermissionsManagerConfig; +import com.activeviam.activepivot.server.spring.api.config.IActivePivotConfig; +import com.activeviam.activepivot.server.spring.api.config.IActivePivotManagerDescriptionConfig; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreConfig; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.api.IDatastore; +import lombok.RequiredArgsConstructor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@RequiredArgsConstructor +public class ActivePivotWithDatastoreConfig implements IDatastoreConfig, IActivePivotConfig { + + private final IActivePivotManagerDescriptionConfig apManagerConfig; + + private final IDatastoreSchemaDescriptionConfig datastoreDescriptionConfig; + + private final IActivePivotBranchPermissionsManagerConfig branchPermissionsManagerConfig; + + @Bean + protected ApplicationWithDatastore applicationWithDatastore() { + return StartBuilding.application() + .withDatastore(this.datastoreDescriptionConfig.datastoreSchemaDescription()) + .withManager(this.apManagerConfig.managerDescription()) + .withEpochPolicy(this.apManagerConfig.epochManagementPolicy()) + .withBranchPermissionsManager( + this.branchPermissionsManagerConfig.branchPermissionsManager()) + .build(); + } + + @Bean + @Override + public IActivePivotManager activePivotManager() { + return applicationWithDatastore().getManager(); + } + + @Bean + @Override + public IDatastore database() { + return applicationWithDatastore().getDatastore(); + } +} diff --git a/src/main/java/com/activeviam/mac/cfg/impl/ActiveUiResourceServerConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/ActiveUiResourceServerConfig.java deleted file mode 100644 index dc965ed7..00000000 --- a/src/main/java/com/activeviam/mac/cfg/impl/ActiveUiResourceServerConfig.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * (C) ActiveViam 2016 - * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY - * property of ActiveViam. Any unauthorized use, - * reproduction or transfer of this material is strictly prohibited - */ - -package com.activeviam.mac.cfg.impl; - -import com.qfs.server.cfg.impl.ASpringResourceServerConfig; -import java.util.Collections; -import java.util.Set; -import org.springframework.context.annotation.Configuration; - -/** - * Spring configuration for Atoti UI web application. - * - * @author ActiveViam - */ -@Configuration -public class ActiveUiResourceServerConfig extends ASpringResourceServerConfig { - - /** The namespace of the Atoti UI web application. */ - public static final String NAMESPACE = "ui"; - - /** Constructor. */ - public ActiveUiResourceServerConfig() { - super("/" + NAMESPACE); - } - - @Override - protected void registerRedirections(final ResourceRegistry registry) { - super.registerRedirections(registry); - // Redirect from the root to ActiveUI - registry.redirectTo(NAMESPACE + "/index.html", "/"); - // Redirect the calls to env*.js to the AP ones rather than the default of the ActiveUI apps - registry.serve("/content/ui/env*.js").addResourceLocations("classpath:/static/content/"); - registry.serve("/ui/env*.js").addResourceLocations("classpath:/static/activeui/"); - registerExtensions(registry); - } - - protected void registerExtensions(final ResourceRegistry registry) { - registry.serve("/ui/extensions*.json").addResourceLocations("classpath:/static/activeui/"); - registry - .serve("/ui/extensions/text-editor-extension/**/*.js") - .addResourceLocations("classpath:/static/activeui/extensions/text-editor-extension/"); - } - - /** - * Registers resources to serve. - * - * @param registry registry to use - */ - @Override - protected void registerResources(final ResourceRegistry registry) { - super.registerResources(registry); - - // ActiveUI web app also serves request to the root, so that the redirection from root to - // ActiveUI works - registry - .serve("/") - .addResourceLocations("/", "classpath:META-INF/resources/") - .setCacheControl(getDefaultCacheControl()); - } - - /** - * Gets the extensions of files to serve. - * - * @return all files extensions - */ - @Override - public Set getServedExtensions() { - return Set.of( - // Default HTML files - "html", - "js", - "css", - "map", - "json", - // Image extensions - "png", - "jpg", - "gif", - "ico", - // Font extensions - "eot", - "svg", - "ttf", - "woff", - "woff2"); - } - - @Override - public Set getServedDirectories() { - return Collections.singleton("/"); - } - - @Override - public Set getResourceLocations() { - // ActiveUI is integrated in the sandbox project thanks to Maven integration. - // You can read more about this feature here - // https://support.activeviam.com/documentation/activeui/4.2.0/dev/setup/maven-integration.html - - return Set.of( - "/activeui/", // index.html, favicon.ico, etc. - "classpath:META-INF/resources/webjars/activeui/"); // ActiveUI SDK UMD scripts - // and supporting assets - } -} diff --git a/src/main/java/com/activeviam/mac/cfg/impl/ContentServiceConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/ContentServiceConfig.java index 39fa04a5..7025e4a1 100644 --- a/src/main/java/com/activeviam/mac/cfg/impl/ContentServiceConfig.java +++ b/src/main/java/com/activeviam/mac/cfg/impl/ContentServiceConfig.java @@ -7,33 +7,39 @@ package com.activeviam.mac.cfg.impl; +import static com.activeviam.tech.contentserver.storage.api.ContentServiceSnapshotter.create; + +import com.activeviam.activepivot.core.intf.api.contextvalues.IContextValue; +import com.activeviam.activepivot.core.intf.api.description.ICalculatedMemberDescription; +import com.activeviam.activepivot.core.intf.api.description.IKpiDescription; +import com.activeviam.activepivot.server.intf.api.entitlements.IActivePivotContentService; +import com.activeviam.activepivot.server.spring.api.config.IActivePivotContentServiceConfig; +import com.activeviam.activepivot.server.spring.api.content.ActivePivotContentServiceBuilder; import com.activeviam.mac.cfg.security.impl.SecurityConfig; +import com.activeviam.tech.contentserver.spring.internal.config.ContentServerRestServicesConfig; +import com.activeviam.tech.contentserver.storage.api.IContentService; +import com.activeviam.tech.contentserver.storage.private_.HibernateContentService; +import com.activeviam.tech.core.internal.monitoring.JmxOperation; import com.activeviam.tools.bookmark.constant.impl.ContentServerConstants.Paths; import com.activeviam.tools.bookmark.constant.impl.ContentServerConstants.Role; import com.activeviam.tools.bookmark.impl.BookmarkTool; -import com.qfs.content.cfg.impl.ContentServerRestServicesConfig; -import com.qfs.content.service.IContentService; -import com.qfs.content.service.impl.HibernateContentService; -import com.qfs.content.snapshot.impl.ContentServiceSnapshotter; -import com.qfs.jmx.JmxOperation; -import com.qfs.pivot.content.IActivePivotContentService; -import com.qfs.pivot.content.impl.ActivePivotContentServiceBuilder; -import com.qfs.server.cfg.content.IActivePivotContentServiceConfig; -import com.quartetfs.biz.pivot.context.IContextValue; -import com.quartetfs.biz.pivot.definitions.ICalculatedMemberDescription; -import com.quartetfs.biz.pivot.definitions.IKpiDescription; +import java.io.IOException; import java.lang.reflect.Method; import java.util.List; import java.util.Map; import java.util.Properties; import javax.sql.DataSource; +import lombok.RequiredArgsConstructor; +import org.hibernate.HibernateException; +import org.hibernate.SessionFactory; import org.hibernate.cfg.AvailableSettings; import org.springframework.beans.factory.BeanInitializationException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; /** * Spring configuration of the Content Service. @@ -46,21 +52,9 @@ * @author ActiveViam */ @Configuration +@RequiredArgsConstructor public class ContentServiceConfig implements IActivePivotContentServiceConfig { - /** - * The name of the property which contains the role allowed to add new calculated members in the - * configuration service. - */ - public static final String CALCULATED_MEMBER_ROLE_PROPERTY = - "contentServer.security.calculatedMemberRole"; - - /** - * The name of the property which contains the role allowed to add new KPIs in the configuration - * service. - */ - public static final String KPI_ROLE_PROPERTY = "contentServer.security.kpiRole"; - /** * The name of the property that controls whether or not to force the reloading of the predefined * bookmarks even if they were already loaded previously. @@ -70,19 +64,22 @@ public class ContentServiceConfig implements IActivePivotContentServiceConfig { /** The name of the property that precise the name of the folder the bookmarks are in. */ public static final String UI_FOLDER_PROPERTY = "bookmarks.folder"; - /** Instance of the Spring context environment. */ - @Autowired public Environment env; + private final Environment env; /** * Loads the Hibernate's configuration from the specified file. * * @return the Hibernate's configuration */ - private static org.hibernate.cfg.Configuration loadConfiguration( - final Properties hibernateProperties) { + private static SessionFactory loadConfiguration(final Properties hibernateProperties) + throws HibernateException, IOException { hibernateProperties.put( AvailableSettings.DATASOURCE, createTomcatJdbcDataSource(hibernateProperties)); - return new org.hibernate.cfg.Configuration().addProperties(hibernateProperties); + final Resource entityMappingFile = new ClassPathResource("content-service-hibernate.xml"); + return new org.hibernate.cfg.Configuration() + .addProperties(hibernateProperties) + .addInputStream(entityMappingFile.getInputStream()) + .buildSessionFactory(); } /** @@ -132,8 +129,17 @@ public Properties contentServiceHibernateProperties() { @Override @Bean public IContentService contentService() { - org.hibernate.cfg.Configuration conf = loadConfiguration(contentServiceHibernateProperties()); - return new HibernateContentService(conf); + if ("db".equals(this.env.getProperty("content-service.type", "db"))) { + return IContentService.builder().inMemory().build(); + } else { + final SessionFactory sessionFactory; + try { + sessionFactory = loadConfiguration(contentServiceHibernateProperties()); + return new HibernateContentService(sessionFactory); + } catch (HibernateException | IOException e) { + throw new BeanInitializationException("Failed to initialize the Content Service", e); + } + } } /** @@ -149,18 +155,16 @@ public IActivePivotContentService activePivotContentService() { return new ActivePivotContentServiceBuilder() .with(contentService()) .withCacheForEntitlements(-1) - .needInitialization( - this.env.getRequiredProperty(CALCULATED_MEMBER_ROLE_PROPERTY), - this.env.getRequiredProperty(KPI_ROLE_PROPERTY)) + .needInitialization(SecurityConfig.ROLE_USER, SecurityConfig.ROLE_USER) .build(); } private Map> defaultBookmarkPermissions() { return Map.of( Role.OWNERS, - List.of(SecurityConfig.ROLE_CS_ROOT), + List.of(SecurityConfig.ROLE_USER), Role.READERS, - List.of(SecurityConfig.ROLE_CS_ROOT)); + List.of(SecurityConfig.ROLE_USER)); } /** @@ -173,16 +177,14 @@ private Map> defaultBookmarkPermissions() { desc = "Export the current bookmark structure", params = {"destination"}) public void exportBookMarks(String destination) { - BookmarkTool.exportBookmarks( - new ContentServiceSnapshotter(contentService().withRootPrivileges()), destination); + BookmarkTool.exportBookmarks(create(contentService().withRootPrivileges()), destination); } /** Loads the bookmarks packaged with the application. */ public void loadPredefinedBookmarks() { final var service = contentService().withRootPrivileges(); if (!service.exists("/" + Paths.UI) || shouldReloadBookmarks()) { - BookmarkTool.importBookmarks( - new ContentServiceSnapshotter(service), defaultBookmarkPermissions()); + BookmarkTool.importBookmarks(create(service), defaultBookmarkPermissions()); } } diff --git a/src/main/java/com/activeviam/mac/cfg/impl/MacServerConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/MacServerConfig.java index 93575e25..b7f81e14 100644 --- a/src/main/java/com/activeviam/mac/cfg/impl/MacServerConfig.java +++ b/src/main/java/com/activeviam/mac/cfg/impl/MacServerConfig.java @@ -7,27 +7,13 @@ package com.activeviam.mac.cfg.impl; +import com.activeviam.activepivot.server.spring.api.config.IActivePivotConfig; +import com.activeviam.activepivot.server.spring.api.config.IActivePivotContentServiceConfig; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreConfig; import com.activeviam.mac.cfg.security.impl.SecurityConfig; -import com.activeviam.mac.cfg.security.impl.UserConfig; -import com.activeviam.properties.cfg.impl.ActiveViamPropertyFromSpringConfig; -import com.qfs.pivot.content.impl.DynamicActivePivotContentServiceMBean; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IActivePivotConfig; -import com.qfs.server.cfg.IDatastoreConfig; -import com.qfs.server.cfg.content.IActivePivotContentServiceConfig; -import com.qfs.server.cfg.i18n.impl.LocalI18nConfig; -import com.qfs.server.cfg.impl.ActivePivotServicesConfig; -import com.qfs.server.cfg.impl.ActivePivotWithDatastoreConfig; -import com.qfs.server.cfg.impl.ActivePivotXmlaServletConfig; -import com.qfs.server.cfg.impl.ActiveViamRestServicesConfig; -import com.qfs.server.cfg.impl.ActiveViamWebSocketServicesConfig; -import com.qfs.server.cfg.impl.FullAccessBranchPermissionsManagerConfig; -import com.qfs.server.cfg.impl.JwtConfig; -import com.qfs.server.cfg.impl.JwtRestServiceConfig; -import com.quartetfs.biz.pivot.monitoring.impl.DynamicActivePivotManagerMBean; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.monitoring.jmx.impl.JMXEnabler; -import java.nio.file.Paths; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.web.spring.internal.JMXEnabler; +import com.activeviam.web.spring.internal.config.JwtConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.Bean; @@ -58,28 +44,17 @@ @Configuration @Import( value = { - ActiveViamPropertyFromSpringConfig.class, - JwtRestServiceConfig.class, JwtConfig.class, ManagerDescriptionConfig.class, // Pivot ActivePivotWithDatastoreConfig.class, - NoWriteDatastoreServiceConfig.class, - FullAccessBranchPermissionsManagerConfig.class, - ActivePivotServicesConfig.class, - ActiveViamRestServicesConfig.class, - ActiveViamWebSocketServicesConfig.class, - ActivePivotXmlaServletConfig.class, // Content server ContentServiceConfig.class, - LocalI18nConfig.class, - ActiveUiResourceServerConfig.class, // Specific to monitoring server SecurityConfig.class, - UserConfig.class, SourceConfig.class, }) public class MacServerConfig { @@ -143,28 +118,6 @@ public JMXEnabler jmxMonitoringConnectorEnabler() { return new JMXEnabler("StatisticSource", this.sourceConfig); } - /** - * Enable JMX Monitoring for the Datastore. - * - * @return the {@link JMXEnabler} attached to the datastore - */ - @Bean - public JMXEnabler jmxDatastoreEnabler() { - return new JMXEnabler(this.datastoreConfig.database()); - } - - /** - * Enable JMX Monitoring for ActivePivot Components. - * - * @return the {@link JMXEnabler} attached to the activePivotManager - */ - @Bean - public JMXEnabler jmxActivePivotEnabler() { - startManager(); - - return new JMXEnabler(new DynamicActivePivotManagerMBean(apConfig.activePivotManager())); - } - /** * [Bean] JMX Bean to export bookmarks. * @@ -174,32 +127,4 @@ public JMXEnabler jmxActivePivotEnabler() { public JMXEnabler jmxBookmarkEnabler() { return new JMXEnabler("Bookmark", this.contentServiceConfig); } - - /** - * Enable JMX Monitoring for the ContentService. - * - * @return the {@link JMXEnabler} attached to the Content Service - */ - @Bean - public JMXEnabler jmxActivePivotContentServiceEnabler() { - // to allow operations from the JMX bean - return new JMXEnabler( - new DynamicActivePivotContentServiceMBean( - this.apContentServiceConfig.activePivotContentService(), - this.apConfig.activePivotManager())); - } - - /** - * Enable Memory JMX Monitoring. - * - * @return the {@link JMXEnabler} attached to the memory analysis service. - */ - @Bean - public JMXEnabler jmxMemoryMonitoringServiceEnabler() { - return new JMXEnabler( - new MemoryAnalysisService( - this.datastoreConfig.database(), - this.apConfig.activePivotManager(), - Paths.get(System.getProperty("java.io.tmpdir")))); - } } diff --git a/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java index a5a3bc45..21c863d1 100644 --- a/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java +++ b/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java @@ -7,45 +7,44 @@ package com.activeviam.mac.cfg.impl; -import com.activeviam.builders.StartBuilding; -import com.activeviam.comparators.ReverseEpochViewComparator; -import com.activeviam.copper.ICopperContext; -import com.activeviam.copper.api.Copper; -import com.activeviam.copper.api.CopperMeasure; -import com.activeviam.copper.api.CopperMeasureToAggregateAbove; -import com.activeviam.copper.api.CopperStore; +import com.activeviam.activepivot.copper.api.Copper; +import com.activeviam.activepivot.copper.api.CopperMeasure; +import com.activeviam.activepivot.copper.api.CopperMeasureToAggregateAbove; +import com.activeviam.activepivot.copper.api.CopperStore; +import com.activeviam.activepivot.core.datastore.api.builder.StartBuilding; +import com.activeviam.activepivot.core.impl.api.contextvalues.QueriesResultLimit; +import com.activeviam.activepivot.core.impl.api.contextvalues.QueriesTimeLimit; +import com.activeviam.activepivot.core.impl.internal.util.impl.MdxNamingUtil; +import com.activeviam.activepivot.core.intf.api.copper.ICopperContext; +import com.activeviam.activepivot.core.intf.api.cube.hierarchy.IDimension; +import com.activeviam.activepivot.core.intf.api.cube.hierarchy.IHierarchy; +import com.activeviam.activepivot.core.intf.api.cube.metadata.ILevelInfo; +import com.activeviam.activepivot.core.intf.api.cube.metadata.LevelIdentifier; +import com.activeviam.activepivot.core.intf.api.description.IActivePivotInstanceDescription; +import com.activeviam.activepivot.core.intf.api.description.IActivePivotManagerDescription; +import com.activeviam.activepivot.core.intf.api.description.ISelectionDescription; +import com.activeviam.activepivot.core.intf.api.description.builder.ICanBuildCubeDescription; +import com.activeviam.activepivot.core.intf.api.description.builder.ICanStartBuildingMeasures; +import com.activeviam.activepivot.core.intf.api.description.builder.IHasAtLeastOneMeasure; +import com.activeviam.activepivot.core.intf.api.description.builder.ISelectionDescriptionBuilder; +import com.activeviam.activepivot.core.intf.api.description.builder.dimension.ICanStartBuildingDimensions; +import com.activeviam.activepivot.server.spring.api.config.IActivePivotManagerDescriptionConfig; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.mac.comparators.ReverseEpochViewComparator; import com.activeviam.database.api.schema.FieldPath; -import com.activeviam.desc.build.ICanBuildCubeDescription; -import com.activeviam.desc.build.ICanStartBuildingMeasures; -import com.activeviam.desc.build.IHasAtLeastOneMeasure; -import com.activeviam.desc.build.ISelectionDescriptionBuilder; -import com.activeviam.desc.build.dimensions.ICanStartBuildingDimensions; -import com.activeviam.formatter.ByteFormatter; -import com.activeviam.formatter.ClassFormatter; -import com.activeviam.formatter.PartitionIdFormatter; +import com.activeviam.mac.formatter.ByteFormatter; +import com.activeviam.mac.formatter.ClassFormatter; +import com.activeviam.mac.formatter.PartitionIdFormatter; import com.activeviam.mac.entities.ChunkOwner; import com.activeviam.mac.entities.ChunkOwner.OwnerType; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; -import com.qfs.agg.impl.SingleValueFunction; -import com.qfs.multiversion.IEpoch; -import com.qfs.pivot.util.impl.MdxNamingUtil; -import com.qfs.server.cfg.IActivePivotManagerDescriptionConfig; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.Types; -import com.quartetfs.biz.pivot.context.impl.QueriesResultLimit; -import com.quartetfs.biz.pivot.context.impl.QueriesTimeLimit; -import com.quartetfs.biz.pivot.cube.dimension.IDimension; -import com.quartetfs.biz.pivot.cube.hierarchy.IHierarchy; -import com.quartetfs.biz.pivot.cube.hierarchy.ILevelInfo; -import com.quartetfs.biz.pivot.definitions.IActivePivotInstanceDescription; -import com.quartetfs.biz.pivot.definitions.IActivePivotManagerDescription; -import com.quartetfs.biz.pivot.definitions.ISelectionDescription; -import com.quartetfs.fwk.format.impl.DateFormatter; -import com.quartetfs.fwk.format.impl.NumberFormatter; -import com.quartetfs.fwk.ordering.impl.NaturalOrderComparator; -import com.quartetfs.fwk.ordering.impl.ReverseOrderComparator; +import com.activeviam.tech.aggregation.internal.impl.SingleValueFunction; +import com.activeviam.tech.chunks.api.types.Types; +import com.activeviam.tech.core.api.format.IFormatter; +import com.activeviam.tech.core.api.ordering.IComparator; +import com.activeviam.tech.mvcc.api.IEpoch; import java.time.Duration; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -74,9 +73,9 @@ public class ManagerDescriptionConfig implements IActivePivotManagerDescriptionC // region formatters /** Formatter for Numbers. */ - public static final String NUMBER_FORMATTER = NumberFormatter.TYPE + "[#,###]"; + public static final String NUMBER_FORMATTER = IFormatter.NUMBER_PLUGIN_KEY + "[#,###]"; /** Formatter for Percentages. */ - public static final String PERCENT_FORMATTER = NumberFormatter.TYPE + "[#.##%]"; + public static final String PERCENT_FORMATTER = IFormatter.NUMBER_PLUGIN_KEY + "[#.##%]"; // endregion // region dimensions @@ -265,8 +264,7 @@ public IActivePivotManagerDescription managerDescription() { } private ISelectionDescription memorySelection() { - return StartBuilding.selection( - datastoreDescriptionConfig.datastoreSchemaDescription().asDatabaseSchema()) + return StartBuilding.selection(datastoreDescriptionConfig.datastoreSchemaDescription()) .fromBaseStore(DatastoreConstants.CHUNK_STORE) .withAllReachableFields( allReachableFields -> { @@ -379,7 +377,7 @@ private ICanBuildCubeDescription defineDimensio "description", "The source folder name from which the statistics were retrieved") .withLevelOfSameName() .withPropertyName(DatastoreConstants.CHUNK__DUMP_NAME) - .withComparator(NaturalOrderComparator.type) + .withComparator(IComparator.NATURAL_ORDER_PLUGIN_KEY) .withLevelProperty( "description", "The source folder name from which the statistics were retrieved") .withHierarchy(DATE_HIERARCHY) @@ -387,7 +385,7 @@ private ICanBuildCubeDescription defineDimensio .withLevelOfSameName() .withPropertyName(DatastoreConstants.APPLICATION__DATE) .withType(ILevelInfo.LevelType.TIME) - .withComparator(ReverseOrderComparator.type) + .withComparator(IComparator.DESCENDING_NATURAL_ORDER_PLUGIN_KEY) .withLevelProperty("description", "Date at which statistics were retrieved") .withDimension(AGGREGATE_PROVIDER_DIMENSION) .withDimensionProperty( @@ -434,7 +432,7 @@ private ICanBuildCubeDescription defineDimensio .withLevelProperty( "description", "The internal epoch ID of the chunk (may be less than the epoch to view)") - .withComparator(ReverseOrderComparator.type) + .withComparator(IComparator.DESCENDING_NATURAL_ORDER_PLUGIN_KEY) .withHierarchy(BRANCH_HIERARCHY) .withHierarchyProperty("description", "The branch of the chunk") .withLevelOfSameName() @@ -485,7 +483,7 @@ private IHasAtLeastOneMeasure nativeMeasures(ICanStartBuildingMeasures builder) .withFormatter(NUMBER_FORMATTER) .withinFolder(INTERNAL_FOLDER) .withUpdateTimestamp() - .withFormatter(DateFormatter.TYPE + "[HH:mm:ss]") + .withFormatter(IFormatter.DATE_PLUGIN_KEY + "[HH:mm:ss]") .withinFolder(INTERNAL_FOLDER); } @@ -509,9 +507,21 @@ private void joinViewVersion(ICopperContext context) { final CopperStore epochViewStore = Copper.store(DatastoreConstants.EPOCH_VIEW_STORE) .joinToCube() - .withMapping(DatastoreConstants.OWNER__OWNER, OWNER_HIERARCHY) - .withMapping(DatastoreConstants.CHUNK__DUMP_NAME, CHUNK_DUMP_NAME_LEVEL) - .withMapping(DatastoreConstants.EPOCH_VIEW__BASE_EPOCH_ID, INTERNAL_EPOCH_ID_HIERARCHY); + .withMapping( + DatastoreConstants.OWNER__OWNER, + Copper.level(OWNER_DIMENSION, OWNER_HIERARCHY, OWNER_HIERARCHY)) + .withMapping( + DatastoreConstants.CHUNK__DUMP_NAME, + Copper.level( + new LevelIdentifier( + CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL))) + .withMapping( + DatastoreConstants.EPOCH_VIEW__BASE_EPOCH_ID, + Copper.level( + new LevelIdentifier( + VERSION_DIMENSION, + INTERNAL_EPOCH_ID_HIERARCHY, + INTERNAL_EPOCH_ID_HIERARCHY))); Copper.newHierarchy(VERSION_DIMENSION, EPOCH_ID_HIERARCHY) .fromField(epochViewStore.field(DatastoreConstants.EPOCH_VIEW__VIEW_EPOCH_ID)) @@ -524,9 +534,22 @@ private void joinReferencesToChunks(ICopperContext context) { final CopperStore chunkToReferenceStore = Copper.store(DatastoreConstants.REFERENCE_STORE) .joinToCube() - .withMapping(DatastoreConstants.REFERENCE_ID, CHUNK_REF_ID_LEVEL) - .withMapping(DatastoreConstants.CHUNK__DUMP_NAME, CHUNK_DUMP_NAME_LEVEL) - .withMapping(DatastoreConstants.VERSION__EPOCH_ID, INTERNAL_EPOCH_ID_HIERARCHY); + .withMapping( + DatastoreConstants.REFERENCE_ID, + Copper.level( + new LevelIdentifier(CHUNK_DIMENSION, CHUNK_REF_ID_LEVEL, CHUNK_REF_ID_LEVEL))) + .withMapping( + DatastoreConstants.CHUNK__DUMP_NAME, + Copper.level( + new LevelIdentifier( + CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL))) + .withMapping( + DatastoreConstants.VERSION__EPOCH_ID, + Copper.level( + new LevelIdentifier( + VERSION_DIMENSION, + INTERNAL_EPOCH_ID_HIERARCHY, + INTERNAL_EPOCH_ID_HIERARCHY))); Copper.newHierarchy(REFERENCE_NAMES_HIERARCHY) .fromField(chunkToReferenceStore.field(DatastoreConstants.REFERENCE_NAME)) @@ -538,9 +561,23 @@ private void joinIndexesToChunks(ICopperContext context) { final CopperStore chunkToIndexStore = Copper.store(DatastoreConstants.INDEX_STORE) .joinToCube() - .withMapping(DatastoreConstants.INDEX_ID, CHUNK_INDEX_ID_LEVEL) - .withMapping(DatastoreConstants.CHUNK__DUMP_NAME, CHUNK_DUMP_NAME_LEVEL) - .withMapping(DatastoreConstants.VERSION__EPOCH_ID, INTERNAL_EPOCH_ID_HIERARCHY); + .withMapping( + DatastoreConstants.INDEX_ID, + Copper.level( + new LevelIdentifier( + CHUNK_DIMENSION, CHUNK_INDEX_ID_LEVEL, CHUNK_INDEX_ID_LEVEL))) + .withMapping( + DatastoreConstants.CHUNK__DUMP_NAME, + Copper.level( + new LevelIdentifier( + CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL))) + .withMapping( + DatastoreConstants.VERSION__EPOCH_ID, + Copper.level( + new LevelIdentifier( + VERSION_DIMENSION, + INTERNAL_EPOCH_ID_HIERARCHY, + INTERNAL_EPOCH_ID_HIERARCHY))); Copper.newHierarchy(INDEX_DIMENSION, INDEXED_FIELDS_HIERARCHY) .fromField(chunkToIndexStore.field(DatastoreConstants.INDEX__FIELDS)) @@ -555,7 +592,9 @@ private void joinIndexesToChunks(ICopperContext context) { private void bucketingHierarchies(final ICopperContext context) { Copper.newHierarchy(OWNER_DIMENSION, OWNER_TYPE_HIERARCHY) - .fromValues(Copper.level(OWNER_HIERARCHY).map(ChunkOwner::getType)) + .fromValues( + Copper.level(OWNER_DIMENSION, OWNER_HIERARCHY, OWNER_HIERARCHY) + .map(ChunkOwner::getType)) .withMemberList((Object[]) OwnerType.values()) .withLevelOfSameName() .publish(context); @@ -728,18 +767,40 @@ private void dictionaryMeasures(ICopperContext context) { final var chunkToDicoStore = Copper.store(DatastoreConstants.DICTIONARY_STORE) .joinToCube() - .withMapping(DatastoreConstants.DICTIONARY_ID, CHUNK_DICO_ID_LEVEL) - .withMapping(DatastoreConstants.CHUNK__DUMP_NAME, CHUNK_DUMP_NAME_LEVEL) - .withMapping(DatastoreConstants.VERSION__EPOCH_ID, INTERNAL_EPOCH_ID_HIERARCHY); + .withMapping( + DatastoreConstants.DICTIONARY_ID, + Copper.level( + new LevelIdentifier(CHUNK_DIMENSION, CHUNK_DICO_ID_LEVEL, CHUNK_DICO_ID_LEVEL))) + .withMapping( + DatastoreConstants.CHUNK__DUMP_NAME, + Copper.level( + new LevelIdentifier( + CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL))) + .withMapping( + DatastoreConstants.VERSION__EPOCH_ID, + Copper.level( + new LevelIdentifier( + VERSION_DIMENSION, + INTERNAL_EPOCH_ID_HIERARCHY, + INTERNAL_EPOCH_ID_HIERARCHY))); Copper.agg( chunkToDicoStore.field(DatastoreConstants.DICTIONARY_SIZE), SingleValueFunction.PLUGIN_KEY) - .filter(Copper.level(COMPONENT_HIERARCHY).eq(ParentType.DICTIONARY)) + .filter( + Copper.level( + new LevelIdentifier( + COMPONENT_DIMENSION, COMPONENT_HIERARCHY, COMPONENT_HIERARCHY)) + .eq(ParentType.DICTIONARY)) .per( - Copper.level(INTERNAL_EPOCH_ID_HIERARCHY), - Copper.level(CHUNK_DUMP_NAME_LEVEL), - Copper.level(CHUNK_DICO_ID_LEVEL)) + Copper.level( + new LevelIdentifier( + VERSION_DIMENSION, INTERNAL_EPOCH_ID_HIERARCHY, INTERNAL_EPOCH_ID_HIERARCHY)), + Copper.level( + new LevelIdentifier( + CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL)), + Copper.level( + new LevelIdentifier(CHUNK_DIMENSION, CHUNK_DICO_ID_LEVEL, CHUNK_DICO_ID_LEVEL))) .sum() .as(DICTIONARY_SIZE) .withFormatter(NUMBER_FORMATTER) @@ -750,10 +811,10 @@ private void dictionaryMeasures(ICopperContext context) { perChunkAggregation(DatastoreConstants.CHUNK__SIZE) .max() .per( - Copper.hierarchy(OWNER_HIERARCHY).level(OWNER_HIERARCHY), - Copper.hierarchy(FIELD_HIERARCHY).level(FIELD_HIERARCHY), - Copper.hierarchy(PARTITION_HIERARCHY).level(PARTITION_HIERARCHY), - Copper.hierarchy(CHUNK_CLASS_LEVEL).level(CHUNK_CLASS_LEVEL)) + Copper.hierarchy(OWNER_DIMENSION, OWNER_HIERARCHY).level(OWNER_HIERARCHY), + Copper.hierarchy(FIELD_DIMENSION, FIELD_HIERARCHY).level(FIELD_HIERARCHY), + Copper.hierarchy(PARTITION_DIMENSION, PARTITION_HIERARCHY).level(PARTITION_HIERARCHY), + Copper.hierarchy(CHUNK_DIMENSION, CHUNK_CLASS_LEVEL).level(CHUNK_CLASS_LEVEL)) .min() .as("Chunk size") .withFormatter(NUMBER_FORMATTER) @@ -775,7 +836,7 @@ private void vectorMeasures(ICopperContext context) { .custom(SingleValueFunction.PLUGIN_KEY) // The underlying vector block length should be the same for all the chunks of an // application - .per(Copper.level(FIELD_HIERARCHY)) + .per(Copper.level(new LevelIdentifier(FIELD_DIMENSION, FIELD_HIERARCHY, FIELD_HIERARCHY))) .doNotAggregateAbove() .as(VECTOR_BLOCK_SIZE) .withinFolder(VECTOR_FOLDER) @@ -788,6 +849,10 @@ private CopperMeasureToAggregateAbove perChunkAggregation(final String fieldName } private CopperMeasureToAggregateAbove perChunkAggregation(final CopperMeasure measure) { - return measure.per(Copper.level(CHUNK_ID_HIERARCHY), Copper.level(CHUNK_DUMP_NAME_LEVEL)); + return measure.per( + Copper.level(new LevelIdentifier(CHUNK_DIMENSION, CHUNK_ID_HIERARCHY, CHUNK_ID_HIERARCHY)), + Copper.level( + new LevelIdentifier( + CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL, CHUNK_DUMP_NAME_LEVEL))); } } diff --git a/src/main/java/com/activeviam/mac/cfg/impl/NoWriteDatastoreServiceConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/NoWriteDatastoreServiceConfig.java deleted file mode 100644 index 18e6e4e6..00000000 --- a/src/main/java/com/activeviam/mac/cfg/impl/NoWriteDatastoreServiceConfig.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * (C) ActiveViam 2019 - * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY - * property of ActiveViam. Any unauthorized use - * reproduction or transfer of this material is strictly prohibited - */ - -package com.activeviam.mac.cfg.impl; - -import com.qfs.desc.ITablePermissions; -import com.qfs.desc.ITableSecurity; -import com.qfs.service.store.impl.ADatabaseServiceConfig; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import org.springframework.context.annotation.Configuration; - -/** Configuration preventing any edition to the datastore from the remote services. */ -@Configuration -public class NoWriteDatastoreServiceConfig extends ADatabaseServiceConfig { - - /** - * A constant for store security. - * - *

This allows to read any fields but forbids all updates. - */ - protected static final ITableSecurity defaultStoreSecurity = - new ITableSecurity() { - - @Override - public boolean isDeletingRecordsAllowed() { - return false; - } - - @Override - public boolean isAddingNewRecordsAllowed() { - return false; - } - - @Override - public ITablePermissions getTablePermissions() { - return new ITablePermissions() { - - @Override - public Set getTableWriterRoles() { - return Collections.emptySet(); - } - - @Override - public Set getTableReaderRoles() { - return Collections.emptySet(); - } - - @Override - public boolean canWriteField(String field, Set roles) { - return false; - } - - @Override - public boolean canReadField(String field, Set roles) { - return true; - } - }; - } - }; - /** - * A constant for the map which will always return the same store security, also defined as a - * constant below. - */ - protected static final Map storesSecurityMap = - new HashMap<>() { - - private static final long serialVersionUID = 5_08_00L; - - @Override - public ITableSecurity get(Object key) { - return containsKey(key) ? super.get(key) : defaultStoreSecurity; - } - }; - - /** Constructor. */ - public NoWriteDatastoreServiceConfig() { - super(Collections.emptyMap(), Collections.emptyMap(), storesSecurityMap, DEFAULT_QUERY_TIMEOUT); - } -} diff --git a/src/main/java/com/activeviam/mac/cfg/impl/RegistryInitializationConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/RegistryInitializationConfig.java new file mode 100644 index 00000000..1257ef77 --- /dev/null +++ b/src/main/java/com/activeviam/mac/cfg/impl/RegistryInitializationConfig.java @@ -0,0 +1,31 @@ +/* + * (C) ActiveViam 2024 + * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY + * property of ActiveViam. Any unauthorized use, + * reproduction or transfer of this material is strictly prohibited + */ + +package com.activeviam.mac.cfg.impl; + +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.core.api.registry.Registry.RegistryContributions; +import java.util.List; +import org.springframework.boot.context.event.ApplicationStartingEvent; +import org.springframework.context.ApplicationListener; + +/** + * Component initializing the registry as soon as possible when an application is starting. + * + * @author ActiveViam + */ +public class RegistryInitializationConfig implements ApplicationListener { + + public void onApplicationEvent(final ApplicationStartingEvent ignored) { + setupRegistry(); + } + + public static void setupRegistry() { + Registry.initialize( + RegistryContributions.builder().packagesToScan(List.of("com.activeviam.mac")).build()); + } +} diff --git a/src/main/java/com/activeviam/mac/cfg/impl/SourceConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/SourceConfig.java index 3a617deb..c9286c3b 100644 --- a/src/main/java/com/activeviam/mac/cfg/impl/SourceConfig.java +++ b/src/main/java/com/activeviam/mac/cfg/impl/SourceConfig.java @@ -10,18 +10,19 @@ import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toUnmodifiableList; -import com.activeviam.fwk.ActiveViamRuntimeException; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.internal.impl.Datastore; import com.activeviam.mac.Loggers; import com.activeviam.mac.memory.AnalysisDatastoreFeeder; -import com.qfs.jmx.JmxOperation; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.msg.csv.ICsvDataProvider; -import com.qfs.msg.csv.IFileEvent; -import com.qfs.msg.csv.filesystem.impl.DirectoryCSVTopic; -import com.qfs.msg.impl.WatcherService; -import com.qfs.pivot.monitoring.impl.MemoryStatisticSerializerUtil; -import com.qfs.store.IDatastore; -import com.qfs.store.impl.Datastore; +import com.activeviam.mac.statistic.memory.deserializer.RetroCompatibleDeserializer; +import com.activeviam.source.common.api.impl.WatcherService; +import com.activeviam.source.csv.api.DirectoryCsvTopic; +import com.activeviam.source.csv.api.ICsvDataProvider; +import com.activeviam.source.csv.api.IFileEvent; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; +import com.activeviam.tech.core.internal.monitoring.JmxOperation; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.io.IOException; import java.net.URI; import java.net.URL; @@ -40,7 +41,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.RequiredArgsConstructor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Lazy; @@ -52,20 +53,22 @@ * @author ActiveViam */ @Configuration +@RequiredArgsConstructor public class SourceConfig { /** The name of the property that holds the path to the statistics folder. */ public static final String STATISTIC_FOLDER_PROPERTY = "statistic.folder"; private static final Logger LOGGER = Logger.getLogger(Loggers.LOADING); + /** Autowired {@link Datastore} to be fed by this source. */ - @Autowired protected IDatastore datastore; + private final IDatastore datastore; /** Spring environment, automatically wired. */ - @Autowired protected Environment env; + private final Environment env; /** - * Provides a {@link DirectoryCSVTopic topic}. + * Provides a {@link DirectoryCsvTopic topic}. * *

The provided topic is based on the content of the folder defined by the {@code * statistic.folder} environment property. By default, the property is defined in the {@code @@ -77,7 +80,7 @@ public class SourceConfig { */ @Bean @Lazy - public DirectoryCSVTopic statisticTopic() throws IllegalStateException { + public DirectoryCsvTopic statisticTopic() throws IllegalStateException { final String statisticFolder = this.env.getRequiredProperty(STATISTIC_FOLDER_PROPERTY); final Path folderPath = Paths.get(statisticFolder); if (LOGGER.isLoggable(Level.INFO)) { @@ -92,7 +95,7 @@ public DirectoryCSVTopic statisticTopic() throws IllegalStateException { + folderPath.toAbsolutePath() + " is not a correct path to a valid directory."); } - return new DirectoryCSVTopic( + return new DirectoryCsvTopic( "StatisticTopic", null, statisticFolder, @@ -128,9 +131,14 @@ protected Path resolveDirectory(final String name) { e); } } - if (url == null - || !Files.isDirectory(directory = Paths.get(URI.create(url.toExternalForm())))) { - throw new IllegalArgumentException("'" + name + "' could not be resolved to a directory."); + if (url != null) { + directory = Paths.get(URI.create(url.toExternalForm())); + if (!Files.isDirectory(directory)) { + throw new IllegalArgumentException( + "'" + name + "' could not be resolved to a directory."); + } + } else { + throw new IllegalArgumentException("could not find '" + name + "' in the classpath."); } } return directory; @@ -206,15 +214,15 @@ private Map> collectDumpFiles( } private Path getStatisticFolder() { - return resolveDirectory(this.env.getRequiredProperty("statistic.folder")); + return resolveDirectory(this.env.getRequiredProperty(STATISTIC_FOLDER_PROPERTY)); } private void loadDumps(final Map> dumpFiles) { dumpFiles.forEach( (dumpName, entry) -> { try { - final Stream inputs = - entry.stream().parallel().map(this::readStatisticFile); + final Stream inputs = + entry.stream().parallel().map(RetroCompatibleDeserializer::readStatisticFile); final String message = feedDatastore(inputs, dumpName); LOGGER.info(message); } catch (final Exception e) { @@ -223,21 +231,6 @@ private void loadDumps(final Map> dumpFiles) { }); } - private IMemoryStatistic readStatisticFile(final Path file) { - try { - if (LOGGER.isLoggable(Level.FINE)) { - LOGGER.fine("Reading statistics from " + file.toAbsolutePath()); - } - final IMemoryStatistic read = MemoryStatisticSerializerUtil.readStatisticFile(file.toFile()); - if (LOGGER.isLoggable(Level.FINE)) { - LOGGER.fine("Statistics read from " + file.toAbsolutePath()); - } - return read; - } catch (final IOException ioe) { - throw new RuntimeException("Cannot read statistics from " + file); - } - } - /** * Feeds the {@link SourceConfig#datastore datastore} with a stream of {@link IMemoryStatistic}. * @@ -246,9 +239,8 @@ private IMemoryStatistic readStatisticFile(final Path file) { * @return message to the user */ public String feedDatastore( - final Stream memoryStatistics, final String dumpName) { - final var info = - new AnalysisDatastoreFeeder(dumpName).loadInto(this.datastore, memoryStatistics); + final Stream memoryStatistics, final String dumpName) { + final var info = new AnalysisDatastoreFeeder(dumpName, datastore).loadInto(memoryStatistics); if (info.isPresent()) { return "Commit successful for dump " + dumpName + " at epoch " + info.get().getId() + "."; } else { diff --git a/src/main/java/com/activeviam/mac/cfg/security/impl/ASecurityConfig.java b/src/main/java/com/activeviam/mac/cfg/security/impl/ASecurityConfig.java deleted file mode 100644 index c6548e8d..00000000 --- a/src/main/java/com/activeviam/mac/cfg/security/impl/ASecurityConfig.java +++ /dev/null @@ -1,321 +0,0 @@ -/* - * (C) ActiveViam 2017-2020 - * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY - * property of ActiveViam. Any unauthorized use, - * reproduction or transfer of this material is strictly prohibited - */ - -package com.activeviam.mac.cfg.security.impl; - -import com.activeviam.security.cfg.ICorsConfig; -import com.qfs.content.service.IContentService; -import com.qfs.jwt.service.IJwtService; -import com.qfs.server.cfg.IJwtConfig; -import com.qfs.server.cfg.impl.VersionServicesConfig; -import com.qfs.servlet.handlers.impl.NoRedirectLogoutSuccessHandler; -import com.quartetfs.biz.pivot.security.IAuthorityComparator; -import com.quartetfs.biz.pivot.security.impl.AuthorityComparatorAdapter; -import com.quartetfs.fwk.ordering.impl.CustomComparator; -import java.util.Collections; -import java.util.List; -import javax.servlet.Filter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; -import org.springframework.http.HttpStatus; -import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; -import org.springframework.security.config.annotation.authentication.configuration.EnableGlobalAuthentication; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.builders.WebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; -import org.springframework.security.crypto.factory.PasswordEncoderFactories; -import org.springframework.security.crypto.password.DelegatingPasswordEncoder; -import org.springframework.security.crypto.password.NoOpPasswordEncoder; -import org.springframework.security.crypto.password.PasswordEncoder; -import org.springframework.security.web.AuthenticationEntryPoint; -import org.springframework.security.web.authentication.HttpStatusEntryPoint; -import org.springframework.security.web.context.SecurityContextPersistenceFilter; -import org.springframework.security.web.firewall.StrictHttpFirewall; -import org.springframework.web.cors.CorsConfiguration; -import org.springframework.web.cors.CorsConfigurationSource; -import org.springframework.web.cors.UrlBasedCorsConfigurationSource; - -/** - * Generic implementation for security configuration of a server hosting ActivePivot, or Content - * server or ActiveMonitor. - * - *

This class contains methods: - * - *

    - *
  • To define authorized users, - *
  • To enable anonymous user access, - *
  • To configure the JWT filter, - *
  • To configure the security for Version service. - *
- * - * @author ActiveViam - */ -@EnableGlobalAuthentication -@Configuration -public abstract class ASecurityConfig implements ICorsConfig { - - /** Set to true to allow anonymous access. */ - public static final boolean useAnonymous = false; - - /** Authentication Bean Name. */ - public static final String BASIC_AUTH_BEAN_NAME = "basicAuthenticationEntryPoint"; - - /** ActivePivot Cookie Name. */ - public static final String AP_COOKIE_NAME = "AP_JSESSIONID"; - - /** Name of the User Role. */ - public static final String ROLE_USER = "ROLE_USER"; - - /** Name of the Admin Role. */ - public static final String ROLE_ADMIN = "ROLE_ADMIN"; - - /** Name of the Tech Role. */ - public static final String ROLE_TECH = "ROLE_TECH"; - - /** Name of the ContentService Root Role. */ - public static final String ROLE_CS_ROOT = IContentService.ROLE_ROOT; - - /** The address the UI is exposed to. */ - public static final String ACTIVEUI_ADDRESS = "activeui.address"; - /** The User Configuration. */ - @Autowired protected UserConfig userDetailsConfig; - - /** The JWT Configuration. */ - @Autowired protected IJwtConfig jwtConfig; - /** The name of the Environment to use. */ - @Autowired protected Environment env; - - /** - * As of Spring Security 5.0, the way the passwords are encoded must be specified. When logging, - * the input password will be encoded and compared with the stored encoded password. To determine - * which encoding function was used to encode the password, the stored encoded passwords are - * prefixed with the id of the encoding function. - * - *

In order to avoid reformatting existing passwords in databases one can set the default - * PasswordEncoder to use for stored passwords that are not prefixed. This is the - * role of the following function. - * - * @return The {@link PasswordEncoder} to encode passwords with. - */ - @Bean - @SuppressWarnings({"deprecation", "unused"}) - public PasswordEncoder passwordEncoder() { - PasswordEncoder passwordEncoder = PasswordEncoderFactories.createDelegatingPasswordEncoder(); - ((DelegatingPasswordEncoder) passwordEncoder) - .setDefaultPasswordEncoderForMatches(NoOpPasswordEncoder.getInstance()); - return passwordEncoder; - } - - /** - * Returns the default {@link AuthenticationEntryPoint} to use for the fallback basic HTTP - * authentication. - * - * @return The default {@link AuthenticationEntryPoint} for the fallback HTTP basic - * authentication. - */ - @Bean(name = BASIC_AUTH_BEAN_NAME) - public AuthenticationEntryPoint basicAuthenticationEntryPoint() { - return new HttpStatusEntryPoint(HttpStatus.UNAUTHORIZED); - } - - /** - * Configures the authentication of the whole application. - * - *

This binds the defined user service to the authentication and sets the source for JWT - * tokens. - * - * @param auth Spring builder to manage authentication - * @throws Exception in case of error - */ - @Autowired - public void configureGlobal(final AuthenticationManagerBuilder auth) throws Exception { - auth.eraseCredentials(false) - // Add an LDAP authentication provider instead of this to support LDAP - .userDetailsService(this.userDetailsConfig.userDetailsService()) - .and() - // Required to allow JWT - .authenticationProvider(this.jwtConfig.jwtAuthenticationProvider()); - } - - /** - * [Bean] Comparator for user roles. - * - *

Defines the comparator used by: - * - *

    - *
  • com.quartetfs.biz.pivot.security.impl.ContextValueManager#setAuthorityComparator( - * IAuthorityComparator) - *
  • {@link IJwtService} - *
- * - * @return a comparator that indicates which authority/role prevails over another. - */ - @Bean - public IAuthorityComparator authorityComparator() { - final CustomComparator comp = new CustomComparator<>(); - comp.setFirstObjects(Collections.singletonList(ROLE_USER)); - comp.setLastObjects(Collections.singletonList(ROLE_ADMIN)); - return new AuthorityComparatorAdapter(comp); - } - - @Override - public List getAllowedOrigins() { - return Collections.singletonList(env.getRequiredProperty(ACTIVEUI_ADDRESS)); - } - - /** - * [Bean] Spring standard way of configuring CORS. - * - *

This simply forwards the configuration of {@link ICorsConfig} to Spring security system. - * - * @return the configuration for the application. - */ - @Bean - public CorsConfigurationSource corsConfigurationSource() { - final CorsConfiguration configuration = new CorsConfiguration(); - configuration.setAllowedOrigins(getAllowedOrigins()); - configuration.setAllowedHeaders(getAllowedHeaders()); - configuration.setExposedHeaders(getExposedHeaders()); - configuration.setAllowedMethods(getAllowedMethods()); - configuration.setAllowCredentials(true); - - final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); - source.registerCorsConfiguration("/**", configuration); - - return source; - } - - /** - * Common configuration for {@link HttpSecurity}. - * - * @author ActiveViam - */ - public abstract static class AWebSecurityConfigurer extends WebSecurityConfigurerAdapter { - - /** {@code true} to enable the logout URL. */ - protected final boolean logout; - /** The name of the cookie to clear. */ - protected final String cookieName; - - /** The name of the Environment to use. */ - @Autowired protected Environment env; - - /** The ApplicationContext which contains the Beans. */ - @Autowired protected ApplicationContext context; - - /** This constructor does not enable the logout URL. */ - public AWebSecurityConfigurer() { - this(null); - } - - /** - * This constructor enables the logout URL. - * - * @param cookieName the name of the cookie to clear - */ - public AWebSecurityConfigurer(String cookieName) { - this.logout = cookieName != null; - this.cookieName = cookieName; - } - - /** - * {@inheritDoc} - * - *

This configures a new firewall accepting `%` in URLs, as none of the core services encode - * information in URL. This prevents from double-decoding exploits.
- * The firewall is also configured to accept `\` - backslash - as none of ActiveViam APIs offer - * to manipulate files from URL parameters.
- * Yet, nor `/` and `.` - slash and point - are accepted, as it may trick the REGEXP matchers - * used for security. Support for those two characters can be added at your own risk, by - * extending this method. As far as ActiveViam APIs are concerned, `/` and `.` in URL parameters - * do not represent any risk. `;` - semi-colon - is also not supported, for various APIs end up - * target an actual database, and because this character is less likely to be used. - */ - @Override - public void configure(WebSecurity web) throws Exception { - super.configure(web); - - final StrictHttpFirewall firewall = new StrictHttpFirewall(); - firewall.setAllowUrlEncodedPercent(true); - firewall.setAllowBackSlash(true); - - firewall.setAllowUrlEncodedSlash(false); - firewall.setAllowUrlEncodedPeriod(false); - firewall.setAllowSemicolon(false); - web.httpFirewall(firewall); - } - - @Override - protected final void configure(final HttpSecurity http) throws Exception { - final Filter jwtFilter = this.context.getBean(IJwtConfig.class).jwtFilter(); - - http - // As of Spring Security 4.0, CSRF protection is enabled by default. - .csrf() - .disable() - .cors() - .and() - // To allow authentication with JWT (Required for ActiveUI) - .addFilterAfter(jwtFilter, SecurityContextPersistenceFilter.class); - - if (this.logout) { - // Configure logout URL - http.logout() - .permitAll() - .deleteCookies(this.cookieName) - .invalidateHttpSession(true) - .logoutSuccessHandler(new NoRedirectLogoutSuccessHandler()); - } - - if (useAnonymous) { - // Handle anonymous users. The granted authority ROLE_USER - // will be assigned to the anonymous request - http.anonymous().principal("guest").authorities(ROLE_USER); - } - - doConfigure(http); - } - - /** - * Applies the specific configuration for the endpoint. - * - * @param http the http endpoint to configure. - * @throws Exception in case of error. - * @see #configure(HttpSecurity) - */ - protected abstract void doConfigure(HttpSecurity http) throws Exception; - } - - /** - * Configuration for Version service to allow anyone to access this service. - * - * @author ActiveViam - * @see HttpStatusEntryPoint - */ - public abstract static class AVersionSecurityConfigurer extends WebSecurityConfigurerAdapter { - - /** The autowired Spring context. */ - @Autowired protected ApplicationContext context; - - @Override - protected void configure(HttpSecurity http) throws Exception { - - http.antMatcher(VersionServicesConfig.REST_API_URL_PREFIX + "/**") - // As of Spring Security 4.0, CSRF protection is enabled by default. - .csrf() - .disable() - .cors() - .and() - .authorizeRequests() - .antMatchers("/**") - .permitAll(); - } - } -} diff --git a/src/main/java/com/activeviam/mac/cfg/security/impl/ActiveUiSecurityConfigurer.java b/src/main/java/com/activeviam/mac/cfg/security/impl/ActiveUiSecurityConfigurer.java deleted file mode 100644 index 1b3a23e9..00000000 --- a/src/main/java/com/activeviam/mac/cfg/security/impl/ActiveUiSecurityConfigurer.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * (C) ActiveViam 2017 - * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY - * property of ActiveViam. Any unauthorized use, - * reproduction or transfer of this material is strictly prohibited - */ - -package com.activeviam.mac.cfg.security.impl; - -import static com.activeviam.mac.cfg.security.impl.ASecurityConfig.ACTIVEUI_ADDRESS; - -import com.activeviam.mac.cfg.security.impl.ASecurityConfig.AWebSecurityConfigurer; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.annotation.Order; -import org.springframework.http.HttpMethod; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; - -/** - * To expose the login page of ActiveUI. - * - * @author ActiveViam - */ -@Configuration -@Order(1) -public class ActiveUiSecurityConfigurer extends AWebSecurityConfigurer { - - @Override - protected void doConfigure(HttpSecurity http) throws Exception { - final String activeUiUrl = env.getRequiredProperty(ACTIVEUI_ADDRESS); - http - // Only theses URLs must be handled by this HttpSecurity - .regexMatcher(activeUiUrl) - .authorizeRequests() - // The order of the matchers matters - .regexMatchers(HttpMethod.OPTIONS, activeUiUrl) - .permitAll() - .regexMatchers(HttpMethod.GET, activeUiUrl) - .permitAll(); - // this allows pre-flight cross-origin requests - http.cors(); - // Authorizing pages to be embedded in iframes to have ActiveUI in ActiveMonitor UI - http.headers().frameOptions().disable(); - } -} diff --git a/src/main/java/com/activeviam/mac/cfg/security/impl/CorsConfig.java b/src/main/java/com/activeviam/mac/cfg/security/impl/CorsConfig.java new file mode 100644 index 00000000..cb5ca029 --- /dev/null +++ b/src/main/java/com/activeviam/mac/cfg/security/impl/CorsConfig.java @@ -0,0 +1,56 @@ +/* + * (C) ActiveViam 2017-2020 + * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY + * property of ActiveViam. Any unauthorized use, + * reproduction or transfer of this material is strictly prohibited + */ + +package com.activeviam.mac.cfg.security.impl; + +import com.activeviam.web.spring.api.config.ICorsConfig; +import java.util.Collections; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.security.config.annotation.authentication.configuration.EnableGlobalAuthentication; +import org.springframework.web.cors.CorsConfiguration; + +/** + * Generic implementation for security configuration of a server hosting ActivePivot, or Content server or + * ActiveMonitor. + * + *

This class contains methods: + * + *

    + *
  • To define authorized users, + *
  • To enable anonymous user access, + *
  • To configure the JWT filter, + *
  • To configure the security for Version service. + *
+ * + * @author ActiveViam + */ +@EnableGlobalAuthentication +@Configuration +public class CorsConfig implements ICorsConfig { + + /** + * The name of the Environment to use. + */ + protected Environment env; + + /** + * The address the UI is exposed to. + */ + public static final String ACTIVEUI_ADDRESS = "activeui.address"; + + public CorsConfig(@Autowired Environment env) { + this.env = env; + } + + @Override + public List getAllowedOrigins() { + return Collections.singletonList(env.getProperty(ACTIVEUI_ADDRESS, CorsConfiguration.ALL)); + } +} diff --git a/src/main/java/com/activeviam/mac/cfg/security/impl/InMemoryUserDetailsManagerBuilder.java b/src/main/java/com/activeviam/mac/cfg/security/impl/InMemoryUserDetailsManagerBuilder.java deleted file mode 100644 index d8e006dd..00000000 --- a/src/main/java/com/activeviam/mac/cfg/security/impl/InMemoryUserDetailsManagerBuilder.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * (C) ActiveViam 2015 - * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY - * property of ActiveViam. Any unauthorized use, - * reproduction or transfer of this material is strictly prohibited - */ - -package com.activeviam.mac.cfg.security.impl; - -import java.util.ArrayList; -import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; -import org.springframework.security.config.annotation.authentication.configurers.provisioning.InMemoryUserDetailsManagerConfigurer; -import org.springframework.security.config.annotation.authentication.configurers.provisioning.UserDetailsManagerConfigurer; -import org.springframework.security.core.userdetails.UserDetailsService; -import org.springframework.security.provisioning.InMemoryUserDetailsManager; -import org.springframework.security.provisioning.UserDetailsManager; - -/** - * An In-memory {@link UserDetailsService} builder which can be used without {@link - * AuthenticationManagerBuilder} contrary to {@link InMemoryUserDetailsManagerConfigurer}. - * - * @author ActiveViam - */ -public class InMemoryUserDetailsManagerBuilder - extends UserDetailsManagerConfigurer< - AuthenticationManagerBuilder, InMemoryUserDetailsManagerBuilder> { - - /** Creates a new instance. */ - public InMemoryUserDetailsManagerBuilder() { - super(new InMemoryUserDetailsManager(new ArrayList<>())); - } - - @Override - public void configure(AuthenticationManagerBuilder builder) throws Exception { - if (null != builder) { - throw new IllegalArgumentException(); - } - initUserDetailsService(); - } - - /** - * Builds the In-memory {@link UserDetailsManager} and returns it. - * - * @return the built object - */ - public UserDetailsManager build() { - try { - configure(null); - return getUserDetailsService(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/com/activeviam/mac/cfg/security/impl/JwtSecurityConfigurer.java b/src/main/java/com/activeviam/mac/cfg/security/impl/JwtSecurityConfigurer.java deleted file mode 100644 index 5b1c4160..00000000 --- a/src/main/java/com/activeviam/mac/cfg/security/impl/JwtSecurityConfigurer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * (C) ActiveViam 2017 - * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY - * property of ActiveViam. Any unauthorized use, - * reproduction or transfer of this material is strictly prohibited - */ - -package com.activeviam.mac.cfg.security.impl; - -import com.qfs.server.cfg.impl.JwtRestServiceConfig; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.annotation.Order; -import org.springframework.http.HttpMethod; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; -import org.springframework.security.web.AuthenticationEntryPoint; - -/** - * To expose the JWT REST service. - * - * @author ActiveViam - */ -@Configuration -@Order(2) // Must be done before ContentServerSecurityConfigurer (because they match common URLs) -public class JwtSecurityConfigurer extends WebSecurityConfigurerAdapter { - - /** The autowired Spring @link {@link ApplicationContext}. */ - @Autowired protected ApplicationContext context; - - @Override - protected void configure(HttpSecurity http) throws Exception { - final AuthenticationEntryPoint basicAuthenticationEntryPoint = - this.context.getBean(ASecurityConfig.BASIC_AUTH_BEAN_NAME, AuthenticationEntryPoint.class); - http.antMatcher(JwtRestServiceConfig.REST_API_URL_PREFIX + "/**") - // As of Spring Security 4.0, CSRF protection is enabled by default. - .csrf() - .disable() - .cors() - .and() - // Configure CORS - .authorizeRequests() - .antMatchers(HttpMethod.OPTIONS, "/**") - .permitAll() - .antMatchers("/**") - .hasAnyAuthority(ASecurityConfig.ROLE_USER) - .and() - .httpBasic() - .authenticationEntryPoint(basicAuthenticationEntryPoint); - } -} diff --git a/src/main/java/com/activeviam/mac/cfg/security/impl/SecurityConfig.java b/src/main/java/com/activeviam/mac/cfg/security/impl/SecurityConfig.java index 59b42e2c..541b6749 100644 --- a/src/main/java/com/activeviam/mac/cfg/security/impl/SecurityConfig.java +++ b/src/main/java/com/activeviam/mac/cfg/security/impl/SecurityConfig.java @@ -1,70 +1,125 @@ -/* - * (C) ActiveViam 2019-2020 - * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY - * property of ActiveViam. Any unauthorized use, - * reproduction or transfer of this material is strictly prohibited - */ - package com.activeviam.mac.cfg.security.impl; -import static com.qfs.QfsWebUtils.url; -import static com.qfs.server.cfg.impl.ActivePivotRemotingServicesConfig.ID_GENERATOR_REMOTING_SERVICE; -import static com.qfs.server.cfg.impl.ActivePivotRemotingServicesConfig.LICENSING_REMOTING_SERVICE; -import static com.qfs.server.cfg.impl.ActivePivotRemotingServicesConfig.LONG_POLLING_REMOTING_SERVICE; -import static com.qfs.server.cfg.impl.ActivePivotRestServicesConfig.PING_SUFFIX; -import static com.qfs.server.cfg.impl.ActivePivotRestServicesConfig.REST_API_URL_PREFIX; - -import com.qfs.server.cfg.IActivePivotConfig; -import com.quartetfs.biz.pivot.security.impl.UserDetailsServiceWrapper; -import com.quartetfs.fwk.security.IUserDetailsService; -import org.springframework.beans.factory.annotation.Autowired; +import com.activeviam.tech.contentserver.storage.api.IContentService; +import com.activeviam.web.spring.api.config.IJwtConfig; +import com.activeviam.web.spring.api.jwt.JwtAuthenticationProvider; +import lombok.RequiredArgsConstructor; import org.springframework.boot.web.servlet.ServletContextInitializer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; import org.springframework.security.authentication.AuthenticationManager; -import org.springframework.security.config.BeanIds; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.authentication.ProviderManager; +import org.springframework.security.authentication.dao.DaoAuthenticationProvider; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.web.authentication.switchuser.SwitchUserFilter; +import org.springframework.security.core.userdetails.User; +import org.springframework.security.core.userdetails.User.UserBuilder; +import org.springframework.security.core.userdetails.UserDetailsService; +import org.springframework.security.crypto.factory.PasswordEncoderFactories; +import org.springframework.security.crypto.password.DelegatingPasswordEncoder; +import org.springframework.security.crypto.password.NoOpPasswordEncoder; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.security.provisioning.InMemoryUserDetailsManager; +import org.springframework.security.web.AuthenticationEntryPoint; +import org.springframework.security.web.authentication.HttpStatusEntryPoint; +import org.springframework.security.web.firewall.StrictHttpFirewall; -/** - * Spring configuration fragment for security on an ActivePivot Server. - * - *

This configuration will in particular load: - * - *

    - *
  • The service to authenticate users - *
  • The Spring configuration that defines security on the Version RESTful service - *
  • The Spring configuration that defines security on the Content server - *
  • The Spring configuration that defines security on the ActivePivot server - *
- * - * @author ActiveViam - */ -@Import( - value = { - ActiveUiSecurityConfigurer.class, - JwtSecurityConfigurer.class, - VersionSecurityConfigurer.class - }) @Configuration @EnableWebSecurity -public class SecurityConfig extends ASecurityConfig { +@RequiredArgsConstructor +public class SecurityConfig { + + /** Name of the User Role. */ + public static final String ROLE_USER = "ROLE_USER"; + + /** Name of the Admin Role. */ + public static final String ROLE_ADMIN = "ROLE_ADMIN"; /** Name of the Cookies of the MAC application. */ public static final String COOKIE_NAME = "MEMORY_ANALYSIS_CUBE"; /** - * Returns the spring security bean user details service wrapper. + * [Bean] Create the users that can access the application. + * + * @return {@link UserDetailsService user data} + */ + @Bean + public UserDetailsService userDetailsService(final PasswordEncoder passwordEncoder) { + final UserBuilder builder = User.builder().passwordEncoder(passwordEncoder::encode); + final InMemoryUserDetailsManager service = new InMemoryUserDetailsManager(); + service.createUser( + builder + .username("admin") + .password("admin") + .authorities(ROLE_USER, ROLE_ADMIN, IContentService.ROLE_ROOT) + .build()); + return service; + } + + /** + * As of Spring Security 5.0, the way the passwords are encoded must be specified. When logging, + * the input password will be encoded and compared with the stored encoded password. To determine + * which encoding function was used to encode the password, the stored encoded passwords are + * prefixed with the id of the encoding function. + * + *

In order to avoid reformatting existing passwords in databases one can set the default + * PasswordEncoder to use for stored passwords that are not prefixed. This is the + * role of the following function. + * + * @return The {@link PasswordEncoder} to encode passwords with. + */ + @Bean + @SuppressWarnings({"deprecation", "unused"}) + public PasswordEncoder passwordEncoder() { + PasswordEncoder passwordEncoder = PasswordEncoderFactories.createDelegatingPasswordEncoder(); + ((DelegatingPasswordEncoder) passwordEncoder) + .setDefaultPasswordEncoderForMatches(NoOpPasswordEncoder.getInstance()); + return passwordEncoder; + } + + /** + * Returns the default {@link AuthenticationEntryPoint} to use for the fallback basic HTTP + * authentication. * - * @return the {@link IUserDetailsService} used as spring security bean user details service - * wrapper. + * @return The default {@link AuthenticationEntryPoint} for the fallback HTTP basic + * authentication. */ @Bean - public IUserDetailsService qfsUserDetailsService() { - return new UserDetailsServiceWrapper(this.userDetailsConfig.userDetailsService()); + public AuthenticationEntryPoint basicAuthenticationEntryPoint() { + return new HttpStatusEntryPoint(HttpStatus.UNAUTHORIZED); + } + + /** + * Configures the authentication of the whole application. + * + *

This binds the defined user service to the authentication and sets the source for JWT + * tokens. + * + * @param inMemoryAuthenticationProvider the in-memory authentication provider + * @param jwtAuthenticationProvider is a provider which can perform authentication from the + * jwtService's tokens. Implementation from the {@link IJwtConfig} . + * @return the authentication manager + */ + @Bean + public AuthenticationManager authenticationManager( + final JwtAuthenticationProvider jwtAuthenticationProvider, + final AuthenticationProvider inMemoryAuthenticationProvider) { + final ProviderManager providerManager = + new ProviderManager(inMemoryAuthenticationProvider, jwtAuthenticationProvider); + providerManager.setEraseCredentialsAfterAuthentication(false); + + return providerManager; + } + + @Bean + public AuthenticationProvider inMemoryAuthenticationProvider( + final UserDetailsService userDetailsService, final PasswordEncoder passwordEncoder) { + final var authenticationProvider = new DaoAuthenticationProvider(); + authenticationProvider.setPasswordEncoder(passwordEncoder); + authenticationProvider.setUserDetailsService(userDetailsService); + + return authenticationProvider; } /** @@ -78,54 +133,27 @@ public ServletContextInitializer servletContextInitializer() { } /** - * To expose the Pivot services. + * {@inheritDoc} * - * @author ActiveViam + *

This configures a new firewall accepting `%` in URLs, as none of the core services encode + * information in URL. This prevents from double-decoding exploits.
+ * The firewall is also configured to accept `\` - backslash - as none of ActiveViam APIs offer to + * manipulate files from URL parameters.
+ * Yet, nor `/` and `.` - slash and point - are accepted, as it may trick the REGEXP matchers used + * for security. Support for those two characters can be added at your own risk, by extending this + * method. As far as ActiveViam APIs are concerned, `/` and `.` in URL parameters do not represent + * any risk. `;` - semi-colon - is also not supported, for various APIs end up target an actual + * database, and because this character is less likely to be used. */ - @Configuration - public static class ActivePivotSecurityConfigurer extends AWebSecurityConfigurer { - - /** The autowired Spring configuration for ActivePivot. */ - @Autowired protected IActivePivotConfig activePivotConfig; - - /** Constructor. */ - public ActivePivotSecurityConfigurer() { - super(COOKIE_NAME); - } - - @Override - protected void doConfigure(HttpSecurity http) throws Exception { - http.authorizeRequests() - // The order of the matchers matters - .antMatchers(HttpMethod.OPTIONS, REST_API_URL_PREFIX + "/**") - .permitAll() - // Spring remoting services used by AP live 3.4 - .antMatchers(url(ID_GENERATOR_REMOTING_SERVICE, "**")) - .hasAnyAuthority(ROLE_USER, ROLE_TECH) - .antMatchers(url(LONG_POLLING_REMOTING_SERVICE, "**")) - .hasAnyAuthority(ROLE_USER, ROLE_TECH) - .antMatchers(url(LICENSING_REMOTING_SERVICE, "**")) - .hasAnyAuthority(ROLE_USER, ROLE_TECH) - // The ping service is temporarily authenticated (see PIVOT-3149) - .antMatchers(url(REST_API_URL_PREFIX, PING_SUFFIX)) - .hasAnyAuthority(ROLE_USER, ROLE_TECH) - // REST services - .antMatchers(REST_API_URL_PREFIX + "/**") - .hasAnyAuthority(ROLE_USER) - // One has to be a user for all the other URLs - .antMatchers("/**") - .hasAuthority(ROLE_USER) - .and() - .httpBasic() - // SwitchUserFilter is the last filter in the chain. See FilterComparator class. - .and() - .addFilterAfter(this.activePivotConfig.contextValueFilter(), SwitchUserFilter.class); - } + @Bean + public StrictHttpFirewall configureFirewall() { + final StrictHttpFirewall firewall = new StrictHttpFirewall(); + firewall.setAllowUrlEncodedPercent(true); + firewall.setAllowBackSlash(true); - @Bean(name = BeanIds.AUTHENTICATION_MANAGER) - @Override - public AuthenticationManager authenticationManagerBean() throws Exception { - return super.authenticationManagerBean(); - } + firewall.setAllowUrlEncodedSlash(false); + firewall.setAllowUrlEncodedPeriod(false); + firewall.setAllowSemicolon(false); + return firewall; } } diff --git a/src/main/java/com/activeviam/mac/cfg/security/impl/UserConfig.java b/src/main/java/com/activeviam/mac/cfg/security/impl/UserConfig.java deleted file mode 100644 index 9800b794..00000000 --- a/src/main/java/com/activeviam/mac/cfg/security/impl/UserConfig.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * (C) ActiveViam 2017 - * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY - * property of ActiveViam. Any unauthorized use, - * reproduction or transfer of this material is strictly prohibited - */ - -package com.activeviam.mac.cfg.security.impl; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.core.userdetails.UserDetailsService; - -/** - * Spring configuration that defines the users and their associated roles in the Sandbox - * application. - * - * @author ActiveViam - */ -@Configuration -public class UserConfig { - - /** - * ROLE_KPI is added to users, to give them permission to read kpis created by other users in the - * content server In order to "share" kpis created in the content server, the kpi reader role is - * set to : ROLE_KPI. - */ - public static final String ROLE_KPI = "ROLE_KPI"; - - /** - * [Bean] Create the users that can access the application. - * - * @return {@link UserDetailsService user data} - */ - @Bean - public UserDetailsService userDetailsService() { - return new InMemoryUserDetailsManagerBuilder() - - // "Real users" - .withUser("admin") - .password("admin") - .authorities( - SecurityConfig.ROLE_USER, - SecurityConfig.ROLE_ADMIN, - ROLE_KPI, - SecurityConfig.ROLE_CS_ROOT) - .and() - - // We're done - .build(); - } -} diff --git a/src/main/java/com/activeviam/mac/cfg/security/impl/VersionSecurityConfigurer.java b/src/main/java/com/activeviam/mac/cfg/security/impl/VersionSecurityConfigurer.java deleted file mode 100644 index e3a4c1e1..00000000 --- a/src/main/java/com/activeviam/mac/cfg/security/impl/VersionSecurityConfigurer.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * (C) ActiveViam 2017 - * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY - * property of ActiveViam. Any unauthorized use, - * reproduction or transfer of this material is strictly prohibited - */ - -package com.activeviam.mac.cfg.security.impl; - -import org.springframework.context.annotation.Configuration; -import org.springframework.core.annotation.Order; - -/** - * To expose the Version REST service. - * - * @author ActiveViam - */ -@Configuration -@Order(3) -public class VersionSecurityConfigurer extends ASecurityConfig.AVersionSecurityConfigurer {} diff --git a/src/main/java/com/activeviam/comparators/ReverseEpochViewComparator.java b/src/main/java/com/activeviam/mac/comparators/ReverseEpochViewComparator.java similarity index 89% rename from src/main/java/com/activeviam/comparators/ReverseEpochViewComparator.java rename to src/main/java/com/activeviam/mac/comparators/ReverseEpochViewComparator.java index 4416de92..45d45d81 100644 --- a/src/main/java/com/activeviam/comparators/ReverseEpochViewComparator.java +++ b/src/main/java/com/activeviam/mac/comparators/ReverseEpochViewComparator.java @@ -5,13 +5,13 @@ * reproduction or transfer of this material is strictly prohibited */ -package com.activeviam.comparators; +package com.activeviam.mac.comparators; import com.activeviam.mac.statistic.memory.visitor.impl.DistributedEpochView; import com.activeviam.mac.statistic.memory.visitor.impl.EpochView; import com.activeviam.mac.statistic.memory.visitor.impl.RegularEpochView; -import com.quartetfs.fwk.QuartetExtendedPluginValue; -import com.quartetfs.fwk.ordering.IComparator; +import com.activeviam.tech.core.api.ordering.IComparator; +import com.activeviam.tech.core.api.registry.AtotiExtendedPluginValue; /** * A comparator for epoch views. @@ -25,7 +25,7 @@ * cube ids first, and their epoch ids second (more recent epochs are lesser than older ones) * */ -@QuartetExtendedPluginValue(intf = IComparator.class, key = ReverseEpochViewComparator.PLUGIN_KEY) +@AtotiExtendedPluginValue(intf = IComparator.class, key = ReverseEpochViewComparator.PLUGIN_KEY) public class ReverseEpochViewComparator implements IComparator { /** The plugin key of the comparator. */ diff --git a/src/main/java/com/activeviam/formatter/ByteFormatter.java b/src/main/java/com/activeviam/mac/formatter/ByteFormatter.java similarity index 90% rename from src/main/java/com/activeviam/formatter/ByteFormatter.java rename to src/main/java/com/activeviam/mac/formatter/ByteFormatter.java index 2b745e7f..eb284192 100644 --- a/src/main/java/com/activeviam/formatter/ByteFormatter.java +++ b/src/main/java/com/activeviam/mac/formatter/ByteFormatter.java @@ -5,17 +5,17 @@ * reproduction or transfer of this material is strictly prohibited */ -package com.activeviam.formatter; +package com.activeviam.mac.formatter; -import com.quartetfs.fwk.QuartetExtendedPluginValue; -import com.quartetfs.fwk.format.IFormatter; +import com.activeviam.tech.core.api.format.IFormatter; +import com.activeviam.tech.core.api.registry.AtotiExtendedPluginValue; /** * Formatter displaying byte amounts with decimal units. * * @author ActiveViam */ -@QuartetExtendedPluginValue(intf = IFormatter.class, key = ByteFormatter.KEY) +@AtotiExtendedPluginValue(intf = IFormatter.class, key = ByteFormatter.KEY) public class ByteFormatter implements IFormatter { /** Plugin key. */ diff --git a/src/main/java/com/activeviam/formatter/ClassFormatter.java b/src/main/java/com/activeviam/mac/formatter/ClassFormatter.java similarity index 84% rename from src/main/java/com/activeviam/formatter/ClassFormatter.java rename to src/main/java/com/activeviam/mac/formatter/ClassFormatter.java index fddfd70d..edcf926c 100644 --- a/src/main/java/com/activeviam/formatter/ClassFormatter.java +++ b/src/main/java/com/activeviam/mac/formatter/ClassFormatter.java @@ -5,10 +5,10 @@ * reproduction or transfer of this material is strictly prohibited */ -package com.activeviam.formatter; +package com.activeviam.mac.formatter; -import com.quartetfs.fwk.QuartetExtendedPluginValue; -import com.quartetfs.fwk.format.IFormatter; +import com.activeviam.tech.core.api.format.IFormatter; +import com.activeviam.tech.core.api.registry.AtotiExtendedPluginValue; import java.util.Arrays; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -18,7 +18,7 @@ * * @author ActiveViam */ -@QuartetExtendedPluginValue(intf = IFormatter.class, key = ClassFormatter.KEY) +@AtotiExtendedPluginValue(intf = IFormatter.class, key = ClassFormatter.KEY) public class ClassFormatter implements IFormatter { /** Plugin key. */ diff --git a/src/main/java/com/activeviam/formatter/PartitionIdFormatter.java b/src/main/java/com/activeviam/mac/formatter/PartitionIdFormatter.java similarity index 79% rename from src/main/java/com/activeviam/formatter/PartitionIdFormatter.java rename to src/main/java/com/activeviam/mac/formatter/PartitionIdFormatter.java index 022569a6..1f627d88 100644 --- a/src/main/java/com/activeviam/formatter/PartitionIdFormatter.java +++ b/src/main/java/com/activeviam/mac/formatter/PartitionIdFormatter.java @@ -5,18 +5,18 @@ * reproduction or transfer of this material is strictly prohibited */ -package com.activeviam.formatter; +package com.activeviam.mac.formatter; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; -import com.quartetfs.fwk.QuartetExtendedPluginValue; -import com.quartetfs.fwk.format.IFormatter; +import com.activeviam.tech.core.api.format.IFormatter; +import com.activeviam.tech.core.api.registry.AtotiExtendedPluginValue; /** * Formatter for partitions. * * @author ActiveViam */ -@QuartetExtendedPluginValue(intf = IFormatter.class, key = PartitionIdFormatter.KEY) +@AtotiExtendedPluginValue(intf = IFormatter.class, key = PartitionIdFormatter.KEY) public class PartitionIdFormatter implements IFormatter { /** Plugin key. */ diff --git a/src/main/java/com/activeviam/mac/memory/AnalysisDatastoreFeeder.java b/src/main/java/com/activeviam/mac/memory/AnalysisDatastoreFeeder.java index 1eaefb7e..c3ff1554 100644 --- a/src/main/java/com/activeviam/mac/memory/AnalysisDatastoreFeeder.java +++ b/src/main/java/com/activeviam/mac/memory/AnalysisDatastoreFeeder.java @@ -7,7 +7,13 @@ package com.activeviam.mac.memory; +import com.activeviam.database.api.conditions.BaseConditions; +import com.activeviam.database.api.query.ListQuery; import com.activeviam.database.api.schema.FieldPath; +import com.activeviam.database.api.schema.IDataTable; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; +import com.activeviam.database.datastore.api.transaction.stats.IDatastoreSchemaTransactionInformation; import com.activeviam.mac.Loggers; import com.activeviam.mac.entities.ChunkOwner; import com.activeviam.mac.entities.DistributedCubeOwner; @@ -15,14 +21,9 @@ import com.activeviam.mac.statistic.memory.visitor.impl.EpochView; import com.activeviam.mac.statistic.memory.visitor.impl.FeedVisitor; import com.activeviam.mac.statistic.memory.visitor.impl.RegularEpochView; -import com.qfs.condition.impl.BaseConditions; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.store.IDatastore; -import com.qfs.store.query.ICursor; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.record.IRecordReader; -import com.qfs.store.transaction.IDatastoreSchemaTransactionInformation; -import com.qfs.store.transaction.IOpenedTransaction; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.records.api.ICursor; +import com.activeviam.tech.records.api.IRecordReader; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -51,18 +52,23 @@ public class AnalysisDatastoreFeeder { * statistics. */ private final Map> regularEpochsPerOwner; + /** * A mapping giving for each distributed owner the epochs that were expressed in their statistics. */ private final Map> distributedEpochsPerOwner; + final IDatastore datastore; + /** * Constructor. * * @param dumpName the dump name to assign to the statistic + * @param datastore */ - public AnalysisDatastoreFeeder(final String dumpName) { + public AnalysisDatastoreFeeder(final String dumpName, IDatastore datastore) { this.dumpName = dumpName; + this.datastore = datastore; this.datastoreEpochs = new HashSet<>(); this.regularEpochsPerOwner = new HashMap<>(); @@ -80,12 +86,12 @@ public AnalysisDatastoreFeeder(final String dumpName) { * @return the record */ private static Object[] generateEpochViewTuple( - final IRecordFormat recordFormat, + final IDataTable recordFormat, final ChunkOwner owner, final String dumpName, final long baseEpochId, final EpochView epochView) { - final Object[] tuple = new Object[recordFormat.getFieldCount()]; + final Object[] tuple = new Object[recordFormat.getFields().size()]; tuple[recordFormat.getFieldIndex(DatastoreConstants.EPOCH_VIEW__OWNER)] = owner; tuple[recordFormat.getFieldIndex(DatastoreConstants.CHUNK__DUMP_NAME)] = dumpName; tuple[recordFormat.getFieldIndex(DatastoreConstants.EPOCH_VIEW__BASE_EPOCH_ID)] = baseEpochId; @@ -99,13 +105,13 @@ private static Object[] generateEpochViewTuple( * @return the result of the transaction */ public Optional loadInto( - final IDatastore datastore, final Stream stats) { + final Stream stats) { return datastore.edit(transaction -> loadWithTransaction(transaction, stats)); } /** Loads the provided statistics within an open transaction. */ public void loadWithTransaction( - final IOpenedTransaction transaction, final Stream stats) { + final IOpenedTransaction transaction, final Stream stats) { stats.forEach(stat -> feedChunk(transaction, stat)); completeTransaction(transaction); } @@ -115,12 +121,13 @@ public void loadWithTransaction( * * @param transaction the transaction to add facts to */ - private void feedChunk(final IOpenedTransaction transaction, final IMemoryStatistic statistic) { + private void feedChunk(final IOpenedTransaction transaction, final AMemoryStatistic statistic) { if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine("Start feeding the application with " + statistic); } - statistic.accept(new FeedVisitor(transaction.getMetadata(), transaction, this.dumpName)); + statistic.accept( + new FeedVisitor(this.datastore.getCurrentSchema(), transaction, this.dumpName)); if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine("Application processed " + statistic); @@ -139,15 +146,17 @@ private void completeTransaction(final IOpenedTransaction transaction) { * @param transaction the transaction to collect the epochs from */ private void collectEpochsFromOpenedTransaction(final IOpenedTransaction transaction) { - final ICursor result = - transaction - .getQueryRunner() - .forStore(DatastoreConstants.CHUNK_STORE) + final ListQuery query = + this.datastore + .getQueryManager() + .listQuery() + .forTable(DatastoreConstants.CHUNK_STORE) .withCondition( BaseConditions.equal( FieldPath.of(DatastoreConstants.CHUNK__DUMP_NAME), this.dumpName)) - .selecting(DatastoreConstants.OWNER__OWNER, DatastoreConstants.VERSION__EPOCH_ID) - .run(); + .withTableFields(DatastoreConstants.OWNER__OWNER, DatastoreConstants.VERSION__EPOCH_ID) + .toQuery(); + final ICursor result = transaction.getQueryRunner().listQuery(query).run(); for (final IRecordReader reader : result) { final ChunkOwner owner = (ChunkOwner) reader.read(0); @@ -173,12 +182,8 @@ private void replicateChunksForMissingEpochs(final IOpenedTransaction transactio } private void replicateDatastoreEpochs(final IOpenedTransaction transaction) { - final IRecordFormat epochViewRecordFormat = - transaction - .getMetadata() - .getStoreMetadata(DatastoreConstants.EPOCH_VIEW_STORE) - .getStoreFormat() - .getRecordFormat(); + final var epochViewRecordFormat = + this.datastore.getCurrentSchema().getTable(DatastoreConstants.EPOCH_VIEW_STORE); for (final var datastoreEpochId : this.datastoreEpochs) { for (final ChunkOwner owner : this.regularEpochsPerOwner.keySet()) { @@ -212,12 +217,8 @@ private void replicateDatastoreEpochs(final IOpenedTransaction transaction) { } private void replicateDistributedEpochs(final IOpenedTransaction transaction) { - final IRecordFormat epochViewRecordFormat = - transaction - .getMetadata() - .getStoreMetadata(DatastoreConstants.EPOCH_VIEW_STORE) - .getStoreFormat() - .getRecordFormat(); + final var epochViewRecordFormat = + this.datastore.getCurrentSchema().getTable(DatastoreConstants.EPOCH_VIEW_STORE); for (final ChunkOwner owner : this.distributedEpochsPerOwner.keySet()) { final Collection epochs = this.distributedEpochsPerOwner.get(owner); diff --git a/src/main/java/com/activeviam/mac/memory/ChunkRecordHandler.java b/src/main/java/com/activeviam/mac/memory/ChunkRecordHandler.java index 6879fb50..4a8e7b22 100644 --- a/src/main/java/com/activeviam/mac/memory/ChunkRecordHandler.java +++ b/src/main/java/com/activeviam/mac/memory/ChunkRecordHandler.java @@ -7,14 +7,14 @@ package com.activeviam.mac.memory; -import com.qfs.chunk.impl.TombStoneChunk; -import com.qfs.desc.IDuplicateKeyHandler; -import com.qfs.dic.IWritableDictionary; -import com.qfs.store.IStoreMetadata; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.record.IRecordReader; -import com.qfs.store.record.IWritableRecord; -import com.qfs.store.record.impl.IDictionaryProvider; +import com.activeviam.database.datastore.api.description.IDuplicateKeyHandler; +import com.activeviam.database.datastore.api.description.IKeyEventContext; +import com.activeviam.tech.chunks.internal.impl.TombStoneChunk; +import com.activeviam.tech.dictionaries.api.IDictionaryProvider; +import com.activeviam.tech.dictionaries.avinternal.IWritableDictionary; +import com.activeviam.tech.records.api.IRecordFormat; +import com.activeviam.tech.records.api.IRecordReader; +import com.activeviam.tech.records.api.IWritableRecord; /** * {@link IDuplicateKeyHandler} implementation defining the process of dealing with duplicated @@ -29,28 +29,16 @@ public class ChunkRecordHandler implements IDuplicateKeyHandler { private int defaultRefId = -1; private int defaultIdxId = -1; - @Override - public IRecordReader selectDuplicateKeyInDatastore( - final IRecordReader duplicateRecord, - final IRecordReader previousRecord, - final IStoreMetadata storeMetadata, - final IDictionaryProvider dictionaryProvider, - final int[] uniqueIndexFields, - final int partitionId) { - return createMergedRecord(duplicateRecord, previousRecord, storeMetadata, dictionaryProvider); - } - private IRecordReader createMergedRecord( IRecordReader duplicateRecord, IRecordReader previousRecord, - IStoreMetadata storeMetadata, - IDictionaryProvider dictionaryProvider) { - init(storeMetadata, dictionaryProvider); + IKeyEventContext keyEventContext) { final int currentPartition = getPartition(previousRecord); - final long currentDicId = getDicId(previousRecord); - final long currentRefId = getRefId(previousRecord); - final long currentIdxId = getIdxId(previousRecord); + final int currentDicId = getDicId(previousRecord); + final int currentRefId = getRefId(previousRecord); + final int currentIdxId = getIdxId(previousRecord); + init(previousRecord, keyEventContext.getDictionaryProvider()); if (currentPartition == this.sharedPartitionId) { // We cannot make any change @@ -67,7 +55,7 @@ private IRecordReader createMergedRecord( } else { // We ignore TombStoneChunks as they are a singleton that has minimal memory footprint // but don't work with the current MAC data model - if (getChunkClassName(duplicateRecord, dictionaryProvider, storeMetadata) + if (getChunkClassName(duplicateRecord, keyEventContext.getDictionaryProvider()) .contains(TombStoneChunk.class.getName())) { return duplicateRecord; } @@ -75,8 +63,7 @@ private IRecordReader createMergedRecord( assert newPartition != MemoryAnalysisDatastoreDescriptionConfig.NO_PARTITION; assert currentPartition != MemoryAnalysisDatastoreDescriptionConfig.NO_PARTITION; - final int partitionIdx = - storeMetadata.getFieldIndex(DatastoreConstants.CHUNK__PARTITION_ID); + final int partitionIdx = getPartition(newRecord); newRecord.writeInt(partitionIdx, this.sharedPartitionId); // Sanity check in case two Chunks have different parents which should never happen @@ -101,26 +88,25 @@ private IRecordReader createMergedRecord( } private void init( - final IStoreMetadata storeMetadata, final IDictionaryProvider dictionaryProvider) { + final IRecordReader recordReader, final IDictionaryProvider dictionaryProvider) { if (this.sharedPartitionId < 0) { - final int partitionIdx = storeMetadata.getFieldIndex(DatastoreConstants.CHUNK__PARTITION_ID); + final int dicIdIdx = getPartition(recordReader); @SuppressWarnings("unchecked") final IWritableDictionary partitionDictionary = - (IWritableDictionary) dictionaryProvider.getDictionary(partitionIdx); + (IWritableDictionary) dictionaryProvider.getDictionary(dicIdIdx); this.sharedPartitionId = partitionDictionary.map(MemoryAnalysisDatastoreDescriptionConfig.MANY_PARTITIONS); } if (this.defaultDicId < 0) { - final int dicIdIdx = storeMetadata.getFieldIndex(DatastoreConstants.CHUNK__PARENT_DICO_ID); + final int dicIdIdx = getDicId(recordReader); @SuppressWarnings("unchecked") final IWritableDictionary dicIdDictionary = (IWritableDictionary) dictionaryProvider.getDictionary(dicIdIdx); this.defaultDicId = dicIdDictionary.map(MemoryAnalysisDatastoreDescriptionConfig.DEFAULT_COMPONENT_ID_VALUE); } - if (this.defaultIdxId < 0) { - final int idxIdIdx = storeMetadata.getFieldIndex(DatastoreConstants.CHUNK__PARENT_INDEX_ID); + final int idxIdIdx = getIdxId(recordReader); @SuppressWarnings("unchecked") final IWritableDictionary idxIdDictionary = (IWritableDictionary) dictionaryProvider.getDictionary(idxIdIdx); @@ -129,7 +115,7 @@ private void init( } if (this.defaultRefId < 0) { - final int refIdIdx = storeMetadata.getFieldIndex(DatastoreConstants.CHUNK__PARENT_REF_ID); + final int refIdIdx = getRefId(recordReader); @SuppressWarnings("unchecked") final IWritableDictionary refIdDictionary = (IWritableDictionary) dictionaryProvider.getDictionary(refIdIdx); @@ -138,37 +124,31 @@ private void init( } } - private int getPartition(final IRecordReader record) { - final int idx = record.getFormat().getFieldIndex(DatastoreConstants.CHUNK__PARTITION_ID); - return record.readInt(idx); + private int getPartition(final IRecordReader recordReader) { + return recordReader.getFormat().getFieldIndex(DatastoreConstants.CHUNK__PARTITION_ID); } - private long getDicId(final IRecordReader record) { - final int idx = record.getFormat().getFieldIndex(DatastoreConstants.CHUNK__PARENT_DICO_ID); - return record.readLong(idx); + private int getDicId(final IRecordReader recordReader) { + return recordReader.getFormat().getFieldIndex(DatastoreConstants.CHUNK__PARENT_DICO_ID); } - private long getIdxId(final IRecordReader record) { - final int idx = record.getFormat().getFieldIndex(DatastoreConstants.CHUNK__PARENT_INDEX_ID); - return record.readLong(idx); + private int getIdxId(final IRecordReader recordReader) { + return recordReader.getFormat().getFieldIndex(DatastoreConstants.CHUNK__PARENT_INDEX_ID); } - private long getRefId(final IRecordReader record) { - final int idx = record.getFormat().getFieldIndex(DatastoreConstants.CHUNK__PARENT_REF_ID); - return record.readLong(idx); + private int getRefId(final IRecordReader recordReader) { + return recordReader.getFormat().getFieldIndex(DatastoreConstants.CHUNK__PARENT_REF_ID); } private String getChunkClassName( - final IRecordReader record, - final IDictionaryProvider dictionaryProvider, - final IStoreMetadata storeMetadata) { - final int partitionIdx = storeMetadata.getFieldIndex(DatastoreConstants.CHUNK__CLASS); - final int idx = record.getFormat().getFieldIndex(DatastoreConstants.CHUNK__CLASS); + final IRecordReader recordReader, final IDictionaryProvider dictionaryProvider) { + final int partitionIdx = getPartition(recordReader); + final int idx = recordReader.getFormat().getFieldIndex(DatastoreConstants.CHUNK__CLASS); @SuppressWarnings("unchecked") final IWritableDictionary refIdDictionary = (IWritableDictionary) dictionaryProvider.getDictionary(partitionIdx); - return (String) refIdDictionary.read((Integer) record.read(idx)); + return (String) refIdDictionary.read((Integer) recordReader.read(idx)); } private IWritableRecord copyRecord(final IRecordReader record) { @@ -180,4 +160,12 @@ private IWritableRecord copyRecord(final IRecordReader record) { return newRecord; } + + @Override + public IRecordReader selectDuplicateKeyInDatastore( + final IRecordReader duplicateRecord, + final IRecordReader previousRecord, + final IKeyEventContext keyEventContext) { + return createMergedRecord(duplicateRecord, previousRecord, keyEventContext); + } } diff --git a/src/main/java/com/activeviam/mac/memory/DatastoreConstants.java b/src/main/java/com/activeviam/mac/memory/DatastoreConstants.java index cb3fe9af..98612628 100644 --- a/src/main/java/com/activeviam/mac/memory/DatastoreConstants.java +++ b/src/main/java/com/activeviam/mac/memory/DatastoreConstants.java @@ -7,8 +7,8 @@ package com.activeviam.mac.memory; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.quartetfs.biz.pivot.impl.ActivePivotManager; +import com.activeviam.activepivot.core.impl.internal.impl.ActivePivotManager; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; /** * List of constants used by stores storing the off-heap memory analysis results. diff --git a/src/main/java/com/activeviam/mac/memory/MemoryAnalysisDatastoreDescriptionConfig.java b/src/main/java/com/activeviam/mac/memory/MemoryAnalysisDatastoreDescriptionConfig.java index ac711bfa..232d90cb 100644 --- a/src/main/java/com/activeviam/mac/memory/MemoryAnalysisDatastoreDescriptionConfig.java +++ b/src/main/java/com/activeviam/mac/memory/MemoryAnalysisDatastoreDescriptionConfig.java @@ -7,20 +7,20 @@ package com.activeviam.mac.memory; -import com.activeviam.builders.StartBuilding; +import com.activeviam.activepivot.core.datastore.api.builder.StartBuilding; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; import com.activeviam.database.api.schema.StoreField; -import com.qfs.chunk.IChunk; -import com.qfs.desc.IDatastoreSchemaDescription; -import com.qfs.desc.IReferenceDescription; -import com.qfs.desc.IStoreDescription; -import com.qfs.desc.impl.DatastoreSchemaDescription; -import com.qfs.desc.impl.DuplicateKeyHandlers; -import com.qfs.desc.impl.StoreDescription; -import com.qfs.literal.ILiteralType; -import com.qfs.pool.impl.QFSPools; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.util.impl.QfsArrays; -import com.quartetfs.fwk.format.IParser; +import com.activeviam.database.api.types.ILiteralType; +import com.activeviam.database.datastore.api.description.IDatastoreSchemaDescription; +import com.activeviam.database.datastore.api.description.IReferenceDescription; +import com.activeviam.database.datastore.api.description.IStoreDescription; +import com.activeviam.database.datastore.api.description.impl.DatastoreSchemaDescription; +import com.activeviam.database.datastore.api.description.impl.DuplicateKeyHandlers; +import com.activeviam.database.datastore.api.description.impl.StoreDescription; +import com.activeviam.tech.chunks.internal.IChunk; +import com.activeviam.tech.chunks.internal.pool.impl.AtotiPools; +import com.activeviam.tech.core.api.format.IParser; +import com.activeviam.tech.core.internal.util.ArrayUtil; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -64,7 +64,7 @@ public class MemoryAnalysisDatastoreDescriptionConfig implements IDatastoreSchem * @return the modulo partitioning value */ private static int partitioningModulo() { - return QFSPools.getMixedWorkloadThreadCount(); + return AtotiPools.getMixedWorkloadThreadCount(); } @Override @@ -369,7 +369,7 @@ protected IStoreDescription applicationStore() { .build(); } - public Collection getStoreDescriptions() { + public List getStoreDescriptions() { return Arrays.asList( chunkStore(), referenceStore(), @@ -584,7 +584,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return QfsArrays.join(", ", this.fieldNames).toString(); + return ArrayUtil.join(", ", this.fieldNames).toString(); } } } diff --git a/src/main/java/com/activeviam/mac/statistic/memory/deserializer/RetroCompatibleDeserializer.java b/src/main/java/com/activeviam/mac/statistic/memory/deserializer/RetroCompatibleDeserializer.java new file mode 100644 index 00000000..bd430363 --- /dev/null +++ b/src/main/java/com/activeviam/mac/statistic/memory/deserializer/RetroCompatibleDeserializer.java @@ -0,0 +1,285 @@ +/* + * (C) ActiveViam 2015 + * ALL RIGHTS RESERVED. This material is the CONFIDENTIAL and PROPRIETARY + * property of ActiveViam. Any unauthorized use, + * reproduction or transfer of this material is strictly prohibited + */ + +package com.activeviam.mac.statistic.memory.deserializer; + +import static com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryStatisticSerializerUtil.COMPRESSED_FILE_EXTENSION; + +import com.activeviam.activepivot.server.impl.private_.observability.memory.AStatisticDeserializer; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryStatisticAdapter; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; +import com.activeviam.tech.core.api.exceptions.service.InternalServiceException; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.api.memory.IStatisticAttribute; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.core.json.JsonReadFeature; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.module.SimpleModule; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.xerial.snappy.SnappyFramedInputStream; + +/** + * {@link JsonDeserializer} for {@link IMemoryStatistic}. + * + * @author ActiveViam + */ +public class RetroCompatibleDeserializer extends AStatisticDeserializer { + + /** Mapping from 6.1 to 6.0 classes */ + private static final Map CLASS_MAPPING_6_0_to_6_1 = + Map.of( + "com.qfs.monitoring.statistic.memory.impl.ChunkStatistic", + "com.activeviam.tech.observability.internal.memory.ChunkStatistic", + "com.qfs.monitoring.statistic.memory.impl.ChunkSetStatistic", + "com.activeviam.tech.observability.internal.memory.ChunkSetStatistic", + "com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic", + "com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic", + "com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic", + "com.activeviam.tech.observability.internal.memory.DictionaryStatistic", + "com.qfs.monitoring.statistic.memory.impl.ReferenceStatistic", + "com.activeviam.tech.observability.internal.memory.ReferenceStatistic", + "com.qfs.monitoring.statistic.memory.impl.IndexStatistic", + "com.activeviam.tech.observability.internal.memory.IndexStatistic"); + + private static final Set AVAILABLE_CLASSES_6_1 = + Set.of( + "com.activeviam.tech.observability.internal.memory.ChunkStatistic", + "com.activeviam.tech.observability.internal.memory.ChunkSetStatistic", + "com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic", + "com.activeviam.tech.observability.internal.memory.DictionaryStatistic", + "com.activeviam.tech.observability.internal.memory.ReferenceStatistic", + "com.activeviam.tech.observability.internal.memory.IndexStatistic"); + + private static final Logger LOGGER = Logger.getLogger("mac.statistic.memory.deserializer"); + private static ObjectMapper serializer; + + static { + serializer = + new ObjectMapper() + .configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false) + .configure(JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER.mappedFeature(), true) + .configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), true) + .configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false) + .configure(JsonReadFeature.ALLOW_NON_NUMERIC_NUMBERS.mappedFeature(), true); + serializer.setSerializationInclusion(JsonInclude.Include.NON_NULL); + + final SimpleModule deserializeModule = new SimpleModule(); + deserializeModule.addDeserializer(AMemoryStatistic.class, new RetroCompatibleDeserializer()); + serializer.registerModule(deserializeModule); + } + + @Override + public AMemoryStatistic deserialize(final JsonParser parser, final DeserializationContext ctx) + throws IOException { + if (!JsonToken.START_OBJECT.equals(parser.currentToken())) { + throw new IllegalArgumentException("Should be called at the start of an object"); + } + + parser.nextToken(); + final String name; + if (Objects.equals(readFieldName(parser), MemoryStatisticAdapter.NAME_ATTR)) { + readAndCheckFieldName(parser, MemoryStatisticAdapter.NAME_ATTR); + name = readStringField(parser); + parser.nextToken(); + } else { + name = null; + } + + final long onHeap; + if (Objects.equals(readFieldName(parser), MemoryStatisticAdapter.ON_HEAP_ATTR)) { + onHeap = readLongField(parser); + parser.nextToken(); + } else { + onHeap = -1; + } + + final long offHeap; + if (Objects.equals(readFieldName(parser), MemoryStatisticAdapter.OFF_HEAP_ATTR)) { + offHeap = readLongField(parser); + parser.nextToken(); + } else { + offHeap = -1; + } + + checkFieldName(MemoryStatisticAdapter.STATISTIC_CLASS_ATTR, readFieldName(parser)); + final String klassName = readStringField(parser); + + parser.nextToken(); + final Map attributes; + if (Objects.equals(readFieldName(parser), MemoryStatisticAdapter.ATTRIBUTES_ATTR)) { + parser.nextToken(); + attributes = parseAttributes(parser, ctx); + } else { + attributes = Collections.emptyMap(); + } + + parser.nextToken(); + final List children; + if (Objects.equals(readFieldName(parser), MemoryStatisticAdapter.CHILDREN_ATTR)) { + parser.nextToken(); + children = parseChildren(parser, ctx, AMemoryStatistic.class); + } else { + children = Collections.emptyList(); + } + + handleExcessiveAttributes(parser); + + if (!Objects.equals(parser.currentToken(), JsonToken.END_OBJECT)) { + throw new IllegalStateException( + "Unexpected additional tokens. First is " + parser.currentToken()); + } + + return createDeserialized(klassName, name, onHeap, offHeap, attributes, children); + } + + /** + * Creates the actual statistic using the parsed attributed. + * + * @param klassName name of the statistic class + * @param name name of the statistic + * @param onHeap value read for the consumed on-heap memory - negative if unset + * @param offHeap value read for the consume off-heap memory - negative if unset + * @param attributes statistic attributes + * @param children child statistics + * @return the created deserialized statistic + */ + protected AMemoryStatistic createDeserialized( + final String klassName, + final String name, + final long onHeap, + final long offHeap, + final Map attributes, + final List children) { + Class klass; + if (AVAILABLE_CLASSES_6_1.contains(klassName)) { + klass = getClass(klassName); + } else if (classNeedsTranslation(klassName)) { + klass = getClass(getClassTranslation(klassName)); + } else { + throw new InternalServiceException("Cannot find statistic class " + klassName); + } + + final AMemoryStatistic objDeserialized; + try { + objDeserialized = (AMemoryStatistic) klass.getDeclaredConstructor().newInstance(); + } catch (final InstantiationException + | IllegalAccessException + | NoSuchMethodException + | InvocationTargetException e) { + throw new InternalServiceException("Cannot create instance of " + klassName, e); + } + + try { + klass.getMethod("setAttributes", Map.class).invoke(objDeserialized, attributes); + } catch (final IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new InternalServiceException("Cannot set attributes for class " + klassName, e); + } + try { + klass.getMethod("setChildren", Collection.class).invoke(objDeserialized, children); + } catch (final IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new InternalServiceException("Cannot set children for class " + klassName, e); + } + + if (name != null) { + objDeserialized.setName(name); + } + if (onHeap >= 0) { + objDeserialized.setShallowOnHeap(onHeap); + } + if (offHeap >= 0) { + objDeserialized.setShallowOffHeap(offHeap); + } + + return objDeserialized; + } + + public static AMemoryStatistic readStatisticFile(final Path file) { + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("Reading statistics from " + file.toAbsolutePath()); + } + final AMemoryStatistic read = readStatistic(file.toFile()); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("Statistics read from " + file.toAbsolutePath()); + } + return read; + } + + public static AMemoryStatistic readStatistic(final File file) { + + try { + try (final InputStream inputStream = inputStream(file); + final InputStreamReader statistic = + new InputStreamReader(inputStream, StandardCharsets.UTF_8)) { + final ObjectReader reader = serializer.readerFor(AMemoryStatistic.class); + return readStatisticObject(reader, statistic); + } + } catch (IOException e) { + throw new ActiveViamRuntimeException(e); + } + } + + private static AMemoryStatistic readStatisticObject( + ObjectReader reader, InputStreamReader statistic) { + try { + return reader.readValue(new BufferedReader(statistic)); + } catch (final IOException e) { + throw new ActiveViamRuntimeException(e); + } + } + + private static InputStream inputStream(final File file) throws IOException { + final boolean isCompressedFile = file.getName().endsWith("." + COMPRESSED_FILE_EXTENSION); + + InputStream inputStream = + new FileInputStream(file); // NOSONAR: used in try-with-resources above + if (isCompressedFile) { + inputStream = new SnappyFramedInputStream(inputStream); + } + return inputStream; + } + + private static String getClassTranslation(final String klassName) { + return CLASS_MAPPING_6_0_to_6_1.get(klassName); + } + + private static boolean classNeedsTranslation(final String klassName) { + return CLASS_MAPPING_6_0_to_6_1.containsKey(klassName); + } + + private static Class getClass(final String klassName) { + Class klass; + try { + klass = Class.forName(klassName); + } catch (final ClassNotFoundException notFound) { + throw new InternalServiceException("Cannot get class " + klassName, notFound); + } + return klass; + } +} diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/ADatastoreFeedVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/ADatastoreFeedVisitor.java index 0d24bcd5..9bc95391 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/ADatastoreFeedVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/ADatastoreFeedVisitor.java @@ -7,8 +7,8 @@ package com.activeviam.mac.statistic.memory.visitor.impl; -import com.qfs.store.IDatastoreSchemaMetadata; -import com.qfs.store.transaction.IOpenedTransaction; +import com.activeviam.database.api.schema.IDatabaseSchema; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; import java.util.Collection; /** @@ -21,8 +21,10 @@ public abstract class ADatastoreFeedVisitor extends AFeedVisitorWithDictionar /** Names of the currently visited fields. */ protected Collection fields; + /** ID of the current index. */ protected Long indexId; + /** ID of the current reference. */ protected Long referenceId; @@ -34,7 +36,7 @@ public abstract class ADatastoreFeedVisitor extends AFeedVisitorWithDictionar * @param dumpName dumpName of the ongoing import */ public ADatastoreFeedVisitor( - IOpenedTransaction transaction, IDatastoreSchemaMetadata storageMetadata, String dumpName) { + IOpenedTransaction transaction, IDatabaseSchema storageMetadata, String dumpName) { super(transaction, storageMetadata, dumpName); } } diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/AFeedVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/AFeedVisitor.java index 32410fc0..5743a952 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/AFeedVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/AFeedVisitor.java @@ -7,13 +7,14 @@ package com.activeviam.mac.statistic.memory.visitor.impl; +import com.activeviam.database.api.schema.IDataTable; +import com.activeviam.database.api.schema.IDatabaseSchema; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; import com.activeviam.mac.entities.ChunkOwner; import com.activeviam.mac.memory.DatastoreConstants; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.visitor.IMemoryStatisticVisitor; -import com.qfs.store.IDatastoreSchemaMetadata; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.transaction.IOpenedTransaction; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.IMemoryStatisticVisitor; +import com.activeviam.tech.records.api.IRecordFormat; /** * Abstract class for {@link IMemoryStatisticVisitor memory statistic visitors}. @@ -25,10 +26,13 @@ public abstract class AFeedVisitor implements IMemoryStatisticVisitor { /** Ongoing transaction. */ protected final IOpenedTransaction transaction; + /** Metadata of the Analysis Datastore. */ - protected final IDatastoreSchemaMetadata storageMetadata; + protected final IDatabaseSchema storageMetadata; + /** Name of the import. */ protected final String dumpName; + /** Owner of the visited statistics. */ protected ChunkOwner owner = null; @@ -41,7 +45,7 @@ public abstract class AFeedVisitor implements IMemoryStatisticVisitor { */ public AFeedVisitor( final IOpenedTransaction transaction, - final IDatastoreSchemaMetadata storageMetadata, + final IDatabaseSchema storageMetadata, final String dumpName) { this.transaction = transaction; this.storageMetadata = storageMetadata; @@ -54,7 +58,7 @@ public AFeedVisitor( * @param storageMetadata metadata of the application datastore * @return the {@link DatastoreConstants#CHUNK_STORE} record format */ - protected static IRecordFormat getChunkFormat(IDatastoreSchemaMetadata storageMetadata) { + protected static IDataTable getChunkFormat(IDatabaseSchema storageMetadata) { return FeedVisitor.getRecordFormat(storageMetadata, DatastoreConstants.CHUNK_STORE); } @@ -64,7 +68,7 @@ protected static IRecordFormat getChunkFormat(IDatastoreSchemaMetadata storageMe * @param storageMetadata metadata of the application datastore * @return the {@link DatastoreConstants#PROVIDER_STORE} record format */ - protected static IRecordFormat getProviderFormat(IDatastoreSchemaMetadata storageMetadata) { + protected static IDataTable getProviderFormat(IDatabaseSchema storageMetadata) { return FeedVisitor.getRecordFormat(storageMetadata, DatastoreConstants.PROVIDER_STORE); } @@ -74,7 +78,7 @@ protected static IRecordFormat getProviderFormat(IDatastoreSchemaMetadata storag * @param storageMetadata metadata of the application datastore * @return the {@link DatastoreConstants#LEVEL_STORE} record format */ - protected static IRecordFormat getLevelFormat(IDatastoreSchemaMetadata storageMetadata) { + protected static IDataTable getLevelFormat(IDatabaseSchema storageMetadata) { return FeedVisitor.getRecordFormat(storageMetadata, DatastoreConstants.LEVEL_STORE); } @@ -84,7 +88,7 @@ protected static IRecordFormat getLevelFormat(IDatastoreSchemaMetadata storageMe * @param storageMetadata metadata of the application datastore * @return the {@link DatastoreConstants#INDEX_STORE} record format */ - protected static IRecordFormat getIndexFormat(IDatastoreSchemaMetadata storageMetadata) { + protected static IDataTable getIndexFormat(IDatabaseSchema storageMetadata) { return FeedVisitor.getRecordFormat(storageMetadata, DatastoreConstants.INDEX_STORE); } @@ -94,7 +98,7 @@ protected static IRecordFormat getIndexFormat(IDatastoreSchemaMetadata storageMe * @param storageMetadata metadata of the application datastore * @return the {@link DatastoreConstants#REFERENCE_STORE} record format */ - protected static IRecordFormat getReferenceFormat(IDatastoreSchemaMetadata storageMetadata) { + protected static IDataTable getReferenceFormat(IDatabaseSchema storageMetadata) { return FeedVisitor.getRecordFormat(storageMetadata, DatastoreConstants.REFERENCE_STORE); } @@ -104,7 +108,7 @@ protected static IRecordFormat getReferenceFormat(IDatastoreSchemaMetadata stora * @param storageMetadata metadata of the application datastore * @return the {@link DatastoreConstants#VERSION_STORE} record format */ - protected static IRecordFormat getVersionStoreFormat(IDatastoreSchemaMetadata storageMetadata) { + protected static IDataTable getVersionStoreFormat(IDatabaseSchema storageMetadata) { return FeedVisitor.getRecordFormat(storageMetadata, DatastoreConstants.VERSION_STORE); } @@ -114,23 +118,17 @@ protected static IRecordFormat getVersionStoreFormat(IDatastoreSchemaMetadata st * @param storageMetadata metadata of the application datastore * @return the {@link DatastoreConstants#DICTIONARY_STORE} record format */ - protected static IRecordFormat getDictionaryFormat(IDatastoreSchemaMetadata storageMetadata) { + protected static IDataTable getDictionaryFormat(IDatabaseSchema storageMetadata) { return FeedVisitor.getRecordFormat(storageMetadata, DatastoreConstants.DICTIONARY_STORE); } - @Override - public R visit(final IMemoryStatistic memoryStatistic) { - visitChildren(memoryStatistic); - return null; - } - /** - * Visits all the children of the given {@link IMemoryStatistic}. + * Visits all the children of the given {@link AMemoryStatistic}. * * @param statistic The statistics whose children to visit. */ - protected void visitChildren(final IMemoryStatistic statistic) { - for (final IMemoryStatistic child : statistic.getChildren()) { + protected void visitChildren(final AMemoryStatistic statistic) { + for (final AMemoryStatistic child : statistic.getChildren()) { child.accept(this); } } diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/AFeedVisitorWithDictionary.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/AFeedVisitorWithDictionary.java index 98d26189..b15d9da6 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/AFeedVisitorWithDictionary.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/AFeedVisitorWithDictionary.java @@ -7,18 +7,17 @@ package com.activeviam.mac.statistic.memory.visitor.impl; -import static com.qfs.monitoring.statistic.memory.MemoryStatisticConstants.ATTR_NAME_CREATOR_CLASS; -import static com.qfs.monitoring.statistic.memory.MemoryStatisticConstants.ATTR_NAME_DICTIONARY_ID; +import static com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants.ATTR_NAME_CREATOR_CLASS; +import static com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants.ATTR_NAME_DICTIONARY_ID; +import com.activeviam.database.api.schema.IDatabaseSchema; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; +import com.activeviam.database.datastore.private_.structure.impl.StructureDictionaryManager; import com.activeviam.mac.Loggers; import com.activeviam.mac.memory.DatastoreConstants; -import com.activeviam.store.structure.impl.StructureDictionaryManager; -import com.qfs.monitoring.statistic.IStatisticAttribute; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic; -import com.qfs.store.IDatastoreSchemaMetadata; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.transaction.IOpenedTransaction; +import com.activeviam.tech.observability.api.memory.IStatisticAttribute; +import com.activeviam.tech.observability.internal.memory.DictionaryStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; import java.util.logging.Logger; import lombok.AllArgsConstructor; import lombok.Data; @@ -40,7 +39,7 @@ public abstract class AFeedVisitorWithDictionary extends AFeedVisitor { * @param dumpName Name of the import */ public AFeedVisitorWithDictionary( - IOpenedTransaction transaction, IDatastoreSchemaMetadata storageMetadata, String dumpName) { + IOpenedTransaction transaction, IDatabaseSchema storageMetadata, String dumpName) { super(transaction, storageMetadata, dumpName); } @@ -88,7 +87,7 @@ protected DictionaryAttributes processDictionaryStatistic( if (!this.dictionaryAttributes .getDictionaryClass() .equals(StructureDictionaryManager.class.getName())) { - final IRecordFormat format = getDictionaryFormat(this.storageMetadata); + final var format = getDictionaryFormat(this.storageMetadata); final Object[] tuple = FeedVisitor.buildDictionaryTupleFrom( format, @@ -114,12 +113,16 @@ public static class DictionaryAttributes { /** An instance representing no dictionary (all attributes set to {@code null}. */ public static final DictionaryAttributes NONE = new DictionaryAttributes(); + /** ID of the current dictionary. */ protected Long dictionaryId; + /** Class of the current dictionary. */ protected String dictionaryClass; + /** Size of the current dictionary. */ protected Integer dictionarySize; + /** Order of the current dictionary (i.e. base-2 log of its size). */ protected Integer dictionaryOrder; diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/ChunkSetStatisticVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/ChunkSetStatisticVisitor.java index 69bcd702..8ca783ab 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/ChunkSetStatisticVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/ChunkSetStatisticVisitor.java @@ -7,63 +7,71 @@ package com.activeviam.mac.statistic.memory.visitor.impl; +import com.activeviam.database.api.schema.IDataTable; +import com.activeviam.database.api.schema.IDatabaseSchema; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; import com.activeviam.mac.Workaround; import com.activeviam.mac.entities.ChunkOwner; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.UsedByVersion; -import com.qfs.fwk.services.InternalServiceException; -import com.qfs.monitoring.statistic.IStatisticAttribute; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.impl.ChunkSetStatistic; -import com.qfs.monitoring.statistic.memory.impl.ChunkStatistic; -import com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic; -import com.qfs.monitoring.statistic.memory.impl.IndexStatistic; -import com.qfs.monitoring.statistic.memory.impl.ReferenceStatistic; -import com.qfs.store.IDatastoreSchemaMetadata; -import com.qfs.store.impl.ChunkSet; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.transaction.IOpenedTransaction; +import com.activeviam.tech.chunks.internal.chunkset.impl.ChunkSet; +import com.activeviam.tech.core.api.exceptions.service.InternalServiceException; +import com.activeviam.tech.observability.api.memory.IStatisticAttribute; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkSetStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkStatistic; +import com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.DictionaryStatistic; +import com.activeviam.tech.observability.internal.memory.IndexStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.observability.internal.memory.ReferenceStatistic; import java.time.Instant; import java.util.Collection; import java.util.Collections; /** - * Implementation of the {@link com.qfs.monitoring.statistic.memory.visitor.IMemoryStatisticVisitor} - * class for {@link ChunkSetStatistic}. + * Implementation of the {@link + * com.activeviam.tech.observability.internal.memory.IMemoryStatisticVisitor} class for {@link + * ChunkSetStatistic}. * * @author ActiveViam */ public class ChunkSetStatisticVisitor extends ADatastoreFeedVisitor { /** The record format of the store that stores the chunks. */ - protected final IRecordFormat chunkRecordFormat; + protected final IDataTable chunkRecordFormat; /** The export date, found on the first statistics we read. */ protected final Instant current; /** Type of the root structure. */ protected final ParentType rootComponent; + /** Type of the direct parent structure. */ protected final ParentType directParentType; + /** id of the direct parent structure. */ protected final String directParentId; + /** Aggregate provider being currently visited. */ protected final Long providerId; /** The partition id of the visited statistic. */ protected final int partitionId; + /** Whether or not to ignore the field attributes of the visited statistics. */ protected final boolean ignoreFieldSpecifications; + /** The epoch id we are currently reading statistics for. */ protected Long epochId; + /** * Whether or not the currently visited statistics were flagged as used by the current version. */ protected UsedByVersion usedByVersion; + /** ID of the current {@link ChunkSet}. */ protected Long chunkSetId = null; @@ -92,7 +100,7 @@ public class ChunkSetStatisticVisitor extends ADatastoreFeedVisitor { * the encountered fields */ public ChunkSetStatisticVisitor( - final IDatastoreSchemaMetadata storageMetadata, + final IDatabaseSchema storageMetadata, final IOpenedTransaction transaction, final String dumpName, final Instant current, @@ -121,11 +129,7 @@ public ChunkSetStatisticVisitor( this.usedByVersion = usedByVersion; this.ignoreFieldSpecifications = ignoreFieldSpecifications; - this.chunkRecordFormat = - this.storageMetadata - .getStoreMetadata(DatastoreConstants.CHUNK_STORE) - .getStoreFormat() - .getRecordFormat(); + this.chunkRecordFormat = this.storageMetadata.getTable(DatastoreConstants.CHUNK_STORE); } @Override @@ -136,7 +140,7 @@ public Void visit(final DefaultMemoryStatistic memoryStatistic) { } else if (VectorStatisticVisitor.isVector(memoryStatistic)) { @Workaround(jira = "PIVOT-4127", solution = "Support for 5.8.4- versions") - final IMemoryStatistic vectorStat = memoryStatistic; + final AMemoryStatistic vectorStat = memoryStatistic; visitVectorBlock(vectorStat); } else if (memoryStatistic .getName() @@ -181,6 +185,12 @@ public Void visit(final DefaultMemoryStatistic memoryStatistic) { return null; } + @Override + public Void visit(AMemoryStatistic memoryStatistic) { + visitChildren(memoryStatistic); + return null; + } + @Override public Void visit(final ChunkSetStatistic statistic) { this.chunkSize = statistic.getAttribute(MemoryStatisticConstants.ATTR_NAME_LENGTH).asInt(); @@ -227,7 +237,7 @@ public Void visit(final ChunkStatistic chunkStatistic) { this.fields = Collections.singleton(fieldAttribute.asText()); } - final IRecordFormat format = this.chunkRecordFormat; + final var format = this.chunkRecordFormat; final Object[] tuple = FeedVisitor.buildChunkTupleFrom(format, chunkStatistic); FeedVisitor.setTupleElement( @@ -313,7 +323,7 @@ public Void visit(DictionaryStatistic dictionaryStatistic) { // endregion - private void visitVectorBlock(final IMemoryStatistic memoryStatistic) { + private void visitVectorBlock(final AMemoryStatistic memoryStatistic) { final VectorStatisticVisitor subVisitor = new VectorStatisticVisitor( this.storageMetadata, diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DatastoreFeederVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DatastoreFeederVisitor.java index b1c5d475..aba87fe5 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DatastoreFeederVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DatastoreFeederVisitor.java @@ -9,24 +9,25 @@ import static com.activeviam.mac.statistic.memory.visitor.impl.DebugVisitor.DEBUG; +import com.activeviam.database.api.schema.IDataTable; +import com.activeviam.database.api.schema.IDatabaseSchema; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; +import com.activeviam.database.datastore.private_.IStore; import com.activeviam.mac.entities.StoreOwner; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.UsedByVersion; -import com.qfs.monitoring.statistic.IStatisticAttribute; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.impl.ChunkSetStatistic; -import com.qfs.monitoring.statistic.memory.impl.ChunkStatistic; -import com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic; -import com.qfs.monitoring.statistic.memory.impl.IndexStatistic; -import com.qfs.monitoring.statistic.memory.impl.ReferenceStatistic; -import com.qfs.store.IDatastoreSchemaMetadata; -import com.qfs.store.IStore; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.transaction.IOpenedTransaction; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.api.memory.IStatisticAttribute; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkSetStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkStatistic; +import com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.DictionaryStatistic; +import com.activeviam.tech.observability.internal.memory.IndexStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.observability.internal.memory.ReferenceStatistic; import java.time.Instant; import java.util.Arrays; import java.util.Collection; @@ -45,17 +46,21 @@ public class DatastoreFeederVisitor extends ADatastoreFeedVisitor { /** The record format of the store that stores the chunks. */ - protected final IRecordFormat chunkRecordFormat; + protected final IDataTable chunkRecordFormat; + /** * A boolean that if true tells us that the currently visited component is responsible for storing * versioning data. */ // FIXME find a cleaner way to do that. (specific stat for instance). protected boolean isVersionColumn = false; + /** The export date, found on the first statistics we read. */ protected Instant current = null; + /** The epoch id we are currently reading statistics for. */ protected Long epochId = null; + /** Branch owning {@link #epochId}. */ protected String branch = null; @@ -66,10 +71,13 @@ public class DatastoreFeederVisitor extends ADatastoreFeedVisitor { /** Type of the root component visited. */ protected ParentType rootComponent; + /** Types of the direct parent component owning the chunk. */ protected ParentType directParentType; + /** Id of the direct parent owning the chunk. */ protected String directParentId; + /** The partition id of the visited statistic. */ protected Integer partitionId = null; @@ -84,20 +92,15 @@ public class DatastoreFeederVisitor extends ADatastoreFeedVisitor { * @param dumpName The name of the off-heap dump. Can be null. */ public DatastoreFeederVisitor( - final IDatastoreSchemaMetadata storageMetadata, + final IDatabaseSchema storageMetadata, final IOpenedTransaction transaction, final String dumpName) { super(transaction, storageMetadata, dumpName); - this.chunkRecordFormat = - this.storageMetadata - .getStoreMetadata(DatastoreConstants.CHUNK_STORE) - .getStoreFormat() - .getRecordFormat(); + this.chunkRecordFormat = this.storageMetadata.getTable(DatastoreConstants.CHUNK_STORE); } - private static Object[] buildIndexTupleFrom( - final IRecordFormat format, final IndexStatistic stat) { - final Object[] tuple = new Object[format.getFieldCount()]; + private static Object[] buildIndexTupleFrom(final IDataTable format, final IndexStatistic stat) { + final Object[] tuple = new Object[format.getFields().size()]; tuple[format.getFieldIndex(DatastoreConstants.INDEX_ID)] = stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_INDEX_ID).asLong(); @@ -120,7 +123,7 @@ private static Object[] buildIndexTupleFrom( * * @param stat Entry point for the traversal of the memory statistics tree */ - public void startFrom(final IMemoryStatistic stat) { + public void startFrom(final AMemoryStatistic stat) { if (this.current == null) { final IStatisticAttribute dateAtt = @@ -220,7 +223,7 @@ public Void visit(final DefaultMemoryStatistic stat) { final String initialBranch = this.branch; if (readEpochAndBranchIfAny(stat)) { - final IRecordFormat versionStoreFormat = getVersionStoreFormat(this.storageMetadata); + final var versionStoreFormat = getVersionStoreFormat(this.storageMetadata); final Object[] tuple = FeedVisitor.buildVersionTupleFrom( versionStoreFormat, stat, this.dumpName, this.epochId, this.branch); @@ -264,6 +267,12 @@ public Void visit(final DefaultMemoryStatistic stat) { return null; } + @Override + public Void visit(AMemoryStatistic memoryStatistic) { + visitChildren(memoryStatistic); + return null; + } + @Override public Void visit(final ChunkSetStatistic stat) { return new ChunkSetStatisticVisitor( @@ -288,7 +297,7 @@ public Void visit(final ChunkSetStatistic stat) { @Override public Void visit(final ReferenceStatistic referenceStatistic) { final Object[] tuple = buildPartOfReferenceStatisticTuple(referenceStatistic); - final IRecordFormat refStoreFormat = getReferenceFormat(this.storageMetadata); + final var refStoreFormat = getReferenceFormat(this.storageMetadata); // fill out the tuple this.referenceId = @@ -325,7 +334,7 @@ public Void visit(final ReferenceStatistic referenceStatistic) { @Override public Void visit(final IndexStatistic stat) { - final IRecordFormat format = getIndexFormat(this.storageMetadata); + final var format = getIndexFormat(this.storageMetadata); final Object[] tuple = buildIndexTupleFrom(format, stat); this.indexId = stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_INDEX_ID).asLong(); @@ -419,7 +428,7 @@ protected boolean isSingleFieldStorage(final String storageSpecification) { * * @param stat statistic to be processed */ - protected void processDictionaryManager(final IMemoryStatistic stat) { + protected void processDictionaryManager(final AMemoryStatistic stat) { this.rootComponent = ParentType.DICTIONARY; visitChildren(stat); this.rootComponent = null; @@ -430,7 +439,7 @@ protected void processDictionaryManager(final IMemoryStatistic stat) { * * @param stat statistic to be processed */ - protected void processStoreStat(final IMemoryStatistic stat) { + protected void processStoreStat(final AMemoryStatistic stat) { final IStatisticAttribute nameAttr = Objects.requireNonNull( stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_STORE_NAME), @@ -448,7 +457,7 @@ protected void processStoreStat(final IMemoryStatistic stat) { * * @param stat statistic to be processed */ - protected void processStorePartition(final IMemoryStatistic stat) { + protected void processStorePartition(final AMemoryStatistic stat) { final IStatisticAttribute partitionAttr = Objects.requireNonNull( stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_PARTITION_ID), @@ -460,7 +469,7 @@ protected void processStorePartition(final IMemoryStatistic stat) { this.partitionId = null; } - private void processRecords(final IMemoryStatistic stat) { + private void processRecords(final AMemoryStatistic stat) { final ParentType previousParentType = this.directParentType; final String previousParentId = this.directParentId; @@ -475,7 +484,7 @@ private void processRecords(final IMemoryStatistic stat) { this.rootComponent = null; } - private void processRecordVersion(final IMemoryStatistic stat) { + private void processRecordVersion(final AMemoryStatistic stat) { this.isVersionColumn = MemoryStatisticConstants.STAT_NAME_VERSIONS_COLUMN.equals(stat.getName()); @@ -483,19 +492,19 @@ private void processRecordVersion(final IMemoryStatistic stat) { this.isVersionColumn = false; } - private void processUniqueIndices(final IMemoryStatistic stat) { + private void processUniqueIndices(final AMemoryStatistic stat) { processIndexList(stat, IndexType.UNIQUE); } - private void processKeyIndices(final IMemoryStatistic stat) { + private void processKeyIndices(final AMemoryStatistic stat) { processIndexList(stat, IndexType.KEY); } - private void processSecondaryIndices(final IMemoryStatistic stat) { + private void processSecondaryIndices(final AMemoryStatistic stat) { processIndexList(stat, IndexType.SECONDARY); } - private void processIndexList(final IMemoryStatistic stat, final IndexType type) { + private void processIndexList(final AMemoryStatistic stat, final IndexType type) { final var oldComponent = this.rootComponent; final var oldParent = this.directParentType; @@ -555,9 +564,9 @@ private void readFieldsIfAny(final IMemoryStatistic stat) { */ protected Object[] buildPartOfReferenceStatisticTuple( final ReferenceStatistic referenceStatistic) { - final IRecordFormat refStoreFormat = getReferenceFormat(this.storageMetadata); + final var refStoreFormat = getReferenceFormat(this.storageMetadata); - final Object[] tuple = new Object[refStoreFormat.getFieldCount()]; + final Object[] tuple = new Object[refStoreFormat.getFields().size()]; tuple[refStoreFormat.getFieldIndex(DatastoreConstants.REFERENCE_FROM_STORE)] = referenceStatistic.getAttribute(DatastoreConstants.REFERENCE_FROM_STORE).asText(); tuple[refStoreFormat.getFieldIndex(DatastoreConstants.REFERENCE_FROM_STORE_PARTITION_ID)] = diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DebugVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DebugVisitor.java index 9fdc8856..786439b2 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DebugVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DebugVisitor.java @@ -7,17 +7,18 @@ package com.activeviam.mac.statistic.memory.visitor.impl; -import com.qfs.monitoring.statistic.IStatisticAttribute; -import com.qfs.monitoring.statistic.impl.IntegerStatisticAttribute; -import com.qfs.monitoring.statistic.impl.LongStatisticAttribute; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.ChunkSetStatistic; -import com.qfs.monitoring.statistic.memory.impl.ChunkStatistic; -import com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic; -import com.qfs.monitoring.statistic.memory.impl.IndexStatistic; -import com.qfs.monitoring.statistic.memory.impl.ReferenceStatistic; -import com.qfs.monitoring.statistic.memory.visitor.IMemoryStatisticVisitor; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.api.memory.IStatisticAttribute; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkSetStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkStatistic; +import com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.DictionaryStatistic; +import com.activeviam.tech.observability.internal.memory.IMemoryStatisticVisitor; +import com.activeviam.tech.observability.internal.memory.IndexStatistic; +import com.activeviam.tech.observability.internal.memory.ReferenceStatistic; +import com.activeviam.tech.observability.internal.memory.attributes.IntegerStatisticAttribute; +import com.activeviam.tech.observability.internal.memory.attributes.LongStatisticAttribute; import java.util.concurrent.atomic.AtomicLong; /** @@ -45,30 +46,24 @@ public class DebugVisitor implements IMemoryStatisticVisitor { * @param root Memory statistic being the root of the printed tree * @return the generated TreePrinter */ - public static StatisticTreePrinter createDebugPrinter(IMemoryStatistic root) { + public static StatisticTreePrinter createDebugPrinter(AMemoryStatistic root) { root.accept(new DebugVisitor()); return new StatisticTreePrinter(); } - @Override - public Void visit(final IMemoryStatistic memoryStatistic) { - enrichStatisticWithDebugAttributes(memoryStatistic); - return null; - } - /** * Enriches a statistic and its children with debug attributes. * - * @param parent root parent of the {@link IMemoryStatistic} to be enriched + * @param parent root parent of the {@link AMemoryStatistic} to be enriched */ - protected void enrichStatisticWithDebugAttributes(IMemoryStatistic parent) { + protected void enrichStatisticWithDebugAttributes(AMemoryStatistic parent) { addDebugAttributes(parent); if (parent.getChildren() == null) { return; } this.depth++; - for (IMemoryStatistic child : parent.getChildren()) { + for (AMemoryStatistic child : parent.getChildren()) { addDebugAttributes(child); child.accept(this); } @@ -102,6 +97,12 @@ public Void visit(DefaultMemoryStatistic memoryStatistic) { return null; } + @Override + public Void visit(AMemoryStatistic memoryStatistic) { + enrichStatisticWithDebugAttributes(memoryStatistic); + return null; + } + @Override public Void visit(ChunkSetStatistic chunkSetStatistic) { enrichStatisticWithDebugAttributes(chunkSetStatistic); diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/FeedVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/FeedVisitor.java index 711a1a75..7d5c8279 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/FeedVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/FeedVisitor.java @@ -7,23 +7,25 @@ package com.activeviam.mac.statistic.memory.visitor.impl; -import static com.qfs.monitoring.statistic.memory.MemoryStatisticConstants.ATTR_NAME_CREATOR_CLASS; +import static com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants.ATTR_NAME_CREATOR_CLASS; +import com.activeviam.database.api.schema.IDataTable; +import com.activeviam.database.api.schema.IDatabaseSchema; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; import com.activeviam.mac.Loggers; import com.activeviam.mac.memory.DatastoreConstants; -import com.qfs.monitoring.statistic.IStatisticAttribute; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.impl.ChunkSetStatistic; -import com.qfs.monitoring.statistic.memory.impl.ChunkStatistic; -import com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic; -import com.qfs.monitoring.statistic.memory.impl.IndexStatistic; -import com.qfs.monitoring.statistic.memory.impl.ReferenceStatistic; -import com.qfs.monitoring.statistic.memory.visitor.IMemoryStatisticVisitor; -import com.qfs.store.IDatastoreSchemaMetadata; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.transaction.IOpenedTransaction; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.api.memory.IStatisticAttribute; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkSetStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkStatistic; +import com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.DictionaryStatistic; +import com.activeviam.tech.observability.internal.memory.IMemoryStatisticVisitor; +import com.activeviam.tech.observability.internal.memory.IndexStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.observability.internal.memory.ReferenceStatistic; +import com.activeviam.tech.records.api.IRecordFormat; import java.time.Instant; import java.util.Arrays; import java.util.Collection; @@ -39,7 +41,7 @@ public class FeedVisitor implements IMemoryStatisticVisitor { private static final Logger LOGGER = Logger.getLogger(Loggers.LOADING); - private final IDatastoreSchemaMetadata storageMetadata; + private final IDatabaseSchema storageMetadata; private final IOpenedTransaction transaction; private final String dumpName; private static final AtomicLong chunkIdGenerator = new AtomicLong(0L); @@ -52,9 +54,7 @@ public class FeedVisitor implements IMemoryStatisticVisitor { * @param dumpName name of the import being currently executed */ public FeedVisitor( - final IDatastoreSchemaMetadata storageMetadata, - final IOpenedTransaction tm, - final String dumpName) { + final IDatabaseSchema storageMetadata, final IOpenedTransaction tm, final String dumpName) { this.storageMetadata = storageMetadata; this.transaction = tm; this.dumpName = dumpName; @@ -73,8 +73,8 @@ protected static void add( transaction.add(store, tuple); } - static Object[] buildChunkTupleFrom(final IRecordFormat format, final ChunkStatistic stat) { - final Object[] tuple = new Object[format.getFieldCount()]; + static Object[] buildChunkTupleFrom(final IDataTable format, final ChunkStatistic stat) { + final Object[] tuple = new Object[format.getFields().size()]; tuple[format.getFieldIndex(DatastoreConstants.CHUNK_ID)] = getChunkId(stat); tuple[format.getFieldIndex(DatastoreConstants.CHUNK__CLASS)] = stat.getAttribute(ATTR_NAME_CREATOR_CLASS).asText(); @@ -124,7 +124,7 @@ static void writeChunkTupleForFields( final ChunkStatistic statistic, final IOpenedTransaction transaction, final Collection fields, - final IRecordFormat format, + final IDataTable format, final Object... tuple) { if (fields == null || fields.isEmpty()) { FeedVisitor.add(statistic, transaction, DatastoreConstants.CHUNK_STORE, tuple); @@ -138,12 +138,12 @@ static void writeChunkTupleForFields( } static Object[] buildDictionaryTupleFrom( - final IRecordFormat format, + final IDataTable format, final long dictionaryId, final String dictionaryClass, final int dictionarySize, final int dictionaryOrder) { - final Object[] tuple = new Object[format.getFieldCount()]; + final Object[] tuple = new Object[format.getFields().size()]; tuple[format.getFieldIndex(DatastoreConstants.DICTIONARY_ID)] = dictionaryId; tuple[format.getFieldIndex(DatastoreConstants.DICTIONARY_CLASS)] = dictionaryClass; tuple[format.getFieldIndex(DatastoreConstants.DICTIONARY_SIZE)] = dictionarySize; @@ -152,12 +152,12 @@ static Object[] buildDictionaryTupleFrom( } static Object[] buildVersionTupleFrom( - final IRecordFormat format, + final IDataTable format, final IMemoryStatistic statistic, final String dumpName, final long epochId, final String branch) { - final Object[] tuple = new Object[format.getFieldCount()]; + final Object[] tuple = new Object[format.getFields().size()]; FeedVisitor.setTupleElement(tuple, format, DatastoreConstants.VERSION__DUMP_NAME, dumpName); FeedVisitor.setTupleElement(tuple, format, DatastoreConstants.VERSION__EPOCH_ID, epochId); FeedVisitor.setTupleElement(tuple, format, DatastoreConstants.VERSION__BRANCH_NAME, branch); @@ -165,15 +165,15 @@ static Object[] buildVersionTupleFrom( } /** - * Visits all the children of the given {@link IMemoryStatistic}. + * Visits all the children of the given {@link AMemoryStatistic}. * * @param visitor the visitor to use to visit the children * @param statistic The statistics whose children to visit */ protected static void visitChildren( - final IMemoryStatisticVisitor visitor, final IMemoryStatistic statistic) { + final IMemoryStatisticVisitor visitor, final AMemoryStatistic statistic) { if (statistic.getChildren() != null) { - for (final IMemoryStatistic child : statistic.getChildren()) { + for (final AMemoryStatistic child : statistic.getChildren()) { child.accept(visitor); } } @@ -208,9 +208,8 @@ static void includeApplicationInfoIfAny( } } - static IRecordFormat getRecordFormat( - final IDatastoreSchemaMetadata storageMetadata, final String storeName) { - return storageMetadata.getStoreMetadata(storeName).getStoreFormat().getRecordFormat(); + static IDataTable getRecordFormat(final IDatabaseSchema storageMetadata, final String storeName) { + return storageMetadata.getTable(storeName); } /** @@ -222,7 +221,7 @@ static IRecordFormat getRecordFormat( * @param value data to insert */ protected static void setTupleElement( - Object[] tuple, IRecordFormat format, String field, final Object value) { + Object[] tuple, IDataTable format, String field, final Object value) { if (value == null) { throw new RuntimeException("Expected a non-null value for field " + field); } @@ -245,17 +244,6 @@ protected static void checkTuple(Object[] tuple, IRecordFormat format) { } } - @Override - public Void visit(final IMemoryStatistic memoryStatistic) { - System.err.println( - "Unexpected type of statistics : " - + memoryStatistic - + " which is a " - + memoryStatistic.getClass().getSimpleName()); - visitChildren(this, memoryStatistic); - return null; - } - @Override public Void visit(DefaultMemoryStatistic stat) { switch (stat.getName()) { @@ -280,6 +268,17 @@ public Void visit(DefaultMemoryStatistic stat) { return null; } + @Override + public Void visit(AMemoryStatistic memoryStatistic) { + System.err.println( + "Unexpected type of statistics : " + + memoryStatistic + + " which is a " + + memoryStatistic.getClass().getSimpleName()); + visitChildren(this, memoryStatistic); + return null; + } + @Override public Void visit(ChunkSetStatistic chunkSetStatistic) { return null; diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/LevelStatisticVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/LevelStatisticVisitor.java index 6945c93a..7f8e37e4 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/LevelStatisticVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/LevelStatisticVisitor.java @@ -7,22 +7,22 @@ package com.activeviam.mac.statistic.memory.visitor.impl; +import com.activeviam.database.api.schema.IDatabaseSchema; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; import com.activeviam.mac.Loggers; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.impl.ChunkSetStatistic; -import com.qfs.monitoring.statistic.memory.impl.ChunkStatistic; -import com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic; -import com.qfs.monitoring.statistic.memory.impl.IndexStatistic; -import com.qfs.monitoring.statistic.memory.impl.ReferenceStatistic; -import com.qfs.monitoring.statistic.memory.visitor.IMemoryStatisticVisitor; -import com.qfs.store.IDatastoreSchemaMetadata; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.transaction.IOpenedTransaction; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkSetStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkStatistic; +import com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.DictionaryStatistic; +import com.activeviam.tech.observability.internal.memory.IMemoryStatisticVisitor; +import com.activeviam.tech.observability.internal.memory.IndexStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.observability.internal.memory.ReferenceStatistic; import java.util.logging.Logger; /** @@ -38,6 +38,7 @@ public class LevelStatisticVisitor extends AFeedVisitorWithDictionary { private final PivotFeederVisitor parent; private final IOpenedTransaction transaction; private final Long epochId; + /** The number of members of the visited level. */ protected Integer memberCount; @@ -56,7 +57,7 @@ public class LevelStatisticVisitor extends AFeedVisitorWithDictionary { public LevelStatisticVisitor( final PivotFeederVisitor parent, final IOpenedTransaction transaction, - final IDatastoreSchemaMetadata storageMetadata, + final IDatabaseSchema storageMetadata, final String dumpName, final Long epochId) { super(transaction, storageMetadata, dumpName); @@ -77,11 +78,11 @@ public LevelStatisticVisitor( } /** - * Initialize the visit of the children {@link IMemoryStatistic}. + * Initialize the visit of the children {@link AMemoryStatistic}. * * @param root parent of the children to ve visited */ - public void analyze(final IMemoryStatistic root) { + public void analyze(final AMemoryStatistic root) { visitChildren(root); } @@ -96,7 +97,7 @@ public Void visit(final ChunkStatistic stat) { recordLevelForStructure(this.directParentType, this.directParentId); - final IRecordFormat format = getChunkFormat(this.storageMetadata); + final var format = getChunkFormat(this.storageMetadata); final Object[] tuple = FeedVisitor.buildChunkTupleFrom(format, stat); FeedVisitor.setTupleElement(tuple, format, DatastoreConstants.CHUNK__DUMP_NAME, this.dumpName); FeedVisitor.setTupleElement(tuple, format, DatastoreConstants.VERSION__EPOCH_ID, this.epochId); @@ -174,10 +175,16 @@ public Void visit(final DefaultMemoryStatistic stat) { return null; } + @Override + public Void visit(AMemoryStatistic memoryStatistic) { + visitChildren(memoryStatistic); + return null; + } + private void recordLevelForStructure(final ParentType type, final String id) { - final IRecordFormat format = + final var format = FeedVisitor.getRecordFormat(this.storageMetadata, DatastoreConstants.CHUNK_TO_LEVEL_STORE); - final Object[] tuple = new Object[format.getFieldCount()]; + final Object[] tuple = new Object[format.getFields().size()]; FeedVisitor.setTupleElement(tuple, format, DatastoreConstants.CHUNK__DUMP_NAME, this.dumpName); FeedVisitor.setTupleElement( tuple, format, DatastoreConstants.CHUNK_TO_LEVEL__MANAGER_ID, this.parent.manager); diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/PivotFeederVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/PivotFeederVisitor.java index 0a3f6b0e..229b895b 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/PivotFeederVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/PivotFeederVisitor.java @@ -9,29 +9,30 @@ import static com.activeviam.mac.statistic.memory.visitor.impl.DebugVisitor.DEBUG; -import com.activeviam.copper.HierarchyIdentifier; -import com.activeviam.copper.LevelIdentifier; -import com.activeviam.fwk.ActiveViamRuntimeException; +import com.activeviam.activepivot.core.impl.internal.impl.ActivePivotManager; +import com.activeviam.activepivot.core.intf.api.cube.metadata.HierarchyIdentifier; +import com.activeviam.activepivot.core.intf.api.cube.metadata.LevelIdentifier; +import com.activeviam.activepivot.dist.impl.api.cube.IMultiVersionDistributedActivePivot; +import com.activeviam.database.api.schema.IDataTable; +import com.activeviam.database.api.schema.IDatabaseSchema; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; import com.activeviam.mac.entities.CubeOwner; import com.activeviam.mac.entities.DistributedCubeOwner; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.UsedByVersion; -import com.qfs.distribution.IMultiVersionDistributedActivePivot; -import com.qfs.monitoring.statistic.IStatisticAttribute; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.impl.ChunkSetStatistic; -import com.qfs.monitoring.statistic.memory.impl.ChunkStatistic; -import com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic; -import com.qfs.monitoring.statistic.memory.impl.IndexStatistic; -import com.qfs.monitoring.statistic.memory.impl.ReferenceStatistic; -import com.qfs.monitoring.statistic.memory.visitor.IMemoryStatisticVisitor; -import com.qfs.store.IDatastoreSchemaMetadata; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.transaction.IOpenedTransaction; -import com.quartetfs.biz.pivot.impl.ActivePivotManager; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.api.memory.IStatisticAttribute; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkSetStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkStatistic; +import com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.DictionaryStatistic; +import com.activeviam.tech.observability.internal.memory.IMemoryStatisticVisitor; +import com.activeviam.tech.observability.internal.memory.IndexStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.observability.internal.memory.ReferenceStatistic; import java.time.Instant; import java.util.Objects; @@ -44,28 +45,40 @@ public class PivotFeederVisitor extends AFeedVisitorWithDictionary { /** The export date, found on the first statistics we read. */ protected Instant current = null; + /** The epoch id we are currently reading statistics for. */ protected Long epochId = null; + /** The branch of the pivot we're currently reading statistics. */ protected String branch = null; + /** Current {@link ActivePivotManager}. */ protected String manager; + /** Aggregate provider being currently visited. */ protected Long providerId; + /** Partition being currently visited. */ protected Integer partition; + /** Dimension being currently visited. */ protected String dimension; + /** Hierarchy being currently visited. */ protected String hierarchy; + /** Level being currently visited. */ protected String level; + /** Type of the aggregate Provider being currently visited. */ protected ProviderComponentType providerComponentType; + /** Type of the root structure. */ protected ParentType rootComponent; + /** Type of the direct parent structure. */ protected ParentType directParentType; + /** Id of the direct parent structure. */ protected String directParentId; @@ -80,9 +93,7 @@ public class PivotFeederVisitor extends AFeedVisitorWithDictionary { * @param dumpName name of the current import */ public PivotFeederVisitor( - final IDatastoreSchemaMetadata storageMetadata, - final IOpenedTransaction tm, - final String dumpName) { + final IDatabaseSchema storageMetadata, final IOpenedTransaction tm, final String dumpName) { super(tm, storageMetadata, dumpName); } @@ -99,8 +110,8 @@ private static boolean isPivotDistributed(final IMemoryStatistic pivotStat) { } private static Object[] buildProviderTupleFrom( - final IRecordFormat format, final IMemoryStatistic stat) { - final Object[] tuple = new Object[format.getFieldCount()]; + final IDataTable format, final IMemoryStatistic stat) { + final Object[] tuple = new Object[format.getFields().size()]; final IStatisticAttribute indexAttr = stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_PROVIDER_NAME); @@ -119,8 +130,8 @@ private static Object[] buildProviderTupleFrom( } private static Object[] buildLevelTupleFrom( - final IRecordFormat format, final IMemoryStatistic stat) { - final Object[] tuple = new Object[format.getFieldCount()]; + final IDataTable format, final IMemoryStatistic stat) { + final Object[] tuple = new Object[format.getFields().size()]; tuple[format.getFieldIndex(DatastoreConstants.LEVEL__ON_HEAP_SIZE)] = stat.getShallowOnHeap(); tuple[format.getFieldIndex(DatastoreConstants.LEVEL__OFF_HEAP_SIZE)] = stat.getShallowOffHeap(); @@ -149,40 +160,38 @@ private static String getProviderCategory(final IMemoryStatistic stat) { * MemoryStatisticConstants#STAT_NAME_PIVOT} or {@link * MemoryStatisticConstants#STAT_NAME_MANAGER} named statistic */ - public void startFrom(final IMemoryStatistic stat) { - - if (this.current == null) { - final IStatisticAttribute dateAtt = - stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_DATE); - - this.current = - Instant.ofEpochSecond(null != dateAtt ? dateAtt.asLong() : System.currentTimeMillis()); - - readEpochAndBranchIfAny(stat); - if (this.epochId == null - && stat.getName().equals(MemoryStatisticConstants.STAT_NAME_MANAGER)) { - // Look amongst the children to find the epoch - for (final IMemoryStatistic child : stat.getChildren()) { - readEpochAndBranchIfAny(child); - if (this.epochId != null) { - break; - } - } - } + public void startFrom(final AMemoryStatistic stat) { + if (this.current != null) { + throw new IllegalArgumentException("Cannot reuse a feed instance"); + } + final IStatisticAttribute dateAtt = stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_DATE); + this.current = + Instant.ofEpochSecond(null != dateAtt ? dateAtt.asLong() : System.currentTimeMillis()); + readEpochAndBranchIfAny(stat); + if (this.epochId == null && stat.getName().equals(MemoryStatisticConstants.STAT_NAME_MANAGER)) { + // Look amongst the children to find the epoch + findEpoch(stat); + } - FeedVisitor.includeApplicationInfoIfAny(this.transaction, this.current, this.dumpName, stat); + FeedVisitor.includeApplicationInfoIfAny(this.transaction, this.current, this.dumpName, stat); - try { - stat.accept(this); - } catch (final RuntimeException e) { - if (Boolean.TRUE.equals(DEBUG)) { - final StatisticTreePrinter printer = DebugVisitor.createDebugPrinter(stat); - printer.print(); - } - throw e; + try { + stat.accept(this); + } catch (final RuntimeException e) { + if (Boolean.TRUE.equals(DEBUG)) { + final StatisticTreePrinter printer = DebugVisitor.createDebugPrinter(stat); + printer.print(); + } + throw e; + } + } + + private void findEpoch(AMemoryStatistic stat) { + for (final IMemoryStatistic child : stat.getChildren()) { + readEpochAndBranchIfAny(child); + if (this.epochId != null) { + break; } - } else { - throw new RuntimeException("Cannot reuse a feed instance"); } } @@ -198,8 +207,8 @@ public Void visit(final DefaultMemoryStatistic stat) { case MemoryStatisticConstants.STAT_NAME_PIVOT: processPivot(stat); break; - // Unless said otherwise we assume all providers are partial,safer than the - // other way + // Unless said otherwise we assume all providers are partial,safer than the + // other way case MemoryStatisticConstants.STAT_NAME_PROVIDER: case MemoryStatisticConstants.STAT_NAME_PARTIAL_PROVIDER: case MemoryStatisticConstants.STAT_NAME_FULL_PROVIDER: @@ -228,6 +237,12 @@ public Void visit(final DefaultMemoryStatistic stat) { return null; } + @Override + public Void visit(AMemoryStatistic memoryStatistic) { + visitChildren(memoryStatistic); + return null; + } + @Override public Void visit(final ChunkSetStatistic stat) { return new ChunkSetStatisticVisitor( @@ -252,7 +267,7 @@ public Void visit(final ChunkSetStatistic stat) { @Override public Void visit(final ChunkStatistic stat) { - final IRecordFormat format = getChunkFormat(this.storageMetadata); + final var format = getChunkFormat(this.storageMetadata); final Object[] tuple = FeedVisitor.buildChunkTupleFrom(format, stat); FeedVisitor.setTupleElement(tuple, format, DatastoreConstants.CHUNK__DUMP_NAME, this.dumpName); FeedVisitor.setTupleElement(tuple, format, DatastoreConstants.VERSION__EPOCH_ID, this.epochId); @@ -344,7 +359,7 @@ public Void visit(IndexStatistic stat) { "An ActivePivot cannot contain indices. Received: " + stat); } - private void processManager(final IMemoryStatistic stat) { + private void processManager(final AMemoryStatistic stat) { final IStatisticAttribute idAttr = Objects.requireNonNull( stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_MANAGER_ID), @@ -356,7 +371,7 @@ private void processManager(final IMemoryStatistic stat) { this.manager = null; } - private void processMultiVersionPivot(final IMemoryStatistic stat) { + private void processMultiVersionPivot(final AMemoryStatistic stat) { final IStatisticAttribute managerAttr = stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_MANAGER_ID); if (managerAttr != null) { @@ -370,11 +385,11 @@ private void processMultiVersionPivot(final IMemoryStatistic stat) { } } - private void processPivot(final IMemoryStatistic stat) { + private void processPivot(final AMemoryStatistic stat) { readEpochAndBranchIfAny(stat); if (readEpochAndBranchIfAny(stat)) { - final IRecordFormat versionStoreFormat = getVersionStoreFormat(this.storageMetadata); + final var versionStoreFormat = getVersionStoreFormat(this.storageMetadata); final Object[] tuple = FeedVisitor.buildVersionTupleFrom( versionStoreFormat, stat, this.dumpName, this.epochId, this.branch); @@ -409,8 +424,8 @@ private void processPivot(final IMemoryStatistic stat) { this.branch = null; } - private void processProvider(final IMemoryStatistic stat) { - final IRecordFormat format = getProviderFormat(this.storageMetadata); + private void processProvider(final AMemoryStatistic stat) { + final var format = getProviderFormat(this.storageMetadata); final Object[] tuple = buildProviderTupleFrom(format, stat); FeedVisitor.setTupleElement( @@ -427,7 +442,7 @@ private void processProvider(final IMemoryStatistic stat) { this.providerId = null; } - private void processPartition(final IMemoryStatistic stat) { + private void processPartition(final AMemoryStatistic stat) { final IStatisticAttribute idAttr = Objects.requireNonNull( stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_PROVIDER_PARTITION_ID), @@ -440,26 +455,26 @@ private void processPartition(final IMemoryStatistic stat) { this.partition = null; } - private void processHierarchy(final IMemoryStatistic stat) { + private void processHierarchy(final AMemoryStatistic stat) { String hierarchyDescription = stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_HIERARCHY_ID).asText(); HierarchyIdentifier hc = HierarchyIdentifier.fromDescription(hierarchyDescription); - this.dimension = hc.dimension; - this.hierarchy = hc.hierarchy; + this.dimension = hc.getDimensionName(); + this.hierarchy = hc.getHierarchyName(); visitChildren(stat); this.hierarchy = null; } - private void processLevel(final IMemoryStatistic stat) { - final IRecordFormat format = getLevelFormat(this.storageMetadata); + private void processLevel(final AMemoryStatistic stat) { + final var format = getLevelFormat(this.storageMetadata); final Object[] tuple = buildLevelTupleFrom(format, stat); String levelDescription = stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_LEVEL_ID).asText(); LevelIdentifier lc = LevelIdentifier.fromDescription(levelDescription); - this.level = lc.level; + this.level = lc.getLevelName(); FeedVisitor.setTupleElement(tuple, format, DatastoreConstants.LEVEL__MANAGER_ID, this.manager); FeedVisitor.setTupleElement( @@ -493,7 +508,7 @@ private void processLevel(final IMemoryStatistic stat) { } private void processProviderComponent( - final IMemoryStatistic stat, final ProviderComponentType type) { + final AMemoryStatistic stat, final ProviderComponentType type) { final ParentType previousParentType = this.directParentType; final String previousParentId = this.directParentId; this.directParentType = getCorrespondingParentType(type); @@ -509,7 +524,7 @@ private void processProviderComponent( this.providerComponentType = null; } - private void processChunkObject(final IMemoryStatistic statistic) { + private void processChunkObject(final AMemoryStatistic statistic) { if (VectorStatisticVisitor.isVector(statistic)) { final VectorStatisticVisitor subVisitor = new VectorStatisticVisitor( @@ -584,9 +599,9 @@ private boolean readEpochAndBranchIfAny(final IMemoryStatistic stat) { final IStatisticAttribute branchAttr = stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_BRANCH); if (branchAttr != null) { - final String branch = branchAttr.asText(); - assert this.branch == null || this.branch.equals(branch); - this.branch = branch; + final String branchAttrText = branchAttr.asText(); + assert this.branch == null || this.branch.equals(branchAttrText); + this.branch = branchAttrText; epochOrBranchChanged = true; } diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/StatisticTreePrinter.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/StatisticTreePrinter.java index a8716a5e..47724f48 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/StatisticTreePrinter.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/StatisticTreePrinter.java @@ -8,7 +8,7 @@ package com.activeviam.mac.statistic.memory.visitor.impl; import com.activeviam.mac.memory.DatastoreConstants; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/VectorStatisticVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/VectorStatisticVisitor.java index 3076c62a..40115b41 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/VectorStatisticVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/VectorStatisticVisitor.java @@ -7,35 +7,36 @@ package com.activeviam.mac.statistic.memory.visitor.impl; +import com.activeviam.database.api.schema.IDataTable; +import com.activeviam.database.api.schema.IDatabaseSchema; +import com.activeviam.database.datastore.api.transaction.IOpenedTransaction; import com.activeviam.mac.entities.ChunkOwner; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.UsedByVersion; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.impl.ChunkSetStatistic; -import com.qfs.monitoring.statistic.memory.impl.ChunkStatistic; -import com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic; -import com.qfs.monitoring.statistic.memory.impl.IndexStatistic; -import com.qfs.monitoring.statistic.memory.impl.ReferenceStatistic; -import com.qfs.store.IDatastoreSchemaMetadata; -import com.qfs.store.record.IRecordFormat; -import com.qfs.store.transaction.IOpenedTransaction; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkSetStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkStatistic; +import com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.DictionaryStatistic; +import com.activeviam.tech.observability.internal.memory.IndexStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.observability.internal.memory.ReferenceStatistic; import java.time.Instant; import java.util.Collection; /** - * Implementation of the {@link com.qfs.monitoring.statistic.memory.visitor.IMemoryStatisticVisitor} - * class for Vectors. + * Implementation of the {@link + * com.activeviam.tech.observability.internal.memory.IMemoryStatisticVisitor} class for Vectors. * * @author ActiveViam */ public class VectorStatisticVisitor extends AFeedVisitor { /** The record format of the store that stores the chunks. */ - protected final IRecordFormat chunkRecordFormat; + protected final IDataTable chunkRecordFormat; /** The export date, found on the first statistics we read. */ protected final Instant current; @@ -44,8 +45,10 @@ public class VectorStatisticVisitor extends AFeedVisitor { protected final int partitionId; private final UsedByVersion usedByVersion; + /** The epoch id we are currently reading statistics for. */ protected Long epochId; + /** The fields corresponding to the vector block statistic. */ protected Collection fields; @@ -63,7 +66,7 @@ public class VectorStatisticVisitor extends AFeedVisitor { * @param usedByVersion the used by version flag for the current statistic */ public VectorStatisticVisitor( - final IDatastoreSchemaMetadata storageMetadata, + final IDatabaseSchema storageMetadata, final IOpenedTransaction transaction, final String dumpName, final Instant current, @@ -80,11 +83,7 @@ public VectorStatisticVisitor( this.epochId = epochId; this.usedByVersion = usedByVersion; - this.chunkRecordFormat = - this.storageMetadata - .getStoreMetadata(DatastoreConstants.CHUNK_STORE) - .getStoreFormat() - .getRecordFormat(); + this.chunkRecordFormat = this.storageMetadata.getTable(DatastoreConstants.CHUNK_STORE); } /** @@ -117,7 +116,7 @@ static boolean isVectorName(final String statisticName) { * * @param statistic root statistic */ - public void process(final IMemoryStatistic statistic) { + public void process(final AMemoryStatistic statistic) { statistic.accept(this); } @@ -128,6 +127,12 @@ public Void visit(DefaultMemoryStatistic memoryStatistic) { return null; } + @Override + public Void visit(AMemoryStatistic memoryStatistic) { + visitChildren(memoryStatistic); + return null; + } + @Override public Void visit(final ChunkStatistic chunkStatistic) { if (chunkStatistic.getName().equals(MemoryStatisticConstants.STAT_NAME_VECTOR_BLOCK)) { @@ -175,7 +180,7 @@ public Void visit(DictionaryStatistic dictionaryStatistic) { protected void visitVectorBlock(final ChunkStatistic statistic) { assert statistic.getChildren().isEmpty() : "Vector statistics with children"; - final IRecordFormat format = this.chunkRecordFormat; + final var format = this.chunkRecordFormat; final Object[] tuple = FeedVisitor.buildChunkTupleFrom(format, statistic); FeedVisitor.setTupleElement( diff --git a/src/main/java/com/activeviam/mac/tools/Tools.java b/src/main/java/com/activeviam/mac/tools/Tools.java index c3be8134..a0225a10 100644 --- a/src/main/java/com/activeviam/mac/tools/Tools.java +++ b/src/main/java/com/activeviam/mac/tools/Tools.java @@ -6,8 +6,8 @@ */ package com.activeviam.mac.tools; -import com.activeviam.fwk.ActiveViamRuntimeException; -import com.qfs.pivot.monitoring.impl.MemoryStatisticSerializerUtil; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryStatisticSerializerUtil; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/src/main/java/com/activeviam/tools/bookmark/impl/BookmarkTool.java b/src/main/java/com/activeviam/tools/bookmark/impl/BookmarkTool.java index 1f8d467e..33206b06 100644 --- a/src/main/java/com/activeviam/tools/bookmark/impl/BookmarkTool.java +++ b/src/main/java/com/activeviam/tools/bookmark/impl/BookmarkTool.java @@ -7,9 +7,9 @@ package com.activeviam.tools.bookmark.impl; +import com.activeviam.tech.contentserver.storage.api.ContentServiceSnapshotter; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.ObjectMapper; -import com.qfs.content.snapshot.impl.ContentServiceSnapshotter; import java.util.List; import java.util.Map; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; diff --git a/src/main/java/com/activeviam/tools/bookmark/impl/ContentServerToJsonUi.java b/src/main/java/com/activeviam/tools/bookmark/impl/ContentServerToJsonUi.java index 84957a10..05c80bbd 100644 --- a/src/main/java/com/activeviam/tools/bookmark/impl/ContentServerToJsonUi.java +++ b/src/main/java/com/activeviam/tools/bookmark/impl/ContentServerToJsonUi.java @@ -7,15 +7,15 @@ package com.activeviam.tools.bookmark.impl; +import com.activeviam.tech.contentserver.storage.api.BasicJsonContentEntry; +import com.activeviam.tech.contentserver.storage.api.ContentServiceSnapshotter; +import com.activeviam.tech.contentserver.storage.api.IContentTree; +import com.activeviam.tech.contentserver.storage.api.SnapshotContentTree; import com.activeviam.tools.bookmark.constant.impl.ContentServerConstants; import com.activeviam.tools.bookmark.constant.impl.ContentServerConstants.Paths; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import com.qfs.content.service.IContentTree; -import com.qfs.content.snapshot.impl.BasicJsonContentEntry; -import com.qfs.content.snapshot.impl.ContentServiceSnapshotter; -import com.qfs.content.snapshot.impl.SnapshotContentTree; import java.io.File; import java.io.IOException; import java.nio.file.Path; diff --git a/src/main/java/com/activeviam/tools/bookmark/impl/JsonUiToContentServer.java b/src/main/java/com/activeviam/tools/bookmark/impl/JsonUiToContentServer.java index 6d102e0e..96b3e57a 100644 --- a/src/main/java/com/activeviam/tools/bookmark/impl/JsonUiToContentServer.java +++ b/src/main/java/com/activeviam/tools/bookmark/impl/JsonUiToContentServer.java @@ -7,12 +7,12 @@ package com.activeviam.tools.bookmark.impl; +import com.activeviam.tech.contentserver.storage.api.ContentServiceSnapshotter; +import com.activeviam.tech.contentserver.storage.api.SnapshotContentTree; import com.activeviam.tools.bookmark.constant.impl.ContentServerConstants; import com.activeviam.tools.bookmark.constant.impl.ContentServerConstants.Paths; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.qfs.content.snapshot.impl.ContentServiceSnapshotter; -import com.qfs.content.snapshot.impl.SnapshotContentTree; import java.io.IOException; import java.io.InputStream; import java.util.Collections; diff --git a/src/main/resources/META-INF/spring.factories b/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000..4dd14a61 --- /dev/null +++ b/src/main/resources/META-INF/spring.factories @@ -0,0 +1 @@ +org.springframework.context.ApplicationListener=com.activeviam.mac.cfg.impl.RegistryInitializationConfig diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 469f97e9..cf9c9d02 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,10 +1,5 @@ server: port: 9092 - remote: - uri: localhost:9090 - -activeui: - address: http://localhost:9092/ui management: endpoints: @@ -12,13 +7,6 @@ management: exposure: include: "*" -contentServer: - security: - calculatedMemberRole: ROLE_USER - kpiRole: ROLE_USER - cache: - entitlementsTTL: 3600 - bookmarks: folder: ui @@ -49,7 +37,7 @@ activeviam: lULiNZ73+TSdoT8yD6mH2hVArkajsnv1PNw0cR2V2aLY1Cg030jyTtmdOky/ZBhMiT/bH7H1M/BNjsC23rcOtdwFRlApUlOkGMecZN\ 8IzqaYV0zuL+cQt2Iwo=" # The time of validity of a token in seconds - expiration: 86400 + expiration: PT24H # The path of the folder containing statistic files in .json or .json.sz (Snappy-compressed) formats # This path is relative to the execution directory of the Application @@ -57,6 +45,7 @@ statistic: folder: input_statistics content-service: + mode: "in-memory" # "in-memory" or "db" db: hibernate: dialect: org.hibernate.dialect.H2Dialect diff --git a/src/main/resources/content-service-hibernate.xml b/src/main/resources/content-service-hibernate.xml new file mode 100644 index 00000000..a82dfc05 --- /dev/null +++ b/src/main/resources/content-service-hibernate.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/main/resources/static/activeui/env.js b/src/main/resources/static/activeui/env.js deleted file mode 100644 index 2ef832bf..00000000 --- a/src/main/resources/static/activeui/env.js +++ /dev/null @@ -1,14 +0,0 @@ -var baseUrl = window.location.href.split('/ui')[0]; - -window.env = { - contentServerUrl: baseUrl, - contentServerVersion: "6.0.6", - // WARNING: Changing the keys of activePivotServers will break previously saved widgets and dashboards. - // If you must do it, then you also need to update each one's serverKey attribute on your content server. - activePivotServers: { - "MAC": { - url: baseUrl, - version: "6.0.6", - }, - }, -}; diff --git a/src/main/resources/static/activeui/extensions.json b/src/main/resources/static/activeui/extensions.json deleted file mode 100644 index ead25337..00000000 --- a/src/main/resources/static/activeui/extensions.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@activeviam/starter": "extensions/@activeviam/starter/@activeviam/starter.js", - "text-editor-extension": "extensions/text-editor-extension/text-editor-extension.js", - "@activeviam/advanced-extension": "extensions/@activeviam/advanced-extension/@activeviam/advanced-extension.js" -} diff --git a/src/main/resources/static/atoti-ui/env.js b/src/main/resources/static/atoti-ui/env.js new file mode 100644 index 00000000..6a783ffd --- /dev/null +++ b/src/main/resources/static/atoti-ui/env.js @@ -0,0 +1,19 @@ +const baseUrl = window.location.href.split('/ui')[0]; +const version = "6.1.3"; + +window.env = { + contentServer: { + url: baseUrl, + version, + }, + atotiServers: { + "MAC": { + url: baseUrl, + version, + }, + }, + jwtServer: { + url: baseUrl, + version, + } +}; diff --git a/src/main/resources/static/atoti-ui/extensions.json b/src/main/resources/static/atoti-ui/extensions.json new file mode 100644 index 00000000..d8471156 --- /dev/null +++ b/src/main/resources/static/atoti-ui/extensions.json @@ -0,0 +1,4 @@ +{ + "@activeviam/starter":"extensions/@activeviam/starter/extensionEntry.js", + "@activeviam/advanced-extension": "extensions/@activeviam/advanced-extension/extensionEntry.js" +} diff --git a/src/main/resources/static/activeui/extensions/text-editor-extension/static/js/492.84c9d447.js b/src/main/resources/static/atoti-ui/extensions/text-editor-extension/static/js/492.84c9d447.js similarity index 100% rename from src/main/resources/static/activeui/extensions/text-editor-extension/static/js/492.84c9d447.js rename to src/main/resources/static/atoti-ui/extensions/text-editor-extension/static/js/492.84c9d447.js diff --git a/src/main/resources/static/activeui/extensions/text-editor-extension/static/js/7.b5b094db.chunk.js b/src/main/resources/static/atoti-ui/extensions/text-editor-extension/static/js/7.b5b094db.chunk.js similarity index 100% rename from src/main/resources/static/activeui/extensions/text-editor-extension/static/js/7.b5b094db.chunk.js rename to src/main/resources/static/atoti-ui/extensions/text-editor-extension/static/js/7.b5b094db.chunk.js diff --git a/src/main/resources/static/activeui/extensions/text-editor-extension/static/js/main.6b2474f9.js b/src/main/resources/static/atoti-ui/extensions/text-editor-extension/static/js/main.6b2474f9.js similarity index 100% rename from src/main/resources/static/activeui/extensions/text-editor-extension/static/js/main.6b2474f9.js rename to src/main/resources/static/atoti-ui/extensions/text-editor-extension/static/js/main.6b2474f9.js diff --git a/src/main/resources/static/activeui/extensions/text-editor-extension/text-editor-extension.js b/src/main/resources/static/atoti-ui/extensions/text-editor-extension/text-editor-extension.js similarity index 100% rename from src/main/resources/static/activeui/extensions/text-editor-extension/text-editor-extension.js rename to src/main/resources/static/atoti-ui/extensions/text-editor-extension/text-editor-extension.js diff --git a/src/test/java/com/activeviam/mac/TestMemoryStatisticBuilder.java b/src/test/java/com/activeviam/mac/TestMemoryStatisticBuilder.java index b4aa9d95..56d72b56 100644 --- a/src/test/java/com/activeviam/mac/TestMemoryStatisticBuilder.java +++ b/src/test/java/com/activeviam/mac/TestMemoryStatisticBuilder.java @@ -6,26 +6,18 @@ */ package com.activeviam.mac; -import com.activeviam.fwk.ActiveViamRuntimeException; -import com.qfs.monitoring.statistic.IStatisticAttribute; -import com.qfs.monitoring.statistic.impl.GenericMonitoringStatisticBuilder; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.IMemoryStatisticBuilder; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.impl.ChunkSetStatistic; -import com.qfs.monitoring.statistic.memory.impl.ChunkStatistic; -import com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.DictionaryStatistic; -import com.qfs.monitoring.statistic.memory.impl.IndexStatistic; -import com.qfs.monitoring.statistic.memory.impl.ReferenceStatistic; -import java.util.Arrays; -import java.util.stream.Collectors; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.AMemoryStatisticBuilder; +import com.activeviam.tech.observability.internal.memory.ChunkSetStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkStatistic; +import com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.DictionaryStatistic; +import com.activeviam.tech.observability.internal.memory.IndexStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.observability.internal.memory.ReferenceStatistic; /** @author ActiveViam */ -public class TestMemoryStatisticBuilder - extends GenericMonitoringStatisticBuilder< - IMemoryStatistic, IStatisticAttribute, IMemoryStatisticBuilder> - implements IMemoryStatisticBuilder { +public class TestMemoryStatisticBuilder extends AMemoryStatisticBuilder { /** Name of the statistic */ protected String name; @@ -37,26 +29,15 @@ public class TestMemoryStatisticBuilder protected long onHeap; /** Default constructor. */ - public TestMemoryStatisticBuilder() { + public TestMemoryStatisticBuilder(final String name) { + this.name = name; this.offHeap = 0; this.onHeap = 0; } @Override - public IMemoryStatistic build() { - IStatisticAttribute parentClass = - this.attributes.get(MemoryStatisticConstants.ATTR_NAME_CREATOR_CLASS); - if (parentClass == null) { - throw new ActiveViamRuntimeException( - "It is mandatory to add a parent class to statistic with name '" + this.name + "'"); - } - - if (this.name == null) { - // Default name - this.name = parentClass.asText(); - } - - IMemoryStatistic statistic; + protected AMemoryStatistic createStatistic() { + AMemoryStatistic statistic; switch (this.name) { case MemoryStatisticConstants.STAT_NAME_CHUNK: statistic = new ChunkStatistic(); @@ -76,51 +57,7 @@ public IMemoryStatistic build() { default: statistic = new DefaultMemoryStatistic(); } - statistic.setShallowOffHeap(this.offHeap); - statistic.setShallowOnHeap(this.onHeap); - statistic.setName(this.name); - statistic.setAttributes(this.attributes); - IMemoryStatistic delegateParent = null; - if (this.children != null && !this.children.isEmpty()) { - // The children should have a distinct name, see setChildren method. - statistic.setChildren(this.children); - for (final IMemoryStatistic child : this.children) { - if (child.getParent() != null && child.getParent() != statistic) { - if (delegateParent == null) { - delegateParent = child.getParent(); - } else { - if (delegateParent != child.getParent()) { - throw new IllegalStateException("Inconsistency in statistics parenthood...Bad."); - } - } - - } else { - child.setParent(statistic); - } - } - } - if (delegateParent == null) return statistic; - else return delegateParent; - } - - @Override - public IMemoryStatisticBuilder withName(String name) { - this.name = name; - return this; - } - - @Override - public IMemoryStatisticBuilder withCreatorClasses(Class... parentClass) { - withAttribute( - MemoryStatisticConstants.ATTR_NAME_CREATOR_CLASS, - Arrays.stream(parentClass).map(Class::getName).collect(Collectors.joining(","))); - return this; - } - - @Override - public IMemoryStatisticBuilder withMemoryFootPrint(long offHeap, long onHeap) { - this.onHeap = onHeap; - this.offHeap = offHeap; - return this; + statistic.setParent(null); + return statistic; } } diff --git a/src/test/java/com/activeviam/mac/app/TestAppLoading.java b/src/test/java/com/activeviam/mac/app/TestAppLoading.java index c137b0a9..14f90173 100644 --- a/src/test/java/com/activeviam/mac/app/TestAppLoading.java +++ b/src/test/java/com/activeviam/mac/app/TestAppLoading.java @@ -7,19 +7,17 @@ package com.activeviam.mac.app; -import com.activeviam.fwk.ActiveViamRuntimeException; import com.activeviam.mac.cfg.impl.MacServerConfig; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.BeanCreationException; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; +import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.web.WebAppConfiguration; /** Test class for tests related to loading cases of the SpringBoot application */ -@WebAppConfiguration -@Import({MacServerConfig.class}) -@PropertySource(value = "classpath:application.yml") +@WebAppConfiguration("classpath:application.yml") +@ContextConfiguration(classes = MacServerConfig.class) public class TestAppLoading { /** Ensures that the default application setup works */ diff --git a/src/test/java/com/activeviam/mac/statistic/memory/ATestMemoryStatistic.java b/src/test/java/com/activeviam/mac/statistic/memory/ATestMemoryStatistic.java index b7cda2cd..68564659 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/ATestMemoryStatistic.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/ATestMemoryStatistic.java @@ -17,50 +17,52 @@ import static com.activeviam.mac.memory.DatastoreConstants.VERSION__EPOCH_ID; import static com.qfs.chunk.impl.MemoryTestUtils.resetAllThreadsVectorAllocator; -import com.activeviam.builders.StartBuilding; -import com.activeviam.copper.HierarchyIdentifier; -import com.activeviam.copper.api.Copper; +import com.activeviam.activepivot.copper.api.Copper; +import com.activeviam.activepivot.core.datastore.api.builder.StartBuilding; +import com.activeviam.activepivot.core.impl.api.description.impl.ActivePivotManagerDescription; +import com.activeviam.activepivot.core.impl.internal.monitoring.memory.impl.OnHeapPivotMemoryQuantifierPlugin; +import com.activeviam.activepivot.core.impl.internal.test.util.PivotTestUtils; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IActivePivotManager; +import com.activeviam.activepivot.core.intf.api.cube.metadata.HierarchyIdentifier; +import com.activeviam.activepivot.core.intf.api.description.IActivePivotManagerDescription; +import com.activeviam.activepivot.core.intf.api.description.IMessengerDefinition; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryStatisticSerializerUtil; +import com.activeviam.activepivot.server.intf.api.observability.IMemoryAnalysisService; +import com.activeviam.database.api.ICondition; +import com.activeviam.database.api.conditions.BaseConditions; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; import com.activeviam.database.api.schema.FieldPath; +import com.activeviam.database.api.types.ILiteralType; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.api.description.IDatastoreSchemaDescription; +import com.activeviam.database.datastore.api.transaction.DatastoreTransactionException; +import com.activeviam.database.datastore.api.transaction.ITransactionManager; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.NoTransactionException; +import com.activeviam.database.datastore.internal.builder.impl.UnitTestDatastoreBuilder; +import com.activeviam.database.datastore.internal.monitoring.MemoryStatisticsTestUtils; import com.activeviam.mac.TestMemoryStatisticBuilder; import com.activeviam.mac.entities.NoOwner; import com.activeviam.mac.memory.AnalysisDatastoreFeeder; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; -import com.activeviam.pivot.utils.ApplicationInTests; +import com.activeviam.tech.chunks.api.types.TypeValues; +import com.activeviam.tech.mvcc.api.policy.KeepAllEpochPolicy; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.DefaultMemoryStatistic; +import com.activeviam.tech.records.api.ICursor; +import com.activeviam.tech.records.api.IRecordReader; +import com.activeviam.tech.test.internal.junit.resources.Resources; +import com.activeviam.tech.test.internal.junit.resources.ResourcesExtension; +import com.activeviam.tech.test.internal.junit.resources.ResourcesHolder; +import com.activeviam.tech.test.internal.util.FileTestUtil; +import com.activeviam.tech.test.internal.util.ThrowingLambda; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import com.qfs.condition.ICondition; -import com.qfs.condition.impl.BaseConditions; -import com.qfs.desc.IDatastoreSchemaDescription; -import com.qfs.junit.LocalResourcesExtension; -import com.qfs.literal.ILiteralType; -import com.qfs.messenger.impl.LocalMessenger; -import com.qfs.monitoring.memory.impl.OnHeapPivotMemoryQuantifierPlugin; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils.StatisticsSummary; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.impl.DefaultMemoryStatistic; -import com.qfs.multiversion.impl.KeepAllEpochPolicy; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.pivot.monitoring.impl.MemoryStatisticSerializerUtil; -import com.qfs.service.monitoring.IMemoryAnalysisService; -import com.qfs.store.IDatastore; -import com.qfs.store.NoTransactionException; -import com.qfs.store.TypeValues; -import com.qfs.store.build.impl.UnitTestDatastoreBuilder; -import com.qfs.store.query.ICursor; -import com.qfs.store.record.IRecordReader; -import com.qfs.store.transaction.DatastoreTransactionException; -import com.qfs.store.transaction.ITransactionManager; -import com.qfs.util.impl.QfsFileTestUtils; -import com.qfs.util.impl.ThrowingLambda; -import com.qfs.util.impl.ThrowingLambda.ThrowingBiConsumer; -import com.quartetfs.biz.pivot.IActivePivotManager; -import com.quartetfs.biz.pivot.definitions.IActivePivotManagerDescription; -import com.quartetfs.biz.pivot.definitions.impl.ActivePivotManagerDescription; -import com.quartetfs.biz.pivot.test.util.PivotTestUtils; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -81,8 +83,9 @@ import org.assertj.core.api.Assertions; import org.assertj.core.api.SoftAssertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.extension.ExtendWith; +@ExtendWith({ResourcesExtension.class}) public abstract class ATestMemoryStatistic { public static final int STORE_PEOPLE_COUNT = 10; @@ -95,7 +98,7 @@ public abstract class ATestMemoryStatistic { public static final int MAX_GC_STEPS = 10; - @RegisterExtension public final LocalResourcesExtension resources = new LocalResourcesExtension(); + @Resources public ResourcesHolder resources; protected static final String VECTOR_STORE_NAME = "vectorStore"; @@ -331,7 +334,7 @@ void createApplication( } void createMinimalApplication( - final ThrowingLambda.ThrowingBiConsumer actions) { + final ThrowingLambda.ThrowingBiConsumer actions) { final IDatastoreSchemaDescription datastoreSchema = StartBuilding.datastoreSchema() @@ -381,7 +384,7 @@ void createMinimalApplication( .build()) .build(); - final ApplicationInTests application = + final ApplicationInTests application = ApplicationInTests.builder() .withDatastore(datastoreSchema) .withManager(userManagerDescription) @@ -392,7 +395,8 @@ void createMinimalApplication( } /** - * Fills the datastore created by {@link #createMinimalApplication(ThrowingBiConsumer)}. + * Fills the datastore created by {@link + * #createMinimalApplication(ThrowingLambda.ThrowingBiConsumer)}. * * @param datastore datastore to fill */ @@ -576,16 +580,16 @@ static void fillApplicationWithBranches( static IMemoryAnalysisService createService( final IDatastore datastore, final IActivePivotManager manager) { - final Path dumpDirectory = - QfsFileTestUtils.createTempDirectory(TestMemoryStatisticLoading.class); + final Path dumpDirectory = FileTestUtil.createTempDirectory(TestMemoryStatisticLoading.class); return new MemoryAnalysisService(datastore, manager, dumpDirectory); } - IDatastore createAnalysisDatastore() { + IInternalDatastore createAnalysisDatastore() { final IDatastoreSchemaDescription desc = new MemoryAnalysisDatastoreDescriptionConfig().datastoreSchemaDescription(); - final IDatastore datastore = new UnitTestDatastoreBuilder().setSchemaDescription(desc).build(); + final IInternalDatastore datastore = + new UnitTestDatastoreBuilder().setSchemaDescription(desc).build(); resources.register(datastore); return datastore; } @@ -705,7 +709,7 @@ static void commitDataInDatastoreWithVectors( * Builds a minimal application with one field id in the store A, loaded into a cube * Cube with a single hierarchy id. */ - ApplicationInTests createMicroApplication() { + ApplicationInTests createMicroApplication() { final IDatastoreSchemaDescription schemaDescription = StartBuilding.datastoreSchema() .withStore( @@ -733,7 +737,7 @@ ApplicationInTests createMicroApplication() { .build()) .build(); - final ApplicationInTests application = + final ApplicationInTests application = ApplicationInTests.builder() .withDatastore(schemaDescription) .withManager(userManagerDescription) @@ -744,7 +748,7 @@ ApplicationInTests createMicroApplication() { return application; } - ApplicationInTests createMicroApplicationWithIndexedFields() { + ApplicationInTests createMicroApplicationWithIndexedFields() { final IDatastoreSchemaDescription schemaDescription = StartBuilding.datastoreSchema() .withStore( @@ -778,7 +782,7 @@ ApplicationInTests createMicroApplicationWithIndexedFields() { final IActivePivotManagerDescription userManagerDescription = new ActivePivotManagerDescription(); - final ApplicationInTests application = + final ApplicationInTests application = ApplicationInTests.builder() .withDatastore(schemaDescription) .withEpochPolicy(new KeepAllEpochPolicy()) @@ -927,7 +931,7 @@ ApplicationInTests createDistributedApplicationWithKeepAllEpochPolic .withClusterDefinition() .withClusterId(clusterName) .withMessengerDefinition() - .withKey(LocalMessenger.PLUGIN_KEY) + .withKey(IMessengerDefinition.LOCAL_PLUGIN_KEY) .withNoProperty() .end() .withApplicationId("app") @@ -948,7 +952,7 @@ ApplicationInTests createDistributedApplicationWithKeepAllEpochPolic .withClusterDefinition() .withClusterId(clusterName) .withMessengerDefinition() - .withKey(LocalMessenger.PLUGIN_KEY) + .withKey(IMessengerDefinition.LOCAL_PLUGIN_KEY) .withNoProperty() .end() .withApplication("app") @@ -961,7 +965,7 @@ ApplicationInTests createDistributedApplicationWithKeepAllEpochPolic .withClusterDefinition() .withClusterId(clusterName) .withMessengerDefinition() - .withKey(LocalMessenger.PLUGIN_KEY) + .withKey(IMessengerDefinition.LOCAL_PLUGIN_KEY) .withNoProperty() .end() .withApplication("app") @@ -1225,12 +1229,13 @@ ApplicationInTests createMicroApplicationWithPartialProviders() { } IDatastore assertLoadsCorrectly( - final Collection statistics, Class klass) { - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final Collection statistics, Class klass) { + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); feedMonitoringApplication(monitoringDatastore, statistics, "storeA"); - final StatisticsSummary statisticsSummary = computeStatisticsSummary(statistics, klass); + final MemoryStatisticsTestUtils.StatisticsSummary statisticsSummary = + computeStatisticsSummary(statistics, klass); assertDatastoreConsistentWithSummary(monitoringDatastore, statisticsSummary); @@ -1240,10 +1245,10 @@ IDatastore assertLoadsCorrectly( return monitoringDatastore; } - protected static StatisticsSummary computeStatisticsSummary( - final Collection statistics, final Class creatorClass) { + protected static MemoryStatisticsTestUtils.StatisticsSummary computeStatisticsSummary( + final Collection statistics, final Class creatorClass) { return MemoryStatisticsTestUtils.getStatisticsSummary( - new TestMemoryStatisticBuilder() + new TestMemoryStatisticBuilder(creatorClass.getName()) .withCreatorClasses(creatorClass) .withChildren(statistics) .build()); @@ -1339,7 +1344,8 @@ static Set retrieveAllChunkIds( * @param statisticsSummary The statistics summary we want to compare the datastore with. */ static void assertDatastoreConsistentWithSummary( - IDatastore monitoringDatastore, StatisticsSummary statisticsSummary) { + IDatastore monitoringDatastore, + MemoryStatisticsTestUtils.StatisticsSummary statisticsSummary) { final Map latestChunkInfos = extractLatestChunkInfos(monitoringDatastore); @@ -1402,31 +1408,31 @@ static Map extractLatestChunkInfos(final IDatastore da } } - static IMemoryStatistic loadMemoryStatFromFolder(final Path folderPath) { + static AMemoryStatistic loadMemoryStatFromFolder(final Path folderPath) { return loadMemoryStatFromFolder(folderPath, __ -> true); } @SuppressWarnings("unchecked") - static Collection loadDatastoreMemoryStatFromFolder(final Path folderPath) { + static Collection loadDatastoreMemoryStatFromFolder(final Path folderPath) { final IMemoryStatistic allStat = loadMemoryStatFromFolder( folderPath, path -> path.getFileName().toString().startsWith(MemoryAnalysisService.STORE_FILE_PREFIX)); - return (Collection) allStat.getChildren(); + return (Collection) allStat.getChildren(); } @SuppressWarnings("unchecked") - static Collection loadPivotMemoryStatFromFolder(final Path folderPath) { - final IMemoryStatistic allStat = + static Collection loadPivotMemoryStatFromFolder(final Path folderPath) { + final AMemoryStatistic allStat = loadMemoryStatFromFolder( folderPath, path -> path.getFileName().toString().startsWith(MemoryAnalysisService.PIVOT_FILE_PREFIX)); - return (Collection) allStat.getChildren(); + return (Collection) allStat.getChildren(); } - static IMemoryStatistic loadMemoryStatFromFolder( + static AMemoryStatistic loadMemoryStatFromFolder( final Path folderPath, final Predicate filter) { final Stream fileList; try { @@ -1434,7 +1440,7 @@ static IMemoryStatistic loadMemoryStatFromFolder( } catch (IOException e) { throw new IllegalArgumentException("Cannot list files under " + folderPath, e); } - final List childStats = + final List childStats = fileList .filter(filter) .map( @@ -1445,6 +1451,7 @@ static IMemoryStatistic loadMemoryStatFromFolder( throw new RuntimeException("Cannot read " + file, e); } }) + .peek(stat -> stat.setParent(null)) .collect(Collectors.toList()); return new DefaultMemoryStatistic.Builder() @@ -1454,10 +1461,10 @@ static IMemoryStatistic loadMemoryStatFromFolder( } public static void feedMonitoringApplication( - final IDatastore monitoringDatastore, - final Collection stats, + final IInternalDatastore monitoringDatastore, + final Collection stats, final String dumpName) { - new AnalysisDatastoreFeeder(dumpName).loadInto(monitoringDatastore, stats.stream()); + new AnalysisDatastoreFeeder(dumpName, monitoringDatastore).loadInto(stats.stream()); } protected static class VersionedChunkInfo { diff --git a/src/test/java/com/activeviam/mac/statistic/memory/CellSetUtils.java b/src/test/java/com/activeviam/mac/statistic/memory/CellSetUtils.java index f06429a8..bd33a574 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/CellSetUtils.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/CellSetUtils.java @@ -1,6 +1,6 @@ package com.activeviam.mac.statistic.memory; -import com.quartetfs.biz.pivot.dto.CellSetDTO; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.assertj.core.api.Assertions; diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestAggregateProvidersBookmark.java b/src/test/java/com/activeviam/mac/statistic/memory/TestAggregateProvidersBookmark.java index cdfe341c..4e9f1d02 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestAggregateProvidersBookmark.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestAggregateProvidersBookmark.java @@ -7,20 +7,22 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.api.query.MDXQuery; +import com.activeviam.activepivot.server.impl.api.query.MdxQueryUtil; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.internal.IInternalDatastore; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.biz.pivot.dto.CellSetDTO; -import com.quartetfs.biz.pivot.query.impl.MDXQuery; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import com.quartetfs.fwk.query.QueryException; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.query.QueryException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; import java.util.stream.IntStream; @@ -32,11 +34,11 @@ public class TestAggregateProvidersBookmark extends ATestMemoryStatistic { protected ApplicationInTests monitoredApp; - protected ApplicationInTests monitoringApp; + protected ApplicationInTests monitoringApp; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -57,7 +59,7 @@ public void setup() throws AgentException { createService(this.monitoredApp.getDatabase(), this.monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testOverview"); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); // Start a monitoring datastore with the exported data final ManagerDescriptionConfig config = new ManagerDescriptionConfig(); @@ -86,12 +88,6 @@ public void setup() throws AgentException { @Test public void testPresentPartials() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery recordQuery = new MDXQuery( "SELECT NON EMPTY Crossjoin(" @@ -107,7 +103,7 @@ public void testPresentPartials() throws QueryException { + "FROM [MemoryCube]" + ")"); - final CellSetDTO result = pivot.execute(recordQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), recordQuery); final String[][][] positions = result.getAxes().get(1).getPositions().stream() @@ -137,12 +133,6 @@ public void testPresentPartials() throws QueryException { @Test public void testFullAggregateStoreFields() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery recordQuery = new MDXQuery( "SELECT NON EMPTY [Fields].[Field].[Field].Members ON ROWS," @@ -155,7 +145,7 @@ public void testFullAggregateStoreFields() throws QueryException { + " [Aggregate Provider].[ProviderType].[ALL].[AllMember].[BITMAP]" + ")"); - final CellSetDTO result = pivot.execute(recordQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), recordQuery); final String[][] positions = result.getAxes().get(1).getPositions().stream() @@ -174,12 +164,6 @@ public void testFullAggregateStoreFields() throws QueryException { @Test public void testFullAggregateStoreTotal() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery recordQuery = new MDXQuery( "SELECT NON EMPTY Except(" @@ -206,8 +190,9 @@ public void testFullAggregateStoreTotal() throws QueryException { + " [Aggregate Provider].[ProviderType].[ALL].[AllMember].[BITMAP]" + ")"); - final CellSetDTO result = pivot.execute(recordQuery); - final CellSetDTO totalResult = pivot.execute(totalQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), recordQuery); + final CellSetDTO totalResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), totalQuery); Assertions.assertThat(CellSetUtils.sumValuesFromCellSetDTO(result)) .isEqualTo(CellSetUtils.extractValueFromSingleCellDTO(totalResult)); @@ -215,12 +200,6 @@ public void testFullAggregateStoreTotal() throws QueryException { @Test public void testPartialBitmapAggregateStoreFields() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery recordQuery = new MDXQuery( "SELECT NON EMPTY [Fields].[Field].[Field].Members ON ROWS," @@ -233,7 +212,7 @@ public void testPartialBitmapAggregateStoreFields() throws QueryException { + " [Aggregate Provider].[ProviderType].[ALL].[AllMember].[BITMAP]" + ")"); - final CellSetDTO result = pivot.execute(recordQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), recordQuery); final String[][] positions = result.getAxes().get(1).getPositions().stream() @@ -251,12 +230,6 @@ public void testPartialBitmapAggregateStoreFields() throws QueryException { @Test public void testPartialBitmapAggregateStoreTotal() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery recordQuery = new MDXQuery( "SELECT NON EMPTY Except(" @@ -283,8 +256,9 @@ public void testPartialBitmapAggregateStoreTotal() throws QueryException { + " [Aggregate Provider].[ProviderType].[ALL].[AllMember].[BITMAP]" + ")"); - final CellSetDTO result = pivot.execute(recordQuery); - final CellSetDTO totalResult = pivot.execute(totalQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), recordQuery); + final CellSetDTO totalResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), totalQuery); Assertions.assertThat(CellSetUtils.sumValuesFromCellSetDTO(result)) .isEqualTo(CellSetUtils.extractValueFromSingleCellDTO(totalResult)); @@ -292,12 +266,6 @@ public void testPartialBitmapAggregateStoreTotal() throws QueryException { @Test public void testPartialLeafAggregateStoreFields() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery recordQuery = new MDXQuery( "SELECT NON EMPTY [Fields].[Field].[Field].Members ON ROWS," @@ -310,7 +278,7 @@ public void testPartialLeafAggregateStoreFields() throws QueryException { + " [Aggregate Provider].[ProviderType].[ALL].[AllMember].[LEAF]" + ")"); - final CellSetDTO result = pivot.execute(recordQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), recordQuery); final String[][] positions = result.getAxes().get(1).getPositions().stream() @@ -328,12 +296,6 @@ public void testPartialLeafAggregateStoreFields() throws QueryException { @Test public void testPartialLeafAggregateStoreTotal() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery recordQuery = new MDXQuery( "SELECT NON EMPTY Except(" @@ -360,8 +322,9 @@ public void testPartialLeafAggregateStoreTotal() throws QueryException { + " [Aggregate Provider].[ProviderType].[ALL].[AllMember].[LEAF]" + ")"); - final CellSetDTO result = pivot.execute(recordQuery); - final CellSetDTO totalResult = pivot.execute(totalQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), recordQuery); + final CellSetDTO totalResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), totalQuery); Assertions.assertThat(CellSetUtils.sumValuesFromCellSetDTO(result)) .isEqualTo(CellSetUtils.extractValueFromSingleCellDTO(totalResult)); @@ -369,12 +332,6 @@ public void testPartialLeafAggregateStoreTotal() throws QueryException { @Test public void testCubeLevels() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery recordQuery = new MDXQuery( "SELECT NON EMPTY [Fields].[Field].[Field].Members ON ROWS," @@ -382,7 +339,7 @@ public void testCubeLevels() throws QueryException { + "FROM [MemoryCube]" + "WHERE [Components].[Component].[ALL].[AllMember].[LEVEL]"); - final CellSetDTO result = pivot.execute(recordQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), recordQuery); final String[][] positions = result.getAxes().get(1).getPositions().stream() diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestAnalysisDatastoreFeeder.java b/src/test/java/com/activeviam/mac/statistic/memory/TestAnalysisDatastoreFeeder.java index 18c84b62..c5d0eaf3 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestAnalysisDatastoreFeeder.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestAnalysisDatastoreFeeder.java @@ -7,32 +7,32 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IActivePivotManager; +import com.activeviam.activepivot.server.intf.api.observability.IMemoryAnalysisService; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; import com.activeviam.database.api.IDatabaseVersion; +import com.activeviam.database.api.conditions.BaseConditions; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; import com.activeviam.database.api.schema.FieldPath; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.api.transaction.DatastoreTransactionException; +import com.activeviam.database.datastore.api.transaction.ITransactionManager; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.query.impl.DatastoreQueryHelper; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.entities.ChunkOwner; import com.activeviam.mac.entities.CubeOwner; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.statistic.memory.visitor.impl.EpochView; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.condition.impl.BaseConditions; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.multiversion.IEpochHistory; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.service.monitoring.IMemoryAnalysisService; -import com.qfs.store.IDatastore; -import com.qfs.store.query.ICursor; -import com.qfs.store.query.impl.DatastoreQueryHelper; -import com.qfs.store.record.IRecordReader; -import com.qfs.store.transaction.DatastoreTransactionException; -import com.qfs.store.transaction.ITransactionManager; -import com.quartetfs.biz.pivot.IActivePivotManager; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.mvcc.api.IEpochHistory; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.records.api.ICursor; +import com.activeviam.tech.records.api.IRecordReader; import gnu.trove.set.TLongSet; import gnu.trove.set.hash.TLongHashSet; import java.nio.file.Path; @@ -48,14 +48,14 @@ public class TestAnalysisDatastoreFeeder extends ATestMemoryStatistic { private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; + private ApplicationInTests monitoringApp; private ApplicationInTests distributedMonitoredApp; - private IMemoryStatistic appStatistics; - private IMemoryStatistic distributedAppHeadStatistics; + private AMemoryStatistic appStatistics; + private AMemoryStatistic distributedAppHeadStatistics; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -83,7 +83,7 @@ public void setup() throws DatastoreTransactionException, AgentException { /** Ensures the same statistics can be loaded in different dump names. */ @Test public void testDifferentDumps() { - final IDatastore monitoringDatastore = this.monitoringApp.getDatabase(); + final IInternalDatastore monitoringDatastore = this.monitoringApp.getDatabase(); ATestMemoryStatistic.feedMonitoringApplication( monitoringDatastore, List.of(this.appStatistics), "app"); @@ -105,7 +105,7 @@ public void testDifferentDumps() { */ @Test public void testEpochReplicationForAlreadyExistingChunks() { - final IDatastore monitoringDatastore = this.monitoringApp.getDatabase(); + final IInternalDatastore monitoringDatastore = this.monitoringApp.getDatabase(); // load a cluster's single epoch statistics ATestMemoryStatistic.feedMonitoringApplication( @@ -163,7 +163,7 @@ public void testEpochReplicationForAlreadyExistingChunks() { @Test public void testEpochReplicationForAlreadyExistingEpochs() { - final IDatastore monitoringDatastore = this.monitoringApp.getDatabase(); + final IInternalDatastore monitoringDatastore = this.monitoringApp.getDatabase(); ATestMemoryStatistic.feedMonitoringApplication( monitoringDatastore, List.of(this.appStatistics), "app"); @@ -290,7 +290,7 @@ private void initializeMonitoringApplication() { final IDatastoreSchemaDescriptionConfig schemaConfig = new MemoryAnalysisDatastoreDescriptionConfig(); - final ApplicationInTests application = + final ApplicationInTests application = ApplicationInTests.builder() .withManager(config.managerDescription()) .withDatastore(schemaConfig.datastoreSchemaDescription()) diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestBranches.java b/src/test/java/com/activeviam/mac/statistic/memory/TestBranches.java index 34b2c7d2..faf2891e 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestBranches.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestBranches.java @@ -7,29 +7,30 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.api.conditions.BaseConditions; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; import com.activeviam.database.api.schema.FieldPath; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.api.transaction.DatastoreTransactionException; +import com.activeviam.database.datastore.api.transaction.ITransactionManager; +import com.activeviam.database.datastore.internal.IInternalDatastore; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; -import com.activeviam.pivot.utils.ApplicationInTests; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.records.api.ICursor; +import com.activeviam.tech.records.api.IRecordReader; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import com.qfs.condition.impl.BaseConditions; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.qfs.store.query.ICursor; -import com.qfs.store.record.IRecordReader; -import com.qfs.store.transaction.DatastoreTransactionException; -import com.qfs.store.transaction.ITransactionManager; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; import java.nio.file.Path; import java.util.Collection; import java.util.HashMap; @@ -47,11 +48,11 @@ public class TestBranches extends ATestMemoryStatistic { private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; + private ApplicationInTests monitoringApp; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -60,7 +61,7 @@ public void setup() throws AgentException, DatastoreTransactionException { final Path exportPath = generateMemoryStatistics(); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); initializeMonitoringApplication(stats); @@ -104,12 +105,12 @@ private Path generateMemoryStatistics() { return analysisService.exportApplication("testBranches"); } - private void initializeMonitoringApplication(final IMemoryStatistic data) { + private void initializeMonitoringApplication(final AMemoryStatistic data) { final ManagerDescriptionConfig config = new ManagerDescriptionConfig(); final IDatastoreSchemaDescriptionConfig schemaConfig = new MemoryAnalysisDatastoreDescriptionConfig(); - ApplicationInTests application = + ApplicationInTests application = ApplicationInTests.builder() .withDatastore(schemaConfig.datastoreSchemaDescription()) .withManager(config.managerDescription()) diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestDistributedCubeEpochs.java b/src/test/java/com/activeviam/mac/statistic/memory/TestDistributedCubeEpochs.java index 8f084b16..75917841 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestDistributedCubeEpochs.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestDistributedCubeEpochs.java @@ -7,28 +7,29 @@ package com.activeviam.mac.statistic.memory; -import static com.qfs.distribution.impl.DistributionUtil.stopDistribution; +import static com.activeviam.activepivot.dist.impl.internal.distribution.impl.DistributionUtil.stopDistribution; import static org.assertj.core.api.Assertions.assertThat; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.dist.impl.internal.impl.MultiVersionDistributedActivePivot; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.internal.IInternalDatastore; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.statistic.memory.visitor.impl.DistributedEpochView; import com.activeviam.mac.statistic.memory.visitor.impl.EpochView; import com.activeviam.mac.statistic.memory.visitor.impl.RegularEpochView; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.impl.MultiVersionDistributedActivePivot; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.qfs.store.query.ICursor; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.records.api.ICursor; import java.nio.file.Path; import java.util.Collections; import java.util.List; @@ -45,11 +46,11 @@ public class TestDistributedCubeEpochs extends ATestMemoryStatistic { private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; + private ApplicationInTests monitoringApp; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -57,7 +58,7 @@ public void setup() throws AgentException { initializeApplication(); final Path exportPath = generateMemoryStatistics(); - final IMemoryStatistic statistics = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic statistics = loadMemoryStatFromFolder(exportPath); initializeMonitoringApplication(statistics); @@ -121,7 +122,7 @@ private Path generateMemoryStatistics() { return analysisService.exportApplication("testEpochs"); } - private void initializeMonitoringApplication(final IMemoryStatistic data) { + private void initializeMonitoringApplication(final AMemoryStatistic data) { final ManagerDescriptionConfig config = new ManagerDescriptionConfig(); final IDatastoreSchemaDescriptionConfig schemaConfig = new MemoryAnalysisDatastoreDescriptionConfig(); @@ -151,12 +152,7 @@ public void testExpectedViewEpochs() { } private long getHeadEpochId(String queryCube) { - return this.monitoredApp - .getManager() - .getActivePivots() - .get(queryCube) - .getMostRecentVersion() - .getEpochId(); + return this.monitoredApp.getManager().getActivePivots().get(queryCube).getHead().getEpochId(); } protected Set retrieveViewEpochIds() { diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestEpochs.java b/src/test/java/com/activeviam/mac/statistic/memory/TestEpochs.java index bb382b15..898887da 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestEpochs.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestEpochs.java @@ -7,30 +7,29 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.api.ICondition; +import com.activeviam.database.api.conditions.BaseConditions; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; import com.activeviam.database.api.schema.FieldPath; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.internal.IInternalDatastore; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.UsedByVersion; -import com.activeviam.pivot.utils.ApplicationInTests; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.records.api.ICursor; +import com.activeviam.tech.records.api.IRecordReader; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import com.qfs.condition.ICondition; -import com.qfs.condition.impl.BaseConditions; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.multiversion.impl.Epoch; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.qfs.store.query.ICursor; -import com.qfs.store.record.IRecordReader; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; import java.nio.file.Path; import java.util.Collection; import java.util.List; @@ -46,12 +45,12 @@ public class TestEpochs extends ATestMemoryStatistic { private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; - private IMemoryStatistic statistics; + private ApplicationInTests monitoringApp; + private AMemoryStatistic statistics; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -115,8 +114,6 @@ private void initializeApplication() { transactionManager -> transactionManager.removeWhere( "A", BaseConditions.greaterOrEqual(FieldPath.of("id"), 20))); - - this.monitoredApp.getManager().getActivePivots().get("Cube").commit(new Epoch(10L)); } private Path generateMemoryStatistics() { @@ -129,7 +126,7 @@ private Path generateMemoryStatistics() { return analysisService.exportApplication("testEpochs"); } - private void initializeMonitoringApplication(final IMemoryStatistic data) throws AgentException { + private void initializeMonitoringApplication(final AMemoryStatistic data) throws AgentException { ManagerDescriptionConfig config = new ManagerDescriptionConfig(); IDatastoreSchemaDescriptionConfig schemaConfig = new MemoryAnalysisDatastoreDescriptionConfig(); @@ -154,7 +151,7 @@ public void testStatisticConsistency() { public void testExpectedEpochs() { final Set epochIds = retrieveEpochIds(); - Assertions.assertThat(epochIds).containsExactlyInAnyOrder(1L, 2L, 3L, 4L, 5L, 10L); + Assertions.assertThat(epochIds).containsExactlyInAnyOrder(1L, 2L, 3L, 4L, 5L); } @Test diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmark.java b/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmark.java index 3e5639ec..62fcb958 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmark.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmark.java @@ -1,21 +1,21 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.api.query.MDXQuery; +import com.activeviam.activepivot.server.impl.api.query.MdxQueryUtil; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.monitoring.MemoryStatisticsTestUtils; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils.StatisticsSummary; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.biz.pivot.dto.CellSetDTO; -import com.quartetfs.biz.pivot.query.impl.MDXQuery; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import com.quartetfs.fwk.query.QueryException; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.query.QueryException; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; import java.util.stream.IntStream; @@ -28,12 +28,12 @@ public class TestFieldsBookmark extends ATestMemoryStatistic { public static final int ADDED_DATA_SIZE = 20; private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; - StatisticsSummary summary; + private ApplicationInTests monitoringApp; + MemoryStatisticsTestUtils.StatisticsSummary summary; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -57,7 +57,7 @@ public void setup() throws AgentException { createService(this.monitoredApp.getDatabase(), this.monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testOverview"); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); this.summary = MemoryStatisticsTestUtils.getStatisticsSummary(stats); // Start a monitoring datastore with the exported data @@ -86,12 +86,6 @@ public void setup() throws AgentException { @Test public void testStoreTotal() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery storeTotal = new MDXQuery( "SELECT NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS " @@ -120,9 +114,12 @@ public void testStoreTotal() throws QueryException { + " FROM [MemoryCube]" + " WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); - final CellSetDTO totalResult = pivot.execute(storeTotal); - final CellSetDTO fieldResult = pivot.execute(perFieldQuery); - final CellSetDTO excessResult = pivot.execute(excessMemoryQuery); + final CellSetDTO totalResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), storeTotal); + final CellSetDTO fieldResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), perFieldQuery); + final CellSetDTO excessResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), excessMemoryQuery); Assertions.assertThat(CellSetUtils.extractValueFromSingleCellDTO(totalResult)) .isEqualTo( diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmarkWithDuplicateFieldName.java b/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmarkWithDuplicateFieldName.java index 3d2920b9..2d9c0692 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmarkWithDuplicateFieldName.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmarkWithDuplicateFieldName.java @@ -7,22 +7,22 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.api.query.MDXQuery; +import com.activeviam.activepivot.server.impl.api.query.MdxQueryUtil; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.monitoring.MemoryStatisticsTestUtils; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils.StatisticsSummary; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.biz.pivot.dto.CellSetDTO; -import com.quartetfs.biz.pivot.query.impl.MDXQuery; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import com.quartetfs.fwk.query.QueryException; +import com.activeviam.tech.core.api.query.QueryException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; import java.util.stream.IntStream; @@ -35,16 +35,16 @@ public class TestFieldsBookmarkWithDuplicateFieldName extends ATestMemoryStatist public static final int ADDED_DATA_SIZE = 20; private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; - StatisticsSummary summary; + private ApplicationInTests monitoringApp; + MemoryStatisticsTestUtils.StatisticsSummary summary; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach - public void setup() throws AgentException { + public void setup() { this.monitoredApp = createMicroApplicationWithReferenceAndSameFieldName(); this.monitoredApp @@ -66,7 +66,7 @@ public void setup() throws AgentException { createService(this.monitoredApp.getDatabase(), this.monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testOverview"); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); this.summary = MemoryStatisticsTestUtils.getStatisticsSummary(stats); // Start a monitoring datastore with the exported data @@ -94,12 +94,6 @@ public void setup() throws AgentException { @Test public void testDifferentMemoryUsagesForBothFields() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery usageQuery = new MDXQuery( "SELECT NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS, " @@ -109,7 +103,7 @@ public void testDifferentMemoryUsagesForBothFields() throws QueryException { + "} ON ROWS " + "FROM [MemoryCube]"); - final CellSetDTO result = pivot.execute(usageQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), usageQuery); Assertions.assertThat((long) result.getCells().get(0).getValue()) .isNotEqualTo((long) result.getCells().get(1).getValue()); diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestIndexAndDictionaryBookmarks.java b/src/test/java/com/activeviam/mac/statistic/memory/TestIndexAndDictionaryBookmarks.java index c0179cc0..f1020435 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestIndexAndDictionaryBookmarks.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestIndexAndDictionaryBookmarks.java @@ -7,24 +7,24 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.api.query.MDXQuery; +import com.activeviam.activepivot.server.impl.api.query.MdxQueryUtil; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.intf.api.dto.AxisDTO; +import com.activeviam.activepivot.server.intf.api.dto.AxisPositionDTO; +import com.activeviam.activepivot.server.intf.api.dto.CellDTO; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.internal.IInternalDatastore; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.qfs.store.record.impl.IDictionaryProvider; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.biz.pivot.dto.AxisDTO; -import com.quartetfs.biz.pivot.dto.AxisPositionDTO; -import com.quartetfs.biz.pivot.dto.CellDTO; -import com.quartetfs.biz.pivot.dto.CellSetDTO; -import com.quartetfs.biz.pivot.query.impl.MDXQuery; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import com.quartetfs.fwk.query.QueryException; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.query.QueryException; +import com.activeviam.tech.dictionaries.api.IDictionaryProvider; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; import java.util.stream.IntStream; @@ -37,12 +37,12 @@ public class TestIndexAndDictionaryBookmarks extends ATestMemoryStatistic { public static final int ADDED_DATA_SIZE = 20; - private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; + private ApplicationInTests monitoredApp; + private ApplicationInTests monitoringApp; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -51,7 +51,7 @@ public void setup() throws AgentException { final Path exportPath = generateMemoryStatistics(); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); initializeMonitoringApplication(stats); @@ -85,7 +85,7 @@ private Path generateMemoryStatistics() { return analysisService.exportMostRecentVersion("testOverview"); } - private void initializeMonitoringApplication(final IMemoryStatistic data) throws AgentException { + private void initializeMonitoringApplication(final AMemoryStatistic data) throws AgentException { ManagerDescriptionConfig config = new ManagerDescriptionConfig(); IDatastoreSchemaDescriptionConfig schemaConfig = new MemoryAnalysisDatastoreDescriptionConfig(); @@ -103,19 +103,13 @@ private void initializeMonitoringApplication(final IMemoryStatistic data) throws @Test public void testIndexedFieldsForStoreA() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery totalQuery = new MDXQuery( "SELECT NON EMPTY [Indices].[Indexed Fields].[Indexed Fields].Members ON COLUMNS" + " FROM [MemoryCube]" + " WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); - final CellSetDTO result = pivot.execute(totalQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), totalQuery); final List axes = result.getAxes(); Assertions.assertThat(axes).hasSize(1); @@ -130,12 +124,6 @@ public void testIndexedFieldsForStoreA() throws QueryException { @Test public void testDictionarizedFieldsForStoreB() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery totalQuery = new MDXQuery( "SELECT NonEmpty(" @@ -150,7 +138,7 @@ public void testDictionarizedFieldsForStoreB() throws QueryException { + " [Owners].[Owner].[ALL].[AllMember].[Store B]" + " )"); - final CellSetDTO result = pivot.execute(totalQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), totalQuery); final List axes = result.getAxes(); Assertions.assertThat(axes).hasSize(1); @@ -164,12 +152,6 @@ public void testDictionarizedFieldsForStoreB() throws QueryException { @Test public void testDictionarySizeTotal() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery totalQuery = new MDXQuery( "SELECT NON EMPTY [Fields].[Field].[ALL].[AllMember] ON COLUMNS," @@ -190,8 +172,9 @@ public void testDictionarySizeTotal() throws QueryException { + " FROM [MemoryCube]" + " WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); - final CellSetDTO total = pivot.execute(totalQuery); - final CellSetDTO perField = pivot.execute(perFieldQuery); + final CellSetDTO total = MdxQueryUtil.execute(this.monitoringApp.getManager(), totalQuery); + final CellSetDTO perField = + MdxQueryUtil.execute(this.monitoringApp.getManager(), perFieldQuery); Assertions.assertThat(perField.getCells().stream().mapToLong(x -> (long) x.getValue()).sum()) .isEqualTo((long) total.getCells().get(0).getValue()); @@ -199,12 +182,6 @@ public void testDictionarySizeTotal() throws QueryException { @Test public void testDictionarySizesPerField() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery totalQuery = new MDXQuery( "SELECT NON EMPTY" @@ -219,7 +196,7 @@ public void testDictionarySizesPerField() throws QueryException { + " [Components].[Component].[ALL].[AllMember].[DICTIONARY]" + " )"); - final CellSetDTO result = pivot.execute(totalQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), totalQuery); final IDictionaryProvider dictionaryProvider = this.monitoredApp diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestMACMeasures.java b/src/test/java/com/activeviam/mac/statistic/memory/TestMACMeasures.java index 468d0f4d..550a34a4 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestMACMeasures.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestMACMeasures.java @@ -1,34 +1,34 @@ package com.activeviam.mac.statistic.memory; -import static com.qfs.util.impl.ThrowingLambda.cast; +import static com.activeviam.tech.test.internal.util.ThrowingLambda.cast; import static java.util.stream.Collectors.toMap; -import com.activeviam.fwk.ActiveViamRuntimeException; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.api.query.MDXQuery; +import com.activeviam.activepivot.server.impl.api.query.MdxQueryUtil; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.intf.api.dto.CellDTO; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.api.transaction.DatastoreTransactionException; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.NoTransactionException; +import com.activeviam.database.datastore.internal.monitoring.MemoryStatisticsTestUtils; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.activeviam.properties.impl.ActiveViamProperty; -import com.activeviam.properties.impl.ActiveViamPropertyExtension; -import com.activeviam.properties.impl.ActiveViamPropertyExtension.ActiveViamPropertyExtensionBuilder; -import com.qfs.junit.LocalResourcesExtension; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils.StatisticsSummary; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.qfs.store.NoTransactionException; -import com.qfs.store.transaction.DatastoreTransactionException; -import com.qfs.util.impl.QfsArrays; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.biz.pivot.dto.CellDTO; -import com.quartetfs.biz.pivot.dto.CellSetDTO; -import com.quartetfs.biz.pivot.query.impl.MDXQuery; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import com.quartetfs.fwk.query.QueryException; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; +import com.activeviam.tech.core.api.properties.ActiveViamProperty; +import com.activeviam.tech.core.api.query.QueryException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.core.internal.properties.ActiveViamPropertyExtension; +import com.activeviam.tech.core.internal.util.ArrayUtil; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.test.internal.junit.resources.ResourcesExtension; import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -38,6 +38,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; /** @@ -47,6 +48,7 @@ * *

Tools.extractSnappyFile(path to file); */ +@ExtendWith({ResourcesExtension.class}) public class TestMACMeasures extends ATestMemoryStatistic { public static final int ADDED_DATA_SIZE = 100; @@ -55,27 +57,24 @@ public class TestMACMeasures extends ATestMemoryStatistic { @RegisterExtension @SuppressWarnings("unused") public static ActiveViamPropertyExtension propertyRule = - new ActiveViamPropertyExtensionBuilder() + new ActiveViamPropertyExtension.ActiveViamPropertyExtensionBuilder() .withProperty(ActiveViamProperty.ACTIVEVIAM_TEST_PROPERTY, true) .build(); - @RegisterExtension - public final LocalResourcesExtension methodResources = new LocalResourcesExtension(); - - private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; - IMemoryStatistic stats; + private ApplicationInTests monitoredApp; + private ApplicationInTests monitoringApp; + AMemoryStatistic stats; Map appStats; - StatisticsSummary statsSumm; + MemoryStatisticsTestUtils.StatisticsSummary statsSumm; @BeforeAll public static void init() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } private static Map extractApplicationStats(final IMemoryStatistic export) { final IMemoryStatistic firstChild = export.getChildren().iterator().next(); - return QfsArrays.mutableMap( + return ArrayUtil.mutableMap( ManagerDescriptionConfig.USED_HEAP, MemoryStatisticConstants.STAT_NAME_GLOBAL_USED_HEAP_MEMORY, ManagerDescriptionConfig.COMMITTED_HEAP, @@ -154,24 +153,19 @@ public void setup() throws AgentException { @Test public void testDirectMemorySum() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); final MDXQuery query = new MDXQuery( "SELECT" + " NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res = pivot.execute(query); + CellSetDTO res = MdxQueryUtil.execute(this.monitoringApp.getManager(), query); Long value = CellSetUtils.extractValueFromSingleCellDTO(res); final MDXQuery query2 = new MDXQuery( "SELECT" + " NON EMPTY [Measures].[Chunks.COUNT] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res2 = pivot.execute(query2); + CellSetDTO res2 = MdxQueryUtil.execute(this.monitoringApp.getManager(), query2); Long nbC = CellSetUtils.extractValueFromSingleCellDTO(res2); final MDXQuery query3 = @@ -180,7 +174,7 @@ public void testDirectMemorySum() throws QueryException { + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res3 = pivot.execute(query3); + CellSetDTO res3 = MdxQueryUtil.execute(this.monitoringApp.getManager(), query3); // Check that the cell size is the expected one (the amount of chunks) Assertions.assertThat(res3.getCells().size()).isEqualTo(nbC.intValue()); @@ -193,24 +187,19 @@ public void testDirectMemorySum() throws QueryException { @Test public void testOnHeapMemorySum() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); final MDXQuery query = new MDXQuery( "SELECT" + " NON EMPTY [Measures].[HeapMemory.SUM] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res = pivot.execute(query); + CellSetDTO res = MdxQueryUtil.execute(this.monitoringApp.getManager(), query); Long value = CellSetUtils.extractValueFromSingleCellDTO(res); final MDXQuery query2 = new MDXQuery( "SELECT" + " NON EMPTY [Measures].[Chunks.COUNT] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res2 = pivot.execute(query2); + CellSetDTO res2 = MdxQueryUtil.execute(this.monitoringApp.getManager(), query2); Long nbC = CellSetUtils.extractValueFromSingleCellDTO(res2); final MDXQuery query3 = @@ -219,7 +208,7 @@ public void testOnHeapMemorySum() throws QueryException { + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[HeapMemory.SUM] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res3 = pivot.execute(query3); + CellSetDTO res3 = MdxQueryUtil.execute(this.monitoringApp.getManager(), query3); // Check that the cell size is the expected one (the amount of chunks) Assertions.assertThat(res3.getCells().size()).isEqualTo(nbC.intValue()); @@ -237,19 +226,13 @@ public void testOnHeapMemorySum() throws QueryException { @Test public void testChunkSize() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery query = new MDXQuery( "SELECT" + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[ChunkSize.SUM] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res = pivot.execute(query); + CellSetDTO res = MdxQueryUtil.execute(this.monitoringApp.getManager(), query); Assertions.assertThat(CellSetUtils.extractValuesFromCellSetDTO(res)) .contains((double) ATestMemoryStatistic.MICROAPP_CHUNK_SIZE); @@ -258,19 +241,13 @@ public void testChunkSize() throws QueryException { @Test public void testNonWrittenCount() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery query = new MDXQuery( "SELECT" + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[Unused rows] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res = pivot.execute(query); + CellSetDTO res = MdxQueryUtil.execute(this.monitoringApp.getManager(), query); Assertions.assertThat(CellSetUtils.extractValuesFromCellSetDTO(res)) .contains((double) ATestMemoryStatistic.MICROAPP_CHUNK_SIZE - ADDED_DATA_SIZE); @@ -278,11 +255,6 @@ public void testNonWrittenCount() throws QueryException { @Test public void testApplicationMeasures() { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); SoftAssertions.assertSoftly( assertions -> @@ -296,7 +268,8 @@ public void testApplicationMeasures() { + measure + "] ON COLUMNS" + " FROM [MemoryCube]"); - final CellSetDTO result = pivot.execute(query); + final CellSetDTO result = + MdxQueryUtil.execute(this.monitoringApp.getManager(), query); final Long resultValue = CellSetUtils.extractValueFromSingleCellDTO(result); assertions.assertThat(resultValue).as("Value of " + measure).isEqualTo(value); }))); @@ -308,11 +281,6 @@ public void testApplicationMeasures() { */ @Test public void testApplicationMeasuresAtAnyPointOfTheCube() { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); SoftAssertions.assertSoftly( assertions -> @@ -327,7 +295,8 @@ public void testApplicationMeasuresAtAnyPointOfTheCube() { + "] ON COLUMNS" + " FROM [MemoryCube]" + " WHERE ([Owners].[Owner].[ALL].[AllMember].FirstChild)"); - final CellSetDTO result = pivot.execute(query); + final CellSetDTO result = + MdxQueryUtil.execute(this.monitoringApp.getManager(), query); final Long resultValue = CellSetUtils.extractValueFromSingleCellDTO(result); assertions.assertThat(resultValue).as("Value of " + measure).isEqualTo(value); }))); @@ -347,7 +316,8 @@ public void testApplicationMeasuresAtAnyPointOfTheCube() { + " WHERE ([" + ManagerDescriptionConfig.CHUNK_DIMENSION + "].[ChunkId].[ALL].[AllMember].FirstChild)"); - final CellSetDTO result = pivot.execute(query); + final CellSetDTO result = + MdxQueryUtil.execute(this.monitoringApp.getManager(), query); final Long resultValue = CellSetUtils.extractValueFromSingleCellDTO(result); assertions.assertThat(resultValue).as("Value of " + measure).isEqualTo(value); }))); @@ -356,19 +326,13 @@ public void testApplicationMeasuresAtAnyPointOfTheCube() { @Test public void testFreedCount() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery query = new MDXQuery( "SELECT" + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " [Measures].[Deleted rows] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res = pivot.execute(query); + CellSetDTO res = MdxQueryUtil.execute(this.monitoringApp.getManager(), query); Assertions.assertThat(CellSetUtils.extractValuesFromCellSetDTO(res)) .contains((double) REMOVED_DATA_SIZE); @@ -377,18 +341,13 @@ public void testFreedCount() throws QueryException { @Test public void testNonWrittenRatio() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); final MDXQuery query = new MDXQuery( "SELECT" + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[ChunkSize.SUM] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res = pivot.execute(query); + CellSetDTO res = MdxQueryUtil.execute(this.monitoringApp.getManager(), query); final MDXQuery query2 = new MDXQuery( @@ -396,7 +355,7 @@ public void testNonWrittenRatio() throws QueryException { + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[Unused rows] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res2 = pivot.execute(query2); + CellSetDTO res2 = MdxQueryUtil.execute(this.monitoringApp.getManager(), query2); final MDXQuery query3 = new MDXQuery( @@ -404,7 +363,7 @@ public void testNonWrittenRatio() throws QueryException { + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[Unused rows ratio] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res3 = pivot.execute(query3); + CellSetDTO res3 = MdxQueryUtil.execute(this.monitoringApp.getManager(), query3); final Double[] chunkSizes = CellSetUtils.extractValuesFromCellSetDTO(res); final Double[] nonWrittenRows = CellSetUtils.extractValuesFromCellSetDTO(res2); @@ -418,18 +377,13 @@ public void testNonWrittenRatio() throws QueryException { @Test public void testDeletedRatio() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); final MDXQuery query = new MDXQuery( "SELECT" + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[ChunkSize.SUM] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res = pivot.execute(query); + CellSetDTO res = MdxQueryUtil.execute(this.monitoringApp.getManager(), query); final MDXQuery query2 = new MDXQuery( @@ -437,7 +391,7 @@ public void testDeletedRatio() throws QueryException { + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[Deleted rows] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res2 = pivot.execute(query2); + CellSetDTO res2 = MdxQueryUtil.execute(this.monitoringApp.getManager(), query2); final MDXQuery query3 = new MDXQuery( @@ -445,7 +399,7 @@ public void testDeletedRatio() throws QueryException { + " NON EMPTY [Chunks].[ChunkId].[ChunkId].Members ON ROWS," + " NON EMPTY [Measures].[Deleted rows ratio] ON COLUMNS" + " FROM [MemoryCube]"); - CellSetDTO res3 = pivot.execute(query3); + CellSetDTO res3 = MdxQueryUtil.execute(this.monitoringApp.getManager(), query3); final Double[] chunkSizes = CellSetUtils.extractValuesFromCellSetDTO(res); final Double[] DeletedRows = CellSetUtils.extractValuesFromCellSetDTO(res2); @@ -458,11 +412,6 @@ public void testDeletedRatio() throws QueryException { @Test public void testDictionarySize() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); final MDXQuery query = new MDXQuery( "SELECT" @@ -473,7 +422,7 @@ public void testDictionarySize() throws QueryException { + " [Owners].[Owner].[Owner].[Store A]," + " [Fields].[Field].[Field].[id]" + " )"); - CellSetDTO res = pivot.execute(query); + CellSetDTO res = MdxQueryUtil.execute(this.monitoringApp.getManager(), query); final long expectedDictionarySize = this.monitoredApp diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryMonitoringDatastoreContent.java b/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryMonitoringDatastoreContent.java index 39dbd308..dc3b46fb 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryMonitoringDatastoreContent.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryMonitoringDatastoreContent.java @@ -1,38 +1,40 @@ package com.activeviam.mac.statistic.memory; import static com.activeviam.mac.memory.DatastoreConstants.CHUNK_STORE; +import static com.activeviam.tech.test.internal.assertj.AssertJConditions.identical; import static java.util.stream.Collectors.toList; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.server.intf.api.observability.IMemoryAnalysisService; +import com.activeviam.database.api.conditions.BaseConditions; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; import com.activeviam.database.api.schema.FieldPath; -import com.activeviam.fwk.ActiveViamRuntimeException; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.api.transaction.DatastoreTransactionException; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.NoTransactionException; +import com.activeviam.database.datastore.internal.impl.Datastore; +import com.activeviam.database.datastore.internal.monitoring.AMemoryStatisticWithPredicate; import com.activeviam.mac.memory.AnalysisDatastoreFeeder; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; -import com.activeviam.pivot.utils.ApplicationInTests; +import com.activeviam.tech.chunks.internal.impl.ChunkSingleVector; +import com.activeviam.tech.chunks.internal.vectors.direct.impl.DirectIntegerVectorBlock; +import com.activeviam.tech.chunks.internal.vectors.direct.impl.DirectLongVectorBlock; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; +import com.activeviam.tech.dictionaries.api.IDictionary; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.ChunkStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.records.api.ICursor; +import com.activeviam.tech.records.api.IRecordReader; import com.google.common.collect.HashMultimap; import com.google.common.collect.SetMultimap; -import com.qfs.assertj.QfsConditions; -import com.qfs.chunk.impl.ChunkSingleVector; -import com.qfs.condition.impl.BaseConditions; -import com.qfs.dic.IDictionary; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.impl.ChunkStatistic; -import com.qfs.monitoring.statistic.memory.visitor.impl.AMemoryStatisticWithPredicate; -import com.qfs.service.monitoring.IMemoryAnalysisService; -import com.qfs.store.IDatastore; -import com.qfs.store.NoTransactionException; -import com.qfs.store.impl.Datastore; -import com.qfs.store.query.ICursor; -import com.qfs.store.record.IRecordReader; -import com.qfs.store.transaction.DatastoreTransactionException; -import com.qfs.vector.direct.impl.DirectIntegerVectorBlock; -import com.qfs.vector.direct.impl.DirectLongVectorBlock; -import com.quartetfs.fwk.AgentException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -66,7 +68,7 @@ public void testDatastoreMonitoringValues() { createService(monitoredDatastore, monitoredManager); final Path exportPath = analysisService.exportApplication("testLoadComplete"); - final IMemoryStatistic fullStats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic fullStats = loadMemoryStatFromFolder(exportPath); final Datastore monitoringDatastore = (Datastore) createAnalysisDatastore(); IDictionary dic = @@ -138,7 +140,7 @@ public void testDatastoreMonitoringValues() { @Test public void testChunkStructureFieldsWithSingleRecord() { - final ApplicationInTests monitoredApp = createMicroApplication(); + final ApplicationInTests monitoredApp = createMicroApplication(); // Add a single record monitoredApp.getDatabase().edit(tm -> tm.add("A", 0)); @@ -149,13 +151,13 @@ public void testChunkStructureFieldsWithSingleRecord() { final IMemoryAnalysisService analysisService = createService(monitoredApp.getDatabase(), monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final Collection storeStats = loadDatastoreMemoryStatFromFolder(exportPath); + final Collection storeStats = loadDatastoreMemoryStatFromFolder(exportPath); // Make sure there is only one loaded store Assertions.assertThat(storeStats).hasSize(1); // Start a monitoring datastore with the exported data - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); ATestMemoryStatistic.feedMonitoringApplication(monitoringDatastore, storeStats, "storeA"); ListQuery query = @@ -197,7 +199,7 @@ public void testChunkStructureFieldsWithSingleRecord() { @Test public void testChunkStructureFieldsWithFullChunk() { - final ApplicationInTests monitoredApp = createMicroApplication(); + final ApplicationInTests monitoredApp = createMicroApplication(); // Add a full chunk monitoredApp @@ -210,13 +212,13 @@ public void testChunkStructureFieldsWithFullChunk() { final IMemoryAnalysisService analysisService = createService(monitoredApp.getDatabase(), monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final Collection storeStats = loadDatastoreMemoryStatFromFolder(exportPath); + final Collection storeStats = loadDatastoreMemoryStatFromFolder(exportPath); // Make sure there is only one loaded store Assertions.assertThat(storeStats).hasSize(1); // Start a monitoring datastore with the exported data - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); ATestMemoryStatistic.feedMonitoringApplication(monitoringDatastore, storeStats, "storeA"); // Query record chunks data : @@ -259,7 +261,7 @@ public void testChunkStructureFieldsWithFullChunk() { @Test public void testChunkStructureFieldsWithTwoChunks() { - final ApplicationInTests monitoredApp = createMicroApplication(); + final ApplicationInTests monitoredApp = createMicroApplication(); // Add a full chunk + 10 records on the second chunk monitoredApp @@ -272,13 +274,13 @@ public void testChunkStructureFieldsWithTwoChunks() { final IMemoryAnalysisService analysisService = createService(monitoredApp.getDatabase(), monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final Collection storeStats = loadDatastoreMemoryStatFromFolder(exportPath); + final Collection storeStats = loadDatastoreMemoryStatFromFolder(exportPath); // Make sure there is only one loaded store Assertions.assertThat(storeStats).hasSize(1); // Start a monitoring datastore with the exported data - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); ATestMemoryStatistic.feedMonitoringApplication(monitoringDatastore, storeStats, "storeA"); ListQuery query = @@ -329,7 +331,7 @@ public void testChunkStructureFieldsWithTwoChunks() { @Test public void testChunkStructureFieldsWithFreedRows() { - final ApplicationInTests monitoredApp = createMicroApplication(); + final ApplicationInTests monitoredApp = createMicroApplication(); // Add 100 records monitoredApp.getDatabase().edit(tm -> IntStream.range(0, 100).forEach(i -> tm.add("A", i * i))); // Delete 10 records @@ -356,9 +358,9 @@ public void testChunkStructureFieldsWithFreedRows() { createService(monitoredApp.getDatabase(), monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final Collection storeStats = loadDatastoreMemoryStatFromFolder(exportPath); + final Collection storeStats = loadDatastoreMemoryStatFromFolder(exportPath); // Start a monitoring datastore with the exported data - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); ATestMemoryStatistic.feedMonitoringApplication(monitoringDatastore, storeStats, "storeA"); // Query record chunks data : @@ -417,13 +419,13 @@ public void testLevelStoreContent() { final IMemoryAnalysisService analysisService = createService(datastore, monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final Collection pivotStats = loadPivotMemoryStatFromFolder(exportPath); + final Collection pivotStats = loadPivotMemoryStatFromFolder(exportPath); // Make sure there is only one loaded store Assertions.assertThat(pivotStats).hasSize(1); // Start a monitoring datastore with the exported data - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); ATestMemoryStatistic.feedMonitoringApplication(monitoringDatastore, pivotStats, "Cube"); // Query record of level data : @@ -460,12 +462,12 @@ public void testMappingFromChunkToFields() { datastore.edit(tm -> IntStream.range(0, 10).forEach(i -> tm.add("A", i * i))); // Initial Dump Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); ATestMemoryStatistic.feedMonitoringApplication(monitoringDatastore, List.of(stats), "appAInit"); - final List dics = + final List dics = collectStatistics( stats, List.of( @@ -480,7 +482,7 @@ public void testMappingFromChunkToFields() { Assertions.assertThat(dics) .extracting( stat -> stat.getAttribute(MemoryStatisticConstants.ATTR_NAME_DICTIONARY_ID).asLong()) - .are(QfsConditions.identical()); + .are(identical()); final var dicForFieldId = dics.get(0).getAttribute(MemoryStatisticConstants.ATTR_NAME_DICTIONARY_ID).asLong(); @@ -548,7 +550,7 @@ public void testCompareDumpsOfDifferentEpochs() { datastore.edit(tm -> IntStream.range(0, 100).forEach(i -> tm.add("A", i * i))); // Initial Dump Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); // Delete 10 records datastore.edit( tm -> @@ -567,13 +569,15 @@ public void testCompareDumpsOfDifferentEpochs() { datastore.getEpochManager().forceDiscardEpochs(__ -> true); exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final IMemoryStatistic statsEpoch2 = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic statsEpoch2 = loadMemoryStatFromFolder(exportPath); - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); monitoringDatastore.edit( tm -> { - new AnalysisDatastoreFeeder("appAInit").loadWithTransaction(tm, Stream.of(stats)); - new AnalysisDatastoreFeeder("appAEpoch2").loadWithTransaction(tm, Stream.of(statsEpoch2)); + new AnalysisDatastoreFeeder("appAInit", monitoringDatastore) + .loadWithTransaction(tm, Stream.of(stats)); + new AnalysisDatastoreFeeder("appAEpoch2", monitoringDatastore) + .loadWithTransaction(tm, Stream.of(statsEpoch2)); }); // Verify that chunkIds are the same for the two dumps by checking that the Ids are there twice @@ -647,14 +651,14 @@ public void testCompareDumpsOfDifferentEpochs() { @Test public void testCompareDumpsOfDifferentApps() throws AgentException { // Create First App - final ApplicationInTests monitoredApp = createMicroApplication(); - IDatastore datastore = (IDatastore) monitoredApp.getDatabase(); + final ApplicationInTests monitoredApp = createMicroApplication(); + IInternalDatastore datastore = monitoredApp.getDatabase(); final IMemoryAnalysisService analysisService = createService(datastore, monitoredApp.getManager()); datastore.edit(tm -> IntStream.range(0, 100).forEach(i -> tm.add("A", i * i))); // Export first app Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); // Create second app final ApplicationInTests monitoredAppWithBitmap = createMicroApplicationWithLeafBitmap(); @@ -666,12 +670,13 @@ public void testCompareDumpsOfDifferentApps() throws AgentException { // Export second app exportPath = analysisServiceWithBitmap.exportMostRecentVersion("testLoadDatastoreStats"); - final IMemoryStatistic statsWithBitmap = loadMemoryStatFromFolder(exportPath); - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final AMemoryStatistic statsWithBitmap = loadMemoryStatFromFolder(exportPath); + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); monitoringDatastore.edit( tm -> { - new AnalysisDatastoreFeeder("App").loadWithTransaction(tm, Stream.of(stats)); - new AnalysisDatastoreFeeder("AppWithBitmap") + new AnalysisDatastoreFeeder("App", monitoringDatastore) + .loadWithTransaction(tm, Stream.of(stats)); + new AnalysisDatastoreFeeder("AppWithBitmap", monitoringDatastore) .loadWithTransaction(tm, Stream.of(statsWithBitmap)); }); @@ -744,8 +749,8 @@ public void testChunkToReferencesDatastoreContentWithReference() throws AgentExc })); // Export first app Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); - final IDatastore monitoringDatastore = createAnalysisDatastore(); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final IInternalDatastore monitoringDatastore = createAnalysisDatastore(); ATestMemoryStatistic.feedMonitoringApplication(monitoringDatastore, List.of(stats), "App"); ListQuery query = @@ -792,7 +797,7 @@ public void testBlocksOfSingleVectors() { createService(monitoredDatastore, monitoredManager); final Path exportPath = analysisService.exportMostRecentVersion("testBlocksOfSingleVectors"); - final Collection datastoreStats = + final Collection datastoreStats = loadDatastoreMemoryStatFromFolder(exportPath); final IDatastore monitoringDatastore = assertLoadsCorrectly(datastoreStats, getClass()); @@ -857,8 +862,8 @@ private Set retrieveClassesOfChunks( // TODO Test content of all stores similarly - private List collectStatistics( - final IMemoryStatistic root, final List> predicates) { + private List collectStatistics( + final AMemoryStatistic root, final List> predicates) { return predicates.stream() .reduce( List.of(root), diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticLoading.java b/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticLoading.java index 369ac692..85208a87 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticLoading.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticLoading.java @@ -8,19 +8,20 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; +import com.activeviam.activepivot.server.intf.api.observability.IMemoryAnalysisService; +import com.activeviam.database.api.conditions.BaseConditions; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; import com.activeviam.database.api.schema.FieldPath; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.internal.monitoring.AMemoryStatisticWithPredicate; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; -import com.qfs.condition.impl.BaseConditions; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.monitoring.statistic.memory.MemoryStatisticConstants; -import com.qfs.monitoring.statistic.memory.visitor.impl.AMemoryStatisticWithPredicate; -import com.qfs.service.monitoring.IMemoryAnalysisService; -import com.qfs.store.IDatastore; -import com.qfs.store.query.ICursor; -import com.quartetfs.fwk.util.impl.TruePredicate; +import com.activeviam.tech.core.internal.util.TruePredicate; +import com.activeviam.tech.observability.api.memory.IMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; +import com.activeviam.tech.records.api.ICursor; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; @@ -46,7 +47,7 @@ public void testLoadDatastoreStats() { final IMemoryAnalysisService analysisService = createService(monitoredDatastore, monitoredManager); final Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final Collection storeStats = + final Collection storeStats = loadDatastoreMemoryStatFromFolder(exportPath); assertNotEquals(0, storeStats.size()); assertLoadsCorrectly(storeStats, getClass()); @@ -64,7 +65,7 @@ public void doTestLoadMonitoringDatastoreWithVectors(boolean duplicateVectors) { final Path exportPath = analysisService.exportMostRecentVersion( "doTestLoadMonitoringDatastoreWithVectors[" + duplicateVectors + "]"); - final Collection datastoreStats = + final Collection datastoreStats = loadDatastoreMemoryStatFromFolder(exportPath); final IDatastore monitoringDatastore = assertLoadsCorrectly(datastoreStats, getClass()); @@ -130,7 +131,7 @@ public void testLoadPivotStats() { final IMemoryAnalysisService analysisService = createService(monitoredDatastore, monitoredManager); final Path exportPath = analysisService.exportMostRecentVersion("testLoadPivotStats"); - final Collection pivotStats = loadPivotMemoryStatFromFolder(exportPath); + final Collection pivotStats = loadPivotMemoryStatFromFolder(exportPath); assertNotEquals(0, pivotStats.size()); assertLoadsCorrectly(pivotStats, getClass()); }); @@ -145,7 +146,7 @@ public void testLoadFullStats() { final IMemoryAnalysisService analysisService = createService(monitoredDatastore, monitoredManager); final Path exportPath = analysisService.exportMostRecentVersion("testLoadFullStats"); - final IMemoryStatistic fullStats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic fullStats = loadMemoryStatFromFolder(exportPath); assertNotEquals(null, fullStats); assertLoadsCorrectly(fullStats); }); @@ -167,7 +168,7 @@ public void testLoadFullStatsWithBranches() { final IMemoryAnalysisService analysisService = createService(monitoredDatastore, monitoredManager); final Path exportPath = analysisService.exportBranches("testLoadFullStats", branchSet); - final IMemoryStatistic fullStats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic fullStats = loadMemoryStatFromFolder(exportPath); assertNotEquals(null, fullStats); assertLoadsCorrectly(fullStats); }); @@ -188,7 +189,7 @@ public void testLoadFullStatsWithEpochs() { final IMemoryAnalysisService analysisService = createService(monitoredDatastore, monitoredManager); final Path exportPath = analysisService.exportVersions("testLoadFullStats", epochs); - final IMemoryStatistic fullStats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic fullStats = loadMemoryStatFromFolder(exportPath); assertNotEquals(null, fullStats); assertLoadsCorrectly(fullStats); }); @@ -208,7 +209,7 @@ public void testLoadMonitoringDatastoreWithDuplicate() throws Exception { * Asserts the chunks number and off-heap memory as computed from the loaded datastore are * consistent with the ones computed by visiting the statistic. */ - protected void assertLoadsCorrectly(IMemoryStatistic statistic) { + protected void assertLoadsCorrectly(AMemoryStatistic statistic) { assertLoadsCorrectly(Collections.singleton(statistic), getClass()); } } diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticLoadingWithReverseOrder.java b/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticLoadingWithReverseOrder.java index 91dbc6a3..57880863 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticLoadingWithReverseOrder.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticLoadingWithReverseOrder.java @@ -7,16 +7,16 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.server.intf.api.observability.IMemoryAnalysisService; +import com.activeviam.database.api.conditions.BaseConditions; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; import com.activeviam.database.api.schema.FieldPath; +import com.activeviam.database.datastore.api.IDatastore; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; -import com.qfs.condition.impl.BaseConditions; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.service.monitoring.IMemoryAnalysisService; -import com.qfs.store.IDatastore; -import com.qfs.store.query.ICursor; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.records.api.ICursor; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -39,7 +39,7 @@ public void testLoadDatastoreStats() { final IMemoryAnalysisService analysisService = createService(monitoredDatastore, monitoredManager); final Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final List storeStats = + final List storeStats = new ArrayList<>(loadMemoryStatFromFolder(exportPath).getChildren()); Collections.shuffle(storeStats); @@ -59,7 +59,7 @@ public void testLoadDatastoreStatsWithVectors() { final IMemoryAnalysisService analysisService = createService(monitoredDatastore, monitoredManager); final Path exportPath = analysisService.exportMostRecentVersion("testLoadDatastoreStats"); - final List storeStats = + final List storeStats = new ArrayList<>(loadMemoryStatFromFolder(exportPath).getChildren()); Collections.shuffle(storeStats); diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestOverviewBookmark.java b/src/test/java/com/activeviam/mac/statistic/memory/TestOverviewBookmark.java index d8a94009..2eb4887e 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestOverviewBookmark.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestOverviewBookmark.java @@ -1,21 +1,21 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.api.query.MDXQuery; +import com.activeviam.activepivot.server.impl.api.query.MdxQueryUtil; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.monitoring.MemoryStatisticsTestUtils; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils.StatisticsSummary; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.biz.pivot.dto.CellSetDTO; -import com.quartetfs.biz.pivot.query.impl.MDXQuery; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import com.quartetfs.fwk.query.QueryException; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.query.QueryException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; import java.util.stream.IntStream; @@ -28,13 +28,13 @@ public class TestOverviewBookmark extends ATestMemoryStatistic { public static final int ADDED_DATA_SIZE = 20; - private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; - private StatisticsSummary summary; + private ApplicationInTests monitoredApp; + private ApplicationInTests monitoringApp; + private MemoryStatisticsTestUtils.StatisticsSummary summary; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -43,7 +43,7 @@ public void setup() throws AgentException { final Path exportPath = generateMemoryStatistics(); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); this.summary = MemoryStatisticsTestUtils.getStatisticsSummary(stats); initializeMonitoringApplication(stats); @@ -75,7 +75,7 @@ private Path generateMemoryStatistics() { return analysisService.exportMostRecentVersion("testOverview"); } - private void initializeMonitoringApplication(final IMemoryStatistic data) throws AgentException { + private void initializeMonitoringApplication(final AMemoryStatistic data) throws AgentException { ManagerDescriptionConfig config = new ManagerDescriptionConfig(); final IDatastoreSchemaDescriptionConfig schemaConfig = new MemoryAnalysisDatastoreDescriptionConfig(); @@ -99,16 +99,11 @@ public void tearDown() throws AgentException { @Test public void testOverviewGrandTotal() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery totalQuery = new MDXQuery("SELECT NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS FROM [MemoryCube]"); - final CellSetDTO totalResult = pivot.execute(totalQuery); + final CellSetDTO totalResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), totalQuery); Assertions.assertThat(CellSetUtils.extractValueFromSingleCellDTO(totalResult)) .isEqualTo(this.summary.offHeapMemory); @@ -116,12 +111,6 @@ public void testOverviewGrandTotal() throws QueryException { @Test public void testOwnerTotal() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery totalQuery = new MDXQuery("SELECT NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS FROM [MemoryCube]"); @@ -143,9 +132,12 @@ public void testOwnerTotal() throws QueryException { + " SELECT [Measures].[ExcessDirectMemory] ON COLUMNS" + " FROM [MemoryCube]"); - final CellSetDTO totalResult = pivot.execute(totalQuery); - final CellSetDTO perOwnerResult = pivot.execute(perOwnerQuery); - final CellSetDTO excessMemoryResult = pivot.execute(excessMemoryQuery); + final CellSetDTO totalResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), totalQuery); + final CellSetDTO perOwnerResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), perOwnerQuery); + final CellSetDTO excessMemoryResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), excessMemoryQuery); Assertions.assertThat( CellSetUtils.sumValuesFromCellSetDTO(perOwnerResult) @@ -155,12 +147,6 @@ public void testOwnerTotal() throws QueryException { @Test public void testStoreTotal() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery storeTotalQuery = new MDXQuery( "SELECT NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS " @@ -187,9 +173,12 @@ public void testStoreTotal() throws QueryException { + " FROM [MemoryCube]" + " WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); - final CellSetDTO storeTotalResult = pivot.execute(storeTotalQuery); - final CellSetDTO perComponentStoreResult = pivot.execute(perComponentsStoreQuery); - final CellSetDTO excessMemoryResult = pivot.execute(excessMemoryQuery); + final CellSetDTO storeTotalResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), storeTotalQuery); + final CellSetDTO perComponentStoreResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), perComponentsStoreQuery); + final CellSetDTO excessMemoryResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), excessMemoryQuery); Assertions.assertThat( CellSetUtils.sumValuesFromCellSetDTO(perComponentStoreResult) @@ -199,12 +188,6 @@ public void testStoreTotal() throws QueryException { @Test public void testCubeTotal() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery cubeTotalQuery = new MDXQuery( "SELECT NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS " @@ -231,9 +214,12 @@ public void testCubeTotal() throws QueryException { + " FROM [MemoryCube]" + " WHERE [Owners].[Owner].[ALL].[AllMember].[Cube Cube]"); - final CellSetDTO cubeTotalResult = pivot.execute(cubeTotalQuery); - final CellSetDTO perComponentCubeResult = pivot.execute(perComponentCubeQuery); - final CellSetDTO excessMemoryResult = pivot.execute(excessMemoryQuery); + final CellSetDTO cubeTotalResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), cubeTotalQuery); + final CellSetDTO perComponentCubeResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), perComponentCubeQuery); + final CellSetDTO excessMemoryResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), excessMemoryQuery); Assertions.assertThat( CellSetUtils.sumValuesFromCellSetDTO(perComponentCubeResult) diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java b/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java index fdd25ac0..f62d3ec5 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java @@ -1,19 +1,20 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.api.query.MDXQuery; +import com.activeviam.activepivot.server.impl.api.query.MdxQueryUtil; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.internal.IInternalDatastore; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.biz.pivot.dto.CellSetDTO; -import com.quartetfs.biz.pivot.query.impl.MDXQuery; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import com.quartetfs.fwk.query.QueryException; +import com.activeviam.tech.core.api.query.QueryException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; import java.util.stream.IntStream; @@ -26,15 +27,15 @@ public class TestRatioMeasures extends ATestMemoryStatistic { public static final int ADDED_DATA_SIZE = 20; private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; + private ApplicationInTests monitoringApp; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach - public void setup() throws AgentException { + public void setup() { this.monitoredApp = createMicroApplicationWithIsolatedStoreAndKeepAllEpochPolicy(); this.monitoredApp @@ -58,7 +59,7 @@ public void setup() throws AgentException { createService(this.monitoredApp.getDatabase(), this.monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testOverview"); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); // Start a monitoring datastore with the exported data final ManagerDescriptionConfig config = new ManagerDescriptionConfig(); @@ -87,12 +88,6 @@ public void setup() throws AgentException { @Test public void testDirectMemoryRatio() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery totalDirectMemory = new MDXQuery( "SELECT NON EMPTY [Measures].[DirectMemory.SUM] ON COLUMNS " @@ -111,9 +106,12 @@ public void testDirectMemoryRatio() throws QueryException { + "FROM [MemoryCube]" + "WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); - final CellSetDTO total = pivot.execute(totalDirectMemory); - final CellSetDTO storeA = pivot.execute(storeADirectMemory); - final CellSetDTO ratio = pivot.execute(storeADirectMemoryRatio); + final CellSetDTO total = + MdxQueryUtil.execute(this.monitoringApp.getManager(), totalDirectMemory); + final CellSetDTO storeA = + MdxQueryUtil.execute(this.monitoringApp.getManager(), storeADirectMemory); + final CellSetDTO ratio = + MdxQueryUtil.execute(this.monitoringApp.getManager(), storeADirectMemoryRatio); Assertions.assertThat(CellSetUtils.extractDoubleValueFromSingleCellDTO(ratio).doubleValue()) .isEqualTo(0.5D); @@ -125,12 +123,6 @@ public void testDirectMemoryRatio() throws QueryException { @Test public void testCommittedRowsRatio() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - final MDXQuery storeAcommittedRows = new MDXQuery( "SELECT NON EMPTY [Measures].[Used rows] ON COLUMNS " @@ -149,9 +141,12 @@ public void testCommittedRowsRatio() throws QueryException { + "FROM [MemoryCube]" + "WHERE [Owners].[Owner].[ALL].[AllMember].[Store A]"); - final CellSetDTO committedRows = pivot.execute(storeAcommittedRows); - final CellSetDTO chunkSize = pivot.execute(storeAchunkSize); - final CellSetDTO ratio = pivot.execute(storeAcommittedRowsRatio); + final CellSetDTO committedRows = + MdxQueryUtil.execute(this.monitoringApp.getManager(), storeAcommittedRows); + final CellSetDTO chunkSize = + MdxQueryUtil.execute(this.monitoringApp.getManager(), storeAchunkSize); + final CellSetDTO ratio = + MdxQueryUtil.execute(this.monitoringApp.getManager(), storeAcommittedRowsRatio); Assertions.assertThat(CellSetUtils.extractDoubleValueFromSingleCellDTO(ratio).doubleValue()) .isNotIn(0D, 1D); diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestVectorBlockBookmark.java b/src/test/java/com/activeviam/mac/statistic/memory/TestVectorBlockBookmark.java index 1ba6fa72..b011c94e 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestVectorBlockBookmark.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestVectorBlockBookmark.java @@ -7,26 +7,27 @@ package com.activeviam.mac.statistic.memory; +import static org.assertj.core.api.Assertions.assertThat; + +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.api.query.MDXQuery; +import com.activeviam.activepivot.server.impl.api.query.MdxQueryUtil; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.monitoring.MemoryStatisticsTestUtils; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils; -import com.qfs.monitoring.offheap.MemoryStatisticsTestUtils.StatisticsSummary; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.biz.pivot.dto.CellSetDTO; -import com.quartetfs.biz.pivot.query.impl.MDXQuery; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import com.quartetfs.fwk.query.QueryException; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.query.QueryException; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; import java.util.stream.IntStream; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -37,16 +38,16 @@ public class TestVectorBlockBookmark extends ATestMemoryStatistic { public static final int ADDED_DATA_SIZE = 40; public static final int FIELD_SHARING_COUNT = 2; private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; - StatisticsSummary summary; + private ApplicationInTests monitoringApp; + MemoryStatisticsTestUtils.StatisticsSummary summary; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach - public void setup() throws AgentException { + public void setup() { this.monitoredApp = createMicroApplicationWithSharedVectorField(); this.monitoredApp @@ -57,7 +58,7 @@ public void setup() throws AgentException { .forEach(i -> tm.add("A", i * i, new double[] {i}, new double[] {-i, -i * i}))); // Force to discard all versions - this.monitoredApp.getDatabase().getEpochManager().forceDiscardEpochs(__ -> true); + this.monitoredApp.getDatabase().getEpochManager().forceDiscardEpochs(epoch -> true); // perform GCs before exporting the store data performGC(); @@ -66,7 +67,7 @@ public void setup() throws AgentException { createService(this.monitoredApp.getDatabase(), this.monitoredApp.getManager()); final Path exportPath = analysisService.exportMostRecentVersion("testOverview"); - final IMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); + final AMemoryStatistic stats = loadMemoryStatFromFolder(exportPath); this.summary = MemoryStatisticsTestUtils.getStatisticsSummary(stats); // Start a monitoring datastore with the exported data @@ -91,7 +92,7 @@ public void setup() throws AgentException { .getManager() .getActivePivots() .get(ManagerDescriptionConfig.MONITORING_CUBE); - Assertions.assertThat(pivot).isNotNull(); + assertThat(pivot).isNotNull(); } @AfterEach @@ -101,13 +102,7 @@ public void tearDown() throws AgentException { } @Test - public void testVectorBlockRecordConsumptionIsZero() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - + void testVectorBlockRecordConsumptionIsZero() throws QueryException { final MDXQuery recordQuery = new MDXQuery( "SELECT [Components].[Component].[Component].[RECORDS] ON ROWS," @@ -118,19 +113,13 @@ public void testVectorBlockRecordConsumptionIsZero() throws QueryException { + " [Fields].[Field].[Field].[vector1]" + " )"); - final CellSetDTO result = pivot.execute(recordQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), recordQuery); - Assertions.assertThat((long) result.getCells().get(0).getValue()).isZero(); + assertThat((long) result.getCells().get(0).getValue()).isZero(); } @Test - public void testVectorBlockConsumption() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - + void testVectorBlockConsumption() throws QueryException { final MDXQuery vectorBlockQueryField1 = new MDXQuery( "SELECT {" @@ -144,7 +133,8 @@ public void testVectorBlockConsumption() throws QueryException { + " [Fields].[Field].[Field].[vector1]" + " )"); - final CellSetDTO result1 = pivot.execute(vectorBlockQueryField1); + final CellSetDTO result1 = + MdxQueryUtil.execute(this.monitoringApp.getManager(), vectorBlockQueryField1); // Vector1 and Vector2 data are intertwined in the same blocks, so the direct memory is only // partially used @@ -153,9 +143,8 @@ public void testVectorBlockConsumption() throws QueryException { ((ADDED_DATA_SIZE * 2 + ADDED_DATA_SIZE) / MICROAPP_VECTOR_BLOCK_SIZE) + 1; final var directMemoryUsedOnVector = blockCountUsed * MICROAPP_VECTOR_BLOCK_SIZE * Double.BYTES; - Assertions.assertThat((long) result1.getCells().get(1).getValue()) - .isEqualTo(directMemoryUsedOnVector); - Assertions.assertThat((long) result1.getCells().get(1).getValue()) + assertThat((long) result1.getCells().get(1).getValue()) + .isEqualTo(directMemoryUsedOnVector) .isEqualTo((long) result1.getCells().get(0).getValue()); final MDXQuery vectorBlockQueryField2 = @@ -171,22 +160,16 @@ public void testVectorBlockConsumption() throws QueryException { + " [Fields].[Field].[Field].[vector2]" + " )"); - final CellSetDTO result2 = pivot.execute(vectorBlockQueryField2); + final CellSetDTO result2 = + MdxQueryUtil.execute(this.monitoringApp.getManager(), vectorBlockQueryField2); - Assertions.assertThat((long) result2.getCells().get(1).getValue()) - .isEqualTo(directMemoryUsedOnVector); - Assertions.assertThat((long) result2.getCells().get(1).getValue()) + assertThat((long) result2.getCells().get(1).getValue()) + .isEqualTo(directMemoryUsedOnVector) .isEqualTo((long) result2.getCells().get(0).getValue()); } @Test - public void testVectorBlockLength() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - + void testVectorBlockLength() throws QueryException { final MDXQuery lengthQuery = new MDXQuery( "SELECT [Measures].[VectorBlock.Length] ON COLUMNS" @@ -196,20 +179,14 @@ public void testVectorBlockLength() throws QueryException { + " [Fields].[Field].[Field].[vector1]" + " )"); - final CellSetDTO result = pivot.execute(lengthQuery); + final CellSetDTO result = MdxQueryUtil.execute(this.monitoringApp.getManager(), lengthQuery); - Assertions.assertThat(CellSetUtils.extractValueFromSingleCellDTO(result)) + assertThat(CellSetUtils.extractValueFromSingleCellDTO(result)) .isEqualTo(MICROAPP_VECTOR_BLOCK_SIZE); } @Test - public void testVectorBlockRefCount() throws QueryException { - final IMultiVersionActivePivot pivot = - this.monitoringApp - .getManager() - .getActivePivots() - .get(ManagerDescriptionConfig.MONITORING_CUBE); - + void testVectorBlockRefCount() throws QueryException { final MDXQuery refCountQuery = new MDXQuery( "SELECT [Measures].[VectorBlock.RefCount] ON COLUMNS" @@ -219,9 +196,10 @@ public void testVectorBlockRefCount() throws QueryException { + " [Fields].[Field].[Field].[vector1]" + " )"); - final CellSetDTO refCountResult = pivot.execute(refCountQuery); + final CellSetDTO refCountResult = + MdxQueryUtil.execute(this.monitoringApp.getManager(), refCountQuery); - Assertions.assertThat(CellSetUtils.extractValueFromSingleCellDTO(refCountResult)) + assertThat(CellSetUtils.extractValueFromSingleCellDTO(refCountResult)) .isEqualTo(FIELD_SHARING_COUNT * ADDED_DATA_SIZE); } } diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestViewEpochs.java b/src/test/java/com/activeviam/mac/statistic/memory/TestViewEpochs.java index 7e56cc47..8ef9feca 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestViewEpochs.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestViewEpochs.java @@ -7,30 +7,30 @@ package com.activeviam.mac.statistic.memory; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IMultiVersionActivePivot; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.api.transaction.DatastoreTransactionException; +import com.activeviam.database.datastore.api.transaction.ITransactionManager; +import com.activeviam.database.datastore.internal.IInternalDatastore; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.entities.ChunkOwner; import com.activeviam.mac.entities.CubeOwner; import com.activeviam.mac.entities.StoreOwner; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.statistic.memory.visitor.impl.EpochView; -import com.activeviam.pivot.utils.ApplicationInTests; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.records.api.ICursor; +import com.activeviam.tech.records.api.IRecordReader; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.server.cfg.IDatastoreSchemaDescriptionConfig; -import com.qfs.store.IDatastore; -import com.qfs.store.query.ICursor; -import com.qfs.store.record.IRecordReader; -import com.qfs.store.transaction.DatastoreTransactionException; -import com.qfs.store.transaction.ITransactionManager; -import com.quartetfs.biz.pivot.IMultiVersionActivePivot; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; @@ -48,12 +48,12 @@ public class TestViewEpochs extends ATestMemoryStatistic { private ApplicationInTests monitoredApp; - private ApplicationInTests monitoringApp; - private IMemoryStatistic statistics; + private ApplicationInTests monitoringApp; + private AMemoryStatistic statistics; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -111,7 +111,7 @@ private Path generateMemoryStatistics() { return analysisService.exportApplication("testEpochs"); } - private void initializeMonitoringApplication(final IMemoryStatistic data) throws AgentException { + private void initializeMonitoringApplication(final AMemoryStatistic data) throws AgentException { final ManagerDescriptionConfig config = new ManagerDescriptionConfig(); final IDatastoreSchemaDescriptionConfig schemaConfig = new MemoryAnalysisDatastoreDescriptionConfig(); diff --git a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestBitmapAggregateProviderWithVectorPrimitiveMeasure.java b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestBitmapAggregateProviderWithVectorPrimitiveMeasure.java index ee3d2169..e5b76763 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestBitmapAggregateProviderWithVectorPrimitiveMeasure.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestBitmapAggregateProviderWithVectorPrimitiveMeasure.java @@ -7,26 +7,27 @@ package com.activeviam.mac.statistic.memory.scenarios; -import com.activeviam.builders.StartBuilding; -import com.activeviam.fwk.ActiveViamRuntimeException; +import com.activeviam.activepivot.core.datastore.api.builder.StartBuilding; +import com.activeviam.activepivot.core.impl.internal.monitoring.memory.impl.OnHeapPivotMemoryQuantifierPlugin; +import com.activeviam.activepivot.core.impl.internal.test.util.PivotTestUtils; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IActivePivotManager; +import com.activeviam.activepivot.core.intf.api.description.IActivePivotManagerDescription; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryStatisticSerializerUtil; +import com.activeviam.activepivot.server.intf.api.observability.IMemoryAnalysisService; +import com.activeviam.database.api.types.ILiteralType; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.api.description.IDatastoreSchemaDescription; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.builder.impl.UnitTestDatastoreBuilder; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.statistic.memory.ATestMemoryStatistic; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.desc.IDatastoreSchemaDescription; -import com.qfs.junit.LocalResourcesExtension; -import com.qfs.literal.ILiteralType; -import com.qfs.monitoring.memory.impl.OnHeapPivotMemoryQuantifierPlugin; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.pivot.monitoring.impl.MemoryStatisticSerializerUtil; -import com.qfs.service.monitoring.IMemoryAnalysisService; -import com.qfs.store.IDatastore; -import com.qfs.store.build.impl.UnitTestDatastoreBuilder; -import com.qfs.util.impl.QfsFileTestUtils; -import com.quartetfs.biz.pivot.IActivePivotManager; -import com.quartetfs.biz.pivot.definitions.IActivePivotManagerDescription; -import com.quartetfs.biz.pivot.test.util.PivotTestUtils; -import com.quartetfs.fwk.AgentException; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.test.internal.junit.resources.ResourcesExtension; +import com.activeviam.tech.test.internal.util.FileTestUtil; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -36,19 +37,18 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.extension.ExtendWith; /** * The scenario this test produces created a situation where the dictionary used by the point index * of the aggregate store of the cube would leak into subsequent chunks. */ +@ExtendWith({ResourcesExtension.class}) public class TestBitmapAggregateProviderWithVectorPrimitiveMeasure extends ATestMemoryStatistic { protected static final Path TEMP_DIRECTORY = - QfsFileTestUtils.createTempDirectory( - TestBitmapAggregateProviderWithVectorPrimitiveMeasure.class); + FileTestUtil.createTempDirectory(TestBitmapAggregateProviderWithVectorPrimitiveMeasure.class); protected static final int RECORD_COUNT = 100; - @RegisterExtension protected LocalResourcesExtension resources = new LocalResourcesExtension(); protected IDatastore datastore; protected IActivePivotManager manager; protected Path statisticsPath; @@ -133,24 +133,25 @@ protected void exportApplicationMemoryStatistics() { @Test public void testLoading() throws IOException { - final Collection memoryStatistics = loadMemoryStatistic(this.statisticsPath); + final Collection memoryStatistics = loadMemoryStatistic(this.statisticsPath); - final IDatastore analysisDatastore = createAnalysisDatastore(); + final IInternalDatastore analysisDatastore = createAnalysisDatastore(); Assertions.assertDoesNotThrow( () -> ATestMemoryStatistic.feedMonitoringApplication( analysisDatastore, memoryStatistics, "test")); } - protected IDatastore createAnalysisDatastore() { + protected IInternalDatastore createAnalysisDatastore() { final IDatastoreSchemaDescription desc = new MemoryAnalysisDatastoreDescriptionConfig().datastoreSchemaDescription(); - final IDatastore datastore = new UnitTestDatastoreBuilder().setSchemaDescription(desc).build(); + final IInternalDatastore datastore = + new UnitTestDatastoreBuilder().setSchemaDescription(desc).build(); this.resources.register(datastore); return datastore; } - protected Collection loadMemoryStatistic(final Path path) throws IOException { + protected Collection loadMemoryStatistic(final Path path) throws IOException { return Files.list(path) .map( file -> { diff --git a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMissingChunkId.java b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMissingChunkId.java index 667cb767..bff4dde3 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMissingChunkId.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMissingChunkId.java @@ -7,22 +7,24 @@ package com.activeviam.mac.statistic.memory.scenarios; -import com.activeviam.fwk.ActiveViamRuntimeException; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.description.IActivePivotManagerDescription; +import com.activeviam.activepivot.server.impl.api.query.MDXQuery; +import com.activeviam.activepivot.server.impl.api.query.MdxQueryUtil; +import com.activeviam.activepivot.server.intf.api.dto.CellSetDTO; +import com.activeviam.database.datastore.api.description.IDatastoreSchemaDescription; +import com.activeviam.database.datastore.internal.IInternalDatastore; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.statistic.memory.ATestMemoryStatistic; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.desc.IDatastoreSchemaDescription; -import com.qfs.junit.LocalResourcesExtension; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryStatisticSerializerUtil; -import com.qfs.store.IDatastore; -import com.quartetfs.biz.pivot.definitions.IActivePivotManagerDescription; -import com.quartetfs.biz.pivot.dto.CellSetDTO; -import com.quartetfs.biz.pivot.query.impl.MDXQuery; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; -import com.quartetfs.fwk.query.QueryException; +import com.activeviam.mac.statistic.memory.deserializer.RetroCompatibleDeserializer; +import com.activeviam.tech.core.api.query.QueryException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.test.internal.junit.resources.Resources; +import com.activeviam.tech.test.internal.junit.resources.ResourcesExtension; +import com.activeviam.tech.test.internal.junit.resources.ResourcesHolder; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -33,18 +35,19 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.extension.ExtendWith; /** The scenario this test produces has a cube level {@code id} with a nullable dictionary. */ +@ExtendWith({ResourcesExtension.class}) public class TestMissingChunkId { - @RegisterExtension protected LocalResourcesExtension resources = new LocalResourcesExtension(); - private Collection memoryStatistics; + @Resources public ResourcesHolder resources; + private Collection memoryStatistics; private final ApplicationInTests analysisApplication = createAnalysisApplication();; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -64,27 +67,23 @@ protected ApplicationInTests createAnalysisApplication() { return ApplicationInTests.builder().withDatastore(desc).withManager(manager).build(); } - protected Collection loadMemoryStatistic(final Path path) throws IOException { + protected Collection loadMemoryStatistic(final Path path) throws IOException { return Files.list(path) - .map( - file -> { - try { - return MemoryStatisticSerializerUtil.readStatisticFile(file.toFile()); - } catch (IOException exception) { - throw new ActiveViamRuntimeException(exception); - } - }) + .map(Path::toFile) + .map(RetroCompatibleDeserializer::readStatistic) .collect(Collectors.toList()); } protected void loadStatisticsIntoDatastore( - final Collection statistics, final IDatastore analysisDatastore) { + final Collection statistics, + final IInternalDatastore analysisDatastore) { ATestMemoryStatistic.feedMonitoringApplication(analysisDatastore, statistics, "test"); } @Test public void testGeneratio0nOfMissingChunkId() throws QueryException { - loadStatisticsIntoDatastore(memoryStatistics, (IDatastore) analysisApplication.getDatabase()); + loadStatisticsIntoDatastore( + memoryStatistics, (IInternalDatastore) analysisApplication.getDatabase()); final MDXQuery query = new MDXQuery( @@ -100,7 +99,7 @@ public void testGeneratio0nOfMissingChunkId() throws QueryException { + " ) ON ROWS" + " FROM [MemoryCube]"); - final CellSetDTO totalResult = analysisApplication.getSingleCube().execute(query); + final CellSetDTO totalResult = MdxQueryUtil.execute(analysisApplication.getManager(), query); final List negativeChunkIds = totalResult.getAxes().get(0).getPositions().stream() .map(p -> p.getMembers().get(0).getCaption()) @@ -112,7 +111,8 @@ public void testGeneratio0nOfMissingChunkId() throws QueryException { @Test public void testStatisticLoading() { - final IDatastore analysisDatastore = (IDatastore) analysisApplication.getDatabase(); + final IInternalDatastore analysisDatastore = + (IInternalDatastore) analysisApplication.getDatabase(); Assertions.assertDoesNotThrow( () -> loadStatisticsIntoDatastore(memoryStatistics, analysisDatastore)); diff --git a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMultipleFieldsDictionary.java b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMultipleFieldsDictionary.java index 85ab45a2..181aa800 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMultipleFieldsDictionary.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMultipleFieldsDictionary.java @@ -7,39 +7,39 @@ package com.activeviam.mac.statistic.memory.scenarios; -import com.activeviam.builders.StartBuilding; +import com.activeviam.activepivot.core.datastore.api.builder.StartBuilding; +import com.activeviam.activepivot.core.impl.api.description.impl.ActivePivotManagerDescription; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IActivePivotManager; +import com.activeviam.activepivot.core.intf.api.description.IActivePivotManagerDescription; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryStatisticSerializerUtil; +import com.activeviam.activepivot.server.intf.api.observability.IMemoryAnalysisService; +import com.activeviam.database.api.conditions.BaseConditions; import com.activeviam.database.api.query.AliasedField; import com.activeviam.database.api.query.ListQuery; import com.activeviam.database.api.schema.FieldPath; -import com.activeviam.fwk.ActiveViamRuntimeException; +import com.activeviam.database.api.types.ILiteralType; +import com.activeviam.database.datastore.api.description.IDatastoreSchemaDescription; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.database.datastore.internal.dictionary.ISchemaDictionaryProvider; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.entities.ChunkOwner; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; import com.activeviam.mac.statistic.memory.ATestMemoryStatistic; -import com.activeviam.pivot.utils.ApplicationInTests; +import com.activeviam.tech.core.api.agent.AgentException; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.records.api.ICursor; +import com.activeviam.tech.records.api.IRecordReader; +import com.activeviam.tech.test.internal.junit.resources.ResourcesExtension; +import com.activeviam.tech.test.internal.util.FileTestUtil; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import com.qfs.condition.impl.BaseConditions; -import com.qfs.desc.IDatastoreSchemaDescription; -import com.qfs.dic.ISchemaDictionaryProvider; -import com.qfs.junit.LocalResourcesExtension; -import com.qfs.literal.ILiteralType; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.pivot.monitoring.impl.MemoryStatisticSerializerUtil; -import com.qfs.service.monitoring.IMemoryAnalysisService; -import com.qfs.store.IDatastore; -import com.qfs.store.query.ICursor; -import com.qfs.store.record.IRecordReader; -import com.qfs.util.impl.QfsFileTestUtils; -import com.quartetfs.biz.pivot.IActivePivotManager; -import com.quartetfs.biz.pivot.definitions.IActivePivotManagerDescription; -import com.quartetfs.biz.pivot.definitions.impl.ActivePivotManagerDescription; -import com.quartetfs.fwk.AgentException; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -54,21 +54,21 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.extension.ExtendWith; +@ExtendWith({ResourcesExtension.class}) public class TestMultipleFieldsDictionary extends ATestMemoryStatistic { protected static final Path TEMP_DIRECTORY = - QfsFileTestUtils.createTempDirectory(TestMultipleFieldsDictionary.class); - @RegisterExtension protected LocalResourcesExtension resources = new LocalResourcesExtension(); - protected IDatastore monitoredDatastore; + FileTestUtil.createTempDirectory(TestMultipleFieldsDictionary.class); + protected IInternalDatastore monitoredDatastore; protected IActivePivotManager monitoredManager; protected Path statisticsPath; - protected IDatastore monitoringDatastore; + protected IInternalDatastore monitoringDatastore; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -76,7 +76,7 @@ public void setupAndExportApplication() throws AgentException, IOException { final IDatastoreSchemaDescription datastoreSchemaDescription = datastoreSchema(); final IActivePivotManagerDescription managerDescription = new ActivePivotManagerDescription(); - final ApplicationInTests application = + final ApplicationInTests application = ApplicationInTests.builder() .withDatastore(datastoreSchemaDescription) .withManager(managerDescription) @@ -151,16 +151,16 @@ protected void exportApplicationMemoryStatistics() { this.statisticsPath = analysisService.exportMostRecentVersion("memoryStats"); } - protected IDatastore createAnalysisDatastore() { + protected IInternalDatastore createAnalysisDatastore() { final IDatastoreSchemaDescription desc = new MemoryAnalysisDatastoreDescriptionConfig().datastoreSchemaDescription(); final IActivePivotManagerDescription manager = new ManagerDescriptionConfig().managerDescription(); - return (IDatastore) + return (IInternalDatastore) ApplicationInTests.builder().withDatastore(desc).withManager(manager).build().getDatabase(); } - protected Collection loadMemoryStatistic(final Path path) throws IOException { + protected Collection loadMemoryStatistic(final Path path) throws IOException { return Files.list(path) .map( file -> { @@ -174,7 +174,8 @@ protected Collection loadMemoryStatistic(final Path path) thro } protected void loadStatisticsIntoDatastore( - final Collection statistics, final IDatastore analysisDatastore) { + final Collection statistics, + final IInternalDatastore analysisDatastore) { ATestMemoryStatistic.feedMonitoringApplication(analysisDatastore, statistics, "test"); } diff --git a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestNullableLevelDictionary.java b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestNullableLevelDictionary.java index 6442f8f1..eaa01870 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestNullableLevelDictionary.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestNullableLevelDictionary.java @@ -9,29 +9,32 @@ import static com.activeviam.mac.statistic.memory.ATestMemoryStatistic.performGC; -import com.activeviam.builders.StartBuilding; -import com.activeviam.fwk.ActiveViamRuntimeException; +import com.activeviam.activepivot.core.datastore.api.builder.StartBuilding; +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.core.intf.api.cube.IActivePivotManager; +import com.activeviam.activepivot.core.intf.api.cube.IActivePivotVersion; +import com.activeviam.activepivot.core.intf.api.description.IActivePivotManagerDescription; +import com.activeviam.activepivot.core.intf.api.location.IHierarchicalMapping; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryAnalysisService; +import com.activeviam.activepivot.server.impl.private_.observability.memory.MemoryStatisticSerializerUtil; +import com.activeviam.activepivot.server.intf.api.observability.IMemoryAnalysisService; +import com.activeviam.database.api.types.ILiteralType; +import com.activeviam.database.datastore.api.IDatastore; +import com.activeviam.database.datastore.api.description.IDatastoreSchemaDescription; +import com.activeviam.database.datastore.internal.IInternalDatastore; import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.statistic.memory.ATestMemoryStatistic; -import com.activeviam.pivot.utils.ApplicationInTests; -import com.qfs.desc.IDatastoreSchemaDescription; -import com.qfs.dic.IDictionary; -import com.qfs.dic.impl.NullableDictionary; -import com.qfs.junit.LocalResourcesExtension; -import com.qfs.literal.ILiteralType; -import com.qfs.monitoring.statistic.memory.IMemoryStatistic; -import com.qfs.pivot.monitoring.impl.MemoryAnalysisService; -import com.qfs.pivot.monitoring.impl.MemoryStatisticSerializerUtil; -import com.qfs.service.monitoring.IMemoryAnalysisService; -import com.qfs.store.IDatastore; -import com.qfs.util.impl.QfsFileTestUtils; -import com.quartetfs.biz.pivot.IActivePivotManager; -import com.quartetfs.biz.pivot.IActivePivotVersion; -import com.quartetfs.biz.pivot.cube.provider.IHierarchicalMapping; -import com.quartetfs.biz.pivot.definitions.IActivePivotManagerDescription; -import com.quartetfs.fwk.Registry; -import com.quartetfs.fwk.contributions.impl.ClasspathContributionProvider; +import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; +import com.activeviam.tech.core.api.registry.Registry; +import com.activeviam.tech.dictionaries.api.IDictionary; +import com.activeviam.tech.dictionaries.internal.impl.NullableDictionary; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import com.activeviam.tech.test.internal.junit.resources.Resources; +import com.activeviam.tech.test.internal.junit.resources.ResourcesExtension; +import com.activeviam.tech.test.internal.junit.resources.ResourcesHolder; +import com.activeviam.tech.test.internal.util.FileTestUtil; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -41,22 +44,23 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.extension.ExtendWith; /** The scenario this test produces has a cube level {@code id} with a nullable dictionary. */ +@ExtendWith({ResourcesExtension.class}) public class TestNullableLevelDictionary { protected static final Path TEMP_DIRECTORY = - QfsFileTestUtils.createTempDirectory(TestNullableLevelDictionary.class); + FileTestUtil.createTempDirectory(TestNullableLevelDictionary.class); protected static final int RECORD_COUNT = 10; - @RegisterExtension protected LocalResourcesExtension resources = new LocalResourcesExtension(); + @Resources public ResourcesHolder resources; protected IDatastore datastore; protected IActivePivotManager manager; protected Path statisticsPath; @BeforeAll public static void setupRegistry() { - Registry.setContributionProvider(new ClasspathContributionProvider()); + RegistryInitializationConfig.setupRegistry(); } @BeforeEach @@ -126,7 +130,7 @@ protected void exportApplicationMemoryStatistics() { @Test public void testLevelHasNullableDictionary() { final IActivePivotVersion cube = this.manager.getActivePivots().get("Cube").getHead(); - final IHierarchicalMapping mapping = cube.getAggregateProvider().getHierarchicalMapping(); + final IHierarchicalMapping mapping = cube.getHierarchicalMapping(); final int hierarchyCoord = mapping.getCoordinate(1, 1); final IDictionary dictionary = mapping.getDictionaries()[hierarchyCoord]; @@ -138,23 +142,23 @@ public void testLevelHasNullableDictionary() { @Test public void testStatisticLoading() throws IOException { - final Collection memoryStatistics = loadMemoryStatistic(this.statisticsPath); + final Collection memoryStatistics = loadMemoryStatistic(this.statisticsPath); - final IDatastore analysisDatastore = createAnalysisDatastore(); + final IInternalDatastore analysisDatastore = createAnalysisDatastore(); Assertions.assertDoesNotThrow( () -> loadStatisticsIntoDatastore(memoryStatistics, analysisDatastore)); } - protected IDatastore createAnalysisDatastore() { + protected IInternalDatastore createAnalysisDatastore() { final IDatastoreSchemaDescription desc = new MemoryAnalysisDatastoreDescriptionConfig().datastoreSchemaDescription(); final IActivePivotManagerDescription manager = new ManagerDescriptionConfig().managerDescription(); - return (IDatastore) + return (IInternalDatastore) ApplicationInTests.builder().withDatastore(desc).withManager(manager).build().getDatabase(); } - protected Collection loadMemoryStatistic(final Path path) throws IOException { + protected Collection loadMemoryStatistic(final Path path) throws IOException { return Files.list(path) .map( file -> { @@ -168,7 +172,8 @@ protected Collection loadMemoryStatistic(final Path path) thro } protected void loadStatisticsIntoDatastore( - final Collection statistics, final IDatastore analysisDatastore) { + final Collection statistics, + final IInternalDatastore analysisDatastore) { ATestMemoryStatistic.feedMonitoringApplication(analysisDatastore, statistics, "test"); } } diff --git a/src/test/java/com/activeviam/mac/statistic/memory/visitor/TestReverseEpochViewComparator.java b/src/test/java/com/activeviam/mac/statistic/memory/visitor/TestReverseEpochViewComparator.java index 9898f82d..a4706aa8 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/visitor/TestReverseEpochViewComparator.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/visitor/TestReverseEpochViewComparator.java @@ -9,7 +9,7 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.activeviam.comparators.ReverseEpochViewComparator; +import com.activeviam.mac.comparators.ReverseEpochViewComparator; import com.activeviam.mac.statistic.memory.visitor.impl.DistributedEpochView; import com.activeviam.mac.statistic.memory.visitor.impl.EpochView; import com.activeviam.mac.statistic.memory.visitor.impl.RegularEpochView; diff --git a/src/test/java/com/qfs/chunk/impl/MemoryTestUtils.java b/src/test/java/com/qfs/chunk/impl/MemoryTestUtils.java index 6e5462ec..04fa1007 100644 --- a/src/test/java/com/qfs/chunk/impl/MemoryTestUtils.java +++ b/src/test/java/com/qfs/chunk/impl/MemoryTestUtils.java @@ -1,10 +1,10 @@ package com.qfs.chunk.impl; import com.activeviam.mac.Workaround; -import com.qfs.pool.impl.NamedThread; -import com.qfs.util.impl.QfsReflection; -import com.qfs.vector.IVectorAllocator; -import com.qfs.vector.impl.Vectors; +import com.activeviam.tech.chunks.api.vectors.IVectorAllocator; +import com.activeviam.tech.chunks.internal.pool.impl.NamedThread; +import com.activeviam.tech.chunks.internal.vectors.impl.Vectors; +import com.activeviam.tech.test.internal.util.ReflectionUtil; import java.lang.reflect.Field; import java.util.Arrays; import java.util.Objects; @@ -24,16 +24,16 @@ public static void resetAllThreadsVectorAllocator() { final int nbrActives = Thread.activeCount(); final Thread[] allThreads = new Thread[nbrActives * 2]; final Field directAllocatorsField = - QfsReflection.findField(NamedThread.class, "directVectorAllocators"); + ReflectionUtil.findField(NamedThread.class, "directVectorAllocators"); final Field transientAllocatorsField = - QfsReflection.findField(NamedThread.class, "transientVectorAllocators"); + ReflectionUtil.findField(NamedThread.class, "transientVectorAllocators"); Thread.enumerate(allThreads); for (final Thread t : allThreads) { if (t instanceof NamedThread) { final IVectorAllocator[] directAllocators = - QfsReflection.getFieldValue(directAllocatorsField, t); + ReflectionUtil.getFieldValue(directAllocatorsField, t); final IVectorAllocator[] transientAllocators = - QfsReflection.getFieldValue(transientAllocatorsField, t); + ReflectionUtil.getFieldValue(transientAllocatorsField, t); Stream.concat(Arrays.stream(directAllocators), Arrays.stream(transientAllocators)) .filter(Objects::nonNull) .forEach(