From d61b2bca567129f562113b85598587da55ab31b9 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Mon, 9 Oct 2023 16:24:35 +0200 Subject: [PATCH 01/54] feat(#2002): Align adapter registration with other pipeline elements --- .../streampipes/client/api/IAdminApi.java | 3 - .../streampipes/client/api/AdminApi.java | 14 -- .../WorkerAdministrationManagement.java | 54 ++++-- .../management/WorkerRestClient.java | 6 +- .../management/locales/LabelGenerator.java | 8 +- .../manager/assets/AssetManager.java | 5 + .../manager/endpoint/EndpointItemParser.java | 11 +- .../ExtensionsServiceEndpointUtils.java | 21 ++- .../recommender/ElementRecommender.java | 4 +- .../manager/verification/AdapterVerifier.java | 65 ++++++++ .../verification/DataProcessorVerifier.java | 10 +- .../verification/DataSinkVerifier.java | 10 +- .../verification/DataStreamVerifier.java | 10 +- .../manager/verification/ElementVerifier.java | 14 +- .../manager/verification/StorageState.java | 2 +- .../verification/extractor/TypeExtractor.java | 22 ++- ...bstractPipelineElementResourceManager.java | 2 +- .../AdapterDescriptionResourceManager.java | 30 ++-- .../management/SpResourceManager.java | 4 + .../management/UserResourceManager.java | 11 ++ .../core/base/impl/AbstractRestResource.java | 9 +- .../AbstractPipelineElementResource.java | 2 +- .../rest/extensions/WelcomePage.java | 23 ++- .../connect/AdapterDescriptionResource.java | 65 ++++++++ .../rest/extensions/html/HTMLGenerator.java | 15 +- .../rest/extensions/html/JSONGenerator.java | 14 +- .../html/page/WelcomePageGenerator.java | 62 ++++--- .../ExtensionsServiceEndpointResource.java | 79 +++++---- .../impl/admin/PipelineElementImport.java | 34 ++-- .../connect/WorkerAdministrationResource.java | 54 ------ .../service/core/PostStartupTask.java | 156 ++++++++++++++++++ .../core/StreamPipesCoreApplication.java | 95 +---------- .../core/StreamPipesResourceConfig.java | 2 - .../extensions/ExtensionsModelSubmitter.java | 2 - .../extensions/ExtensionsResourceConfig.java | 2 + .../extensions/connect/ConnectRestClient.java | 46 ------ .../ConnectWorkerRegistrationService.java | 48 ------ .../storage/api/IAdapterStorage.java | 2 +- .../storage/api/INoSqlStorage.java | 2 +- .../IPipelineElementDescriptionStorage.java | 33 ++-- ...ipelineElementDescriptionStorageCache.java | 27 --- .../couchdb/CouchDbStorageManager.java | 4 +- .../impl/AdapterDescriptionStorageImpl.java | 38 ++++- .../impl/AdapterInstanceStorageImpl.java | 32 +++- ...PipelineElementDescriptionStorageImpl.java | 108 ++++++------ .../storage/management/StorageManager.java | 4 +- ui/deployment/_variables.scss | 1 + ui/deployment/modules.yml | 4 +- ui/src/app/add/add.component.html | 7 +- ui/src/app/add/add.routes.ts | 2 +- .../endpoint-item.component.scss | 2 - .../endpoint-item/endpoint-item.component.ts | 8 +- .../adapter-description.component.html | 42 +---- .../adapter-description.component.scss | 22 ++- .../adapter-description.component.ts | 26 ++- .../existing-adapters.component.scss | 4 + .../existing-adapters.component.ts | 7 +- ui/src/scss/sp/main.scss | 16 +- 58 files changed, 768 insertions(+), 637 deletions(-) create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/AdapterVerifier.java rename streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/model/DataSourceDescriptionHtml.java => streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AdapterDescriptionResourceManager.java (55%) create mode 100644 streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/connect/AdapterDescriptionResource.java delete mode 100644 streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/WorkerAdministrationResource.java create mode 100644 streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PostStartupTask.java delete mode 100644 streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/connect/ConnectRestClient.java delete mode 100644 streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/connect/ConnectWorkerRegistrationService.java delete mode 100644 streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IPipelineElementDescriptionStorageCache.java diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java index 1d6042d59b..2b9d9e45eb 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java @@ -19,7 +19,6 @@ package org.apache.streampipes.client.api; import org.apache.streampipes.model.configuration.MessagingSettings; -import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.function.FunctionDefinition; @@ -36,8 +35,6 @@ public interface IAdminApi { SpServiceConfiguration getServiceConfiguration(String serviceGroup); - void registerAdapters(List adapters); - void registerFunctions(List functions); void deregisterFunction(String functionId); diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java index 3f9403614b..b502632839 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java @@ -20,7 +20,6 @@ import org.apache.streampipes.client.model.StreamPipesClientConfig; import org.apache.streampipes.client.util.StreamPipesApiPath; import org.apache.streampipes.model.configuration.MessagingSettings; -import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.function.FunctionDefinition; @@ -57,11 +56,6 @@ public SpServiceConfiguration getServiceConfiguration(String serviceGroup) { return opt.orElseGet(SpServiceConfiguration::new); } - @Override - public void registerAdapters(List adapters) { - post(getConnectPath(), adapters); - } - @Override public void registerFunctions(List functions) { post(getFunctionsPath(), functions); @@ -95,14 +89,6 @@ private StreamPipesApiPath getMessagingSettingsPath() { .addToPath("messaging"); } - private StreamPipesApiPath getConnectPath() { - return StreamPipesApiPath - .fromBaseApiPath() - .addToPath("connect") - .addToPath("master") - .addToPath("administration"); - } - private StreamPipesApiPath getFunctionsPath() { return StreamPipesApiPath .fromBaseApiPath() diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java index 2b6300dccd..15ec1f55a3 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java @@ -18,15 +18,22 @@ package org.apache.streampipes.connect.management.management; +import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; import org.apache.streampipes.connect.management.health.AdapterHealthCheck; import org.apache.streampipes.connect.management.health.AdapterOperationLock; +import org.apache.streampipes.manager.assets.AssetManager; import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.resource.management.PermissionResourceManager; +import org.apache.streampipes.resource.management.SpResourceManager; import org.apache.streampipes.storage.api.IAdapterStorage; import org.apache.streampipes.storage.couchdb.CouchDbStorageManager; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.List; import java.util.concurrent.TimeUnit; @@ -44,24 +51,7 @@ public WorkerAdministrationManagement() { this.adapterDescriptionStorage = CouchDbStorageManager.INSTANCE.getAdapterDescriptionStorage(); } - public void register(List availableAdapterDescription) { - List alreadyRegisteredAdapters = this.adapterDescriptionStorage.getAllAdapters(); - - availableAdapterDescription.forEach(adapterDescription -> { - - // only install once adapter description per service group - boolean alreadyInstalled = - alreadyRegisteredAdapters.stream().anyMatch(a -> a.getAppId().equals(adapterDescription.getAppId())); - if (!alreadyInstalled) { - this.adapterDescriptionStorage.storeAdapter(adapterDescription); - } - }); - - int retryCount = 0; - checkAndRestore(retryCount); - } - - private void checkAndRestore(int retryCount) { + public void checkAndRestore(int retryCount) { if (AdapterOperationLock.INSTANCE.isLocked()) { LOG.info("Adapter operation already in progress, {}/{}", (retryCount + 1), MAX_RETRIES); if (retryCount <= MAX_RETRIES) { @@ -83,4 +73,32 @@ private void checkAndRestore(int retryCount) { AdapterOperationLock.INSTANCE.unlock(); } } + + public void performAdapterMigrations(List tags) { + var installedAdapters = CouchDbStorageManager.INSTANCE.getAdapterDescriptionStorage().getAllAdapters(); + var adminSid = new SpResourceManager().manageUsers().getAdminUser().getPrincipalId(); + installedAdapters.stream() + .filter(adapter -> tags.stream().anyMatch(tag -> tag.getValue().equals(adapter.getAppId()))) + .forEach(adapter -> { + if (!AssetManager.existsAssetDir(adapter.getAppId())) { + try { + LOG.info("Updating assets for adapter {}", adapter.getAppId()); + AssetManager.storeAsset(SpServiceUrlProvider.ADAPTER, adapter.getAppId()); + } catch (IOException | NoServiceEndpointsAvailableException e) { + LOG.error( + "Could not fetch asset for adapter {}, please try to manually update this adapter.", + adapter.getAppId(), + e); + } + } + var permissionStorage = CouchDbStorageManager.INSTANCE.getPermissionStorage(); + var elementId = adapter.getElementId(); + var permissions = permissionStorage.getUserPermissionsForObject(elementId); + if (permissions.isEmpty()) { + LOG.info("Adding default permission for adapter {}", adapter.getAppId()); + new PermissionResourceManager() + .createDefault(elementId, AdapterDescription.class, adminSid, true); + } + }); + } } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java index 58f6a49bc1..ba0a21383f 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java @@ -77,9 +77,7 @@ public static List getAllRunningAdapterInstanceDescriptions( .extServiceGetRequest(url) .execute().returnContent().asString(); - List result = JacksonSerializer.getObjectMapper().readValue(responseString, List.class); - - return result; + return JacksonSerializer.getObjectMapper().readValue(responseString, List.class); } catch (IOException e) { logger.error("List of running adapters could not be fetched", e); throw new AdapterException("List of running adapters could not be fetched from: " + url); @@ -106,7 +104,7 @@ private static void triggerAdapterStateChange(AdapterDescription ad, try { String adapterDescription = JacksonSerializer.getObjectMapper().writeValueAsString(ad); - var response = triggerPost(url, ad.getElementId(), adapterDescription); + var response = triggerPost(url, ad.getCorrespondingDataStreamElementId(), adapterDescription); var responseString = getResponseBody(response); if (response.getStatusLine().getStatusCode() != 200) { diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/locales/LabelGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/locales/LabelGenerator.java index 3ce1807a3d..8e47e20923 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/locales/LabelGenerator.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/locales/LabelGenerator.java @@ -40,7 +40,7 @@ import static org.apache.streampipes.extensions.management.util.LocalesUtil.makePath; -public class LabelGenerator { +public class LabelGenerator { private static final Logger LOG = LoggerFactory.getLogger(LabelGenerator.class); @@ -48,13 +48,13 @@ public class LabelGenerator { private static final String Title = "title"; private static final String Description = "description"; - private NamedStreamPipesEntity desc; + private T desc; - public LabelGenerator(NamedStreamPipesEntity desc) { + public LabelGenerator(T desc) { this.desc = desc; } - public NamedStreamPipesEntity generateLabels() throws IOException { + public T generateLabels() throws IOException { if (existsLocalesFile()) { Properties props = makeProperties(); desc.setName(getTitle(props, desc.getAppId())); diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetManager.java index a570864ae7..a4e9abd31f 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetManager.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetManager.java @@ -44,6 +44,11 @@ public static byte[] getAsset(String appId, String assetName) throws IOException return Files.readAllBytes(Paths.get(getAssetPath(appId, assetName))); } + public static boolean existsAssetDir(String appId) { + var directory = new File(getAssetDir(appId)); + return directory.exists() && directory.isDirectory(); + } + public static void storeAsset(SpServiceUrlProvider spServiceUrlProvider, String appId) throws IOException, NoServiceEndpointsAvailableException { InputStream assetStream = new AssetFetcher(spServiceUrlProvider, appId) diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/endpoint/EndpointItemParser.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/endpoint/EndpointItemParser.java index 88a3a22c6e..9ab4bf48e6 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/endpoint/EndpointItemParser.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/endpoint/EndpointItemParser.java @@ -17,15 +17,15 @@ */ package org.apache.streampipes.manager.endpoint; +import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; import org.apache.streampipes.manager.operations.Operations; import org.apache.streampipes.model.message.Message; import org.apache.streampipes.model.message.NotificationType; import org.apache.streampipes.model.message.Notifications; -import org.apache.http.client.fluent.Request; - import java.io.IOException; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; public class EndpointItemParser { @@ -33,7 +33,7 @@ public Message parseAndAddEndpointItem(String url, String principalSid, boolean publicElement) { try { - url = URLDecoder.decode(url, "UTF-8"); + url = URLDecoder.decode(url, StandardCharsets.UTF_8); String payload = parseURIContent(url); return Operations.verifyAndAddElement(payload, principalSid, publicElement); } catch (Exception e) { @@ -44,9 +44,8 @@ public Message parseAndAddEndpointItem(String url, } private String parseURIContent(String url) throws IOException { - return Request - .Get(url) - .addHeader("Accept", "application/json") + return ExtensionServiceExecutions + .extServiceGetRequest(url) .execute() .returnContent() .asString(); diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointUtils.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointUtils.java index af6f81a803..7c63a315bf 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointUtils.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointUtils.java @@ -18,8 +18,11 @@ package org.apache.streampipes.manager.execution.endpoint; import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.graph.DataProcessorDescription; import org.apache.streampipes.model.graph.DataProcessorInvocation; +import org.apache.streampipes.model.graph.DataSinkDescription; +import org.apache.streampipes.model.graph.DataSinkInvocation; import org.apache.streampipes.storage.management.StorageDispatcher; import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; @@ -28,7 +31,15 @@ public class ExtensionsServiceEndpointUtils { public static SpServiceUrlProvider getPipelineElementType(NamedStreamPipesEntity entity) { - return isDataProcessor(entity) ? SpServiceUrlProvider.DATA_PROCESSOR : SpServiceUrlProvider.DATA_SINK; + if (isDataProcessor(entity)) { + return SpServiceUrlProvider.DATA_PROCESSOR; + } else if (isDataSink(entity)) { + return SpServiceUrlProvider.DATA_SINK; + } else if (isAdapter(entity)) { + return SpServiceUrlProvider.ADAPTER; + } else { + throw new RuntimeException("Could not find service url for entity " + entity.getClass().getCanonicalName()); + } } public static SpServiceUrlProvider getPipelineElementType(String appId) { @@ -43,4 +54,12 @@ public static SpServiceUrlProvider getPipelineElementType(String appId) { private static boolean isDataProcessor(NamedStreamPipesEntity entity) { return entity instanceof DataProcessorInvocation || entity instanceof DataProcessorDescription; } + + private static boolean isDataSink(NamedStreamPipesEntity entity) { + return entity instanceof DataSinkInvocation || entity instanceof DataSinkDescription; + } + + private static boolean isAdapter(NamedStreamPipesEntity entity) { + return entity instanceof AdapterDescription; + } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/recommender/ElementRecommender.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/recommender/ElementRecommender.java index d6268d661b..196c06ce43 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/recommender/ElementRecommender.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/recommender/ElementRecommender.java @@ -156,7 +156,7 @@ private List getAllDataProcessors() { return getTripleStore() .getAllDataProcessors() .stream() - .filter(e -> userObjects.stream().anyMatch(u -> u.equals(e.getElementId()))) + .filter(e -> userObjects.stream().anyMatch(u -> u.equals(e.getAppId()))) .map(DataProcessorDescription::new) .collect(Collectors.toList()); } @@ -167,7 +167,7 @@ private List getAllDataSinks() { return getTripleStore() .getAllDataSinks() .stream() - .filter(e -> userObjects.stream().anyMatch(u -> u.equals(e.getElementId()))) + .filter(e -> userObjects.stream().anyMatch(u -> u.equals(e.getAppId()))) .map(DataSinkDescription::new) .collect(Collectors.toList()); } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/AdapterVerifier.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/AdapterVerifier.java new file mode 100644 index 0000000000..e7545675d7 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/AdapterVerifier.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.verification; + +import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; +import org.apache.streampipes.commons.exceptions.SepaParseException; +import org.apache.streampipes.manager.assets.AssetManager; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; + +import java.io.IOException; + +public class AdapterVerifier extends ElementVerifier { + + public AdapterVerifier(String graphData) throws SepaParseException { + super(graphData, AdapterDescription.class); + } + + @Override + protected StorageState store() { + var storageState = StorageState.STORED; + if (!storageApi.exists(elementDescription)) { + storageApi.storeAdapterDescription(elementDescription); + } else { + storageState = StorageState.ALREADY_STORED; + } + return storageState; + } + + @Override + protected void update() { + storageApi.update(elementDescription); + } + + @Override + protected void storeAssets() throws IOException, NoServiceEndpointsAvailableException { + if (elementDescription.isIncludesAssets()) { + AssetManager.storeAsset(SpServiceUrlProvider.ADAPTER, elementDescription.getAppId()); + } + } + + @Override + protected void updateAssets() throws IOException, NoServiceEndpointsAvailableException { + if (elementDescription.isIncludesAssets()) { + AssetManager.deleteAsset(elementDescription.getAppId()); + storeAssets(); + } + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataProcessorVerifier.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataProcessorVerifier.java index 851b634873..c5a86d8ed7 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataProcessorVerifier.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataProcessorVerifier.java @@ -46,7 +46,7 @@ protected StorageState store() { if (!storageApi.exists(elementDescription)) { storageApi.storeDataProcessor(elementDescription); } else { - storageState = StorageState.ALREADY_IN_SESAME; + storageState = StorageState.ALREADY_STORED; } return storageState; } @@ -63,12 +63,4 @@ protected void storeAssets() throws IOException, NoServiceEndpointsAvailableExce } } - @Override - protected void updateAssets() throws IOException, NoServiceEndpointsAvailableException { - if (elementDescription.isIncludesAssets()) { - AssetManager.deleteAsset(elementDescription.getAppId()); - storeAssets(); - } - } - } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataSinkVerifier.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataSinkVerifier.java index 2b3370942c..6bbd41e29e 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataSinkVerifier.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataSinkVerifier.java @@ -41,7 +41,7 @@ protected StorageState store() { if (!storageApi.exists(elementDescription)) { storageApi.storeDataSink(elementDescription); } else { - storageState = StorageState.ALREADY_IN_SESAME; + storageState = StorageState.ALREADY_STORED; } return storageState; } @@ -63,12 +63,4 @@ protected void storeAssets() throws IOException, NoServiceEndpointsAvailableExce AssetManager.storeAsset(SpServiceUrlProvider.DATA_SINK, elementDescription.getAppId()); } } - - @Override - protected void updateAssets() throws IOException, NoServiceEndpointsAvailableException { - if (elementDescription.isIncludesAssets()) { - AssetManager.deleteAsset(elementDescription.getAppId()); - storeAssets(); - } - } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataStreamVerifier.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataStreamVerifier.java index 9d37a5c6c0..f29c999c14 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataStreamVerifier.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/DataStreamVerifier.java @@ -47,7 +47,7 @@ protected StorageState store() { if (!storageApi.exists(elementDescription)) { storageApi.storeDataStream(elementDescription); } else { - storageState = StorageState.ALREADY_IN_SESAME; + storageState = StorageState.ALREADY_STORED; } return storageState; } @@ -63,12 +63,4 @@ protected void storeAssets() throws IOException, NoServiceEndpointsAvailableExce AssetManager.storeAsset(SpServiceUrlProvider.DATA_STREAM, elementDescription.getAppId()); } } - - @Override - protected void updateAssets() throws IOException, NoServiceEndpointsAvailableException { - if (elementDescription.isIncludesAssets()) { - AssetManager.deleteAsset(elementDescription.getAppId()); - storeAssets(); - } - } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/ElementVerifier.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/ElementVerifier.java index cfdaf9870e..2537057698 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/ElementVerifier.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/ElementVerifier.java @@ -20,6 +20,7 @@ import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; import org.apache.streampipes.commons.exceptions.SepaParseException; +import org.apache.streampipes.manager.assets.AssetManager; import org.apache.streampipes.manager.verification.messages.VerificationError; import org.apache.streampipes.manager.verification.messages.VerificationResult; import org.apache.streampipes.manager.verification.structure.GeneralVerifier; @@ -34,7 +35,7 @@ import org.apache.streampipes.model.message.SuccessMessage; import org.apache.streampipes.resource.management.SpResourceManager; import org.apache.streampipes.serializers.json.JacksonSerializer; -import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorageCache; +import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage; import org.apache.streampipes.storage.management.StorageManager; import com.fasterxml.jackson.core.JsonProcessingException; @@ -57,7 +58,7 @@ public abstract class ElementVerifier { protected List validationResults; protected List validators; - protected IPipelineElementDescriptionStorageCache storageApi = + protected IPipelineElementDescriptionStorage storageApi = StorageManager.INSTANCE.getPipelineElementStorage(); public ElementVerifier(String graphData, Class elementClass) { @@ -107,7 +108,7 @@ public Message verifyAndAdd(String principalSid, boolean publicElement) throws S e.printStackTrace(); } return successMessage(); - } else if (state == StorageState.ALREADY_IN_SESAME) { + } else if (state == StorageState.ALREADY_STORED) { return addedToUserSuccessMessage(); } else { return skippedSuccessMessage(); @@ -141,7 +142,12 @@ public Message verifyAndUpdate() throws SepaParseException { protected abstract void storeAssets() throws IOException, NoServiceEndpointsAvailableException; - protected abstract void updateAssets() throws IOException, NoServiceEndpointsAvailableException; + protected void updateAssets() throws IOException, NoServiceEndpointsAvailableException { + if (elementDescription.isIncludesAssets()) { + AssetManager.deleteAsset(elementDescription.getAppId()); + storeAssets(); + } + } private Message errorMessage() { return new ErrorMessage(elementDescription.getName(), collectNotifications()); diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/StorageState.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/StorageState.java index 3d30411fb1..35e85aa2a5 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/StorageState.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/StorageState.java @@ -19,5 +19,5 @@ package org.apache.streampipes.manager.verification; public enum StorageState { - STORED, ALREADY_IN_SESAME + STORED, ALREADY_STORED } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/extractor/TypeExtractor.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/extractor/TypeExtractor.java index a7e6c928b4..32f5122f45 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/extractor/TypeExtractor.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/extractor/TypeExtractor.java @@ -19,11 +19,13 @@ package org.apache.streampipes.manager.verification.extractor; import org.apache.streampipes.commons.exceptions.SepaParseException; +import org.apache.streampipes.manager.verification.AdapterVerifier; import org.apache.streampipes.manager.verification.DataProcessorVerifier; import org.apache.streampipes.manager.verification.DataSinkVerifier; import org.apache.streampipes.manager.verification.DataStreamVerifier; import org.apache.streampipes.manager.verification.ElementVerifier; import org.apache.streampipes.model.SpDataStream; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.graph.DataProcessorDescription; import org.apache.streampipes.model.graph.DataSinkDescription; import org.apache.streampipes.serializers.json.JacksonSerializer; @@ -37,17 +39,17 @@ public class TypeExtractor { private static final Logger logger = Logger.getAnonymousLogger(); - private String pipelineElementDescription; + private final String extensionElementDescription; - public TypeExtractor(String pipelineElementDescription) { - this.pipelineElementDescription = pipelineElementDescription; + public TypeExtractor(String extensionElementDescription) { + this.extensionElementDescription = extensionElementDescription; } public ElementVerifier getTypeVerifier() throws SepaParseException { try { ObjectNode jsonNode = - JacksonSerializer.getObjectMapper().readValue(this.pipelineElementDescription, ObjectNode.class); + JacksonSerializer.getObjectMapper().readValue(this.extensionElementDescription, ObjectNode.class); String jsonClassName = jsonNode.get("@class").asText(); return getTypeDef(jsonClassName); } catch (JsonProcessingException e) { @@ -61,13 +63,15 @@ private ElementVerifier getTypeDef(String jsonClassName) throws SepaParseExce } else { if (jsonClassName.equals(ep())) { logger.info("Detected type data stream"); - return new DataStreamVerifier(pipelineElementDescription); + return new DataStreamVerifier(extensionElementDescription); } else if (jsonClassName.equals(epa())) { logger.info("Detected type data processor"); - return new DataProcessorVerifier(pipelineElementDescription); + return new DataProcessorVerifier(extensionElementDescription); } else if (jsonClassName.equals(ec())) { logger.info("Detected type data sink"); - return new DataSinkVerifier(pipelineElementDescription); + return new DataSinkVerifier(extensionElementDescription); + } else if (jsonClassName.equals(adapter())) { + return new AdapterVerifier(extensionElementDescription); } else { throw new SepaParseException(); } @@ -86,4 +90,8 @@ private static String ec() { return DataSinkDescription.class.getCanonicalName(); } + private static String adapter() { + return AdapterDescription.class.getCanonicalName(); + } + } diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AbstractPipelineElementResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AbstractPipelineElementResourceManager.java index da3aed9814..255e3f127d 100644 --- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AbstractPipelineElementResourceManager.java +++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AbstractPipelineElementResourceManager.java @@ -38,7 +38,7 @@ public List findAll() { } public List findAllIdsOnly() { - return db.getAll().stream().map(NamedStreamPipesEntity::getElementId).collect(Collectors.toList()); + return db.getAll().stream().map(NamedStreamPipesEntity::getAppId).collect(Collectors.toList()); } public List findAllAsInvocation() { diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/model/DataSourceDescriptionHtml.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AdapterDescriptionResourceManager.java similarity index 55% rename from streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/model/DataSourceDescriptionHtml.java rename to streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AdapterDescriptionResourceManager.java index f549b634e9..da842896de 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/model/DataSourceDescriptionHtml.java +++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AdapterDescriptionResourceManager.java @@ -16,29 +16,21 @@ * */ -package org.apache.streampipes.rest.extensions.html.model; +package org.apache.streampipes.resource.management; -import java.util.ArrayList; -import java.util.List; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.storage.api.IAdapterStorage; +import org.apache.streampipes.storage.management.StorageDispatcher; -public class DataSourceDescriptionHtml extends Description { +public class AdapterDescriptionResourceManager + extends AbstractPipelineElementResourceManager { - private List streams; - - public DataSourceDescriptionHtml(String name, String description, String descriptionUrl, List streams) { - super(name, description, descriptionUrl); - this.streams = streams; - } - - public DataSourceDescriptionHtml() { - streams = new ArrayList<>(); - } - - public List getStreams() { - return streams; + public AdapterDescriptionResourceManager() { + super(StorageDispatcher.INSTANCE.getNoSqlStore().getAdapterDescriptionStorage()); } - public void setStreams(List streams) { - this.streams = streams; + @Override + protected AdapterDescription toInvocation(AdapterDescription description) { + return description; } } diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/SpResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/SpResourceManager.java index 3b362e3225..37df63dfc7 100644 --- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/SpResourceManager.java +++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/SpResourceManager.java @@ -23,6 +23,10 @@ public AdapterResourceManager manageAdapters() { return new AdapterResourceManager(); } + public AdapterDescriptionResourceManager manageAdapterDescriptions() { + return new AdapterDescriptionResourceManager(); + } + public DashboardResourceManager manageDashboards() { return new DashboardResourceManager(); } diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/UserResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/UserResourceManager.java index b42cbc0cd6..719ea614c2 100644 --- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/UserResourceManager.java +++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/UserResourceManager.java @@ -33,6 +33,7 @@ import org.apache.streampipes.storage.api.IPasswordRecoveryTokenStorage; import org.apache.streampipes.storage.api.IUserActivationTokenStorage; import org.apache.streampipes.storage.api.IUserStorage; +import org.apache.streampipes.storage.couchdb.CouchDbStorageManager; import org.apache.streampipes.storage.management.StorageDispatcher; import org.apache.streampipes.user.management.util.PasswordUtil; @@ -71,6 +72,16 @@ public Principal getServiceAdmin() { ); } + public Principal getAdminUser() { + return CouchDbStorageManager.INSTANCE + .getUserStorageAPI() + .getAllUserAccounts() + .stream() + .filter(u -> u.getRoles().contains(Role.ROLE_ADMIN)) + .findFirst() + .orElseThrow(IllegalArgumentException::new); + } + public boolean registerUser(RegistrationData data) throws UsernameAlreadyTakenException { try { diff --git a/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java b/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java index b231450db9..18ac8be032 100644 --- a/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java +++ b/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java @@ -29,14 +29,13 @@ import org.apache.streampipes.storage.api.IFileMetadataStorage; import org.apache.streampipes.storage.api.INoSqlStorage; import org.apache.streampipes.storage.api.INotificationStorage; -import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorageCache; +import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage; import org.apache.streampipes.storage.api.IPipelineElementTemplateStorage; import org.apache.streampipes.storage.api.IPipelineStorage; import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; import org.apache.streampipes.storage.api.IUserStorage; import org.apache.streampipes.storage.api.IVisualizationStorage; import org.apache.streampipes.storage.management.StorageDispatcher; -import org.apache.streampipes.storage.management.StorageManager; import org.apache.http.client.ClientProtocolException; @@ -53,11 +52,11 @@ protected ISpCoreConfigurationStorage getSpCoreConfigurationStorage() { return getNoSqlStorage().getSpCoreConfigurationStorage(); } - protected IPipelineElementDescriptionStorageCache getPipelineElementRdfStorage() { - return StorageManager.INSTANCE.getPipelineElementStorage(); + protected IPipelineElementDescriptionStorage getPipelineElementRdfStorage() { + return getPipelineElementStorage(); } - protected IPipelineElementDescriptionStorageCache getPipelineElementStorage() { + protected IPipelineElementDescriptionStorage getPipelineElementStorage() { return getNoSqlStorage().getPipelineElementDescriptionStorage(); } diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/AbstractPipelineElementResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/AbstractPipelineElementResource.java index 20e24dbeef..eb0eeb47d8 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/AbstractPipelineElementResource.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/AbstractPipelineElementResource.java @@ -129,7 +129,7 @@ protected NamedStreamPipesEntity rewrite(NamedStreamPipesEntity desc) { // TODO remove after full internationalization support has been implemented if (desc.isIncludesLocales()) { try { - desc = new LabelGenerator(desc).generateLabels(); + desc = new LabelGenerator<>(desc).generateLabels(); } catch (IOException e) { e.printStackTrace(); } diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/WelcomePage.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/WelcomePage.java index 288404cb5b..38867a73af 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/WelcomePage.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/WelcomePage.java @@ -18,6 +18,8 @@ package org.apache.streampipes.rest.extensions; +import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter; +import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; import org.apache.streampipes.extensions.management.init.DeclarersSingleton; import org.apache.streampipes.rest.extensions.html.HTMLGenerator; import org.apache.streampipes.rest.extensions.html.JSONGenerator; @@ -28,6 +30,8 @@ import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; +import java.util.Collection; + @Path("/") public class WelcomePage { @@ -48,11 +52,22 @@ public String getWelcomePageJson() { } private WelcomePageGenerator getWelcomePageGenerator() { - return new WelcomePageGenerator(DeclarersSingleton - .getInstance() - .getBaseUri(), DeclarersSingleton + return new WelcomePageGenerator( + DeclarersSingleton.getInstance().getBaseUri(), + getPipelineElements(), + getAdapters()); + } + + private Collection> getPipelineElements() { + return DeclarersSingleton .getInstance() .getDeclarers() - .values()); + .values(); + } + + private Collection getAdapters() { + return DeclarersSingleton + .getInstance() + .getAdapters(); } } diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/connect/AdapterDescriptionResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/connect/AdapterDescriptionResource.java new file mode 100644 index 0000000000..cdc0f75146 --- /dev/null +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/connect/AdapterDescriptionResource.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.extensions.connect; + +import org.apache.streampipes.extensions.management.init.DeclarersSingleton; +import org.apache.streampipes.extensions.management.locales.LabelGenerator; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; +import org.apache.streampipes.rest.shared.impl.AbstractSharedRestInterface; + +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +import java.io.IOException; + +@Path("/api/v1/worker/adapters") +public class AdapterDescriptionResource extends AbstractSharedRestInterface { + + @GET + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + @JacksonSerialized + public Response getAdapterDescription(@PathParam("id") String id) { + var adapterDescriptionOpt = DeclarersSingleton.getInstance().getAdapter(id); + if (adapterDescriptionOpt.isPresent()) { + try { + var adapterDescription = adapterDescriptionOpt.get().declareConfig().getAdapterDescription(); + var localizedDescription = applyLocales(adapterDescription); + return ok(localizedDescription); + } catch (IOException e) { + return serverError(e); + } + } else { + return notFound(); + } + } + + private AdapterDescription applyLocales(AdapterDescription adapterDescription) throws IOException { + if (adapterDescription.isIncludesLocales()) { + return new LabelGenerator<>(adapterDescription).generateLabels(); + } else { + return adapterDescription; + } + } +} diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/HTMLGenerator.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/HTMLGenerator.java index bb26d9d72f..8d5dec7d5a 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/HTMLGenerator.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/HTMLGenerator.java @@ -18,7 +18,6 @@ package org.apache.streampipes.rest.extensions.html; -import org.apache.streampipes.rest.extensions.html.model.DataSourceDescriptionHtml; import org.apache.streampipes.rest.extensions.html.model.Description; import org.rendersnake.HtmlCanvas; @@ -54,7 +53,7 @@ public String buildHtml() { .div(class_("container")) .div(class_("navbar-header")) .a(class_("navbar-brand").style("color:white;")) - .content("StreamPipes Pipeline Element Container") + .content("StreamPipes Extensions Service") ._div() ._div() ._nav() @@ -69,17 +68,9 @@ public String buildHtml() { html.h3(); html.write(description.getName()); html._h3(); - html.h4().write("URI: ").a(href(description.getDescriptionUrl().toString())) - .content(description.getDescriptionUrl().toString())._h4(); + html.h4().write("URI: ").a(href(description.getDescriptionUrl())) + .content(description.getDescriptionUrl())._h4(); html.h4().write("Description: ").write(description.getDescription())._h4(); - if (description instanceof DataSourceDescriptionHtml) { - DataSourceDescriptionHtml semanticEventProducerDescription = (DataSourceDescriptionHtml) description; - for (Description agentDesc : semanticEventProducerDescription.getStreams()) { - html.h5().b().write(agentDesc.getName())._b()._h5(); - html.h5().write(agentDesc.getDescription())._h5(); - - } - } } html._div(); html._body(); diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/JSONGenerator.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/JSONGenerator.java index bbb7b0ebf2..28e772842f 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/JSONGenerator.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/JSONGenerator.java @@ -18,7 +18,6 @@ package org.apache.streampipes.rest.extensions.html; -import org.apache.streampipes.rest.extensions.html.model.DataSourceDescriptionHtml; import org.apache.streampipes.rest.extensions.html.model.Description; import com.google.gson.JsonArray; @@ -29,7 +28,7 @@ public class JSONGenerator { - private List description; + private final List description; public JSONGenerator(List description) { this.description = description; @@ -42,16 +41,7 @@ public String buildJson() { } private JsonObject getJsonElement(Description d) { - JsonObject obj = makeDescription(d); - if (d instanceof DataSourceDescriptionHtml) { - DataSourceDescriptionHtml producerDesc = (DataSourceDescriptionHtml) d; - JsonArray array = new JsonArray(); - - producerDesc.getStreams().forEach(s -> array.add(makeDescription(s))); - - obj.add("streams", array); - } - return obj; + return makeDescription(d); } private JsonObject makeDescription(Description d) { diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/page/WelcomePageGenerator.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/page/WelcomePageGenerator.java index 052b61c606..d4ff6fe381 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/page/WelcomePageGenerator.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/html/page/WelcomePageGenerator.java @@ -18,6 +18,7 @@ package org.apache.streampipes.rest.extensions.html.page; +import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter; import org.apache.streampipes.extensions.api.pe.IStreamPipesDataProcessor; import org.apache.streampipes.extensions.api.pe.IStreamPipesDataSink; import org.apache.streampipes.extensions.api.pe.IStreamPipesDataStream; @@ -25,6 +26,8 @@ import org.apache.streampipes.extensions.management.locales.LabelGenerator; import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.graph.DataProcessorDescription; import org.apache.streampipes.model.graph.DataSinkDescription; import org.apache.streampipes.rest.extensions.html.model.Description; import org.apache.streampipes.sdk.utils.Assets; @@ -39,34 +42,44 @@ public class WelcomePageGenerator { protected List descriptions; protected Collection> pipelineElements; + protected Collection adapters; protected String baseUri; public WelcomePageGenerator(String baseUri, - Collection> pipelineElements) { + Collection> pipelineElements, + Collection adapters) { this.pipelineElements = pipelineElements; + this.adapters = adapters; this.baseUri = baseUri; this.descriptions = new ArrayList<>(); } - public List getDescriptions() { - return descriptions; - } - public List buildUris() { List descriptions = new ArrayList<>(); - for (IStreamPipesPipelineElement pipelineElement : pipelineElements) { - descriptions.add(getDescription(pipelineElement)); - } + pipelineElements.forEach(pipelineElement -> descriptions.add(getPipelineElementDescription(pipelineElement))); + adapters.forEach(adapter -> descriptions.add(getAdapterDescription(adapter))); + return descriptions; } - private Description getDescription(IStreamPipesPipelineElement declarer) { - NamedStreamPipesEntity entity = declarer.declareConfig().getDescription(); + private Description getAdapterDescription(StreamPipesAdapter adapter) { + var entity = adapter.declareConfig().getAdapterDescription(); + return getDescription(entity, "adapter", "api/v1/worker/adapters/"); + } + + private Description getPipelineElementDescription(IStreamPipesPipelineElement declarer) { + var entity = declarer.declareConfig().getDescription(); + return getDescription(entity, getType(declarer), getPathPrefix(declarer)); + } + + private Description getDescription(NamedStreamPipesEntity entity, + String type, + String pathPrefix) { Description desc = new Description(); // TODO remove after full internationalization support has been implemented updateLabel(entity, desc); - desc.setType(getType(declarer)); + desc.setType(type); desc.setElementId(entity.getElementId()); desc.setAppId(entity.getAppId()); desc.setEditable(!(entity.isInternallyManaged())); @@ -74,26 +87,35 @@ private Description getDescription(IStreamPipesPipelineElement declarer) { && entity.getIncludedAssets().contains(Assets.DOCUMENTATION)); desc.setIncludesIcon(entity.isIncludesAssets() && entity.getIncludedAssets().contains(Assets.ICON)); - String uri = baseUri; - if (declarer instanceof IStreamPipesDataSink) { - uri += "sec/"; - } else if (declarer instanceof IStreamPipesDataProcessor) { - uri += "sepa/"; - } else if (declarer instanceof IStreamPipesDataStream) { - uri += "stream/"; - } + String uri = baseUri + pathPrefix; desc.setDescriptionUrl(uri + entity.getAppId()); return desc; } + private String getPathPrefix(IStreamPipesPipelineElement pipelineElement) { + if (pipelineElement instanceof IStreamPipesDataSink) { + return "sec/"; + } else if (pipelineElement instanceof IStreamPipesDataProcessor) { + return "sepa/"; + } else if (pipelineElement instanceof IStreamPipesDataStream) { + return "stream/"; + } else { + return ""; + } + } + private String getType(IStreamPipesPipelineElement pipelineElement) { var elementDescription = pipelineElement.declareConfig().getDescription(); if (elementDescription instanceof DataSinkDescription) { return "action"; } else if (elementDescription instanceof SpDataStream) { return "stream"; - } else { + } else if (elementDescription instanceof DataProcessorDescription) { return "sepa"; + } else if (elementDescription instanceof AdapterDescription) { + return "adapter"; + } else { + throw new RuntimeException("Could not get type for element " + elementDescription.getClass()); } } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java index 65ff8a299d..6b65ec8770 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java @@ -20,9 +20,13 @@ import org.apache.streampipes.manager.endpoint.EndpointFetcher; import org.apache.streampipes.manager.operations.Operations; +import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.NamedStreamPipesEntity; import org.apache.streampipes.model.client.endpoint.ExtensionsServiceEndpoint; import org.apache.streampipes.model.client.endpoint.ExtensionsServiceEndpointItem; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.graph.DataProcessorDescription; +import org.apache.streampipes.model.graph.DataSinkDescription; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; @@ -92,10 +96,12 @@ public Response getEndpointContents() { List endpoints = getEndpoints(); String username = getAuthenticatedUsername(); + var installedExtensions = getAllInstalledExtensions(); List items = Operations.getEndpointUriContents(endpoints); - items.forEach(item -> item.setInstalled(isInstalled(item.getElementId()))); + items.forEach(item -> item.setInstalled(isInstalled(installedExtensions, item.getAppId()))); // also add installed elements that are currently not running or available + items.addAll(getAllAdapterEndpoints(items)); items.addAll(getAllDataStreamEndpoints(username, items)); items.addAll(getAllDataProcessorEndpoints(username, items)); items.addAll(getAllDataSinkEndpoints(username, items)); @@ -123,46 +129,56 @@ private List getEndpoints() { return new EndpointFetcher().getEndpoints(); } - private boolean isInstalled(String elementId) { - return getAllPipelineElements() + private boolean isInstalled(List installedElements, + String appId) { + return installedElements .stream() - .anyMatch(e -> e.equals(elementId)); + .anyMatch(e -> e.getAppId().equals(appId)); } - private List getAllPipelineElements() { - List elementUris = new ArrayList<>(); - elementUris.addAll(getAllDataStreamUris()); - elementUris.addAll(getAllDataProcessorUris()); - elementUris.addAll(getAllDataSinkUris()); - return elementUris; + private List getAllInstalledExtensions() { + List elements = new ArrayList<>(); + elements.addAll(getAllAdapters()); + elements.addAll(getAllDataStreams()); + elements.addAll(getAllDataProcessors()); + elements.addAll(getAllDataSinks()); + return elements; + } + + private List getAllAdapterEndpoints( + List existingItems) { + return getAllAdapters() + .stream() + .filter(s -> existingItems.stream().noneMatch(item -> s.getAppId().equals(item.getAppId()))) + .map(adapter -> makeItem(adapter, "adapter")) + .collect(Collectors.toList()); } private List getAllDataStreamEndpoints(String username, - List existingItems) { - return getAllDataStreamUris() + List existingItems) { + return getAllDataStreams() .stream() - .filter(s -> existingItems.stream().noneMatch(item -> s.equals(item.getElementId()))) - .map(s -> getPipelineElementStorage().getDataStreamById(s)) + .filter(s -> existingItems.stream().noneMatch(item -> s.getAppId().equals(item.getAppId()))) + .filter(s -> !s.isInternallyManaged()) .map(stream -> makeItem(stream, "stream")) .collect(Collectors.toList()); } + private List getAllDataProcessorEndpoints(String username, - List existingItems) { - return getAllDataProcessorUris() + List existingItems) { + return getAllDataProcessors() .stream() - .filter(s -> existingItems.stream().noneMatch(item -> s.equals(item.getElementId()))) - .map(s -> getPipelineElementStorage().getDataProcessorById(s)) + .filter(s -> existingItems.stream().noneMatch(item -> s.getAppId().equals(item.getAppId()))) .map(source -> makeItem(source, "sepa")) .collect(Collectors.toList()); } private List getAllDataSinkEndpoints(String username, - List existingItems) { - return getAllDataSinkUris() + List existingItems) { + return getAllDataSinks() .stream() - .filter(s -> existingItems.stream().noneMatch(item -> s.equals(item.getElementId()))) - .map(s -> getPipelineElementStorage().getDataSinkById(s)) + .filter(s -> existingItems.stream().noneMatch(item -> s.getAppId().equals(item.getAppId()))) .map(source -> makeItem(source, "action")) .collect(Collectors.toList()); } @@ -183,16 +199,23 @@ private ExtensionsServiceEndpointItem makeItem(NamedStreamPipesEntity entity, St return endpoint; } - private List getAllDataStreamUris() { - return getSpResourceManager().manageDataStreams().findAllIdsOnly(); + private List getAllAdapters() { + return getNoSqlStorage().getAdapterDescriptionStorage().getAllAdapters(); + } + + private List getAllDataStreams() { + return getNoSqlStorage().getDataStreamStorage().getAll() + .stream() + .filter(stream -> !stream.isInternallyManaged()) + .toList(); } - private List getAllDataProcessorUris() { - return getSpResourceManager().manageDataProcessors().findAllIdsOnly(); + private List getAllDataProcessors() { + return getNoSqlStorage().getDataProcessorStorage().getAll(); } - private List getAllDataSinkUris() { - return getSpResourceManager().manageDataSinks().findAllIdsOnly(); + private List getAllDataSinks() { + return getNoSqlStorage().getDataSinkStorage().getAll(); } private IExtensionsServiceEndpointStorage getRdfEndpointStorage() { diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/PipelineElementImport.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/PipelineElementImport.java index 4bc0d6fdb3..b69d599437 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/PipelineElementImport.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/PipelineElementImport.java @@ -30,10 +30,8 @@ import org.apache.streampipes.model.message.NotificationType; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; -import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorageCache; +import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; @@ -55,8 +53,6 @@ @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) public class PipelineElementImport extends AbstractAuthGuardedRestResource { - private static final Logger LOG = LoggerFactory.getLogger(PipelineElementImport.class); - @POST @Produces(MediaType.APPLICATION_JSON) public Response addElement(@FormParam("uri") String uri, @@ -83,18 +79,22 @@ public Response updateElement(@PathParam("id") String elementId) { @DELETE @Produces(MediaType.APPLICATION_JSON) public Response deleteElement(@PathParam("id") String elementId) { - IPipelineElementDescriptionStorageCache requestor = getPipelineElementRdfStorage(); + IPipelineElementDescriptionStorage requestor = getPipelineElementStorage(); + var resourceManager = getSpResourceManager(); String appId; try { if (requestor.existsDataProcessor(elementId)) { appId = requestor.getDataProcessorById(elementId).getAppId(); - getSpResourceManager().manageDataProcessors().delete(elementId); + resourceManager.manageDataProcessors().delete(elementId); } else if (requestor.existsDataStream(elementId)) { appId = requestor.getDataStreamById(elementId).getAppId(); - getSpResourceManager().manageDataStreams().delete(elementId); + resourceManager.manageDataStreams().delete(elementId); } else if (requestor.existsDataSink(elementId)) { appId = requestor.getDataSinkById(elementId).getAppId(); - getSpResourceManager().manageDataSinks().delete(elementId); + resourceManager.manageDataSinks().delete(elementId); + } else if (requestor.existsAdapterDescription(elementId)) { + appId = requestor.getAdapterById(elementId).getAppId(); + resourceManager.manageAdapterDescriptions().delete(elementId); } else { return constructErrorMessage(new Notification(NotificationType.STORAGE_ERROR.title(), NotificationType.STORAGE_ERROR.description())); @@ -114,15 +114,17 @@ private Message verifyAndAddElement(String uri, } private NamedStreamPipesEntity find(String elementId) { - if (getPipelineElementStorage().existsDataSink(elementId)) { - return getPipelineElementStorage().getDataSinkById(elementId); - } else if (getPipelineElementStorage().existsDataProcessor(elementId)) { - return getPipelineElementStorage().getDataProcessorById(elementId); - } else if (getPipelineElementStorage().existsDataStream(elementId)) { - return getPipelineElementStorage().getDataStreamById(elementId); + var extensionStorage = getPipelineElementStorage(); + if (extensionStorage.existsDataSink(elementId)) { + return extensionStorage.getDataSinkById(elementId); + } else if (extensionStorage.existsDataProcessor(elementId)) { + return extensionStorage.getDataProcessorById(elementId); + } else if (extensionStorage.existsDataStream(elementId)) { + return extensionStorage.getDataStreamById(elementId); + } else if (extensionStorage.existsAdapterDescription(elementId)) { + return extensionStorage.getAdapterById(elementId); } else { throw new IllegalArgumentException("Could not find element for ID " + elementId); } } - } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/WorkerAdministrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/WorkerAdministrationResource.java deleted file mode 100644 index d62602a4eb..0000000000 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/WorkerAdministrationResource.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.rest.impl.connect; - -import org.apache.streampipes.connect.management.management.WorkerAdministrationManagement; -import org.apache.streampipes.model.connect.adapter.AdapterDescription; -import org.apache.streampipes.model.message.Notifications; -import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; -import org.apache.streampipes.rest.shared.impl.AbstractSharedRestInterface; - -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; - -import java.util.List; - -@Path("v2/connect/master/administration") -public class WorkerAdministrationResource extends AbstractSharedRestInterface { - - private WorkerAdministrationManagement workerAdministrationManagement; - - public WorkerAdministrationResource() { - this.workerAdministrationManagement = new WorkerAdministrationManagement(); - } - - @POST - @JacksonSerialized - @Produces(MediaType.APPLICATION_JSON) - public Response addWorkerContainer(List availableAdapterDescription) { - - this.workerAdministrationManagement.register(availableAdapterDescription); - - return ok(Notifications.success("Worker Container successfully added")); - } - -} diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PostStartupTask.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PostStartupTask.java new file mode 100644 index 0000000000..9355c6b895 --- /dev/null +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PostStartupTask.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.service.core; + +import org.apache.streampipes.connect.management.management.WorkerAdministrationManagement; +import org.apache.streampipes.manager.operations.Operations; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.model.pipeline.Pipeline; +import org.apache.streampipes.model.pipeline.PipelineOperationStatus; +import org.apache.streampipes.storage.api.IPipelineStorage; +import org.apache.streampipes.storage.couchdb.CouchDbStorageManager; +import org.apache.streampipes.storage.management.StorageDispatcher; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class PostStartupTask implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(PostStartupTask.class); + + private static final int MAX_PIPELINE_START_RETRIES = 3; + private static final int WAIT_TIME_AFTER_FAILURE_IN_SECONDS = 10; + + private final List allPipelines; + private final Map failedPipelines = new HashMap<>(); + private final ScheduledExecutorService executorService; + private final WorkerAdministrationManagement workerAdministrationManagement; + + public PostStartupTask(List allPipelines) { + this.allPipelines = allPipelines; + this.executorService = Executors.newSingleThreadScheduledExecutor(); + this.workerAdministrationManagement = new WorkerAdministrationManagement(); + } + + @Override + public void run() { + performAdapterAssetUpdate(); + startAllPreviouslyStoppedPipelines(); + startAdapters(); + } + + private void performAdapterAssetUpdate() { + var installedAppIds = CouchDbStorageManager.INSTANCE.getExtensionsServiceStorage().getAll() + .stream() + .flatMap(config -> config.getTags().stream()) + .filter(tag -> tag.getPrefix() == SpServiceTagPrefix.ADAPTER) + .toList(); + workerAdministrationManagement.performAdapterMigrations(installedAppIds); + } + + private void startAdapters() { + workerAdministrationManagement.checkAndRestore(0); + } + + private void startAllPreviouslyStoppedPipelines() { + LOG.info("Checking for orphaned pipelines..."); + List orphanedPipelines = allPipelines + .stream() + .filter(Pipeline::isRunning) + .toList(); + + LOG.info("Found {} orphaned pipelines", orphanedPipelines.size()); + + orphanedPipelines.forEach(pipeline -> { + LOG.info("Restoring orphaned pipeline {}", pipeline.getName()); + startPipeline(pipeline, false); + }); + + LOG.info("Checking for gracefully shut down pipelines to be restarted..."); + + List pipelinesToRestart = allPipelines + .stream() + .filter(p -> !(p.isRunning())) + .filter(Pipeline::isRestartOnSystemReboot) + .toList(); + + LOG.info("Found {} pipelines that we are attempting to restart...", pipelinesToRestart.size()); + + pipelinesToRestart.forEach(pipeline -> { + startPipeline(pipeline, false); + }); + + LOG.info("No more pipelines to restore..."); + } + + private void startPipeline(Pipeline pipeline, boolean restartOnReboot) { + PipelineOperationStatus status = Operations.startPipeline(pipeline); + if (status.isSuccess()) { + LOG.info("Pipeline {} successfully restarted", status.getPipelineName()); + Pipeline storedPipeline = getPipelineStorage().getPipeline(pipeline.getPipelineId()); + storedPipeline.setRestartOnSystemReboot(restartOnReboot); + getPipelineStorage().updatePipeline(storedPipeline); + } else { + storeFailedRestartAttempt(pipeline); + int failedAttemptCount = failedPipelines.get(pipeline.getPipelineId()); + if (failedAttemptCount <= MAX_PIPELINE_START_RETRIES) { + LOG.error("Pipeline {} could not be restarted - I'll try again in {} seconds ({}/{} failed attempts)", + pipeline.getName(), + WAIT_TIME_AFTER_FAILURE_IN_SECONDS, + failedAttemptCount, + MAX_PIPELINE_START_RETRIES); + + schedulePipelineStart(pipeline, restartOnReboot); + } else { + LOG.error("Pipeline {} could not be restarted - are all pipeline element containers running?", + status.getPipelineName()); + } + } + } + + private void schedulePipelineStart(Pipeline pipeline, boolean restartOnReboot) { + executorService.schedule(() -> { + startPipeline(pipeline, restartOnReboot); + }, WAIT_TIME_AFTER_FAILURE_IN_SECONDS, TimeUnit.SECONDS); + } + + private void storeFailedRestartAttempt(Pipeline pipeline) { + String pipelineId = pipeline.getPipelineId(); + if (!failedPipelines.containsKey(pipelineId)) { + failedPipelines.put(pipelineId, 1); + } else { + int failedAttempts = failedPipelines.get(pipelineId) + 1; + failedPipelines.put(pipelineId, failedAttempts); + } + } + + private IPipelineStorage getPipelineStorage() { + return StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getPipelineStorageAPI(); + } +} diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java index 56304c5dcf..e03d3291d1 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java @@ -51,11 +51,8 @@ import jakarta.annotation.PreDestroy; import java.net.UnknownHostException; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @Configuration @@ -72,9 +69,6 @@ public class StreamPipesCoreApplication extends StreamPipesServiceBase { private static final Logger LOG = LoggerFactory.getLogger(StreamPipesCoreApplication.class.getCanonicalName()); - private static final int MAX_PIPELINE_START_RETRIES = 3; - private static final int WAIT_TIME_AFTER_FAILURE_IN_SECONDS = 10; - private static final int LOG_FETCH_INTERVAL = 60; private static final TimeUnit LOG_FETCH_UNIT = TimeUnit.SECONDS; @@ -84,13 +78,6 @@ public class StreamPipesCoreApplication extends StreamPipesServiceBase { private static final int SERVICE_HEALTH_CHECK_INTERVAL = 60; private static final TimeUnit SERVICE_HEALTH_CHECK_UNIT = TimeUnit.SECONDS; - private ScheduledExecutorService executorService; - private ScheduledExecutorService healthCheckExecutorService; - private ScheduledExecutorService serviceHealthCheckExecutorService; - private ScheduledExecutorService logCheckExecutorService; - - private final Map failedPipelines = new HashMap<>(); - public static void main(String[] args) { StreamPipesCoreApplication application = new StreamPipesCoreApplication(); application.initialize(() -> List.of( @@ -121,10 +108,10 @@ protected void registerProtocols(SupportedProtocols protocols) { @PostConstruct public void init() { - this.executorService = Executors.newSingleThreadScheduledExecutor(); - this.healthCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); - this.logCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); - this.serviceHealthCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); + var executorService = Executors.newSingleThreadScheduledExecutor(); + var healthCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); + var logCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); + var serviceHealthCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); new StreamPipesEnvChecker().updateEnvironmentVariables(); new CouchDbViewGenerator().createGenericDatabaseIfNotExists(); @@ -135,7 +122,7 @@ public void init() { new MigrationsHandler().performMigrations(); } - executorService.schedule(this::startAllPreviouslyStoppedPipelines, 10, TimeUnit.SECONDS); + executorService.schedule(new PostStartupTask(getAllPipelines()), 10, TimeUnit.SECONDS); LOG.info("Service health check will run every {} seconds", SERVICE_HEALTH_CHECK_INTERVAL); serviceHealthCheckExecutorService.scheduleAtFixedRate(new ServiceHealthCheck(), @@ -176,11 +163,7 @@ private void doInitialSetup() { } } - private void schedulePipelineStart(Pipeline pipeline, boolean restartOnReboot) { - executorService.schedule(() -> { - startPipeline(pipeline, restartOnReboot); - }, WAIT_TIME_AFTER_FAILURE_IN_SECONDS, TimeUnit.SECONDS); - } + @PreDestroy public void onExit() { @@ -211,72 +194,6 @@ public void onExit() { LOG.info("Thanks for using Apache StreamPipes - see you next time!"); } - private void startAllPreviouslyStoppedPipelines() { - LOG.info("Checking for orphaned pipelines..."); - List orphanedPipelines = getAllPipelines() - .stream() - .filter(Pipeline::isRunning) - .toList(); - - LOG.info("Found {} orphaned pipelines", orphanedPipelines.size()); - - orphanedPipelines.forEach(pipeline -> { - LOG.info("Restoring orphaned pipeline {}", pipeline.getName()); - startPipeline(pipeline, false); - }); - - LOG.info("Checking for gracefully shut down pipelines to be restarted..."); - - List pipelinesToRestart = getAllPipelines() - .stream() - .filter(p -> !(p.isRunning())) - .filter(Pipeline::isRestartOnSystemReboot) - .toList(); - - LOG.info("Found {} pipelines that we are attempting to restart...", pipelinesToRestart.size()); - - pipelinesToRestart.forEach(pipeline -> { - startPipeline(pipeline, false); - }); - - LOG.info("No more pipelines to restore..."); - } - - private void startPipeline(Pipeline pipeline, boolean restartOnReboot) { - PipelineOperationStatus status = Operations.startPipeline(pipeline); - if (status.isSuccess()) { - LOG.info("Pipeline {} successfully restarted", status.getPipelineName()); - Pipeline storedPipeline = getPipelineStorage().getPipeline(pipeline.getPipelineId()); - storedPipeline.setRestartOnSystemReboot(restartOnReboot); - getPipelineStorage().updatePipeline(storedPipeline); - } else { - storeFailedRestartAttempt(pipeline); - int failedAttemptCount = failedPipelines.get(pipeline.getPipelineId()); - if (failedAttemptCount <= MAX_PIPELINE_START_RETRIES) { - LOG.error("Pipeline {} could not be restarted - I'll try again in {} seconds ({}/{} failed attempts)", - pipeline.getName(), - WAIT_TIME_AFTER_FAILURE_IN_SECONDS, - failedAttemptCount, - MAX_PIPELINE_START_RETRIES); - - schedulePipelineStart(pipeline, restartOnReboot); - } else { - LOG.error("Pipeline {} could not be restarted - are all pipeline element containers running?", - status.getPipelineName()); - } - } - } - - private void storeFailedRestartAttempt(Pipeline pipeline) { - String pipelineId = pipeline.getPipelineId(); - if (!failedPipelines.containsKey(pipelineId)) { - failedPipelines.put(pipelineId, 1); - } else { - int failedAttempts = failedPipelines.get(pipelineId) + 1; - failedPipelines.put(pipelineId, failedAttempts); - } - } - private List getAllPipelines() { return getPipelineStorage() .getAllPipelines(); diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java index 49c29bdd1d..db54d3c3d0 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java @@ -73,7 +73,6 @@ import org.apache.streampipes.rest.impl.connect.GuessResource; import org.apache.streampipes.rest.impl.connect.RuntimeResolvableResource; import org.apache.streampipes.rest.impl.connect.UnitResource; -import org.apache.streampipes.rest.impl.connect.WorkerAdministrationResource; import org.apache.streampipes.rest.impl.dashboard.Dashboard; import org.apache.streampipes.rest.impl.dashboard.DashboardWidget; import org.apache.streampipes.rest.impl.dashboard.VisualizablePipelineResource; @@ -179,7 +178,6 @@ public Set> getClassesToRegister() { GuessResource.class, UnitResource.class, - WorkerAdministrationResource.class, RuntimeResolvableResource.class ); } diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java index a67e9175eb..cbd3ffa8bf 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java @@ -20,7 +20,6 @@ import org.apache.streampipes.extensions.management.init.DeclarersSingleton; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; -import org.apache.streampipes.service.extensions.connect.ConnectWorkerRegistrationService; import org.apache.streampipes.service.extensions.function.StreamPipesFunctionHandler; import org.apache.streampipes.service.extensions.security.WebSecurityConfig; @@ -46,7 +45,6 @@ public void onExit() { @Override public void afterServiceRegistered(SpServiceDefinition serviceDef) { - new ConnectWorkerRegistrationService().registerWorker(); StreamPipesFunctionHandler.INSTANCE.initializeFunctions(serviceDef.getServiceGroup()); } diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java index 005dd4ca53..e85be4593f 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java @@ -20,6 +20,7 @@ import org.apache.streampipes.rest.extensions.WelcomePage; import org.apache.streampipes.rest.extensions.connect.AdapterAssetResource; +import org.apache.streampipes.rest.extensions.connect.AdapterDescriptionResource; import org.apache.streampipes.rest.extensions.connect.AdapterWorkerResource; import org.apache.streampipes.rest.extensions.connect.GuessResource; import org.apache.streampipes.rest.extensions.connect.HttpServerAdapterResource; @@ -50,6 +51,7 @@ public Set> getClassesToRegister() { AdapterWorkerResource.class, MultiPartFeature.class, AdapterAssetResource.class, + AdapterDescriptionResource.class, HttpServerAdapterResource.class, DataSinkPipelineElementResource.class, diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/connect/ConnectRestClient.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/connect/ConnectRestClient.java deleted file mode 100644 index cacba70d93..0000000000 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/connect/ConnectRestClient.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.service.extensions.connect; - -import org.apache.streampipes.client.StreamPipesClient; -import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver; -import org.apache.streampipes.model.connect.adapter.AdapterDescription; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -public class ConnectRestClient { - - private static final Logger LOG = LoggerFactory.getLogger(ConnectRestClient.class); - - public static boolean register(List allAvailableAdapters) { - - try { - StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); - client.adminApi().registerAdapters(allAvailableAdapters); - return true; - } catch (Exception e) { - LOG.error("Could not register adapter at url - is a 'StreamPipes Core' service running?", e); - return false; - } - } - -} diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/connect/ConnectWorkerRegistrationService.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/connect/ConnectWorkerRegistrationService.java deleted file mode 100644 index 6c00eb17bb..0000000000 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/connect/ConnectWorkerRegistrationService.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.service.extensions.connect; - -import org.apache.streampipes.extensions.management.connect.ConnectWorkerDescriptionProvider; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ConnectWorkerRegistrationService { - - private static final Logger LOG = LoggerFactory.getLogger(ConnectWorkerRegistrationService.class); - - public void registerWorker() { - var connected = false; - - while (!connected) { - connected = ConnectRestClient.register( - new ConnectWorkerDescriptionProvider().getAdapterDescriptions()); - - if (connected) { - LOG.info("Successfully connected to master. Worker is now running."); - } else { - LOG.info("Retrying in 5 seconds"); - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } -} diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IAdapterStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IAdapterStorage.java index 51ca65f6ba..ed88faf134 100644 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IAdapterStorage.java +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IAdapterStorage.java @@ -22,7 +22,7 @@ import java.util.List; -public interface IAdapterStorage { +public interface IAdapterStorage extends CRUDStorage{ List getAllAdapters(); diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java index 01e9fdbea6..fc4a640f73 100644 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java @@ -70,7 +70,7 @@ public interface INoSqlStorage { IPipelineCanvasMetadataStorage getPipelineCanvasMetadataStorage(); - IPipelineElementDescriptionStorageCache getPipelineElementDescriptionStorage(); + IPipelineElementDescriptionStorage getPipelineElementDescriptionStorage(); IPermissionStorage getPermissionStorage(); diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IPipelineElementDescriptionStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IPipelineElementDescriptionStorage.java index 76e289c1d4..b67029147e 100644 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IPipelineElementDescriptionStorage.java +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IPipelineElementDescriptionStorage.java @@ -20,11 +20,10 @@ import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.graph.DataProcessorDescription; import org.apache.streampipes.model.graph.DataSinkDescription; -import org.apache.streampipes.model.staticproperty.StaticProperty; -import java.net.URI; import java.util.List; public interface IPipelineElementDescriptionStorage { @@ -33,34 +32,30 @@ public interface IPipelineElementDescriptionStorage { boolean storeDataStream(SpDataStream stream); - boolean storeDataStream(String jsonld); - boolean storeDataProcessor(DataProcessorDescription processorDescription); - boolean storeDataProcessor(String jsonld); - - SpDataStream getDataStreamById(URI rdfId); - SpDataStream getDataStreamByAppId(String appId); SpDataStream getDataStreamById(String rdfId); DataProcessorDescription getDataProcessorById(String rdfId); - DataProcessorDescription getDataProcessorById(URI rdfId); - DataProcessorDescription getDataProcessorByAppId(String appId); DataSinkDescription getDataSinkById(String rdfId); - DataSinkDescription getDataSinkById(URI rdfId); - DataSinkDescription getDataSinkByAppId(String appId); + AdapterDescription getAdapterById(String elementId); + + AdapterDescription getAdapterByAppId(String appId); + List getAllDataStreams(); List getAllDataProcessors(); + List getAllAdapterDescriptions(); + boolean deleteDataStream(SpDataStream sep); boolean deleteDataStream(String rdfId); @@ -69,8 +64,14 @@ public interface IPipelineElementDescriptionStorage { boolean deleteDataProcessor(String rdfId); + boolean deleteAdapterDescription(AdapterDescription adapterDescription); + + boolean deleteAdapterDescription(String elementId); + boolean exists(SpDataStream stream); + boolean exists(AdapterDescription adapterDescription); + boolean exists(DataProcessorDescription processorDescription); boolean existsDataProcessorByAppId(String appId); @@ -83,6 +84,8 @@ public interface IPipelineElementDescriptionStorage { boolean existsDataSink(String elementId); + boolean existsAdapterDescription(String elementId); + boolean update(SpDataStream stream); boolean update(DataProcessorDescription processorDescription); @@ -91,15 +94,17 @@ public interface IPipelineElementDescriptionStorage { boolean update(DataSinkDescription sec); + boolean update(AdapterDescription adapter); + boolean deleteDataSink(DataSinkDescription sec); boolean deleteDataSink(String rdfId); boolean storeDataSink(DataSinkDescription sec); - List getAllDataSinks(); + boolean storeAdapterDescription(AdapterDescription adapterDescription); - StaticProperty getStaticPropertyById(String rdfId); + List getAllDataSinks(); SpDataStream getEventStreamById(String rdfId); diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IPipelineElementDescriptionStorageCache.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IPipelineElementDescriptionStorageCache.java deleted file mode 100644 index 8f3be0c96d..0000000000 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IPipelineElementDescriptionStorageCache.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.storage.api; - -public interface IPipelineElementDescriptionStorageCache extends IPipelineElementDescriptionStorage { - - void refreshDataProcessorCache(); - - void refreshDataSinkCache(); - - void refreshDataSourceCache(); -} diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java index 988b45ff80..c5bfbfa27c 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java @@ -42,7 +42,7 @@ import org.apache.streampipes.storage.api.IPipelineCanvasMetadataStorage; import org.apache.streampipes.storage.api.IPipelineCategoryStorage; import org.apache.streampipes.storage.api.IPipelineElementConnectionStorage; -import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorageCache; +import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage; import org.apache.streampipes.storage.api.IPipelineElementTemplateStorage; import org.apache.streampipes.storage.api.IPipelineMonitoringDataStorage; import org.apache.streampipes.storage.api.IPipelineStorage; @@ -212,7 +212,7 @@ public IPipelineCanvasMetadataStorage getPipelineCanvasMetadataStorage() { } @Override - public IPipelineElementDescriptionStorageCache getPipelineElementDescriptionStorage() { + public IPipelineElementDescriptionStorage getPipelineElementDescriptionStorage() { return new PipelineElementDescriptionStorageImpl(); } diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterDescriptionStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterDescriptionStorageImpl.java index 3dfe691d45..1fafc6e045 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterDescriptionStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterDescriptionStorageImpl.java @@ -25,17 +25,11 @@ import org.apache.streampipes.storage.couchdb.dao.FindCommand; import org.apache.streampipes.storage.couchdb.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.List; import java.util.Optional; public class AdapterDescriptionStorageImpl extends AbstractDao implements IAdapterStorage { - private static final String SYSTEM_USER = "system"; - Logger logger = LoggerFactory.getLogger(AdapterDescriptionStorageImpl.class); - public AdapterDescriptionStorageImpl() { super(Utils::getCouchDbAdapterDescriptionClient, AdapterDescription.class); } @@ -69,4 +63,36 @@ public void deleteAdapter(String adapterId) { couchDbClientSupplier.get().remove(adapterDescription.getElementId(), adapterDescription.getRev()); } + + @Override + public List getAll() { + return findAll(); + } + + @Override + public void createElement(AdapterDescription adapter) { + persist(adapter); + } + + @Override + public AdapterDescription getElementById(String id) { + return findWithNullIfEmpty(id); + } + + @Override + public AdapterDescription updateElement(AdapterDescription element) { + var rev = getCurrentRev(element.getElementId()); + element.setRev(rev); + update(element); + return getElementById(element.getElementId()); + } + + @Override + public void deleteElement(AdapterDescription element) { + delete(element.getElementId()); + } + + private String getCurrentRev(String elementId) { + return find(elementId).get().getRev(); + } } diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterInstanceStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterInstanceStorageImpl.java index a1653db43d..9dbdcab883 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterInstanceStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterInstanceStorageImpl.java @@ -25,17 +25,11 @@ import org.apache.streampipes.storage.couchdb.dao.FindCommand; import org.apache.streampipes.storage.couchdb.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.List; import java.util.Optional; public class AdapterInstanceStorageImpl extends AbstractDao implements IAdapterStorage { - private static final String SYSTEM_USER = "system"; - Logger logger = LoggerFactory.getLogger(AdapterInstanceStorageImpl.class); - public AdapterInstanceStorageImpl() { super(Utils::getCouchDbAdapterInstanceClient, AdapterDescription.class); } @@ -69,4 +63,30 @@ public void deleteAdapter(String adapterId) { couchDbClientSupplier.get().remove(adapterDescription.getElementId(), adapterDescription.getRev()); } + + @Override + public List getAll() { + return findAll(); + } + + @Override + public void createElement(AdapterDescription adapter) { + persist(adapter); + } + + @Override + public AdapterDescription getElementById(String id) { + return findWithNullIfEmpty(id); + } + + @Override + public AdapterDescription updateElement(AdapterDescription element) { + update(element); + return getElementById(element.getElementId()); + } + + @Override + public void deleteElement(AdapterDescription element) { + delete(element.getElementId()); + } } diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/PipelineElementDescriptionStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/PipelineElementDescriptionStorageImpl.java index ce1b095652..92a7d5ad7b 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/PipelineElementDescriptionStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/PipelineElementDescriptionStorageImpl.java @@ -19,28 +19,30 @@ import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.graph.DataProcessorDescription; import org.apache.streampipes.model.graph.DataSinkDescription; -import org.apache.streampipes.model.staticproperty.StaticProperty; +import org.apache.streampipes.storage.api.IAdapterStorage; import org.apache.streampipes.storage.api.IDataProcessorStorage; import org.apache.streampipes.storage.api.IDataSinkStorage; import org.apache.streampipes.storage.api.IDataStreamStorage; -import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorageCache; +import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage; -import java.net.URI; import java.util.List; import java.util.NoSuchElementException; -public class PipelineElementDescriptionStorageImpl implements IPipelineElementDescriptionStorageCache { +public class PipelineElementDescriptionStorageImpl implements IPipelineElementDescriptionStorage { - private IDataProcessorStorage dataProcessorStorage; - private IDataStreamStorage dataStreamStorage; - private IDataSinkStorage dataSinkStorage; + private final IDataProcessorStorage dataProcessorStorage; + private final IDataStreamStorage dataStreamStorage; + private final IDataSinkStorage dataSinkStorage; + private final IAdapterStorage adapterStorage; public PipelineElementDescriptionStorageImpl() { this.dataProcessorStorage = new DataProcessorStorageImpl(); this.dataStreamStorage = new DataStreamStorageImpl(); this.dataSinkStorage = new DataSinkStorageImpl(); + this.adapterStorage = new AdapterDescriptionStorageImpl(); } @Override @@ -55,28 +57,12 @@ public boolean storeDataStream(SpDataStream stream) { return true; } - @Override - public boolean storeDataStream(String jsonld) { - return false; - } - @Override public boolean storeDataProcessor(DataProcessorDescription processorDescription) { this.dataProcessorStorage.createElement(processorDescription); return true; } - @Override - public boolean storeDataProcessor(String jsonld) { - // TODO check if this can be deleted - return true; - } - - @Override - public SpDataStream getDataStreamById(URI rdfId) { - return getDataStreamById(rdfId.toString()); - } - @Override public SpDataStream getDataStreamByAppId(String appId) { return this.dataStreamStorage.getDataStreamByAppId(appId); @@ -92,11 +78,6 @@ public DataProcessorDescription getDataProcessorById(String rdfId) { return new DataProcessorStorageImpl().getElementById(rdfId); } - @Override - public DataProcessorDescription getDataProcessorById(URI rdfId) { - return this.getDataProcessorById(rdfId.toString()); - } - @Override public DataProcessorDescription getDataProcessorByAppId(String appId) { return this.dataProcessorStorage.getDataProcessorByAppId(appId); @@ -108,13 +89,18 @@ public DataSinkDescription getDataSinkById(String rdfId) { } @Override - public DataSinkDescription getDataSinkById(URI rdfId) { - return getDataSinkById(rdfId.toString()); + public DataSinkDescription getDataSinkByAppId(String appId) { + return this.dataSinkStorage.getDataSinkByAppId(appId); + } + + @Override + public AdapterDescription getAdapterById(String elementId) { + return adapterStorage.getElementById(elementId); } @Override - public DataSinkDescription getDataSinkByAppId(String appId) { - return this.dataSinkStorage.getDataSinkByAppId(appId); + public AdapterDescription getAdapterByAppId(String appId) { + throw new IllegalArgumentException("Not yet implemented"); } @Override @@ -127,6 +113,11 @@ public List getAllDataProcessors() { return this.dataProcessorStorage.getAll(); } + @Override + public List getAllAdapterDescriptions() { + return null; + } + @Override public boolean deleteDataStream(SpDataStream sep) { this.dataStreamStorage.deleteElement(sep); @@ -149,11 +140,28 @@ public boolean deleteDataProcessor(String rdfId) { return deleteDataProcessor(getDataProcessorById(rdfId)); } + @Override + public boolean deleteAdapterDescription(AdapterDescription adapterDescription) { + adapterStorage.deleteAdapter(adapterDescription.getElementId()); + return true; + } + + @Override + public boolean deleteAdapterDescription(String elementId) { + adapterStorage.deleteAdapter(elementId); + return true; + } + @Override public boolean exists(SpDataStream stream) { return getEventStreamById(stream.getElementId()) != null; } + @Override + public boolean exists(AdapterDescription adapterDescription) { + return existsAdapterDescription(adapterDescription.getElementId()); + } + @Override public boolean exists(DataProcessorDescription processorDescription) { return getDataProcessorById(processorDescription.getElementId()) != null; @@ -194,6 +202,11 @@ public boolean existsDataSink(String elementId) { return getDataSinkById(elementId) != null; } + @Override + public boolean existsAdapterDescription(String elementId) { + return getAdapterById(elementId) != null; + } + @Override public boolean update(SpDataStream stream) { this.dataStreamStorage.updateElement(stream); @@ -217,6 +230,12 @@ public boolean update(DataSinkDescription sec) { return true; } + @Override + public boolean update(AdapterDescription adapter) { + adapterStorage.updateElement(adapter); + return true; + } + @Override public boolean deleteDataSink(DataSinkDescription sec) { this.dataSinkStorage.deleteElement(sec); @@ -235,33 +254,18 @@ public boolean storeDataSink(DataSinkDescription sec) { } @Override - public List getAllDataSinks() { - return this.dataSinkStorage.getAll(); + public boolean storeAdapterDescription(AdapterDescription adapterDescription) { + this.adapterStorage.storeAdapter(adapterDescription); + return true; } @Override - public StaticProperty getStaticPropertyById(String rdfId) { - // TODO Check if this is needed - return null; + public List getAllDataSinks() { + return this.dataSinkStorage.getAll(); } @Override public SpDataStream getEventStreamById(String rdfId) { return dataStreamStorage.getElementById(rdfId); } - - @Override - public void refreshDataProcessorCache() { - // TODO no longer needed - } - - @Override - public void refreshDataSinkCache() { - // TODO no longer needed - } - - @Override - public void refreshDataSourceCache() { - // TODO no longer needed - } } diff --git a/streampipes-storage-management/src/main/java/org/apache/streampipes/storage/management/StorageManager.java b/streampipes-storage-management/src/main/java/org/apache/streampipes/storage/management/StorageManager.java index ae74e3ab99..01e9a58117 100644 --- a/streampipes-storage-management/src/main/java/org/apache/streampipes/storage/management/StorageManager.java +++ b/streampipes-storage-management/src/main/java/org/apache/streampipes/storage/management/StorageManager.java @@ -18,14 +18,14 @@ package org.apache.streampipes.storage.management; -import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorageCache; +import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage; import org.apache.streampipes.storage.couchdb.CouchDbStorageManager; public enum StorageManager { INSTANCE; - public IPipelineElementDescriptionStorageCache getPipelineElementStorage() { + public IPipelineElementDescriptionStorage getPipelineElementStorage() { return CouchDbStorageManager.INSTANCE.getPipelineElementDescriptionStorage(); } diff --git a/ui/deployment/_variables.scss b/ui/deployment/_variables.scss index c8de0a5a40..e78c116ffe 100644 --- a/ui/deployment/_variables.scss +++ b/ui/deployment/_variables.scss @@ -27,6 +27,7 @@ $sp-color-accent-light-transparent: rgba(156, 156, 156, 0.4); $sp-color-accent-dark: #83a3de; +$sp-color-adapter: #7f007f; $sp-color-stream: #ffeb3b; $sp-color-set: #ffa23b; $sp-color-processor: #009688; diff --git a/ui/deployment/modules.yml b/ui/deployment/modules.yml index 3c89fa2bce..69e807f615 100644 --- a/ui/deployment/modules.yml +++ b/ui/deployment/modules.yml @@ -77,8 +77,8 @@ spAdd: path: './add/add.module' link: 'add' url: '/add' - title: 'Install Pipeline Elements' - description: 'The pipeline element installation module can be used to extend StreamPipes with new algorithms and data sinks.' + title: 'Install Extensions' + description: 'The extension installation module can be used to install additional adapters, data streams, data processors, and sinks.' icon: 'cloud_download' homeImage: '/assets/img/home/add.png' admin: True diff --git a/ui/src/app/add/add.component.html b/ui/src/app/add/add.component.html index aba281ebb3..624b881aea 100644 --- a/ui/src/app/add/add.component.html +++ b/ui/src/app/add/add.component.html @@ -77,7 +77,7 @@ (selectionChange)="filterByCatergory($event)" > All - Sets + Adapters Streams Processors Sinks @@ -149,7 +149,7 @@ >
@@ -183,8 +183,7 @@ color="accent" >

-   Searching for available pipeline elements, please - wait... +   Searching for available extensions, please wait...

diff --git a/ui/src/app/add/add.routes.ts b/ui/src/app/add/add.routes.ts index b6945d3bf0..dc74251230 100644 --- a/ui/src/app/add/add.routes.ts +++ b/ui/src/app/add/add.routes.ts @@ -21,7 +21,7 @@ import { SpBreadcrumbItem } from '@streampipes/shared-ui'; export class SpAddRoutes { static ADD_BASE_LINK = 'add'; static BASE: SpBreadcrumbItem = { - label: 'Install Pipeline Elements', + label: 'Install Extensions', link: [SpAddRoutes.ADD_BASE_LINK], }; } diff --git a/ui/src/app/add/components/endpoint-item/endpoint-item.component.scss b/ui/src/app/add/components/endpoint-item/endpoint-item.component.scss index 12731c0b16..47bc675003 100644 --- a/ui/src/app/add/components/endpoint-item/endpoint-item.component.scss +++ b/ui/src/app/add/components/endpoint-item/endpoint-item.component.scss @@ -37,8 +37,6 @@ } .icon { - height: 50px; - margin-top: -5px; width: 50px; } diff --git a/ui/src/app/add/components/endpoint-item/endpoint-item.component.ts b/ui/src/app/add/components/endpoint-item/endpoint-item.component.ts index f77b1cedf1..73c8f331a1 100644 --- a/ui/src/app/add/components/endpoint-item/endpoint-item.component.ts +++ b/ui/src/app/add/components/endpoint-item/endpoint-item.component.ts @@ -99,7 +99,9 @@ export class EndpointItemComponent implements OnInit { } findItemTypeTitle() { - if (this.item.type === 'stream') { + if (this.item.type === 'adapter') { + this.itemTypeTitle = 'Adapter'; + } else if (this.item.type === 'stream') { this.itemTypeTitle = 'Data Stream'; } else if (this.item.type === 'sepa') { this.itemTypeTitle = 'Data Processor'; @@ -112,8 +114,8 @@ export class EndpointItemComponent implements OnInit { const baseType = 'pe-label '; if (this.item.type === 'stream') { this.itemTypeStyle = baseType + 'stream-label'; - } else if (this.item.type === 'set') { - this.itemTypeStyle = baseType + 'set-label'; + } else if (this.item.type === 'adapter') { + this.itemTypeStyle = baseType + 'adapter-label'; } else if (this.item.type === 'sepa') { this.itemTypeStyle = baseType + 'processor-label'; } else { diff --git a/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.html b/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.html index 10784aacc9..66639d2d09 100644 --- a/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.html +++ b/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.html @@ -23,46 +23,14 @@ fxLayout="row" fxLayoutAlign="start start" fxFlex="100" - > -
- - - - -
+ > +
+
-
- - -
-

{{ adapter.name }}

+

{{ adapter.name }}

-
+

diff --git a/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.scss b/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.scss index 9390988753..7122b43da4 100644 --- a/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.scss +++ b/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.scss @@ -67,7 +67,25 @@ color: #ffeb3b; } +.adapter-icon { + margin-left: 10px; + box-shadow: + rgba(0, 0, 0, 0.05) 0px 6px 24px 0px, + rgba(0, 0, 0, 0.08) 0px 0px 0px 1px; + text-align: center; + display: inline-block; +} + .icon { - margin-top: 20px; - height: 50px; + max-width: 60px; +} + +.large-icon { + line-height: 90px; + width: 90px; + height: 90px; +} + +.ml-10 { + margin-left: 10px; } diff --git a/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.ts b/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.ts index 5dfab94aea..2bcd062365 100644 --- a/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.ts +++ b/ui/src/app/connect/components/data-marketplace/adapter-description/adapter-description.component.ts @@ -17,7 +17,6 @@ */ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; -import { ConnectService } from '../../../services/connect.service'; import { MatDialog } from '@angular/material/dialog'; import { AdapterDescription, @@ -25,6 +24,8 @@ import { } from '@streampipes/platform-services'; import { DialogService } from '@streampipes/shared-ui'; import { MatSnackBar } from '@angular/material/snack-bar'; +import { RestApi } from '../../../../services/rest-api.service'; +import { DomSanitizer, SafeUrl } from '@angular/platform-browser'; @Component({ selector: 'sp-adapter-description', @@ -45,11 +46,12 @@ export class AdapterDescriptionComponent implements OnInit { className = ''; isRunningAdapter = false; adapterLabel: string; + iconUrl: SafeUrl; constructor( - private connectService: ConnectService, + private restApi: RestApi, private dataMarketplaceService: AdapterService, - private dialogService: DialogService, + private sanitizer: DomSanitizer, public dialog: MatDialog, private _snackBar: MatSnackBar, ) {} @@ -63,6 +65,9 @@ export class AdapterDescriptionComponent implements OnInit { !(this.adapter as any).isTemplate; this.adapterLabel = this.adapter.name.split(' ').join('_'); this.className = this.getClassName(); + this.iconUrl = this.sanitizer.bypassSecurityTrustUrl( + this.makeAssetIconUrl(), + ); } getClassName() { @@ -87,18 +92,7 @@ export class AdapterDescriptionComponent implements OnInit { } } - removeAdapter(): void { - this.dataMarketplaceService - .deleteAdapterDescription(this.adapter.elementId) - .subscribe({ - next: () => { - this.updateAdapterEmitter.emit(); - }, - error: () => { - this._snackBar.open( - 'Cannot delete an adapter which has an active instance running.', - ); - }, - }); + makeAssetIconUrl() { + return this.restApi.getAssetUrl(this.adapter.appId) + '/icon'; } } diff --git a/ui/src/app/connect/components/existing-adapters/existing-adapters.component.scss b/ui/src/app/connect/components/existing-adapters/existing-adapters.component.scss index 2962aaee66..4cc00fa1f5 100644 --- a/ui/src/app/connect/components/existing-adapters/existing-adapters.component.scss +++ b/ui/src/app/connect/components/existing-adapters/existing-adapters.component.scss @@ -52,3 +52,7 @@ padding: 5px; text-align: center; } + +.icon { + max-width: 60px; +} diff --git a/ui/src/app/connect/components/existing-adapters/existing-adapters.component.ts b/ui/src/app/connect/components/existing-adapters/existing-adapters.component.ts index 4cfb1c7b8e..e7c6453ea2 100644 --- a/ui/src/app/connect/components/existing-adapters/existing-adapters.component.ts +++ b/ui/src/app/connect/components/existing-adapters/existing-adapters.component.ts @@ -42,7 +42,6 @@ import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; import { ObjectPermissionDialogComponent } from '../../../core-ui/object-permission-dialog/object-permission-dialog.component'; import { UserRole } from '../../../_enums/user-role.enum'; -import { AuthService } from '../../../services/auth.service'; import { HelpComponent } from '../../../editor/dialog/help/help.component'; import { Router } from '@angular/router'; import { AdapterFilterSettingsModel } from '../../model/adapter-filter-settings.model'; @@ -50,6 +49,7 @@ import { AdapterFilterPipe } from '../../filter/adapter-filter.pipe'; import { SpConnectRoutes } from '../../connect.routes'; import { CanNotEditAdapterDialog } from '../../dialog/can-not-edit-adapter-dialog/can-not-edit-adapter-dialog.component'; import { zip } from 'rxjs'; +import { RestApi } from '../../../services/rest-api.service'; @Component({ selector: 'sp-existing-adapters', @@ -89,6 +89,7 @@ export class ExistingAdaptersComponent implements OnInit { private pipelineElementService: PipelineElementService, private pipelineService: PipelineService, private router: Router, + private restApi: RestApi, private adapterFilter: AdapterFilterPipe, private breadcrumbService: SpBreadcrumbService, private adapterMonitoringService: AdapterMonitoringService, @@ -195,9 +196,7 @@ export class ExistingAdaptersComponent implements OnInit { getIconUrl(adapter: AdapterDescription) { if (adapter.includedAssets.length > 0) { - return this.adapterService.getAssetUrl(adapter.appId) + '/icon'; - } else { - return 'assets/img/connect/' + adapter.iconUrl; + return this.restApi.getAssetUrl(adapter.appId) + '/icon'; } } diff --git a/ui/src/scss/sp/main.scss b/ui/src/scss/sp/main.scss index 80ebae4a33..538ec8b3a9 100644 --- a/ui/src/scss/sp/main.scss +++ b/ui/src/scss/sp/main.scss @@ -38,6 +38,7 @@ body { height: auto !important; width: 100%; + --color-adapter: #{$sp-color-adapter}; --color-set: #{$sp-color-set}; --color-stream: #{$sp-color-stream}; --color-processor: #{$sp-color-processor}; @@ -526,13 +527,13 @@ md-content { padding: 2px 5px; } -.stream-label { - background: $sp-color-stream; - color: black; +.adapter-label { + background: $sp-color-adapter; + color: white; } -.set-label { - background: $sp-color-set; +.stream-label { + background: $sp-color-stream; color: black; } @@ -544,6 +545,11 @@ md-content { background: $sp-color-sink; } +.adapter { + border-radius: 50%; + border: 2px solid $sp-color-adapter; +} + .stream { border-radius: 50%; border: 2px solid $sp-color-stream; From dbf4cec792b33e9272f986248cde314da7f4fe9a Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Mon, 9 Oct 2023 17:05:15 +0200 Subject: [PATCH 02/54] Fix checkstyle --- .../ExtensionsServiceEndpointResource.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java index 6b65ec8770..13ef985588 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java @@ -154,8 +154,10 @@ private List getAllAdapterEndpoints( .collect(Collectors.toList()); } - private List getAllDataStreamEndpoints(String username, - List existingItems) { + private List getAllDataStreamEndpoints( + String username, + List existingItems) { + return getAllDataStreams() .stream() .filter(s -> existingItems.stream().noneMatch(item -> s.getAppId().equals(item.getAppId()))) @@ -165,8 +167,10 @@ private List getAllDataStreamEndpoints(String use } - private List getAllDataProcessorEndpoints(String username, - List existingItems) { + private List getAllDataProcessorEndpoints( + String username, + List existingItems) { + return getAllDataProcessors() .stream() .filter(s -> existingItems.stream().noneMatch(item -> s.getAppId().equals(item.getAppId()))) @@ -174,8 +178,10 @@ private List getAllDataProcessorEndpoints(String .collect(Collectors.toList()); } - private List getAllDataSinkEndpoints(String username, - List existingItems) { + private List getAllDataSinkEndpoints( + String username, + List existingItems) { + return getAllDataSinks() .stream() .filter(s -> existingItems.stream().noneMatch(item -> s.getAppId().equals(item.getAppId()))) From d437edcf2ef96f3e0e6fccc85854e6744c68068d Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 9 Oct 2023 17:54:05 +0200 Subject: [PATCH 03/54] style: remove trailing whitespace --- .../rest/impl/admin/ExtensionsServiceEndpointResource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java index 13ef985588..134aeac28d 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java @@ -157,7 +157,6 @@ private List getAllAdapterEndpoints( private List getAllDataStreamEndpoints( String username, List existingItems) { - return getAllDataStreams() .stream() .filter(s -> existingItems.stream().noneMatch(item -> s.getAppId().equals(item.getAppId()))) From 969d2ac02c81a94bef56d382972bcc84099830ec Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Tue, 17 Oct 2023 16:15:08 +0200 Subject: [PATCH 04/54] Add initial draft of migration concept --- .../exceptions/SpAutoMigrateException.java | 50 ++++++++++++++ .../api/migration/AdapterMigrator.java | 25 +++++++ .../api/migration/DataProcessorMigrator.java | 27 ++++++++ .../api/migration/DataSinkMigrator.java | 25 +++++++ .../api/migration/MigrationResult.java | 67 +++++++++++++++++++ .../api/migration/ModelMigrator.java | 36 ++++++++++ .../management/model/SpServiceDefinition.java | 11 +++ .../model/SpServiceDefinitionBuilder.java | 6 ++ .../internal/jvm/DataLakeMigrationv1.java | 56 ++++++++++++++++ .../internal/jvm/SinksInternalJvmInit.java | 3 + 10 files changed, 306 insertions(+) create mode 100644 streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/SpAutoMigrateException.java create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java create mode 100644 streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/DataLakeMigrationv1.java diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/SpAutoMigrateException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/SpAutoMigrateException.java new file mode 100644 index 0000000000..3542ef2d71 --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/SpAutoMigrateException.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class SpAutoMigrateException extends Exception { + + /** + * Creates a new Exception with the given message and null as the cause. + * + * @param message The exception message + */ + public SpAutoMigrateException(String message) { + super(message); + } + + /** + * Creates a new exception with a null message and the given cause. + * + * @param cause The exception that caused this exception + */ + public SpAutoMigrateException(Throwable cause) { + super(cause); + } + + /** + * Creates a new exception with the given message and cause + * + * @param message The exception message + * @param cause The exception that caused this exception + */ + public SpAutoMigrateException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java new file mode 100644 index 0000000000..f8f5f73d54 --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.extensions.api.extractor.IAdapterParameterExtractor; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; + +public interface AdapterMigrator extends ModelMigrator { +} diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java new file mode 100644 index 0000000000..2c395ef862 --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.extensions.api.extractor.IDataProcessorParameterExtractor; +import org.apache.streampipes.model.graph.DataProcessorInvocation; + +public interface DataProcessorMigrator + extends ModelMigrator { + +} diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java new file mode 100644 index 0000000000..dafee59c41 --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.extensions.api.extractor.IDataSinkParameterExtractor; +import org.apache.streampipes.model.graph.DataSinkInvocation; + +public interface DataSinkMigrator extends ModelMigrator { +} diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java new file mode 100644 index 0000000000..aa7b17838e --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.model.Notification; + +import java.util.List; + +public class MigrationResult { + + boolean success; + List messages; + + T element; + + public static MigrationResult success(T element) { + return new MigrationResult<>(true, element, List.of()); + } + + private MigrationResult(boolean success, + T element, + List messages) { + this.success = success; + this.element = element; + this.messages = messages; + } + + public boolean isSuccess() { + return success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + public List getMessages() { + return messages; + } + + public void setMessages(List messages) { + this.messages = messages; + } + + public T getElement() { + return element; + } + + public void setElement(T element) { + this.element = element; + } +} diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java new file mode 100644 index 0000000000..201546106b --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.commons.exceptions.SpAutoMigrateException; +import org.apache.streampipes.commons.exceptions.SpConfigurationException; +import org.apache.streampipes.extensions.api.extractor.IParameterExtractor; +import org.apache.streampipes.model.base.NamedStreamPipesEntity; + +public interface ModelMigrator> { + + String targetsAppId(); + + Integer fromVersion(); + + Integer toVersion(); + + MigrationResult migrate(T element, ExT extractor) throws SpConfigurationException, SpAutoMigrateException; + +} diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java index ca22193907..30a3bcd063 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java @@ -20,6 +20,7 @@ import org.apache.streampipes.dataformat.SpDataFormatFactory; import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter; import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer; +import org.apache.streampipes.extensions.api.migration.ModelMigrator; import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; import org.apache.streampipes.extensions.api.pe.runtime.IStreamPipesRuntimeProvider; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; @@ -47,6 +48,7 @@ public class SpServiceDefinition { private List kvConfigs; private List runtimeProviders; + private List> migrators; public SpServiceDefinition() { this.serviceId = UUID.randomUUID().toString(); @@ -57,6 +59,7 @@ public SpServiceDefinition() { this.functions = new ArrayList<>(); this.adapters = new ArrayList<>(); this.runtimeProviders = new ArrayList<>(); + this.migrators = new ArrayList<>(); } public String getServiceGroup() { @@ -182,4 +185,12 @@ public List getRuntimeProviders() { public void addRuntimeProvider(IStreamPipesRuntimeProvider runtimeProvider) { this.runtimeProviders.add(runtimeProvider); } + + public List> getMigrators() { + return this.migrators; + } + + public void addMigrators(List> migrators) { + this.migrators = migrators; + } } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java index a350950c9e..18d7adadf5 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java @@ -20,6 +20,7 @@ import org.apache.streampipes.dataformat.SpDataFormatFactory; import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter; import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer; +import org.apache.streampipes.extensions.api.migration.ModelMigrator; import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; import org.apache.streampipes.extensions.api.pe.runtime.IStreamPipesRuntimeProvider; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; @@ -123,6 +124,11 @@ public SpServiceDefinitionBuilder registerMessagingProtocols(SpProtocolDefinitio return this; } + public SpServiceDefinitionBuilder registerMigrators(ModelMigrator... migrations) { + this.serviceDefinition.addMigrators(List.of(migrations)); + return this; + } + public SpServiceDefinitionBuilder merge(SpServiceDefinition other) { this.serviceDefinition.addDeclarers(other.getDeclarers()); this.serviceDefinition.addAdapters(other.getAdapters()); diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/DataLakeMigrationv1.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/DataLakeMigrationv1.java new file mode 100644 index 0000000000..be70b0e83f --- /dev/null +++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/DataLakeMigrationv1.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.sinks.internal.jvm; + +import org.apache.streampipes.commons.exceptions.SpConfigurationException; +import org.apache.streampipes.extensions.api.extractor.IDataSinkParameterExtractor; +import org.apache.streampipes.extensions.api.migration.MigrationResult; +import org.apache.streampipes.extensions.api.migration.ModelMigrator; +import org.apache.streampipes.model.graph.DataSinkInvocation; +import org.apache.streampipes.sdk.StaticProperties; +import org.apache.streampipes.sdk.helpers.Labels; +import org.apache.streampipes.sdk.utils.Datatypes; + +public class DataLakeMigrationv1 implements ModelMigrator { + + @Override + public String targetsAppId() { + return "org.apache.streampipes.sinks.internal.jvm.datalake"; + } + + @Override + public Integer fromVersion() { + return 0; + } + + @Override + public Integer toVersion() { + return 1; + } + + @Override + public MigrationResult migrate(DataSinkInvocation element, + IDataSinkParameterExtractor extractor) throws SpConfigurationException { + element.getStaticProperties().add( + StaticProperties.freeTextProperty(Labels.empty(), Datatypes.String) + ); + + return MigrationResult.success(element); + } +} diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/SinksInternalJvmInit.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/SinksInternalJvmInit.java index 38961624c8..d18b68625d 100644 --- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/SinksInternalJvmInit.java +++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/SinksInternalJvmInit.java @@ -53,6 +53,9 @@ public SpServiceDefinition provideServiceDefinition() { new SpJmsProtocolFactory(), new SpMqttProtocolFactory(), new SpNatsProtocolFactory()) + .registerMigrators( + new DataLakeMigrationv1() + ) .build(); From 6f8ad83651a21c546a461731dd619d1fc48366c6 Mon Sep 17 00:00:00 2001 From: bossenti Date: Thu, 19 Oct 2023 08:44:52 +0200 Subject: [PATCH 05/54] refactor: fix logger configuration --- .../management/management/WorkerAdministrationManagement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java index 15ec1f55a3..df88329323 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java @@ -39,7 +39,7 @@ public class WorkerAdministrationManagement { - private static final Logger LOG = LoggerFactory.getLogger(AdapterMasterManagement.class); + private static final Logger LOG = LoggerFactory.getLogger(WorkerAdministrationManagement.class); private static final int MAX_RETRIES = 7; private final IAdapterStorage adapterDescriptionStorage; From 01ec0327bbf91e6b36860019d30f6f5400702a47 Mon Sep 17 00:00:00 2001 From: bossenti Date: Thu, 19 Oct 2023 09:55:31 +0200 Subject: [PATCH 06/54] refactor: extend storage implementations by method to get all instances by the app id --- .../storage/api/IAdapterStorage.java | 4 ++++ .../storage/api/IDataProcessorStorage.java | 6 +++++- .../storage/api/IDataSinkStorage.java | 6 +++++- .../impl/AdapterDescriptionStorageImpl.java | 18 +++++++++++++++++ .../impl/AdapterInstanceStorageImpl.java | 20 ++++++++++++++++++- .../impl/DataProcessorStorageImpl.java | 10 +++++++++- .../couchdb/impl/DataSinkStorageImpl.java | 10 +++++++++- ...PipelineElementDescriptionStorageImpl.java | 6 +++--- 8 files changed, 72 insertions(+), 8 deletions(-) diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IAdapterStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IAdapterStorage.java index ed88faf134..651d7274ba 100644 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IAdapterStorage.java +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IAdapterStorage.java @@ -33,4 +33,8 @@ public interface IAdapterStorage extends CRUDStorage AdapterDescription getAdapter(String adapterId); void deleteAdapter(String adapterId); + + AdapterDescription getFirstAdapterByAppId(String appId); + + List getAdaptersByAppId(String appId); } diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IDataProcessorStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IDataProcessorStorage.java index dfc02b8649..c9f4d6630c 100644 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IDataProcessorStorage.java +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IDataProcessorStorage.java @@ -19,7 +19,11 @@ import org.apache.streampipes.model.graph.DataProcessorDescription; +import java.util.List; + public interface IDataProcessorStorage extends CRUDStorage { - DataProcessorDescription getDataProcessorByAppId(String appId); + DataProcessorDescription getFirstDataProcessorByAppId(String appId); + + List getDataProcessorsByAppId(String appId); } diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IDataSinkStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IDataSinkStorage.java index 7f67c07304..e740816d70 100644 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IDataSinkStorage.java +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IDataSinkStorage.java @@ -19,7 +19,11 @@ import org.apache.streampipes.model.graph.DataSinkDescription; +import java.util.List; + public interface IDataSinkStorage extends CRUDStorage { - DataSinkDescription getDataSinkByAppId(String appId); + DataSinkDescription getFirstDataSinkByAppId(String appId); + + List getDataSinksByAppId(String appId); } diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterDescriptionStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterDescriptionStorageImpl.java index 1fafc6e045..9a771a408b 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterDescriptionStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterDescriptionStorageImpl.java @@ -26,6 +26,7 @@ import org.apache.streampipes.storage.couchdb.utils.Utils; import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; public class AdapterDescriptionStorageImpl extends AbstractDao implements IAdapterStorage { @@ -64,6 +65,23 @@ public void deleteAdapter(String adapterId) { } + @Override + public AdapterDescription getFirstAdapterByAppId(String appId) { + return getAll() + .stream() + .filter(p -> p.getAppId().equals(appId)) + .findFirst() + .orElseThrow(NoSuchElementException::new); + } + + @Override + public List getAdaptersByAppId(String appId) { + return getAll() + .stream() + .filter(p -> p.getAppId().equals(appId)) + .toList(); + } + @Override public List getAll() { return findAll(); diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterInstanceStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterInstanceStorageImpl.java index 9dbdcab883..e8cc511374 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterInstanceStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/AdapterInstanceStorageImpl.java @@ -26,6 +26,7 @@ import org.apache.streampipes.storage.couchdb.utils.Utils; import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; public class AdapterInstanceStorageImpl extends AbstractDao implements IAdapterStorage { @@ -52,7 +53,7 @@ public void updateAdapter(AdapterDescription adapter) { @Override public AdapterDescription getAdapter(String adapterId) { DbCommand, AdapterDescription> cmd = - new FindCommand<>(couchDbClientSupplier, adapterId, AdapterDescription.class); + new FindCommand<>(couchDbClientSupplier, adapterId, AdapterDescription.class); return cmd.execute().orElse(null); } @@ -79,6 +80,23 @@ public AdapterDescription getElementById(String id) { return findWithNullIfEmpty(id); } + @Override + public AdapterDescription getFirstAdapterByAppId(String appId) { + return getAll() + .stream() + .filter(p -> p.getAppId().equals(appId)) + .findFirst() + .orElseThrow(NoSuchElementException::new); + } + + @Override + public List getAdaptersByAppId(String appId) { + return getAll() + .stream() + .filter(p -> p.getAppId().equals(appId)) + .toList(); + } + @Override public AdapterDescription updateElement(AdapterDescription element) { update(element); diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/DataProcessorStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/DataProcessorStorageImpl.java index c046e314b0..cf013b56ea 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/DataProcessorStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/DataProcessorStorageImpl.java @@ -60,7 +60,7 @@ public void deleteElement(DataProcessorDescription element) { } @Override - public DataProcessorDescription getDataProcessorByAppId(String appId) { + public DataProcessorDescription getFirstDataProcessorByAppId(String appId) { return getAll() .stream() .filter(p -> p.getAppId().equals(appId)) @@ -68,6 +68,14 @@ public DataProcessorDescription getDataProcessorByAppId(String appId) { .orElseThrow(NoSuchElementException::new); } + @Override + public List getDataProcessorsByAppId(String appId) { + return getAll() + .stream() + .filter(p -> p.getAppId().equals(appId)) + .toList(); + } + private String getCurrentRev(String elementId) { return find(elementId).get().getRev(); } diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/DataSinkStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/DataSinkStorageImpl.java index 5f80ea77fb..f15149c1b1 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/DataSinkStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/DataSinkStorageImpl.java @@ -59,7 +59,7 @@ public void deleteElement(DataSinkDescription element) { } @Override - public DataSinkDescription getDataSinkByAppId(String appId) { + public DataSinkDescription getFirstDataSinkByAppId(String appId) { return getAll() .stream() .filter(s -> s.getAppId().equals(appId)) @@ -67,6 +67,14 @@ public DataSinkDescription getDataSinkByAppId(String appId) { .orElseThrow(IllegalArgumentException::new); } + @Override + public List getDataSinksByAppId(String appId) { + return getAll() + .stream() + .filter(s -> s.getAppId().equals(appId)) + .toList(); + } + private String getCurrentRev(String elementId) { return find(elementId).get().getRev(); } diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/PipelineElementDescriptionStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/PipelineElementDescriptionStorageImpl.java index 92a7d5ad7b..0b8e26ca6a 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/PipelineElementDescriptionStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/PipelineElementDescriptionStorageImpl.java @@ -80,7 +80,7 @@ public DataProcessorDescription getDataProcessorById(String rdfId) { @Override public DataProcessorDescription getDataProcessorByAppId(String appId) { - return this.dataProcessorStorage.getDataProcessorByAppId(appId); + return this.dataProcessorStorage.getFirstDataProcessorByAppId(appId); } @Override @@ -90,7 +90,7 @@ public DataSinkDescription getDataSinkById(String rdfId) { @Override public DataSinkDescription getDataSinkByAppId(String appId) { - return this.dataSinkStorage.getDataSinkByAppId(appId); + return this.dataSinkStorage.getFirstDataSinkByAppId(appId); } @Override @@ -100,7 +100,7 @@ public AdapterDescription getAdapterById(String elementId) { @Override public AdapterDescription getAdapterByAppId(String appId) { - throw new IllegalArgumentException("Not yet implemented"); + return this.adapterStorage.getFirstAdapterByAppId(appId); } @Override From a23df74dcdc2de988b01120505708eee240c39ba Mon Sep 17 00:00:00 2001 From: bossenti Date: Thu, 19 Oct 2023 11:23:34 +0200 Subject: [PATCH 07/54] refactor: update generated typescript model --- .../src/lib/model/gen/streampipes-model.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts index 36a810b513..1608dddbc8 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts @@ -16,11 +16,10 @@ * specific language governing permissions and limitations * under the License. */ - /* tslint:disable */ /* eslint-disable */ // @ts-nocheck -// Generated using typescript-generator version 3.1.1185 on 2023-08-06 11:37:37. +// Generated using typescript-generator version 3.2.1263 on 2023-10-19 10:40:49. export class NamedStreamPipesEntity { '@class': @@ -102,6 +101,7 @@ export class AdapterDescription extends NamedStreamPipesEntity { 'selectedEndpointUrl': string; 'streamRules': TransformationRuleDescriptionUnion[]; 'valueRules': TransformationRuleDescriptionUnion[]; + 'version': number; static 'fromData'( data: AdapterDescription, @@ -140,6 +140,7 @@ export class AdapterDescription extends NamedStreamPipesEntity { instance.valueRules = __getCopyArrayFn( TransformationRuleDescription.fromDataUnion, )(data.valueRules); + instance.version = data.version; return instance; } } @@ -198,6 +199,7 @@ export class TransformationRuleDescription { | 'org.apache.streampipes.model.connect.rules.schema.DeleteRuleDescription' | 'org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription' | 'org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription'; + 'rulePriority': number; static 'fromData'( data: TransformationRuleDescription, @@ -208,6 +210,7 @@ export class TransformationRuleDescription { } const instance = target || new TransformationRuleDescription(); instance['@class'] = data['@class']; + instance.rulePriority = data.rulePriority; return instance; } @@ -3521,6 +3524,7 @@ export class SpServiceRegistration { scheme: string; svcGroup: string; svcId: string; + svcType: string; tags: SpServiceTag[]; static fromData( @@ -3540,6 +3544,7 @@ export class SpServiceRegistration { instance.scheme = data.scheme; instance.svcGroup = data.svcGroup; instance.svcId = data.svcId; + instance.svcType = data.svcType; instance.tags = __getCopyArrayFn(SpServiceTag.fromData)(data.tags); return instance; } From ecd20c4e8f6ef8636996e799064abba43c5ff269 Mon Sep 17 00:00:00 2001 From: bossenti Date: Thu, 19 Oct 2023 11:26:38 +0200 Subject: [PATCH 08/54] feat: add version to models & builders --- .../base/ConsumableStreamPipesEntity.java | 15 ++++++++++- .../base/VersionedStreamPipesEntity.java | 25 +++++++++++++++++++ .../connect/adapter/AdapterDescription.java | 17 +++++++++++-- .../AbstractProcessingElementBuilder.java | 4 +++ .../adapter/AdapterConfigurationBuilder.java | 5 +++- 5 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedStreamPipesEntity.java diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java index 67be04100a..c3550bf1ce 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java @@ -26,10 +26,12 @@ import java.util.ArrayList; import java.util.List; -public abstract class ConsumableStreamPipesEntity extends NamedStreamPipesEntity { +public abstract class ConsumableStreamPipesEntity extends NamedStreamPipesEntity implements VersionedStreamPipesEntity{ private static final long serialVersionUID = -6617391345752016449L; + private int version; + protected List spDataStreams; protected List staticProperties; @@ -57,6 +59,7 @@ public ConsumableStreamPipesEntity(ConsumableStreamPipesEntity other) { if (other.getSupportedGrounding() != null) { this.supportedGrounding = new EventGrounding(other.getSupportedGrounding()); } + this.version = other.version; } public List getSpDataStreams() { @@ -87,4 +90,14 @@ public void setSupportedGrounding(EventGrounding supportedGrounding) { this.supportedGrounding = supportedGrounding; } + @Override + public int getVersion(){ + return this.version; + } + + @Override + public void setVersion(int version){ + this.version = version; + } + } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedStreamPipesEntity.java new file mode 100644 index 0000000000..5194340dc6 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedStreamPipesEntity.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.base; + +public interface VersionedStreamPipesEntity { + int getVersion(); + + void setVersion(int version); +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java index d77d8567a3..f78df953a2 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java @@ -20,6 +20,7 @@ import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.base.VersionedStreamPipesEntity; import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; import org.apache.streampipes.model.connect.rules.schema.SchemaTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.stream.StreamTransformationRuleDescription; @@ -35,11 +36,12 @@ import java.util.List; @TsModel -public class AdapterDescription extends NamedStreamPipesEntity { +public class AdapterDescription extends NamedStreamPipesEntity implements VersionedStreamPipesEntity { protected SpDataStream dataStream; protected boolean running; + private int version; private EventGrounding eventGrounding; private String icon; @@ -74,11 +76,12 @@ public AdapterDescription() { this.dataStream = new SpDataStream(); } - public AdapterDescription(String elementId, String name, String description) { + public AdapterDescription(String elementId, String name, String description, int version) { super(elementId, name, description); this.rules = new ArrayList<>(); this.category = new ArrayList<>(); this.dataStream = new SpDataStream(); + this.version = version; } @@ -87,6 +90,7 @@ public AdapterDescription(AdapterDescription other) { this.config = new Cloner().staticProperties(other.getConfig()); this.rules = other.getRules(); this.icon = other.getIcon(); + this.version = other.getVersion(); this.category = new Cloner().epaTypes(other.getCategory()); this.createdAt = other.getCreatedAt(); this.selectedEndpointUrl = other.getSelectedEndpointUrl(); @@ -250,4 +254,13 @@ public void setRunning(boolean running) { this.running = running; } + @Override + public int getVersion() { + return version; + } + + @Override + public void setVersion(int newVersion) { + this.version = newVersion; + } } diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/AbstractProcessingElementBuilder.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/AbstractProcessingElementBuilder.java index 2a91e74ae0..8db25a0578 100644 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/AbstractProcessingElementBuilder.java +++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/AbstractProcessingElementBuilder.java @@ -264,6 +264,10 @@ public K setStream2() { stream2 = true; return me(); } + public K withVersion(int version) { + this.elementDescription.setVersion(version); + return me(); + } @Override diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterConfigurationBuilder.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterConfigurationBuilder.java index 3d626e2e59..4dd39e1c80 100644 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterConfigurationBuilder.java +++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterConfigurationBuilder.java @@ -87,5 +87,8 @@ public AdapterConfigurationBuilder withCategory(AdapterType... categories) { return me(); } - + public AdapterConfigurationBuilder withVersion(int version) { + this.elementDescription.setVersion(version); + return this; + } } From 646e792d50b3022748e99be738c390270b87c345 Mon Sep 17 00:00:00 2001 From: bossenti Date: Fri, 20 Oct 2023 10:54:58 +0200 Subject: [PATCH 09/54] refactor: implement string representation of Notification --- .../apache/streampipes/model/Notification.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/Notification.java b/streampipes-model/src/main/java/org/apache/streampipes/model/Notification.java index 3b9f47f84c..48d8c75497 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/Notification.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/Notification.java @@ -26,8 +26,10 @@ public class Notification { - private @JsonProperty("_id") @SerializedName("_id") String id; - private @JsonProperty("_rev") @SerializedName("_rev") String rev; + private @JsonProperty("_id") + @SerializedName("_id") String id; + private @JsonProperty("_rev") + @SerializedName("_rev") String rev; private String title; private Date createdAt; @@ -120,4 +122,13 @@ public long getCreatedAtTimestamp() { public void setCreatedAtTimestamp(long createdAtTimestamp) { this.createdAtTimestamp = createdAtTimestamp; } + + public String toString() { + return String.format("Notification '%s' with target '%s' created at %s: '%s'", + title, + targetedAt, + createdAt, + message + ); + } } From 0792a0112ce6a01d6fee9e79f135b5ddb545e564 Mon Sep 17 00:00:00 2001 From: bossenti Date: Fri, 20 Oct 2023 15:41:45 +0200 Subject: [PATCH 10/54] feat: implement data model for migration --- .../api/migration/AdapterMigrator.java | 25 ++++++++++++ .../api/migration/DataProcessorMigrator.java | 27 +++++++++++++ .../api/migration/DataSinkMigrator.java | 25 ++++++++++++ .../api/migration/MigrationResult.java | 38 +++++++++++++++++++ .../api/migration/ModelMigrator.java | 38 +++++++++++++++++++ .../migration/AdapterMigrationRequest.java | 30 +++++++++++++++ .../model/migration/ModelMigratorConfig.java | 33 ++++++++++++++++ 7 files changed, 216 insertions(+) create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java create mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/AdapterMigrationRequest.java create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java new file mode 100644 index 0000000000..33c3d19209 --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; + +public interface AdapterMigrator extends ModelMigrator { +} diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java new file mode 100644 index 0000000000..2c395ef862 --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.extensions.api.extractor.IDataProcessorParameterExtractor; +import org.apache.streampipes.model.graph.DataProcessorInvocation; + +public interface DataProcessorMigrator + extends ModelMigrator { + +} diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java new file mode 100644 index 0000000000..dafee59c41 --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.extensions.api.extractor.IDataSinkParameterExtractor; +import org.apache.streampipes.model.graph.DataSinkInvocation; + +public interface DataSinkMigrator extends ModelMigrator { +} diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java new file mode 100644 index 0000000000..bb18c95670 --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.model.Notification; + +import java.util.List; + +public record MigrationResult (boolean success, T element, List messages){ + + public static MigrationResult failure(T element, List messages) { + return new MigrationResult<>(false, element, messages); + } + + public static MigrationResult failure(T element, Notification message) { + return failure(element, List.of(message)); + } + + public static MigrationResult success(T element) { + return new MigrationResult<>(true, element, List.of()); + } +} diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java new file mode 100644 index 0000000000..d734cdfd82 --- /dev/null +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.apache.streampipes.extensions.api.extractor.IParameterExtractor; +import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.migration.ModelMigratorConfig; + +public interface ModelMigrator> { + + ModelMigratorConfig config(); + + /** + * Defines the migration to be performed. + * @param element Entity to be transformed. + * @param extractor Extractor that allows to handle static properties. + * @return Result of the migration that describes both outcomes: successful and failed migrations + * @throws RuntimeException in case any unexpected error occurs + */ + MigrationResult migrate(T element, ExT extractor) throws RuntimeException; + +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/AdapterMigrationRequest.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/AdapterMigrationRequest.java new file mode 100644 index 0000000000..4b372a9e22 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/AdapterMigrationRequest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.extensions.migration; + +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.migration.ModelMigratorConfig; + +/** + * Models a request that is sent from the core to an extensions service to mandate an adapter migration. + * @param adapterDescription adapter to be migrated + * @param modelMigratorConfig migration config that describes the migration to be applied to the adapter. + */ +public record AdapterMigrationRequest(AdapterDescription adapterDescription, ModelMigratorConfig modelMigratorConfig) { +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java b/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java new file mode 100644 index 0000000000..d667b1a995 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.migration; + +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; + +/** + * The configuration element for a 'ModelMigrator'. + * + * @param targetAppId 'appId' of the model to be migrated + * @param modelType Type of the model to be migrated, e.g., adapter + * @param fromVersion Base version from which the migration starts + * @param toVersion Target version that the migration aims to achieve + */ +public record ModelMigratorConfig(String targetAppId, SpServiceTagPrefix modelType, + Integer fromVersion, Integer toVersion) { +} From 6bc8bc189956176c8dff989d649a0b76552abc81 Mon Sep 17 00:00:00 2001 From: bossenti Date: Fri, 20 Oct 2023 15:43:53 +0200 Subject: [PATCH 11/54] feat: register migrations at service --- .../apache/streampipes/client/api/IAdminApi.java | 3 +++ .../apache/streampipes/client/api/AdminApi.java | 16 ++++++++++++++++ .../management/model/SpServiceDefinition.java | 11 +++++++++++ .../model/SpServiceDefinitionBuilder.java | 16 ++++++++++++++-- .../extensions/ExtensionsModelSubmitter.java | 11 +++++++++++ 5 files changed, 55 insertions(+), 2 deletions(-) diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java index 2b9d9e45eb..ba776ce51c 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java @@ -22,6 +22,7 @@ import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.function.FunctionDefinition; +import org.apache.streampipes.model.migration.ModelMigratorConfig; import java.util.List; @@ -39,5 +40,7 @@ public interface IAdminApi { void deregisterFunction(String functionId); + void registerMigrations(List migrationConfigs, String serviceId); + MessagingSettings getMessagingSettings(); } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java index b502632839..5ed1e4f1dc 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java @@ -24,6 +24,7 @@ import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.function.FunctionDefinition; import org.apache.streampipes.model.message.SuccessMessage; +import org.apache.streampipes.model.migration.ModelMigratorConfig; import java.util.List; @@ -66,6 +67,15 @@ public void deregisterFunction(String functionId) { delete(getDeleteFunctionPath(functionId), SuccessMessage.class); } + /** + * Register migration configs {@link ModelMigratorConfig} at the StreamPipes Core service. + * @param migrationConfigs list of migration configs to be registered + */ + @Override + public void registerMigrations(List migrationConfigs, String serviceId) { + post(getMigrationsPath().addToPath(serviceId), migrationConfigs); + } + @Override public MessagingSettings getMessagingSettings() { return getSingle(getMessagingSettingsPath(), MessagingSettings.class); @@ -98,4 +108,10 @@ private StreamPipesApiPath getFunctionsPath() { private StreamPipesApiPath getDeleteFunctionPath(String functionId) { return getFunctionsPath().addToPath(functionId); } + + private StreamPipesApiPath getMigrationsPath() { + return StreamPipesApiPath + .fromBaseApiPath() + .addToPath("migrations"); + } } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java index ca22193907..49cefee013 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java @@ -20,6 +20,7 @@ import org.apache.streampipes.dataformat.SpDataFormatFactory; import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter; import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer; +import org.apache.streampipes.extensions.api.migration.ModelMigrator; import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; import org.apache.streampipes.extensions.api.pe.runtime.IStreamPipesRuntimeProvider; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; @@ -47,6 +48,7 @@ public class SpServiceDefinition { private List kvConfigs; private List runtimeProviders; + private List> migrators; public SpServiceDefinition() { this.serviceId = UUID.randomUUID().toString(); @@ -57,6 +59,7 @@ public SpServiceDefinition() { this.functions = new ArrayList<>(); this.adapters = new ArrayList<>(); this.runtimeProviders = new ArrayList<>(); + this.migrators = new ArrayList<>(); } public String getServiceGroup() { @@ -182,4 +185,12 @@ public List getRuntimeProviders() { public void addRuntimeProvider(IStreamPipesRuntimeProvider runtimeProvider) { this.runtimeProviders.add(runtimeProvider); } + + public List> getMigrators() { + return this.migrators; + } + + public void addMigrators(List> migrators) { + this.migrators.addAll(migrators); + } } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java index a350950c9e..9593c47225 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java @@ -20,6 +20,7 @@ import org.apache.streampipes.dataformat.SpDataFormatFactory; import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter; import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer; +import org.apache.streampipes.extensions.api.migration.ModelMigrator; import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; import org.apache.streampipes.extensions.api.pe.runtime.IStreamPipesRuntimeProvider; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; @@ -36,7 +37,6 @@ public class SpServiceDefinitionBuilder { private static final Logger LOG = LoggerFactory.getLogger(SpServiceDefinitionBuilder.class); private SpServiceDefinition serviceDefinition; - //private SpConfig config; private SpServiceDefinitionBuilder(String serviceGroup, String serviceName, @@ -47,7 +47,6 @@ private SpServiceDefinitionBuilder(String serviceGroup, this.serviceDefinition.setServiceName(serviceName); this.serviceDefinition.setServiceDescription(serviceDescription); this.serviceDefinition.setDefaultPort(defaultPort); - //this.config = new ConsulSpConfig(serviceGroup); } public static SpServiceDefinitionBuilder create(String serviceGroup, @@ -123,9 +122,22 @@ public SpServiceDefinitionBuilder registerMessagingProtocols(SpProtocolDefinitio return this; } + /** + * Include migrations in the service definition. + *
+ * Please refrain from providing {@link ModelMigrator}s with overlapping version definitions for one application id. + * @param migrations List of migrations to be registered + * @return {@link SpServiceDefinitionBuilder} + */ + public SpServiceDefinitionBuilder registerMigrators(ModelMigrator... migrations) { + this.serviceDefinition.addMigrators(List.of(migrations)); + return this; + } + public SpServiceDefinitionBuilder merge(SpServiceDefinition other) { this.serviceDefinition.addDeclarers(other.getDeclarers()); this.serviceDefinition.addAdapters(other.getAdapters()); + this.serviceDefinition.addMigrators(other.getMigrators()); other.getKvConfigs().forEach(value -> { if (this.serviceDefinition.getKvConfigs().stream().anyMatch(c -> c.getKey().equals(value.getKey()))) { LOG.warn("Config key {} already exists and will be overridden by merge, which might lead to strange results.", diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java index cbd3ffa8bf..a67ab9ebbe 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java @@ -17,6 +17,9 @@ */ package org.apache.streampipes.service.extensions; +import org.apache.streampipes.client.StreamPipesClient; +import org.apache.streampipes.extensions.api.migration.ModelMigrator; +import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver; import org.apache.streampipes.extensions.management.init.DeclarersSingleton; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; @@ -45,6 +48,14 @@ public void onExit() { @Override public void afterServiceRegistered(SpServiceDefinition serviceDef) { + // register all migrations at the StreamPipes Core + StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); + client.adminApi().registerMigrations( + serviceDef.getMigrators().stream().map(ModelMigrator::config).toList(), + serviceId() + ); + + // initialize all function instances StreamPipesFunctionHandler.INSTANCE.initializeFunctions(serviceDef.getServiceGroup()); } From 9f88b9ad6062378192572b172d1a4bc9e65b911f Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 11:44:57 +0200 Subject: [PATCH 12/54] Revert "refactor: implement string representation of Notification" This reverts commit 646e792d50b3022748e99be738c390270b87c345. --- .../apache/streampipes/model/Notification.java | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/Notification.java b/streampipes-model/src/main/java/org/apache/streampipes/model/Notification.java index 48d8c75497..3b9f47f84c 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/Notification.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/Notification.java @@ -26,10 +26,8 @@ public class Notification { - private @JsonProperty("_id") - @SerializedName("_id") String id; - private @JsonProperty("_rev") - @SerializedName("_rev") String rev; + private @JsonProperty("_id") @SerializedName("_id") String id; + private @JsonProperty("_rev") @SerializedName("_rev") String rev; private String title; private Date createdAt; @@ -122,13 +120,4 @@ public long getCreatedAtTimestamp() { public void setCreatedAtTimestamp(long createdAtTimestamp) { this.createdAtTimestamp = createdAtTimestamp; } - - public String toString() { - return String.format("Notification '%s' with target '%s' created at %s: '%s'", - title, - targetedAt, - createdAt, - message - ); - } } From 879ced48451289d9d89f95fc24dbfc895f89b6cf Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 11:49:18 +0200 Subject: [PATCH 13/54] refactor: use correct Notification class --- .../streampipes/extensions/api/migration/MigrationResult.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java index bb18c95670..09204ceaba 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java @@ -18,7 +18,7 @@ package org.apache.streampipes.extensions.api.migration; -import org.apache.streampipes.model.Notification; +import org.apache.streampipes.model.message.Notification; import java.util.List; From e65fe82fdde93fc45a3875d32156e173a71ec00e Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 13:11:42 +0200 Subject: [PATCH 14/54] feat: introduce migrate extensions resource --- .../migration/MigrateExtensionsResource.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResource.java diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResource.java new file mode 100644 index 0000000000..d437d1e55a --- /dev/null +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResource.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.extensions.migration; + +import org.apache.streampipes.extensions.api.migration.ModelMigrator; +import org.apache.streampipes.extensions.management.init.DeclarersSingleton; +import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.apache.streampipes.rest.extensions.AbstractExtensionsResource; + +import java.util.Optional; + +public class MigrateExtensionsResource extends AbstractExtensionsResource { + + /** + * Find and return the corresponding {@link ModelMigrator} instance within the registered migrators. + * This allows to pass the corresponding model migrator to a {@link ModelMigratorConfig} which is exchanged + * between Core and Extensions service. + * @param modelMigratorConfig config that describes the model migrator to be returned + * @return Optional model migrator which is empty in case no appropriate migrator is found among the registered. + */ + public Optional getMigrator(ModelMigratorConfig modelMigratorConfig) { + return DeclarersSingleton.getInstance().getServiceDefinition().getMigrators() + .stream() + .filter(modelMigrator -> modelMigrator.config().equals(modelMigratorConfig)) + .map(modelMigrator -> (T) modelMigrator) + .findFirst(); + } +} From 5ae832ade90d5d3cc39ff368c2559bccc5f64e2c Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 13:13:00 +0200 Subject: [PATCH 15/54] feat: introduce migrate adapter endpoint --- streampipes-rest-extensions/pom.xml | 10 +- .../migration/MigrationResource.java | 127 ++++++++++++++++++ .../extensions/ExtensionsResourceConfig.java | 22 +-- 3 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrationResource.java diff --git a/streampipes-rest-extensions/pom.xml b/streampipes-rest-extensions/pom.xml index b3a39e4319..211d2e73cb 100644 --- a/streampipes-rest-extensions/pom.xml +++ b/streampipes-rest-extensions/pom.xml @@ -32,12 +32,16 @@ + + org.apache.streampipes + streampipes-rest + 0.93.0-SNAPSHOT + org.apache.streampipes streampipes-rest-shared 0.93.0-SNAPSHOT - org.apache.streampipes streampipes-extensions-management @@ -64,6 +68,10 @@ + + org.springframework.security + spring-security-core + diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrationResource.java new file mode 100644 index 0000000000..5137c0cdc9 --- /dev/null +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrationResource.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.extensions.migration; + +import org.apache.streampipes.extensions.api.migration.AdapterMigrator; +import org.apache.streampipes.extensions.api.migration.MigrationResult; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.extensions.migration.AdapterMigrationRequest; +import org.apache.streampipes.model.message.Notification; +import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; +import org.apache.streampipes.sdk.extractor.StaticPropertyExtractor; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.access.prepost.PreAuthorize; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + + +@Path("/api/v1/migrations") +public class MigrationResource extends MigrateExtensionsResource { + + private static final Logger LOG = LoggerFactory.getLogger(MigrationResource.class); + + /** + * Migrates an adapter instance based on the provided {@link AdapterMigrationRequest}. + * The outcome of the migration is described in {@link MigrationResult}. + * The result is always part of the response. + * Independent, of the migration outcome, the returned response always has OK as status code. + * It is the responsibility of the recipient to interpret the migration result and act accordingly. + * @param adapterMigrateRequest Request that contains both the adapter to be migrated and the migration config. + * @return A response with status code ok, that contains a migration result reflecting the outcome of the operation. + */ + @POST + @Path("adapter") + @Consumes(MediaType.APPLICATION_JSON) + @JacksonSerialized + @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) + public Response migrateAdapter(AdapterMigrationRequest adapterMigrateRequest) { + + var adapterDescription = adapterMigrateRequest.adapterDescription(); + var migrationConfig = adapterMigrateRequest.modelMigratorConfig(); + + LOG.info("Received migration request for adapter '{}' to migrate from version {} to {}", + adapterDescription.getElementId(), + migrationConfig.fromVersion(), + migrationConfig.toVersion() + ); + + var migratorOptional = getMigrator(migrationConfig); + + if (migratorOptional.isPresent()) { + LOG.info("Migrator found for request, starting migration..."); + var migrator = migratorOptional.get(); + + var extractor = StaticPropertyExtractor.from(adapterDescription.getConfig()); + + try { + var result = migrator.migrate(adapterDescription, extractor); + + if (result.success()) { + LOG.info("Migration successfully finished."); + + // Since adapter migration was successful, version can be adapted to the target version. + // this step is explicitly performed here and not left to the migration itself to + // prevent leaving this step out + AdapterDescription migratedAdapterDescription = result.element(); + migratedAdapterDescription.setVersion(migrationConfig.toVersion()); + return ok(MigrationResult.success(migratedAdapterDescription)); + + } else { + LOG.error("Migration failed with the following reasons: {}", StringUtils.join(result.messages(), ", ")); + // The failed migration is documented in the MigrationResult + // The core is expected to handle the response accordingly, so we can safely return a positive status code + return ok(result); + } + + } catch (RuntimeException e) { + LOG.error("An unexpected exception caused the migration to fail - " + + "sending exception report in migration result"); + return ok(MigrationResult.failure( + adapterDescription, + new Notification( + "MigrationFailure", + String.format( + "Adapter Migration failed due to an unexpected exception: %s", + StringUtils.join(e.getStackTrace(), "\n") + ) + ) + )); + } + } + LOG.error("Migrator for migration config {} could not be found. Migration is cancelled.", migrationConfig); + return ok(MigrationResult.failure( + adapterDescription, + new Notification( + "MigrationNotFound", + String.format( + "The given migration config '%s' could not be mapped to a registered migrator.", + migrationConfig + ) + ) + )); + } +} diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java index e85be4593f..a061865c73 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java @@ -25,6 +25,7 @@ import org.apache.streampipes.rest.extensions.connect.GuessResource; import org.apache.streampipes.rest.extensions.connect.HttpServerAdapterResource; import org.apache.streampipes.rest.extensions.connect.RuntimeResolvableResource; +import org.apache.streampipes.rest.extensions.migration.MigrationResource; import org.apache.streampipes.rest.extensions.monitoring.MonitoringResource; import org.apache.streampipes.rest.extensions.pe.DataProcessorPipelineElementResource; import org.apache.streampipes.rest.extensions.pe.DataSinkPipelineElementResource; @@ -46,22 +47,21 @@ public class ExtensionsResourceConfig extends BaseResourceConfig { @Override public Set> getClassesToRegister() { return Set.of( - GuessResource.class, - RuntimeResolvableResource.class, - AdapterWorkerResource.class, - MultiPartFeature.class, AdapterAssetResource.class, AdapterDescriptionResource.class, - HttpServerAdapterResource.class, - - DataSinkPipelineElementResource.class, + AdapterWorkerResource.class, DataProcessorPipelineElementResource.class, + DataSinkPipelineElementResource.class, DataStreamPipelineElementResource.class, - WelcomePage.class, - - ServiceHealthResource.class, + GuessResource.class, + HttpServerAdapterResource.class, JacksonSerializationProvider.class, - MonitoringResource.class + MigrationResource.class, + MonitoringResource.class, + MultiPartFeature.class, + RuntimeResolvableResource.class, + ServiceHealthResource.class, + WelcomePage.class ); } From 56964914bc7ffa92df140ef58d752d844733b3e0 Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 13:16:51 +0200 Subject: [PATCH 16/54] feat: implement adapter migration at the core --- .../management/WorkerRestClient.java | 77 ++++++++-- .../rest/impl/admin/MigrationResource.java | 142 ++++++++++++++++++ .../core/StreamPipesResourceConfig.java | 2 + 3 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java index bbb92edc71..5dfb0bfbc1 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java @@ -21,8 +21,11 @@ import org.apache.streampipes.commons.exceptions.SpConfigurationException; import org.apache.streampipes.commons.exceptions.connect.AdapterException; import org.apache.streampipes.connect.management.util.WorkerPaths; +import org.apache.streampipes.extensions.api.migration.MigrationResult; import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.extensions.migration.AdapterMigrationRequest; +import org.apache.streampipes.model.message.Notification; import org.apache.streampipes.model.runtime.RuntimeOptionsRequest; import org.apache.streampipes.model.runtime.RuntimeOptionsResponse; import org.apache.streampipes.model.util.Cloner; @@ -32,8 +35,11 @@ import org.apache.streampipes.storage.couchdb.impl.AdapterInstanceStorageImpl; import org.apache.streampipes.storage.management.StorageDispatcher; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpResponse; import org.apache.http.client.fluent.Request; import org.slf4j.Logger; @@ -49,6 +55,7 @@ public class WorkerRestClient { private static final Logger LOG = LoggerFactory.getLogger(WorkerRestClient.class); + private static final String ADAPTER_MIGRATION_ENDPOINT = "api/v1/migrations/adapter"; public static void invokeStreamAdapter(String endpointUrl, String elementId) throws AdapterException { @@ -84,20 +91,68 @@ public static List getAllRunningAdapterInstanceDescriptions( } } - public static void startAdapter(String url, + private static void startAdapter(String url, AdapterDescription ad) throws AdapterException { LOG.info("Trying to start adapter on endpoint {} ", url); triggerAdapterStateChange(ad, url, "started"); } - public static void stopAdapter(AdapterDescription ad, + private static void stopAdapter(AdapterDescription ad, String url) throws AdapterException { LOG.info("Trying to stop adapter on endpoint {} ", url); triggerAdapterStateChange(ad, url, "stopped"); } + /** + * Sends a migration request for an adapter to the extensions service. + * Encapsulates the communication with the service fully so that only a migration result is returned + * which reports about the outcome of the migration. + * @param adapterMigrationRequest request that describes the adapter migration + * @param serviceUrl URL of the extensions service where the migration should be requested + * @return The outcome of the migration in form of a {@link MigrationResult} + */ + public static MigrationResult migrateAdapter(AdapterMigrationRequest adapterMigrationRequest, + String serviceUrl + ) { + try { + String serializedRequest = JacksonSerializer.getObjectMapper().writeValueAsString(adapterMigrationRequest); + var migrationResponse = ExtensionServiceExecutions.extServicePostRequest( + "%s/%s".formatted(serviceUrl, ADAPTER_MIGRATION_ENDPOINT), + serializedRequest + ).execute(); + + TypeReference> typeReference = new TypeReference<>() { + }; + + return JacksonSerializer.getObjectMapper().readValue( + migrationResponse.returnContent().asString(), + typeReference + ); + + } catch (JsonProcessingException e) { + return MigrationResult.failure(adapterMigrationRequest.adapterDescription(), + List.of(new Notification( + "MigrationNotSend", + String.format( + "Serialization of AdapterMigrationRequest failed, " + + "could not sent request to extensions service: %s.", + StringUtils.join(e.getStackTrace(), "\n") + ) + ))); + } catch (IOException e) { + LOG.error("Migration of adapter failed at the extensions service, adapter is not migrated."); + return MigrationResult.failure(adapterMigrationRequest.adapterDescription(), + new Notification( + "MigrationCommunicationError", + String.format("An error occurred in the communication between StreamPipes core and " + + "the extensions service: %s", + StringUtils.join(e.getStackTrace(), "\n")) + )); + } + } + private static void triggerAdapterStateChange(AdapterDescription ad, String url, String action) throws AdapterException { @@ -134,14 +189,14 @@ private static HttpResponse triggerPost(String url, public static RuntimeOptionsResponse getConfiguration(String workerEndpoint, String appId, RuntimeOptionsRequest runtimeOptionsRequest) - throws AdapterException, SpConfigurationException { + throws AdapterException, SpConfigurationException { String url = workerEndpoint + WorkerPaths.getRuntimeResolvablePath(appId); try { String payload = JacksonSerializer.getObjectMapper().writeValueAsString(runtimeOptionsRequest); var response = ExtensionServiceExecutions.extServicePostRequest(url, payload) - .execute() - .returnResponse(); + .execute() + .returnResponse(); String responseString = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); @@ -178,9 +233,9 @@ public static byte[] getIconAsset(String baseUrl) throws AdapterException { try { byte[] responseString = Request.Get(url) - .connectTimeout(1000) - .socketTimeout(100000) - .execute().returnContent().asBytes(); + .connectTimeout(1000) + .socketTimeout(100000) + .execute().returnContent().asBytes(); return responseString; } catch (IOException e) { LOG.error(e.getMessage()); @@ -193,9 +248,9 @@ public static String getDocumentationAsset(String baseUrl) throws AdapterExcepti try { return Request.Get(url) - .connectTimeout(1000) - .socketTimeout(100000) - .execute().returnContent().asString(); + .connectTimeout(1000) + .socketTimeout(100000) + .execute().returnContent().asString(); } catch (IOException e) { LOG.error(e.getMessage()); throw new AdapterException("Could not get documentation endpoint: " + url); diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java new file mode 100644 index 0000000000..7760d07beb --- /dev/null +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.impl.admin; + +import org.apache.streampipes.commons.exceptions.connect.AdapterException; +import org.apache.streampipes.connect.management.management.WorkerRestClient; +import org.apache.streampipes.model.extensions.migration.AdapterMigrationRequest; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; +import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.api.IAdapterStorage; +import org.apache.streampipes.storage.api.IDataProcessorStorage; +import org.apache.streampipes.storage.api.IDataSinkStorage; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.stereotype.Component; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +import java.util.List; + + +@Path("v2/migrations") +@Component +@PreAuthorize(AuthConstants.IS_ADMIN_ROLE) +public class MigrationResource extends AbstractAuthGuardedRestResource { + + private static final Logger LOG = LoggerFactory.getLogger(MigrationResource.class); + + private final IAdapterStorage adapterStorage = getNoSqlStorage().getAdapterInstanceStorage(); + private final IDataProcessorStorage dataProcessorStorage = getNoSqlStorage().getDataProcessorStorage(); + private final IDataSinkStorage dataSinkStorage = getNoSqlStorage().getDataSinkStorage(); + + private final CRUDStorage extensionsServiceStorage = + getNoSqlStorage().getExtensionsServiceStorage(); + + @POST + @Path("{serviceId}") + @Consumes(MediaType.APPLICATION_JSON) + public Response registerMigrations( + @PathParam("serviceId") String serviceId, + List migrationConfigs) { + + LOG.info("Received {} migrations from extension service.", migrationConfigs.size()); + LOG.info("Checking migrations for existing assets in StreamPipes Core ..."); + for (var migrationConfig: migrationConfigs) { + LOG.info("Searching for assets of '{}'", migrationConfig.targetAppId()); + LOG.debug("Searching for assets of '{}' with config {}", migrationConfig.targetAppId(), migrationConfig); + switch (migrationConfig.modelType()) { + case ADAPTER -> { + var adapterDescriptions = adapterStorage.getAdaptersByAppId(migrationConfig.targetAppId()); + LOG.info("Found {} instances for appId '{}'", adapterDescriptions.size(), migrationConfig.targetAppId()); + for (var adapterDescription: adapterDescriptions) { + + var adapterVersion = adapterDescription.getVersion(); + + if (adapterVersion == migrationConfig.fromVersion()) { + LOG.info("Migration is required for adapter '{}'. Migrating from version '{}' to '{}' ...", + adapterDescription.getElementId(), + adapterVersion, migrationConfig.toVersion() + ); + + var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); + + var adapterMigrationRequest = new AdapterMigrationRequest(adapterDescription, migrationConfig); + + var migrationResult = WorkerRestClient.migrateAdapter( + adapterMigrationRequest, + extensionsServiceConfig.getServiceUrl() + ); + + if (migrationResult.success()) { + LOG.info("Migration successfully performed by extensions service. Updating adapter description ..."); + LOG.debug( + "Migration was performed by extensions service '{}'", + extensionsServiceConfig.getServiceUrl()); + + adapterStorage.updateAdapter(migrationResult.element()); + LOG.info("Adapter description is updated - Migration successfully completed at core."); + } else { + LOG.error( + "Migration failed - Failure report: {}", + StringUtils.join(migrationResult.messages(), ",") + ); + LOG.error( + "Migration for adapter '{}' failed - Stopping adapter ...", + migrationResult.element().getElementId() + ); + try { + WorkerRestClient.stopStreamAdapter(extensionsServiceConfig.getServiceUrl(), adapterDescription); + } catch (AdapterException e) { + LOG.error("Stopping adapter failed: {}", StringUtils.join(e.getStackTrace(), "\n")); + } + LOG.info("Adapter successfully stopped."); + } + } else { + LOG.info("Migration is not applicable for adapter '{}' because of a version mismatch - " + + "adapter version: '{}', migration starts at: '{}'", + adapterDescription.getElementId(), + adapterVersion, + migrationConfig.fromVersion() + ); + + } + } + } + case DATA_PROCESSOR -> {} + case DATA_SINK -> {} + default -> LOG.error("Can't handle migration for model type {}", migrationConfig.modelType() + ); + } + + } + return ok(); + } +} diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java index db54d3c3d0..cc628981b8 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java @@ -62,6 +62,7 @@ import org.apache.streampipes.rest.impl.admin.ExtensionsServiceEndpointResource; import org.apache.streampipes.rest.impl.admin.GeneralConfigurationResource; import org.apache.streampipes.rest.impl.admin.MessagingConfigurationResource; +import org.apache.streampipes.rest.impl.admin.MigrationResource; import org.apache.streampipes.rest.impl.admin.PermissionResource; import org.apache.streampipes.rest.impl.admin.PipelineElementImport; import org.apache.streampipes.rest.impl.admin.ServiceConfigurationResource; @@ -130,6 +131,7 @@ public Set> getClassesToRegister() { LabelResource.class, MeasurementUnitResource.class, MessagingConfigurationResource.class, + MigrationResource.class, Notification.class, OntologyMeasurementUnit.class, PermissionResource.class, From 20a9ddf4a7223a2078c2e1e4f049838507bb9392 Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 14:08:27 +0200 Subject: [PATCH 17/54] remove data lake migration --- .../internal/jvm/DataLakeMigrationv1.java | 56 ------------------- .../internal/jvm/SinksInternalJvmInit.java | 3 - 2 files changed, 59 deletions(-) delete mode 100644 streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/DataLakeMigrationv1.java diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/DataLakeMigrationv1.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/DataLakeMigrationv1.java deleted file mode 100644 index be70b0e83f..0000000000 --- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/DataLakeMigrationv1.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.sinks.internal.jvm; - -import org.apache.streampipes.commons.exceptions.SpConfigurationException; -import org.apache.streampipes.extensions.api.extractor.IDataSinkParameterExtractor; -import org.apache.streampipes.extensions.api.migration.MigrationResult; -import org.apache.streampipes.extensions.api.migration.ModelMigrator; -import org.apache.streampipes.model.graph.DataSinkInvocation; -import org.apache.streampipes.sdk.StaticProperties; -import org.apache.streampipes.sdk.helpers.Labels; -import org.apache.streampipes.sdk.utils.Datatypes; - -public class DataLakeMigrationv1 implements ModelMigrator { - - @Override - public String targetsAppId() { - return "org.apache.streampipes.sinks.internal.jvm.datalake"; - } - - @Override - public Integer fromVersion() { - return 0; - } - - @Override - public Integer toVersion() { - return 1; - } - - @Override - public MigrationResult migrate(DataSinkInvocation element, - IDataSinkParameterExtractor extractor) throws SpConfigurationException { - element.getStaticProperties().add( - StaticProperties.freeTextProperty(Labels.empty(), Datatypes.String) - ); - - return MigrationResult.success(element); - } -} diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/SinksInternalJvmInit.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/SinksInternalJvmInit.java index d18b68625d..38961624c8 100644 --- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/SinksInternalJvmInit.java +++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/SinksInternalJvmInit.java @@ -53,9 +53,6 @@ public SpServiceDefinition provideServiceDefinition() { new SpJmsProtocolFactory(), new SpMqttProtocolFactory(), new SpNatsProtocolFactory()) - .registerMigrators( - new DataLakeMigrationv1() - ) .build(); From cd0e41d13c8a8af366ff243f165ead719d6adfa1 Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 14:09:01 +0200 Subject: [PATCH 18/54] ensure order & uniqueness of migrations --- .../api/migration/AdapterMigrator.java | 10 ++++++- .../api/migration/DataSinkMigrator.java | 10 ++++++- .../api/migration/ModelMigrator.java | 17 ++++++++++-- .../management/model/SpServiceDefinition.java | 15 ++++++++++- .../model/migration/ModelMigratorConfig.java | 26 ++++++++++++++++++- 5 files changed, 72 insertions(+), 6 deletions(-) diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java index 33c3d19209..96f32d1e62 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/AdapterMigrator.java @@ -21,5 +21,13 @@ import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor; import org.apache.streampipes.model.connect.adapter.AdapterDescription; -public interface AdapterMigrator extends ModelMigrator { +public abstract class AdapterMigrator implements ModelMigrator { + + @Override + public boolean equals(Object obj) { + if (obj instanceof ModelMigrator) { + return this.config().equals(((ModelMigrator) obj).config()); + } + return false; + } } diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java index dafee59c41..f88dbbf5ce 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataSinkMigrator.java @@ -21,5 +21,13 @@ import org.apache.streampipes.extensions.api.extractor.IDataSinkParameterExtractor; import org.apache.streampipes.model.graph.DataSinkInvocation; -public interface DataSinkMigrator extends ModelMigrator { +public abstract class DataSinkMigrator implements ModelMigrator { + + @Override + public boolean equals(Object obj) { + if (obj instanceof ModelMigrator) { + return this.config().equals(((ModelMigrator) obj).config()); + } + return false; + } } diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java index d734cdfd82..d95ff49c13 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java @@ -22,17 +22,30 @@ import org.apache.streampipes.model.base.NamedStreamPipesEntity; import org.apache.streampipes.model.migration.ModelMigratorConfig; -public interface ModelMigrator> { +public interface ModelMigrator< + T extends NamedStreamPipesEntity, + ExT extends IParameterExtractor + > extends Comparable { ModelMigratorConfig config(); /** * Defines the migration to be performed. - * @param element Entity to be transformed. + * + * @param element Entity to be transformed. * @param extractor Extractor that allows to handle static properties. * @return Result of the migration that describes both outcomes: successful and failed migrations * @throws RuntimeException in case any unexpected error occurs */ MigrationResult migrate(T element, ExT extractor) throws RuntimeException; + @Override + default int compareTo(Object o) { + if (!(o instanceof ModelMigrator)) { + throw new ClassCastException("Given object is not an instance of `ModelMigrator` - " + + "only instances of `ModelMigrator` can be compared."); + } else { + return config().compareTo(((ModelMigrator) o).config()); + } + } } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java index 49cefee013..dfa1489dda 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java @@ -27,6 +27,7 @@ import org.apache.streampipes.model.extensions.configuration.ConfigItem; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -190,7 +191,19 @@ public void addRuntimeProvider(IStreamPipesRuntimeProvider runtimeProvider) { return this.migrators; } + /** + * Add a list of migrations to the service definition. + * This inherently checks for duplicates and sorts the migrations as such that + * migrations affecting lower versions always come first. + * + * @param migrators migrators to add + */ public void addMigrators(List> migrators) { - this.migrators.addAll(migrators); + for (var migratorToAdd : migrators) { + if (!this.migrators.contains(migratorToAdd)) { + this.migrators.add(migratorToAdd); + } + } + Collections.sort(this.migrators); } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java b/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java index d667b1a995..65713d905a 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java @@ -29,5 +29,29 @@ * @param toVersion Target version that the migration aims to achieve */ public record ModelMigratorConfig(String targetAppId, SpServiceTagPrefix modelType, - Integer fromVersion, Integer toVersion) { + int fromVersion, int toVersion) implements Comparable{ + + + @Override + public int compareTo(Object o) { + + if (o == null) { + throw new NullPointerException(); + } + + if (!(o instanceof ModelMigratorConfig)){ + throw new ClassCastException("Given object is not an instance of `ModelMigratorConfig` - " + + "only instances of `ModelMigratorConfig` can be compared."); + } + + if (targetAppId.equals(((ModelMigratorConfig) o).targetAppId())) { + + if (fromVersion != ((ModelMigratorConfig) o).fromVersion()) { + return this.toVersion() - ((ModelMigratorConfig) o).toVersion(); + } + return this.fromVersion() - ((ModelMigratorConfig) o).fromVersion(); + } + + return 0; + } } From 18c1bc6713bbc6590cb3284018056bf5b6aa9581 Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 14:12:40 +0200 Subject: [PATCH 19/54] remove redundant exception --- .../exceptions/SpAutoMigrateException.java | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/SpAutoMigrateException.java diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/SpAutoMigrateException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/SpAutoMigrateException.java deleted file mode 100644 index 3542ef2d71..0000000000 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/SpAutoMigrateException.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.commons.exceptions; - -public class SpAutoMigrateException extends Exception { - - /** - * Creates a new Exception with the given message and null as the cause. - * - * @param message The exception message - */ - public SpAutoMigrateException(String message) { - super(message); - } - - /** - * Creates a new exception with a null message and the given cause. - * - * @param cause The exception that caused this exception - */ - public SpAutoMigrateException(Throwable cause) { - super(cause); - } - - /** - * Creates a new exception with the given message and cause - * - * @param message The exception message - * @param cause The exception that caused this exception - */ - public SpAutoMigrateException(String message, Throwable cause) { - super(message, cause); - } -} From b31a3ea05469d9d9260da9777a8261f3802c8f72 Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 14:30:09 +0200 Subject: [PATCH 20/54] remove redundant exception --- .../api/migration/DataProcessorMigrator.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java index 2c395ef862..003bf4539c 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java @@ -21,7 +21,15 @@ import org.apache.streampipes.extensions.api.extractor.IDataProcessorParameterExtractor; import org.apache.streampipes.model.graph.DataProcessorInvocation; -public interface DataProcessorMigrator - extends ModelMigrator { +public abstract class DataProcessorMigrator + implements ModelMigrator { + + @Override + public boolean equals(Object obj) { + if (obj instanceof ModelMigrator) { + return this.config().equals(((ModelMigrator) obj).config()); + } + return false; + } } From 7277ddb50698b2482907308b9889f8d0d7350b84 Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 15:04:26 +0200 Subject: [PATCH 21/54] add tests --- .../api/migration/AdapterMigratorTest.java | 42 +++++++++++++++ .../model/SpServiceDefinitionBuilderTest.java | 52 +++++++++++++++++++ .../model/migration/ModelMigratorConfig.java | 4 -- .../migration/ModelMigratorConfigTest.java | 49 +++++++++++++++++ 4 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 streampipes-extensions-api/src/test/java/org/apache/streampipes/extensions/api/migration/AdapterMigratorTest.java create mode 100644 streampipes-model/src/test/java/org/apache/streampipes/model/migration/ModelMigratorConfigTest.java diff --git a/streampipes-extensions-api/src/test/java/org/apache/streampipes/extensions/api/migration/AdapterMigratorTest.java b/streampipes-extensions-api/src/test/java/org/apache/streampipes/extensions/api/migration/AdapterMigratorTest.java new file mode 100644 index 0000000000..6061f1bc43 --- /dev/null +++ b/streampipes-extensions-api/src/test/java/org/apache/streampipes/extensions/api/migration/AdapterMigratorTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.api.migration; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class AdapterMigratorTest { + + @Test + public void migrationOrdering(){ + var migratorA1 = new MigratorTestClass("app-id", 1, 2); + var migratorA2 = new MigratorTestClass("app-id", 2, 3); + var migratorB = new MigratorTestClass("other-id", 1, 2); + + List> migrators = new ArrayList<>(List.of(migratorA2, migratorA1, migratorB)); + Collections.sort(migrators); + + assertEquals(List.of(migratorA1, migratorA2, migratorB), migrators); + } +} diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilderTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilderTest.java index 523d9c1f62..e36a624a9b 100644 --- a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilderTest.java +++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilderTest.java @@ -24,7 +24,13 @@ import org.apache.streampipes.extensions.api.connect.context.IAdapterGuessSchemaContext; import org.apache.streampipes.extensions.api.connect.context.IAdapterRuntimeContext; import org.apache.streampipes.extensions.api.extractor.IAdapterParameterExtractor; +import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor; +import org.apache.streampipes.extensions.api.migration.AdapterMigrator; +import org.apache.streampipes.extensions.api.migration.MigrationResult; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.connect.guess.GuessSchema; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.model.migration.ModelMigratorConfig; import org.junit.Test; @@ -43,6 +49,27 @@ public void registerAdapter() { assertEquals(expected, result.getAdapters().get(0)); } + @Test + public void registerMigration() { + + var migration1 = new TestMigration("app-id", 1, 2); + var migration2 = new TestMigration("app-id", 0, 1); + var migration3 = new TestMigration("other-id", 0, 1); + var migration4 = new TestMigration("app-id", 0, 1); + + var result = SpServiceDefinitionBuilder.create("", "", "", 1) + .registerMigrators(migration1, migration2, migration3, migration4) + .build(); + + // assert de-duplication (last migration should not be registered) + assertEquals(3, result.getMigrators().size()); + + // assert ordering + assertEquals(migration2, result.getMigrators().get(0)); + assertEquals(migration1, result.getMigrators().get(1)); + assertEquals(migration3, result.getMigrators().get(2)); + } + private static class TestAdapter implements StreamPipesAdapter { @Override @@ -67,4 +94,29 @@ public GuessSchema onSchemaRequested(IAdapterParameterExtractor extractor, return null; } } + + private static class TestMigration extends AdapterMigrator { + + String appId; + int fromVersion; + int toVersion; + + public TestMigration(String appId, int fromVersion, int toVersion) { + this.appId = appId; + this.fromVersion = fromVersion; + this.toVersion = toVersion; + } + + @Override + public ModelMigratorConfig config() { + return new ModelMigratorConfig(appId, SpServiceTagPrefix.ADAPTER, fromVersion, toVersion); + } + + @Override + public MigrationResult migrate( + AdapterDescription element, + IStaticPropertyExtractor extractor) throws RuntimeException { + return null; + } + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java b/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java index 65713d905a..23356e7502 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/migration/ModelMigratorConfig.java @@ -45,10 +45,6 @@ public int compareTo(Object o) { } if (targetAppId.equals(((ModelMigratorConfig) o).targetAppId())) { - - if (fromVersion != ((ModelMigratorConfig) o).fromVersion()) { - return this.toVersion() - ((ModelMigratorConfig) o).toVersion(); - } return this.fromVersion() - ((ModelMigratorConfig) o).fromVersion(); } diff --git a/streampipes-model/src/test/java/org/apache/streampipes/model/migration/ModelMigratorConfigTest.java b/streampipes-model/src/test/java/org/apache/streampipes/model/migration/ModelMigratorConfigTest.java new file mode 100644 index 0000000000..914e1ed4ee --- /dev/null +++ b/streampipes-model/src/test/java/org/apache/streampipes/model/migration/ModelMigratorConfigTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.migration; + +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class ModelMigratorConfigTest { + + static ModelMigratorConfig config = new ModelMigratorConfig("app-id", SpServiceTagPrefix.ADAPTER, 0, 1); + + @Test + public void compareToWithNullPointer() { + assertThrows(NullPointerException.class, () -> config.compareTo(null)); + } + + @Test + public void compareToWithClassCastException() { + assertThrows(ClassCastException.class, () -> config.compareTo(5)); + } + + @Test + public void compareTo() { + assertEquals(0, config.compareTo(new ModelMigratorConfig("other-app-id", SpServiceTagPrefix.ADAPTER, 0, 1))); + assertEquals(0, config.compareTo(new ModelMigratorConfig("app-id", SpServiceTagPrefix.ADAPTER, 0, 1))); + assertEquals(-1, config.compareTo(new ModelMigratorConfig("app-id", SpServiceTagPrefix.ADAPTER, 1, 2))); + assertEquals(0, config.compareTo(new ModelMigratorConfig("app-id", SpServiceTagPrefix.ADAPTER, 0, 3))); + } +} From 37370f765aa3ced74b0c17cf0e5205e0f4f5953f Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 23 Oct 2023 15:57:39 +0200 Subject: [PATCH 22/54] remove outdated test --- .../api/migration/AdapterMigratorTest.java | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 streampipes-extensions-api/src/test/java/org/apache/streampipes/extensions/api/migration/AdapterMigratorTest.java diff --git a/streampipes-extensions-api/src/test/java/org/apache/streampipes/extensions/api/migration/AdapterMigratorTest.java b/streampipes-extensions-api/src/test/java/org/apache/streampipes/extensions/api/migration/AdapterMigratorTest.java deleted file mode 100644 index 6061f1bc43..0000000000 --- a/streampipes-extensions-api/src/test/java/org/apache/streampipes/extensions/api/migration/AdapterMigratorTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.extensions.api.migration; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -public class AdapterMigratorTest { - - @Test - public void migrationOrdering(){ - var migratorA1 = new MigratorTestClass("app-id", 1, 2); - var migratorA2 = new MigratorTestClass("app-id", 2, 3); - var migratorB = new MigratorTestClass("other-id", 1, 2); - - List> migrators = new ArrayList<>(List.of(migratorA2, migratorA1, migratorB)); - Collections.sort(migrators); - - assertEquals(List.of(migratorA1, migratorA2, migratorB), migrators); - } -} From 86056b81ae4988426124b4fbb6a1e50d3f8d582a Mon Sep 17 00:00:00 2001 From: bossenti Date: Tue, 24 Oct 2023 09:02:49 +0200 Subject: [PATCH 23/54] refactor: separate adapter migration from pipeline element migrations --- .../streampipes/client/api/IAdminApi.java | 2 +- .../streampipes/client/api/AdminApi.java | 8 +- .../rest/impl/admin/MigrationResource.java | 192 ++++++++++++------ .../extensions/ExtensionsModelSubmitter.java | 10 +- 4 files changed, 137 insertions(+), 75 deletions(-) diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java index ba776ce51c..d07123550a 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java @@ -40,7 +40,7 @@ public interface IAdminApi { void deregisterFunction(String functionId); - void registerMigrations(List migrationConfigs, String serviceId); + void registerAdapterMigrations(List migrationConfigs, String serviceId); MessagingSettings getMessagingSettings(); } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java index 5ed1e4f1dc..7f1a9e06d7 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java @@ -72,8 +72,8 @@ public void deregisterFunction(String functionId) { * @param migrationConfigs list of migration configs to be registered */ @Override - public void registerMigrations(List migrationConfigs, String serviceId) { - post(getMigrationsPath().addToPath(serviceId), migrationConfigs); + public void registerAdapterMigrations(List migrationConfigs, String serviceId) { + post(getAdapterMigrationPath().addToPath(serviceId), migrationConfigs); } @Override @@ -109,9 +109,9 @@ private StreamPipesApiPath getDeleteFunctionPath(String functionId) { return getFunctionsPath().addToPath(functionId); } - private StreamPipesApiPath getMigrationsPath() { + private StreamPipesApiPath getAdapterMigrationPath() { return StreamPipesApiPath .fromBaseApiPath() - .addToPath("migrations"); + .addToPath("migrations/adapter"); } } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index 7760d07beb..d77ade1395 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -27,8 +27,7 @@ import org.apache.streampipes.rest.security.AuthConstants; import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.storage.api.IAdapterStorage; -import org.apache.streampipes.storage.api.IDataProcessorStorage; -import org.apache.streampipes.storage.api.IDataSinkStorage; +import org.apache.streampipes.storage.api.IPipelineStorage; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -54,89 +53,148 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { private static final Logger LOG = LoggerFactory.getLogger(MigrationResource.class); private final IAdapterStorage adapterStorage = getNoSqlStorage().getAdapterInstanceStorage(); - private final IDataProcessorStorage dataProcessorStorage = getNoSqlStorage().getDataProcessorStorage(); - private final IDataSinkStorage dataSinkStorage = getNoSqlStorage().getDataSinkStorage(); + private final IPipelineStorage pipelineStorage = getNoSqlStorage().getPipelineStorageAPI(); private final CRUDStorage extensionsServiceStorage = getNoSqlStorage().getExtensionsServiceStorage(); @POST - @Path("{serviceId}") + @Path("adapter/{serviceId}") @Consumes(MediaType.APPLICATION_JSON) - public Response registerMigrations( + public Response registerAdapterMigrations( @PathParam("serviceId") String serviceId, List migrationConfigs) { - LOG.info("Received {} migrations from extension service.", migrationConfigs.size()); + var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); + + LOG.info("Received {} migrations from extension service {}.", + migrationConfigs.size(), + extensionsServiceConfig.getServiceUrl()); LOG.info("Checking migrations for existing assets in StreamPipes Core ..."); for (var migrationConfig: migrationConfigs) { LOG.info("Searching for assets of '{}'", migrationConfig.targetAppId()); LOG.debug("Searching for assets of '{}' with config {}", migrationConfig.targetAppId(), migrationConfig); - switch (migrationConfig.modelType()) { - case ADAPTER -> { - var adapterDescriptions = adapterStorage.getAdaptersByAppId(migrationConfig.targetAppId()); - LOG.info("Found {} instances for appId '{}'", adapterDescriptions.size(), migrationConfig.targetAppId()); - for (var adapterDescription: adapterDescriptions) { - - var adapterVersion = adapterDescription.getVersion(); - - if (adapterVersion == migrationConfig.fromVersion()) { - LOG.info("Migration is required for adapter '{}'. Migrating from version '{}' to '{}' ...", - adapterDescription.getElementId(), - adapterVersion, migrationConfig.toVersion() - ); - - var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); - - var adapterMigrationRequest = new AdapterMigrationRequest(adapterDescription, migrationConfig); - - var migrationResult = WorkerRestClient.migrateAdapter( - adapterMigrationRequest, - extensionsServiceConfig.getServiceUrl() - ); - - if (migrationResult.success()) { - LOG.info("Migration successfully performed by extensions service. Updating adapter description ..."); - LOG.debug( - "Migration was performed by extensions service '{}'", - extensionsServiceConfig.getServiceUrl()); - - adapterStorage.updateAdapter(migrationResult.element()); - LOG.info("Adapter description is updated - Migration successfully completed at core."); - } else { - LOG.error( - "Migration failed - Failure report: {}", - StringUtils.join(migrationResult.messages(), ",") - ); - LOG.error( - "Migration for adapter '{}' failed - Stopping adapter ...", - migrationResult.element().getElementId() - ); - try { - WorkerRestClient.stopStreamAdapter(extensionsServiceConfig.getServiceUrl(), adapterDescription); - } catch (AdapterException e) { - LOG.error("Stopping adapter failed: {}", StringUtils.join(e.getStackTrace(), "\n")); - } - LOG.info("Adapter successfully stopped."); - } - } else { - LOG.info("Migration is not applicable for adapter '{}' because of a version mismatch - " - + "adapter version: '{}', migration starts at: '{}'", - adapterDescription.getElementId(), - adapterVersion, - migrationConfig.fromVersion() - ); - + var adapterDescriptions = adapterStorage.getAdaptersByAppId(migrationConfig.targetAppId()); + LOG.info("Found {} instances for appId '{}'", adapterDescriptions.size(), migrationConfig.targetAppId()); + for (var adapterDescription: adapterDescriptions) { + + var adapterVersion = adapterDescription.getVersion(); + + if (adapterVersion == migrationConfig.fromVersion()) { + LOG.info("Migration is required for adapter '{}'. Migrating from version '{}' to '{}' ...", + adapterDescription.getElementId(), + adapterVersion, migrationConfig.toVersion() + ); + + var adapterMigrationRequest = new AdapterMigrationRequest(adapterDescription, migrationConfig); + + var migrationResult = WorkerRestClient.migrateAdapter( + adapterMigrationRequest, + extensionsServiceConfig.getServiceUrl() + ); + + if (migrationResult.success()) { + LOG.info("Migration successfully performed by extensions service. Updating adapter description ..."); + LOG.debug( + "Migration was performed by extensions service '{}'", + extensionsServiceConfig.getServiceUrl()); + + adapterStorage.updateAdapter(migrationResult.element()); + LOG.info("Adapter description is updated - Migration successfully completed at Core."); + } else { + LOG.error( + "Migration failed - Failure report: {}", + StringUtils.join(migrationResult.messages(), ",") + ); + LOG.error( + "Migration for adapter '{}' failed - Stopping adapter ...", + migrationResult.element().getElementId() + ); + try { + WorkerRestClient.stopStreamAdapter(extensionsServiceConfig.getServiceUrl(), adapterDescription); + } catch (AdapterException e) { + LOG.error("Stopping adapter failed: {}", StringUtils.join(e.getStackTrace(), "\n")); } + LOG.info("Adapter successfully stopped."); } + } else { + LOG.info("Migration is not applicable for adapter '{}' because of a version mismatch - " + + "adapter version: '{}', migration starts at: '{}'", + adapterDescription.getElementId(), + adapterVersion, + migrationConfig.fromVersion() + ); + } - case DATA_PROCESSOR -> {} - case DATA_SINK -> {} - default -> LOG.error("Can't handle migration for model type {}", migrationConfig.modelType() - ); } - } return ok(); } + + // //TODO: abstract method: data processor description & data sink description are both consumableStreamPipesEntiteis +// var dataProcessorDescriptions = dataProcessorStorage.getDataProcessorsByAppId(migrationConfig.targetAppId()); +// LOG.info("Found {} instances for appId '{}'", +// dataProcessorDescriptions.size(), +// migrationConfig.targetAppId() +// ); +// for (var dataProcessorDescription : dataProcessorDescriptions) { +// var processorVersion = dataProcessorDescription.getVersion(); +// +// if (processorVersion == migrationConfig.fromVersion()) { +// LOG.info("Migration is required for {} '{}'. Migrating from version '{}' to '{}' ...", +// migrationConfig.modelType(), +// dataProcessorDescription.getElementId(), +// processorVersion, migrationConfig.toVersion() +// ); +// +// pipelineStorage.getPipeline(dataProcessorDescription.getConnectedTo()) +// +// var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); +// +// var migrationRequest = new PipelineElementMigrationRequest( +// dataProcessorDescription, +// migrationConfig +// ); +// var migrationResult = WorkerRestClient.migrateAdapter( +// migrationRequest, +// extensionsServiceConfig.getServiceUrl() +// ); +// +// if (migrationResult.success()) { +// LOG.info("Migration successfully performed by extensions service. Updating {}} description ...", +// migrationConfig.modelType()); +// LOG.debug( +// "Migration was performed by extensions service '{}'", +// extensionsServiceConfig.getServiceUrl()); +// +// adapterStorage.updateAdapter(migrationResult.element()); +// LOG.info("{}} description is updated - Migration successfully completed at core.", migrationConfig.modelType()); +// } else { +// LOG.error( +// "Migration failed - Failure report: {}", +// StringUtils.join(migrationResult.messages(), ",") +// ); +// LOG.error( +// "Migration for {}} '{}' failed - Stopping adapter ...", +// migrationConfig.modelType(), +// migrationResult.element().getElementId() +// ); +// try { +// WorkerRestClient.stopStreamAdapter(extensionsServiceConfig.getServiceUrl(), adapterDescription); +// } catch (AdapterException e) { +// LOG.error("Stopping adapter failed: {}", StringUtils.join(e.getStackTrace(), "\n")); +// } +// LOG.info("Adapter successfully stopped."); +// } +// } else { +// LOG.info("Migration is not applicable for {} '{}' because of a version mismatch - " +// + "adapter version: '{}', migration starts at: '{}'", +// migrationConfig.modelType(), +// dataProcessorDescription.getElementId(), +// processorVersion, +// migrationConfig.fromVersion() +// ); +// +// } +// } } diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java index a67ab9ebbe..c7e6b316df 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java @@ -23,6 +23,7 @@ import org.apache.streampipes.extensions.management.init.DeclarersSingleton; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.service.extensions.function.StreamPipesFunctionHandler; import org.apache.streampipes.service.extensions.security.WebSecurityConfig; @@ -48,10 +49,13 @@ public void onExit() { @Override public void afterServiceRegistered(SpServiceDefinition serviceDef) { - // register all migrations at the StreamPipes Core StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); - client.adminApi().registerMigrations( - serviceDef.getMigrators().stream().map(ModelMigrator::config).toList(), + + // register all adapter migrations at StreamPipes Core + var adapterMigrations = serviceDef.getMigrators().stream() + .filter(modelMigrator -> modelMigrator.config().modelType() == SpServiceTagPrefix.ADAPTER).toList(); + client.adminApi().registerAdapterMigrations( + adapterMigrations.stream().map(ModelMigrator::config).toList(), serviceId() ); From 4cbd9c4ef992c7cafc15e7f22a76527af55a882d Mon Sep 17 00:00:00 2001 From: bossenti Date: Tue, 24 Oct 2023 15:37:30 +0200 Subject: [PATCH 24/54] refactor: move MigrationResult to StreamPipes model --- .../connect/management/management/WorkerRestClient.java | 2 +- .../streampipes/extensions/api/migration/ModelMigrator.java | 1 + .../{extensions/api => model}/migration/MigrationResult.java | 2 +- .../management/model/SpServiceDefinitionBuilderTest.java | 2 +- .../{MigrationResource.java => AdapterMigrationResource.java} | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) rename streampipes-extensions-api/src/main/java/org/apache/streampipes/{extensions/api => model}/migration/MigrationResult.java (96%) rename streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/{MigrationResource.java => AdapterMigrationResource.java} (98%) diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java index 5dfb0bfbc1..11144e6dfd 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java @@ -21,11 +21,11 @@ import org.apache.streampipes.commons.exceptions.SpConfigurationException; import org.apache.streampipes.commons.exceptions.connect.AdapterException; import org.apache.streampipes.connect.management.util.WorkerPaths; -import org.apache.streampipes.extensions.api.migration.MigrationResult; import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.extensions.migration.AdapterMigrationRequest; import org.apache.streampipes.model.message.Notification; +import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.runtime.RuntimeOptionsRequest; import org.apache.streampipes.model.runtime.RuntimeOptionsResponse; import org.apache.streampipes.model.util.Cloner; diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java index d95ff49c13..e1b6731ab2 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java @@ -20,6 +20,7 @@ import org.apache.streampipes.extensions.api.extractor.IParameterExtractor; import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.migration.ModelMigratorConfig; public interface ModelMigrator< diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java similarity index 96% rename from streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java rename to streampipes-extensions-api/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java index 09204ceaba..0788c9b32e 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/MigrationResult.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.extensions.api.migration; +package org.apache.streampipes.model.migration; import org.apache.streampipes.model.message.Notification; diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilderTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilderTest.java index e36a624a9b..293cbdb173 100644 --- a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilderTest.java +++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilderTest.java @@ -26,10 +26,10 @@ import org.apache.streampipes.extensions.api.extractor.IAdapterParameterExtractor; import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor; import org.apache.streampipes.extensions.api.migration.AdapterMigrator; -import org.apache.streampipes.extensions.api.migration.MigrationResult; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.connect.guess.GuessSchema; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.migration.ModelMigratorConfig; import org.junit.Test; diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java similarity index 98% rename from streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrationResource.java rename to streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java index 5137c0cdc9..7e515dd88e 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrationResource.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java @@ -19,10 +19,10 @@ package org.apache.streampipes.rest.extensions.migration; import org.apache.streampipes.extensions.api.migration.AdapterMigrator; -import org.apache.streampipes.extensions.api.migration.MigrationResult; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.extensions.migration.AdapterMigrationRequest; import org.apache.streampipes.model.message.Notification; +import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.rest.security.AuthConstants; import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; import org.apache.streampipes.sdk.extractor.StaticPropertyExtractor; From 06f349f1f4e1e69664ef1a530662541d4b34dcc7 Mon Sep 17 00:00:00 2001 From: bossenti Date: Tue, 24 Oct 2023 15:51:29 +0200 Subject: [PATCH 25/54] refactor: migration result --- .../management/WorkerRestClient.java | 56 +++++++++---------- .../model/migration/MigrationResult.java | 23 ++++---- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java index 11144e6dfd..46285db2a9 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java @@ -24,7 +24,6 @@ import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.extensions.migration.AdapterMigrationRequest; -import org.apache.streampipes.model.message.Notification; import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.runtime.RuntimeOptionsRequest; import org.apache.streampipes.model.runtime.RuntimeOptionsResponse; @@ -70,8 +69,7 @@ public static void stopStreamAdapter(String baseUrl, AdapterDescription adapterStreamDescription) throws AdapterException { String url = baseUrl + WorkerPaths.getStreamStopPath(); - var ad = - getAdapterDescriptionById(new AdapterInstanceStorageImpl(), adapterStreamDescription.getElementId()); + var ad = getAdapterDescriptionById(new AdapterInstanceStorageImpl(), adapterStreamDescription.getElementId()); stopAdapter(ad, url); updateStreamAdapterStatus(adapterStreamDescription.getElementId(), false); @@ -81,8 +79,8 @@ public static List getAllRunningAdapterInstanceDescriptions( try { LOG.info("Requesting all running adapter description instances: " + url); var responseString = ExtensionServiceExecutions - .extServiceGetRequest(url) - .execute().returnContent().asString(); + .extServiceGetRequest(url) + .execute().returnContent().asString(); return JacksonSerializer.getObjectMapper().readValue(responseString, List.class); } catch (IOException e) { @@ -92,14 +90,14 @@ public static List getAllRunningAdapterInstanceDescriptions( } private static void startAdapter(String url, - AdapterDescription ad) throws AdapterException { + AdapterDescription ad) throws AdapterException { LOG.info("Trying to start adapter on endpoint {} ", url); triggerAdapterStateChange(ad, url, "started"); } private static void stopAdapter(AdapterDescription ad, - String url) throws AdapterException { + String url) throws AdapterException { LOG.info("Trying to stop adapter on endpoint {} ", url); triggerAdapterStateChange(ad, url, "stopped"); @@ -109,12 +107,13 @@ private static void stopAdapter(AdapterDescription ad, * Sends a migration request for an adapter to the extensions service. * Encapsulates the communication with the service fully so that only a migration result is returned * which reports about the outcome of the migration. + * * @param adapterMigrationRequest request that describes the adapter migration - * @param serviceUrl URL of the extensions service where the migration should be requested + * @param serviceUrl URL of the extensions service where the migration should be requested * @return The outcome of the migration in form of a {@link MigrationResult} */ public static MigrationResult migrateAdapter(AdapterMigrationRequest adapterMigrationRequest, - String serviceUrl + String serviceUrl ) { try { String serializedRequest = JacksonSerializer.getObjectMapper().writeValueAsString(adapterMigrationRequest); @@ -132,24 +131,25 @@ public static MigrationResult migrateAdapter(AdapterMigratio ); } catch (JsonProcessingException e) { - return MigrationResult.failure(adapterMigrationRequest.adapterDescription(), - List.of(new Notification( - "MigrationNotSend", - String.format( - "Serialization of AdapterMigrationRequest failed, " - + "could not sent request to extensions service: %s.", - StringUtils.join(e.getStackTrace(), "\n") - ) - ))); + LOG.error("Migration of adapter failed before sending to the extensions service, adapter is not migrated."); + return MigrationResult.failure( + adapterMigrationRequest.adapterDescription(), + String.format( + "Serialization of AdapterMigrationRequest failed, " + + "could not sent request to extensions service: %s.", + StringUtils.join(e.getStackTrace(), "\n") + ) + ); } catch (IOException e) { LOG.error("Migration of adapter failed at the extensions service, adapter is not migrated."); - return MigrationResult.failure(adapterMigrationRequest.adapterDescription(), - new Notification( - "MigrationCommunicationError", - String.format("An error occurred in the communication between StreamPipes core and " - + "the extensions service: %s", - StringUtils.join(e.getStackTrace(), "\n")) - )); + return MigrationResult.failure( + adapterMigrationRequest.adapterDescription(), + String.format( + "An error occurred in the communication between StreamPipes core and " + + "the extensions service: %s", + StringUtils.join(e.getStackTrace(), "\n") + ) + ); } } @@ -218,9 +218,9 @@ public static String getAssets(String workerPath) throws AdapterException { try { return Request.Get(url) - .connectTimeout(1000) - .socketTimeout(100000) - .execute().returnContent().asString(); + .connectTimeout(1000) + .socketTimeout(100000) + .execute().returnContent().asString(); } catch (IOException e) { LOG.error(e.getMessage()); throw new AdapterException("Could not get assets endpoint: " + url); diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java index 0788c9b32e..805dc65ea4 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java @@ -18,21 +18,20 @@ package org.apache.streampipes.model.migration; -import org.apache.streampipes.model.message.Notification; - -import java.util.List; - -public record MigrationResult (boolean success, T element, List messages){ - - public static MigrationResult failure(T element, List messages) { - return new MigrationResult<>(false, element, messages); - } +/** + * Models the outcome of a migration. + * @param success whether the migration was successfully or not. + * @param element the migrated pipeline element or in case of a failure the original one + * @param message message that describes the outcome of the migration + * @param type of the migration element + */ +public record MigrationResult (boolean success, T element, String message){ - public static MigrationResult failure(T element, Notification message) { - return failure(element, List.of(message)); + public static MigrationResult failure(T element, String message) { + return new MigrationResult<>(false, element, message); } public static MigrationResult success(T element) { - return new MigrationResult<>(true, element, List.of()); + return new MigrationResult<>(true, element, "SUCCESS"); } } From 4656bb248a1611a5fcf850e06341959e8729690a Mon Sep 17 00:00:00 2001 From: bossenti Date: Wed, 25 Oct 2023 12:45:43 +0200 Subject: [PATCH 26/54] refactor: introduce generic migration request --- ...tionRequest.java => MigrationRequest.java} | 10 +++---- .../migration/AdapterMigrationResource.java | 26 +++++++------------ 2 files changed, 14 insertions(+), 22 deletions(-) rename streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/{AdapterMigrationRequest.java => MigrationRequest.java} (76%) diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/AdapterMigrationRequest.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/MigrationRequest.java similarity index 76% rename from streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/AdapterMigrationRequest.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/MigrationRequest.java index 4b372a9e22..13837d12e1 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/AdapterMigrationRequest.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/migration/MigrationRequest.java @@ -18,13 +18,11 @@ package org.apache.streampipes.model.extensions.migration; -import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.migration.ModelMigratorConfig; /** - * Models a request that is sent from the core to an extensions service to mandate an adapter migration. - * @param adapterDescription adapter to be migrated - * @param modelMigratorConfig migration config that describes the migration to be applied to the adapter. + * Models a request that is sent from the core to an extensions service to mandate a migration. + * @param migrationElement element that needs to be migrated + * @param modelMigratorConfig migration config that describes the migration to be applied. */ -public record AdapterMigrationRequest(AdapterDescription adapterDescription, ModelMigratorConfig modelMigratorConfig) { -} +public record MigrationRequest(T migrationElement, ModelMigratorConfig modelMigratorConfig) {} diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java index 7e515dd88e..89cddbd17f 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java @@ -20,8 +20,7 @@ import org.apache.streampipes.extensions.api.migration.AdapterMigrator; import org.apache.streampipes.model.connect.adapter.AdapterDescription; -import org.apache.streampipes.model.extensions.migration.AdapterMigrationRequest; -import org.apache.streampipes.model.message.Notification; +import org.apache.streampipes.model.extensions.migration.MigrationRequest; import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.rest.security.AuthConstants; import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; @@ -39,13 +38,13 @@ import jakarta.ws.rs.core.Response; -@Path("/api/v1/migrations") -public class MigrationResource extends MigrateExtensionsResource { +@Path("/api/v1/migrations/adapter") +public class AdapterMigrationResource extends MigrateExtensionsResource { - private static final Logger LOG = LoggerFactory.getLogger(MigrationResource.class); + private static final Logger LOG = LoggerFactory.getLogger(AdapterMigrationResource.class); /** - * Migrates an adapter instance based on the provided {@link AdapterMigrationRequest}. + * Migrates an adapter instance based on the provided {@link MigrationRequest}. * The outcome of the migration is described in {@link MigrationResult}. * The result is always part of the response. * Independent, of the migration outcome, the returned response always has OK as status code. @@ -54,13 +53,12 @@ public class MigrationResource extends MigrateExtensionsResource adapterMigrateRequest) { - var adapterDescription = adapterMigrateRequest.adapterDescription(); + var adapterDescription = adapterMigrateRequest.migrationElement(); var migrationConfig = adapterMigrateRequest.modelMigratorConfig(); LOG.info("Received migration request for adapter '{}' to migrate from version {} to {}", @@ -91,7 +89,7 @@ public Response migrateAdapter(AdapterMigrationRequest adapterMigrateRequest) { return ok(MigrationResult.success(migratedAdapterDescription)); } else { - LOG.error("Migration failed with the following reasons: {}", StringUtils.join(result.messages(), ", ")); + LOG.error("Migration failed with the following reason: {}", result.message()); // The failed migration is documented in the MigrationResult // The core is expected to handle the response accordingly, so we can safely return a positive status code return ok(result); @@ -102,26 +100,22 @@ public Response migrateAdapter(AdapterMigrationRequest adapterMigrateRequest) { + "sending exception report in migration result"); return ok(MigrationResult.failure( adapterDescription, - new Notification( - "MigrationFailure", String.format( "Adapter Migration failed due to an unexpected exception: %s", StringUtils.join(e.getStackTrace(), "\n") ) ) - )); + ); } } LOG.error("Migrator for migration config {} could not be found. Migration is cancelled.", migrationConfig); return ok(MigrationResult.failure( adapterDescription, - new Notification( - "MigrationNotFound", String.format( "The given migration config '%s' could not be mapped to a registered migrator.", migrationConfig ) ) - )); + ); } } From d4390bb1714fe31ebb4e6c2d51eade88972a1da5 Mon Sep 17 00:00:00 2001 From: bossenti Date: Wed, 25 Oct 2023 12:49:30 +0200 Subject: [PATCH 27/54] feat: send migration requests to core --- .../streampipes/client/api/IAdminApi.java | 2 ++ .../streampipes/client/api/AdminApi.java | 10 ++++++ .../base/InvocableStreamPipesEntity.java | 35 ++++++++++++++++--- .../model/graph/DataProcessorInvocation.java | 9 +++-- .../model/graph/DataSinkInvocation.java | 5 +++ .../extensions/ExtensionsModelSubmitter.java | 23 +++++++++--- .../extensions/ExtensionsResourceConfig.java | 4 +-- 7 files changed, 76 insertions(+), 12 deletions(-) diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java index d07123550a..35047ca9e6 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java @@ -42,5 +42,7 @@ public interface IAdminApi { void registerAdapterMigrations(List migrationConfigs, String serviceId); + void registerPipelineElementMigrations(List migratorConfigs, String serviceId); + MessagingSettings getMessagingSettings(); } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java index 7f1a9e06d7..0670d94483 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java @@ -76,6 +76,10 @@ public void registerAdapterMigrations(List migrationConfigs post(getAdapterMigrationPath().addToPath(serviceId), migrationConfigs); } + public void registerPipelineElementMigrations(List migratorConfigs, String serviceId) { + post(getPipelineElementMigrationPath().addToPath(serviceId), migratorConfigs); + } + @Override public MessagingSettings getMessagingSettings() { return getSingle(getMessagingSettingsPath(), MessagingSettings.class); @@ -114,4 +118,10 @@ private StreamPipesApiPath getAdapterMigrationPath() { .fromBaseApiPath() .addToPath("migrations/adapter"); } + + private StreamPipesApiPath getPipelineElementMigrationPath() { + return StreamPipesApiPath + .fromBaseApiPath() + .addToPath("migrations/pipeline-element"); + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java index 0d64297fc9..925274e922 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java @@ -23,6 +23,7 @@ import org.apache.streampipes.logging.api.Logger; import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.api.EndpointSelectable; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.grounding.EventGrounding; import org.apache.streampipes.model.monitoring.ElementStatusInfoSettings; import org.apache.streampipes.model.staticproperty.StaticProperty; @@ -32,7 +33,9 @@ import java.util.List; -public abstract class InvocableStreamPipesEntity extends NamedStreamPipesEntity implements EndpointSelectable { +public abstract class InvocableStreamPipesEntity + extends NamedStreamPipesEntity + implements EndpointSelectable, VersionedStreamPipesEntity { protected List inputStreams; @@ -56,6 +59,9 @@ public abstract class InvocableStreamPipesEntity extends NamedStreamPipesEntity private String selectedEndpointUrl; + private int version; + protected SpServiceTagPrefix serviceTagPrefix; + public InvocableStreamPipesEntity() { super(); } @@ -69,6 +75,8 @@ public InvocableStreamPipesEntity(InvocableStreamPipesEntity other) { this.uncompleted = other.isUncompleted(); this.correspondingUser = other.getCorrespondingUser(); this.selectedEndpointUrl = other.getSelectedEndpointUrl(); + this.serviceTagPrefix = other.serviceTagPrefix; + this.version = other.getVersion(); if (other.getStreamRequirements() != null) { this.streamRequirements = new Cloner().streams(other.getStreamRequirements()); } @@ -81,8 +89,15 @@ public InvocableStreamPipesEntity(InvocableStreamPipesEntity other) { } } - public InvocableStreamPipesEntity(String uri, String name, String description, String iconUrl) { + public InvocableStreamPipesEntity( + String uri, + String name, + String description, + String iconUrl, + SpServiceTagPrefix serviceTagPrefix + ) { super(uri, name, description, iconUrl); + this.serviceTagPrefix = serviceTagPrefix; this.configured = false; } @@ -172,6 +187,10 @@ public void setUncompleted(boolean uncompleted) { this.uncompleted = uncompleted; } + public SpServiceTagPrefix getServiceTagPrefix() { + return serviceTagPrefix; + } + @Override public String getSelectedEndpointUrl() { return selectedEndpointUrl; @@ -188,9 +207,17 @@ public String getDetachPath() { return "/" + InstanceIdExtractor.extractId(getElementId()); } - //public Logger getLogger(Class clazz, PeConfig peConfig) { public Logger getLogger(Class clazz) { - //return LoggerFactory.getPeLogger(clazz, getCorrespondingPipeline(), getUri(), peConfig); return LoggerFactory.getPeLogger(clazz, getCorrespondingPipeline(), getUri()); } + + @Override + public int getVersion(){ + return this.version; + } + + @Override + public void setVersion(int version){ + this.version = version; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorInvocation.java b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorInvocation.java index 89cac11dff..64aed3b49b 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorInvocation.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorInvocation.java @@ -20,6 +20,7 @@ import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.output.OutputStrategy; import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.model.util.Cloner; @@ -57,6 +58,7 @@ public DataProcessorInvocation(DataProcessorDescription other) { this.setIncludesAssets(other.isIncludesAssets()); this.setIncludedAssets(other.getIncludedAssets()); this.setElementId(ElementIdGenerator.makeElementId(this)); + this.serviceTagPrefix = SpServiceTagPrefix.DATA_PROCESSOR; } public DataProcessorInvocation(DataProcessorInvocation other) { @@ -67,28 +69,31 @@ public DataProcessorInvocation(DataProcessorInvocation other) { } this.pathName = other.getPathName(); this.category = new Cloner().epaTypes(other.getCategory()); + this.serviceTagPrefix = SpServiceTagPrefix.DATA_PROCESSOR; } public DataProcessorInvocation(DataProcessorDescription sepa, String domId) { this(sepa); this.dom = domId; + this.serviceTagPrefix = SpServiceTagPrefix.DATA_PROCESSOR; } public DataProcessorInvocation() { super(); inputStreams = new ArrayList<>(); + this.serviceTagPrefix = SpServiceTagPrefix.DATA_PROCESSOR; } public DataProcessorInvocation(String uri, String name, String description, String iconUrl, String pathName, List spDataStreams, List staticProperties) { - super(uri, name, description, iconUrl); + super(uri, name, description, iconUrl, SpServiceTagPrefix.DATA_PROCESSOR); this.pathName = pathName; this.inputStreams = spDataStreams; this.staticProperties = staticProperties; } public DataProcessorInvocation(String uri, String name, String description, String iconUrl, String pathName) { - super(uri, name, description, iconUrl); + super(uri, name, description, iconUrl, SpServiceTagPrefix.DATA_PROCESSOR); this.pathName = pathName; inputStreams = new ArrayList<>(); staticProperties = new ArrayList<>(); diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkInvocation.java b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkInvocation.java index f334778508..4476c6651b 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkInvocation.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkInvocation.java @@ -19,6 +19,7 @@ package org.apache.streampipes.model.graph; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.model.util.ElementIdGenerator; @@ -34,6 +35,7 @@ public class DataSinkInvocation extends InvocableStreamPipesEntity { public DataSinkInvocation(DataSinkInvocation sec) { super(sec); this.category = sec.getCategory(); + this.serviceTagPrefix = SpServiceTagPrefix.DATA_SINK; } public DataSinkInvocation(DataSinkDescription other) { @@ -51,16 +53,19 @@ public DataSinkInvocation(DataSinkDescription other) { this.setIncludesAssets(other.isIncludesAssets()); this.setElementId(ElementIdGenerator.makeElementId(this)); this.setIncludedAssets(other.getIncludedAssets()); + this.serviceTagPrefix = SpServiceTagPrefix.DATA_SINK; } public DataSinkInvocation(DataSinkDescription sec, String domId) { this(sec); this.setDom(domId); + this.serviceTagPrefix = SpServiceTagPrefix.DATA_SINK; } public DataSinkInvocation() { super(); inputStreams = new ArrayList<>(); + this.serviceTagPrefix = SpServiceTagPrefix.DATA_SINK; } public List getStaticProperties() { diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java index c7e6b316df..c562ae02fa 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java @@ -52,11 +52,26 @@ public void afterServiceRegistered(SpServiceDefinition serviceDef) { StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); // register all adapter migrations at StreamPipes Core - var adapterMigrations = serviceDef.getMigrators().stream() - .filter(modelMigrator -> modelMigrator.config().modelType() == SpServiceTagPrefix.ADAPTER).toList(); + var adapterMigrations = serviceDef.getMigrators() + .stream() + .filter(modelMigrator -> modelMigrator.config().modelType() == SpServiceTagPrefix.ADAPTER) + .toList(); client.adminApi().registerAdapterMigrations( - adapterMigrations.stream().map(ModelMigrator::config).toList(), - serviceId() + adapterMigrations.stream().map(ModelMigrator::config).toList(), + serviceId() + ); + + // register all pipeline element migrations at StreamPipes Core + var pipelineElementMigrations = serviceDef.getMigrators() + .stream() + .filter(modelMigrator -> + modelMigrator.config().modelType() == SpServiceTagPrefix.DATA_PROCESSOR + || modelMigrator.config().modelType() == SpServiceTagPrefix.DATA_SINK + ) + .toList(); + client.adminApi().registerPipelineElementMigrations( + pipelineElementMigrations.stream().map(ModelMigrator::config).toList(), + serviceId() ); // initialize all function instances diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java index a061865c73..ff785a3e40 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java @@ -25,7 +25,7 @@ import org.apache.streampipes.rest.extensions.connect.GuessResource; import org.apache.streampipes.rest.extensions.connect.HttpServerAdapterResource; import org.apache.streampipes.rest.extensions.connect.RuntimeResolvableResource; -import org.apache.streampipes.rest.extensions.migration.MigrationResource; +import org.apache.streampipes.rest.extensions.migration.AdapterMigrationResource; import org.apache.streampipes.rest.extensions.monitoring.MonitoringResource; import org.apache.streampipes.rest.extensions.pe.DataProcessorPipelineElementResource; import org.apache.streampipes.rest.extensions.pe.DataSinkPipelineElementResource; @@ -49,6 +49,7 @@ public Set> getClassesToRegister() { return Set.of( AdapterAssetResource.class, AdapterDescriptionResource.class, + AdapterMigrationResource.class, AdapterWorkerResource.class, DataProcessorPipelineElementResource.class, DataSinkPipelineElementResource.class, @@ -56,7 +57,6 @@ public Set> getClassesToRegister() { GuessResource.class, HttpServerAdapterResource.class, JacksonSerializationProvider.class, - MigrationResource.class, MonitoringResource.class, MultiPartFeature.class, RuntimeResolvableResource.class, From 16dc3b25b9da6bb64a9e6eae2f9b480737c6e0cf Mon Sep 17 00:00:00 2001 From: bossenti Date: Wed, 25 Oct 2023 12:50:04 +0200 Subject: [PATCH 28/54] feat: process migrations at core --- .../management/WorkerRestClient.java | 56 --- .../rest/impl/admin/MigrationResource.java | 365 ++++++++++++++---- .../impl/admin/MigrationResourceTest.java | 77 ++++ 3 files changed, 358 insertions(+), 140 deletions(-) create mode 100644 streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java index 46285db2a9..c45506d0f5 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java @@ -23,8 +23,6 @@ import org.apache.streampipes.connect.management.util.WorkerPaths; import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; import org.apache.streampipes.model.connect.adapter.AdapterDescription; -import org.apache.streampipes.model.extensions.migration.AdapterMigrationRequest; -import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.runtime.RuntimeOptionsRequest; import org.apache.streampipes.model.runtime.RuntimeOptionsResponse; import org.apache.streampipes.model.util.Cloner; @@ -34,11 +32,8 @@ import org.apache.streampipes.storage.couchdb.impl.AdapterInstanceStorageImpl; import org.apache.streampipes.storage.management.StorageDispatcher; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpResponse; import org.apache.http.client.fluent.Request; import org.slf4j.Logger; @@ -54,7 +49,6 @@ public class WorkerRestClient { private static final Logger LOG = LoggerFactory.getLogger(WorkerRestClient.class); - private static final String ADAPTER_MIGRATION_ENDPOINT = "api/v1/migrations/adapter"; public static void invokeStreamAdapter(String endpointUrl, String elementId) throws AdapterException { @@ -103,56 +97,6 @@ private static void stopAdapter(AdapterDescription ad, triggerAdapterStateChange(ad, url, "stopped"); } - /** - * Sends a migration request for an adapter to the extensions service. - * Encapsulates the communication with the service fully so that only a migration result is returned - * which reports about the outcome of the migration. - * - * @param adapterMigrationRequest request that describes the adapter migration - * @param serviceUrl URL of the extensions service where the migration should be requested - * @return The outcome of the migration in form of a {@link MigrationResult} - */ - public static MigrationResult migrateAdapter(AdapterMigrationRequest adapterMigrationRequest, - String serviceUrl - ) { - try { - String serializedRequest = JacksonSerializer.getObjectMapper().writeValueAsString(adapterMigrationRequest); - var migrationResponse = ExtensionServiceExecutions.extServicePostRequest( - "%s/%s".formatted(serviceUrl, ADAPTER_MIGRATION_ENDPOINT), - serializedRequest - ).execute(); - - TypeReference> typeReference = new TypeReference<>() { - }; - - return JacksonSerializer.getObjectMapper().readValue( - migrationResponse.returnContent().asString(), - typeReference - ); - - } catch (JsonProcessingException e) { - LOG.error("Migration of adapter failed before sending to the extensions service, adapter is not migrated."); - return MigrationResult.failure( - adapterMigrationRequest.adapterDescription(), - String.format( - "Serialization of AdapterMigrationRequest failed, " - + "could not sent request to extensions service: %s.", - StringUtils.join(e.getStackTrace(), "\n") - ) - ); - } catch (IOException e) { - LOG.error("Migration of adapter failed at the extensions service, adapter is not migrated."); - return MigrationResult.failure( - adapterMigrationRequest.adapterDescription(), - String.format( - "An error occurred in the communication between StreamPipes core and " - + "the extensions service: %s", - StringUtils.join(e.getStackTrace(), "\n") - ) - ); - } - } - private static void triggerAdapterStateChange(AdapterDescription ad, String url, String action) throws AdapterException { diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index d77ade1395..310ac56113 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -20,15 +20,30 @@ import org.apache.streampipes.commons.exceptions.connect.AdapterException; import org.apache.streampipes.connect.management.management.WorkerRestClient; -import org.apache.streampipes.model.extensions.migration.AdapterMigrationRequest; +import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; +import org.apache.streampipes.manager.execution.PipelineExecutor; +import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.base.VersionedStreamPipesEntity; +import org.apache.streampipes.model.extensions.migration.MigrationRequest; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.graph.DataProcessorInvocation; +import org.apache.streampipes.model.graph.DataSinkInvocation; +import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.apache.streampipes.model.pipeline.Pipeline; +import org.apache.streampipes.model.pipeline.PipelineHealthStatus; +import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.serializers.json.JacksonSerializer; import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.storage.api.IAdapterStorage; +import org.apache.streampipes.storage.api.IDataProcessorStorage; +import org.apache.streampipes.storage.api.IDataSinkStorage; import org.apache.streampipes.storage.api.IPipelineStorage; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +57,10 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; @Path("v2/migrations") @@ -52,12 +70,23 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { private static final Logger LOG = LoggerFactory.getLogger(MigrationResource.class); - private final IAdapterStorage adapterStorage = getNoSqlStorage().getAdapterInstanceStorage(); - private final IPipelineStorage pipelineStorage = getNoSqlStorage().getPipelineStorageAPI(); + private static final String ADAPTER_MIGRATION_ENDPOINT = "api/v1/migrations/adapter"; + private static final String PIPELINE_ELEMENT_MIGRATION_ENDPOINT = "api/v1/migrations/pipeline-elements"; private final CRUDStorage extensionsServiceStorage = getNoSqlStorage().getExtensionsServiceStorage(); + private final IAdapterStorage adapterStorage = getNoSqlStorage().getAdapterInstanceStorage(); + + private final IDataProcessorStorage dataProcessorStorage = getNoSqlStorage().getDataProcessorStorage(); + + private final IDataSinkStorage dataSinkStorage = getNoSqlStorage().getDataSinkStorage(); + private final IPipelineStorage pipelineStorage = getNoSqlStorage().getPipelineStorageAPI(); + + + // TODO: override all existing descriptions(adapter, pipeline elements) + // Use functionality of update button in UI + // increase version as well @POST @Path("adapter/{serviceId}") @Consumes(MediaType.APPLICATION_JSON) @@ -70,13 +99,13 @@ public Response registerAdapterMigrations( LOG.info("Received {} migrations from extension service {}.", migrationConfigs.size(), extensionsServiceConfig.getServiceUrl()); - LOG.info("Checking migrations for existing assets in StreamPipes Core ..."); - for (var migrationConfig: migrationConfigs) { + LOG.info("Checking migrations for existing adapters in StreamPipes Core ..."); + for (var migrationConfig : migrationConfigs) { LOG.info("Searching for assets of '{}'", migrationConfig.targetAppId()); LOG.debug("Searching for assets of '{}' with config {}", migrationConfig.targetAppId(), migrationConfig); var adapterDescriptions = adapterStorage.getAdaptersByAppId(migrationConfig.targetAppId()); LOG.info("Found {} instances for appId '{}'", adapterDescriptions.size(), migrationConfig.targetAppId()); - for (var adapterDescription: adapterDescriptions) { + for (var adapterDescription : adapterDescriptions) { var adapterVersion = adapterDescription.getVersion(); @@ -86,12 +115,11 @@ public Response registerAdapterMigrations( adapterVersion, migrationConfig.toVersion() ); - var adapterMigrationRequest = new AdapterMigrationRequest(adapterDescription, migrationConfig); - - var migrationResult = WorkerRestClient.migrateAdapter( - adapterMigrationRequest, - extensionsServiceConfig.getServiceUrl() - ); + var migrationResult = performMigration( + adapterDescription, + migrationConfig, + extensionsServiceConfig.getServiceUrl(), + ADAPTER_MIGRATION_ENDPOINT); if (migrationResult.success()) { LOG.info("Migration successfully performed by extensions service. Updating adapter description ..."); @@ -102,10 +130,7 @@ public Response registerAdapterMigrations( adapterStorage.updateAdapter(migrationResult.element()); LOG.info("Adapter description is updated - Migration successfully completed at Core."); } else { - LOG.error( - "Migration failed - Failure report: {}", - StringUtils.join(migrationResult.messages(), ",") - ); + LOG.error("Migration failed with the following reason: {}", migrationResult.message()); LOG.error( "Migration for adapter '{}' failed - Stopping adapter ...", migrationResult.element().getElementId() @@ -118,83 +143,255 @@ public Response registerAdapterMigrations( LOG.info("Adapter successfully stopped."); } } else { - LOG.info("Migration is not applicable for adapter '{}' because of a version mismatch - " + LOG.info( + "Migration is not applicable for adapter '{}' because of a version mismatch - " + "adapter version: '{}', migration starts at: '{}'", adapterDescription.getElementId(), adapterVersion, migrationConfig.fromVersion() ); - } } } return ok(); } - // //TODO: abstract method: data processor description & data sink description are both consumableStreamPipesEntiteis -// var dataProcessorDescriptions = dataProcessorStorage.getDataProcessorsByAppId(migrationConfig.targetAppId()); -// LOG.info("Found {} instances for appId '{}'", -// dataProcessorDescriptions.size(), -// migrationConfig.targetAppId() -// ); -// for (var dataProcessorDescription : dataProcessorDescriptions) { -// var processorVersion = dataProcessorDescription.getVersion(); -// -// if (processorVersion == migrationConfig.fromVersion()) { -// LOG.info("Migration is required for {} '{}'. Migrating from version '{}' to '{}' ...", -// migrationConfig.modelType(), -// dataProcessorDescription.getElementId(), -// processorVersion, migrationConfig.toVersion() -// ); -// -// pipelineStorage.getPipeline(dataProcessorDescription.getConnectedTo()) -// -// var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); -// -// var migrationRequest = new PipelineElementMigrationRequest( -// dataProcessorDescription, -// migrationConfig -// ); -// var migrationResult = WorkerRestClient.migrateAdapter( -// migrationRequest, -// extensionsServiceConfig.getServiceUrl() -// ); -// -// if (migrationResult.success()) { -// LOG.info("Migration successfully performed by extensions service. Updating {}} description ...", -// migrationConfig.modelType()); -// LOG.debug( -// "Migration was performed by extensions service '{}'", -// extensionsServiceConfig.getServiceUrl()); -// -// adapterStorage.updateAdapter(migrationResult.element()); -// LOG.info("{}} description is updated - Migration successfully completed at core.", migrationConfig.modelType()); -// } else { -// LOG.error( -// "Migration failed - Failure report: {}", -// StringUtils.join(migrationResult.messages(), ",") -// ); -// LOG.error( -// "Migration for {}} '{}' failed - Stopping adapter ...", -// migrationConfig.modelType(), -// migrationResult.element().getElementId() -// ); -// try { -// WorkerRestClient.stopStreamAdapter(extensionsServiceConfig.getServiceUrl(), adapterDescription); -// } catch (AdapterException e) { -// LOG.error("Stopping adapter failed: {}", StringUtils.join(e.getStackTrace(), "\n")); -// } -// LOG.info("Adapter successfully stopped."); -// } -// } else { -// LOG.info("Migration is not applicable for {} '{}' because of a version mismatch - " -// + "adapter version: '{}', migration starts at: '{}'", -// migrationConfig.modelType(), -// dataProcessorDescription.getElementId(), -// processorVersion, -// migrationConfig.fromVersion() -// ); -// -// } -// } + @POST + @Path("pipeline-element/{serviceId}") + @Consumes(MediaType.APPLICATION_JSON) + public Response registerPipelineElementMigrations( + @PathParam("serviceId") String serviceId, + List migrationConfigs) { + + var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); + LOG.info("Received {} pipeline element migrations from extension service {}.", + migrationConfigs.size(), + extensionsServiceConfig.getServiceUrl()); + var availablePipelines = pipelineStorage.getAllPipelines(); + if (!availablePipelines.isEmpty()) { + LOG.info("Found {} available pipelines. Checking pipelines for applicable migrations...", + availablePipelines.size() + ); + } + + for (var pipeline : availablePipelines) { + List> failedMigrations = new ArrayList<>(); + + var migratedDataProcessors = pipeline.getSepas() + .stream() + .map(processor -> { + if (getApplicableMigration(processor, migrationConfigs).isPresent()) { + return migratePipelineElement( + processor, + migrationConfigs, + extensionsServiceConfig.getServiceUrl(), + failedMigrations + ); + } else { + LOG.info("No migration applicable for data processor '{}'.", processor.getElementId()); + return processor; + } + } + ).toList(); + pipeline.setSepas(migratedDataProcessors); + + var migratedDataSinks = pipeline.getActions() + .stream() + .map(sink -> { + if (getApplicableMigration(sink, migrationConfigs).isPresent()) { + return migratePipelineElement( + sink, + migrationConfigs, + extensionsServiceConfig.getServiceUrl(), + failedMigrations + ); + } else { + LOG.info("No migration applicable for data sink '{}'.", sink.getElementId()); + return sink; + } + } + ).toList(); + pipeline.setActions(migratedDataSinks); + + if (failedMigrations.isEmpty()) { + LOG.info("Migration for pipeline successfully completed."); + } else { + handleFailedMigrations(pipeline, failedMigrations); + } + } + + return ok(); + } + + /** + * Takes care about the failed migrations of pipeline elements. + * This includes the following steps: + *
    + *
  • logging of failed pipeline elements + *
  • setting migration results as pipeline notifications + *
  • updating pipeline health status + *
  • stopping the pipeline + *
+ * + * @param pipeline the pipeline affected by failed migrations + * @param failedMigrations the list of failed migrations + */ + protected static void handleFailedMigrations(Pipeline pipeline, List> failedMigrations) { + LOG.error("Failures in migration detected - The following pipeline elements could to be migrated:\n" + + StringUtils.join(failedMigrations.stream().map(Record::toString)), "\n"); + + pipeline.setPipelineNotifications(failedMigrations.stream().map(MigrationResult::message).toList()); + pipeline.setHealthStatus(PipelineHealthStatus.REQUIRES_ATTENTION); + + var pipelineExecutor = new PipelineExecutor(pipeline, true); + var pipelineStopResult = pipelineExecutor.stopPipeline(); + + if (pipelineStopResult.isSuccess()) { + LOG.info("Pipeline successfully stopped."); + } else { + LOG.error("Pipeline stop failed."); + } + } + + /** + * Filter the application migration for a pipeline definition. + * By definition, there is only one migration config that fulfills the requirements. + * Otherwise, it should have been detected as duplicate by the extensions service. + * + * @param pipelineElement pipeline element that should be migrated + * @param migrationConfigs available migration configs to pick the applicable from + * @return config that is applicable for the given pipeline element + */ + protected static Optional getApplicableMigration( + InvocableStreamPipesEntity pipelineElement, + List migrationConfigs + ) { + return migrationConfigs + .stream() + .filter( + config -> config.modelType().equals(pipelineElement.getServiceTagPrefix()) + && config.targetAppId().equals(pipelineElement.getAppId()) + && config.fromVersion() == pipelineElement.getVersion() + ) + .findFirst(); + } + + /** + * Handle the migration of a pipeline element with respect to the given model migration configs. + * All applicable migrations found in the provided configs are executed for the given pipeline element. + * In case a migration fails, the related pipeline element receives the latest definition of its static properties, + * so that the pipeline element can be adapted by the user to resolve the failed migration. + * @param pipelineElement pipeline element to be migrated + * @param modelMigrations list of model migrations that might be applicable for this pipeline element + * @param serviceUrl url of the extensions service that handles the migration + * @param failedMigrations collection of failed migrations which is extended by occurring migration failures + * @param type of the pipeline element (e.g., DataProcessorInvocation) + * @return the migrated (or - in case of a failure - updated) pipeline element + */ + protected T migratePipelineElement( + T pipelineElement, + List modelMigrations, + String serviceUrl, + List> failedMigrations + ) { + + // loop until no migrations are available anymore + // this allows to apply multiple migrations for a pipeline element sequentially + // For example, first migration from 0 to 1 and the second migration from 1 to 2 + while (getApplicableMigration(pipelineElement, modelMigrations).isPresent()) { + + var migrationConfig = getApplicableMigration(pipelineElement, modelMigrations).get(); + LOG.info( + "Found applicable migration for pipeline element '{}': {}", + pipelineElement.getElementId(), + migrationConfig + ); + + var migrationResult = performMigration( + pipelineElement, + migrationConfig, + serviceUrl, + PIPELINE_ELEMENT_MIGRATION_ENDPOINT + ); + + if (migrationResult.success()) { + LOG.info("Migration successfully performed by extensions service. Updating pipeline element invocation ..."); + LOG.debug("Migration was performed by extensions service '{}'", serviceUrl); + return migrationResult.element(); + } else { + LOG.error("Migration failed with the following reason: {}", migrationResult.message()); + failedMigrations.add(migrationResult); + } + } + updateFailedPipelineElement(pipelineElement); + return pipelineElement; + } + + /** + * Performs the actual migration of a pipeline element. + * This includes the communication with the extensions service which runs the migration. + * @param pipelineElement pipeline element to be migrated + * @param migrationConfig config of the migration to be performed + * @param serviceUrl url of the extensions service where the migration should be performed + * @param migrationEndpoint endpoint at which the migration should be performed + * @param type of the processing element + * @return result of the migration + */ + protected MigrationResult performMigration( + T pipelineElement, + ModelMigratorConfig migrationConfig, + String serviceUrl, + String migrationEndpoint + ) { + + try { + + var migrationRequest = new MigrationRequest<>(pipelineElement, migrationConfig); + + String serializedRequest = JacksonSerializer.getObjectMapper().writeValueAsString(migrationRequest); + + var migrationResponse = ExtensionServiceExecutions.extServicePostRequest( + "%s/%s".formatted(serviceUrl, migrationEndpoint), + serializedRequest + ).execute(); + + TypeReference> typeReference = new TypeReference<>() { + }; + + return JacksonSerializer + .getObjectMapper() + .readValue(migrationResponse.returnContent().asString(), typeReference); + } catch (JsonProcessingException e) { + LOG.error( + "Migration of pipeline element failed before sending to the extensions service, " + + "pipeline element is not migrated. Serialization of migration request failed: {}", + StringUtils.join(e.getStackTrace(), "\n") + ); + } catch (IOException e) { + LOG.error("Migration of pipeline element failed at the extensions service, pipeline element is not migrated: {}.", + StringUtils.join(e.getStackTrace(), "\n") + ); + } + return MigrationResult.failure(pipelineElement, "Internal error during migration at StreamPipes Core"); + } + + /** + * Update the static properties of the failed pipeline element with its description. + * This allows to adapt the failed pipeline element in the UI to overcome the failed migration. + * + * @param pipelineElement pipeline element with failed migration + */ + protected void updateFailedPipelineElement(InvocableStreamPipesEntity pipelineElement) { + List updatedStaticProperties = new ArrayList<>(); + if (pipelineElement instanceof DataProcessorInvocation) { + updatedStaticProperties = dataProcessorStorage + .getFirstDataProcessorByAppId(pipelineElement.getAppId()) + .getStaticProperties(); + } else if (pipelineElement instanceof DataSinkInvocation) { + updatedStaticProperties = dataSinkStorage + .getFirstDataSinkByAppId(pipelineElement.getAppId()) + .getStaticProperties(); + } + pipelineElement.setStaticProperties(updatedStaticProperties); + } } diff --git a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java new file mode 100644 index 0000000000..eb3c743253 --- /dev/null +++ b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.impl.admin; + +import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.model.graph.DataProcessorInvocation; +import org.apache.streampipes.model.graph.DataSinkInvocation; +import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class MigrationResourceTest { + + List migrationConfigs = List.of( + new ModelMigratorConfig("app-id", SpServiceTagPrefix.DATA_PROCESSOR, 0, 1), + new ModelMigratorConfig("app-id", SpServiceTagPrefix.DATA_PROCESSOR, 1, 2), + new ModelMigratorConfig("other-app-id", SpServiceTagPrefix.DATA_PROCESSOR, 0, 1), + new ModelMigratorConfig("other-app-id", SpServiceTagPrefix.DATA_SINK, 1, 2) + ); + + @Test + public void findMigrations() { + + var pipelineElement1 = new DataProcessorInvocation(); + pipelineElement1.setAppId("app-id"); + pipelineElement1.setVersion(0); + + var pipelineElement2 = new DataProcessorInvocation(); + pipelineElement2.setAppId("app-id"); + pipelineElement2.setVersion(1); + + var pipelineElement3 = new DataProcessorInvocation(); + pipelineElement3.setAppId("other-app-id"); + pipelineElement3.setVersion(0); + + var pipelineElement4 = new DataSinkInvocation(); + pipelineElement4.setAppId("other-app-id"); + pipelineElement4.setVersion(0); + + assertEquals( + migrationConfigs.get(0), + MigrationResource.getApplicableMigration(pipelineElement1, migrationConfigs).get() + ); + assertEquals( + migrationConfigs.get(1), + MigrationResource.getApplicableMigration(pipelineElement2, migrationConfigs).get() + ); + assertEquals( + migrationConfigs.get(2), + MigrationResource.getApplicableMigration(pipelineElement3, migrationConfigs).get() + ); + assertTrue( + MigrationResource.getApplicableMigration(pipelineElement4, migrationConfigs).isEmpty() + ); + } +} From 0f9475a4b380ecee841d5e02d585079d8af6a980 Mon Sep 17 00:00:00 2001 From: bossenti Date: Wed, 25 Oct 2023 15:23:10 +0200 Subject: [PATCH 29/54] refactor: remove legacy generic --- .../api/extractor/IDataProcessorParameterExtractor.java | 4 +--- .../extensions/api/extractor/IDataSinkParameterExtractor.java | 4 +--- .../extensions/api/extractor/IParameterExtractor.java | 3 +-- .../extensions/api/extractor/IStaticPropertyExtractor.java | 4 +--- .../extensions/api/pe/param/IParameterGenerator.java | 2 +- .../extensions/api/pe/param/IPipelineElementParameters.java | 2 +- .../api/runtime/ResolvesContainerProvidedOutputStrategy.java | 2 +- 7 files changed, 7 insertions(+), 14 deletions(-) diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IDataProcessorParameterExtractor.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IDataProcessorParameterExtractor.java index 30e9e52525..986c0d0810 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IDataProcessorParameterExtractor.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IDataProcessorParameterExtractor.java @@ -18,11 +18,9 @@ package org.apache.streampipes.extensions.api.extractor; -import org.apache.streampipes.model.graph.DataProcessorInvocation; - import java.util.List; -public interface IDataProcessorParameterExtractor extends IParameterExtractor { +public interface IDataProcessorParameterExtractor extends IParameterExtractor { String outputTopic(); List outputKeySelectors(); diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IDataSinkParameterExtractor.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IDataSinkParameterExtractor.java index 16c69c572c..fac25b1ca2 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IDataSinkParameterExtractor.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IDataSinkParameterExtractor.java @@ -18,7 +18,5 @@ package org.apache.streampipes.extensions.api.extractor; -import org.apache.streampipes.model.graph.DataSinkInvocation; - -public interface IDataSinkParameterExtractor extends IParameterExtractor { +public interface IDataSinkParameterExtractor extends IParameterExtractor { } diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IParameterExtractor.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IParameterExtractor.java index 9c091b7d2e..7b3e7539b6 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IParameterExtractor.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IParameterExtractor.java @@ -19,7 +19,6 @@ package org.apache.streampipes.extensions.api.extractor; import org.apache.streampipes.commons.exceptions.SpRuntimeException; -import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.schema.EventProperty; import org.apache.streampipes.model.schema.EventPropertyPrimitive; import org.apache.streampipes.model.schema.PropertyScope; @@ -30,7 +29,7 @@ import java.io.InputStream; import java.util.List; -public interface IParameterExtractor { +public interface IParameterExtractor { String measurementUnit(String runtimeName, Integer streamIndex); String inputTopic(Integer streamIndex); diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IStaticPropertyExtractor.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IStaticPropertyExtractor.java index 017c3ad777..67692b45d9 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IStaticPropertyExtractor.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IStaticPropertyExtractor.java @@ -18,7 +18,5 @@ package org.apache.streampipes.extensions.api.extractor; -import org.apache.streampipes.model.graph.DataSinkInvocation; - -public interface IStaticPropertyExtractor extends IParameterExtractor { +public interface IStaticPropertyExtractor extends IParameterExtractor { } diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/param/IParameterGenerator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/param/IParameterGenerator.java index fef867b047..a9edb8b48e 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/param/IParameterGenerator.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/param/IParameterGenerator.java @@ -23,7 +23,7 @@ public interface IParameterGenerator< IvT extends InvocableStreamPipesEntity, - PeT extends IParameterExtractor, + PeT extends IParameterExtractor, K extends IPipelineElementParameters> { diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/param/IPipelineElementParameters.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/param/IPipelineElementParameters.java index 2ef9dc3d1a..39a1a3be37 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/param/IPipelineElementParameters.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/pe/param/IPipelineElementParameters.java @@ -28,7 +28,7 @@ public interface IPipelineElementParameters< IvT extends InvocableStreamPipesEntity, - ExT extends IParameterExtractor> { + ExT extends IParameterExtractor> { IvT getModel(); diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/runtime/ResolvesContainerProvidedOutputStrategy.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/runtime/ResolvesContainerProvidedOutputStrategy.java index b37871daff..2db19a880a 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/runtime/ResolvesContainerProvidedOutputStrategy.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/runtime/ResolvesContainerProvidedOutputStrategy.java @@ -23,7 +23,7 @@ import org.apache.streampipes.model.schema.EventSchema; public interface ResolvesContainerProvidedOutputStrategy> { + extends IParameterExtractor> { EventSchema resolveOutputStrategy(T processingElement, K parameterExtractor) throws SpConfigurationException; } From c91544064cb88e7ba6cca303ed06e4a45c285c79 Mon Sep 17 00:00:00 2001 From: bossenti Date: Wed, 25 Oct 2023 15:23:45 +0200 Subject: [PATCH 30/54] refactor: introduce versioned StreamPipes entity --- .../base/ConsumableStreamPipesEntity.java | 19 ++--------- .../base/InvocableStreamPipesEntity.java | 17 ++-------- ...a => VersionedNamedStreamPipesEntity.java} | 32 ++++++++++++++++--- .../connect/adapter/AdapterDescription.java | 20 ++---------- 4 files changed, 36 insertions(+), 52 deletions(-) rename streampipes-model/src/main/java/org/apache/streampipes/model/base/{VersionedStreamPipesEntity.java => VersionedNamedStreamPipesEntity.java} (52%) diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java index c3550bf1ce..e2fcb83f7d 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java @@ -26,12 +26,10 @@ import java.util.ArrayList; import java.util.List; -public abstract class ConsumableStreamPipesEntity extends NamedStreamPipesEntity implements VersionedStreamPipesEntity{ +public abstract class ConsumableStreamPipesEntity extends VersionedNamedStreamPipesEntity { private static final long serialVersionUID = -6617391345752016449L; - private int version; - protected List spDataStreams; protected List staticProperties; @@ -44,8 +42,8 @@ public ConsumableStreamPipesEntity() { this.staticProperties = new ArrayList<>(); } - public ConsumableStreamPipesEntity(String uri, String name, String description, String iconUrl) { - super(uri, name, description, iconUrl); + public ConsumableStreamPipesEntity(String elementId, String name, String description, String iconUrl) { + super(elementId, name, description, iconUrl); this.spDataStreams = new ArrayList<>(); this.staticProperties = new ArrayList<>(); } @@ -59,7 +57,6 @@ public ConsumableStreamPipesEntity(ConsumableStreamPipesEntity other) { if (other.getSupportedGrounding() != null) { this.supportedGrounding = new EventGrounding(other.getSupportedGrounding()); } - this.version = other.version; } public List getSpDataStreams() { @@ -90,14 +87,4 @@ public void setSupportedGrounding(EventGrounding supportedGrounding) { this.supportedGrounding = supportedGrounding; } - @Override - public int getVersion(){ - return this.version; - } - - @Override - public void setVersion(int version){ - this.version = version; - } - } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java index 925274e922..876152ce85 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java @@ -34,8 +34,8 @@ import java.util.List; public abstract class InvocableStreamPipesEntity - extends NamedStreamPipesEntity - implements EndpointSelectable, VersionedStreamPipesEntity { + extends VersionedNamedStreamPipesEntity + implements EndpointSelectable { protected List inputStreams; @@ -58,8 +58,6 @@ public abstract class InvocableStreamPipesEntity private boolean uncompleted; private String selectedEndpointUrl; - - private int version; protected SpServiceTagPrefix serviceTagPrefix; public InvocableStreamPipesEntity() { @@ -76,7 +74,6 @@ public InvocableStreamPipesEntity(InvocableStreamPipesEntity other) { this.correspondingUser = other.getCorrespondingUser(); this.selectedEndpointUrl = other.getSelectedEndpointUrl(); this.serviceTagPrefix = other.serviceTagPrefix; - this.version = other.getVersion(); if (other.getStreamRequirements() != null) { this.streamRequirements = new Cloner().streams(other.getStreamRequirements()); } @@ -210,14 +207,4 @@ public String getDetachPath() { public Logger getLogger(Class clazz) { return LoggerFactory.getPeLogger(clazz, getCorrespondingPipeline(), getUri()); } - - @Override - public int getVersion(){ - return this.version; - } - - @Override - public void setVersion(int version){ - this.version = version; - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedNamedStreamPipesEntity.java similarity index 52% rename from streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedStreamPipesEntity.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedNamedStreamPipesEntity.java index 5194340dc6..4d533eaf3e 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedNamedStreamPipesEntity.java @@ -15,11 +15,35 @@ * limitations under the License. * */ - package org.apache.streampipes.model.base; -public interface VersionedStreamPipesEntity { - int getVersion(); +public abstract class VersionedNamedStreamPipesEntity extends NamedStreamPipesEntity { + + private int version; + + public VersionedNamedStreamPipesEntity() { + super(); + this.version = 0; + } + + public VersionedNamedStreamPipesEntity(VersionedNamedStreamPipesEntity other) { + super(other); + this.version = other.version; + } + + public VersionedNamedStreamPipesEntity(String elementId, String name, String description){ + super(elementId, name, description); + } + + public VersionedNamedStreamPipesEntity(String elementId, String name, String description, String iconUrl) { + super(elementId, name, description, iconUrl); + } + + public int getVersion(){ + return version; + } - void setVersion(int version); + public void setVersion(int version) { + this.version = version; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java index f78df953a2..e386a101f3 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java @@ -19,8 +19,7 @@ package org.apache.streampipes.model.connect.adapter; import org.apache.streampipes.model.SpDataStream; -import org.apache.streampipes.model.base.NamedStreamPipesEntity; -import org.apache.streampipes.model.base.VersionedStreamPipesEntity; +import org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity; import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; import org.apache.streampipes.model.connect.rules.schema.SchemaTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.stream.StreamTransformationRuleDescription; @@ -36,12 +35,11 @@ import java.util.List; @TsModel -public class AdapterDescription extends NamedStreamPipesEntity implements VersionedStreamPipesEntity { +public class AdapterDescription extends VersionedNamedStreamPipesEntity { protected SpDataStream dataStream; protected boolean running; - private int version; private EventGrounding eventGrounding; private String icon; @@ -76,12 +74,11 @@ public AdapterDescription() { this.dataStream = new SpDataStream(); } - public AdapterDescription(String elementId, String name, String description, int version) { + public AdapterDescription(String elementId, String name, String description) { super(elementId, name, description); this.rules = new ArrayList<>(); this.category = new ArrayList<>(); this.dataStream = new SpDataStream(); - this.version = version; } @@ -90,7 +87,6 @@ public AdapterDescription(AdapterDescription other) { this.config = new Cloner().staticProperties(other.getConfig()); this.rules = other.getRules(); this.icon = other.getIcon(); - this.version = other.getVersion(); this.category = new Cloner().epaTypes(other.getCategory()); this.createdAt = other.getCreatedAt(); this.selectedEndpointUrl = other.getSelectedEndpointUrl(); @@ -253,14 +249,4 @@ public boolean isRunning() { public void setRunning(boolean running) { this.running = running; } - - @Override - public int getVersion() { - return version; - } - - @Override - public void setVersion(int newVersion) { - this.version = newVersion; - } } From a63423a0033f79d5c54c2d5aa297fcf30ede45ad Mon Sep 17 00:00:00 2001 From: bossenti Date: Wed, 25 Oct 2023 15:43:53 +0200 Subject: [PATCH 31/54] refactor: remove deprecated generic type --- .../streampipes/connect/adapters/netio/NetioRestAdapter.java | 2 +- .../extensions/connectors/influx/shared/InfluxConfigs.java | 2 +- .../connectors/opcua/config/SpOpcUaConfigExtractor.java | 4 ++-- .../streampipes/pe/shared/config/mqtt/MqttConnectUtils.java | 4 ++-- .../streampipes/pe/shared/config/nats/NatsConfigUtils.java | 2 +- .../processor/csvmetadata/CsvMetadataEnrichmentProcessor.java | 2 +- .../streampipes/sdk/extractor/AbstractParameterExtractor.java | 2 +- .../standalone/runtime/StandalonePipelineElementRuntime.java | 2 +- .../streampipes/wrapper/params/PipelineElementParameters.java | 2 +- .../streampipes/wrapper/runtime/PipelineElementRuntime.java | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/netio/NetioRestAdapter.java b/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/netio/NetioRestAdapter.java index e4898fb45e..02b7ee85ce 100644 --- a/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/netio/NetioRestAdapter.java +++ b/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/netio/NetioRestAdapter.java @@ -113,7 +113,7 @@ public PollingSettings getPollingInterval() { * * @param extractor StaticPropertyExtractor */ - private void applyConfiguration(IParameterExtractor extractor) { + private void applyConfiguration(IParameterExtractor extractor) { this.ip = extractor.singleValueParameter(NETIO_IP, String.class); this.username = extractor.singleValueParameter(NETIO_USERNAME, String.class); diff --git a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/InfluxConfigs.java b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/InfluxConfigs.java index d3eee716e1..4c6e1d4cd5 100644 --- a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/InfluxConfigs.java +++ b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/InfluxConfigs.java @@ -64,7 +64,7 @@ public static void appendSharedInfluxConfig(AbstractConfigurablePipelineElementB ); } - public static InfluxConnectionSettings fromExtractor(IParameterExtractor extractor) { + public static InfluxConnectionSettings fromExtractor(IParameterExtractor extractor) { String protocol = extractor.selectedSingleValueInternalName(DATABASE_PROTOCOL, String.class); String hostname = extractor.singleValueParameter(DATABASE_HOST_KEY, String.class); Integer port = extractor.singleValueParameter(DATABASE_PORT_KEY, Integer.class); diff --git a/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/config/SpOpcUaConfigExtractor.java b/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/config/SpOpcUaConfigExtractor.java index 4f461c9774..2c3e3b1383 100644 --- a/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/config/SpOpcUaConfigExtractor.java +++ b/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/config/SpOpcUaConfigExtractor.java @@ -62,11 +62,11 @@ public static OpcUaAdapterConfig extractAdapterConfig(IStaticPropertyExtractor e return config; } - public static OpcUaConfig extractSinkConfig(IParameterExtractor extractor) { + public static OpcUaConfig extractSinkConfig(IParameterExtractor extractor) { return extractSharedConfig(extractor, new OpcUaConfig()); } - public static T extractSharedConfig(IParameterExtractor extractor, + public static T extractSharedConfig(IParameterExtractor extractor, T config) { String selectedAlternativeConnection = diff --git a/streampipes-extensions/streampipes-pipeline-elements-shared/src/main/java/org/apache/streampipes/pe/shared/config/mqtt/MqttConnectUtils.java b/streampipes-extensions/streampipes-pipeline-elements-shared/src/main/java/org/apache/streampipes/pe/shared/config/mqtt/MqttConnectUtils.java index 102384d2d4..ad6879f864 100644 --- a/streampipes-extensions/streampipes-pipeline-elements-shared/src/main/java/org/apache/streampipes/pe/shared/config/mqtt/MqttConnectUtils.java +++ b/streampipes-extensions/streampipes-pipeline-elements-shared/src/main/java/org/apache/streampipes/pe/shared/config/mqtt/MqttConnectUtils.java @@ -69,11 +69,11 @@ public static StaticPropertyAlternative getAlternativesTwo() { } - public static MqttConfig getMqttConfig(IParameterExtractor extractor) { + public static MqttConfig getMqttConfig(IParameterExtractor extractor) { return getMqttConfig(extractor, null); } - public static MqttConfig getMqttConfig(IParameterExtractor extractor, String topicInput) { + public static MqttConfig getMqttConfig(IParameterExtractor extractor, String topicInput) { MqttConfig mqttConfig; String brokerUrl = extractor.singleValueParameter(BROKER_URL, String.class); diff --git a/streampipes-extensions/streampipes-pipeline-elements-shared/src/main/java/org/apache/streampipes/pe/shared/config/nats/NatsConfigUtils.java b/streampipes-extensions/streampipes-pipeline-elements-shared/src/main/java/org/apache/streampipes/pe/shared/config/nats/NatsConfigUtils.java index 32560de1e8..6de6f7c0d7 100644 --- a/streampipes-extensions/streampipes-pipeline-elements-shared/src/main/java/org/apache/streampipes/pe/shared/config/nats/NatsConfigUtils.java +++ b/streampipes-extensions/streampipes-pipeline-elements-shared/src/main/java/org/apache/streampipes/pe/shared/config/nats/NatsConfigUtils.java @@ -39,7 +39,7 @@ public class NatsConfigUtils { public static final String CONNECTION_PROPERTIES_GROUP = "connection-group"; public static final String PROPERTIES_KEY = "properties"; - public static NatsConfig from(IParameterExtractor extractor) { + public static NatsConfig from(IParameterExtractor extractor) { String subject = extractor.singleValueParameter(SUBJECT_KEY, String.class); String natsUrls = extractor.singleValueParameter(URLS_KEY, String.class); String authentication = extractor.selectedAlternativeInternalId(ACCESS_MODE); diff --git a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java index 20372678eb..3bb61d2654 100644 --- a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java +++ b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java @@ -184,7 +184,7 @@ private List getColumnNames(String fileContents, List columnsToI .collect(Collectors.toList()); } - private String getFileContents(IParameterExtractor extractor) { + private String getFileContents(IParameterExtractor extractor) { String filename = extractor.selectedFilename(CSV_FILE_KEY); return getStreamPipesClientInstance().fileApi().getFileContentAsString(filename); } diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java index 3ec7ee6299..a86a1ad587 100644 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java +++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/extractor/AbstractParameterExtractor.java @@ -64,7 +64,7 @@ import java.util.stream.Collectors; public abstract class AbstractParameterExtractor - implements IParameterExtractor { + implements IParameterExtractor { protected T sepaElement; private TypeParser typeParser; diff --git a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/runtime/StandalonePipelineElementRuntime.java b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/runtime/StandalonePipelineElementRuntime.java index d35317cda5..a8e48596c0 100644 --- a/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/runtime/StandalonePipelineElementRuntime.java +++ b/streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/runtime/StandalonePipelineElementRuntime.java @@ -43,7 +43,7 @@ public abstract class StandalonePipelineElementRuntime< PeT extends IStreamPipesPipelineElement, IvT extends InvocableStreamPipesEntity, RcT extends RuntimeContext, - ExT extends IParameterExtractor, + ExT extends IParameterExtractor, PepT extends IPipelineElementParameters> extends PipelineElementRuntime implements RawDataProcessor { diff --git a/streampipes-wrapper/src/main/java/org/apache/streampipes/wrapper/params/PipelineElementParameters.java b/streampipes-wrapper/src/main/java/org/apache/streampipes/wrapper/params/PipelineElementParameters.java index 607b7f437b..6fcd1cfc11 100644 --- a/streampipes-wrapper/src/main/java/org/apache/streampipes/wrapper/params/PipelineElementParameters.java +++ b/streampipes-wrapper/src/main/java/org/apache/streampipes/wrapper/params/PipelineElementParameters.java @@ -31,7 +31,7 @@ import java.util.List; import java.util.Map; -public class PipelineElementParameters> +public class PipelineElementParameters implements IPipelineElementParameters { private final List inputStreamParams; diff --git a/streampipes-wrapper/src/main/java/org/apache/streampipes/wrapper/runtime/PipelineElementRuntime.java b/streampipes-wrapper/src/main/java/org/apache/streampipes/wrapper/runtime/PipelineElementRuntime.java index 84d66579d0..0d33e47457 100644 --- a/streampipes-wrapper/src/main/java/org/apache/streampipes/wrapper/runtime/PipelineElementRuntime.java +++ b/streampipes-wrapper/src/main/java/org/apache/streampipes/wrapper/runtime/PipelineElementRuntime.java @@ -32,7 +32,7 @@ public abstract class PipelineElementRuntime< PeT extends IStreamPipesPipelineElement, IvT extends InvocableStreamPipesEntity, RcT extends RuntimeContext, - ExT extends IParameterExtractor, + ExT extends IParameterExtractor, PepT extends IPipelineElementParameters> implements IStreamPipesRuntime { From 83089715f94f3b2f7e95c0b259f304c041fed4a8 Mon Sep 17 00:00:00 2001 From: bossenti Date: Wed, 25 Oct 2023 16:53:27 +0200 Subject: [PATCH 32/54] feat: implement migration for processing elements & data sinks --- .../api/migration/DataProcessorMigrator.java | 3 +- .../api/migration/ModelMigrator.java | 6 +- .../migration/AdapterMigrationResource.java | 89 ++---------- .../DataProcessorMigrationResource.java | 56 ++++++++ .../migration/DataSinkMigrationResource.java | 53 +++++++ .../migration/MigrateExtensionsResource.java | 103 +++++++++++++- .../MigrateExtensionsResourceTest.java | 131 ++++++++++++++++++ .../rest/impl/admin/MigrationResource.java | 88 +++++++----- .../extensions/ExtensionsResourceConfig.java | 4 + 9 files changed, 416 insertions(+), 117 deletions(-) create mode 100644 streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataProcessorMigrationResource.java create mode 100644 streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java create mode 100644 streampipes-rest-extensions/src/test/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResourceTest.java diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java index 003bf4539c..5f6faffb27 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/DataProcessorMigrator.java @@ -22,7 +22,7 @@ import org.apache.streampipes.model.graph.DataProcessorInvocation; public abstract class DataProcessorMigrator - implements ModelMigrator { + implements ModelMigrator { @Override public boolean equals(Object obj) { @@ -31,5 +31,4 @@ public boolean equals(Object obj) { } return false; } - } diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java index e1b6731ab2..cf492cd67d 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/migration/ModelMigrator.java @@ -19,13 +19,13 @@ package org.apache.streampipes.extensions.api.migration; import org.apache.streampipes.extensions.api.extractor.IParameterExtractor; -import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity; import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.migration.ModelMigratorConfig; public interface ModelMigrator< - T extends NamedStreamPipesEntity, - ExT extends IParameterExtractor + T extends VersionedNamedStreamPipesEntity, + ExT extends IParameterExtractor > extends Comparable { ModelMigratorConfig config(); diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java index 89cddbd17f..f83d2bda3c 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java @@ -18,17 +18,14 @@ package org.apache.streampipes.rest.extensions.migration; +import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor; import org.apache.streampipes.extensions.api.migration.AdapterMigrator; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.extensions.migration.MigrationRequest; -import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.rest.security.AuthConstants; import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; import org.apache.streampipes.sdk.extractor.StaticPropertyExtractor; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.security.access.prepost.PreAuthorize; import jakarta.ws.rs.Consumes; @@ -39,83 +36,21 @@ @Path("/api/v1/migrations/adapter") -public class AdapterMigrationResource extends MigrateExtensionsResource { - - private static final Logger LOG = LoggerFactory.getLogger(AdapterMigrationResource.class); - - /** - * Migrates an adapter instance based on the provided {@link MigrationRequest}. - * The outcome of the migration is described in {@link MigrationResult}. - * The result is always part of the response. - * Independent, of the migration outcome, the returned response always has OK as status code. - * It is the responsibility of the recipient to interpret the migration result and act accordingly. - * @param adapterMigrateRequest Request that contains both the adapter to be migrated and the migration config. - * @return A response with status code ok, that contains a migration result reflecting the outcome of the operation. - */ +public class AdapterMigrationResource extends MigrateExtensionsResource< + AdapterDescription, + IStaticPropertyExtractor, + AdapterMigrator + > { @POST @Consumes(MediaType.APPLICATION_JSON) @JacksonSerialized @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response migrateAdapter(MigrationRequest adapterMigrateRequest) { - - var adapterDescription = adapterMigrateRequest.migrationElement(); - var migrationConfig = adapterMigrateRequest.modelMigratorConfig(); - - LOG.info("Received migration request for adapter '{}' to migrate from version {} to {}", - adapterDescription.getElementId(), - migrationConfig.fromVersion(), - migrationConfig.toVersion() - ); - - var migratorOptional = getMigrator(migrationConfig); - - if (migratorOptional.isPresent()) { - LOG.info("Migrator found for request, starting migration..."); - var migrator = migratorOptional.get(); - - var extractor = StaticPropertyExtractor.from(adapterDescription.getConfig()); - - try { - var result = migrator.migrate(adapterDescription, extractor); - - if (result.success()) { - LOG.info("Migration successfully finished."); - - // Since adapter migration was successful, version can be adapted to the target version. - // this step is explicitly performed here and not left to the migration itself to - // prevent leaving this step out - AdapterDescription migratedAdapterDescription = result.element(); - migratedAdapterDescription.setVersion(migrationConfig.toVersion()); - return ok(MigrationResult.success(migratedAdapterDescription)); - - } else { - LOG.error("Migration failed with the following reason: {}", result.message()); - // The failed migration is documented in the MigrationResult - // The core is expected to handle the response accordingly, so we can safely return a positive status code - return ok(result); - } + public Response migrateAdapter(MigrationRequest adapterMigrationRequest) { + return ok(handleMigration(adapterMigrationRequest)); + } - } catch (RuntimeException e) { - LOG.error("An unexpected exception caused the migration to fail - " - + "sending exception report in migration result"); - return ok(MigrationResult.failure( - adapterDescription, - String.format( - "Adapter Migration failed due to an unexpected exception: %s", - StringUtils.join(e.getStackTrace(), "\n") - ) - ) - ); - } - } - LOG.error("Migrator for migration config {} could not be found. Migration is cancelled.", migrationConfig); - return ok(MigrationResult.failure( - adapterDescription, - String.format( - "The given migration config '%s' could not be mapped to a registered migrator.", - migrationConfig - ) - ) - ); + @Override + protected IStaticPropertyExtractor getPropertyExtractor(AdapterDescription pipelineElementDescription) { + return StaticPropertyExtractor.from(pipelineElementDescription.getConfig()); } } diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataProcessorMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataProcessorMigrationResource.java new file mode 100644 index 0000000000..84fe627d8c --- /dev/null +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataProcessorMigrationResource.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.extensions.migration; + +import org.apache.streampipes.extensions.api.extractor.IDataProcessorParameterExtractor; +import org.apache.streampipes.extensions.api.migration.DataProcessorMigrator; +import org.apache.streampipes.model.extensions.migration.MigrationRequest; +import org.apache.streampipes.model.graph.DataProcessorInvocation; +import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; +import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor; + +import org.springframework.security.access.prepost.PreAuthorize; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +@Path("api/v1/migrations/processor") +public class DataProcessorMigrationResource extends MigrateExtensionsResource< + DataProcessorInvocation, + IDataProcessorParameterExtractor, + DataProcessorMigrator + > { + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @JacksonSerialized + @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) + public Response migrateDataProcessor(MigrationRequest processorMigrationRequest) { + return ok(handleMigration(processorMigrationRequest)); + } + + @Override + protected IDataProcessorParameterExtractor getPropertyExtractor(DataProcessorInvocation pipelineElementDescription) { + return ProcessingElementParameterExtractor.from(pipelineElementDescription); + } +} diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java new file mode 100644 index 0000000000..6fdde09805 --- /dev/null +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.extensions.migration; + +import org.apache.streampipes.extensions.api.extractor.IDataSinkParameterExtractor; +import org.apache.streampipes.extensions.api.migration.DataSinkMigrator; +import org.apache.streampipes.model.extensions.migration.MigrationRequest; +import org.apache.streampipes.model.graph.DataSinkInvocation; +import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; +import org.apache.streampipes.sdk.extractor.DataSinkParameterExtractor; + +import org.springframework.security.access.prepost.PreAuthorize; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +public class DataSinkMigrationResource extends MigrateExtensionsResource< + DataSinkInvocation, + IDataSinkParameterExtractor, + DataSinkMigrator + > { + @POST + @Consumes(MediaType.APPLICATION_JSON) + @JacksonSerialized + @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) + public Response migrateDataSink(MigrationRequest sinkMigrationRequest) { + return ok(handleMigration(sinkMigrationRequest)); + } + + @Override + protected IDataSinkParameterExtractor getPropertyExtractor(DataSinkInvocation pipelineElementDescription) { + return DataSinkParameterExtractor.from(pipelineElementDescription); + } +} diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResource.java index d437d1e55a..66788e002e 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResource.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResource.java @@ -18,27 +18,124 @@ package org.apache.streampipes.rest.extensions.migration; +import org.apache.streampipes.extensions.api.extractor.IParameterExtractor; import org.apache.streampipes.extensions.api.migration.ModelMigrator; import org.apache.streampipes.extensions.management.init.DeclarersSingleton; +import org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity; +import org.apache.streampipes.model.extensions.migration.MigrationRequest; +import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.migration.ModelMigratorConfig; import org.apache.streampipes.rest.extensions.AbstractExtensionsResource; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.Optional; -public class MigrateExtensionsResource extends AbstractExtensionsResource { +public abstract class MigrateExtensionsResource< + T extends VersionedNamedStreamPipesEntity, + ExT extends IParameterExtractor, + MmT extends ModelMigrator> extends AbstractExtensionsResource { + + private static final Logger LOG = LoggerFactory.getLogger(MigrateExtensionsResource.class); /** * Find and return the corresponding {@link ModelMigrator} instance within the registered migrators. * This allows to pass the corresponding model migrator to a {@link ModelMigratorConfig} which is exchanged * between Core and Extensions service. + * * @param modelMigratorConfig config that describes the model migrator to be returned * @return Optional model migrator which is empty in case no appropriate migrator is found among the registered. */ - public Optional getMigrator(ModelMigratorConfig modelMigratorConfig) { + public Optional getMigrator(ModelMigratorConfig modelMigratorConfig) { return DeclarersSingleton.getInstance().getServiceDefinition().getMigrators() .stream() .filter(modelMigrator -> modelMigrator.config().equals(modelMigratorConfig)) - .map(modelMigrator -> (T) modelMigrator) + .map(modelMigrator -> (MmT) modelMigrator) .findFirst(); } + + /** + * Migrates a pipeline element instance based on the provided {@link MigrationRequest}. + * The outcome of the migration is described in {@link MigrationResult}. + * The result is always part of the response. + * Independent, of the migration outcome, the returned response always has OK as status code. + * It is the responsibility of the recipient to interpret the migration result and act accordingly. + * @param migrationRequest Request that contains both the pipeline element to be migrated and the migration config. + * @return A response with status code ok, that contains a migration result reflecting the outcome of the operation. + */ + protected MigrationResult handleMigration(MigrationRequest migrationRequest) { + + var pipelineElementDescription = migrationRequest.migrationElement(); + var migrationConfig = migrationRequest.modelMigratorConfig(); + + LOG.info("Received migration request for pipeline element '{}' to migrate from version {} to {}", + pipelineElementDescription.getElementId(), + migrationConfig.fromVersion(), + migrationConfig.toVersion() + ); + + var migratorOptional = getMigrator(migrationConfig); + + if (migratorOptional.isPresent()) { + LOG.info("Migrator found for request, starting migration..."); + return executeMigration(migratorOptional.get(), pipelineElementDescription); + } + LOG.error("Migrator for migration config {} could not be found. Migration is cancelled.", migrationConfig); + return MigrationResult.failure( + pipelineElementDescription, + String.format( + "The given migration config '%s' could not be mapped to a registered migrator.", + migrationConfig + ) + ); + } + + /** + * Executes the migration for the given pipeline element based on the given migrator. + * @param migrator migrator that executes the migration + * @param pipelineElementDescription pipeline element to be migrated + * @return the migration result containing either the migrated element or the original one in case of a failure + */ + protected MigrationResult executeMigration( + MmT migrator, + T pipelineElementDescription + ) { + + var extractor = getPropertyExtractor(pipelineElementDescription); + + try { + var result = migrator.migrate(pipelineElementDescription, extractor); + + if (result.success()) { + LOG.info("Migration successfully finished."); + + // Since adapter migration was successful, version can be adapted to the target version. + // this step is explicitly performed here and not left to the migration itself to + // prevent leaving this step out + var migratedProcessor = result.element(); + migratedProcessor.setVersion(migrator.config().toVersion()); + return MigrationResult.success(migratedProcessor); + + } else { + LOG.error("Migration failed with the following reason: {}", result.message()); + // The failed migration is documented in the MigrationResult + // The core is expected to handle the response accordingly, so we can safely return a positive status code + return result; + } + } catch (RuntimeException e) { + LOG.error("An unexpected exception caused the migration to fail - " + + "sending exception report in migration result"); + return MigrationResult.failure( + pipelineElementDescription, + String.format( + "Migration failed due to an unexpected exception: %s", + StringUtils.join(e.getStackTrace(), "\n") + ) + ); + } + } + + protected abstract ExT getPropertyExtractor(T pipelineElementDescription); } diff --git a/streampipes-rest-extensions/src/test/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResourceTest.java b/streampipes-rest-extensions/src/test/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResourceTest.java new file mode 100644 index 0000000000..98a2ea33a9 --- /dev/null +++ b/streampipes-rest-extensions/src/test/java/org/apache/streampipes/rest/extensions/migration/MigrateExtensionsResourceTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.rest.extensions.migration; + +import org.apache.streampipes.extensions.api.extractor.IDataProcessorParameterExtractor; +import org.apache.streampipes.extensions.api.migration.DataProcessorMigrator; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.model.graph.DataProcessorInvocation; +import org.apache.streampipes.model.migration.MigrationResult; +import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.apache.streampipes.sdk.StaticProperties; +import org.apache.streampipes.sdk.helpers.Labels; +import org.apache.streampipes.sdk.utils.Datatypes; + +import org.junit.Test; + +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MigrateExtensionsResourceTest { + + @Test + public void executeMigration() { + var migrationsResource = new DataProcessorMigrationResource(); + + var migrator = new DataProcessorMigrator() { + @Override + public ModelMigratorConfig config() { + return new ModelMigratorConfig("app-id", SpServiceTagPrefix.DATA_PROCESSOR, 0, 1); + } + + @Override + public MigrationResult migrate( + DataProcessorInvocation element, + IDataProcessorParameterExtractor extractor + ) throws RuntimeException { + var properties = element.getStaticProperties(); + properties.add( + StaticProperties.freeTextProperty(Labels.empty(), Datatypes.String) + ); + element.setStaticProperties(properties); + return MigrationResult.success(element); + } + }; + + var dataProcessor = new DataProcessorInvocation(); + dataProcessor.setStaticProperties(new ArrayList<>()); + + var result = migrationsResource.executeMigration(migrator, dataProcessor); + + assertTrue(result.success()); + assertEquals("SUCCESS", result.message()); + assertEquals(1, result.element().getVersion()); + assertEquals(1, result.element().getStaticProperties().size()); + + } + + @Test + public void executeMigrationWithFailure() { + var migrationsResource = new DataProcessorMigrationResource(); + + var migrator = new DataProcessorMigrator() { + @Override + public ModelMigratorConfig config() { + return new ModelMigratorConfig("app-id", SpServiceTagPrefix.DATA_PROCESSOR, 0, 1); + } + + @Override + public MigrationResult migrate( + DataProcessorInvocation element, + IDataProcessorParameterExtractor extractor + ) throws RuntimeException { + return MigrationResult.failure(element, "This should fail"); + } + }; + + var dataProcessor = new DataProcessorInvocation(); + + var result = migrationsResource.executeMigration(migrator, dataProcessor); + + assertFalse(result.success()); + assertEquals("This should fail", result.message()); + assertEquals(0, result.element().getVersion()); + } + + @Test + public void executeMigrationWithUnknownFailure() { + var migrationsResource = new DataProcessorMigrationResource(); + + var migrator = new DataProcessorMigrator() { + @Override + public ModelMigratorConfig config() { + return new ModelMigratorConfig("app-id", SpServiceTagPrefix.DATA_PROCESSOR, 0, 1); + } + + @Override + public MigrationResult migrate( + DataProcessorInvocation element, + IDataProcessorParameterExtractor extractor + ) throws RuntimeException { + throw new NullPointerException(); + } + }; + + var dataProcessor = new DataProcessorInvocation(); + + var result = migrationsResource.executeMigration(migrator, dataProcessor); + + assertFalse(result.success()); + assertTrue(result.message().startsWith("Migration failed due to an unexpected exception:")); + assertEquals(0, result.element().getVersion()); + } +} diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index 310ac56113..9c59b42da7 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -23,7 +23,7 @@ import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; import org.apache.streampipes.manager.execution.PipelineExecutor; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; -import org.apache.streampipes.model.base.VersionedStreamPipesEntity; +import org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity; import org.apache.streampipes.model.extensions.migration.MigrationRequest; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.graph.DataProcessorInvocation; @@ -69,9 +69,7 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { private static final Logger LOG = LoggerFactory.getLogger(MigrationResource.class); - - private static final String ADAPTER_MIGRATION_ENDPOINT = "api/v1/migrations/adapter"; - private static final String PIPELINE_ELEMENT_MIGRATION_ENDPOINT = "api/v1/migrations/pipeline-elements"; + private static final String MIGRATION_ENDPOINT = "api/v1/migrations"; private final CRUDStorage extensionsServiceStorage = getNoSqlStorage().getExtensionsServiceStorage(); @@ -118,8 +116,11 @@ public Response registerAdapterMigrations( var migrationResult = performMigration( adapterDescription, migrationConfig, - extensionsServiceConfig.getServiceUrl(), - ADAPTER_MIGRATION_ENDPOINT); + String.format("%s/%s/adapter", + extensionsServiceConfig.getServiceUrl(), + MIGRATION_ENDPOINT + ) + ); if (migrationResult.success()) { LOG.info("Migration successfully performed by extensions service. Updating adapter description ..."); @@ -145,7 +146,7 @@ public Response registerAdapterMigrations( } else { LOG.info( "Migration is not applicable for adapter '{}' because of a version mismatch - " - + "adapter version: '{}', migration starts at: '{}'", + + "adapter version: '{}', migration starts at: '{}'", adapterDescription.getElementId(), adapterVersion, migrationConfig.fromVersion() @@ -184,7 +185,10 @@ public Response registerPipelineElementMigrations( return migratePipelineElement( processor, migrationConfigs, - extensionsServiceConfig.getServiceUrl(), + String.format("%s/%s/processor", + extensionsServiceConfig.getServiceUrl(), + MIGRATION_ENDPOINT + ), failedMigrations ); } else { @@ -202,7 +206,10 @@ public Response registerPipelineElementMigrations( return migratePipelineElement( sink, migrationConfigs, - extensionsServiceConfig.getServiceUrl(), + String.format("%s/%s/sink", + extensionsServiceConfig.getServiceUrl(), + MIGRATION_ENDPOINT + ), failedMigrations ); } else { @@ -213,13 +220,15 @@ public Response registerPipelineElementMigrations( ).toList(); pipeline.setActions(migratedDataSinks); + pipelineStorage.updatePipeline(pipeline); + if (failedMigrations.isEmpty()) { LOG.info("Migration for pipeline successfully completed."); } else { - handleFailedMigrations(pipeline, failedMigrations); + // pass most recent version of pipeline + handleFailedMigrations(pipelineStorage.getPipeline(pipeline.getPipelineId()), failedMigrations); } } - return ok(); } @@ -236,13 +245,25 @@ public Response registerPipelineElementMigrations( * @param pipeline the pipeline affected by failed migrations * @param failedMigrations the list of failed migrations */ - protected static void handleFailedMigrations(Pipeline pipeline, List> failedMigrations) { + protected void handleFailedMigrations(Pipeline pipeline, List> failedMigrations) { LOG.error("Failures in migration detected - The following pipeline elements could to be migrated:\n" - + StringUtils.join(failedMigrations.stream().map(Record::toString)), "\n"); + + StringUtils.join(failedMigrations.stream().map(Record::toString).toList()), "\n"); - pipeline.setPipelineNotifications(failedMigrations.stream().map(MigrationResult::message).toList()); + pipeline.setPipelineNotifications(failedMigrations.stream().map( + failedMigration -> "Failed migration of pipeline element: %s".formatted(failedMigration.message()) + ).toList()); pipeline.setHealthStatus(PipelineHealthStatus.REQUIRES_ATTENTION); + pipelineStorage.updatePipeline(pipeline); + + // get updated version of pipeline after modification + pipeline = pipelineStorage.getPipeline(pipeline.getPipelineId()); + + stopPipeline(pipeline); + } + + + public void stopPipeline(Pipeline pipeline) { var pipelineExecutor = new PipelineExecutor(pipeline, true); var pipelineStopResult = pipelineExecutor.stopPipeline(); @@ -281,24 +302,25 @@ protected static Optional getApplicableMigration( * All applicable migrations found in the provided configs are executed for the given pipeline element. * In case a migration fails, the related pipeline element receives the latest definition of its static properties, * so that the pipeline element can be adapted by the user to resolve the failed migration. - * @param pipelineElement pipeline element to be migrated - * @param modelMigrations list of model migrations that might be applicable for this pipeline element - * @param serviceUrl url of the extensions service that handles the migration + * + * @param pipelineElement pipeline element to be migrated + * @param modelMigrations list of model migrations that might be applicable for this pipeline element + * @param url url of the extensions service endpoint that handles the migration * @param failedMigrations collection of failed migrations which is extended by occurring migration failures - * @param type of the pipeline element (e.g., DataProcessorInvocation) + * @param type of the pipeline element (e.g., DataProcessorInvocation) * @return the migrated (or - in case of a failure - updated) pipeline element */ protected T migratePipelineElement( T pipelineElement, List modelMigrations, - String serviceUrl, + String url, List> failedMigrations ) { // loop until no migrations are available anymore // this allows to apply multiple migrations for a pipeline element sequentially // For example, first migration from 0 to 1 and the second migration from 1 to 2 - while (getApplicableMigration(pipelineElement, modelMigrations).isPresent()) { + while (getApplicableMigration(pipelineElement, modelMigrations).isPresent() && failedMigrations.isEmpty()) { var migrationConfig = getApplicableMigration(pipelineElement, modelMigrations).get(); LOG.info( @@ -310,38 +332,40 @@ protected T migratePipelineElement( var migrationResult = performMigration( pipelineElement, migrationConfig, - serviceUrl, - PIPELINE_ELEMENT_MIGRATION_ENDPOINT + url ); if (migrationResult.success()) { LOG.info("Migration successfully performed by extensions service. Updating pipeline element invocation ..."); - LOG.debug("Migration was performed by extensions service '{}'", serviceUrl); - return migrationResult.element(); + LOG.debug("Migration was performed at extensions service endpoint '{}'", url); + pipelineElement = migrationResult.element(); } else { LOG.error("Migration failed with the following reason: {}", migrationResult.message()); failedMigrations.add(migrationResult); } } - updateFailedPipelineElement(pipelineElement); + if (!failedMigrations.isEmpty()) { + updateFailedPipelineElement(pipelineElement); + LOG.info("Updated pipeline elements with new description where automatic migration failed."); + } return pipelineElement; } /** * Performs the actual migration of a pipeline element. * This includes the communication with the extensions service which runs the migration. + * * @param pipelineElement pipeline element to be migrated * @param migrationConfig config of the migration to be performed - * @param serviceUrl url of the extensions service where the migration should be performed - * @param migrationEndpoint endpoint at which the migration should be performed - * @param type of the processing element + * @param url url of the migration endpoint at the extensions service + * where the migration should be performed + * @param type of the processing element * @return result of the migration */ - protected MigrationResult performMigration( + protected MigrationResult performMigration( T pipelineElement, ModelMigratorConfig migrationConfig, - String serviceUrl, - String migrationEndpoint + String url ) { try { @@ -351,7 +375,7 @@ protected MigrationResult performMigra String serializedRequest = JacksonSerializer.getObjectMapper().writeValueAsString(migrationRequest); var migrationResponse = ExtensionServiceExecutions.extServicePostRequest( - "%s/%s".formatted(serviceUrl, migrationEndpoint), + url, serializedRequest ).execute(); diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java index ff785a3e40..574d2d8e30 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsResourceConfig.java @@ -26,6 +26,8 @@ import org.apache.streampipes.rest.extensions.connect.HttpServerAdapterResource; import org.apache.streampipes.rest.extensions.connect.RuntimeResolvableResource; import org.apache.streampipes.rest.extensions.migration.AdapterMigrationResource; +import org.apache.streampipes.rest.extensions.migration.DataProcessorMigrationResource; +import org.apache.streampipes.rest.extensions.migration.DataSinkMigrationResource; import org.apache.streampipes.rest.extensions.monitoring.MonitoringResource; import org.apache.streampipes.rest.extensions.pe.DataProcessorPipelineElementResource; import org.apache.streampipes.rest.extensions.pe.DataSinkPipelineElementResource; @@ -51,7 +53,9 @@ public Set> getClassesToRegister() { AdapterDescriptionResource.class, AdapterMigrationResource.class, AdapterWorkerResource.class, + DataProcessorMigrationResource.class, DataProcessorPipelineElementResource.class, + DataSinkMigrationResource.class, DataSinkPipelineElementResource.class, DataStreamPipelineElementResource.class, GuessResource.class, From 796be9a728313ef08af1126bf65141abcf28170d Mon Sep 17 00:00:00 2001 From: bossenti Date: Thu, 26 Oct 2023 09:57:28 +0200 Subject: [PATCH 33/54] docs: add endpoint documentation --- .../migration/AdapterMigrationResource.java | 33 ++++++++++++++- .../DataProcessorMigrationResource.java | 33 ++++++++++++++- .../migration/DataSinkMigrationResource.java | 33 ++++++++++++++- .../rest/impl/admin/MigrationResource.java | 41 +++++++++++++++++++ 4 files changed, 137 insertions(+), 3 deletions(-) diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java index f83d2bda3c..2a79c1eec4 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java @@ -26,6 +26,12 @@ import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; import org.apache.streampipes.sdk.extractor.StaticPropertyExtractor; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import org.apache.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; import jakarta.ws.rs.Consumes; @@ -45,7 +51,32 @@ public class AdapterMigrationResource extends MigrateExtensionsResource< @Consumes(MediaType.APPLICATION_JSON) @JacksonSerialized @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response migrateAdapter(MigrationRequest adapterMigrationRequest) { + @Operation( + summary = "Execute the migration for a specific adapter instance", tags = {"Extensions", "Migration"}, + responses = { + @ApiResponse( + responseCode = "" + HttpStatus.SC_OK, + description = "The migration was executed. It's result is described in the response. " + + "The Response needs to be handled accordingly.", + content = @Content( + examples = @ExampleObject( + name = "Successful migration", + value = "{\"success\": true,\"messages\": \"SUCCESS\", \"element\": {}}" + ), + mediaType = MediaType.APPLICATION_JSON + ) + ) + } + ) + public Response migrateAdapter( + @Parameter( + description = "request that encompasses the adapter description(AdapterDescription) and " + + "the configuration of the migration", + example = "{\"migrationElement\": {}, \"modelMigratorConfig\": {\"targetAppId\": \"app-id\"," + + "\"modelType\": \"adapter\", \"fromVersion\": 0, \"toVersion\": 1}}", + required = true + ) + MigrationRequest adapterMigrationRequest) { return ok(handleMigration(adapterMigrationRequest)); } diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataProcessorMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataProcessorMigrationResource.java index 84fe627d8c..33754180c2 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataProcessorMigrationResource.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataProcessorMigrationResource.java @@ -26,6 +26,12 @@ import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; import org.apache.streampipes.sdk.extractor.ProcessingElementParameterExtractor; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import org.apache.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; import jakarta.ws.rs.Consumes; @@ -45,7 +51,32 @@ public class DataProcessorMigrationResource extends MigrateExtensionsResource< @Consumes(MediaType.APPLICATION_JSON) @JacksonSerialized @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response migrateDataProcessor(MigrationRequest processorMigrationRequest) { + @Operation( + summary = "Execute the migration for a specific data processor instance", tags = {"Extensions", "Migration"}, + responses = { + @ApiResponse( + responseCode = "" + HttpStatus.SC_OK, + description = "The migration was executed. It's result is described in the response. " + + "The Response needs to be handled accordingly.", + content = @Content( + examples = @ExampleObject( + name = "Successful migration", + value = "{\"success\": true,\"messages\": \"SUCCESS\", \"element\": {}}" + ), + mediaType = MediaType.APPLICATION_JSON + ) + ) + } + ) + public Response migrateDataProcessor( + @Parameter( + description = "Request that encompasses the data processor description (DataProcessorInvocation) and " + + "the configuration of the migration to be performed", + example = "{\"migrationElement\": {}, \"modelMigratorConfig\": {\"targetAppId\": \"app-id\", " + + "\"modelType\": \"dprocessor\", \"fromVersion\": 0, \"toVersion\": 1}}", + required = true + ) + MigrationRequest processorMigrationRequest) { return ok(handleMigration(processorMigrationRequest)); } diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java index 6fdde09805..98569ffae7 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java @@ -26,6 +26,12 @@ import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; import org.apache.streampipes.sdk.extractor.DataSinkParameterExtractor; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import org.apache.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; import jakarta.ws.rs.Consumes; @@ -42,7 +48,32 @@ public class DataSinkMigrationResource extends MigrateExtensionsResource< @Consumes(MediaType.APPLICATION_JSON) @JacksonSerialized @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response migrateDataSink(MigrationRequest sinkMigrationRequest) { + @Operation( + summary = "Execute the migration for a specific data sink instance", tags = {"Extensions", "Migration"}, + responses = { + @ApiResponse( + responseCode = "" + HttpStatus.SC_OK, + description = "The migration was executed. It's result is described in the response. " + + "The Response needs to be handled accordingly.", + content = @Content( + examples = @ExampleObject( + name = "Successful migration", + value = "{\"success\": true,\"messages\": \"SUCCESS\", \"element\": {}}" + ), + mediaType = MediaType.APPLICATION_JSON + ) + ) + } + ) + public Response migrateDataSink( + @Parameter( + description = "Request that encompasses the data sink description (DataSinkInvocation) and " + + "the configuration of the migration to be performed", + example = "{\"migrationElement\": {}, \"modelMigratorConfig\": {\"targetAppId\": \"app-id\", " + + "\"modelType\": \"dsink\", \"fromVersion\": 0, \"toVersion\": 1}}", + required = true + ) + MigrationRequest sinkMigrationRequest) { return ok(handleMigration(sinkMigrationRequest)); } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index 9c59b42da7..5923601263 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -44,7 +44,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.responses.ApiResponse; import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.security.access.prepost.PreAuthorize; @@ -88,8 +93,26 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { @POST @Path("adapter/{serviceId}") @Consumes(MediaType.APPLICATION_JSON) + @Operation( + summary = "Migrate adapters based on migration configs", tags = {"Core", "Migration"}, + responses = { + @ApiResponse( + responseCode = "" + HttpStatus.SC_OK, + description = "All provided migrations are handled. If an error appeared, " + + "the corresponding actions are taken.") + } + ) public Response registerAdapterMigrations( + @Parameter( + in = ParameterIn.PATH, + description = "the id of the extensions service that requests migrations", + required = true + ) @PathParam("serviceId") String serviceId, + @Parameter( + description = "list of configs (ModelMigratorConfig) that describe the requested migrations", + required = true + ) List migrationConfigs) { var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); @@ -160,8 +183,26 @@ public Response registerAdapterMigrations( @POST @Path("pipeline-element/{serviceId}") @Consumes(MediaType.APPLICATION_JSON) + @Operation( + summary = "Migrate pipeline elements based on migration configs", tags = {"Core", "Migration"}, + responses = { + @ApiResponse( + responseCode = "200" + HttpStatus.SC_OK, + description = "All provided migrations are handled. " + + "If an error appeared, the corresponding actions are taken." + ) + } + ) public Response registerPipelineElementMigrations( + @Parameter( + in = ParameterIn.PATH, + description = "the id of the extensions service that requests migrations", + required = true + ) @PathParam("serviceId") String serviceId, + @Parameter( + description = "list of config that describe the requested migrations" + ) List migrationConfigs) { var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); From 2658ae0a2aafd60006b8387dbff6c8c14fbb9d69 Mon Sep 17 00:00:00 2001 From: bossenti Date: Thu, 26 Oct 2023 14:45:20 +0200 Subject: [PATCH 34/54] refactor: move to correct module --- .../org/apache/streampipes/model/migration/MigrationResult.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {streampipes-extensions-api => streampipes-model}/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java (100%) diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java b/streampipes-model/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java similarity index 100% rename from streampipes-extensions-api/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/migration/MigrationResult.java From 71297932feecfc43a2394dd2f81bc906ea87fe29 Mon Sep 17 00:00:00 2001 From: bossenti Date: Thu, 26 Oct 2023 14:50:12 +0200 Subject: [PATCH 35/54] feature: add update for descriptions --- .../rest/impl/admin/MigrationResource.java | 97 ++++++++++++++++++- 1 file changed, 92 insertions(+), 5 deletions(-) diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index 5923601263..3fec34f38f 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -18,16 +18,20 @@ package org.apache.streampipes.rest.impl.admin; +import org.apache.streampipes.commons.exceptions.SepaParseException; import org.apache.streampipes.commons.exceptions.connect.AdapterException; import org.apache.streampipes.connect.management.management.WorkerRestClient; import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; import org.apache.streampipes.manager.execution.PipelineExecutor; +import org.apache.streampipes.manager.operations.Operations; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity; import org.apache.streampipes.model.extensions.migration.MigrationRequest; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.graph.DataProcessorInvocation; import org.apache.streampipes.model.graph.DataSinkInvocation; +import org.apache.streampipes.model.message.Notification; import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.migration.ModelMigratorConfig; import org.apache.streampipes.model.pipeline.Pipeline; @@ -41,6 +45,7 @@ import org.apache.streampipes.storage.api.IDataProcessorStorage; import org.apache.streampipes.storage.api.IDataSinkStorage; import org.apache.streampipes.storage.api.IPipelineStorage; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -63,10 +68,12 @@ import jakarta.ws.rs.core.Response; import java.io.IOException; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Optional; - +import java.util.function.Function; +import java.util.stream.Collectors; @Path("v2/migrations") @Component @@ -86,10 +93,6 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { private final IDataSinkStorage dataSinkStorage = getNoSqlStorage().getDataSinkStorage(); private final IPipelineStorage pipelineStorage = getNoSqlStorage().getPipelineStorageAPI(); - - // TODO: override all existing descriptions(adapter, pipeline elements) - // Use functionality of update button in UI - // increase version as well @POST @Path("adapter/{serviceId}") @Consumes(MediaType.APPLICATION_JSON) @@ -120,6 +123,10 @@ public Response registerAdapterMigrations( LOG.info("Received {} migrations from extension service {}.", migrationConfigs.size(), extensionsServiceConfig.getServiceUrl()); + LOG.info("Updating adapter descriptions by replacement..."); + updateDescriptions(migrationConfigs, extensionsServiceConfig.getServiceUrl()); + LOG.info("Adapter descriptions are up to date."); + LOG.info("Checking migrations for existing adapters in StreamPipes Core ..."); for (var migrationConfig : migrationConfigs) { LOG.info("Searching for assets of '{}'", migrationConfig.targetAppId()); @@ -206,6 +213,10 @@ public Response registerPipelineElementMigrations( List migrationConfigs) { var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); + LOG.info("Updating pipeline element descriptions by replacement..."); + updateDescriptions(migrationConfigs, extensionsServiceConfig.getServiceUrl()); + LOG.info("Pipeline element descriptions are up to date."); + LOG.info("Received {} pipeline element migrations from extension service {}.", migrationConfigs.size(), extensionsServiceConfig.getServiceUrl()); @@ -440,6 +451,82 @@ protected MigrationResult perform return MigrationResult.failure(pipelineElement, "Internal error during migration at StreamPipes Core"); } + /** + * Update all descriptions of entities in the Core that are affected by migrations. + * + * @param migrationConfigs List of migrations to take in account + * @param serviceUrl Url of the extension service that provides the migrations. + */ + protected void updateDescriptions(List migrationConfigs, String serviceUrl) { + migrationConfigs + .stream() + .collect( + // We only need to update the description once per appId, + // because this is directly done with the newest version of the description and + // there is iterative migration required. + // To avoid unnecessary, multiple updates, + // we filter the migration configs such that every appId is unique. + // This ensures that every description is only updated once. + Collectors.toMap( + ModelMigratorConfig::targetAppId, + Function.identity(), + (existing, replacement) -> existing + ) + ) + .values() + .stream() + .peek(config -> { + var requestUrl = getRequestUrl(config.modelType(), config.targetAppId(), serviceUrl); + performUpdate(requestUrl); + } + ); + } + + /** + * Get the URL that provides the description for an entity. + * + * @param entityType Type of the entity to be updated. + * @param appId AppId of the entity to be updated + * @param serviceUrl URL of the extensions service to which the entity belongs + * @return + */ + protected String getRequestUrl(SpServiceTagPrefix entityType, String appId, String serviceUrl) { + + SpServiceUrlProvider urlProvider; + switch (entityType) { + case ADAPTER -> urlProvider = SpServiceUrlProvider.ADAPTER; + case DATA_PROCESSOR -> urlProvider = SpServiceUrlProvider.DATA_PROCESSOR; + case DATA_SINK -> urlProvider = SpServiceUrlProvider.DATA_SINK; + default -> throw new RuntimeException("Unexpected instance type."); + } + return urlProvider.getInvocationUrl(serviceUrl, appId); + } + + /** + * Perform the update of the description based on the given requestUrl + * + * @param requestUrl URl that references the description to be updated at the extensions service. + */ + protected void performUpdate(String requestUrl) { + + try { + var entityPayload = parseURIContent(requestUrl); + var updateResult = Operations.verifyAndUpdateElement(entityPayload); + if (!updateResult.isSuccess()) { + LOG.error( + "Updating the pipeline element description failed: {}", + StringUtils.join( + updateResult.getNotifications().stream().map(Notification::toString).toList(), + "\n") + ); + } + } catch (IOException | URISyntaxException | SepaParseException e) { + LOG.error("Updating the pipeline element description failed due to the following exception:\n{}", + StringUtils.join(e.getStackTrace(), "\n") + ); + } + } + /** * Update the static properties of the failed pipeline element with its description. * This allows to adapt the failed pipeline element in the UI to overcome the failed migration. From 509b6b40e2e205c2c8a5d7c0f8786f46386dc69c Mon Sep 17 00:00:00 2001 From: bossenti Date: Fri, 27 Oct 2023 08:21:57 +0200 Subject: [PATCH 36/54] refactor: adapt ProcessingElementBuilder to be capable of versions --- .../connect/adapter/AdapterDescription.java | 10 ++++ .../sdk/builder/ProcessingElementBuilder.java | 59 ++++++++++++++++--- .../adapter/AdapterConfigurationBuilder.java | 27 ++++++++- 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java index e386a101f3..5fd6b21b3f 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java @@ -74,6 +74,16 @@ public AdapterDescription() { this.dataStream = new SpDataStream(); } + public AdapterDescription(int version) { + super(); + this.rules = new ArrayList<>(); + this.eventGrounding = new EventGrounding(); + this.config = new ArrayList<>(); + this.category = new ArrayList<>(); + this.dataStream = new SpDataStream(); + this.setVersion(version); + } + public AdapterDescription(String elementId, String name, String description) { super(elementId, name, description); this.rules = new ArrayList<>(); diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/ProcessingElementBuilder.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/ProcessingElementBuilder.java index 057820a47a..c244a2cf11 100644 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/ProcessingElementBuilder.java +++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/ProcessingElementBuilder.java @@ -35,43 +35,88 @@ public class ProcessingElementBuilder private List outputStrategies; - private ProcessingElementBuilder(String id, String name, String description) { + private ProcessingElementBuilder(String id, String name, String description, int version) { super(id, name, description, new DataProcessorDescription()); this.outputStrategies = new ArrayList<>(); + this.elementDescription.setVersion(version); } - private ProcessingElementBuilder(String id) { + private ProcessingElementBuilder(String id, int version) { super(id, new DataProcessorDescription()); this.outputStrategies = new ArrayList<>(); + this.elementDescription.setVersion(version); } /** * Creates a new processing element using the builder pattern. - * + * @deprecated + * This method is no longer recommend since we rely on a version for migration purposes. + *

Please adopt {@link #create(String, String, String, int)} instead. * @param id A unique identifier of the new element, e.g., com.mycompany.processor.mynewdataprocessor * @param label A human-readable name of the element. * Will later be shown as the element name in the StreamPipes UI. * @param description A human-readable description of the element. */ + @Deprecated(since = "0.93.0", forRemoval = true) public static ProcessingElementBuilder create(String id, String label, String description) { - return new ProcessingElementBuilder(id, label, description); + return new ProcessingElementBuilder(id, label, description, 0); } + /** + * Creates a new processing element based on a label using the builder pattern. + * @param id A unique identifier of the new element, e.g., com.mycompany.processor.mynewdataprocessor + * @param label A human-readable name of the element. + * Will later be shown as the element name in the StreamPipes UI. + * @param description A human-readable description of the element. + * @param version version of the processing element for migration purposes. Should be 0 in standard cases. + * Only in case there exist migrations for the specific element the version needs to be aligned. + * @return Builder for the pre-defined processing element. + */ + public static ProcessingElementBuilder create(String id, String label, String description, int version) { + return new ProcessingElementBuilder(id, label, description, version); + } + + /** + * Creates a new processing element based on a label using the builder pattern. + * @deprecated + * This method is no longer recommend since we rely on a version for migration purposes. + *

Please adopt {@link #create(String, String, String, int)} instead. + */ + @Deprecated(since = "0.93.0", forRemoval = true) public static ProcessingElementBuilder create(Label label) { - return new ProcessingElementBuilder(label.getInternalId(), label.getLabel(), label.getDescription()); + return new ProcessingElementBuilder(label.getInternalId(), label.getLabel(), label.getDescription(), 0); } /** - * Creates a new processing element using the builder pattern. If no label and description is + * Creates a new processing element using the builder pattern. + * @deprecated + * This method is no longer recommend since we rely on a version for migration purposes. + *

Please adopt {@link #create(String, int)} instead. + * If no label and description is * given * for an element, * {@link org.apache.streampipes.sdk.builder.AbstractProcessingElementBuilder#withLocales(Locales...)} * must be called. * * @param id A unique identifier of the new element, e.g., com.mycompany.sink.mynewdatasink + * */ + @Deprecated(since = "0.93.0", forRemoval = true) public static ProcessingElementBuilder create(String id) { - return new ProcessingElementBuilder(id); + return new ProcessingElementBuilder(id, 0); + } + + /** + * Creates a new processing element using the builder pattern. If no label and description is + * given + * for an element, + * {@link org.apache.streampipes.sdk.builder.AbstractProcessingElementBuilder#withLocales(Locales...)} + * must be called. + * + * @param id A unique identifier of the new element, e.g., com.mycompany.sink.mynewdatasink + */ + public static ProcessingElementBuilder create(String id, int version) { + return new ProcessingElementBuilder(id, version); } /** diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterConfigurationBuilder.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterConfigurationBuilder.java index 4dd39e1c80..fcebdbfaf6 100644 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterConfigurationBuilder.java +++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterConfigurationBuilder.java @@ -38,15 +38,38 @@ public class AdapterConfigurationBuilder extends private final Supplier supplier; protected AdapterConfigurationBuilder(String appId, + int version, Supplier supplier) { - super(appId, new AdapterDescription()); + super(appId, new AdapterDescription(version)); supportedParsers = new ArrayList<>(); this.supplier = supplier; } + /** + * Creates a new adapter configuration using the builder pattern. + * @deprecated + * This method is no longer recommend since we rely on a version for migration purposes. + *

Please adopt {@link #create(String, int, Supplier)} instead. + * @param appId A unique identifier of the new adapter, e.g., com.mycompany.processor.mynewdataprocessor + * @param supplier instance of the adapter to be described + */ + @Deprecated(since = "0.93.0", forRemoval = true) public static AdapterConfigurationBuilder create(String appId, Supplier supplier) { - return new AdapterConfigurationBuilder(appId, supplier); + return new AdapterConfigurationBuilder(appId, 0, supplier); + } + + /** + * Creates a new adapter configuration using the builder pattern. + * @param appId A unique identifier of the new adapter, e.g., com.mycompany.processor.mynewdataprocessor + * @param supplier instance of the adapter to be described + * @param version version of the processing element for migration purposes. Should be 0 in standard cases. + * Only in case there exist migrations for the specific element the version needs to be aligned. + */ + public static AdapterConfigurationBuilder create(String appId, + int version, + Supplier supplier) { + return new AdapterConfigurationBuilder(appId, version, supplier); } @Override From 203bd5461dc05440a569ec99f6331eb650e97357 Mon Sep 17 00:00:00 2001 From: bossenti Date: Fri, 27 Oct 2023 09:03:54 +0200 Subject: [PATCH 37/54] refactor: minor improvements --- .../streampipes/rest/impl/admin/MigrationResource.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index 3fec34f38f..7342bcc211 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -479,7 +479,8 @@ protected void updateDescriptions(List migrationConfigs, St var requestUrl = getRequestUrl(config.modelType(), config.targetAppId(), serviceUrl); performUpdate(requestUrl); } - ); + ) + .toList(); } /** @@ -488,7 +489,7 @@ protected void updateDescriptions(List migrationConfigs, St * @param entityType Type of the entity to be updated. * @param appId AppId of the entity to be updated * @param serviceUrl URL of the extensions service to which the entity belongs - * @return + * @return URL of the endpoint that provides the description for the given entity */ protected String getRequestUrl(SpServiceTagPrefix entityType, String appId, String serviceUrl) { From 1b1b4c0e6db0acf1e8a7a69e89dea9f1c23bb20a Mon Sep 17 00:00:00 2001 From: bossenti Date: Fri, 27 Oct 2023 09:30:39 +0200 Subject: [PATCH 38/54] style: fix checkstyle issues --- .../streampipes/rest/impl/admin/MigrationResourceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java index eb3c743253..63f0b92a77 100644 --- a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java +++ b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java @@ -18,11 +18,11 @@ package org.apache.streampipes.rest.impl.admin; -import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.graph.DataProcessorInvocation; import org.apache.streampipes.model.graph.DataSinkInvocation; import org.apache.streampipes.model.migration.ModelMigratorConfig; + import org.junit.Test; import java.util.List; From 542d6325a9473e89a6f1e63474a7dd887ffdcc60 Mon Sep 17 00:00:00 2001 From: bossenti Date: Fri, 27 Oct 2023 10:17:07 +0200 Subject: [PATCH 39/54] refactor: remove legacy type definition --- .../wrapper/distributed/runtime/DistributedRuntime.java | 2 +- .../java/org/apache/streampipes/wrapper/flink/FlinkRuntime.java | 2 +- .../apache/streampipes/wrapper/kafka/KafkaStreamsRuntime.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/streampipes-wrapper-distributed/src/main/java/org/apache/streampipes/wrapper/distributed/runtime/DistributedRuntime.java b/streampipes-wrapper-distributed/src/main/java/org/apache/streampipes/wrapper/distributed/runtime/DistributedRuntime.java index d24e9a65af..9e9b0578ce 100644 --- a/streampipes-wrapper-distributed/src/main/java/org/apache/streampipes/wrapper/distributed/runtime/DistributedRuntime.java +++ b/streampipes-wrapper-distributed/src/main/java/org/apache/streampipes/wrapper/distributed/runtime/DistributedRuntime.java @@ -43,7 +43,7 @@ public abstract class DistributedRuntime< PeT extends IStreamPipesPipelineElement, IvT extends InvocableStreamPipesEntity, RcT extends RuntimeContext, - ExT extends IParameterExtractor, + ExT extends IParameterExtractor, PepT extends IPipelineElementParameters> extends PipelineElementRuntime implements IStreamPipesRuntime { diff --git a/streampipes-wrapper-flink/src/main/java/org/apache/streampipes/wrapper/flink/FlinkRuntime.java b/streampipes-wrapper-flink/src/main/java/org/apache/streampipes/wrapper/flink/FlinkRuntime.java index af3afce599..5b77768f4d 100644 --- a/streampipes-wrapper-flink/src/main/java/org/apache/streampipes/wrapper/flink/FlinkRuntime.java +++ b/streampipes-wrapper-flink/src/main/java/org/apache/streampipes/wrapper/flink/FlinkRuntime.java @@ -69,7 +69,7 @@ public abstract class FlinkRuntime< PeT extends IStreamPipesPipelineElement, IvT extends InvocableStreamPipesEntity, RcT extends RuntimeContext, - ExT extends IParameterExtractor, + ExT extends IParameterExtractor, PepT extends IPipelineElementParameters, FpT extends IFlinkProgram> extends DistributedRuntime diff --git a/streampipes-wrapper-kafka-streams/src/main/java/org/apache/streampipes/wrapper/kafka/KafkaStreamsRuntime.java b/streampipes-wrapper-kafka-streams/src/main/java/org/apache/streampipes/wrapper/kafka/KafkaStreamsRuntime.java index d9f22287ca..6ba4311e04 100644 --- a/streampipes-wrapper-kafka-streams/src/main/java/org/apache/streampipes/wrapper/kafka/KafkaStreamsRuntime.java +++ b/streampipes-wrapper-kafka-streams/src/main/java/org/apache/streampipes/wrapper/kafka/KafkaStreamsRuntime.java @@ -39,7 +39,7 @@ public abstract class KafkaStreamsRuntime< PeT extends IStreamPipesPipelineElement, IvT extends InvocableStreamPipesEntity, RcT extends RuntimeContext, - ExT extends IParameterExtractor, + ExT extends IParameterExtractor, PepT extends IPipelineElementParameters> extends DistributedRuntime implements IStreamPipesRuntime { From 798ed3a9034b156401c79f434b9f8c447da4caac Mon Sep 17 00:00:00 2001 From: bossenti Date: Fri, 27 Oct 2023 10:46:24 +0200 Subject: [PATCH 40/54] refactor: update generated TS models --- .../lib/model/gen/streampipes-model-client.ts | 20 +------ .../src/lib/model/gen/streampipes-model.ts | 54 ++++++++++--------- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts index 0f076fb290..04daf55734 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts @@ -1,25 +1,7 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ /* tslint:disable */ /* eslint-disable */ // @ts-nocheck -// Generated using typescript-generator version 3.1.1185 on 2023-05-15 15:33:05. +// Generated using typescript-generator version 3.2.1263 on 2023-10-27 10:44:54. export class ExtensionsServiceEndpointItem { appId: string; diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts index 2cec24e173..cea5962535 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts @@ -1,32 +1,15 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ /* tslint:disable */ /* eslint-disable */ // @ts-nocheck -// Generated using typescript-generator version 3.2.1263 on 2023-10-24 20:02:02. +// Generated using typescript-generator version 3.2.1263 on 2023-10-27 10:43:45. export class NamedStreamPipesEntity { '@class': - | 'org.apache.streampipes.model.connect.adapter.AdapterDescription' | 'org.apache.streampipes.model.connect.grounding.ProtocolDescription' | 'org.apache.streampipes.model.template.PipelineTemplateDescription' | 'org.apache.streampipes.model.SpDataStream' + | 'org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity' + | 'org.apache.streampipes.model.connect.adapter.AdapterDescription' | 'org.apache.streampipes.model.base.InvocableStreamPipesEntity' | 'org.apache.streampipes.model.graph.DataProcessorInvocation' | 'org.apache.streampipes.model.graph.DataSinkInvocation'; @@ -81,7 +64,30 @@ export class NamedStreamPipesEntity { } } -export class AdapterDescription extends NamedStreamPipesEntity { +export class VersionedNamedStreamPipesEntity extends NamedStreamPipesEntity { + '@class': + | 'org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity' + | 'org.apache.streampipes.model.connect.adapter.AdapterDescription' + | 'org.apache.streampipes.model.base.InvocableStreamPipesEntity' + | 'org.apache.streampipes.model.graph.DataProcessorInvocation' + | 'org.apache.streampipes.model.graph.DataSinkInvocation'; + 'version': number; + + static 'fromData'( + data: VersionedNamedStreamPipesEntity, + target?: VersionedNamedStreamPipesEntity, + ): VersionedNamedStreamPipesEntity { + if (!data) { + return data; + } + const instance = target || new VersionedNamedStreamPipesEntity(); + super.fromData(data, instance); + instance.version = data.version; + return instance; + } +} + +export class AdapterDescription extends VersionedNamedStreamPipesEntity { '@class': 'org.apache.streampipes.model.connect.adapter.AdapterDescription'; 'category': string[]; 'config': StaticPropertyUnion[]; @@ -101,7 +107,6 @@ export class AdapterDescription extends NamedStreamPipesEntity { 'selectedEndpointUrl': string; 'streamRules': TransformationRuleDescriptionUnion[]; 'valueRules': TransformationRuleDescriptionUnion[]; - 'version': number; static 'fromData'( data: AdapterDescription, @@ -140,7 +145,6 @@ export class AdapterDescription extends NamedStreamPipesEntity { instance.valueRules = __getCopyArrayFn( TransformationRuleDescription.fromDataUnion, )(data.valueRules); - instance.version = data.version; return instance; } } @@ -1160,7 +1164,7 @@ export class DataLakeMeasure { } export class InvocableStreamPipesEntity - extends NamedStreamPipesEntity + extends VersionedNamedStreamPipesEntity implements EndpointSelectable { '@class': @@ -1174,6 +1178,7 @@ export class InvocableStreamPipesEntity 'detachPath': string; 'inputStreams': SpDataStream[]; 'selectedEndpointUrl': string; + 'serviceTagPrefix': SpServiceTagPrefix; 'staticProperties': StaticPropertyUnion[]; 'statusInfoSettings': ElementStatusInfoSettings; 'streamRequirements': SpDataStream[]; @@ -1198,6 +1203,7 @@ export class InvocableStreamPipesEntity data.inputStreams, ); instance.selectedEndpointUrl = data.selectedEndpointUrl; + instance.serviceTagPrefix = data.serviceTagPrefix; instance.staticProperties = __getCopyArrayFn( StaticProperty.fromDataUnion, )(data.staticProperties); From d0d28c07afc392160c604257b3931e4cb6d1e068 Mon Sep 17 00:00:00 2001 From: bossenti Date: Fri, 27 Oct 2023 10:48:27 +0200 Subject: [PATCH 41/54] fix: add missing license header --- .../lib/model/gen/streampipes-model-client.ts | 18 ++++++++++++++++++ .../src/lib/model/gen/streampipes-model.ts | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts index 04daf55734..fe8e13571f 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ /* tslint:disable */ /* eslint-disable */ // @ts-nocheck diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts index cea5962535..47f06b81f2 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ /* tslint:disable */ /* eslint-disable */ // @ts-nocheck From 672745e7ce14407e4442cb852431be2d4cf96c84 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Fri, 27 Oct 2023 22:17:14 +0200 Subject: [PATCH 42/54] Fix adapter model migration, add OPC adapter migration as sample --- .../connect/iiot/ConnectAdapterIiotInit.java | 3 ++ .../opcua/adapter/OpcUaAdapter.java | 2 +- .../opcua/config/SharedUserConfiguration.java | 18 ++++--- .../migration/OpcUaAdapterMigrationV1.java | 51 +++++++++++++++++++ .../migration/GenericAdapterConverter.java | 17 +++++-- .../migrations/v093/AdapterMigration.java | 29 ++++++----- 6 files changed, 94 insertions(+), 26 deletions(-) create mode 100644 streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/migration/OpcUaAdapterMigrationV1.java diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/ConnectAdapterIiotInit.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/ConnectAdapterIiotInit.java index 72270d31fd..fa651ea92d 100644 --- a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/ConnectAdapterIiotInit.java +++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/ConnectAdapterIiotInit.java @@ -30,6 +30,7 @@ import org.apache.streampipes.extensions.connectors.mqtt.adapter.MqttProtocol; import org.apache.streampipes.extensions.connectors.nats.adapter.NatsProtocol; import org.apache.streampipes.extensions.connectors.opcua.adapter.OpcUaAdapter; +import org.apache.streampipes.extensions.connectors.opcua.migration.OpcUaAdapterMigrationV1; import org.apache.streampipes.extensions.connectors.pulsar.adapter.PulsarProtocol; import org.apache.streampipes.extensions.connectors.rocketmq.adapter.RocketMQProtocol; import org.apache.streampipes.extensions.connectors.tubemq.adapter.TubeMQProtocol; @@ -63,6 +64,8 @@ public SpServiceDefinition provideServiceDefinition() { .registerAdapter(new RocketMQProtocol()) .registerAdapter(new HttpServerProtocol()) .registerAdapter(new TubeMQProtocol()) + + .registerMigrators(new OpcUaAdapterMigrationV1()) .build(); } } diff --git a/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/adapter/OpcUaAdapter.java b/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/adapter/OpcUaAdapter.java index bfa035c0cc..5df4f6329d 100644 --- a/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/adapter/OpcUaAdapter.java +++ b/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/adapter/OpcUaAdapter.java @@ -231,7 +231,7 @@ public StaticProperty resolveConfiguration(String staticPropertyInternalName, @Override public IAdapterConfiguration declareConfig() { - var builder = AdapterConfigurationBuilder.create(ID, OpcUaAdapter::new) + var builder = AdapterConfigurationBuilder.create(ID, 1, OpcUaAdapter::new) .withAssets(Assets.DOCUMENTATION, Assets.ICON) .withLocales(Locales.EN) .withCategory(AdapterType.Generic, AdapterType.Manufacturing) diff --git a/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/config/SharedUserConfiguration.java b/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/config/SharedUserConfiguration.java index 647e66fdb7..a5691da311 100644 --- a/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/config/SharedUserConfiguration.java +++ b/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/config/SharedUserConfiguration.java @@ -45,13 +45,7 @@ public class SharedUserConfiguration { public static void appendSharedOpcUaConfig(AbstractConfigurablePipelineElementBuilder builder, boolean adapterConfig) { - var dependsOn = adapterConfig ? List.of( - ADAPTER_TYPE.name(), - ACCESS_MODE.name(), - OPC_HOST_OR_URL.name() - ) : List.of( - ACCESS_MODE.name(), - OPC_HOST_OR_URL.name()); + var dependsOn = getDependsOn(adapterConfig); builder .requiredAlternatives(Labels.withId(ACCESS_MODE), @@ -86,4 +80,14 @@ public static void appendSharedOpcUaConfig(AbstractConfigurablePipelineElementBu adapterConfig ); } + + public static List getDependsOn(boolean adapterConfig) { + return adapterConfig ? List.of( + ADAPTER_TYPE.name(), + ACCESS_MODE.name(), + OPC_HOST_OR_URL.name() + ) : List.of( + ACCESS_MODE.name(), + OPC_HOST_OR_URL.name()); + } } diff --git a/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/migration/OpcUaAdapterMigrationV1.java b/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/migration/OpcUaAdapterMigrationV1.java new file mode 100644 index 0000000000..4f056d40cc --- /dev/null +++ b/streampipes-extensions/streampipes-connectors-opcua/src/main/java/org/apache/streampipes/extensions/connectors/opcua/migration/OpcUaAdapterMigrationV1.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.connectors.opcua.migration; + +import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor; +import org.apache.streampipes.extensions.api.migration.AdapterMigrator; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.model.migration.MigrationResult; +import org.apache.streampipes.model.migration.ModelMigratorConfig; + +public class OpcUaAdapterMigrationV1 extends AdapterMigrator { + + private static final String OldNamespaceIndexKey = "NAMESPACE_INDEX"; + private static final String OldNodeId = "NODE_ID"; + + @Override + public ModelMigratorConfig config() { + return new ModelMigratorConfig( + "org.apache.streampipes.connect.iiot.adapters.opcua", + SpServiceTagPrefix.ADAPTER, + 0, + 1); + } + + @Override + public MigrationResult migrate(AdapterDescription element, + IStaticPropertyExtractor extractor) throws RuntimeException { + + element.getConfig().removeIf(c -> c.getInternalName().equals(OldNamespaceIndexKey)); + element.getConfig().removeIf(c -> c.getInternalName().equals(OldNodeId)); + + return MigrationResult.success(element); + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/migration/GenericAdapterConverter.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/migration/GenericAdapterConverter.java index 8b1f17e72e..abd51c8187 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/migration/GenericAdapterConverter.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/migration/GenericAdapterConverter.java @@ -72,14 +72,21 @@ public JsonObject convert(JsonObject adapter) { helpers.updateFieldType(adapter); } - JsonObject formatDescription = getProperties(adapter).get(FORMAT_DESC_KEY).getAsJsonObject(); - JsonObject protocolDescription = getProperties(adapter).get(PROTOCOL_DESC_KEY).getAsJsonObject(); + var properties = getProperties(adapter); + if (!properties.has(CONFIG_KEY)) { + properties.add(CONFIG_KEY, new JsonArray()); + } + + JsonObject protocolDescription = properties.get(PROTOCOL_DESC_KEY).getAsJsonObject(); migrateProtocolDescription(adapter, protocolDescription); - migrateFormatDescription(adapter, formatDescription); + properties.remove(PROTOCOL_DESC_KEY); - getProperties(adapter).remove(FORMAT_DESC_KEY); - getProperties(adapter).remove(PROTOCOL_DESC_KEY); + if (properties.has(FORMAT_DESC_KEY)) { + JsonObject formatDescription = properties.get(FORMAT_DESC_KEY).getAsJsonObject(); + migrateFormatDescription(adapter, formatDescription); + properties.remove(FORMAT_DESC_KEY); + } return adapter; } diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java index 7e8d29824f..94d2d7eecd 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java @@ -31,11 +31,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import static org.apache.streampipes.model.connect.adapter.migration.utils.AdapterModels.GENERIC_STREAM; +import static org.apache.streampipes.model.connect.adapter.migration.utils.AdapterModels.isSetAdapter; public class AdapterMigration implements Migration { @@ -45,7 +45,7 @@ public class AdapterMigration implements Migration { private final CouchDbClient adapterInstanceClient; private final CouchDbClient adapterDescriptionClient; private final List adaptersToMigrate; - private final List adapterDescriptionsToDelete; + private final List adapterDescriptionsToMigrate; private final MigrationHelpers helpers; @@ -54,7 +54,7 @@ public AdapterMigration() { this.adapterInstanceClient = Utils.getCouchDbAdapterInstanceClient(); this.adapterDescriptionClient = Utils.getCouchDbAdapterDescriptionClient(); this.adaptersToMigrate = new ArrayList<>(); - this.adapterDescriptionsToDelete = new ArrayList<>(); + this.adapterDescriptionsToMigrate = new ArrayList<>(); this.helpers = new MigrationHelpers(); } @@ -64,9 +64,9 @@ public boolean shouldExecute() { var adapterDescriptionUri = getAllDocsUri(adapterDescriptionClient); findDocsToMigrate(adapterInstanceClient, adapterInstanceUri, adaptersToMigrate); - findDocsToMigrate(adapterDescriptionClient, adapterDescriptionUri, adapterDescriptionsToDelete); + findDocsToMigrate(adapterDescriptionClient, adapterDescriptionUri, adapterDescriptionsToMigrate); - return adaptersToMigrate.size() > 0 || adapterDescriptionsToDelete.size() > 0; + return !adaptersToMigrate.isEmpty() || !adapterDescriptionsToMigrate.isEmpty(); } private void findDocsToMigrate(CouchDbClient adapterClient, @@ -86,16 +86,19 @@ private void findDocsToMigrate(CouchDbClient adapterClient, } @Override - public void executeMigration() throws IOException { + public void executeMigration() { var adapterInstanceBackupClient = Utils.getCouchDbAdapterInstanceBackupClient(); - LOG.info("Deleting {} adapter descriptions, which will be regenerated after migration", - adapterDescriptionsToDelete.size()); - - adapterDescriptionsToDelete.forEach(ad -> { - String docId = helpers.getDocId(ad); - String rev = helpers.getRev(ad); - adapterDescriptionClient.remove(docId, rev); + adapterDescriptionsToMigrate.forEach(ad -> { + var adapterType = ad.get("type").getAsString(); + var appId = ad.get("appId"); + if (isSetAdapter(adapterType)) { + LOG.info("Deleting adapter description data set {}", appId); + adapterDescriptionClient.remove(helpers.getDocId(ad), helpers.getRev(ad)); + } else { + LOG.info("Migrating adapter description {} to new adapter model", appId); + getAdapterMigrator(adapterType).migrate(adapterDescriptionClient, ad); + } }); LOG.info("Migrating {} adapter models", adaptersToMigrate.size()); From 28b21c6e1c9b9bc02606a35baa0685cd2eca4772 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Fri, 27 Oct 2023 22:34:39 +0200 Subject: [PATCH 43/54] Fix typo --- .../migration/AdapterMigrationResource.java | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java index 2a79c1eec4..222a63e7c1 100644 --- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java +++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/AdapterMigrationResource.java @@ -43,40 +43,41 @@ @Path("/api/v1/migrations/adapter") public class AdapterMigrationResource extends MigrateExtensionsResource< - AdapterDescription, - IStaticPropertyExtractor, - AdapterMigrator - > { + AdapterDescription, + IStaticPropertyExtractor, + AdapterMigrator + > { + @POST @Consumes(MediaType.APPLICATION_JSON) @JacksonSerialized @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) @Operation( - summary = "Execute the migration for a specific adapter instance", tags = {"Extensions", "Migration"}, - responses = { - @ApiResponse( - responseCode = "" + HttpStatus.SC_OK, - description = "The migration was executed. It's result is described in the response. " - + "The Response needs to be handled accordingly.", - content = @Content( - examples = @ExampleObject( - name = "Successful migration", - value = "{\"success\": true,\"messages\": \"SUCCESS\", \"element\": {}}" - ), - mediaType = MediaType.APPLICATION_JSON - ) + summary = "Execute the migration for a specific adapter instance", tags = {"Extensions", "Migration"}, + responses = { + @ApiResponse( + responseCode = "" + HttpStatus.SC_OK, + description = "The migration was executed. Its result is described in the response. " + + "The Response needs to be handled accordingly.", + content = @Content( + examples = @ExampleObject( + name = "Successful migration", + value = "{\"success\": true,\"messages\": \"SUCCESS\", \"element\": {}}" + ), + mediaType = MediaType.APPLICATION_JSON ) - } + ) + } ) public Response migrateAdapter( - @Parameter( - description = "request that encompasses the adapter description(AdapterDescription) and " - + "the configuration of the migration", - example = "{\"migrationElement\": {}, \"modelMigratorConfig\": {\"targetAppId\": \"app-id\"," - + "\"modelType\": \"adapter\", \"fromVersion\": 0, \"toVersion\": 1}}", - required = true - ) - MigrationRequest adapterMigrationRequest) { + @Parameter( + description = "request that encompasses the adapter description(AdapterDescription) and " + + "the configuration of the migration", + example = "{\"migrationElement\": {}, \"modelMigratorConfig\": {\"targetAppId\": \"app-id\"," + + "\"modelType\": \"adapter\", \"fromVersion\": 0, \"toVersion\": 1}}", + required = true + ) + MigrationRequest adapterMigrationRequest) { return ok(handleMigration(adapterMigrationRequest)); } From 44055febc7e5c591b66fd000c231697ea3a5b0b0 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Sat, 28 Oct 2023 11:57:03 +0200 Subject: [PATCH 44/54] Extract MigrationResource logic into smaller units --- .../management/AdapterMigrationManager.java | 113 ++++ .../migration/AbstractMigrationManager.java | 155 ++++++ .../manager/migration/IMigrationHandler.java | 30 ++ .../manager/migration/MigrationUtils.java | 73 +++ .../PipelineElementMigrationManager.java | 249 +++++++++ .../manager/migration/MigrationUtilsTest.java | 12 +- .../rest/impl/admin/MigrationResource.java | 508 ++---------------- 7 files changed, 671 insertions(+), 469 deletions(-) create mode 100644 streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMigrationManager.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AbstractMigrationManager.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/IMigrationHandler.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/MigrationUtils.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/PipelineElementMigrationManager.java rename streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java => streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/migration/MigrationUtilsTest.java (83%) diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMigrationManager.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMigrationManager.java new file mode 100644 index 0000000000..26f158a5b7 --- /dev/null +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMigrationManager.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.management.management; + +import org.apache.streampipes.commons.exceptions.connect.AdapterException; +import org.apache.streampipes.manager.migration.AbstractMigrationManager; +import org.apache.streampipes.manager.migration.IMigrationHandler; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.apache.streampipes.storage.api.IAdapterStorage; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class AdapterMigrationManager extends AbstractMigrationManager implements IMigrationHandler { + + private static final Logger LOG = LoggerFactory.getLogger(AdapterMigrationManager.class); + + private final IAdapterStorage adapterStorage; + + public AdapterMigrationManager(IAdapterStorage adapterStorage) { + this.adapterStorage = adapterStorage; + } + + @Override + public void handleMigrations(SpServiceRegistration extensionsServiceConfig, + List migrationConfigs) { + + LOG.info("Received {} migrations from extension service {}.", + migrationConfigs.size(), + extensionsServiceConfig.getServiceUrl()); + LOG.info("Updating adapter descriptions by replacement..."); + updateDescriptions(migrationConfigs, extensionsServiceConfig.getServiceUrl()); + LOG.info("Adapter descriptions are up to date."); + + LOG.info("Checking migrations for existing adapters in StreamPipes Core ..."); + for (var migrationConfig : migrationConfigs) { + LOG.info("Searching for assets of '{}'", migrationConfig.targetAppId()); + LOG.debug("Searching for assets of '{}' with config {}", migrationConfig.targetAppId(), migrationConfig); + var adapterDescriptions = adapterStorage.getAdaptersByAppId(migrationConfig.targetAppId()); + LOG.info("Found {} instances for appId '{}'", adapterDescriptions.size(), migrationConfig.targetAppId()); + for (var adapterDescription : adapterDescriptions) { + + var adapterVersion = adapterDescription.getVersion(); + + if (adapterVersion == migrationConfig.fromVersion()) { + LOG.info("Migration is required for adapter '{}'. Migrating from version '{}' to '{}' ...", + adapterDescription.getElementId(), + adapterVersion, migrationConfig.toVersion() + ); + + var migrationResult = performMigration( + adapterDescription, + migrationConfig, + String.format("%s/%s/adapter", + extensionsServiceConfig.getServiceUrl(), + MIGRATION_ENDPOINT + ) + ); + + if (migrationResult.success()) { + LOG.info("Migration successfully performed by extensions service. Updating adapter description ..."); + LOG.debug( + "Migration was performed by extensions service '{}'", + extensionsServiceConfig.getServiceUrl()); + + adapterStorage.updateAdapter(migrationResult.element()); + LOG.info("Adapter description is updated - Migration successfully completed at Core."); + } else { + LOG.error("Migration failed with the following reason: {}", migrationResult.message()); + LOG.error( + "Migration for adapter '{}' failed - Stopping adapter ...", + migrationResult.element().getElementId() + ); + try { + WorkerRestClient.stopStreamAdapter(extensionsServiceConfig.getServiceUrl(), adapterDescription); + } catch (AdapterException e) { + LOG.error("Stopping adapter failed: {}", StringUtils.join(e.getStackTrace(), "\n")); + } + LOG.info("Adapter successfully stopped."); + } + } else { + LOG.info( + "Migration is not applicable for adapter '{}' because of a version mismatch - " + + "adapter version: '{}', migration starts at: '{}'", + adapterDescription.getElementId(), + adapterVersion, + migrationConfig.fromVersion() + ); + } + } + } + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AbstractMigrationManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AbstractMigrationManager.java new file mode 100644 index 0000000000..3641f0b7c7 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AbstractMigrationManager.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.migration; + +import org.apache.streampipes.commons.exceptions.SepaParseException; +import org.apache.streampipes.manager.endpoint.HttpJsonParser; +import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; +import org.apache.streampipes.manager.operations.Operations; +import org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity; +import org.apache.streampipes.model.extensions.migration.MigrationRequest; +import org.apache.streampipes.model.message.Notification; +import org.apache.streampipes.model.migration.MigrationResult; +import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.apache.streampipes.serializers.json.JacksonSerializer; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.apache.streampipes.manager.migration.MigrationUtils.getRequestUrl; + +public abstract class AbstractMigrationManager { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractMigrationManager.class); + + protected static final String MIGRATION_ENDPOINT = "api/v1/migrations"; + + /** + * Performs the actual migration of a pipeline element. + * This includes the communication with the extensions service which runs the migration. + * + * @param pipelineElement pipeline element to be migrated + * @param migrationConfig config of the migration to be performed + * @param url url of the migration endpoint at the extensions service + * where the migration should be performed + * @param type of the processing element + * @return result of the migration + */ + protected MigrationResult performMigration( + T pipelineElement, + ModelMigratorConfig migrationConfig, + String url + ) { + + try { + + var migrationRequest = new MigrationRequest<>(pipelineElement, migrationConfig); + + String serializedRequest = JacksonSerializer.getObjectMapper().writeValueAsString(migrationRequest); + + var migrationResponse = ExtensionServiceExecutions.extServicePostRequest( + url, + serializedRequest + ).execute(); + + TypeReference> typeReference = new TypeReference<>() { + }; + + return JacksonSerializer + .getObjectMapper() + .readValue(migrationResponse.returnContent().asString(), typeReference); + } catch (JsonProcessingException e) { + LOG.error( + "Migration of pipeline element failed before sending to the extensions service, " + + "pipeline element is not migrated. Serialization of migration request failed: {}", + StringUtils.join(e.getStackTrace(), "\n") + ); + } catch (IOException e) { + LOG.error("Migration of pipeline element failed at the extensions service, pipeline element is not migrated: {}.", + StringUtils.join(e.getStackTrace(), "\n") + ); + } + return MigrationResult.failure(pipelineElement, "Internal error during migration at StreamPipes Core"); + } + + /** + * Update all descriptions of entities in the Core that are affected by migrations. + * + * @param migrationConfigs List of migrations to take in account + * @param serviceUrl Url of the extension service that provides the migrations. + */ + protected void updateDescriptions(List migrationConfigs, String serviceUrl) { + migrationConfigs + .stream() + .collect( + // We only need to update the description once per appId, + // because this is directly done with the newest version of the description and + // there is iterative migration required. + // To avoid unnecessary, multiple updates, + // we filter the migration configs such that every appId is unique. + // This ensures that every description is only updated once. + Collectors.toMap( + ModelMigratorConfig::targetAppId, + Function.identity(), + (existing, replacement) -> existing + ) + ) + .values() + .stream() + .peek(config -> { + var requestUrl = getRequestUrl(config.modelType(), config.targetAppId(), serviceUrl); + performUpdate(requestUrl); + }) + .toList(); + } + + /** + * Perform the update of the description based on the given requestUrl + * + * @param requestUrl URl that references the description to be updated at the extensions service. + */ + protected void performUpdate(String requestUrl) { + + try { + var entityPayload = HttpJsonParser.getContentFromUrl(URI.create(requestUrl)); + var updateResult = Operations.verifyAndUpdateElement(entityPayload); + if (!updateResult.isSuccess()) { + LOG.error( + "Updating the pipeline element description failed: {}", + StringUtils.join( + updateResult.getNotifications().stream().map(Notification::toString).toList(), + "\n") + ); + } + } catch (IOException | SepaParseException e) { + LOG.error("Updating the pipeline element description failed due to the following exception:\n{}", + StringUtils.join(e.getStackTrace(), "\n") + ); + } + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/IMigrationHandler.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/IMigrationHandler.java new file mode 100644 index 0000000000..7847e24324 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/IMigrationHandler.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.migration; + +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.migration.ModelMigratorConfig; + +import java.util.List; + +public interface IMigrationHandler { + + void handleMigrations(SpServiceRegistration serviceRegistration, + List migrationConfigs); +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/MigrationUtils.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/MigrationUtils.java new file mode 100644 index 0000000000..8a96c3c1f4 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/MigrationUtils.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.migration; + +import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; + +import java.util.List; +import java.util.Optional; + +public class MigrationUtils { + + /** + * Filter the application migration for a pipeline definition. + * By definition, there is only one migration config that fulfills the requirements. + * Otherwise, it should have been detected as duplicate by the extensions service. + * + * @param pipelineElement pipeline element that should be migrated + * @param migrationConfigs available migration configs to pick the applicable from + * @return config that is applicable for the given pipeline element + */ + public static Optional getApplicableMigration( + InvocableStreamPipesEntity pipelineElement, + List migrationConfigs + ) { + return migrationConfigs + .stream() + .filter( + config -> config.modelType().equals(pipelineElement.getServiceTagPrefix()) + && config.targetAppId().equals(pipelineElement.getAppId()) + && config.fromVersion() == pipelineElement.getVersion() + ) + .findFirst(); + } + + /** + * Get the URL that provides the description for an entity. + * + * @param entityType Type of the entity to be updated. + * @param appId AppId of the entity to be updated + * @param serviceUrl URL of the extensions service to which the entity belongs + * @return URL of the endpoint that provides the description for the given entity + */ + public static String getRequestUrl(SpServiceTagPrefix entityType, String appId, String serviceUrl) { + + SpServiceUrlProvider urlProvider; + switch (entityType) { + case ADAPTER -> urlProvider = SpServiceUrlProvider.ADAPTER; + case DATA_PROCESSOR -> urlProvider = SpServiceUrlProvider.DATA_PROCESSOR; + case DATA_SINK -> urlProvider = SpServiceUrlProvider.DATA_SINK; + default -> throw new RuntimeException("Unexpected instance type."); + } + return urlProvider.getInvocationUrl(serviceUrl, appId); + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/PipelineElementMigrationManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/PipelineElementMigrationManager.java new file mode 100644 index 0000000000..28338e1ac0 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/PipelineElementMigrationManager.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.migration; + +import org.apache.streampipes.manager.execution.PipelineExecutor; +import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.graph.DataProcessorInvocation; +import org.apache.streampipes.model.graph.DataSinkInvocation; +import org.apache.streampipes.model.migration.MigrationResult; +import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.apache.streampipes.model.pipeline.Pipeline; +import org.apache.streampipes.model.pipeline.PipelineHealthStatus; +import org.apache.streampipes.model.staticproperty.StaticProperty; +import org.apache.streampipes.storage.api.IDataProcessorStorage; +import org.apache.streampipes.storage.api.IDataSinkStorage; +import org.apache.streampipes.storage.api.IPipelineStorage; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.streampipes.manager.migration.MigrationUtils.getApplicableMigration; + +public class PipelineElementMigrationManager extends AbstractMigrationManager implements IMigrationHandler { + + private static final Logger LOG = LoggerFactory.getLogger(PipelineElementMigrationManager.class); + + private final IPipelineStorage pipelineStorage; + private final IDataProcessorStorage dataProcessorStorage; + private final IDataSinkStorage dataSinkStorage; + + public PipelineElementMigrationManager(IPipelineStorage pipelineStorage, + IDataProcessorStorage dataProcessorStorage, + IDataSinkStorage dataSinkStorage) { + this.pipelineStorage = pipelineStorage; + this.dataProcessorStorage = dataProcessorStorage; + this.dataSinkStorage = dataSinkStorage; + } + + @Override + public void handleMigrations(SpServiceRegistration extensionsServiceConfig, + List migrationConfigs) { + + LOG.info("Updating pipeline element descriptions by replacement..."); + updateDescriptions(migrationConfigs, extensionsServiceConfig.getServiceUrl()); + LOG.info("Pipeline element descriptions are up to date."); + + LOG.info("Received {} pipeline element migrations from extension service {}.", + migrationConfigs.size(), + extensionsServiceConfig.getServiceUrl()); + var availablePipelines = pipelineStorage.getAllPipelines(); + if (!availablePipelines.isEmpty()) { + LOG.info("Found {} available pipelines. Checking pipelines for applicable migrations...", + availablePipelines.size() + ); + } + + for (var pipeline : availablePipelines) { + List> failedMigrations = new ArrayList<>(); + + var migratedDataProcessors = pipeline.getSepas() + .stream() + .map(processor -> { + if (getApplicableMigration(processor, migrationConfigs).isPresent()) { + return migratePipelineElement( + processor, + migrationConfigs, + String.format("%s/%s/processor", + extensionsServiceConfig.getServiceUrl(), + MIGRATION_ENDPOINT + ), + failedMigrations + ); + } else { + LOG.info("No migration applicable for data processor '{}'.", processor.getElementId()); + return processor; + } + }) + .toList(); + pipeline.setSepas(migratedDataProcessors); + + var migratedDataSinks = pipeline.getActions() + .stream() + .map(sink -> { + if (getApplicableMigration(sink, migrationConfigs).isPresent()) { + return migratePipelineElement( + sink, + migrationConfigs, + String.format("%s/%s/sink", + extensionsServiceConfig.getServiceUrl(), + MIGRATION_ENDPOINT + ), + failedMigrations + ); + } else { + LOG.info("No migration applicable for data sink '{}'.", sink.getElementId()); + return sink; + } + }) + .toList(); + pipeline.setActions(migratedDataSinks); + + pipelineStorage.updatePipeline(pipeline); + + if (failedMigrations.isEmpty()) { + LOG.info("Migration for pipeline successfully completed."); + } else { + // pass most recent version of pipeline + handleFailedMigrations(pipelineStorage.getPipeline(pipeline.getPipelineId()), failedMigrations); + } + } + } + + /** + * Takes care about the failed migrations of pipeline elements. + * This includes the following steps: + *

    + *
  • logging of failed pipeline elements + *
  • setting migration results as pipeline notifications + *
  • updating pipeline health status + *
  • stopping the pipeline + *
+ * + * @param pipeline the pipeline affected by failed migrations + * @param failedMigrations the list of failed migrations + */ + protected void handleFailedMigrations(Pipeline pipeline, List> failedMigrations) { + LOG.error("Failures in migration detected - The following pipeline elements could to be migrated:\n" + + StringUtils.join(failedMigrations.stream().map(Record::toString).toList()), "\n"); + + pipeline.setPipelineNotifications(failedMigrations.stream().map( + failedMigration -> "Failed migration of pipeline element: %s".formatted(failedMigration.message()) + ).toList()); + pipeline.setHealthStatus(PipelineHealthStatus.REQUIRES_ATTENTION); + + pipelineStorage.updatePipeline(pipeline); + + // get updated version of pipeline after modification + pipeline = pipelineStorage.getPipeline(pipeline.getPipelineId()); + + stopPipeline(pipeline); + } + + + public void stopPipeline(Pipeline pipeline) { + var pipelineExecutor = new PipelineExecutor(pipeline, true); + var pipelineStopResult = pipelineExecutor.stopPipeline(); + + if (pipelineStopResult.isSuccess()) { + LOG.info("Pipeline successfully stopped."); + } else { + LOG.error("Pipeline stop failed."); + } + } + + /** + * Handle the migration of a pipeline element with respect to the given model migration configs. + * All applicable migrations found in the provided configs are executed for the given pipeline element. + * In case a migration fails, the related pipeline element receives the latest definition of its static properties, + * so that the pipeline element can be adapted by the user to resolve the failed migration. + * + * @param pipelineElement pipeline element to be migrated + * @param modelMigrations list of model migrations that might be applicable for this pipeline element + * @param url url of the extensions service endpoint that handles the migration + * @param failedMigrations collection of failed migrations which is extended by occurring migration failures + * @param type of the pipeline element (e.g., DataProcessorInvocation) + * @return the migrated (or - in case of a failure - updated) pipeline element + */ + protected T migratePipelineElement( + T pipelineElement, + List modelMigrations, + String url, + List> failedMigrations + ) { + + // loop until no migrations are available anymore + // this allows to apply multiple migrations for a pipeline element sequentially + // For example, first migration from 0 to 1 and the second migration from 1 to 2 + while (getApplicableMigration(pipelineElement, modelMigrations).isPresent() && failedMigrations.isEmpty()) { + + var migrationConfig = getApplicableMigration(pipelineElement, modelMigrations).get(); + LOG.info( + "Found applicable migration for pipeline element '{}': {}", + pipelineElement.getElementId(), + migrationConfig + ); + + var migrationResult = performMigration( + pipelineElement, + migrationConfig, + url + ); + + if (migrationResult.success()) { + LOG.info("Migration successfully performed by extensions service. Updating pipeline element invocation ..."); + LOG.debug("Migration was performed at extensions service endpoint '{}'", url); + pipelineElement = migrationResult.element(); + } else { + LOG.error("Migration failed with the following reason: {}", migrationResult.message()); + failedMigrations.add(migrationResult); + } + } + if (!failedMigrations.isEmpty()) { + updateFailedPipelineElement(pipelineElement); + LOG.info("Updated pipeline elements with new description where automatic migration failed."); + } + return pipelineElement; + } + + /** + * Update the static properties of the failed pipeline element with its description. + * This allows to adapt the failed pipeline element in the UI to overcome the failed migration. + * + * @param pipelineElement pipeline element with failed migration + */ + protected void updateFailedPipelineElement(InvocableStreamPipesEntity pipelineElement) { + List updatedStaticProperties = new ArrayList<>(); + if (pipelineElement instanceof DataProcessorInvocation) { + updatedStaticProperties = dataProcessorStorage + .getFirstDataProcessorByAppId(pipelineElement.getAppId()) + .getStaticProperties(); + } else if (pipelineElement instanceof DataSinkInvocation) { + updatedStaticProperties = dataSinkStorage + .getFirstDataSinkByAppId(pipelineElement.getAppId()) + .getStaticProperties(); + } + pipelineElement.setStaticProperties(updatedStaticProperties); + } +} diff --git a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/migration/MigrationUtilsTest.java similarity index 83% rename from streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java rename to streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/migration/MigrationUtilsTest.java index 63f0b92a77..05545db197 100644 --- a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/admin/MigrationResourceTest.java +++ b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/migration/MigrationUtilsTest.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.rest.impl.admin; +package org.apache.streampipes.manager.migration; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.graph.DataProcessorInvocation; @@ -30,7 +30,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class MigrationResourceTest { +public class MigrationUtilsTest { List migrationConfigs = List.of( new ModelMigratorConfig("app-id", SpServiceTagPrefix.DATA_PROCESSOR, 0, 1), @@ -60,18 +60,18 @@ public void findMigrations() { assertEquals( migrationConfigs.get(0), - MigrationResource.getApplicableMigration(pipelineElement1, migrationConfigs).get() + MigrationUtils.getApplicableMigration(pipelineElement1, migrationConfigs).get() ); assertEquals( migrationConfigs.get(1), - MigrationResource.getApplicableMigration(pipelineElement2, migrationConfigs).get() + MigrationUtils.getApplicableMigration(pipelineElement2, migrationConfigs).get() ); assertEquals( migrationConfigs.get(2), - MigrationResource.getApplicableMigration(pipelineElement3, migrationConfigs).get() + MigrationUtils.getApplicableMigration(pipelineElement3, migrationConfigs).get() ); assertTrue( - MigrationResource.getApplicableMigration(pipelineElement4, migrationConfigs).isEmpty() + MigrationUtils.getApplicableMigration(pipelineElement4, migrationConfigs).isEmpty() ); } } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index 7342bcc211..2b6f065cbf 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -18,42 +18,22 @@ package org.apache.streampipes.rest.impl.admin; -import org.apache.streampipes.commons.exceptions.SepaParseException; -import org.apache.streampipes.commons.exceptions.connect.AdapterException; -import org.apache.streampipes.connect.management.management.WorkerRestClient; -import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; -import org.apache.streampipes.manager.execution.PipelineExecutor; -import org.apache.streampipes.manager.operations.Operations; -import org.apache.streampipes.model.base.InvocableStreamPipesEntity; -import org.apache.streampipes.model.base.VersionedNamedStreamPipesEntity; -import org.apache.streampipes.model.extensions.migration.MigrationRequest; +import org.apache.streampipes.connect.management.management.AdapterMigrationManager; +import org.apache.streampipes.manager.migration.PipelineElementMigrationManager; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; -import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; -import org.apache.streampipes.model.graph.DataProcessorInvocation; -import org.apache.streampipes.model.graph.DataSinkInvocation; -import org.apache.streampipes.model.message.Notification; -import org.apache.streampipes.model.migration.MigrationResult; import org.apache.streampipes.model.migration.ModelMigratorConfig; -import org.apache.streampipes.model.pipeline.Pipeline; -import org.apache.streampipes.model.pipeline.PipelineHealthStatus; -import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; -import org.apache.streampipes.serializers.json.JacksonSerializer; import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.storage.api.IAdapterStorage; import org.apache.streampipes.storage.api.IDataProcessorStorage; import org.apache.streampipes.storage.api.IDataSinkStorage; import org.apache.streampipes.storage.api.IPipelineStorage; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.responses.ApiResponse; -import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,13 +47,7 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; @Path("v2/migrations") @Component @@ -81,10 +55,9 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { private static final Logger LOG = LoggerFactory.getLogger(MigrationResource.class); - private static final String MIGRATION_ENDPOINT = "api/v1/migrations"; private final CRUDStorage extensionsServiceStorage = - getNoSqlStorage().getExtensionsServiceStorage(); + getNoSqlStorage().getExtensionsServiceStorage(); private final IAdapterStorage adapterStorage = getNoSqlStorage().getAdapterInstanceStorage(); @@ -97,93 +70,29 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { @Path("adapter/{serviceId}") @Consumes(MediaType.APPLICATION_JSON) @Operation( - summary = "Migrate adapters based on migration configs", tags = {"Core", "Migration"}, - responses = { - @ApiResponse( - responseCode = "" + HttpStatus.SC_OK, - description = "All provided migrations are handled. If an error appeared, " - + "the corresponding actions are taken.") - } + summary = "Migrate adapters based on migration configs", tags = {"Core", "Migration"}, + responses = { + @ApiResponse( + responseCode = "" + HttpStatus.SC_OK, + description = "All provided migrations are handled. If an error appeared, " + + "the corresponding actions are taken.") + } ) public Response registerAdapterMigrations( - @Parameter( - in = ParameterIn.PATH, - description = "the id of the extensions service that requests migrations", - required = true - ) - @PathParam("serviceId") String serviceId, - @Parameter( - description = "list of configs (ModelMigratorConfig) that describe the requested migrations", - required = true - ) - List migrationConfigs) { + @Parameter( + in = ParameterIn.PATH, + description = "the id of the extensions service that requests migrations", + required = true + ) + @PathParam("serviceId") String serviceId, + @Parameter( + description = "list of configs (ModelMigratorConfig) that describe the requested migrations", + required = true + ) + List migrationConfigs) { var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); - - LOG.info("Received {} migrations from extension service {}.", - migrationConfigs.size(), - extensionsServiceConfig.getServiceUrl()); - LOG.info("Updating adapter descriptions by replacement..."); - updateDescriptions(migrationConfigs, extensionsServiceConfig.getServiceUrl()); - LOG.info("Adapter descriptions are up to date."); - - LOG.info("Checking migrations for existing adapters in StreamPipes Core ..."); - for (var migrationConfig : migrationConfigs) { - LOG.info("Searching for assets of '{}'", migrationConfig.targetAppId()); - LOG.debug("Searching for assets of '{}' with config {}", migrationConfig.targetAppId(), migrationConfig); - var adapterDescriptions = adapterStorage.getAdaptersByAppId(migrationConfig.targetAppId()); - LOG.info("Found {} instances for appId '{}'", adapterDescriptions.size(), migrationConfig.targetAppId()); - for (var adapterDescription : adapterDescriptions) { - - var adapterVersion = adapterDescription.getVersion(); - - if (adapterVersion == migrationConfig.fromVersion()) { - LOG.info("Migration is required for adapter '{}'. Migrating from version '{}' to '{}' ...", - adapterDescription.getElementId(), - adapterVersion, migrationConfig.toVersion() - ); - - var migrationResult = performMigration( - adapterDescription, - migrationConfig, - String.format("%s/%s/adapter", - extensionsServiceConfig.getServiceUrl(), - MIGRATION_ENDPOINT - ) - ); - - if (migrationResult.success()) { - LOG.info("Migration successfully performed by extensions service. Updating adapter description ..."); - LOG.debug( - "Migration was performed by extensions service '{}'", - extensionsServiceConfig.getServiceUrl()); - - adapterStorage.updateAdapter(migrationResult.element()); - LOG.info("Adapter description is updated - Migration successfully completed at Core."); - } else { - LOG.error("Migration failed with the following reason: {}", migrationResult.message()); - LOG.error( - "Migration for adapter '{}' failed - Stopping adapter ...", - migrationResult.element().getElementId() - ); - try { - WorkerRestClient.stopStreamAdapter(extensionsServiceConfig.getServiceUrl(), adapterDescription); - } catch (AdapterException e) { - LOG.error("Stopping adapter failed: {}", StringUtils.join(e.getStackTrace(), "\n")); - } - LOG.info("Adapter successfully stopped."); - } - } else { - LOG.info( - "Migration is not applicable for adapter '{}' because of a version mismatch - " - + "adapter version: '{}', migration starts at: '{}'", - adapterDescription.getElementId(), - adapterVersion, - migrationConfig.fromVersion() - ); - } - } - } + new AdapterMigrationManager(adapterStorage).handleMigrations(extensionsServiceConfig, migrationConfigs); return ok(); } @@ -191,360 +100,33 @@ public Response registerAdapterMigrations( @Path("pipeline-element/{serviceId}") @Consumes(MediaType.APPLICATION_JSON) @Operation( - summary = "Migrate pipeline elements based on migration configs", tags = {"Core", "Migration"}, - responses = { - @ApiResponse( - responseCode = "200" + HttpStatus.SC_OK, - description = "All provided migrations are handled. " - + "If an error appeared, the corresponding actions are taken." - ) - } + summary = "Migrate pipeline elements based on migration configs", tags = {"Core", "Migration"}, + responses = { + @ApiResponse( + responseCode = "200" + HttpStatus.SC_OK, + description = "All provided migrations are handled. " + + "If an error appeared, the corresponding actions are taken." + ) + } ) public Response registerPipelineElementMigrations( - @Parameter( - in = ParameterIn.PATH, - description = "the id of the extensions service that requests migrations", - required = true - ) - @PathParam("serviceId") String serviceId, - @Parameter( - description = "list of config that describe the requested migrations" - ) - List migrationConfigs) { + @Parameter( + in = ParameterIn.PATH, + description = "the id of the extensions service that requests migrations", + required = true + ) + @PathParam("serviceId") String serviceId, + @Parameter( + description = "list of config that describe the requested migrations" + ) + List migrationConfigs) { var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); - LOG.info("Updating pipeline element descriptions by replacement..."); - updateDescriptions(migrationConfigs, extensionsServiceConfig.getServiceUrl()); - LOG.info("Pipeline element descriptions are up to date."); - - LOG.info("Received {} pipeline element migrations from extension service {}.", - migrationConfigs.size(), - extensionsServiceConfig.getServiceUrl()); - var availablePipelines = pipelineStorage.getAllPipelines(); - if (!availablePipelines.isEmpty()) { - LOG.info("Found {} available pipelines. Checking pipelines for applicable migrations...", - availablePipelines.size() - ); - } - - for (var pipeline : availablePipelines) { - List> failedMigrations = new ArrayList<>(); - - var migratedDataProcessors = pipeline.getSepas() - .stream() - .map(processor -> { - if (getApplicableMigration(processor, migrationConfigs).isPresent()) { - return migratePipelineElement( - processor, - migrationConfigs, - String.format("%s/%s/processor", - extensionsServiceConfig.getServiceUrl(), - MIGRATION_ENDPOINT - ), - failedMigrations - ); - } else { - LOG.info("No migration applicable for data processor '{}'.", processor.getElementId()); - return processor; - } - } - ).toList(); - pipeline.setSepas(migratedDataProcessors); - - var migratedDataSinks = pipeline.getActions() - .stream() - .map(sink -> { - if (getApplicableMigration(sink, migrationConfigs).isPresent()) { - return migratePipelineElement( - sink, - migrationConfigs, - String.format("%s/%s/sink", - extensionsServiceConfig.getServiceUrl(), - MIGRATION_ENDPOINT - ), - failedMigrations - ); - } else { - LOG.info("No migration applicable for data sink '{}'.", sink.getElementId()); - return sink; - } - } - ).toList(); - pipeline.setActions(migratedDataSinks); - - pipelineStorage.updatePipeline(pipeline); - - if (failedMigrations.isEmpty()) { - LOG.info("Migration for pipeline successfully completed."); - } else { - // pass most recent version of pipeline - handleFailedMigrations(pipelineStorage.getPipeline(pipeline.getPipelineId()), failedMigrations); - } - } + new PipelineElementMigrationManager( + pipelineStorage, + dataProcessorStorage, + dataSinkStorage) + .handleMigrations(extensionsServiceConfig, migrationConfigs); return ok(); } - - /** - * Takes care about the failed migrations of pipeline elements. - * This includes the following steps: - *
    - *
  • logging of failed pipeline elements - *
  • setting migration results as pipeline notifications - *
  • updating pipeline health status - *
  • stopping the pipeline - *
- * - * @param pipeline the pipeline affected by failed migrations - * @param failedMigrations the list of failed migrations - */ - protected void handleFailedMigrations(Pipeline pipeline, List> failedMigrations) { - LOG.error("Failures in migration detected - The following pipeline elements could to be migrated:\n" - + StringUtils.join(failedMigrations.stream().map(Record::toString).toList()), "\n"); - - pipeline.setPipelineNotifications(failedMigrations.stream().map( - failedMigration -> "Failed migration of pipeline element: %s".formatted(failedMigration.message()) - ).toList()); - pipeline.setHealthStatus(PipelineHealthStatus.REQUIRES_ATTENTION); - - pipelineStorage.updatePipeline(pipeline); - - // get updated version of pipeline after modification - pipeline = pipelineStorage.getPipeline(pipeline.getPipelineId()); - - stopPipeline(pipeline); - } - - - public void stopPipeline(Pipeline pipeline) { - var pipelineExecutor = new PipelineExecutor(pipeline, true); - var pipelineStopResult = pipelineExecutor.stopPipeline(); - - if (pipelineStopResult.isSuccess()) { - LOG.info("Pipeline successfully stopped."); - } else { - LOG.error("Pipeline stop failed."); - } - } - - /** - * Filter the application migration for a pipeline definition. - * By definition, there is only one migration config that fulfills the requirements. - * Otherwise, it should have been detected as duplicate by the extensions service. - * - * @param pipelineElement pipeline element that should be migrated - * @param migrationConfigs available migration configs to pick the applicable from - * @return config that is applicable for the given pipeline element - */ - protected static Optional getApplicableMigration( - InvocableStreamPipesEntity pipelineElement, - List migrationConfigs - ) { - return migrationConfigs - .stream() - .filter( - config -> config.modelType().equals(pipelineElement.getServiceTagPrefix()) - && config.targetAppId().equals(pipelineElement.getAppId()) - && config.fromVersion() == pipelineElement.getVersion() - ) - .findFirst(); - } - - /** - * Handle the migration of a pipeline element with respect to the given model migration configs. - * All applicable migrations found in the provided configs are executed for the given pipeline element. - * In case a migration fails, the related pipeline element receives the latest definition of its static properties, - * so that the pipeline element can be adapted by the user to resolve the failed migration. - * - * @param pipelineElement pipeline element to be migrated - * @param modelMigrations list of model migrations that might be applicable for this pipeline element - * @param url url of the extensions service endpoint that handles the migration - * @param failedMigrations collection of failed migrations which is extended by occurring migration failures - * @param type of the pipeline element (e.g., DataProcessorInvocation) - * @return the migrated (or - in case of a failure - updated) pipeline element - */ - protected T migratePipelineElement( - T pipelineElement, - List modelMigrations, - String url, - List> failedMigrations - ) { - - // loop until no migrations are available anymore - // this allows to apply multiple migrations for a pipeline element sequentially - // For example, first migration from 0 to 1 and the second migration from 1 to 2 - while (getApplicableMigration(pipelineElement, modelMigrations).isPresent() && failedMigrations.isEmpty()) { - - var migrationConfig = getApplicableMigration(pipelineElement, modelMigrations).get(); - LOG.info( - "Found applicable migration for pipeline element '{}': {}", - pipelineElement.getElementId(), - migrationConfig - ); - - var migrationResult = performMigration( - pipelineElement, - migrationConfig, - url - ); - - if (migrationResult.success()) { - LOG.info("Migration successfully performed by extensions service. Updating pipeline element invocation ..."); - LOG.debug("Migration was performed at extensions service endpoint '{}'", url); - pipelineElement = migrationResult.element(); - } else { - LOG.error("Migration failed with the following reason: {}", migrationResult.message()); - failedMigrations.add(migrationResult); - } - } - if (!failedMigrations.isEmpty()) { - updateFailedPipelineElement(pipelineElement); - LOG.info("Updated pipeline elements with new description where automatic migration failed."); - } - return pipelineElement; - } - - /** - * Performs the actual migration of a pipeline element. - * This includes the communication with the extensions service which runs the migration. - * - * @param pipelineElement pipeline element to be migrated - * @param migrationConfig config of the migration to be performed - * @param url url of the migration endpoint at the extensions service - * where the migration should be performed - * @param type of the processing element - * @return result of the migration - */ - protected MigrationResult performMigration( - T pipelineElement, - ModelMigratorConfig migrationConfig, - String url - ) { - - try { - - var migrationRequest = new MigrationRequest<>(pipelineElement, migrationConfig); - - String serializedRequest = JacksonSerializer.getObjectMapper().writeValueAsString(migrationRequest); - - var migrationResponse = ExtensionServiceExecutions.extServicePostRequest( - url, - serializedRequest - ).execute(); - - TypeReference> typeReference = new TypeReference<>() { - }; - - return JacksonSerializer - .getObjectMapper() - .readValue(migrationResponse.returnContent().asString(), typeReference); - } catch (JsonProcessingException e) { - LOG.error( - "Migration of pipeline element failed before sending to the extensions service, " - + "pipeline element is not migrated. Serialization of migration request failed: {}", - StringUtils.join(e.getStackTrace(), "\n") - ); - } catch (IOException e) { - LOG.error("Migration of pipeline element failed at the extensions service, pipeline element is not migrated: {}.", - StringUtils.join(e.getStackTrace(), "\n") - ); - } - return MigrationResult.failure(pipelineElement, "Internal error during migration at StreamPipes Core"); - } - - /** - * Update all descriptions of entities in the Core that are affected by migrations. - * - * @param migrationConfigs List of migrations to take in account - * @param serviceUrl Url of the extension service that provides the migrations. - */ - protected void updateDescriptions(List migrationConfigs, String serviceUrl) { - migrationConfigs - .stream() - .collect( - // We only need to update the description once per appId, - // because this is directly done with the newest version of the description and - // there is iterative migration required. - // To avoid unnecessary, multiple updates, - // we filter the migration configs such that every appId is unique. - // This ensures that every description is only updated once. - Collectors.toMap( - ModelMigratorConfig::targetAppId, - Function.identity(), - (existing, replacement) -> existing - ) - ) - .values() - .stream() - .peek(config -> { - var requestUrl = getRequestUrl(config.modelType(), config.targetAppId(), serviceUrl); - performUpdate(requestUrl); - } - ) - .toList(); - } - - /** - * Get the URL that provides the description for an entity. - * - * @param entityType Type of the entity to be updated. - * @param appId AppId of the entity to be updated - * @param serviceUrl URL of the extensions service to which the entity belongs - * @return URL of the endpoint that provides the description for the given entity - */ - protected String getRequestUrl(SpServiceTagPrefix entityType, String appId, String serviceUrl) { - - SpServiceUrlProvider urlProvider; - switch (entityType) { - case ADAPTER -> urlProvider = SpServiceUrlProvider.ADAPTER; - case DATA_PROCESSOR -> urlProvider = SpServiceUrlProvider.DATA_PROCESSOR; - case DATA_SINK -> urlProvider = SpServiceUrlProvider.DATA_SINK; - default -> throw new RuntimeException("Unexpected instance type."); - } - return urlProvider.getInvocationUrl(serviceUrl, appId); - } - - /** - * Perform the update of the description based on the given requestUrl - * - * @param requestUrl URl that references the description to be updated at the extensions service. - */ - protected void performUpdate(String requestUrl) { - - try { - var entityPayload = parseURIContent(requestUrl); - var updateResult = Operations.verifyAndUpdateElement(entityPayload); - if (!updateResult.isSuccess()) { - LOG.error( - "Updating the pipeline element description failed: {}", - StringUtils.join( - updateResult.getNotifications().stream().map(Notification::toString).toList(), - "\n") - ); - } - } catch (IOException | URISyntaxException | SepaParseException e) { - LOG.error("Updating the pipeline element description failed due to the following exception:\n{}", - StringUtils.join(e.getStackTrace(), "\n") - ); - } - } - - /** - * Update the static properties of the failed pipeline element with its description. - * This allows to adapt the failed pipeline element in the UI to overcome the failed migration. - * - * @param pipelineElement pipeline element with failed migration - */ - protected void updateFailedPipelineElement(InvocableStreamPipesEntity pipelineElement) { - List updatedStaticProperties = new ArrayList<>(); - if (pipelineElement instanceof DataProcessorInvocation) { - updatedStaticProperties = dataProcessorStorage - .getFirstDataProcessorByAppId(pipelineElement.getAppId()) - .getStaticProperties(); - } else if (pipelineElement instanceof DataSinkInvocation) { - updatedStaticProperties = dataSinkStorage - .getFirstDataSinkByAppId(pipelineElement.getAppId()) - .getStaticProperties(); - } - pipelineElement.setStaticProperties(updatedStaticProperties); - } } From 096dcf650e2a8ca7edee70227ba9ca575f1f7def Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Sat, 28 Oct 2023 13:25:10 +0200 Subject: [PATCH 45/54] Use single request for submitting migrations from extensions to core --- .../streampipes/client/api/IAdminApi.java | 4 +- .../streampipes/client/api/AdminApi.java | 18 ++----- .../rest/impl/admin/MigrationResource.java | 52 +++++++------------ .../extensions/ExtensionsModelSubmitter.java | 27 ++-------- 4 files changed, 28 insertions(+), 73 deletions(-) diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java index 35047ca9e6..ba776ce51c 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java @@ -40,9 +40,7 @@ public interface IAdminApi { void deregisterFunction(String functionId); - void registerAdapterMigrations(List migrationConfigs, String serviceId); - - void registerPipelineElementMigrations(List migratorConfigs, String serviceId); + void registerMigrations(List migrationConfigs, String serviceId); MessagingSettings getMessagingSettings(); } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java index 0670d94483..970eb62249 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java @@ -72,12 +72,8 @@ public void deregisterFunction(String functionId) { * @param migrationConfigs list of migration configs to be registered */ @Override - public void registerAdapterMigrations(List migrationConfigs, String serviceId) { - post(getAdapterMigrationPath().addToPath(serviceId), migrationConfigs); - } - - public void registerPipelineElementMigrations(List migratorConfigs, String serviceId) { - post(getPipelineElementMigrationPath().addToPath(serviceId), migratorConfigs); + public void registerMigrations(List migrationConfigs, String serviceId) { + post(getMigrationPath().addToPath(serviceId), migrationConfigs); } @Override @@ -113,15 +109,9 @@ private StreamPipesApiPath getDeleteFunctionPath(String functionId) { return getFunctionsPath().addToPath(functionId); } - private StreamPipesApiPath getAdapterMigrationPath() { - return StreamPipesApiPath - .fromBaseApiPath() - .addToPath("migrations/adapter"); - } - - private StreamPipesApiPath getPipelineElementMigrationPath() { + private StreamPipesApiPath getMigrationPath() { return StreamPipesApiPath .fromBaseApiPath() - .addToPath("migrations/pipeline-element"); + .addToPath("migrations"); } } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index 2b6f065cbf..56c940e7c5 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -21,6 +21,7 @@ import org.apache.streampipes.connect.management.management.AdapterMigrationManager; import org.apache.streampipes.manager.migration.PipelineElementMigrationManager; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.migration.ModelMigratorConfig; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; @@ -67,10 +68,10 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { private final IPipelineStorage pipelineStorage = getNoSqlStorage().getPipelineStorageAPI(); @POST - @Path("adapter/{serviceId}") + @Path("{serviceId}") @Consumes(MediaType.APPLICATION_JSON) @Operation( - summary = "Migrate adapters based on migration configs", tags = {"Core", "Migration"}, + summary = "Migrate adapters and pipeline elements based on migration configs", tags = {"Core", "Migration"}, responses = { @ApiResponse( responseCode = "" + HttpStatus.SC_OK, @@ -78,7 +79,7 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { + "the corresponding actions are taken.") } ) - public Response registerAdapterMigrations( + public Response performMigrations( @Parameter( in = ParameterIn.PATH, description = "the id of the extensions service that requests migrations", @@ -92,41 +93,26 @@ public Response registerAdapterMigrations( List migrationConfigs) { var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); - new AdapterMigrationManager(adapterStorage).handleMigrations(extensionsServiceConfig, migrationConfigs); - return ok(); - } - - @POST - @Path("pipeline-element/{serviceId}") - @Consumes(MediaType.APPLICATION_JSON) - @Operation( - summary = "Migrate pipeline elements based on migration configs", tags = {"Core", "Migration"}, - responses = { - @ApiResponse( - responseCode = "200" + HttpStatus.SC_OK, - description = "All provided migrations are handled. " - + "If an error appeared, the corresponding actions are taken." - ) - } - ) - public Response registerPipelineElementMigrations( - @Parameter( - in = ParameterIn.PATH, - description = "the id of the extensions service that requests migrations", - required = true - ) - @PathParam("serviceId") String serviceId, - @Parameter( - description = "list of config that describe the requested migrations" - ) - List migrationConfigs) { + var adapterMigrations = filterConfigs(migrationConfigs, List.of(SpServiceTagPrefix.ADAPTER)); + var pipelineElementMigrations = filterConfigs( + migrationConfigs, + List.of(SpServiceTagPrefix.DATA_PROCESSOR, SpServiceTagPrefix.DATA_SINK) + ); - var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); + new AdapterMigrationManager(adapterStorage).handleMigrations(extensionsServiceConfig, adapterMigrations); new PipelineElementMigrationManager( pipelineStorage, dataProcessorStorage, dataSinkStorage) - .handleMigrations(extensionsServiceConfig, migrationConfigs); + .handleMigrations(extensionsServiceConfig, pipelineElementMigrations); return ok(); } + + private List filterConfigs(List migrationConfigs, + List modelTypes) { + return migrationConfigs + .stream() + .filter(config -> modelTypes.stream().anyMatch(modelType -> modelType == config.modelType())) + .toList(); + } } diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java index c562ae02fa..cd2721007e 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java @@ -23,7 +23,6 @@ import org.apache.streampipes.extensions.management.init.DeclarersSingleton; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; -import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.service.extensions.function.StreamPipesFunctionHandler; import org.apache.streampipes.service.extensions.security.WebSecurityConfig; @@ -51,28 +50,10 @@ public void onExit() { public void afterServiceRegistered(SpServiceDefinition serviceDef) { StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); - // register all adapter migrations at StreamPipes Core - var adapterMigrations = serviceDef.getMigrators() - .stream() - .filter(modelMigrator -> modelMigrator.config().modelType() == SpServiceTagPrefix.ADAPTER) - .toList(); - client.adminApi().registerAdapterMigrations( - adapterMigrations.stream().map(ModelMigrator::config).toList(), - serviceId() - ); - - // register all pipeline element migrations at StreamPipes Core - var pipelineElementMigrations = serviceDef.getMigrators() - .stream() - .filter(modelMigrator -> - modelMigrator.config().modelType() == SpServiceTagPrefix.DATA_PROCESSOR - || modelMigrator.config().modelType() == SpServiceTagPrefix.DATA_SINK - ) - .toList(); - client.adminApi().registerPipelineElementMigrations( - pipelineElementMigrations.stream().map(ModelMigrator::config).toList(), - serviceId() - ); + // register all migrations at StreamPipes Core + var migrationConfigs = serviceDef.getMigrators().stream().map(ModelMigrator::config).toList(); + client.adminApi(). + registerMigrations(migrationConfigs, serviceId()); // initialize all function instances StreamPipesFunctionHandler.INSTANCE.initializeFunctions(serviceDef.getServiceGroup()); From 67fe516edbfe508035c3ee91edff7b4a9e06801d Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Sun, 29 Oct 2023 21:42:47 +0100 Subject: [PATCH 46/54] Improve execution order of migrations and service startup tasks --- .../management/health/AdapterHealthCheck.java | 8 +- .../configuration/SpCoreConfiguration.java | 9 ++ .../SpCoreConfigurationStatus.java | 25 ++++++ .../svcdiscovery/SpServiceRegistration.java | 18 ++-- .../svcdiscovery/SpServiceStatus.java | 26 ++++++ .../health/CoreServiceStatusManager.java | 59 +++++++++++++ .../manager/health/ServiceHealthCheck.java | 9 +- .../health/ServiceRegistrationManager.java | 85 +++++++++++++++++++ .../setup/SpCoreConfigurationStep.java | 10 ++- .../manager/setup/StreamPipesEnvChecker.java | 2 +- .../rest/impl/admin/MigrationResource.java | 51 ++++++++--- .../admin/ServiceRegistrationResource.java | 12 ++- .../service/core/PostStartupTask.java | 7 +- .../core/StreamPipesCoreApplication.java | 53 +++++++----- .../v093/ConsulConfigMigration.java | 2 + .../svcdiscovery/SpServiceDiscoveryCore.java | 3 +- .../extensions/CoreRequestSubmitter.java | 54 ++++++++++++ .../extensions/ExtensionsModelSubmitter.java | 9 +- .../StreamPipesExtensionsServiceBase.java | 30 +++---- .../api/ISpCoreConfigurationStorage.java | 2 + .../impl/CoreConfigurationStorageImpl.java | 5 ++ 21 files changed, 401 insertions(+), 78 deletions(-) create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfigurationStatus.java create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceStatus.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/CoreServiceStatusManager.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceRegistrationManager.java create mode 100644 streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CoreRequestSubmitter.java diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java index 07d70cf7dd..b5bdc27394 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java @@ -34,7 +34,7 @@ import java.util.List; import java.util.Map; -public class AdapterHealthCheck { +public class AdapterHealthCheck implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(AdapterHealthCheck.class); @@ -52,6 +52,11 @@ public AdapterHealthCheck(IAdapterStorage adapterStorage, this.adapterMasterManagement = adapterMasterManagement; } + @Override + public void run() { + this.checkAndRestoreAdapters(); + } + /** * In this method it is checked which adapters are currently running. * Then it calls all workers to validate if the adapter instance is @@ -135,5 +140,4 @@ public void recoverAdapters(Map adaptersToRecover) { } } - } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java index 3e3bd251d0..6aad62a172 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java @@ -34,6 +34,7 @@ public class SpCoreConfiguration { private GeneralConfig generalConfig; private boolean isConfigured; + private SpCoreConfigurationStatus serviceStatus; private String assetDir; private String filesDir; @@ -120,4 +121,12 @@ public EmailTemplateConfig getEmailTemplateConfig() { public void setEmailTemplateConfig(EmailTemplateConfig emailTemplateConfig) { this.emailTemplateConfig = emailTemplateConfig; } + + public SpCoreConfigurationStatus getServiceStatus() { + return this.serviceStatus; + } + + public void setServiceStatus(SpCoreConfigurationStatus serviceStatus) { + this.serviceStatus = serviceStatus; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfigurationStatus.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfigurationStatus.java new file mode 100644 index 0000000000..394b9a8aba --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfigurationStatus.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.configuration; + +public enum SpCoreConfigurationStatus { + INSTALLING, + MIGRATING, + READY +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java index b1490adfaa..8edd9932fa 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java @@ -36,8 +36,8 @@ public class SpServiceRegistration { private int port; private List tags; private String healthCheckPath; - private boolean healthy = true; private long firstTimeSeenUnhealthy = 0; + private SpServiceStatus status = SpServiceStatus.REGISTERED; public SpServiceRegistration() { } @@ -133,14 +133,6 @@ public void setRev(String rev) { this.rev = rev; } - public boolean isHealthy() { - return healthy; - } - - public void setHealthy(boolean healthy) { - this.healthy = healthy; - } - public String getScheme() { return scheme; } @@ -168,4 +160,12 @@ public String getSvcType() { public void setSvcType(String svcType) { this.svcType = svcType; } + + public SpServiceStatus getStatus() { + return status; + } + + public void setStatus(SpServiceStatus status) { + this.status = status; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceStatus.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceStatus.java new file mode 100644 index 0000000000..56b388d166 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceStatus.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.extensions.svcdiscovery; + +public enum SpServiceStatus { + REGISTERED, + MIGRATING, + HEALTHY, + UNHEALTHY +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/CoreServiceStatusManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/CoreServiceStatusManager.java new file mode 100644 index 0000000000..3d2c407ded --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/CoreServiceStatusManager.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.health; + +import org.apache.streampipes.model.configuration.SpCoreConfiguration; +import org.apache.streampipes.model.configuration.SpCoreConfigurationStatus; +import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CoreServiceStatusManager { + + private static final Logger LOG = LoggerFactory.getLogger(CoreServiceStatusManager.class); + + private final ISpCoreConfigurationStorage storage; + + public CoreServiceStatusManager(ISpCoreConfigurationStorage storage) { + this.storage = storage; + } + + public boolean existsConfig() { + return storage.exists(); + } + + public boolean isCoreReady() { + return existsConfig() && storage.get().getServiceStatus() == SpCoreConfigurationStatus.READY; + } + + public void updateCoreStatus(SpCoreConfigurationStatus status) { + var config = storage.get(); + config.setServiceStatus(status); + storage.updateElement(config); + logService(config); + } + + private void logService(SpCoreConfiguration coreConfig) { + LOG.info( + "Core is now in {} state", + coreConfig.getServiceStatus() + ); + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java index fda4741a3e..7f48e9c236 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java @@ -20,6 +20,7 @@ import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceStatus; import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.storage.management.StorageDispatcher; @@ -56,8 +57,8 @@ private void checkServiceHealth(SpServiceRegistration service) { if (response.returnResponse().getStatusLine().getStatusCode() != 200) { processUnhealthyService(service); } else { - if (!service.isHealthy()) { - service.setHealthy(true); + if (service.getStatus() == SpServiceStatus.UNHEALTHY) { + service.setStatus(SpServiceStatus.HEALTHY); updateService(service); } } @@ -67,8 +68,8 @@ private void checkServiceHealth(SpServiceRegistration service) { } private void processUnhealthyService(SpServiceRegistration service) { - if (service.isHealthy()) { - service.setHealthy(false); + if (service.getStatus() == SpServiceStatus.HEALTHY) { + service.setStatus(SpServiceStatus.UNHEALTHY); service.setFirstTimeSeenUnhealthy(System.currentTimeMillis()); updateService(service); } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceRegistrationManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceRegistrationManager.java new file mode 100644 index 0000000000..cbae440a8d --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceRegistrationManager.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.health; + +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceStatus; +import org.apache.streampipes.storage.api.CRUDStorage; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ServiceRegistrationManager { + + private static final Logger LOG = LoggerFactory.getLogger(ServiceRegistrationManager.class); + + private final CRUDStorage storage; + + public ServiceRegistrationManager(CRUDStorage storage) { + this.storage = storage; + } + + public void applyServiceStatus(String serviceId, + SpServiceStatus status) { + var serviceRegistration = storage.getElementById(serviceId); + serviceRegistration.setStatus(status); + storage.updateElement(serviceRegistration); + logService(serviceRegistration); + } + + public void addService(SpServiceRegistration serviceRegistration, + SpServiceStatus status) { + serviceRegistration.setStatus(status); + storage.createElement(serviceRegistration); + logService(serviceRegistration); + } + + public SpServiceRegistration getService(String serviceId) { + return storage.getElementById(serviceId); + } + + public boolean isAnyServiceMigrating() { + return storage.getAll() + .stream() + .anyMatch(service -> service.getStatus() == SpServiceStatus.MIGRATING); + } + + public void removeService(String serviceId) { + var serviceRegistration = storage.getElementById(serviceId); + storage.deleteElement(serviceRegistration); + LOG.info( + "Service {} (id={}) has been removed", + serviceRegistration.getSvcGroup(), + serviceRegistration.getSvcId()) + ; + } + + public SpServiceStatus getServiceStatus(String serviceId) { + return storage.getElementById(serviceId).getStatus(); + } + + private void logService(SpServiceRegistration serviceRegistration) { + LOG.info( + "Service {} (id={}) is now in {} state", + serviceRegistration.getSvcGroup(), + serviceRegistration.getSvcId(), + serviceRegistration.getStatus() + ); + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/SpCoreConfigurationStep.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/SpCoreConfigurationStep.java index 187d677efc..b5e0a8bd3b 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/SpCoreConfigurationStep.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/SpCoreConfigurationStep.java @@ -19,14 +19,22 @@ package org.apache.streampipes.manager.setup; import org.apache.streampipes.model.configuration.DefaultSpCoreConfiguration; +import org.apache.streampipes.model.configuration.SpCoreConfigurationStatus; import org.apache.streampipes.storage.management.StorageDispatcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class SpCoreConfigurationStep extends InstallationStep { + + private static final Logger LOG = LoggerFactory.getLogger(SpCoreConfigurationStep.class); + @Override public void install() { var coreCfg = new DefaultSpCoreConfiguration().make(); - + coreCfg.setServiceStatus(SpCoreConfigurationStatus.INSTALLING); StorageDispatcher.INSTANCE.getNoSqlStore().getSpCoreConfigurationStorage().createElement(coreCfg); + LOG.info("Core is now in {} state", coreCfg.getServiceStatus()); new StreamPipesEnvChecker().updateEnvironmentVariables(); } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/StreamPipesEnvChecker.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/StreamPipesEnvChecker.java index c4fc1bba08..70e5a8aa59 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/StreamPipesEnvChecker.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/StreamPipesEnvChecker.java @@ -52,7 +52,7 @@ public void updateEnvironmentVariables() { .getNoSqlStore() .getSpCoreConfigurationStorage(); - if (configStorage.getAll().size() > 0) { + if (configStorage.exists()) { this.coreConfig = configStorage.get(); LOG.info("Checking and updating environment variables..."); diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index 56c940e7c5..e715a1c02a 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -18,9 +18,13 @@ package org.apache.streampipes.rest.impl.admin; +import org.apache.streampipes.config.backend.BackendConfig; import org.apache.streampipes.connect.management.management.AdapterMigrationManager; +import org.apache.streampipes.manager.health.CoreServiceStatusManager; +import org.apache.streampipes.manager.health.ServiceRegistrationManager; import org.apache.streampipes.manager.migration.PipelineElementMigrationManager; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceStatus; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.migration.ModelMigratorConfig; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; @@ -67,6 +71,10 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { private final IDataSinkStorage dataSinkStorage = getNoSqlStorage().getDataSinkStorage(); private final IPipelineStorage pipelineStorage = getNoSqlStorage().getPipelineStorageAPI(); + private final CoreServiceStatusManager coreServiceStatusManager = new CoreServiceStatusManager( + getNoSqlStorage().getSpCoreConfigurationStorage() + ); + @POST @Path("{serviceId}") @Consumes(MediaType.APPLICATION_JSON) @@ -92,24 +100,39 @@ public Response performMigrations( ) List migrationConfigs) { - var extensionsServiceConfig = extensionsServiceStorage.getElementById(serviceId); - var adapterMigrations = filterConfigs(migrationConfigs, List.of(SpServiceTagPrefix.ADAPTER)); - var pipelineElementMigrations = filterConfigs( - migrationConfigs, - List.of(SpServiceTagPrefix.DATA_PROCESSOR, SpServiceTagPrefix.DATA_SINK) - ); - - new AdapterMigrationManager(adapterStorage).handleMigrations(extensionsServiceConfig, adapterMigrations); - new PipelineElementMigrationManager( - pipelineStorage, - dataProcessorStorage, - dataSinkStorage) - .handleMigrations(extensionsServiceConfig, pipelineElementMigrations); + var serviceManager = new ServiceRegistrationManager(extensionsServiceStorage); + var extensionsServiceConfig = serviceManager.getService(serviceId); + if (!migrationConfigs.isEmpty() && BackendConfig.INSTANCE.isConfigured()) { + if (serviceManager.isAnyServiceMigrating() || !isCoreReady()) { + LOG.info("Refusing migration request since precondition is not met."); + return Response.status(HttpStatus.SC_CONFLICT).build(); + } else { + serviceManager.applyServiceStatus(serviceId, SpServiceStatus.MIGRATING); + var adapterMigrations = filterConfigs(migrationConfigs, List.of(SpServiceTagPrefix.ADAPTER)); + var pipelineElementMigrations = filterConfigs( + migrationConfigs, + List.of(SpServiceTagPrefix.DATA_PROCESSOR, SpServiceTagPrefix.DATA_SINK) + ); + + new AdapterMigrationManager(adapterStorage).handleMigrations(extensionsServiceConfig, adapterMigrations); + new PipelineElementMigrationManager( + pipelineStorage, + dataProcessorStorage, + dataSinkStorage) + .handleMigrations(extensionsServiceConfig, pipelineElementMigrations); + } + } + new ServiceRegistrationManager(extensionsServiceStorage) + .applyServiceStatus(extensionsServiceConfig.getSvcId(), SpServiceStatus.HEALTHY); return ok(); } + private boolean isCoreReady() { + return coreServiceStatusManager.isCoreReady(); + } + private List filterConfigs(List migrationConfigs, - List modelTypes) { + List modelTypes) { return migrationConfigs .stream() .filter(config -> modelTypes.stream().anyMatch(modelType -> modelType == config.modelType())) diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java index f89cf34ace..d97653d866 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java @@ -18,11 +18,15 @@ package org.apache.streampipes.rest.impl.admin; +import org.apache.streampipes.manager.health.ServiceRegistrationManager; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceStatus; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; import org.apache.streampipes.storage.api.CRUDStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; @@ -40,6 +44,8 @@ @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) public class ServiceRegistrationResource extends AbstractAuthGuardedRestResource { + private static final Logger LOG = LoggerFactory.getLogger(ServiceRegistrationResource.class); + private final CRUDStorage extensionsServiceStorage = getNoSqlStorage().getExtensionsServiceStorage(); @@ -52,7 +58,8 @@ public Response getRegisteredServices() { @POST @Consumes(MediaType.APPLICATION_JSON) public Response registerService(SpServiceRegistration serviceRegistration) { - extensionsServiceStorage.createElement(serviceRegistration); + new ServiceRegistrationManager(extensionsServiceStorage) + .addService(serviceRegistration, SpServiceStatus.REGISTERED); return ok(); } @@ -60,8 +67,7 @@ public Response registerService(SpServiceRegistration serviceRegistration) { @Path("/{serviceId}") public Response unregisterService(@PathParam("serviceId") String serviceId) { try { - var serviceRegistration = extensionsServiceStorage.getElementById(serviceId); - extensionsServiceStorage.deleteElement(serviceRegistration); + new ServiceRegistrationManager(extensionsServiceStorage).removeService(serviceId); return ok(); } catch (IllegalArgumentException e) { return badRequest("Could not find registered service with id " + serviceId); diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PostStartupTask.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PostStartupTask.java index 9355c6b895..2442816746 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PostStartupTask.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PostStartupTask.java @@ -44,13 +44,13 @@ public class PostStartupTask implements Runnable { private static final int MAX_PIPELINE_START_RETRIES = 3; private static final int WAIT_TIME_AFTER_FAILURE_IN_SECONDS = 10; - private final List allPipelines; + private final IPipelineStorage pipelineStorage; private final Map failedPipelines = new HashMap<>(); private final ScheduledExecutorService executorService; private final WorkerAdministrationManagement workerAdministrationManagement; - public PostStartupTask(List allPipelines) { - this.allPipelines = allPipelines; + public PostStartupTask(IPipelineStorage pipelineStorage) { + this.pipelineStorage = pipelineStorage; this.executorService = Executors.newSingleThreadScheduledExecutor(); this.workerAdministrationManagement = new WorkerAdministrationManagement(); } @@ -76,6 +76,7 @@ private void startAdapters() { } private void startAllPreviouslyStoppedPipelines() { + var allPipelines = pipelineStorage.getAllPipelines(); LOG.info("Checking for orphaned pipelines..."); List orphanedPipelines = allPipelines .stream() diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java index e03d3291d1..49b218f738 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java @@ -18,6 +18,8 @@ package org.apache.streampipes.service.core; import org.apache.streampipes.config.backend.BackendConfig; +import org.apache.streampipes.connect.management.health.AdapterHealthCheck; +import org.apache.streampipes.manager.health.CoreServiceStatusManager; import org.apache.streampipes.manager.health.PipelineHealthCheck; import org.apache.streampipes.manager.health.ServiceHealthCheck; import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsServiceLogExecutor; @@ -30,6 +32,7 @@ import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory; import org.apache.streampipes.messaging.nats.SpNatsProtocolFactory; import org.apache.streampipes.messaging.pulsar.SpPulsarProtocolFactory; +import org.apache.streampipes.model.configuration.SpCoreConfigurationStatus; import org.apache.streampipes.model.pipeline.Pipeline; import org.apache.streampipes.model.pipeline.PipelineOperationStatus; import org.apache.streampipes.rest.security.SpPermissionEvaluator; @@ -37,6 +40,7 @@ import org.apache.streampipes.service.base.StreamPipesServiceBase; import org.apache.streampipes.service.core.migrations.MigrationsHandler; import org.apache.streampipes.storage.api.IPipelineStorage; +import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; import org.apache.streampipes.storage.couchdb.utils.CouchDbViewGenerator; import org.apache.streampipes.storage.management.StorageDispatcher; @@ -72,11 +76,13 @@ public class StreamPipesCoreApplication extends StreamPipesServiceBase { private static final int LOG_FETCH_INTERVAL = 60; private static final TimeUnit LOG_FETCH_UNIT = TimeUnit.SECONDS; - private static final int HEALTH_CHECK_INTERVAL = 60; + private static final int HEALTH_CHECK_INTERVAL = 30; private static final TimeUnit HEALTH_CHECK_UNIT = TimeUnit.SECONDS; - private static final int SERVICE_HEALTH_CHECK_INTERVAL = 60; - private static final TimeUnit SERVICE_HEALTH_CHECK_UNIT = TimeUnit.SECONDS; + private final ISpCoreConfigurationStorage coreConfigStorage = StorageDispatcher.INSTANCE + .getNoSqlStore().getSpCoreConfigurationStorage(); + + private final CoreServiceStatusManager coreStatusManager = new CoreServiceStatusManager(coreConfigStorage); public static void main(String[] args) { StreamPipesCoreApplication application = new StreamPipesCoreApplication(); @@ -109,9 +115,7 @@ protected void registerProtocols(SupportedProtocols protocols) { @PostConstruct public void init() { var executorService = Executors.newSingleThreadScheduledExecutor(); - var healthCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); var logCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); - var serviceHealthCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); new StreamPipesEnvChecker().updateEnvironmentVariables(); new CouchDbViewGenerator().createGenericDatabaseIfNotExists(); @@ -119,22 +123,20 @@ public void init() { if (!isConfigured()) { doInitialSetup(); } else { + // Check needs to be present since core configuration is part of migration + if (coreConfigStorage.exists()) { + coreStatusManager.updateCoreStatus(SpCoreConfigurationStatus.MIGRATING); + } new MigrationsHandler().performMigrations(); } + coreStatusManager.updateCoreStatus(SpCoreConfigurationStatus.READY); - executorService.schedule(new PostStartupTask(getAllPipelines()), 10, TimeUnit.SECONDS); + executorService.schedule(new PostStartupTask(getPipelineStorage()), 10, TimeUnit.SECONDS); - LOG.info("Service health check will run every {} seconds", SERVICE_HEALTH_CHECK_INTERVAL); - serviceHealthCheckExecutorService.scheduleAtFixedRate(new ServiceHealthCheck(), - SERVICE_HEALTH_CHECK_INTERVAL, - SERVICE_HEALTH_CHECK_INTERVAL, - SERVICE_HEALTH_CHECK_UNIT); - - LOG.info("Pipeline health check will run every {} seconds", HEALTH_CHECK_INTERVAL); - healthCheckExecutorService.scheduleAtFixedRate(new PipelineHealthCheck(), - HEALTH_CHECK_INTERVAL, - HEALTH_CHECK_INTERVAL, - HEALTH_CHECK_UNIT); + scheduleHealthChecks(List.of( + new ServiceHealthCheck(), + new PipelineHealthCheck(), + new AdapterHealthCheck())); LOG.info("Extensions logs will be fetched every {} seconds", LOG_FETCH_INTERVAL); logCheckExecutorService.scheduleAtFixedRate(new ExtensionsServiceLogExecutor(), @@ -143,6 +145,21 @@ public void init() { LOG_FETCH_UNIT); } + private void scheduleHealthChecks(List checks) { + var healthCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); + checks.forEach(check -> { + LOG.info( + "Health check {} configured to run every {} {}", + check.getClass().getCanonicalName(), + HEALTH_CHECK_INTERVAL, + HEALTH_CHECK_UNIT); + healthCheckExecutorService.scheduleAtFixedRate(check, + HEALTH_CHECK_INTERVAL, + HEALTH_CHECK_INTERVAL, + HEALTH_CHECK_UNIT); + }); + } + private boolean isConfigured() { return BackendConfig.INSTANCE.isConfigured(); } @@ -163,8 +180,6 @@ private void doInitialSetup() { } } - - @PreDestroy public void onExit() { LOG.info("Shutting down StreamPipes..."); diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/ConsulConfigMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/ConsulConfigMigration.java index 5da0d0bef7..53857b12ca 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/ConsulConfigMigration.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/ConsulConfigMigration.java @@ -22,6 +22,7 @@ import org.apache.streampipes.config.backend.BackendConfigKeys; import org.apache.streampipes.model.configuration.DefaultMessagingSettings; import org.apache.streampipes.model.configuration.SpCoreConfiguration; +import org.apache.streampipes.model.configuration.SpCoreConfigurationStatus; import org.apache.streampipes.service.core.migrations.Migration; import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; import org.apache.streampipes.storage.management.StorageDispatcher; @@ -71,6 +72,7 @@ public void executeMigration() { newConf.setFilesDir(currConf.getFilesDir()); newConf.setMessagingSettings(messagingSettings); + newConf.setServiceStatus(SpCoreConfigurationStatus.MIGRATING); storage.createElement(newConf); } diff --git a/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java b/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java index 8170c36ad9..0455dc1d8d 100644 --- a/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java +++ b/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java @@ -19,6 +19,7 @@ package org.apache.streampipes.svcdiscovery; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceStatus; import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.storage.management.StorageDispatcher; import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery; @@ -62,7 +63,7 @@ public List getServiceEndpoints(String serviceGroup, .stream() .filter(service -> allFiltersSupported(service, filterByTags)) .filter(service -> !restrictToHealthy - || service.isHealthy()) + || service.getStatus() != SpServiceStatus.UNHEALTHY) .map(this::makeServiceUrl) .collect(Collectors.toList()); } diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CoreRequestSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CoreRequestSubmitter.java new file mode 100644 index 0000000000..4ba4ed0807 --- /dev/null +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CoreRequestSubmitter.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.service.extensions; + +import org.apache.streampipes.commons.exceptions.SpRuntimeException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +public class CoreRequestSubmitter { + + private static final Logger LOG = LoggerFactory.getLogger(CoreRequestSubmitter.class); + + private static final int RETRY_INTERVAL_SECONDS = 3; + + public void submitRepeatedRequest(Supplier request, + Supplier successMessage, + Supplier failureMessage) { + try { + request.get(); + LOG.info(successMessage.get()); + } catch (SpRuntimeException e) { + LOG.warn( + failureMessage.get() + " Trying again in {} seconds", + RETRY_INTERVAL_SECONDS + ); + try { + TimeUnit.SECONDS.sleep(RETRY_INTERVAL_SECONDS); + submitRepeatedRequest(request, successMessage, failureMessage); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } + } +} diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java index cd2721007e..b990a90ebc 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java @@ -52,8 +52,13 @@ public void afterServiceRegistered(SpServiceDefinition serviceDef) { // register all migrations at StreamPipes Core var migrationConfigs = serviceDef.getMigrators().stream().map(ModelMigrator::config).toList(); - client.adminApi(). - registerMigrations(migrationConfigs, serviceId()); + new CoreRequestSubmitter().submitRepeatedRequest( + () -> { + client.adminApi().registerMigrations(migrationConfigs, serviceId()); + return true; + }, + () -> "Successfully sent migration request", + () -> "Core currently doesn't accept migration requests."); // initialize all function instances StreamPipesFunctionHandler.INSTANCE.initializeFunctions(serviceDef.getServiceGroup()); diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java index c13b1153a8..6570cd7023 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java @@ -19,7 +19,6 @@ package org.apache.streampipes.service.extensions; import org.apache.streampipes.client.StreamPipesClient; -import org.apache.streampipes.commons.exceptions.SpRuntimeException; import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver; import org.apache.streampipes.extensions.management.init.DeclarersSingleton; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; @@ -40,7 +39,6 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; public abstract class StreamPipesExtensionsServiceBase extends StreamPipesServiceBase { @@ -96,23 +94,17 @@ public void startExtensionsService(Class serviceClass, } private void registerService(SpServiceRegistration serviceRegistration) { - StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); - try { - client.adminApi().registerService(serviceRegistration); - LOG.info("Successfully registered service at core."); - } catch (SpRuntimeException e) { - LOG.warn( - "Could not register at core at url {}. Trying again in {} seconds", - client.getConnectionConfig().getBaseUrl(), - RETRY_INTERVAL_SECONDS - ); - try { - TimeUnit.SECONDS.sleep(RETRY_INTERVAL_SECONDS); - registerService(serviceRegistration); - } catch (InterruptedException ex) { - throw new RuntimeException(ex); - } - } + var client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); + new CoreRequestSubmitter().submitRepeatedRequest( + () -> { + client.adminApi().registerService(serviceRegistration); + return true; + }, + () -> "Successfully registered service at core.", + () -> String.format( + "Could not register service at core at url %s", + client.getConnectionConfig().getBaseUrl() + )); } protected List getServiceTags() { diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/ISpCoreConfigurationStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/ISpCoreConfigurationStorage.java index 829dc09ce9..2d0f8ae657 100644 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/ISpCoreConfigurationStorage.java +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/ISpCoreConfigurationStorage.java @@ -24,6 +24,8 @@ public interface ISpCoreConfigurationStorage { + boolean exists(); + List getAll(); void createElement(SpCoreConfiguration element); diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/CoreConfigurationStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/CoreConfigurationStorageImpl.java index 7ff0a99c04..9b25b5ac80 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/CoreConfigurationStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/CoreConfigurationStorageImpl.java @@ -33,6 +33,11 @@ public CoreConfigurationStorageImpl() { super(Utils::getCouchDbGeneralConfigStorage, SpCoreConfiguration.class); } + @Override + public boolean exists() { + return !findAll().isEmpty(); + } + @Override public List getAll() { return findAll(); From 17a070b51bde8cf3c7eba4de824f0d495193db9e Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Mon, 30 Oct 2023 22:46:19 +0100 Subject: [PATCH 47/54] Improve exception logging --- .../connect/management/health/AdapterHealthCheck.java | 4 ++-- .../management/management/WorkerRestClient.java | 11 ++--------- .../endpoint/ExtensionsServiceEndpointGenerator.java | 2 +- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java index b5bdc27394..e59187ea69 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java @@ -119,7 +119,7 @@ public Map getAdaptersToRecover( allRunningInstancesOfOneWorker.forEach(adapterDescription -> allRunningInstancesAdapterDescription.remove(adapterDescription.getElementId())); } catch (AdapterException e) { - e.printStackTrace(); + LOG.info("Could not recover adapter at endpoint {} due to {}", adapterEndpointUrl, e.getMessage()); } }); @@ -135,7 +135,7 @@ public void recoverAdapters(Map adaptersToRecover) { this.adapterMasterManagement.startStreamAdapter(adapterDescription.getElementId()); } } catch (AdapterException e) { - LOG.warn("Could not start adapter {}", adapterDescription.getName(), e); + LOG.warn("Could not start adapter {} ({})", adapterDescription.getName(), e.getMessage()); } } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java index a551201e29..c715459070 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java @@ -73,14 +73,12 @@ public static void stopStreamAdapter(String baseUrl, public static List getAllRunningAdapterInstanceDescriptions(String url) throws AdapterException { try { - LOG.info("Requesting all running adapter description instances: " + url); var responseString = ExtensionServiceExecutions .extServiceGetRequest(url) .execute().returnContent().asString(); return JacksonSerializer.getObjectMapper().readValue(responseString, List.class); } catch (IOException e) { - LOG.error("List of running adapters could not be fetched", e); throw new AdapterException("List of running adapters could not be fetched from: " + url); } } @@ -112,9 +110,6 @@ private static void triggerAdapterStateChange(AdapterDescription ad, var exception = getSerializer().readValue(responseString, AdapterException.class); throw new AdapterException(exception.getMessage(), exception.getCause()); } - - LOG.info("Adapter {} on endpoint: " + url + " with Response: ", ad.getName() + responseString); - } catch (IOException e) { LOG.error("Adapter was not {} successfully", action, e); throw new AdapterException("Adapter was not " + action + " successfully with url " + url, e); @@ -153,8 +148,7 @@ public static RuntimeOptionsResponse getConfiguration(String workerEndpoint, throw new SpConfigurationException(exception.getMessage(), exception.getCause()); } } catch (IOException e) { - e.printStackTrace(); - throw new AdapterException("Could not resolve runtime configurations from " + url); + throw new AdapterException("Could not resolve runtime configurations from " + url, e); } } @@ -178,11 +172,10 @@ public static byte[] getIconAsset(String baseUrl) throws AdapterException { String url = baseUrl + "/assets/icon"; try { - byte[] responseString = Request.Get(url) + return Request.Get(url) .connectTimeout(1000) .socketTimeout(100000) .execute().returnContent().asBytes(); - return responseString; } catch (IOException e) { LOG.error(e.getMessage()); throw new AdapterException("Could not get icon endpoint: " + url); diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java index 970d81a0d3..cb2d28408a 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java @@ -72,7 +72,7 @@ private List getServiceEndpoints() { private String selectService() throws NoServiceEndpointsAvailableException { List serviceEndpoints = getServiceEndpoints(); - if (serviceEndpoints.size() > 0) { + if (!serviceEndpoints.isEmpty()) { return getServiceEndpoints().get(0); } else { LOG.error("Could not find any service endpoints for appId {}, serviceTag {}", appId, From bcbcf1417b15fc7a17813dd3a1fdea4472aea800 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Mon, 30 Oct 2023 23:50:10 +0100 Subject: [PATCH 48/54] Improve pipeline health check --- .../manager/health/PipelineHealthCheck.java | 21 +-- .../PipelineElementMigrationManager.java | 143 ++++++++++-------- .../src/lib/model/gen/streampipes-model.ts | 13 +- ...istered-extensions-services.component.html | 7 +- 4 files changed, 104 insertions(+), 80 deletions(-) diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/PipelineHealthCheck.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/PipelineHealthCheck.java index d8064055a2..66a448be02 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/PipelineHealthCheck.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/PipelineHealthCheck.java @@ -45,6 +45,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import static org.apache.streampipes.manager.pipeline.PipelineManager.getPipeline; + public class PipelineHealthCheck implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(PipelineHealthCheck.class); @@ -68,7 +70,7 @@ public void checkAndRestorePipelineElements() { pipelinesStats.setRunningPipelines(runningPipelines.size()); pipelinesStats.setStoppedPipelines(pipelinesStats.getAllPipelines() - pipelinesStats.getRunningPipelines()); - if (runningPipelines.size() > 0) { + if (!runningPipelines.isEmpty()) { Map> endpointMap = generateEndpointMap(); List allRunningInstances = findRunningInstances(endpointMap.keySet()); @@ -115,15 +117,16 @@ public void checkAndRestorePipelineElements() { } }); if (shouldUpdatePipeline.get()) { - if (failedInstances.size() > 0) { - pipeline.setHealthStatus(PipelineHealthStatus.FAILURE); + var currentPipeline = getPipeline(pipeline.getPipelineId()); + if (!failedInstances.isEmpty()) { + currentPipeline.setHealthStatus(PipelineHealthStatus.FAILURE); pipelinesStats.failedIncrease(); - } else if (recoveredInstances.size() > 0) { - pipeline.setHealthStatus(PipelineHealthStatus.REQUIRES_ATTENTION); + } else if (!recoveredInstances.isEmpty()) { + currentPipeline.setHealthStatus(PipelineHealthStatus.REQUIRES_ATTENTION); pipelinesStats.attentionRequiredIncrease(); } - pipeline.setPipelineNotifications(pipelineNotifications); - StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineStorageAPI().updatePipeline(pipeline); + currentPipeline.setPipelineNotifications(pipelineNotifications); + StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineStorageAPI().updatePipeline(currentPipeline); } }); int healthNum = pipelinesStats.getRunningPipelines() - pipelinesStats.getFailedPipelines() @@ -233,13 +236,11 @@ private List getRunningPipelines(List allPipelines) { } private List getAllPipelines() { - List allPipelines = StorageDispatcher + return StorageDispatcher .INSTANCE .getNoSqlStore() .getPipelineStorageAPI() .getAllPipelines(); - - return allPipelines; } private int getElementsCount(List allPipelines){ diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/PipelineElementMigrationManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/PipelineElementMigrationManager.java index 28338e1ac0..166d24c8e3 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/PipelineElementMigrationManager.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/PipelineElementMigrationManager.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; import static org.apache.streampipes.manager.migration.MigrationUtils.getApplicableMigration; @@ -60,77 +61,89 @@ public PipelineElementMigrationManager(IPipelineStorage pipelineStorage, @Override public void handleMigrations(SpServiceRegistration extensionsServiceConfig, List migrationConfigs) { + if (!migrationConfigs.isEmpty()) { + LOG.info("Updating pipeline element descriptions by replacement..."); + updateDescriptions(migrationConfigs, extensionsServiceConfig.getServiceUrl()); + LOG.info("Pipeline element descriptions are up to date."); - LOG.info("Updating pipeline element descriptions by replacement..."); - updateDescriptions(migrationConfigs, extensionsServiceConfig.getServiceUrl()); - LOG.info("Pipeline element descriptions are up to date."); - - LOG.info("Received {} pipeline element migrations from extension service {}.", - migrationConfigs.size(), - extensionsServiceConfig.getServiceUrl()); - var availablePipelines = pipelineStorage.getAllPipelines(); - if (!availablePipelines.isEmpty()) { - LOG.info("Found {} available pipelines. Checking pipelines for applicable migrations...", - availablePipelines.size() - ); - } + LOG.info("Received {} pipeline element migrations from extension service {}.", + migrationConfigs.size(), + extensionsServiceConfig.getServiceUrl()); + var availablePipelines = pipelineStorage.getAllPipelines(); + if (!availablePipelines.isEmpty()) { + LOG.info("Found {} available pipelines. Checking pipelines for applicable migrations...", + availablePipelines.size() + ); + } - for (var pipeline : availablePipelines) { - List> failedMigrations = new ArrayList<>(); - - var migratedDataProcessors = pipeline.getSepas() - .stream() - .map(processor -> { - if (getApplicableMigration(processor, migrationConfigs).isPresent()) { - return migratePipelineElement( - processor, - migrationConfigs, - String.format("%s/%s/processor", - extensionsServiceConfig.getServiceUrl(), - MIGRATION_ENDPOINT - ), - failedMigrations - ); - } else { - LOG.info("No migration applicable for data processor '{}'.", processor.getElementId()); - return processor; - } - }) - .toList(); - pipeline.setSepas(migratedDataProcessors); - - var migratedDataSinks = pipeline.getActions() - .stream() - .map(sink -> { - if (getApplicableMigration(sink, migrationConfigs).isPresent()) { - return migratePipelineElement( - sink, - migrationConfigs, - String.format("%s/%s/sink", - extensionsServiceConfig.getServiceUrl(), - MIGRATION_ENDPOINT - ), - failedMigrations - ); - } else { - LOG.info("No migration applicable for data sink '{}'.", sink.getElementId()); - return sink; - } - }) - .toList(); - pipeline.setActions(migratedDataSinks); - - pipelineStorage.updatePipeline(pipeline); - - if (failedMigrations.isEmpty()) { - LOG.info("Migration for pipeline successfully completed."); - } else { - // pass most recent version of pipeline - handleFailedMigrations(pipelineStorage.getPipeline(pipeline.getPipelineId()), failedMigrations); + for (var pipeline : availablePipelines) { + if (shouldMigratePipeline(pipeline, migrationConfigs)) { + List> failedMigrations = new ArrayList<>(); + + var migratedDataProcessors = pipeline.getSepas() + .stream() + .map(processor -> { + if (getApplicableMigration(processor, migrationConfigs).isPresent()) { + return migratePipelineElement( + processor, + migrationConfigs, + String.format("%s/%s/processor", + extensionsServiceConfig.getServiceUrl(), + MIGRATION_ENDPOINT + ), + failedMigrations + ); + } else { + LOG.info("No migration applicable for data processor '{}'.", processor.getElementId()); + return processor; + } + }) + .toList(); + pipeline.setSepas(migratedDataProcessors); + + var migratedDataSinks = pipeline.getActions() + .stream() + .map(sink -> { + if (getApplicableMigration(sink, migrationConfigs).isPresent()) { + return migratePipelineElement( + sink, + migrationConfigs, + String.format("%s/%s/sink", + extensionsServiceConfig.getServiceUrl(), + MIGRATION_ENDPOINT + ), + failedMigrations + ); + } else { + LOG.info("No migration applicable for data sink '{}'.", sink.getElementId()); + return sink; + } + }) + .toList(); + pipeline.setActions(migratedDataSinks); + + pipelineStorage.updatePipeline(pipeline); + + if (failedMigrations.isEmpty()) { + LOG.info("Migration for pipeline successfully completed."); + } else { + // pass most recent version of pipeline + handleFailedMigrations(pipelineStorage.getPipeline(pipeline.getPipelineId()), failedMigrations); + } + } } + } else { + LOG.info("No pipeline element migrations to perform"); } } + private boolean shouldMigratePipeline(Pipeline pipeline, + List migrationConfigs) { + return Stream + .concat(pipeline.getSepas().stream(), pipeline.getActions().stream()) + .anyMatch(element -> getApplicableMigration(element, migrationConfigs).isPresent()); + } + /** * Takes care about the failed migrations of pipeline elements. * This includes the following steps: diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts index 47f06b81f2..85f9a230b6 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ + /* tslint:disable */ /* eslint-disable */ // @ts-nocheck -// Generated using typescript-generator version 3.2.1263 on 2023-10-27 10:43:45. +// Generated using typescript-generator version 3.2.1263 on 2023-10-30 22:49:29. export class NamedStreamPipesEntity { '@class': @@ -3599,12 +3600,12 @@ export class SpServiceConfiguration { export class SpServiceRegistration { firstTimeSeenUnhealthy: number; healthCheckPath: string; - healthy: boolean; host: string; port: number; rev: string; scheme: string; serviceUrl: string; + status: SpServiceStatus; svcGroup: string; svcId: string; svcType: string; @@ -3620,12 +3621,12 @@ export class SpServiceRegistration { const instance = target || new SpServiceRegistration(); instance.firstTimeSeenUnhealthy = data.firstTimeSeenUnhealthy; instance.healthCheckPath = data.healthCheckPath; - instance.healthy = data.healthy; instance.host = data.host; instance.port = data.port; instance.rev = data.rev; instance.scheme = data.scheme; instance.serviceUrl = data.serviceUrl; + instance.status = data.status; instance.svcGroup = data.svcGroup; instance.svcId = data.svcId; instance.svcType = data.svcType; @@ -4105,6 +4106,12 @@ export type SpProtocol = 'KAFKA' | 'JMS' | 'MQTT' | 'NATS' | 'PULSAR'; export type SpQueryStatus = 'OK' | 'TOO_MUCH_DATA'; +export type SpServiceStatus = + | 'REGISTERED' + | 'MIGRATING' + | 'HEALTHY' + | 'UNHEALTHY'; + export type SpServiceTagPrefix = | 'SYSTEM' | 'SP_GROUP' diff --git a/ui/src/app/configuration/extensions-service-management/registered-extensions-services/registered-extensions-services.component.html b/ui/src/app/configuration/extensions-service-management/registered-extensions-services/registered-extensions-services.component.html index 32f1534086..c0a1e74b5e 100644 --- a/ui/src/app/configuration/extensions-service-management/registered-extensions-services/registered-extensions-services.component.html +++ b/ui/src/app/configuration/extensions-service-management/registered-extensions-services/registered-extensions-services.component.html @@ -38,11 +38,14 @@ mat-cell *matCellDef="let element" > - + lens lens From 63cedf1aeab90bee2bd9d376723239eccec2a12c Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Tue, 31 Oct 2023 08:08:20 +0100 Subject: [PATCH 49/54] Properly execute migration of adapter models --- .../AdapterDescriptionMigration093.java | 73 +++++++++++++++++++ ...dapterDescriptionMigration093Provider.java | 45 ++++++++++++ .../rest/impl/admin/MigrationResource.java | 41 ++++++----- .../migrations/v093/AdapterMigration.java | 29 ++++---- 4 files changed, 155 insertions(+), 33 deletions(-) create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AdapterDescriptionMigration093.java create mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AdapterDescriptionMigration093Provider.java diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AdapterDescriptionMigration093.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AdapterDescriptionMigration093.java new file mode 100644 index 0000000000..d10d279993 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AdapterDescriptionMigration093.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.migration; + +import org.apache.streampipes.commons.exceptions.SepaParseException; +import org.apache.streampipes.manager.endpoint.HttpJsonParser; +import org.apache.streampipes.manager.operations.Operations; +import org.apache.streampipes.manager.util.AuthTokenUtils; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.storage.api.IAdapterStorage; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; + +import static org.apache.streampipes.manager.migration.MigrationUtils.getRequestUrl; + +public class AdapterDescriptionMigration093 extends AbstractMigrationManager { + + private static final Logger LOG = LoggerFactory.getLogger(AdapterDescriptionMigration093.class); + + private final IAdapterStorage adapterDescriptionStorage; + + public AdapterDescriptionMigration093(IAdapterStorage adapterDescriptionStorage) { + this.adapterDescriptionStorage = adapterDescriptionStorage; + } + + public void reinstallAdapters(SpServiceRegistration extensionsServiceConfig) { + var migrationProvider = AdapterDescriptionMigration093Provider.INSTANCE; + if (migrationProvider.hasAppIdsToReinstall()) { + var appIdsToReinstall = migrationProvider.getAppIdsToReinstall(); + var serviceUrl = extensionsServiceConfig.getServiceUrl(); + extensionsServiceConfig.getTags() + .stream() + .filter(tag -> tag.getPrefix() == SpServiceTagPrefix.ADAPTER) + .filter(tag -> appIdsToReinstall.contains(tag.getValue())) + .forEach(tag -> { + var appId = tag.getValue(); + try { + if (adapterDescriptionStorage.getAdaptersByAppId(appId).isEmpty()) { + var requestUrl = getRequestUrl(SpServiceTagPrefix.ADAPTER, appId, serviceUrl); + var entityPayload = HttpJsonParser.getContentFromUrl(URI.create(requestUrl)); + Operations.verifyAndAddElement( + entityPayload, + AuthTokenUtils.getAuthTokenForCurrentUser(), + true); + } + } catch (IOException | SepaParseException e) { + LOG.warn("Could not reinstall adapter description {}", appId); + } + }); + } + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AdapterDescriptionMigration093Provider.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AdapterDescriptionMigration093Provider.java new file mode 100644 index 0000000000..8ad025d5c4 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/AdapterDescriptionMigration093Provider.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.migration; + +import java.util.ArrayList; +import java.util.List; + +public enum AdapterDescriptionMigration093Provider { + + INSTANCE; + + private final List appIdsToReinstall; + + AdapterDescriptionMigration093Provider() { + this.appIdsToReinstall = new ArrayList<>(); + } + + public void addAppId(String appId) { + this.appIdsToReinstall.add(appId); + } + + public List getAppIdsToReinstall() { + return appIdsToReinstall; + } + + public boolean hasAppIdsToReinstall() { + return !appIdsToReinstall.isEmpty(); + } +} diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java index e715a1c02a..6b28afe0a3 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MigrationResource.java @@ -22,6 +22,7 @@ import org.apache.streampipes.connect.management.management.AdapterMigrationManager; import org.apache.streampipes.manager.health.CoreServiceStatusManager; import org.apache.streampipes.manager.health.ServiceRegistrationManager; +import org.apache.streampipes.manager.migration.AdapterDescriptionMigration093; import org.apache.streampipes.manager.migration.PipelineElementMigrationManager; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceStatus; @@ -64,6 +65,7 @@ public class MigrationResource extends AbstractAuthGuardedRestResource { private final CRUDStorage extensionsServiceStorage = getNoSqlStorage().getExtensionsServiceStorage(); + private final IAdapterStorage adapterDescriptionStorage = getNoSqlStorage().getAdapterDescriptionStorage(); private final IAdapterStorage adapterStorage = getNoSqlStorage().getAdapterInstanceStorage(); private final IDataProcessorStorage dataProcessorStorage = getNoSqlStorage().getDataProcessorStorage(); @@ -102,24 +104,27 @@ public Response performMigrations( var serviceManager = new ServiceRegistrationManager(extensionsServiceStorage); var extensionsServiceConfig = serviceManager.getService(serviceId); - if (!migrationConfigs.isEmpty() && BackendConfig.INSTANCE.isConfigured()) { - if (serviceManager.isAnyServiceMigrating() || !isCoreReady()) { - LOG.info("Refusing migration request since precondition is not met."); - return Response.status(HttpStatus.SC_CONFLICT).build(); - } else { - serviceManager.applyServiceStatus(serviceId, SpServiceStatus.MIGRATING); - var adapterMigrations = filterConfigs(migrationConfigs, List.of(SpServiceTagPrefix.ADAPTER)); - var pipelineElementMigrations = filterConfigs( - migrationConfigs, - List.of(SpServiceTagPrefix.DATA_PROCESSOR, SpServiceTagPrefix.DATA_SINK) - ); - - new AdapterMigrationManager(adapterStorage).handleMigrations(extensionsServiceConfig, adapterMigrations); - new PipelineElementMigrationManager( - pipelineStorage, - dataProcessorStorage, - dataSinkStorage) - .handleMigrations(extensionsServiceConfig, pipelineElementMigrations); + if (BackendConfig.INSTANCE.isConfigured()) { + new AdapterDescriptionMigration093(adapterDescriptionStorage).reinstallAdapters(extensionsServiceConfig); + if (!migrationConfigs.isEmpty()) { + if (serviceManager.isAnyServiceMigrating() || !isCoreReady()) { + LOG.info("Refusing migration request since precondition is not met."); + return Response.status(HttpStatus.SC_CONFLICT).build(); + } else { + serviceManager.applyServiceStatus(serviceId, SpServiceStatus.MIGRATING); + var adapterMigrations = filterConfigs(migrationConfigs, List.of(SpServiceTagPrefix.ADAPTER)); + var pipelineElementMigrations = filterConfigs( + migrationConfigs, + List.of(SpServiceTagPrefix.DATA_PROCESSOR, SpServiceTagPrefix.DATA_SINK) + ); + + new AdapterMigrationManager(adapterStorage).handleMigrations(extensionsServiceConfig, adapterMigrations); + new PipelineElementMigrationManager( + pipelineStorage, + dataProcessorStorage, + dataSinkStorage) + .handleMigrations(extensionsServiceConfig, pipelineElementMigrations); + } } } new ServiceRegistrationManager(extensionsServiceStorage) diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java index 94d2d7eecd..a3042dffd3 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java @@ -18,6 +18,7 @@ package org.apache.streampipes.service.core.migrations.v093; +import org.apache.streampipes.manager.migration.AdapterDescriptionMigration093Provider; import org.apache.streampipes.model.connect.adapter.migration.MigrationHelpers; import org.apache.streampipes.model.connect.adapter.migration.utils.AdapterModels; import org.apache.streampipes.service.core.migrations.Migration; @@ -35,7 +36,6 @@ import java.util.List; import static org.apache.streampipes.model.connect.adapter.migration.utils.AdapterModels.GENERIC_STREAM; -import static org.apache.streampipes.model.connect.adapter.migration.utils.AdapterModels.isSetAdapter; public class AdapterMigration implements Migration { @@ -45,7 +45,7 @@ public class AdapterMigration implements Migration { private final CouchDbClient adapterInstanceClient; private final CouchDbClient adapterDescriptionClient; private final List adaptersToMigrate; - private final List adapterDescriptionsToMigrate; + private final List adapterDescriptionsToRemove; private final MigrationHelpers helpers; @@ -54,7 +54,7 @@ public AdapterMigration() { this.adapterInstanceClient = Utils.getCouchDbAdapterInstanceClient(); this.adapterDescriptionClient = Utils.getCouchDbAdapterDescriptionClient(); this.adaptersToMigrate = new ArrayList<>(); - this.adapterDescriptionsToMigrate = new ArrayList<>(); + this.adapterDescriptionsToRemove = new ArrayList<>(); this.helpers = new MigrationHelpers(); } @@ -64,9 +64,9 @@ public boolean shouldExecute() { var adapterDescriptionUri = getAllDocsUri(adapterDescriptionClient); findDocsToMigrate(adapterInstanceClient, adapterInstanceUri, adaptersToMigrate); - findDocsToMigrate(adapterDescriptionClient, adapterDescriptionUri, adapterDescriptionsToMigrate); + findDocsToMigrate(adapterDescriptionClient, adapterDescriptionUri, adapterDescriptionsToRemove); - return !adaptersToMigrate.isEmpty() || !adapterDescriptionsToMigrate.isEmpty(); + return !adaptersToMigrate.isEmpty() || !adapterDescriptionsToRemove.isEmpty(); } private void findDocsToMigrate(CouchDbClient adapterClient, @@ -89,16 +89,15 @@ private void findDocsToMigrate(CouchDbClient adapterClient, public void executeMigration() { var adapterInstanceBackupClient = Utils.getCouchDbAdapterInstanceBackupClient(); - adapterDescriptionsToMigrate.forEach(ad -> { - var adapterType = ad.get("type").getAsString(); - var appId = ad.get("appId"); - if (isSetAdapter(adapterType)) { - LOG.info("Deleting adapter description data set {}", appId); - adapterDescriptionClient.remove(helpers.getDocId(ad), helpers.getRev(ad)); - } else { - LOG.info("Migrating adapter description {} to new adapter model", appId); - getAdapterMigrator(adapterType).migrate(adapterDescriptionClient, ad); - } + LOG.info("Deleting {} adapter descriptions, which will be regenerated after migration", + adapterDescriptionsToRemove.size()); + + adapterDescriptionsToRemove.forEach(ad -> { + String docId = helpers.getDocId(ad); + String rev = helpers.getRev(ad); + String appId = ad.get("appId").getAsString(); + AdapterDescriptionMigration093Provider.INSTANCE.addAppId(appId); + adapterDescriptionClient.remove(docId, rev); }); LOG.info("Migrating {} adapter models", adaptersToMigrate.size()); From 67e93db7d7994f15fba457b2f6a95453ddcca6ef Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Tue, 31 Oct 2023 08:34:47 +0100 Subject: [PATCH 50/54] Fix adapter model migration --- .../model/connect/adapter/migration/MigrationHelpers.java | 4 ++++ .../service/core/migrations/v093/AdapterMigration.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/migration/MigrationHelpers.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/migration/MigrationHelpers.java index d518086e9e..2343d41bb0 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/migration/MigrationHelpers.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/migration/MigrationHelpers.java @@ -40,6 +40,10 @@ public String getRev(JsonObject adapter) { return adapter.get(REV).getAsString(); } + public String getAppId(JsonObject adapter) { + return adapter.get("properties").getAsJsonObject().get(APP_ID).getAsString(); + } + public void updateType(JsonObject adapter, String typeFieldName) { adapter.add(typeFieldName, new JsonPrimitive(AdapterModels.NEW_MODEL)); diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java index a3042dffd3..98681b5100 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java @@ -95,7 +95,7 @@ public void executeMigration() { adapterDescriptionsToRemove.forEach(ad -> { String docId = helpers.getDocId(ad); String rev = helpers.getRev(ad); - String appId = ad.get("appId").getAsString(); + String appId = helpers.getAppId(ad); AdapterDescriptionMigration093Provider.INSTANCE.addAppId(appId); adapterDescriptionClient.remove(docId, rev); }); From e4ef56f8f1ff5a09a4ba10169e4f6b37551c1876 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Wed, 1 Nov 2023 19:18:48 +0100 Subject: [PATCH 51/54] Improve service health check --- .../manager/health/ServiceHealthCheck.java | 26 ++++++++----------- .../health/ServiceRegistrationManager.java | 19 ++++++++++++++ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java index 21121f358a..46d2792eaa 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java @@ -22,7 +22,6 @@ import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceStatus; -import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.storage.management.StorageDispatcher; import org.apache.http.HttpStatus; @@ -36,12 +35,13 @@ public class ServiceHealthCheck implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(ServiceHealthCheck.class); - private static final int MAX_UNHEALTHY_DURATION_BEFORE_REMOVAL_MS = 60000; + private static final int MAX_UNHEALTHY_DURATION_BEFORE_REMOVAL_MS = 20000; - private final CRUDStorage storage; + private final ServiceRegistrationManager serviceRegistrationManager; public ServiceHealthCheck() { - this.storage = StorageDispatcher.INSTANCE.getNoSqlStore().getExtensionsServiceStorage(); + var storage = StorageDispatcher.INSTANCE.getNoSqlStore().getExtensionsServiceStorage(); + this.serviceRegistrationManager = new ServiceRegistrationManager(storage); } @Override @@ -60,8 +60,7 @@ private void checkServiceHealth(SpServiceRegistration service) { processUnhealthyService(service); } else { if (service.getStatus() == SpServiceStatus.UNHEALTHY) { - service.setStatus(SpServiceStatus.HEALTHY); - updateService(service); + serviceRegistrationManager.applyServiceStatus(service.getSvcId(), SpServiceStatus.HEALTHY); } } } catch (IOException e) { @@ -71,14 +70,15 @@ private void checkServiceHealth(SpServiceRegistration service) { private void processUnhealthyService(SpServiceRegistration service) { if (service.getStatus() == SpServiceStatus.HEALTHY) { - service.setStatus(SpServiceStatus.UNHEALTHY); - service.setFirstTimeSeenUnhealthy(System.currentTimeMillis()); - updateService(service); + serviceRegistrationManager.applyServiceStatus( + service.getSvcId(), + SpServiceStatus.UNHEALTHY, + System.currentTimeMillis()); } if (shouldDeleteService(service)) { LOG.info("Removing service {} which has been unhealthy for more than {} seconds.", service.getSvcId(), MAX_UNHEALTHY_DURATION_BEFORE_REMOVAL_MS / 1000); - storage.deleteElement(service); + serviceRegistrationManager.removeService(service.getSvcId()); } } @@ -87,15 +87,11 @@ private boolean shouldDeleteService(SpServiceRegistration service) { return (currentTimeMillis - service.getFirstTimeSeenUnhealthy() > MAX_UNHEALTHY_DURATION_BEFORE_REMOVAL_MS); } - private void updateService(SpServiceRegistration service) { - storage.updateElement(service); - } - private String makeHealthCheckUrl(SpServiceRegistration service) { return service.getServiceUrl() + service.getHealthCheckPath(); } private List getRegisteredServices() { - return storage.getAll(); + return serviceRegistrationManager.getAllServices(); } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceRegistrationManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceRegistrationManager.java index cbae440a8d..f29b39d11e 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceRegistrationManager.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceRegistrationManager.java @@ -25,6 +25,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + public class ServiceRegistrationManager { private static final Logger LOG = LoggerFactory.getLogger(ServiceRegistrationManager.class); @@ -35,9 +37,22 @@ public ServiceRegistrationManager(CRUDStorage sto this.storage = storage; } + public void applyServiceStatus(String serviceId, + SpServiceStatus status, + long firstTimeSeenUnhealthy) { + var serviceRegistration = storage.getElementById(serviceId); + serviceRegistration.setFirstTimeSeenUnhealthy(firstTimeSeenUnhealthy); + applyServiceStatus(status, serviceRegistration); + } + public void applyServiceStatus(String serviceId, SpServiceStatus status) { var serviceRegistration = storage.getElementById(serviceId); + applyServiceStatus(status, serviceRegistration); + } + + private void applyServiceStatus(SpServiceStatus status, + SpServiceRegistration serviceRegistration) { serviceRegistration.setStatus(status); storage.updateElement(serviceRegistration); logService(serviceRegistration); @@ -50,6 +65,10 @@ public void addService(SpServiceRegistration serviceRegistration, logService(serviceRegistration); } + public List getAllServices() { + return storage.getAll(); + } + public SpServiceRegistration getService(String serviceId) { return storage.getElementById(serviceId); } From d2e8cc3fc760e116dfb8ebdbb8bda91cbd7fa495 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Wed, 1 Nov 2023 23:08:52 +0100 Subject: [PATCH 52/54] Add more checks to adapter migration --- .../service/core/migrations/v093/AdapterMigration.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java index 98681b5100..29fd205ab3 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/AdapterMigration.java @@ -36,6 +36,7 @@ import java.util.List; import static org.apache.streampipes.model.connect.adapter.migration.utils.AdapterModels.GENERIC_STREAM; +import static org.apache.streampipes.model.connect.adapter.migration.utils.AdapterModels.isSetAdapter; public class AdapterMigration implements Migration { @@ -94,10 +95,15 @@ public void executeMigration() { adapterDescriptionsToRemove.forEach(ad -> { String docId = helpers.getDocId(ad); + var adapterType = ad.get("type").getAsString(); String rev = helpers.getRev(ad); String appId = helpers.getAppId(ad); - AdapterDescriptionMigration093Provider.INSTANCE.addAppId(appId); - adapterDescriptionClient.remove(docId, rev); + if (!isSetAdapter(adapterType)) { + AdapterDescriptionMigration093Provider.INSTANCE.addAppId(appId); + } + if (docId != null && rev != null) { + adapterDescriptionClient.remove(docId, rev); + } }); LOG.info("Migrating {} adapter models", adaptersToMigrate.size()); From 3d1dfbb506e58ac0c9c999c06b7ad5ff59b6464d Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Thu, 2 Nov 2023 15:42:30 +0100 Subject: [PATCH 53/54] Simplify migration request handling --- .../service/extensions/CoreRequestSubmitter.java | 8 ++++---- .../service/extensions/ExtensionsModelSubmitter.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CoreRequestSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CoreRequestSubmitter.java index 4ba4ed0807..ab7f631d02 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CoreRequestSubmitter.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CoreRequestSubmitter.java @@ -33,14 +33,14 @@ public class CoreRequestSubmitter { private static final int RETRY_INTERVAL_SECONDS = 3; public void submitRepeatedRequest(Supplier request, - Supplier successMessage, - Supplier failureMessage) { + String successMessage, + String failureMessage) { try { request.get(); - LOG.info(successMessage.get()); + LOG.info(successMessage); } catch (SpRuntimeException e) { LOG.warn( - failureMessage.get() + " Trying again in {} seconds", + failureMessage + " Trying again in {} seconds", RETRY_INTERVAL_SECONDS ); try { diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java index 897075dd96..24f128cc47 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java @@ -57,8 +57,8 @@ public void afterServiceRegistered(SpServiceDefinition serviceDef) { client.adminApi().registerMigrations(migrationConfigs, serviceId()); return true; }, - () -> "Successfully sent migration request", - () -> "Core currently doesn't accept migration requests."); + "Successfully sent migration request", + "Core currently doesn't accept migration requests."); // initialize all function instances StreamPipesFunctionHandler.INSTANCE.initializeFunctions(serviceDef.getServiceGroup()); From 51accf430fce86c5b9299d356dddbe7846b6b0d7 Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Thu, 2 Nov 2023 16:46:47 +0100 Subject: [PATCH 54/54] Fix registration --- .../service/extensions/StreamPipesExtensionsServiceBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java index 6570cd7023..0913fd2587 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java @@ -100,8 +100,8 @@ private void registerService(SpServiceRegistration serviceRegistration) { client.adminApi().registerService(serviceRegistration); return true; }, - () -> "Successfully registered service at core.", - () -> String.format( + "Successfully registered service at core.", + String.format( "Could not register service at core at url %s", client.getConnectionConfig().getBaseUrl() ));