From 629c919fe16f353fef8fa3db01a2b3ea19647e6b Mon Sep 17 00:00:00 2001 From: Wareek Date: Fri, 14 Feb 2020 16:56:41 +0100 Subject: [PATCH 01/11] temporary commit, Kubernetes settings and service select options --- pom.xml | 8 + .../dto/CloudProviderEndpoint.java | 3 +- .../orchestrator/dto/cmdb/CloudService.java | 15 +- .../dto/cmdb/CloudServiceResolver.java | 4 + .../dto/cmdb/KubernetesService.java | 48 +++ .../enums/DeploymentProvider.java | 5 +- .../orchestrator/enums/DeploymentType.java | 5 +- .../CloudProviderEndpointServiceImpl.java | 4 + .../orchestrator/service/CmdbServiceImpl.java | 12 +- .../service/DeploymentServiceImpl.java | 9 +- .../commands/PrefilterCloudProviders.java | 11 +- .../service/commands/UpdateDeployment.java | 20 ++ .../providers/KubernetesServiceImpl.java | 316 ++++++++++++++++++ .../orchestrator/utils/ToscaConstants.java | 2 + .../tosca-definitions/custom_types.yaml | 23 ++ .../service/DeploymentServiceTest.java | 3 +- .../providers/KubernetesServiceTest.java | 209 ++++++++++++ src/test/resources/tosca/kubernetes_app.yaml | 36 ++ 18 files changed, 718 insertions(+), 15 deletions(-) create mode 100644 src/main/java/it/reply/orchestrator/dto/cmdb/KubernetesService.java create mode 100644 src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java create mode 100644 src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java create mode 100644 src/test/resources/tosca/kubernetes_app.yaml diff --git a/pom.xml b/pom.xml index a2669d9208..aa1a4e045c 100644 --- a/pom.xml +++ b/pom.xml @@ -95,6 +95,7 @@ 1.1.0 1.16.20 2.0.1 + 7.0.0 @@ -456,6 +457,13 @@ test + + io.kubernetes + client-java + ${kubernetes.version} + compile + + diff --git a/src/main/java/it/reply/orchestrator/dto/CloudProviderEndpoint.java b/src/main/java/it/reply/orchestrator/dto/CloudProviderEndpoint.java index 2d2e3e0004..19a9a312f2 100644 --- a/src/main/java/it/reply/orchestrator/dto/CloudProviderEndpoint.java +++ b/src/main/java/it/reply/orchestrator/dto/CloudProviderEndpoint.java @@ -51,7 +51,8 @@ public enum IaaSType { AZURE, CHRONOS, MARATHON, - QCG; + QCG, + KUBERNETES; } @Nullable diff --git a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java index e6c0c943e3..5ba27ae991 100644 --- a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java +++ b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java @@ -96,6 +96,7 @@ public class CloudService implements CmdbIdentifiable { private static final String COMPUTE_SERVICE_PREFIX = EGI_SERVICE_PREFIX + ".vm-management"; private static final String STORAGE_SERVICE_PREFIX = EGI_SERVICE_PREFIX + ".storage-management"; + private static final String DEEP_SERVICE_PREFIX = "eu.deep"; public static final String OPENSTACK_COMPUTE_SERVICE = "org.openstack.nova"; public static final String OPENNEBULA_COMPUTE_SERVICE = COMPUTE_SERVICE_PREFIX + ".opennebula"; @@ -112,7 +113,8 @@ public class CloudService implements CmdbIdentifiable { public static final String MARATHON_COMPUTE_SERVICE = INDIGO_SERVICE_PREFIX + ".marathon"; public static final String CHRONOS_COMPUTE_SERVICE = INDIGO_SERVICE_PREFIX + ".chronos"; - public static final String QCG_COMPUTE_SERVICE = "eu.deep.qcg"; + public static final String QCG_COMPUTE_SERVICE = DEEP_SERVICE_PREFIX + ".qcg"; + public static final String KUBERNETES_COMPUTE_SERVICE = DEEP_SERVICE_PREFIX + ".kubernetes"; /** * Get if the the service is a OpenStack compute service. @@ -234,9 +236,14 @@ public boolean isQcgComputeProviderService() { return QCG_COMPUTE_SERVICE.equals(this.serviceType); } + /** + * Get if the the service is a Kubernetes compute service. + * + * @return true if the service is a Kubernetes compute service + */ @JsonIgnore - public boolean isCredentialsRequired() { - return isAwsComputeProviderService() || isAzureComputeProviderService() - || isOtcComputeProviderService(); + public boolean isKubernetesComputeProviderService() { + return KUBERNETES_COMPUTE_SERVICE.equals(this.serviceType); } + } diff --git a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java index 3cb6777eb3..1b8463d4b5 100644 --- a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java +++ b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java @@ -26,6 +26,7 @@ import static it.reply.orchestrator.dto.cmdb.CloudService.OPENSTACK_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.OTC_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.QCG_COMPUTE_SERVICE; +import static it.reply.orchestrator.dto.cmdb.CloudService.KUBERNETES_COMPUTE_SERVICE; import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; import com.fasterxml.jackson.databind.DatabindContext; @@ -64,6 +65,9 @@ public JavaType typeFromId(DatabindContext context, String id) { case QCG_COMPUTE_SERVICE: subType = QcgService.class; break; + case KUBERNETES_COMPUTE_SERVICE: + subType = KubernetesService.class; + break; case OCCI_COMPUTE_SERVICE: case OPENNEBULA_COMPUTE_SERVICE: case OPENNEBULA_TOSCA_SERVICE: diff --git a/src/main/java/it/reply/orchestrator/dto/cmdb/KubernetesService.java b/src/main/java/it/reply/orchestrator/dto/cmdb/KubernetesService.java new file mode 100644 index 0000000000..3fda33094c --- /dev/null +++ b/src/main/java/it/reply/orchestrator/dto/cmdb/KubernetesService.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.dto.cmdb; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class KubernetesService extends CloudService { + + @Builder(builderMethodName = "kubernetesBuilder") + public KubernetesService( + @NonNull String id, + @NonNull String serviceType, + @NonNull String endpoint, + @NonNull String providerId, + @NonNull CloudServiceType type, + boolean publicService, + @Nullable String region, + @NonNull String hostname, + @Nullable String parentServiceId, + boolean iamEnabled) { + super(id, serviceType, endpoint, providerId, type, publicService, region, hostname, + parentServiceId, iamEnabled); + } +} diff --git a/src/main/java/it/reply/orchestrator/enums/DeploymentProvider.java b/src/main/java/it/reply/orchestrator/enums/DeploymentProvider.java index 5dbfc7fb11..eb927b73bf 100644 --- a/src/main/java/it/reply/orchestrator/enums/DeploymentProvider.java +++ b/src/main/java/it/reply/orchestrator/enums/DeploymentProvider.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ public enum DeploymentProvider { HEAT, CHRONOS, MARATHON, - QCG; + QCG, + KUBERNETES; } diff --git a/src/main/java/it/reply/orchestrator/enums/DeploymentType.java b/src/main/java/it/reply/orchestrator/enums/DeploymentType.java index a2287ffecc..e186a76002 100644 --- a/src/main/java/it/reply/orchestrator/enums/DeploymentType.java +++ b/src/main/java/it/reply/orchestrator/enums/DeploymentType.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,8 @@ public enum DeploymentType { CHRONOS, MARATHON, TOSCA, - QCG; + QCG, + KUBERNETES; public static boolean isMesosDeployment(DeploymentType deploymentType) { return deploymentType == CHRONOS || deploymentType == DeploymentType.MARATHON; diff --git a/src/main/java/it/reply/orchestrator/service/CloudProviderEndpointServiceImpl.java b/src/main/java/it/reply/orchestrator/service/CloudProviderEndpointServiceImpl.java index 18b4396d2a..56e70da456 100644 --- a/src/main/java/it/reply/orchestrator/service/CloudProviderEndpointServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/CloudProviderEndpointServiceImpl.java @@ -123,6 +123,8 @@ public CloudProviderEndpoint getCloudProviderEndpoint(CloudService computeServic iaasType = IaaSType.MARATHON; } else if (computeService.isQcgComputeProviderService()) { iaasType = IaaSType.QCG; + } else if (computeService.isKubernetesComputeProviderService()) { + iaasType = IaaSType.KUBERNETES; } else { throw new IllegalArgumentException("Unknown Cloud Provider type: " + computeService); } @@ -164,6 +166,8 @@ public DeploymentProvider getDeploymentProvider(DeploymentType deploymentType, return DeploymentProvider.QCG; case TOSCA: return DeploymentProvider.IM; + case KUBERNETES: + return DeploymentProvider.KUBERNETES; default: throw new DeploymentException("Unknown DeploymentType: " + deploymentType.toString()); } diff --git a/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java b/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java index 84d899b948..2d264a79eb 100644 --- a/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java @@ -24,6 +24,7 @@ import it.reply.orchestrator.dto.cmdb.ComputeService; import it.reply.orchestrator.dto.cmdb.Flavor; import it.reply.orchestrator.dto.cmdb.Image; +import it.reply.orchestrator.dto.cmdb.KubernetesService; import it.reply.orchestrator.dto.cmdb.Tenant; import it.reply.orchestrator.dto.cmdb.wrappers.CmdbDataWrapper; import it.reply.orchestrator.dto.cmdb.wrappers.CmdbHasManyList; @@ -286,7 +287,16 @@ public CloudProvider fillCloudProviderInfo(String providerId, return cloudService; }) .collect(Collectors.toMap(CloudService::getId, Function.identity())); - + +// services.put(providerId+"_kuberServiceId123", KubernetesService.kubernetesBuilder() +// .id(providerId+"kuberServiceId123") +// .providerId(providerId) +// .serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) +// .type(CloudServiceType.COMPUTE) +// .hostname("localhost:8001") +// .endpoint("localhost:8001") +// .build()); + provider.setServices(services); return provider; } diff --git a/src/main/java/it/reply/orchestrator/service/DeploymentServiceImpl.java b/src/main/java/it/reply/orchestrator/service/DeploymentServiceImpl.java index 0422b472c9..f541f51046 100644 --- a/src/main/java/it/reply/orchestrator/service/DeploymentServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/DeploymentServiceImpl.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -280,6 +280,8 @@ private DeploymentType inferDeploymentType(Map nodes) { return DeploymentType.MARATHON; } else if (toscaService.isOfToscaType(node, ToscaConstants.Nodes.Types.QCG)) { return DeploymentType.QCG; + } else if (toscaService.isOfToscaType(node, ToscaConstants.Nodes.Types.KUBERNETES)) { + return DeploymentType.KUBERNETES; } } return DeploymentType.TOSCA; @@ -293,6 +295,8 @@ private static DeploymentType inferDeploymentType(DeploymentProvider deploymentP return DeploymentType.MARATHON; case QCG: return DeploymentType.QCG; + case KUBERNETES: + return DeploymentType.KUBERNETES; case HEAT: case IM: default: @@ -361,7 +365,8 @@ public void updateDeployment(String id, DeploymentRequest request) { if (deployment.getDeploymentProvider() == DeploymentProvider.CHRONOS || deployment.getDeploymentProvider() == DeploymentProvider.MARATHON - || deployment.getDeploymentProvider() == DeploymentProvider.QCG) { + || deployment.getDeploymentProvider() == DeploymentProvider.QCG + || deployment.getDeploymentProvider() == DeploymentProvider.KUBERNETES) { throw new BadRequestException(String.format("%s deployments cannot be updated.", deployment.getDeploymentProvider().toString())); } diff --git a/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java b/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java index 4daace1d40..ba9a4405a1 100644 --- a/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java +++ b/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java @@ -25,6 +25,7 @@ import it.reply.orchestrator.dto.cmdb.CloudService; import it.reply.orchestrator.dto.cmdb.CloudServiceType; import it.reply.orchestrator.dto.cmdb.ComputeService; +import it.reply.orchestrator.dto.cmdb.KubernetesService; import it.reply.orchestrator.dto.cmdb.MarathonService; import it.reply.orchestrator.dto.cmdb.MesosFrameworkService; import it.reply.orchestrator.dto.cmdb.QcgService; @@ -143,6 +144,13 @@ public void execute(DelegateExecution execution, addServiceToDiscard(servicesToDiscard, cloudProviderService); } break; + case KUBERNETES: + if ((cloudProviderService instanceof KubernetesService)) { + KubernetesService kubernetesService = (KubernetesService) cloudProviderService; + } else { + addServiceToDiscard(servicesToDiscard, cloudProviderService); + } + break; default: throw new DeploymentException("Unknown Deployment Type: " + type); } @@ -243,8 +251,7 @@ private void discardOnPlacementPolicies(Map placementPolici .stream() .flatMap(policy -> policy.getServicesId().stream()) .anyMatch(serviceId -> serviceId.equals(cloudService.getId())); - boolean credentialsRequired = cloudService.isCredentialsRequired(); - if (!serviceIsInSlaPolicy && (slaPlacementRequired || credentialsRequired)) { + if (!serviceIsInSlaPolicy && slaPlacementRequired) { LOG.debug( "Discarded service {} of provider {} because it doesn't match SLA policies", cloudService.getId(), cloudProvider.getId()); diff --git a/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java b/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java index 8a5358b9e9..ba2ef1d44d 100644 --- a/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java +++ b/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java @@ -20,9 +20,12 @@ import it.reply.orchestrator.dto.CloudProviderEndpoint; import it.reply.orchestrator.dto.RankCloudProvidersMessage; import it.reply.orchestrator.dto.cmdb.CloudService; +import it.reply.orchestrator.dto.cmdb.CloudServiceType; +import it.reply.orchestrator.dto.cmdb.KubernetesService; import it.reply.orchestrator.dto.deployment.DeploymentMessage; import it.reply.orchestrator.dto.onedata.OneData; import it.reply.orchestrator.dto.onedata.OneData.OneDataProviderInfo; +import it.reply.orchestrator.dto.workflow.CloudServiceWf; import it.reply.orchestrator.dto.workflow.CloudServicesOrderedIterator; import it.reply.orchestrator.enums.DeploymentProvider; import it.reply.orchestrator.enums.Status; @@ -33,6 +36,7 @@ import it.reply.orchestrator.utils.WorkflowConstants; import it.reply.orchestrator.utils.WorkflowConstants.ErrorCode; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -46,6 +50,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import antlr.collections.List; + /** * Choose Cloud Provider and update Deployment/Message with the selected one data. */ @@ -66,6 +72,20 @@ public void execute(DelegateExecution execution, DeploymentMessage deploymentMes RankCloudProvidersMessage.class); CloudServicesOrderedIterator servicesIt = deploymentMessage.getCloudServicesOrderedIterator(); + +// ArrayList k = new ArrayList(); +// +// k.add(KubernetesService.kubernetesBuilder() +// .id("localhost_kuberServiceId123") +// .providerId("localhost") +// .serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) +// .type(CloudServiceType.COMPUTE) +// .hostname("localhost:8001") +// .endpoint("localhost:8001") +// .build()); +// +// servicesIt = new CloudServicesOrderedIterator(k); + if (servicesIt == null) { servicesIt = cloudProviderEndpointService .generateCloudProvidersOrderedIterator(rankCloudProvidersMessage, diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java new file mode 100644 index 0000000000..2fd325c40e --- /dev/null +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -0,0 +1,316 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.service.deployment.providers; + +import io.kubernetes.client.custom.IntOrString; +import io.kubernetes.client.openapi.ApiCallback; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.Configuration; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1APIService; +import io.kubernetes.client.openapi.apis.BatchV1Api; +import io.kubernetes.client.openapi.apis.AppsV1Api; +import io.kubernetes.client.openapi.models.V1Container; +import io.kubernetes.client.openapi.models.V1ContainerBuilder; +import io.kubernetes.client.openapi.models.V1ContainerPort; +import io.kubernetes.client.openapi.models.V1DeleteOptions; +import io.kubernetes.client.openapi.models.V1Deployment; +import io.kubernetes.client.openapi.models.V1DeploymentBuilder; +import io.kubernetes.client.openapi.models.V1DeploymentCondition; +import io.kubernetes.client.openapi.models.V1DeploymentSpec; +import io.kubernetes.client.openapi.models.V1DeploymentSpecBuilder; +import io.kubernetes.client.openapi.models.V1DeploymentStatus; +import io.kubernetes.client.openapi.models.V1DeploymentStatusBuilder; +import io.kubernetes.client.openapi.models.V1EnvVar; +import io.kubernetes.client.openapi.models.V1Job; +import io.kubernetes.client.openapi.models.V1JobBuilder; +import io.kubernetes.client.openapi.models.V1Namespace; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.kubernetes.client.openapi.models.V1ObjectMetaBuilder; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodBuilder; +import io.kubernetes.client.openapi.models.V1PodList; +import io.kubernetes.client.openapi.models.V1ResourceRequirements; +import io.kubernetes.client.openapi.models.V1ResourceRequirementsBuilder; +import io.kubernetes.client.openapi.models.V1Service; +import io.kubernetes.client.openapi.models.V1ServiceBuilder; +import io.kubernetes.client.openapi.models.V1Status; +import io.kubernetes.client.util.exception.ObjectMetaReflectException; + +import io.kubernetes.client.util.Config; +import io.kubernetes.client.util.Yaml; +import io.kubernetes.client.util.exception.ObjectMetaReflectException; + +import com.google.common.base.Strings; + +import es.upv.i3m.grycap.im.InfrastructureManager; +import es.upv.i3m.grycap.im.exceptions.ImClientErrorException; +import es.upv.i3m.grycap.im.exceptions.ImClientException; +import es.upv.i3m.grycap.im.pojo.Property; + +import it.reply.orchestrator.annotation.DeploymentProviderQualifier; +import it.reply.orchestrator.dal.entity.Deployment; +import it.reply.orchestrator.dal.entity.OidcTokenId; +import it.reply.orchestrator.dto.CloudProviderEndpoint; +import it.reply.orchestrator.dto.deployment.DeploymentMessage; +import it.reply.orchestrator.enums.DeploymentProvider; +import it.reply.orchestrator.enums.Task; +import it.reply.orchestrator.exception.service.BusinessWorkflowException; +import it.reply.orchestrator.exception.service.DeploymentException; +import it.reply.orchestrator.function.ThrowingFunction; +import it.reply.orchestrator.service.ToscaService; +import it.reply.orchestrator.service.VaultService; +import it.reply.orchestrator.service.security.OAuth2TokenService; +import it.reply.orchestrator.utils.WorkflowConstants.ErrorCode; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import lombok.extern.slf4j.Slf4j; + +import mesosphere.marathon.client.model.v2.App; +import mesosphere.marathon.client.model.v2.Group; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +@DeploymentProviderQualifier(DeploymentProvider.KUBERNETES) +@Slf4j +public class KubernetesServiceImpl extends AbstractDeploymentProviderService { + + @Autowired + private ToscaService toscaService; + + @Autowired + private OAuth2TokenService oauth2TokenService; + + @Autowired + private VaultService vaultService; + + private void test(Deployment deployment) throws IOException, ApiException { + + // access configuration TODO + ApiClient client = Config.defaultClient(); + Configuration.setDefaultApiClient(client); + AppsV1Api app = new AppsV1Api(client); + //-- + + deployment.getCallback(); + String name = deployment.getId(); + + V1Deployment v1Deployment = new V1DeploymentBuilder() + .withApiVersion("apps/v1") + .withKind("Deployment") + .withNewMetadata() + .withName(name) + .endMetadata() + .withNewSpec() + .withReplicas((Integer) deployment.getParameters().get("replicas")) + .withNewSelector() + .addToMatchLabels("app", "nginx") + .endSelector() + .withNewTemplate() + .withNewMetadata() + .addToLabels("app", "nginx") + .endMetadata() + .withNewSpec() + .addNewContainer() + .withImage("nginx:1.7.9") + .withName("nginx") + .addNewPort() + .withContainerPort(80) + .endPort() + .endContainer() + .endSpec() + .endTemplate() + .endSpec() + .build(); + + System.out.println(Yaml.dump(v1Deployment)); + + V1Deployment outDepl = app.createNamespacedDeployment("default", v1Deployment, "true", null, null); + String ref = outDepl.getMetadata().getName(); + + outDepl.getStatus(); + + printPodList(); + } + + @Override + public boolean doDeploy(DeploymentMessage deploymentMessage) { + Deployment deployment = getDeployment(deploymentMessage); + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + + // Update status of the deployment - if not already done (remember the Iterative + // mode) + if (deployment.getTask() != Task.DEPLOYER) { + deployment.setTask(Task.DEPLOYER); + } + if (deployment.getEndpoint() == null) { + deployment.setEndpoint(""); + } + +// test(deployment); + doUndeploy(deploymentMessage); + + LOG.info("Creating Kubernetes App Group for deployment {} with definition:\n{}", deployment.getId(), "x"); + // deployment.getId(), group); + CloudProviderEndpoint cloudProviderEndpoint = deployment.getCloudProviderEndpoint(); + vaultService.getServiceUri() + .map(URI::toString) + .ifPresent(cloudProviderEndpoint::setVaultEndpoint); +// executeWithClient(cloudProviderEndpoint, requestedWithToken, +// client -> client.createGroup(group)); + return true; + } + + @Override + public boolean isDeployed(DeploymentMessage deploymentMessage) { + // TODO Auto-generated method stub + Deployment deployment = getDeployment(deploymentMessage); + + AppsV1Api app; + V1Deployment v1Deployment = new V1Deployment(); + CoreV1Api api; + try { + app = new AppsV1Api(Config.defaultClient()); + api = new CoreV1Api(); + V1Namespace namespace = api.readNamespaceStatus("default", null); + v1Deployment = app.readNamespacedDeploymentStatus("nginx-deployment-5754944d6c-c8kpv", "default", "true"); + + printPodList(); + } catch (ApiException e) { + LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + } catch (IOException e) { + LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + } + + boolean isDeployed = v1Deployment.getStatus().equals(V1DeploymentStatus.SERIALIZED_NAME_READY_REPLICAS); + LOG.debug("Kubernetes App Group for deployment {} is deployed? {}", deployment.getId(), + isDeployed); + return isDeployed; + } + + @Override + public void cleanFailedDeploy(DeploymentMessage deploymentMessage) { + doUndeploy(deploymentMessage); + } + + @Override + public boolean doUpdate(DeploymentMessage deploymentMessage, String template) { + // TODO Auto-generated method stub + Deployment deployment = getDeployment(deploymentMessage); + + V1Deployment v1Deployment = new V1DeploymentBuilder() + .withNewMetadata() + .withName(deployment.getId()) + .endMetadata() + .build(); + AppsV1Api app; + CoreV1Api api; + try { + app = new AppsV1Api(Config.defaultClient()); + api = new CoreV1Api(); + + app.patchNamespacedDeployment(deployment.getId(), "default", v1Deployment, "true", null, null, null); +// app.replaceNamespacedDeployment(name, namespace, body, pretty, dryRun, fieldManager); + + } catch (ApiException e) { + LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + } catch (IOException e) { + LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + } + throw new UnsupportedOperationException("Marathon app deployments do not support update."); + } + + @Override + public void cleanFailedUpdate(DeploymentMessage deploymentMessage) { + doUndeploy(deploymentMessage); + } + + @Override + public boolean doUndeploy(DeploymentMessage deploymentMessage) { + + Deployment deployment = getDeployment(deploymentMessage); + + V1Deployment v1Deployment = new V1Deployment(); + AppsV1Api app; + CoreV1Api api; + try { + app = new AppsV1Api(Config.defaultClient()); + api = new CoreV1Api(); + + app.deleteNamespacedDeployment("nginx-deployment-5754944d6c", "default", "true", null, null, null, null, null); +// app.deleteCollectionNamespacedDeployment(namespace, pretty, allowWatchBookmarks, _continue, dryRun, fieldSelector, gracePeriodSeconds, labelSelector, limit, orphanDependents, propagationPolicy, resourceVersion, timeoutSeconds, watch, body) +// api.deleteCollectionNamespacedPod(namespace, pretty, allowWatchBookmarks, _continue, dryRun, fieldSelector, gracePeriodSeconds, labelSelector, limit, orphanDependents, propagationPolicy, resourceVersion, timeoutSeconds, watch, body) + } catch (ApiException e) { + LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); +// if(e.getCode()!=404) { +// throw new HttpResponseException(e.getCode(), "KubernetesApiException"); +// } + } catch (IOException e) { + LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + } + return true; + } + + @Override + public boolean isUndeployed(DeploymentMessage deploymentMessage) { + // TODO Auto-generated method stub + return false; + } + + @Override + public void doProviderTimeout(DeploymentMessage deploymentMessage) { + throw new BusinessWorkflowException(ErrorCode.CLOUD_PROVIDER_ERROR, + "Error executing request to Kubernetes", + new DeploymentException("Kubernetes provider timeout during deployment")); + } + + @Override + protected Optional getAdditionalErrorInfoInternal(DeploymentMessage deploymentMessage) { + Deployment deployment = getDeployment(deploymentMessage); + + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + + List cloudProviderEndpoints = + deployment.getCloudProviderEndpoint().getAllCloudProviderEndpoint(); + //TODO + return Optional.empty(); + } + + private void printPodList() throws ApiException { + CoreV1Api api = new CoreV1Api(); + V1PodList list = api.listPodForAllNamespaces(null, null, null, null, null, null, null, null, null); + for (V1Pod item : list.getItems()) { + System.out.println(item.getMetadata().getName()); + } + } + +} diff --git a/src/main/java/it/reply/orchestrator/utils/ToscaConstants.java b/src/main/java/it/reply/orchestrator/utils/ToscaConstants.java index 7c5f3bd7c0..af1f8bb133 100644 --- a/src/main/java/it/reply/orchestrator/utils/ToscaConstants.java +++ b/src/main/java/it/reply/orchestrator/utils/ToscaConstants.java @@ -32,6 +32,8 @@ public static class Types { BASE_INDIGO_NAME + "Container.Application.Docker.Chronos"; public static final String MARATHON = BASE_INDIGO_NAME + "Container.Application.Docker.Marathon"; + public static final String KUBERNETES = + BASE_INDIGO_NAME + "Container.Application.Docker.Kubernetes"; public static final String COMPUTE = BASE_INDIGO_NAME + "Compute"; public static final String QCG = BASE_INDIGO_NAME + "Qcg.Job"; public static final String ELASTIC_CLUSTER = BASE_INDIGO_NAME + "ElasticCluster"; diff --git a/src/main/resources/tosca-definitions/custom_types.yaml b/src/main/resources/tosca-definitions/custom_types.yaml index 1973e6d501..72bea5955a 100644 --- a/src/main/resources/tosca-definitions/custom_types.yaml +++ b/src/main/resources/tosca-definitions/custom_types.yaml @@ -2349,6 +2349,29 @@ node_types: kube_admin_username: { get_property: [ SELF, admin_username] } kube_admin_token: { get_property: [ SELF, admin_token] } + tosca.nodes.indigo.Container.Application.Docker.Kubernetes: + derived_from: tosca.nodes.indigo.Container.Application.Docker + metadata: + icon: /images/kubernetesWN.png + attributes: + some_attrib: + type: list + entry_schema: string + properties: + secrets: + entry_schema: + type: string + required: no + type: map + replicas: + type: integer + default: 1 + required: no + container_port: + type: integer + description: container port + default: 80 + tosca.nodes.indigo.JupyterHub: derived_from: tosca.nodes.SoftwareComponent metadata: diff --git a/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java b/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java index 4d3d44d5c1..c64b26ee16 100644 --- a/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java @@ -506,7 +506,8 @@ public void deleteDeploymentSuccesfulWithProvider(Status status) throws Exceptio @Parameters({ "CHRONOS", "MARATHON", - "QCG"}) + "QCG", + "KUBERNETES"}) public void updateDeploymentBadRequest(DeploymentProvider provider) throws Exception { String id = UUID.randomUUID().toString(); diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java new file mode 100644 index 0000000000..0833435631 --- /dev/null +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -0,0 +1,209 @@ +package it.reply.orchestrator.service.deployment.providers; + +import com.google.common.collect.Lists; + +import it.reply.orchestrator.config.specific.ToscaParserAwareTest; +import it.reply.orchestrator.controller.ControllerTestUtils; +import it.reply.orchestrator.dal.entity.Deployment; +import it.reply.orchestrator.dal.entity.Resource; +import it.reply.orchestrator.dal.repository.DeploymentRepository; +import it.reply.orchestrator.dal.repository.ResourceRepository; +import it.reply.orchestrator.dto.CloudProviderEndpoint; +import it.reply.orchestrator.dto.CloudProviderEndpoint.IaaSType; +import it.reply.orchestrator.dto.cmdb.CloudService; +import it.reply.orchestrator.dto.cmdb.CloudServiceType; +import it.reply.orchestrator.dto.cmdb.KubernetesService; +import it.reply.orchestrator.dto.deployment.DeploymentMessage; +import it.reply.orchestrator.dto.onedata.OneData; +import it.reply.orchestrator.dto.onedata.OneData.OneDataProviderInfo; +import it.reply.orchestrator.dto.workflow.CloudServicesOrderedIterator; +import it.reply.orchestrator.enums.NodeStates; +import it.reply.orchestrator.function.ThrowingFunction; +import it.reply.orchestrator.service.ToscaService; +import it.reply.orchestrator.service.ToscaServiceTest; +import it.reply.orchestrator.service.VaultService; +import it.reply.orchestrator.util.TestUtil; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +import org.assertj.core.api.Assertions; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; + +import junitparams.JUnitParamsRunner; + +@RunWith(JUnitParamsRunner.class) +public class KubernetesServiceTest extends ToscaParserAwareTest { + + @InjectMocks + private KubernetesServiceImpl kubernetesServiceImpl; + + @SpyBean + @Autowired + protected ToscaService toscaService; + + @MockBean + private ResourceRepository resourceRepository; + + @MockBean + private DeploymentRepository deploymentRepository; + + @MockBean + private VaultService vaultService; + + private static final String defaultVaultEndpoint = "https://default.vault.com:8200"; + + @Before + public void setup() throws Exception { + MockitoAnnotations.initMocks(this); + Mockito + .when(oauth2tokenService.executeWithClientForResult( + Mockito.any(), Mockito.any(), Mockito.any())) + .thenAnswer(y -> ((ThrowingFunction) y.getArguments()[1]).apply("token")); + } + + @Test + public void testDoDeploy() throws IOException, URISyntaxException { + Deployment deployment = generateDeployment(); + DeploymentMessage dm = generateDeployDm(deployment); + + KubernetesService cs = buildService(); + + CloudServicesOrderedIterator csi = new CloudServicesOrderedIterator(Lists.newArrayList(cs)); + csi.next(); + dm.setCloudServicesOrderedIterator(csi); + + Mockito + .when(deploymentRepository.findOne(deployment.getId())) + .thenReturn(deployment); + Mockito + .when(vaultService.getServiceUri()) + .thenReturn(Optional.of(new URI(defaultVaultEndpoint))); + Assertions + .assertThat(kubernetesServiceImpl.doDeploy(dm)) + .isTrue(); + } + + private DeploymentMessage generateDeployDmKuber(Deployment deployment) { + DeploymentMessage dm = new DeploymentMessage(); + dm.setDeploymentId(deployment.getId()); + CloudProviderEndpoint chosenCloudProviderEndpoint = CloudProviderEndpoint + .builder() + .cpComputeServiceId(UUID.randomUUID().toString()) + .cpEndpoint("http://www.example.com/api") + .iaasType(IaaSType.KUBERNETES) + .build(); + dm.setChosenCloudProviderEndpoint(chosenCloudProviderEndpoint); + deployment.setCloudProviderEndpoint(chosenCloudProviderEndpoint); + return dm; + } + + private DeploymentMessage generateDeployDm(Deployment deployment) { + DeploymentMessage dm = new DeploymentMessage(); + dm.setDeploymentId(deployment.getId()); + CloudProviderEndpoint chosenCloudProviderEndpoint = CloudProviderEndpoint + .builder() + .cpComputeServiceId(UUID.randomUUID().toString()) + .cpEndpoint("http://example.com") + .iaasType(IaaSType.KUBERNETES) + .build(); + dm.setChosenCloudProviderEndpoint(chosenCloudProviderEndpoint); + deployment.setCloudProviderEndpoint(chosenCloudProviderEndpoint); + Map oneDataParameters = new HashMap<>(); + OneDataProviderInfo providerInfo = OneDataProviderInfo + .builder() + .cloudProviderId("provider-1") + .cloudServiceId(UUID.randomUUID().toString()) + .endpoint("http://example.onedata.com") + .id("test") + .build(); + List oneproviders = new ArrayList(); + oneproviders.add(providerInfo); + OneData parameter = OneData + .builder() + .oneproviders(oneproviders) + .onezone("test") + .path("/tmp/") + .selectedOneprovider(providerInfo) + .serviceSpace(true) + .smartScheduling(false) + .space("test") + .token("0123456789-onedata-token") + .build(); + oneDataParameters.put("provider-1", parameter); + dm.setOneDataParameters(oneDataParameters); + return dm; + } + + private Deployment generateDeployment() throws IOException { + Deployment deployment = ControllerTestUtils.createDeployment(); + deployment.setCloudProviderEndpoint(CloudProviderEndpoint.builder() + .cpComputeServiceId(UUID.randomUUID().toString()) + .cpEndpoint("example.com") + .iaasType(IaaSType.KUBERNETES) + .build()); + deployment.setTemplate( + TestUtil.getFileContentAsString(ToscaServiceTest.TEMPLATES_BASE_DIR + "kubernetes_app.yaml")); + + Resource runtime = new Resource(); + runtime.setDeployment(deployment); + runtime.setId("1"); + runtime.setState(NodeStates.INITIAL); + runtime.setToscaNodeName("Docker"); + runtime.setToscaNodeType("tosca.nodes.indigo.Container.Runtime.Docker"); + deployment.getResources().add(runtime); + + Resource app = new Resource(); + app.setDeployment(deployment); + app.setId("2"); + app.setState(NodeStates.INITIAL); + app.setToscaNodeName("kubernetes"); + app.setToscaNodeType("tosca.nodes.indigo.Container.Application.Docker.Kubernetes"); + app.addRequiredResource(runtime); + deployment.getResources().add(app); + + Map paramMap = new HashMap(); + paramMap.put("replicas", 1); + + deployment.setParameters(paramMap); + + Mockito + .when(resourceRepository.findByToscaNodeNameAndDeployment_id("Docker", + deployment.getId())) + .thenReturn(Lists.newArrayList(runtime)); + + Mockito + .when(resourceRepository.findByToscaNodeNameAndDeployment_id("kubernetes", + deployment.getId())) + .thenReturn(Lists.newArrayList(app)); + return deployment; + } + + private KubernetesService buildService() { + KubernetesService cs = KubernetesService + .kubernetesBuilder() + .endpoint("example.com/kubernetes") + .serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) + .hostname("example.com") + .providerId("provider-1") + .id("provider-1-service-1") + .type(CloudServiceType.COMPUTE) + .build(); + return cs; + } +} diff --git a/src/test/resources/tosca/kubernetes_app.yaml b/src/test/resources/tosca/kubernetes_app.yaml new file mode 100644 index 0000000000..a048553d28 --- /dev/null +++ b/src/test/resources/tosca/kubernetes_app.yaml @@ -0,0 +1,36 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +imports: + - indigo_custom_types: https://raw.githubusercontent.com/indigo-dc/tosca-types/master/custom_types.yaml + +topology_template: + + node_templates: + + kubernetes: + type: tosca.nodes.indigo.Container.Application.Docker.Kubernetes + properties: + uris: [] + enable_https: true + artifacts: + image: + file: hello-world + type: tosca.artifacts.Deployment.Image.Container.Docker + requirements: + - host: Docker + + Docker: + type: tosca.nodes.indigo.Container.Runtime.Docker + capabilities: + host: + properties: + num_cpus: 1.0 + mem_size: 256 Mb + publish_ports: + - protocol: tcp + source: 8080 + volumes: [ 'kubernetes:/data:rw:dvdi:rexray', '/data:rw' ] + + outputs: + endpoint: + value: { concat: [ { get_attribute : [ kubernetes, attr, 0 ] }, ':', { get_attribute : [ Docker, host, publish_ports, 0, target ] } ] } From ec4b2101af925e4d87ca17d7508b80f72eb2b406 Mon Sep 17 00:00:00 2001 From: Wareek Date: Mon, 17 Feb 2020 17:06:29 +0100 Subject: [PATCH 02/11] temporary improvement of insert delete deployment --- .../dto/cmdb/CloudServiceResolver.java | 4 +- .../orchestrator/service/CmdbServiceImpl.java | 20 +- .../commands/PrefilterCloudProviders.java | 5 +- .../service/commands/UpdateDeployment.java | 23 +- .../providers/KubernetesServiceImpl.java | 244 +++++++++--------- .../service/DeploymentServiceTest.java | 2 +- .../providers/KubernetesServiceTest.java | 18 +- 7 files changed, 154 insertions(+), 162 deletions(-) diff --git a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java index 1b8463d4b5..678f097ccd 100644 --- a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java +++ b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import static it.reply.orchestrator.dto.cmdb.CloudService.AWS_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.AZURE_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.CHRONOS_COMPUTE_SERVICE; +import static it.reply.orchestrator.dto.cmdb.CloudService.KUBERNETES_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.MARATHON_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.OCCI_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.OPENNEBULA_COMPUTE_SERVICE; @@ -26,7 +27,6 @@ import static it.reply.orchestrator.dto.cmdb.CloudService.OPENSTACK_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.OTC_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.QCG_COMPUTE_SERVICE; -import static it.reply.orchestrator.dto.cmdb.CloudService.KUBERNETES_COMPUTE_SERVICE; import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; import com.fasterxml.jackson.databind.DatabindContext; diff --git a/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java b/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java index 2d264a79eb..5cff0a6ecd 100644 --- a/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java @@ -287,16 +287,16 @@ public CloudProvider fillCloudProviderInfo(String providerId, return cloudService; }) .collect(Collectors.toMap(CloudService::getId, Function.identity())); - -// services.put(providerId+"_kuberServiceId123", KubernetesService.kubernetesBuilder() -// .id(providerId+"kuberServiceId123") -// .providerId(providerId) -// .serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) -// .type(CloudServiceType.COMPUTE) -// .hostname("localhost:8001") -// .endpoint("localhost:8001") -// .build()); - + + //services.put(providerId+"_kuberServiceId123", KubernetesService.kubernetesBuilder() + //.id(providerId+"kuberServiceId123") + //.providerId(providerId) + //.serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) + //.type(CloudServiceType.COMPUTE) + //.hostname("localhost:8001") + //.endpoint("localhost:8001") + //.build()); + provider.setServices(services); return provider; } diff --git a/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java b/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java index ba9a4405a1..ed715fc2fd 100644 --- a/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java +++ b/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -146,7 +146,8 @@ public void execute(DelegateExecution execution, break; case KUBERNETES: if ((cloudProviderService instanceof KubernetesService)) { - KubernetesService kubernetesService = (KubernetesService) cloudProviderService; + KubernetesService kubernetesService = + (KubernetesService) cloudProviderService; } else { addServiceToDiscard(servicesToDiscard, cloudProviderService); } diff --git a/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java b/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java index ba2ef1d44d..121db4e5b0 100644 --- a/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java +++ b/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,12 +20,9 @@ import it.reply.orchestrator.dto.CloudProviderEndpoint; import it.reply.orchestrator.dto.RankCloudProvidersMessage; import it.reply.orchestrator.dto.cmdb.CloudService; -import it.reply.orchestrator.dto.cmdb.CloudServiceType; -import it.reply.orchestrator.dto.cmdb.KubernetesService; import it.reply.orchestrator.dto.deployment.DeploymentMessage; import it.reply.orchestrator.dto.onedata.OneData; import it.reply.orchestrator.dto.onedata.OneData.OneDataProviderInfo; -import it.reply.orchestrator.dto.workflow.CloudServiceWf; import it.reply.orchestrator.dto.workflow.CloudServicesOrderedIterator; import it.reply.orchestrator.enums.DeploymentProvider; import it.reply.orchestrator.enums.Status; @@ -36,7 +33,6 @@ import it.reply.orchestrator.utils.WorkflowConstants; import it.reply.orchestrator.utils.WorkflowConstants.ErrorCode; -import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -50,8 +46,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import antlr.collections.List; - /** * Choose Cloud Provider and update Deployment/Message with the selected one data. */ @@ -72,20 +66,7 @@ public void execute(DelegateExecution execution, DeploymentMessage deploymentMes RankCloudProvidersMessage.class); CloudServicesOrderedIterator servicesIt = deploymentMessage.getCloudServicesOrderedIterator(); - -// ArrayList k = new ArrayList(); -// -// k.add(KubernetesService.kubernetesBuilder() -// .id("localhost_kuberServiceId123") -// .providerId("localhost") -// .serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) -// .type(CloudServiceType.COMPUTE) -// .hostname("localhost:8001") -// .endpoint("localhost:8001") -// .build()); -// -// servicesIt = new CloudServicesOrderedIterator(k); - + if (servicesIt == null) { servicesIt = cloudProviderEndpointService .generateCloudProvidersOrderedIterator(rankCloudProvidersMessage, diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index 2fd325c40e..7f897a495c 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -16,52 +16,19 @@ package it.reply.orchestrator.service.deployment.providers; -import io.kubernetes.client.custom.IntOrString; -import io.kubernetes.client.openapi.ApiCallback; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.ApiResponse; import io.kubernetes.client.openapi.Configuration; -import io.kubernetes.client.openapi.apis.CoreV1Api; -import io.kubernetes.client.openapi.models.V1APIService; -import io.kubernetes.client.openapi.apis.BatchV1Api; import io.kubernetes.client.openapi.apis.AppsV1Api; -import io.kubernetes.client.openapi.models.V1Container; -import io.kubernetes.client.openapi.models.V1ContainerBuilder; -import io.kubernetes.client.openapi.models.V1ContainerPort; -import io.kubernetes.client.openapi.models.V1DeleteOptions; +import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1Deployment; import io.kubernetes.client.openapi.models.V1DeploymentBuilder; -import io.kubernetes.client.openapi.models.V1DeploymentCondition; -import io.kubernetes.client.openapi.models.V1DeploymentSpec; -import io.kubernetes.client.openapi.models.V1DeploymentSpecBuilder; -import io.kubernetes.client.openapi.models.V1DeploymentStatus; -import io.kubernetes.client.openapi.models.V1DeploymentStatusBuilder; -import io.kubernetes.client.openapi.models.V1EnvVar; -import io.kubernetes.client.openapi.models.V1Job; -import io.kubernetes.client.openapi.models.V1JobBuilder; import io.kubernetes.client.openapi.models.V1Namespace; -import io.kubernetes.client.openapi.models.V1ObjectMeta; -import io.kubernetes.client.openapi.models.V1ObjectMetaBuilder; import io.kubernetes.client.openapi.models.V1Pod; -import io.kubernetes.client.openapi.models.V1PodBuilder; import io.kubernetes.client.openapi.models.V1PodList; -import io.kubernetes.client.openapi.models.V1ResourceRequirements; -import io.kubernetes.client.openapi.models.V1ResourceRequirementsBuilder; -import io.kubernetes.client.openapi.models.V1Service; -import io.kubernetes.client.openapi.models.V1ServiceBuilder; import io.kubernetes.client.openapi.models.V1Status; -import io.kubernetes.client.util.exception.ObjectMetaReflectException; - import io.kubernetes.client.util.Config; -import io.kubernetes.client.util.Yaml; -import io.kubernetes.client.util.exception.ObjectMetaReflectException; - -import com.google.common.base.Strings; - -import es.upv.i3m.grycap.im.InfrastructureManager; -import es.upv.i3m.grycap.im.exceptions.ImClientErrorException; -import es.upv.i3m.grycap.im.exceptions.ImClientException; -import es.upv.i3m.grycap.im.pojo.Property; import it.reply.orchestrator.annotation.DeploymentProviderQualifier; import it.reply.orchestrator.dal.entity.Deployment; @@ -72,30 +39,18 @@ import it.reply.orchestrator.enums.Task; import it.reply.orchestrator.exception.service.BusinessWorkflowException; import it.reply.orchestrator.exception.service.DeploymentException; -import it.reply.orchestrator.function.ThrowingFunction; import it.reply.orchestrator.service.ToscaService; import it.reply.orchestrator.service.VaultService; import it.reply.orchestrator.service.security.OAuth2TokenService; import it.reply.orchestrator.utils.WorkflowConstants.ErrorCode; -import java.io.File; import java.io.IOException; import java.net.URI; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; -import mesosphere.marathon.client.model.v2.App; -import mesosphere.marathon.client.model.v2.Group; - -import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -113,58 +68,10 @@ public class KubernetesServiceImpl extends AbstractDeploymentProviderService { @Autowired private VaultService vaultService; - private void test(Deployment deployment) throws IOException, ApiException { - - // access configuration TODO - ApiClient client = Config.defaultClient(); - Configuration.setDefaultApiClient(client); - AppsV1Api app = new AppsV1Api(client); - //-- - - deployment.getCallback(); - String name = deployment.getId(); - - V1Deployment v1Deployment = new V1DeploymentBuilder() - .withApiVersion("apps/v1") - .withKind("Deployment") - .withNewMetadata() - .withName(name) - .endMetadata() - .withNewSpec() - .withReplicas((Integer) deployment.getParameters().get("replicas")) - .withNewSelector() - .addToMatchLabels("app", "nginx") - .endSelector() - .withNewTemplate() - .withNewMetadata() - .addToLabels("app", "nginx") - .endMetadata() - .withNewSpec() - .addNewContainer() - .withImage("nginx:1.7.9") - .withName("nginx") - .addNewPort() - .withContainerPort(80) - .endPort() - .endContainer() - .endSpec() - .endTemplate() - .endSpec() - .build(); - - System.out.println(Yaml.dump(v1Deployment)); - - V1Deployment outDepl = app.createNamespacedDeployment("default", v1Deployment, "true", null, null); - String ref = outDepl.getMetadata().getName(); - - outDepl.getStatus(); - - printPodList(); - } - @Override public boolean doDeploy(DeploymentMessage deploymentMessage) { Deployment deployment = getDeployment(deploymentMessage); + V1Deployment outDepl = new V1Deployment(); final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); // Update status of the deployment - if not already done (remember the Iterative @@ -175,24 +82,73 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { if (deployment.getEndpoint() == null) { deployment.setEndpoint(""); } - -// test(deployment); - doUndeploy(deploymentMessage); - LOG.info("Creating Kubernetes App Group for deployment {} with definition:\n{}", deployment.getId(), "x"); - // deployment.getId(), group); + try { + // access configuration TODO + ApiClient client = Config.defaultClient(); + Configuration.setDefaultApiClient(client); + AppsV1Api app = new AppsV1Api(client); + + String name = deployment.getId(); + + //TODO manage needed field for deployment and get them from DeploymentMessage + V1Deployment v1Deployment = new V1DeploymentBuilder() + .withApiVersion("apps/v1") + .withKind("Deployment") + .withNewMetadata() + .withName(name) + .endMetadata() + .withNewSpec() + .withReplicas((Integer) deployment.getParameters().get("replicas")) + .withNewSelector() + .addToMatchLabels("app", "nginx") + .endSelector() + .withNewTemplate() + .withNewMetadata() + .addToLabels("app", "nginx") + .endMetadata() + .withNewSpec() + .addNewContainer() + .withImage("nginx:1.7.9") + .withName("nginx") + .addNewPort() + .withContainerPort(80) + .endPort() + .endContainer() + .endSpec() + .endTemplate() + .endSpec() + .build(); + + outDepl = app.createNamespacedDeployment( + "default", + v1Deployment, + "true", + null, + null); + + String ref = outDepl.getMetadata().getName(); + + } catch (ApiException e) { + LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + } catch (IOException e) { + LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + } + + LOG.info("Creating Kubernetes App Group for deployment {} with definition:\n{}", + deployment.getId(), outDepl.getMetadata().getName()); + // deployment.getId(), group); CloudProviderEndpoint cloudProviderEndpoint = deployment.getCloudProviderEndpoint(); vaultService.getServiceUri() .map(URI::toString) .ifPresent(cloudProviderEndpoint::setVaultEndpoint); -// executeWithClient(cloudProviderEndpoint, requestedWithToken, -// client -> client.createGroup(group)); + // executeWithClient(cloudProviderEndpoint, requestedWithToken, + // client -> client.createGroup(group)); return true; } @Override public boolean isDeployed(DeploymentMessage deploymentMessage) { - // TODO Auto-generated method stub Deployment deployment = getDeployment(deploymentMessage); AppsV1Api app; @@ -201,17 +157,19 @@ public boolean isDeployed(DeploymentMessage deploymentMessage) { try { app = new AppsV1Api(Config.defaultClient()); api = new CoreV1Api(); + V1Namespace namespace = api.readNamespaceStatus("default", null); - v1Deployment = app.readNamespacedDeploymentStatus("nginx-deployment-5754944d6c-c8kpv", "default", "true"); - + v1Deployment = app.readNamespacedDeploymentStatus(deployment.getId(), "default", "true"); + printPodList(); } catch (ApiException e) { LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); } catch (IOException e) { LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); } - - boolean isDeployed = v1Deployment.getStatus().equals(V1DeploymentStatus.SERIALIZED_NAME_READY_REPLICAS); + + boolean isDeployed = + v1Deployment.getStatus().getReplicas() == v1Deployment.getStatus().getReadyReplicas(); LOG.debug("Kubernetes App Group for deployment {} is deployed? {}", deployment.getId(), isDeployed); return isDeployed; @@ -224,7 +182,6 @@ public void cleanFailedDeploy(DeploymentMessage deploymentMessage) { @Override public boolean doUpdate(DeploymentMessage deploymentMessage, String template) { - // TODO Auto-generated method stub Deployment deployment = getDeployment(deploymentMessage); V1Deployment v1Deployment = new V1DeploymentBuilder() @@ -237,10 +194,17 @@ public boolean doUpdate(DeploymentMessage deploymentMessage, String template) { try { app = new AppsV1Api(Config.defaultClient()); api = new CoreV1Api(); - - app.patchNamespacedDeployment(deployment.getId(), "default", v1Deployment, "true", null, null, null); -// app.replaceNamespacedDeployment(name, namespace, body, pretty, dryRun, fieldManager); - + + app.patchNamespacedDeployment( + deployment.getId(), + "default", + v1Deployment, + "true", + null, + null, + null); + // app.replaceNamespacedDeployment(name, namespace, body, pretty, dryRun, fieldManager); + } catch (ApiException e) { LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); } catch (IOException e) { @@ -258,22 +222,43 @@ public void cleanFailedUpdate(DeploymentMessage deploymentMessage) { public boolean doUndeploy(DeploymentMessage deploymentMessage) { Deployment deployment = getDeployment(deploymentMessage); - - V1Deployment v1Deployment = new V1Deployment(); AppsV1Api app; - CoreV1Api api; try { app = new AppsV1Api(Config.defaultClient()); - api = new CoreV1Api(); - app.deleteNamespacedDeployment("nginx-deployment-5754944d6c", "default", "true", null, null, null, null, null); -// app.deleteCollectionNamespacedDeployment(namespace, pretty, allowWatchBookmarks, _continue, dryRun, fieldSelector, gracePeriodSeconds, labelSelector, limit, orphanDependents, propagationPolicy, resourceVersion, timeoutSeconds, watch, body) -// api.deleteCollectionNamespacedPod(namespace, pretty, allowWatchBookmarks, _continue, dryRun, fieldSelector, gracePeriodSeconds, labelSelector, limit, orphanDependents, propagationPolicy, resourceVersion, timeoutSeconds, watch, body) + ApiResponse response = app.deleteNamespacedDeploymentWithHttpInfo( + deployment.getId(), + "default", + "true", + null, + null, + null, + null, + null); + V1Status status = app.deleteNamespacedDeployment( + deployment.getId(), + "default", + "true", + null, + null, + null, + null, + null); + + response.getStatusCode(); + LOG.debug("Deleting deployment exited with :" + + status.getCode() + + " - " + + status.getMessage() + + " - " + + status.getStatus()); + } catch (ApiException e) { LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); -// if(e.getCode()!=404) { -// throw new HttpResponseException(e.getCode(), "KubernetesApiException"); -// } + // TODO manage throwing errorCode exception + // if(e.getCode()!=404) { + // throw new HttpResponseException(e.getCode(), "KubernetesApiException"); + // } } catch (IOException e) { LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); } @@ -304,10 +289,19 @@ protected Optional getAdditionalErrorInfoInternal(DeploymentMessage depl //TODO return Optional.empty(); } - + private void printPodList() throws ApiException { CoreV1Api api = new CoreV1Api(); - V1PodList list = api.listPodForAllNamespaces(null, null, null, null, null, null, null, null, null); + V1PodList list = api.listPodForAllNamespaces( + null, + null, + null, + null, + null, + null, + null, + null, + null); for (V1Pod item : list.getItems()) { System.out.println(item.getMetadata().getName()); } diff --git a/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java b/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java index c64b26ee16..842ac6f6e4 100644 --- a/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java index 0833435631..0cb88135df 100644 --- a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package it.reply.orchestrator.service.deployment.providers; import com.google.common.collect.Lists; @@ -112,7 +128,7 @@ private DeploymentMessage generateDeployDmKuber(Deployment deployment) { deployment.setCloudProviderEndpoint(chosenCloudProviderEndpoint); return dm; } - + private DeploymentMessage generateDeployDm(Deployment deployment) { DeploymentMessage dm = new DeploymentMessage(); dm.setDeploymentId(deployment.getId()); From 813f45a84f5e4aeb6de603d0b2f9ff79bee9878f Mon Sep 17 00:00:00 2001 From: Wareek Date: Tue, 18 Feb 2020 17:09:11 +0100 Subject: [PATCH 03/11] tem commit wit token auth --- .../providers/KubernetesServiceImpl.java | 152 ++++++++++++++---- .../factory/KubernetesClientFactory.java | 65 ++++++++ 2 files changed, 184 insertions(+), 33 deletions(-) create mode 100644 src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index 7f897a495c..dc3da6d763 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -16,20 +16,26 @@ package it.reply.orchestrator.service.deployment.providers; +import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.ApiResponse; import io.kubernetes.client.openapi.Configuration; import io.kubernetes.client.openapi.apis.AppsV1Api; import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1Deployment; import io.kubernetes.client.openapi.models.V1DeploymentBuilder; import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1PodList; +import io.kubernetes.client.openapi.models.V1ResourceRequirements; +import io.kubernetes.client.openapi.models.V1ResourceRequirementsBuilder; import io.kubernetes.client.openapi.models.V1Status; import io.kubernetes.client.util.Config; +import alien4cloud.tosca.model.ArchiveRoot; + import it.reply.orchestrator.annotation.DeploymentProviderQualifier; import it.reply.orchestrator.dal.entity.Deployment; import it.reply.orchestrator.dal.entity.OidcTokenId; @@ -39,18 +45,42 @@ import it.reply.orchestrator.enums.Task; import it.reply.orchestrator.exception.service.BusinessWorkflowException; import it.reply.orchestrator.exception.service.DeploymentException; +import it.reply.orchestrator.function.ThrowingConsumer; +import it.reply.orchestrator.function.ThrowingFunction; +import it.reply.orchestrator.service.IndigoInputsPreProcessorService; import it.reply.orchestrator.service.ToscaService; import it.reply.orchestrator.service.VaultService; +import it.reply.orchestrator.service.IndigoInputsPreProcessorService.RuntimeProperties; +import it.reply.orchestrator.service.deployment.providers.factory.KubernetesClientFactory; import it.reply.orchestrator.service.security.OAuth2TokenService; +import it.reply.orchestrator.utils.CommonUtils; +import it.reply.orchestrator.utils.OneDataUtils; +import it.reply.orchestrator.utils.ToscaConstants; import it.reply.orchestrator.utils.WorkflowConstants.ErrorCode; import java.io.IOException; import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; + +import javax.ws.rs.client.ClientBuilder; import lombok.extern.slf4j.Slf4j; +import mesosphere.marathon.client.Marathon; +import mesosphere.marathon.client.MarathonException; +import mesosphere.marathon.client.model.v2.App; + +import org.alien4cloud.tosca.model.templates.NodeTemplate; +import org.alien4cloud.tosca.model.templates.RelationshipTemplate; +import org.alien4cloud.tosca.model.templates.Topology; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.jgrapht.graph.DirectedMultigraph; +import org.jgrapht.traverse.TopologicalOrderIterator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -59,38 +89,98 @@ @Slf4j public class KubernetesServiceImpl extends AbstractDeploymentProviderService { + // @Autowired + // private KubernetesClientFactory kubernetesClientFactory; + + @Autowired + private IndigoInputsPreProcessorService indigoInputsPreProcessorService; + @Autowired private ToscaService toscaService; @Autowired private OAuth2TokenService oauth2TokenService; - @Autowired - private VaultService vaultService; +// protected R executeWithClientForResult(CloudProviderEndpoint cloudProviderEndpoint, +// @Nullable OidcTokenId requestedWithToken, +// ThrowingFunction function) throws ApiException { +// return oauth2TokenService.executeWithClientForResult(requestedWithToken, +// token -> function.apply(kubernetesClientFactory.build(cloudProviderEndpoint, token)), +// ex -> ex instanceof ApiException && ((ApiException) ex).getCode() == 401); +// } +// +// protected void executeWithClient(CloudProviderEndpoint cloudProviderEndpoint, +// @Nullable OidcTokenId requestedWithToken, +// ThrowingConsumer consumer) throws ApiException { +// executeWithClientForResult(cloudProviderEndpoint, requestedWithToken, +// client -> consumer.asFunction().apply(client)); +// } + + //TODO if needed + protected ArchiveRoot prepareTemplate(Deployment deployment, + DeploymentMessage deploymentMessage) { + RuntimeProperties runtimeProperties = + OneDataUtils.getOneDataRuntimeProperties(deploymentMessage); + Map inputs = deployment.getParameters(); + ArchiveRoot ar = toscaService.parseAndValidateTemplate(deployment.getTemplate(), inputs); + if (runtimeProperties.getVaules().size() > 0) { + indigoInputsPreProcessorService.processGetInputAttributes(ar, inputs, runtimeProperties); + } else { + indigoInputsPreProcessorService.processGetInput(ar, inputs); + } + return ar; + } + + protected void createX(DeploymentMessage deploymentMessage, OidcTokenId requestedWithToken) { + Deployment deployment = getDeployment(deploymentMessage); + ArchiveRoot ar = prepareTemplate(deployment, deploymentMessage); + + Map nodes = Optional + .ofNullable(ar.getTopology()) + .map(Topology::getNodeTemplates) + .orElseGet(HashMap::new); + + DirectedMultigraph graph = + toscaService.buildNodeGraph(nodes, false); + + TopologicalOrderIterator orderIterator = + new TopologicalOrderIterator<>(graph); + + List orderedKubernetesApps = CommonUtils + .iteratorToStream(orderIterator) + .filter(node -> toscaService.isOfToscaType(node, ToscaConstants.Nodes.Types.KUBERNETES)) + .collect(Collectors.toList()); + + List containers = new ArrayList<>(); + } @Override public boolean doDeploy(DeploymentMessage deploymentMessage) { Deployment deployment = getDeployment(deploymentMessage); V1Deployment outDepl = new V1Deployment(); - final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); - // Update status of the deployment - if not already done (remember the Iterative - // mode) - if (deployment.getTask() != Task.DEPLOYER) { - deployment.setTask(Task.DEPLOYER); - } - if (deployment.getEndpoint() == null) { - deployment.setEndpoint(""); - } + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + String accessToken = + oauth2TokenService.getAccessToken(CommonUtils.checkNotNull(requestedWithToken)); try { // access configuration TODO - ApiClient client = Config.defaultClient(); - Configuration.setDefaultApiClient(client); + //ApiClient client = Config.defaultClient(); + //Configuration.setDefaultApiClient(client); + ApiClient client = Config.fromToken( + deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint(), accessToken); + AppsV1Api app = new AppsV1Api(client); String name = deployment.getId(); + Map res = new HashMap (); + V1ResourceRequirements resources = new V1ResourceRequirementsBuilder() + .withRequests(res) + .build(); + /* note , cpu and ram are shared between all containers in the pod + * so it is enought difine it once*/ + //TODO manage needed field for deployment and get them from DeploymentMessage V1Deployment v1Deployment = new V1DeploymentBuilder() .withApiVersion("apps/v1") @@ -111,6 +201,7 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { .addNewContainer() .withImage("nginx:1.7.9") .withName("nginx") + .withResources(resources) .addNewPort() .withContainerPort(80) .endPort() @@ -131,19 +222,14 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { } catch (ApiException e) { LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); - } catch (IOException e) { - LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); - } + } +// catch (IOException e) { +// LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); +// } LOG.info("Creating Kubernetes App Group for deployment {} with definition:\n{}", deployment.getId(), outDepl.getMetadata().getName()); - // deployment.getId(), group); - CloudProviderEndpoint cloudProviderEndpoint = deployment.getCloudProviderEndpoint(); - vaultService.getServiceUri() - .map(URI::toString) - .ifPresent(cloudProviderEndpoint::setVaultEndpoint); - // executeWithClient(cloudProviderEndpoint, requestedWithToken, - // client -> client.createGroup(group)); + return true; } @@ -226,15 +312,15 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { try { app = new AppsV1Api(Config.defaultClient()); - ApiResponse response = app.deleteNamespacedDeploymentWithHttpInfo( - deployment.getId(), - "default", - "true", - null, - null, - null, - null, - null); +// ApiResponse response = app.deleteNamespacedDeploymentWithHttpInfo( +// deployment.getId(), +// "default", +// "true", +// null, +// null, +// null, +// null, +// null); V1Status status = app.deleteNamespacedDeployment( deployment.getId(), "default", @@ -245,7 +331,7 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { null, null); - response.getStatusCode(); + //response.getStatusCode(); LOG.debug("Deleting deployment exited with :" + status.getCode() + " - " diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java new file mode 100644 index 0000000000..fa981d42b0 --- /dev/null +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java @@ -0,0 +1,65 @@ +package it.reply.orchestrator.service.deployment.providers.factory; + +import feign.Feign; +import feign.RequestInterceptor; +import feign.Logger.Level; +import feign.slf4j.Slf4jLogger; + +import it.infn.ba.deep.qcg.client.Qcg; +import it.infn.ba.deep.qcg.client.utils.QcgDecoder; +import it.infn.ba.deep.qcg.client.utils.QcgEncoder; +import it.infn.ba.deep.qcg.client.utils.QcgException; +import it.reply.orchestrator.dto.CloudProviderEndpoint; +import it.reply.orchestrator.dto.cmdb.KubernetesService; + +import java.util.Objects; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.http.HttpHeaders; +import org.springframework.stereotype.Service; + +import io.kubernetes.client.openapi.ApiClient; + +@Service +@Slf4j +public class KubernetesClientFactory { + + /** + * Build a Kubernetes Api client object. + * @param cloudProviderEndpoint the service endpoint. + * @param accessToken the input accesstoken. + * @return the Kubernetes Api client object. + */ + public ApiClient build(CloudProviderEndpoint cloudProviderEndpoint, String accessToken) { + final RequestInterceptor requestInterceptor; + Objects.requireNonNull(accessToken, "Access Token must not be null"); + requestInterceptor = requestTemplate -> + requestTemplate.header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken); + + return build(cloudProviderEndpoint.getCpEndpoint(), requestInterceptor); + } + + /** + * Build a Qcg client object. + * @param qcgEndpoint the input qcg service endpoint. + * @param authInterceptor the input request interceptor. + * @return the Qcg client object. + */ + public ApiClient build(String apiClientEndpoint, RequestInterceptor authInterceptor) { + LOG.info("Generating Qcg client with endpoint {}", apiClientEndpoint); + + //TODO * new ApiEncoder()).decoder(new QcgDecoder() + return Feign.builder().encoder(null/* * */) + .logger(new Slf4jLogger(ApiClient.class)) + .logLevel(Level.FULL) + .errorDecoder((methodKey, response) -> new QcgException(response.status(), + response.reason())) + .requestInterceptor(authInterceptor).requestInterceptor(template -> { + template.header(HttpHeaders.ACCEPT, "application/json"); + template.header(HttpHeaders.CONTENT_TYPE, "application/json"); + }).target(ApiClient.class, apiClientEndpoint); + } + +} From b6d87db9566c49658df6ea793883fdcf54f81fc0 Mon Sep 17 00:00:00 2001 From: Wareek Date: Tue, 18 Feb 2020 20:20:07 +0100 Subject: [PATCH 04/11] temp resource management --- .../providers/KubernetesServiceImpl.java | 113 ++++++++++-------- .../factory/KubernetesClientFactory.java | 35 ++++-- .../providers/KubernetesServiceTest.java | 8 ++ 3 files changed, 92 insertions(+), 64 deletions(-) diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index dc3da6d763..a62572bd73 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -16,6 +16,8 @@ package it.reply.orchestrator.service.deployment.providers; +import alien4cloud.tosca.model.ArchiveRoot; + import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; @@ -24,8 +26,11 @@ import io.kubernetes.client.openapi.apis.AppsV1Api; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1Container; +import io.kubernetes.client.openapi.models.V1ContainerBuilder; import io.kubernetes.client.openapi.models.V1Deployment; import io.kubernetes.client.openapi.models.V1DeploymentBuilder; +import io.kubernetes.client.openapi.models.V1DeploymentSpec; +import io.kubernetes.client.openapi.models.V1DeploymentSpecBuilder; import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1PodList; @@ -34,8 +39,6 @@ import io.kubernetes.client.openapi.models.V1Status; import io.kubernetes.client.util.Config; -import alien4cloud.tosca.model.ArchiveRoot; - import it.reply.orchestrator.annotation.DeploymentProviderQualifier; import it.reply.orchestrator.dal.entity.Deployment; import it.reply.orchestrator.dal.entity.OidcTokenId; @@ -48,9 +51,9 @@ import it.reply.orchestrator.function.ThrowingConsumer; import it.reply.orchestrator.function.ThrowingFunction; import it.reply.orchestrator.service.IndigoInputsPreProcessorService; +import it.reply.orchestrator.service.IndigoInputsPreProcessorService.RuntimeProperties; import it.reply.orchestrator.service.ToscaService; import it.reply.orchestrator.service.VaultService; -import it.reply.orchestrator.service.IndigoInputsPreProcessorService.RuntimeProperties; import it.reply.orchestrator.service.deployment.providers.factory.KubernetesClientFactory; import it.reply.orchestrator.service.security.OAuth2TokenService; import it.reply.orchestrator.utils.CommonUtils; @@ -67,18 +70,11 @@ import java.util.Optional; import java.util.stream.Collectors; -import javax.ws.rs.client.ClientBuilder; - import lombok.extern.slf4j.Slf4j; -import mesosphere.marathon.client.Marathon; -import mesosphere.marathon.client.MarathonException; -import mesosphere.marathon.client.model.v2.App; - import org.alien4cloud.tosca.model.templates.NodeTemplate; import org.alien4cloud.tosca.model.templates.RelationshipTemplate; import org.alien4cloud.tosca.model.templates.Topology; -import org.checkerframework.checker.nullness.qual.Nullable; import org.jgrapht.graph.DirectedMultigraph; import org.jgrapht.traverse.TopologicalOrderIterator; import org.springframework.beans.factory.annotation.Autowired; @@ -91,7 +87,7 @@ public class KubernetesServiceImpl extends AbstractDeploymentProviderService { // @Autowired // private KubernetesClientFactory kubernetesClientFactory; - + @Autowired private IndigoInputsPreProcessorService indigoInputsPreProcessorService; @@ -101,22 +97,22 @@ public class KubernetesServiceImpl extends AbstractDeploymentProviderService { @Autowired private OAuth2TokenService oauth2TokenService; -// protected R executeWithClientForResult(CloudProviderEndpoint cloudProviderEndpoint, -// @Nullable OidcTokenId requestedWithToken, -// ThrowingFunction function) throws ApiException { -// return oauth2TokenService.executeWithClientForResult(requestedWithToken, -// token -> function.apply(kubernetesClientFactory.build(cloudProviderEndpoint, token)), -// ex -> ex instanceof ApiException && ((ApiException) ex).getCode() == 401); -// } -// -// protected void executeWithClient(CloudProviderEndpoint cloudProviderEndpoint, -// @Nullable OidcTokenId requestedWithToken, -// ThrowingConsumer consumer) throws ApiException { -// executeWithClientForResult(cloudProviderEndpoint, requestedWithToken, -// client -> consumer.asFunction().apply(client)); -// } - - //TODO if needed + // protected R executeWithClientForResult(CloudProviderEndpoint cloudProviderEndpoint, + // @Nullable OidcTokenId requestedWithToken, + // ThrowingFunction function) throws ApiException { + // return oauth2TokenService.executeWithClientForResult(requestedWithToken, + // token -> function.apply(kubernetesClientFactory.build(cloudProviderEndpoint, token)), + // ex -> ex instanceof ApiException && ((ApiException) ex).getCode() == 401); + // } + // + // protected void executeWithClient(CloudProviderEndpoint cloudProviderEndpoint, + // @Nullable OidcTokenId requestedWithToken, + // ThrowingConsumer consumer) throws ApiException { + // executeWithClientForResult(cloudProviderEndpoint, requestedWithToken, + // client -> consumer.asFunction().apply(client)); + // } + + //TODO if needed protected ArchiveRoot prepareTemplate(Deployment deployment, DeploymentMessage deploymentMessage) { RuntimeProperties runtimeProperties = @@ -130,7 +126,7 @@ protected ArchiveRoot prepareTemplate(Deployment deployment, } return ar; } - + protected void createX(DeploymentMessage deploymentMessage, OidcTokenId requestedWithToken) { Deployment deployment = getDeployment(deploymentMessage); ArchiveRoot ar = prepareTemplate(deployment, deploymentMessage); @@ -150,8 +146,26 @@ protected void createX(DeploymentMessage deploymentMessage, OidcTokenId requeste .iteratorToStream(orderIterator) .filter(node -> toscaService.isOfToscaType(node, ToscaConstants.Nodes.Types.KUBERNETES)) .collect(Collectors.toList()); - + List containers = new ArrayList<>(); + + Map requestsRes = new HashMap(); + /*The expression 0.1 is equivalent to the expression 100m, + *which can be read as “one hundred millicpu”.*/ + requestsRes.put("cpu", new Quantity("32Mi")); + requestsRes.put("memory", new Quantity("100m")); + + Map limitRes = new HashMap(); + limitRes.put("cpu", new Quantity("64Mi")); + limitRes.put("memory", new Quantity("200m")); + + V1ResourceRequirements resources = new V1ResourceRequirementsBuilder() + .withRequests(requestsRes) + .withLimits(limitRes) + .build(); + + V1DeploymentSpec spec = new V1DeploymentSpec(); + V1ContainerBuilder cont = new V1ContainerBuilder().withResources(resources); } @Override @@ -160,7 +174,7 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { V1Deployment outDepl = new V1Deployment(); final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); - String accessToken = + String accessToken = oauth2TokenService.getAccessToken(CommonUtils.checkNotNull(requestedWithToken)); try { @@ -171,16 +185,12 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint(), accessToken); AppsV1Api app = new AppsV1Api(client); - String name = deployment.getId(); - Map res = new HashMap (); - V1ResourceRequirements resources = new V1ResourceRequirementsBuilder() - .withRequests(res) - .build(); + - /* note , cpu and ram are shared between all containers in the pod + /* note , cpu and ram are shared between all containers in the pod * so it is enought difine it once*/ - + //TODO manage needed field for deployment and get them from DeploymentMessage V1Deployment v1Deployment = new V1DeploymentBuilder() .withApiVersion("apps/v1") @@ -201,7 +211,6 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { .addNewContainer() .withImage("nginx:1.7.9") .withName("nginx") - .withResources(resources) .addNewPort() .withContainerPort(80) .endPort() @@ -222,14 +231,14 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { } catch (ApiException e) { LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); - } -// catch (IOException e) { -// LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); -// } + } + // catch (IOException e) { + // LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + // } LOG.info("Creating Kubernetes App Group for deployment {} with definition:\n{}", deployment.getId(), outDepl.getMetadata().getName()); - + return true; } @@ -312,15 +321,15 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { try { app = new AppsV1Api(Config.defaultClient()); -// ApiResponse response = app.deleteNamespacedDeploymentWithHttpInfo( -// deployment.getId(), -// "default", -// "true", -// null, -// null, -// null, -// null, -// null); + // ApiResponse response = app.deleteNamespacedDeploymentWithHttpInfo( + // deployment.getId(), + // "default", + // "true", + // null, + // null, + // null, + // null, + // null); V1Status status = app.deleteNamespacedDeployment( deployment.getId(), "default", diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java index fa981d42b0..883ae36092 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java @@ -1,27 +1,38 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package it.reply.orchestrator.service.deployment.providers.factory; import feign.Feign; -import feign.RequestInterceptor; import feign.Logger.Level; +import feign.RequestInterceptor; import feign.slf4j.Slf4jLogger; -import it.infn.ba.deep.qcg.client.Qcg; -import it.infn.ba.deep.qcg.client.utils.QcgDecoder; -import it.infn.ba.deep.qcg.client.utils.QcgEncoder; +import io.kubernetes.client.openapi.ApiClient; + import it.infn.ba.deep.qcg.client.utils.QcgException; import it.reply.orchestrator.dto.CloudProviderEndpoint; -import it.reply.orchestrator.dto.cmdb.KubernetesService; import java.util.Objects; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpHeaders; import org.springframework.stereotype.Service; -import io.kubernetes.client.openapi.ApiClient; - @Service @Slf4j public class KubernetesClientFactory { @@ -35,17 +46,17 @@ public class KubernetesClientFactory { public ApiClient build(CloudProviderEndpoint cloudProviderEndpoint, String accessToken) { final RequestInterceptor requestInterceptor; Objects.requireNonNull(accessToken, "Access Token must not be null"); - requestInterceptor = requestTemplate -> + requestInterceptor = requestTemplate -> requestTemplate.header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken); return build(cloudProviderEndpoint.getCpEndpoint(), requestInterceptor); } /** - * Build a Qcg client object. - * @param qcgEndpoint the input qcg service endpoint. + * Build a Kubernetes client object. + * @param apiClientEndpoint the input Kubernetes service endpoint. * @param authInterceptor the input request interceptor. - * @return the Qcg client object. + * @return the Kubernetes client object. */ public ApiClient build(String apiClientEndpoint, RequestInterceptor authInterceptor) { LOG.info("Generating Qcg client with endpoint {}", apiClientEndpoint); diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java index 0cb88135df..a37474f2a0 100644 --- a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -39,6 +39,7 @@ import it.reply.orchestrator.service.ToscaServiceTest; import it.reply.orchestrator.service.VaultService; import it.reply.orchestrator.util.TestUtil; +import it.reply.orchestrator.utils.CommonUtils; import java.io.IOException; import java.net.URI; @@ -91,6 +92,10 @@ public void setup() throws Exception { .when(oauth2tokenService.executeWithClientForResult( Mockito.any(), Mockito.any(), Mockito.any())) .thenAnswer(y -> ((ThrowingFunction) y.getArguments()[1]).apply("token")); + + Mockito + .when(oauth2tokenService.getAccessToken(Mockito.any())) + .thenAnswer(y -> ((ThrowingFunction) y.getArguments()[1]).apply("token")); } @Test @@ -110,6 +115,9 @@ public void testDoDeploy() throws IOException, URISyntaxException { Mockito .when(vaultService.getServiceUri()) .thenReturn(Optional.of(new URI(defaultVaultEndpoint))); +// Mockito +// .when(marathonClientFactory.build(deployment.getCloudProviderEndpoint(), "token")) +// .thenReturn(marathonClient); Assertions .assertThat(kubernetesServiceImpl.doDeploy(dm)) .isTrue(); From cd3d0813fe47bd079bd2fedcf8a751352caa4036 Mon Sep 17 00:00:00 2001 From: Wareek Date: Wed, 19 Feb 2020 18:26:44 +0100 Subject: [PATCH 05/11] temp dto for kuber deployments --- .../dto/kubernetes/KubernetesContainer.java | 36 ++ .../dto/kubernetes/KubernetesTask.java | 34 ++ .../providers/KubernetesServiceImpl.java | 307 +++++++++++++----- .../providers/KubernetesServiceTest.java | 17 +- 4 files changed, 305 insertions(+), 89 deletions(-) create mode 100644 src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java create mode 100644 src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java new file mode 100644 index 0000000000..b6f0349539 --- /dev/null +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.dto.kubernetes; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + +import lombok.Data; + +@Data +public class KubernetesContainer { + + private String id; + + private final String name; + + private final String toscaName; + + private String image; + + private Multimap parameters = ArrayListMultimap.create(); +} diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java new file mode 100644 index 0000000000..c8fe3e0104 --- /dev/null +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.dto.kubernetes; + +import lombok.Data; + +@Data +public class KubernetesTask { + + private String id; + + private String cmd; + + private KubernetesContainer container; + + private Double cpus; + + private Double memSize; + +} diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index a62572bd73..11fe356a3d 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -18,11 +18,11 @@ import alien4cloud.tosca.model.ArchiveRoot; +import com.google.common.collect.MoreCollectors; + import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; -import io.kubernetes.client.openapi.ApiResponse; -import io.kubernetes.client.openapi.Configuration; import io.kubernetes.client.openapi.apis.AppsV1Api; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1Container; @@ -30,8 +30,6 @@ import io.kubernetes.client.openapi.models.V1Deployment; import io.kubernetes.client.openapi.models.V1DeploymentBuilder; import io.kubernetes.client.openapi.models.V1DeploymentSpec; -import io.kubernetes.client.openapi.models.V1DeploymentSpecBuilder; -import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1PodList; import io.kubernetes.client.openapi.models.V1ResourceRequirements; @@ -39,42 +37,60 @@ import io.kubernetes.client.openapi.models.V1Status; import io.kubernetes.client.util.Config; +import it.infn.ba.deep.qcg.client.model.Job; +import it.infn.ba.deep.qcg.client.model.JobDescription; +import it.infn.ba.deep.qcg.client.model.JobDescriptionExecution; +import it.infn.ba.deep.qcg.client.model.JobWorkingDirectoryPolicy; +import it.infn.ba.deep.qcg.client.model.RemoveConditionCreateMode; +import it.infn.ba.deep.qcg.client.model.RemoveConditionWhen; import it.reply.orchestrator.annotation.DeploymentProviderQualifier; import it.reply.orchestrator.dal.entity.Deployment; import it.reply.orchestrator.dal.entity.OidcTokenId; +import it.reply.orchestrator.dal.entity.Resource; +import it.reply.orchestrator.dal.repository.ResourceRepository; import it.reply.orchestrator.dto.CloudProviderEndpoint; import it.reply.orchestrator.dto.deployment.DeploymentMessage; +import it.reply.orchestrator.dto.kubernetes.KubernetesContainer; +import it.reply.orchestrator.dto.kubernetes.KubernetesTask; import it.reply.orchestrator.enums.DeploymentProvider; -import it.reply.orchestrator.enums.Task; import it.reply.orchestrator.exception.service.BusinessWorkflowException; import it.reply.orchestrator.exception.service.DeploymentException; -import it.reply.orchestrator.function.ThrowingConsumer; -import it.reply.orchestrator.function.ThrowingFunction; +import it.reply.orchestrator.exception.service.ToscaException; import it.reply.orchestrator.service.IndigoInputsPreProcessorService; import it.reply.orchestrator.service.IndigoInputsPreProcessorService.RuntimeProperties; +import it.reply.orchestrator.service.deployment.providers.QcgServiceImpl.DeepJob; import it.reply.orchestrator.service.ToscaService; -import it.reply.orchestrator.service.VaultService; -import it.reply.orchestrator.service.deployment.providers.factory.KubernetesClientFactory; import it.reply.orchestrator.service.security.OAuth2TokenService; import it.reply.orchestrator.utils.CommonUtils; import it.reply.orchestrator.utils.OneDataUtils; import it.reply.orchestrator.utils.ToscaConstants; +import it.reply.orchestrator.utils.ToscaUtils; import it.reply.orchestrator.utils.WorkflowConstants.ErrorCode; import java.io.IOException; -import java.net.URI; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import javax.validation.constraints.NotNull; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.alien4cloud.tosca.model.templates.Capability; import org.alien4cloud.tosca.model.templates.NodeTemplate; import org.alien4cloud.tosca.model.templates.RelationshipTemplate; import org.alien4cloud.tosca.model.templates.Topology; +import org.alien4cloud.tosca.normative.types.BooleanType; +import org.alien4cloud.tosca.normative.types.FloatType; +import org.checkerframework.checker.nullness.qual.NonNull; import org.jgrapht.graph.DirectedMultigraph; import org.jgrapht.traverse.TopologicalOrderIterator; import org.springframework.beans.factory.annotation.Autowired; @@ -97,6 +113,13 @@ public class KubernetesServiceImpl extends AbstractDeploymentProviderService { @Autowired private OAuth2TokenService oauth2TokenService; + @Autowired + private ResourceRepository resourceRepository; + + private static CoreV1Api COREV1_API; + + private static final String HOST_CAPABILITY_NAME = "host"; + // protected R executeWithClientForResult(CloudProviderEndpoint cloudProviderEndpoint, // @Nullable OidcTokenId requestedWithToken, // ThrowingFunction function) throws ApiException { @@ -127,7 +150,8 @@ protected ArchiveRoot prepareTemplate(Deployment deployment, return ar; } - protected void createX(DeploymentMessage deploymentMessage, OidcTokenId requestedWithToken) { + protected List getContanersList(DeploymentMessage deploymentMessage, + OidcTokenId requestedWithToken) { Deployment deployment = getDeployment(deploymentMessage); ArchiveRoot ar = prepareTemplate(deployment, deploymentMessage); @@ -147,98 +171,185 @@ protected void createX(DeploymentMessage deploymentMessage, OidcTokenId requeste .filter(node -> toscaService.isOfToscaType(node, ToscaConstants.Nodes.Types.KUBERNETES)) .collect(Collectors.toList()); - List containers = new ArrayList<>(); + Map resources = deployment.getResources().stream() + .filter(resource -> toscaService.isOfToscaType(resource, + ToscaConstants.Nodes.Types.KUBERNETES)) + .collect(Collectors.toMap(Resource::getToscaNodeName, res -> res)); + + LinkedHashMap containersByKuberNode = new LinkedHashMap(); + + List containersList = new ArrayList<>(); + + for (NodeTemplate kuberNode : orderedKubernetesApps) { + + Resource kuberResource = resources.get(kuberNode.getName()); + String id = Optional.ofNullable(kuberResource.getIaasId()).orElseGet(() -> { + kuberResource.setIaasId(kuberResource.getId()); + return kuberResource.getIaasId(); + }); + + + KubernetesTask kuberTask = buildTask(graph, kuberNode, id); + containersByKuberNode.put(kuberNode.getName(), kuberTask); + + V1Deployment kubernetesContainer = generateExternalTaskRepresentation(kuberTask); + DeepKubernetesContainer deepV1Container = new DeepKubernetesContainer(kubernetesContainer, kuberNode.getName()); + containersList.add(deepV1Container); + + List requestsRes = resourceRepository + .findByToscaNodeNameAndDeployment_id(kuberNode.getName(), deployment.getId()); + + } + + return containersList; + } + + /** + * Build a Kubernetes task object. + * @param graph the input nodegraph. + * @param taskNode the input tasknode. + * @param taskId the input taskid. + * @return the KubernetesTask. + */ + public KubernetesTask buildTask(DirectedMultigraph graph, + NodeTemplate taskNode, String taskId) { + + KubernetesTask kubernetesTask = new KubernetesTask(); + + // orchestrator internal + kubernetesTask.setId(taskId); + + // TODO MAP ALL PROPETIES FROM TOSCA + + Capability containerCapability = getHostCapability(graph, taskNode); + ToscaUtils + .extractScalar(containerCapability.getProperties(), "num_cpus", FloatType.class) + .ifPresent(kubernetesTask::setCpus); + + + return kubernetesTask; + } + + protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kubernetesTask) { + + V1Deployment v1Deployment = new V1Deployment(); + + v1Deployment.setApiVersion("apps/v1"); + //(kubernetesTask.getId()); +// if (qcgjob.getAttributes() != null) { +// job.setAttributes((HashMap) ((HashMap) +// qcgjob.getAttributes()).clone()); +// } +// job.setUser(qcgjob.getUser()); +// job.setState(qcgjob.getState()); +// job.setOperation(qcgjob.getOperation()); +// job.setNote(qcgjob.getNote()); + + return v1Deployment; + } + + private AppsV1Api connectApi(DeploymentMessage deploymentMessage) throws IOException { + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + String accessToken = oauth2TokenService + .getAccessToken(CommonUtils.checkNotNull(requestedWithToken)); + + ApiClient client = Config.fromToken( + deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint(), accessToken); + if (client == null) { + client = Config.defaultClient(); + } + return new AppsV1Api(client); + //TODO manage ApiException or throw some exception + } + + private V1Deployment createV1Deployment(DeploymentMessage deploymentMessage) { + /* note , cpu and ram are shared between all containers in the pod + * so it is enought difine it once*/ + + //TODO manage needed field for deployment and get them from DeploymentMessage + List containers = new ArrayList<>(); + Map requestsRes = new HashMap(); /*The expression 0.1 is equivalent to the expression 100m, *which can be read as “one hundred millicpu”.*/ requestsRes.put("cpu", new Quantity("32Mi")); requestsRes.put("memory", new Quantity("100m")); - + Map limitRes = new HashMap(); limitRes.put("cpu", new Quantity("64Mi")); limitRes.put("memory", new Quantity("200m")); - + V1ResourceRequirements resources = new V1ResourceRequirementsBuilder() .withRequests(requestsRes) .withLimits(limitRes) .build(); - + V1DeploymentSpec spec = new V1DeploymentSpec(); - V1ContainerBuilder cont = new V1ContainerBuilder().withResources(resources); + V1Container cont = new V1ContainerBuilder() + .withName("nginx") + .withImage("nginx:1.7.9") + .withResources(resources) + .addNewPort() + .withContainerPort(80) + .endPort() + .build(); + + Deployment deployment = getDeployment(deploymentMessage); + + V1Deployment v1Deployment = new V1DeploymentBuilder() + .withApiVersion("apps/v1") + .withKind("Deployment") + .withNewMetadata() + .withName(deployment.getId()) + .endMetadata() + .withNewSpec() + .withReplicas((Integer) deployment.getParameters().get("replicas")) + .withNewSelector() + .addToMatchLabels("app", "nginx") + .endSelector() + .withNewTemplate() + .withNewMetadata() + .addToLabels("app", "nginx") + .endMetadata() + .withNewSpec() + .addNewContainer() + .withImage("nginx:1.7.9") + .withName("nginx") + .addNewPort() + .withContainerPort(80) + .endPort() + .endContainer() + .withContainers(cont) + .endSpec() + .endTemplate() + .endSpec() + .build(); + + return v1Deployment; } @Override public boolean doDeploy(DeploymentMessage deploymentMessage) { - Deployment deployment = getDeployment(deploymentMessage); - V1Deployment outDepl = new V1Deployment(); - final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); - String accessToken = - oauth2TokenService.getAccessToken(CommonUtils.checkNotNull(requestedWithToken)); + V1Deployment v1Deployment = createV1Deployment(deploymentMessage); try { - // access configuration TODO - //ApiClient client = Config.defaultClient(); - //Configuration.setDefaultApiClient(client); - ApiClient client = Config.fromToken( - deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint(), accessToken); - - AppsV1Api app = new AppsV1Api(client); - String name = deployment.getId(); - - - /* note , cpu and ram are shared between all containers in the pod - * so it is enought difine it once*/ - - //TODO manage needed field for deployment and get them from DeploymentMessage - V1Deployment v1Deployment = new V1DeploymentBuilder() - .withApiVersion("apps/v1") - .withKind("Deployment") - .withNewMetadata() - .withName(name) - .endMetadata() - .withNewSpec() - .withReplicas((Integer) deployment.getParameters().get("replicas")) - .withNewSelector() - .addToMatchLabels("app", "nginx") - .endSelector() - .withNewTemplate() - .withNewMetadata() - .addToLabels("app", "nginx") - .endMetadata() - .withNewSpec() - .addNewContainer() - .withImage("nginx:1.7.9") - .withName("nginx") - .addNewPort() - .withContainerPort(80) - .endPort() - .endContainer() - .endSpec() - .endTemplate() - .endSpec() - .build(); - - outDepl = app.createNamespacedDeployment( + AppsV1Api app = connectApi(deploymentMessage); + app.createNamespacedDeployment( "default", v1Deployment, "true", null, null); - String ref = outDepl.getMetadata().getName(); - } catch (ApiException e) { - LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + LOG.error("Error in doDeploy:" + e.getCode() + " - " + e.getMessage()); + } catch (IOException e) { + LOG.error("Error in doDeploy:" + e.getCause() + " - " + e.getMessage()); } - // catch (IOException e) { - // LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); - // } - - LOG.info("Creating Kubernetes App Group for deployment {} with definition:\n{}", - deployment.getId(), outDepl.getMetadata().getName()); - + LOG.info("Creating Kubernetes V1Deployment for deployment {} with definition:\n{}", + deploymentMessage.getDeploymentId(), v1Deployment.getMetadata().getName()); return true; } @@ -248,12 +359,9 @@ public boolean isDeployed(DeploymentMessage deploymentMessage) { AppsV1Api app; V1Deployment v1Deployment = new V1Deployment(); - CoreV1Api api; try { app = new AppsV1Api(Config.defaultClient()); - api = new CoreV1Api(); - V1Namespace namespace = api.readNamespaceStatus("default", null); v1Deployment = app.readNamespacedDeploymentStatus(deployment.getId(), "default", "true"); printPodList(); @@ -267,6 +375,9 @@ public boolean isDeployed(DeploymentMessage deploymentMessage) { v1Deployment.getStatus().getReplicas() == v1Deployment.getStatus().getReadyReplicas(); LOG.debug("Kubernetes App Group for deployment {} is deployed? {}", deployment.getId(), isDeployed); + if (!isDeployed) { + LOG.warn(v1Deployment.getStatus().getConditions().get(0).getMessage()); + } return isDeployed; } @@ -386,8 +497,8 @@ protected Optional getAdditionalErrorInfoInternal(DeploymentMessage depl } private void printPodList() throws ApiException { - CoreV1Api api = new CoreV1Api(); - V1PodList list = api.listPodForAllNamespaces( + COREV1_API = new CoreV1Api(); + V1PodList list = COREV1_API.listPodForAllNamespaces( null, null, null, @@ -401,5 +512,43 @@ private void printPodList() throws ApiException { System.out.println(item.getMetadata().getName()); } } + + @Data + @NoArgsConstructor(access = AccessLevel.PROTECTED) + @RequiredArgsConstructor + public static class DeepKubernetesContainer { + + @NonNull + @NotNull + private V1Deployment v1Deployment; + + @NonNull + @NotNull + private String toscaNodeName; + + } + + protected Capability getHostCapability( + DirectedMultigraph graph, NodeTemplate taskNode) { + + NodeTemplate hostNode = getHostNode(graph, taskNode); + + // at this point we're sure that it exists + return hostNode.getCapabilities().get(HOST_CAPABILITY_NAME); + } + + protected NodeTemplate getHostNode(DirectedMultigraph graph, + NodeTemplate taskNode) { + return graph + .incomingEdgesOf(taskNode) + .stream() + .filter( + relationship -> HOST_CAPABILITY_NAME.equals(relationship.getTargetedCapabilityName())) + .map(graph::getEdgeSource) + // if more than 1 node is present -> IllegalArgumentException + .collect(MoreCollectors.toOptional()) + .orElseThrow(() -> new IllegalArgumentException( + String.format("No hosting node provided for node <%s>", taskNode.getName()))); + } } diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java index a37474f2a0..2577d050d4 100644 --- a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -38,6 +38,7 @@ import it.reply.orchestrator.service.ToscaService; import it.reply.orchestrator.service.ToscaServiceTest; import it.reply.orchestrator.service.VaultService; +import it.reply.orchestrator.service.security.OAuth2TokenService; import it.reply.orchestrator.util.TestUtil; import it.reply.orchestrator.utils.CommonUtils; @@ -62,6 +63,7 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; +import io.kubernetes.client.util.Config; import junitparams.JUnitParamsRunner; @RunWith(JUnitParamsRunner.class) @@ -83,6 +85,9 @@ public class KubernetesServiceTest extends ToscaParserAwareTest { @MockBean private VaultService vaultService; + @SpyBean + private Config config; + private static final String defaultVaultEndpoint = "https://default.vault.com:8200"; @Before @@ -92,10 +97,6 @@ public void setup() throws Exception { .when(oauth2tokenService.executeWithClientForResult( Mockito.any(), Mockito.any(), Mockito.any())) .thenAnswer(y -> ((ThrowingFunction) y.getArguments()[1]).apply("token")); - - Mockito - .when(oauth2tokenService.getAccessToken(Mockito.any())) - .thenAnswer(y -> ((ThrowingFunction) y.getArguments()[1]).apply("token")); } @Test @@ -109,15 +110,11 @@ public void testDoDeploy() throws IOException, URISyntaxException { csi.next(); dm.setCloudServicesOrderedIterator(csi); + Mockito .when(deploymentRepository.findOne(deployment.getId())) .thenReturn(deployment); - Mockito - .when(vaultService.getServiceUri()) - .thenReturn(Optional.of(new URI(defaultVaultEndpoint))); -// Mockito -// .when(marathonClientFactory.build(deployment.getCloudProviderEndpoint(), "token")) -// .thenReturn(marathonClient); + Assertions .assertThat(kubernetesServiceImpl.doDeploy(dm)) .isTrue(); From 857b9c27185322c24908a0ab12095043736310a3 Mon Sep 17 00:00:00 2001 From: Wareek Date: Thu, 20 Feb 2020 16:55:20 +0100 Subject: [PATCH 06/11] tosca to kubernetes deployment --- .../dto/kubernetes/KubernetesContainer.java | 35 +- .../dto/kubernetes/KubernetesTask.java | 20 +- .../providers/KubernetesServiceImpl.java | 311 +++++++++++------- 3 files changed, 233 insertions(+), 133 deletions(-) diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java index b6f0349539..a09c3c2969 100644 --- a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java @@ -16,21 +16,34 @@ package it.reply.orchestrator.dto.kubernetes; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; - import lombok.Data; +import lombok.Getter; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; @Data public class KubernetesContainer { - - private String id; - - private final String name; - - private final String toscaName; + @Nullable private String image; - - private Multimap parameters = ArrayListMultimap.create(); + + private Integer port; + + @NonNull + private Type type; + + @Getter + public enum Type { + DOCKER("DOCKER", "tosca.artifacts.Deployment.Image.Container.Docker"); + + Type(String name, String toscaName) { + this.name = name; + this.toscaName = toscaName; + } + + private final String name; + + private final String toscaName; + } } diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java index c8fe3e0104..a17ac08560 100644 --- a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java @@ -16,19 +16,31 @@ package it.reply.orchestrator.dto.kubernetes; +import io.kubernetes.client.custom.Quantity; + +import it.reply.orchestrator.utils.ToscaConstants; + +import java.util.List; + import lombok.Data; @Data public class KubernetesTask { - + + public final String getToscaNodeName() { + return ToscaConstants.Nodes.Types.KUBERNETES; + } + private String id; - private String cmd; + private List containers; private KubernetesContainer container; - private Double cpus; + private String cpu; + + private String memory; - private Double memSize; + private Double replicas; } diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index 11fe356a3d..f0766b9145 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -37,12 +37,6 @@ import io.kubernetes.client.openapi.models.V1Status; import io.kubernetes.client.util.Config; -import it.infn.ba.deep.qcg.client.model.Job; -import it.infn.ba.deep.qcg.client.model.JobDescription; -import it.infn.ba.deep.qcg.client.model.JobDescriptionExecution; -import it.infn.ba.deep.qcg.client.model.JobWorkingDirectoryPolicy; -import it.infn.ba.deep.qcg.client.model.RemoveConditionCreateMode; -import it.infn.ba.deep.qcg.client.model.RemoveConditionWhen; import it.reply.orchestrator.annotation.DeploymentProviderQualifier; import it.reply.orchestrator.dal.entity.Deployment; import it.reply.orchestrator.dal.entity.OidcTokenId; @@ -52,16 +46,16 @@ import it.reply.orchestrator.dto.deployment.DeploymentMessage; import it.reply.orchestrator.dto.kubernetes.KubernetesContainer; import it.reply.orchestrator.dto.kubernetes.KubernetesTask; +import it.reply.orchestrator.dto.mesos.MesosContainer; import it.reply.orchestrator.enums.DeploymentProvider; import it.reply.orchestrator.exception.service.BusinessWorkflowException; import it.reply.orchestrator.exception.service.DeploymentException; -import it.reply.orchestrator.exception.service.ToscaException; import it.reply.orchestrator.service.IndigoInputsPreProcessorService; import it.reply.orchestrator.service.IndigoInputsPreProcessorService.RuntimeProperties; -import it.reply.orchestrator.service.deployment.providers.QcgServiceImpl.DeepJob; import it.reply.orchestrator.service.ToscaService; import it.reply.orchestrator.service.security.OAuth2TokenService; import it.reply.orchestrator.utils.CommonUtils; +import it.reply.orchestrator.utils.EnumUtils; import it.reply.orchestrator.utils.OneDataUtils; import it.reply.orchestrator.utils.ToscaConstants; import it.reply.orchestrator.utils.ToscaUtils; @@ -84,12 +78,13 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.alien4cloud.tosca.model.definitions.DeploymentArtifact; import org.alien4cloud.tosca.model.templates.Capability; import org.alien4cloud.tosca.model.templates.NodeTemplate; import org.alien4cloud.tosca.model.templates.RelationshipTemplate; import org.alien4cloud.tosca.model.templates.Topology; -import org.alien4cloud.tosca.normative.types.BooleanType; import org.alien4cloud.tosca.normative.types.FloatType; +import org.alien4cloud.tosca.normative.types.StringType; import org.checkerframework.checker.nullness.qual.NonNull; import org.jgrapht.graph.DirectedMultigraph; import org.jgrapht.traverse.TopologicalOrderIterator; @@ -117,25 +112,9 @@ public class KubernetesServiceImpl extends AbstractDeploymentProviderService { private ResourceRepository resourceRepository; private static CoreV1Api COREV1_API; - + private static final String HOST_CAPABILITY_NAME = "host"; - // protected R executeWithClientForResult(CloudProviderEndpoint cloudProviderEndpoint, - // @Nullable OidcTokenId requestedWithToken, - // ThrowingFunction function) throws ApiException { - // return oauth2TokenService.executeWithClientForResult(requestedWithToken, - // token -> function.apply(kubernetesClientFactory.build(cloudProviderEndpoint, token)), - // ex -> ex instanceof ApiException && ((ApiException) ex).getCode() == 401); - // } - // - // protected void executeWithClient(CloudProviderEndpoint cloudProviderEndpoint, - // @Nullable OidcTokenId requestedWithToken, - // ThrowingConsumer consumer) throws ApiException { - // executeWithClientForResult(cloudProviderEndpoint, requestedWithToken, - // client -> consumer.asFunction().apply(client)); - // } - - //TODO if needed protected ArchiveRoot prepareTemplate(Deployment deployment, DeploymentMessage deploymentMessage) { RuntimeProperties runtimeProperties = @@ -150,8 +129,7 @@ protected ArchiveRoot prepareTemplate(Deployment deployment, return ar; } - protected List getContanersList(DeploymentMessage deploymentMessage, - OidcTokenId requestedWithToken) { + protected V1Deployment createV1Deployment(DeploymentMessage deploymentMessage) { Deployment deployment = getDeployment(deploymentMessage); ArchiveRoot ar = prepareTemplate(deployment, deploymentMessage); @@ -176,10 +154,13 @@ protected List getContanersList(DeploymentMessage deplo ToscaConstants.Nodes.Types.KUBERNETES)) .collect(Collectors.toMap(Resource::getToscaNodeName, res -> res)); - LinkedHashMap containersByKuberNode = new LinkedHashMap(); + LinkedHashMap containersByKuberNode = + new LinkedHashMap(); + + List deploymentList = new ArrayList<>(); + + V1Deployment kubernetesDeployment = new V1Deployment(); - List containersList = new ArrayList<>(); - for (NodeTemplate kuberNode : orderedKubernetesApps) { Resource kuberResource = resources.get(kuberNode.getName()); @@ -187,23 +168,21 @@ protected List getContanersList(DeploymentMessage deplo kuberResource.setIaasId(kuberResource.getId()); return kuberResource.getIaasId(); }); - - + KubernetesTask kuberTask = buildTask(graph, kuberNode, id); containersByKuberNode.put(kuberNode.getName(), kuberTask); - V1Deployment kubernetesContainer = generateExternalTaskRepresentation(kuberTask); - DeepKubernetesContainer deepV1Container = new DeepKubernetesContainer(kubernetesContainer, kuberNode.getName()); - containersList.add(deepV1Container); - - List requestsRes = resourceRepository - .findByToscaNodeNameAndDeployment_id(kuberNode.getName(), deployment.getId()); + kubernetesDeployment = generateExternalTaskRepresentation(kuberTask, deployment.getId()); + + DeepKubernetesDeployment deepV1Deployment = + new DeepKubernetesDeployment(kubernetesDeployment, kuberNode.getName()); + deploymentList.add(deepV1Deployment); } - return containersList; + return kubernetesDeployment; } - + /** * Build a Kubernetes task object. * @param graph the input nodegraph. @@ -222,105 +201,120 @@ public KubernetesTask buildTask(DirectedMultigraph new IllegalArgumentException( + String.format(" artifact not found in node <%s> of type <%s>", + taskNode.getName(), taskNode.getType()))); + // artifact type check + List supportedTypes = EnumUtils + .toList(MesosContainer.Type.class, MesosContainer.Type::getToscaName); + + KubernetesContainer.Type containerType = EnumUtils + .fromPredicate(KubernetesContainer.Type.class, + enumItem -> enumItem.getToscaName().equals(image.getArtifactType())) + .orElseThrow(() -> new IllegalArgumentException(String.format( + "Unsupported artifact type for artifact in node <%s> of type <%s>." + + " Given <%s>, supported <%s>", + taskNode, taskNode.getType(), image.getArtifactType(), supportedTypes))); + + KubernetesContainer container = new KubernetesContainer(containerType); + + String imageName = Optional + .ofNullable(image.getArtifactRef()) + .orElseThrow(() -> + new IllegalArgumentException( + " field for artifact in node <" + taskNode.getName() + + "> must be provided") + ); + + container.setImage(imageName); + + kubernetesTask.setContainer(container); + + /* TODO + * cpu and memory in Kubernetes Container are rappresented as Quantity.class, + * + * The expression 0.1 is equivalent to the expression 100m, + * which can be read as “one hundred millicpu”. */ - V1Deployment v1Deployment = new V1Deployment(); + ToscaUtils + .extractScalar(containerCapability.getProperties(), "num_cpus", StringType.class) + .ifPresent(kubernetesTask::setCpu); - v1Deployment.setApiVersion("apps/v1"); - //(kubernetesTask.getId()); -// if (qcgjob.getAttributes() != null) { -// job.setAttributes((HashMap) ((HashMap) -// qcgjob.getAttributes()).clone()); -// } -// job.setUser(qcgjob.getUser()); -// job.setState(qcgjob.getState()); -// job.setOperation(qcgjob.getOperation()); -// job.setNote(qcgjob.getNote()); + /* Limits and requests for memory are measured in bytes. + * You can express memory as a plain integer or as a fixed-point integer + * using one of these suffixes: E, P, T, G, M, K. + * You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki.*/ - return v1Deployment; - } + ToscaUtils + .extractScalar(containerCapability.getProperties(), "mem_size", StringType.class) + .ifPresent(kubernetesTask::setMemory); - private AppsV1Api connectApi(DeploymentMessage deploymentMessage) throws IOException { - final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); - String accessToken = oauth2TokenService - .getAccessToken(CommonUtils.checkNotNull(requestedWithToken)); + ToscaUtils + .extractScalar(containerCapability.getProperties(), "replicas", FloatType.class) + .ifPresent(kubernetesTask::setReplicas); - ApiClient client = Config.fromToken( - deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint(), accessToken); - if (client == null) { - client = Config.defaultClient(); - } - return new AppsV1Api(client); - //TODO manage ApiException or throw some exception + return kubernetesTask; } - private V1Deployment createV1Deployment(DeploymentMessage deploymentMessage) { - /* note , cpu and ram are shared between all containers in the pod - * so it is enought difine it once*/ - - //TODO manage needed field for deployment and get them from DeploymentMessage - List containers = new ArrayList<>(); + protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kubernetesTask, + String deploymentId) { - Map requestsRes = new HashMap(); - /*The expression 0.1 is equivalent to the expression 100m, - *which can be read as “one hundred millicpu”.*/ - requestsRes.put("cpu", new Quantity("32Mi")); - requestsRes.put("memory", new Quantity("100m")); + V1Deployment v1Deployment = new V1Deployment(); - Map limitRes = new HashMap(); - limitRes.put("cpu", new Quantity("64Mi")); - limitRes.put("memory", new Quantity("200m")); + v1Deployment.setApiVersion("apps/v1"); + v1Deployment.setKind("Deployment"); - V1ResourceRequirements resources = new V1ResourceRequirementsBuilder() - .withRequests(requestsRes) - .withLimits(limitRes) - .build(); + Map requestsRes = new HashMap(); - V1DeploymentSpec spec = new V1DeploymentSpec(); - V1Container cont = new V1ContainerBuilder() - .withName("nginx") - .withImage("nginx:1.7.9") - .withResources(resources) - .addNewPort() - .withContainerPort(80) - .endPort() - .build(); + //TODO cpu and ram to string and not quantity maybe + requestsRes.put("cpu", new Quantity(kubernetesTask.getCpu())); + requestsRes.put("memory", new Quantity(kubernetesTask.getMemory())); - Deployment deployment = getDeployment(deploymentMessage); + Map limitRes = new HashMap(); + limitRes.put("cpu", new Quantity(kubernetesTask.getCpu())); + limitRes.put("memory", new Quantity(kubernetesTask.getMemory())); + + List v1Containers = new ArrayList(); + for (KubernetesContainer cont : kubernetesTask.getContainers()) { + V1Container contV1 = new V1ContainerBuilder() + .withName(cont.getType().getName()) + .withImage(cont.getImage()) + .withNewResources() + .withRequests(requestsRes) + .endResources() + .addNewPort() + .withContainerPort(cont.getPort()) + .endPort() + .build(); + v1Containers.add(contV1); + } - V1Deployment v1Deployment = new V1DeploymentBuilder() + V1Deployment v1Deployment2 = new V1DeploymentBuilder() .withApiVersion("apps/v1") .withKind("Deployment") .withNewMetadata() - .withName(deployment.getId()) + .withName(kubernetesTask.getId()) .endMetadata() .withNewSpec() - .withReplicas((Integer) deployment.getParameters().get("replicas")) + .withReplicas(kubernetesTask.getReplicas().intValue()) /*TODO check if good value*/ .withNewSelector() - .addToMatchLabels("app", "nginx") + .addToMatchLabels("app", kubernetesTask.getToscaNodeName()) .endSelector() .withNewTemplate() .withNewMetadata() - .addToLabels("app", "nginx") + .addToLabels("app", kubernetesTask.getToscaNodeName()) .endMetadata() .withNewSpec() - .addNewContainer() - .withImage("nginx:1.7.9") - .withName("nginx") - .addNewPort() - .withContainerPort(80) - .endPort() - .endContainer() - .withContainers(cont) + .addAllToContainers(v1Containers) .endSpec() .endTemplate() .endSpec() @@ -329,6 +323,20 @@ private V1Deployment createV1Deployment(DeploymentMessage deploymentMessage) { return v1Deployment; } + private AppsV1Api connectApi(DeploymentMessage deploymentMessage) throws IOException { + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + String accessToken = oauth2TokenService + .getAccessToken(CommonUtils.checkNotNull(requestedWithToken)); + + ApiClient client = Config.fromToken( + deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint(), accessToken); + if (client == null) { + client = Config.defaultClient(); + } + return new AppsV1Api(client); + //TODO manage ApiException or throw some exception + } + @Override public boolean doDeploy(DeploymentMessage deploymentMessage) { @@ -512,11 +520,11 @@ private void printPodList() throws ApiException { System.out.println(item.getMetadata().getName()); } } - + @Data @NoArgsConstructor(access = AccessLevel.PROTECTED) @RequiredArgsConstructor - public static class DeepKubernetesContainer { + public static class DeepKubernetesDeployment { @NonNull @NotNull @@ -527,7 +535,7 @@ public static class DeepKubernetesContainer { private String toscaNodeName; } - + protected Capability getHostCapability( DirectedMultigraph graph, NodeTemplate taskNode) { @@ -536,7 +544,7 @@ protected Capability getHostCapability( // at this point we're sure that it exists return hostNode.getCapabilities().get(HOST_CAPABILITY_NAME); } - + protected NodeTemplate getHostNode(DirectedMultigraph graph, NodeTemplate taskNode) { return graph @@ -551,4 +559,71 @@ protected NodeTemplate getHostNode(DirectedMultigraph", taskNode.getName()))); } + private V1Deployment createV1DeploymentNotToUse(DeploymentMessage deploymentMessage) { + /* note , cpu and ram are shared between all containers in the pod + * so it is enought difine it once*/ + + //TODO manage needed field for deployment and get them from DeploymentMessage + List containers = new ArrayList<>(); + + Map requestsRes = new HashMap(); + /*The expression 0.1 is equivalent to the expression 100m, + *which can be read as “one hundred millicpu”.*/ + requestsRes.put("cpu", new Quantity("32Mi")); + requestsRes.put("memory", new Quantity("100m")); + + Map limitRes = new HashMap(); + limitRes.put("cpu", new Quantity("64Mi")); + limitRes.put("memory", new Quantity("200m")); + + V1ResourceRequirements resources = new V1ResourceRequirementsBuilder() + .withRequests(requestsRes) + .withLimits(limitRes) + .build(); + + V1DeploymentSpec spec = new V1DeploymentSpec(); + V1Container cont = new V1ContainerBuilder() + .withName("nginx") + .withImage("nginx:1.7.9") + .withResources(resources) + .addNewPort() + .withContainerPort(80) + .endPort() + .build(); + + Deployment deployment = getDeployment(deploymentMessage); + + V1Deployment v1Deployment = new V1DeploymentBuilder() + .withApiVersion("apps/v1") + .withKind("Deployment") + .withNewMetadata() + .withName(deployment.getId()) + .endMetadata() + .withNewSpec() + .withReplicas((Integer) deployment.getParameters().get("replicas")) + .withNewSelector() + .addToMatchLabels("app", "nginx") + .endSelector() + .withNewTemplate() + .withNewMetadata() + .addToLabels("app", "nginx") + .endMetadata() + .withNewSpec() + .addNewContainer() + .withImage("nginx:1.7.9") + .withName("nginx") + .addNewPort() + .withContainerPort(80) + .endPort() + .endContainer() + .withContainers(cont) + + .endSpec() + .endTemplate() + .endSpec() + .build(); + + return v1Deployment; + } + } From 234cc051ea18dc23f925250e57889ecb2ad86845 Mon Sep 17 00:00:00 2001 From: Wareek Date: Fri, 21 Feb 2020 19:18:07 +0100 Subject: [PATCH 07/11] edited isUndeployed method --- .../orchestrator/dto/cmdb/CloudService.java | 2 +- .../providers/KubernetesServiceImpl.java | 65 +++++++++++++------ .../providers/KubernetesServiceTest.java | 11 ++-- 3 files changed, 53 insertions(+), 25 deletions(-) diff --git a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java index 5ba27ae991..c4284474bf 100644 --- a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java +++ b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java @@ -114,7 +114,7 @@ public class CloudService implements CmdbIdentifiable { public static final String MARATHON_COMPUTE_SERVICE = INDIGO_SERVICE_PREFIX + ".marathon"; public static final String CHRONOS_COMPUTE_SERVICE = INDIGO_SERVICE_PREFIX + ".chronos"; public static final String QCG_COMPUTE_SERVICE = DEEP_SERVICE_PREFIX + ".qcg"; - public static final String KUBERNETES_COMPUTE_SERVICE = DEEP_SERVICE_PREFIX + ".kubernetes"; + public static final String KUBERNETES_COMPUTE_SERVICE = INDIGO_SERVICE_PREFIX + ".kubernetes"; /** * Get if the the service is a OpenStack compute service. diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index f0766b9145..a638e448b8 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -89,6 +89,7 @@ import org.jgrapht.graph.DirectedMultigraph; import org.jgrapht.traverse.TopologicalOrderIterator; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; @Service @@ -164,6 +165,7 @@ protected V1Deployment createV1Deployment(DeploymentMessage deploymentMessage) { for (NodeTemplate kuberNode : orderedKubernetesApps) { Resource kuberResource = resources.get(kuberNode.getName()); + //TODO Check what id it is String id = Optional.ofNullable(kuberResource.getIaasId()).orElseGet(() -> { kuberResource.setIaasId(kuberResource.getId()); return kuberResource.getIaasId(); @@ -268,10 +270,10 @@ public KubernetesTask buildTask(DirectedMultigraph requestsRes = new HashMap(); @@ -298,7 +300,7 @@ protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kuberne v1Containers.add(contV1); } - V1Deployment v1Deployment2 = new V1DeploymentBuilder() + V1Deployment v1Deployment = new V1DeploymentBuilder() .withApiVersion("apps/v1") .withKind("Deployment") .withNewMetadata() @@ -332,6 +334,7 @@ private AppsV1Api connectApi(DeploymentMessage deploymentMessage) throws IOExcep deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint(), accessToken); if (client == null) { client = Config.defaultClient(); + //client.setBasePath("https://kubernetes.docker.internal:6443"); } return new AppsV1Api(client); //TODO manage ApiException or throw some exception @@ -341,6 +344,7 @@ private AppsV1Api connectApi(DeploymentMessage deploymentMessage) throws IOExcep public boolean doDeploy(DeploymentMessage deploymentMessage) { V1Deployment v1Deployment = createV1Deployment(deploymentMessage); + //V1Deployment v1Deployment = createV1DeploymentForTest(deploymentMessage); try { AppsV1Api app = connectApi(deploymentMessage); @@ -368,7 +372,7 @@ public boolean isDeployed(DeploymentMessage deploymentMessage) { AppsV1Api app; V1Deployment v1Deployment = new V1Deployment(); try { - app = new AppsV1Api(Config.defaultClient()); + app = connectApi(deploymentMessage); v1Deployment = app.readNamespacedDeploymentStatus(deployment.getId(), "default", "true"); @@ -438,17 +442,8 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { Deployment deployment = getDeployment(deploymentMessage); AppsV1Api app; try { - app = new AppsV1Api(Config.defaultClient()); + app = connectApi(deploymentMessage); - // ApiResponse response = app.deleteNamespacedDeploymentWithHttpInfo( - // deployment.getId(), - // "default", - // "true", - // null, - // null, - // null, - // null, - // null); V1Status status = app.deleteNamespacedDeployment( deployment.getId(), "default", @@ -459,7 +454,6 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { null, null); - //response.getStatusCode(); LOG.debug("Deleting deployment exited with :" + status.getCode() + " - " @@ -481,8 +475,40 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { @Override public boolean isUndeployed(DeploymentMessage deploymentMessage) { - // TODO Auto-generated method stub - return false; + boolean isUndeployed = false; + Deployment deployment = getDeployment(deploymentMessage); + + AppsV1Api app; + try { + app = connectApi(deploymentMessage); + + V1Deployment deplSimleToCheck = app.readNamespacedDeploymentStatus( + deployment.getId(), + "default", + "true"); + + V1Deployment depl = app.readNamespacedDeployment( + deployment.getId(), + "default", + "true", + null, + null); + + if (depl==null) { + isUndeployed = true; + } + + } catch (ApiException e) { + LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + if (e.getCode() == 404) { + isUndeployed = true; + } + } catch (IOException e) { + LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + } + + + return isUndeployed; } @Override @@ -559,12 +585,11 @@ protected NodeTemplate getHostNode(DirectedMultigraph", taskNode.getName()))); } - private V1Deployment createV1DeploymentNotToUse(DeploymentMessage deploymentMessage) { + private V1Deployment createV1DeploymentForTest(DeploymentMessage deploymentMessage) { /* note , cpu and ram are shared between all containers in the pod * so it is enought difine it once*/ //TODO manage needed field for deployment and get them from DeploymentMessage - List containers = new ArrayList<>(); Map requestsRes = new HashMap(); /*The expression 0.1 is equivalent to the expression 100m, diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java index 2577d050d4..558a37fca9 100644 --- a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -97,6 +97,7 @@ public void setup() throws Exception { .when(oauth2tokenService.executeWithClientForResult( Mockito.any(), Mockito.any(), Mockito.any())) .thenAnswer(y -> ((ThrowingFunction) y.getArguments()[1]).apply("token")); + } @Test @@ -218,12 +219,14 @@ private Deployment generateDeployment() throws IOException { private KubernetesService buildService() { KubernetesService cs = KubernetesService .kubernetesBuilder() - .endpoint("example.com/kubernetes") + .endpoint("http://localhost:8080") .serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) - .hostname("example.com") - .providerId("provider-1") - .id("provider-1-service-1") + .hostname("localhost") + .providerId("RECAS-BARI") + .id("http://localhost:8080") .type(CloudServiceType.COMPUTE) + .iamEnabled(false) + .publicService(true) .build(); return cs; } From 7d3af5b25b5f3e67a4e28e584822d713ebd2d4ec Mon Sep 17 00:00:00 2001 From: Wareek Date: Tue, 25 Feb 2020 16:11:29 +0100 Subject: [PATCH 08/11] doDeployment edited --- .../dto/kubernetes/KubernetesTask.java | 2 + .../providers/KubernetesServiceImpl.java | 163 +++++++++++------- .../providers/KubernetesServiceTest.java | 69 +++++--- src/test/resources/tosca/kubernetes_app.yaml | 3 - 4 files changed, 149 insertions(+), 88 deletions(-) diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java index a17ac08560..8ef8d6b317 100644 --- a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java @@ -43,4 +43,6 @@ public final String getToscaNodeName() { private Double replicas; + private Integer instances; + } diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index a638e448b8..0a069b0a22 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -85,6 +85,8 @@ import org.alien4cloud.tosca.model.templates.Topology; import org.alien4cloud.tosca.normative.types.FloatType; import org.alien4cloud.tosca.normative.types.StringType; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; import org.checkerframework.checker.nullness.qual.NonNull; import org.jgrapht.graph.DirectedMultigraph; import org.jgrapht.traverse.TopologicalOrderIterator; @@ -150,10 +152,10 @@ protected V1Deployment createV1Deployment(DeploymentMessage deploymentMessage) { .filter(node -> toscaService.isOfToscaType(node, ToscaConstants.Nodes.Types.KUBERNETES)) .collect(Collectors.toList()); - Map resources = deployment.getResources().stream() - .filter(resource -> toscaService.isOfToscaType(resource, - ToscaConstants.Nodes.Types.KUBERNETES)) - .collect(Collectors.toMap(Resource::getToscaNodeName, res -> res)); + // Map resources = deployment.getResources().stream() + // .filter(resource -> toscaService.isOfToscaType(resource, + // ToscaConstants.Nodes.Types.KUBERNETES)) + // .collect(Collectors.toMap(Resource::getToscaNodeName, res -> res)); LinkedHashMap containersByKuberNode = new LinkedHashMap(); @@ -164,14 +166,26 @@ protected V1Deployment createV1Deployment(DeploymentMessage deploymentMessage) { for (NodeTemplate kuberNode : orderedKubernetesApps) { - Resource kuberResource = resources.get(kuberNode.getName()); - //TODO Check what id it is + KubernetesTask kuberTask = buildTask(graph, kuberNode, kuberNode.getName()); + + List resources = resourceRepository + .findByToscaNodeNameAndDeployment_id(kuberNode.getName(), deployment.getId()); + + resources.forEach(resource -> resource.setIaasId(kuberTask.getId())); + kuberTask.setInstances(resources.size()); + + // Resource kuberResource = resources.get(resources.indexOf(kuberNode.getName())); + Resource kuberResource = resources.stream() + .filter(resource -> kuberNode.getName().equals(resource.getToscaNodeName())) + .findAny() + .orElse(null); + String id = Optional.ofNullable(kuberResource.getIaasId()).orElseGet(() -> { kuberResource.setIaasId(kuberResource.getId()); return kuberResource.getIaasId(); }); - KubernetesTask kuberTask = buildTask(graph, kuberNode, id); + //TODO Check what id it is containersByKuberNode.put(kuberNode.getName(), kuberTask); kubernetesDeployment = generateExternalTaskRepresentation(kuberTask, deployment.getId()); @@ -257,12 +271,13 @@ public KubernetesTask buildTask(DirectedMultigraph requestsRes = new HashMap(); + String mem = kubernetesTask.getMemory().replace("Mb", "M").replaceAll(" ", ""); + //TODO cpu and ram to string and not quantity maybe requestsRes.put("cpu", new Quantity(kubernetesTask.getCpu())); - requestsRes.put("memory", new Quantity(kubernetesTask.getMemory())); + requestsRes.put("memory", new Quantity(mem)); Map limitRes = new HashMap(); limitRes.put("cpu", new Quantity(kubernetesTask.getCpu())); - limitRes.put("memory", new Quantity(kubernetesTask.getMemory())); + limitRes.put("memory", new Quantity(mem)); List v1Containers = new ArrayList(); - for (KubernetesContainer cont : kubernetesTask.getContainers()) { + if (kubernetesTask.getContainers() == null || kubernetesTask.getContainers().isEmpty()) { V1Container contV1 = new V1ContainerBuilder() - .withName(cont.getType().getName()) - .withImage(cont.getImage()) + .withName(kubernetesTask.getContainer().getType().getName()) + .withImage(kubernetesTask.getContainer().getImage()) .withNewResources() .withRequests(requestsRes) .endResources() .addNewPort() - .withContainerPort(cont.getPort()) + .withContainerPort(kubernetesTask.getContainer().getPort()) .endPort() .build(); v1Containers.add(contV1); + } else { + for (KubernetesContainer cont : kubernetesTask.getContainers()) { + V1Container contV1 = new V1ContainerBuilder() + .withName(cont.getType().getName()) + .withImage(cont.getImage()) + .withNewResources() + .withRequests(requestsRes) + .endResources() + .addNewPort() + .withContainerPort(cont.getPort()) + .endPort() + .build(); + v1Containers.add(contV1); + } } V1Deployment v1Deployment = new V1DeploymentBuilder() @@ -325,11 +356,22 @@ protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kuberne return v1Deployment; } - private AppsV1Api connectApi(DeploymentMessage deploymentMessage) throws IOException { + /** + * Connecting Kubernetes Api config. + * @param deploymentMessage DeploymentMessage as parameter + * @return + */ + public AppsV1Api connectApi(DeploymentMessage deploymentMessage) throws IOException { final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); String accessToken = oauth2TokenService .getAccessToken(CommonUtils.checkNotNull(requestedWithToken)); + ApiClient x = new ApiClient(); + x.setAccessToken(accessToken); + x.setReadTimeout(Integer.parseInt(deploymentMessage.getTimeout())); + x.setConnectTimeout(Integer.parseInt(deploymentMessage.getTimeout())); + x.setBasePath(deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint()); + ApiClient client = Config.fromToken( deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint(), accessToken); if (client == null) { @@ -355,6 +397,7 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { null, null); + //TODO handle exception in out } catch (ApiException e) { LOG.error("Error in doDeploy:" + e.getCode() + " - " + e.getMessage()); } catch (IOException e) { @@ -378,9 +421,9 @@ public boolean isDeployed(DeploymentMessage deploymentMessage) { printPodList(); } catch (ApiException e) { - LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + LOG.error("Error in isDeployed:" + e.getCode() + " - " + e.getMessage()); } catch (IOException e) { - LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + LOG.error("Error in isDeployed:" + e.getCause() + " - " + e.getMessage()); } boolean isDeployed = @@ -424,9 +467,9 @@ public boolean doUpdate(DeploymentMessage deploymentMessage, String template) { // app.replaceNamespacedDeployment(name, namespace, body, pretty, dryRun, fieldManager); } catch (ApiException e) { - LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + LOG.error("Error in doUpdate:" + e.getCode() + " - " + e.getMessage()); } catch (IOException e) { - LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + LOG.error("Error in doUpdate:" + e.getCause() + " - " + e.getMessage()); } throw new UnsupportedOperationException("Marathon app deployments do not support update."); } @@ -440,35 +483,38 @@ public void cleanFailedUpdate(DeploymentMessage deploymentMessage) { public boolean doUndeploy(DeploymentMessage deploymentMessage) { Deployment deployment = getDeployment(deploymentMessage); - AppsV1Api app; - try { - app = connectApi(deploymentMessage); - - V1Status status = app.deleteNamespacedDeployment( - deployment.getId(), - "default", - "true", - null, - null, - null, - null, - null); - - LOG.debug("Deleting deployment exited with :" - + status.getCode() - + " - " - + status.getMessage() - + " - " - + status.getStatus()); - - } catch (ApiException e) { - LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); - // TODO manage throwing errorCode exception - // if(e.getCode()!=404) { - // throw new HttpResponseException(e.getCode(), "KubernetesApiException"); - // } - } catch (IOException e) { - LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + CloudProviderEndpoint cloudProviderEndpoint = deployment.getCloudProviderEndpoint(); + if (cloudProviderEndpoint != null) { + AppsV1Api app; + try { + app = connectApi(deploymentMessage); + + V1Status status = app.deleteNamespacedDeployment( + deployment.getId(), + "default", + "true", + null, + null, + null, + null, + null); + + LOG.debug("Deleting deployment exited with :" + + status.getCode() + + " - " + + status.getMessage() + + " - " + + status.getStatus()); + + } catch (ApiException e) { + LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + // TODO manage throwing errorCode exception + // if(e.getCode()!=404) { + // throw new HttpResponseException(e.getCode(), "KubernetesApiException"); + // } + } catch (IOException e) { + LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + } } return true; } @@ -477,7 +523,7 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { public boolean isUndeployed(DeploymentMessage deploymentMessage) { boolean isUndeployed = false; Deployment deployment = getDeployment(deploymentMessage); - + AppsV1Api app; try { app = connectApi(deploymentMessage); @@ -486,7 +532,7 @@ public boolean isUndeployed(DeploymentMessage deploymentMessage) { deployment.getId(), "default", "true"); - + V1Deployment depl = app.readNamespacedDeployment( deployment.getId(), "default", @@ -494,9 +540,9 @@ public boolean isUndeployed(DeploymentMessage deploymentMessage) { null, null); - if (depl==null) { - isUndeployed = true; - } + if (depl == null) { + isUndeployed = true; + } } catch (ApiException e) { LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); @@ -506,8 +552,7 @@ public boolean isUndeployed(DeploymentMessage deploymentMessage) { } catch (IOException e) { LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); } - - + return isUndeployed; } diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java index 558a37fca9..549e785384 100644 --- a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -16,8 +16,37 @@ package it.reply.orchestrator.service.deployment.providers; +import static org.mockito.Mockito.doReturn; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.assertj.core.api.Assertions; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.junit4.rules.SpringClassRule; +import org.springframework.test.context.junit4.rules.SpringMethodRule; + import com.google.common.collect.Lists; +import io.kubernetes.client.openapi.apis.AppsV1Api; +import io.kubernetes.client.util.Config; import it.reply.orchestrator.config.specific.ToscaParserAwareTest; import it.reply.orchestrator.controller.ControllerTestUtils; import it.reply.orchestrator.dal.entity.Deployment; @@ -38,37 +67,19 @@ import it.reply.orchestrator.service.ToscaService; import it.reply.orchestrator.service.ToscaServiceTest; import it.reply.orchestrator.service.VaultService; -import it.reply.orchestrator.service.security.OAuth2TokenService; import it.reply.orchestrator.util.TestUtil; -import it.reply.orchestrator.utils.CommonUtils; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; - -import org.assertj.core.api.Assertions; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.boot.test.mock.mockito.SpyBean; - -import io.kubernetes.client.util.Config; import junitparams.JUnitParamsRunner; @RunWith(JUnitParamsRunner.class) public class KubernetesServiceTest extends ToscaParserAwareTest { + @ClassRule + public static final SpringClassRule SPRING_CLASS_RULE = new SpringClassRule(); + + @Rule + public final SpringMethodRule springMethodRule = new SpringMethodRule(); + + @Spy @InjectMocks private KubernetesServiceImpl kubernetesServiceImpl; @@ -97,7 +108,7 @@ public void setup() throws Exception { .when(oauth2tokenService.executeWithClientForResult( Mockito.any(), Mockito.any(), Mockito.any())) .thenAnswer(y -> ((ThrowingFunction) y.getArguments()[1]).apply("token")); - + } @Test @@ -115,7 +126,13 @@ public void testDoDeploy() throws IOException, URISyntaxException { Mockito .when(deploymentRepository.findOne(deployment.getId())) .thenReturn(deployment); + ; + + doReturn(new AppsV1Api(Config.defaultClient())).when(kubernetesServiceImpl).connectApi(dm); +// Mockito +// .when(toscaService.parseAndValidateTemplate(deployment.getTemplate(), inputs)) +// .thenReturn(deployment); Assertions .assertThat(kubernetesServiceImpl.doDeploy(dm)) .isTrue(); diff --git a/src/test/resources/tosca/kubernetes_app.yaml b/src/test/resources/tosca/kubernetes_app.yaml index a048553d28..809c9279cd 100644 --- a/src/test/resources/tosca/kubernetes_app.yaml +++ b/src/test/resources/tosca/kubernetes_app.yaml @@ -9,9 +9,6 @@ topology_template: kubernetes: type: tosca.nodes.indigo.Container.Application.Docker.Kubernetes - properties: - uris: [] - enable_https: true artifacts: image: file: hello-world From d0f0a32a1f4dc156f213cecfa06e0df5731b3d32 Mon Sep 17 00:00:00 2001 From: "A.Lapshynov" Date: Tue, 25 Feb 2020 17:57:05 +0100 Subject: [PATCH 09/11] deployment and undeployment tests --- .../providers/KubernetesServiceImpl.java | 13 +++-- .../providers/KubernetesServiceTest.java | 47 ++++++++++++++++--- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index 0a069b0a22..d9d5c71996 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -85,13 +85,10 @@ import org.alien4cloud.tosca.model.templates.Topology; import org.alien4cloud.tosca.normative.types.FloatType; import org.alien4cloud.tosca.normative.types.StringType; -import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; -import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; import org.checkerframework.checker.nullness.qual.NonNull; import org.jgrapht.graph.DirectedMultigraph; import org.jgrapht.traverse.TopologicalOrderIterator; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; @Service @@ -400,8 +397,10 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { //TODO handle exception in out } catch (ApiException e) { LOG.error("Error in doDeploy:" + e.getCode() + " - " + e.getMessage()); + throw new DeploymentException("Error in doDeploy:" + e.getCode() + " - " + e.getMessage(), e); } catch (IOException e) { LOG.error("Error in doDeploy:" + e.getCause() + " - " + e.getMessage()); + throw new DeploymentException("Error in doDeploy:" + e.getCause() + " - " + e.getMessage(), e); } LOG.info("Creating Kubernetes V1Deployment for deployment {} with definition:\n{}", deploymentMessage.getDeploymentId(), v1Deployment.getMetadata().getName()); @@ -422,8 +421,10 @@ public boolean isDeployed(DeploymentMessage deploymentMessage) { printPodList(); } catch (ApiException e) { LOG.error("Error in isDeployed:" + e.getCode() + " - " + e.getMessage()); + throw new DeploymentException(e.getMessage(), e); } catch (IOException e) { LOG.error("Error in isDeployed:" + e.getCause() + " - " + e.getMessage()); + throw new DeploymentException(e.getMessage(), e); } boolean isDeployed = @@ -468,8 +469,10 @@ public boolean doUpdate(DeploymentMessage deploymentMessage, String template) { } catch (ApiException e) { LOG.error("Error in doUpdate:" + e.getCode() + " - " + e.getMessage()); + throw new DeploymentException(e.getMessage(), e); } catch (IOException e) { LOG.error("Error in doUpdate:" + e.getCause() + " - " + e.getMessage()); + throw new DeploymentException(e.getMessage(), e); } throw new UnsupportedOperationException("Marathon app deployments do not support update."); } @@ -512,8 +515,10 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { // if(e.getCode()!=404) { // throw new HttpResponseException(e.getCode(), "KubernetesApiException"); // } + throw new DeploymentException("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage(), e); } catch (IOException e) { LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + throw new DeploymentException("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage(), e); } } return true; @@ -549,8 +554,10 @@ public boolean isUndeployed(DeploymentMessage deploymentMessage) { if (e.getCode() == 404) { isUndeployed = true; } + throw new DeploymentException(e.getMessage(), e); } catch (IOException e) { LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); + throw new DeploymentException(e.getMessage(), e); } return isUndeployed; diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java index 549e785384..0faff2f902 100644 --- a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -16,16 +16,20 @@ package it.reply.orchestrator.service.deployment.providers; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.mockito.Mockito.doReturn; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; +import org.assertj.core.api.AbstractThrowableAssert; import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.ClassRule; @@ -33,7 +37,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; -import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; @@ -45,6 +48,7 @@ import com.google.common.collect.Lists; +import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.AppsV1Api; import io.kubernetes.client.util.Config; import it.reply.orchestrator.config.specific.ToscaParserAwareTest; @@ -58,11 +62,14 @@ import it.reply.orchestrator.dto.cmdb.CloudService; import it.reply.orchestrator.dto.cmdb.CloudServiceType; import it.reply.orchestrator.dto.cmdb.KubernetesService; +import it.reply.orchestrator.dto.cmdb.MarathonService; import it.reply.orchestrator.dto.deployment.DeploymentMessage; import it.reply.orchestrator.dto.onedata.OneData; import it.reply.orchestrator.dto.onedata.OneData.OneDataProviderInfo; import it.reply.orchestrator.dto.workflow.CloudServicesOrderedIterator; import it.reply.orchestrator.enums.NodeStates; +import it.reply.orchestrator.exception.service.BusinessWorkflowException; +import it.reply.orchestrator.exception.service.DeploymentException; import it.reply.orchestrator.function.ThrowingFunction; import it.reply.orchestrator.service.ToscaService; import it.reply.orchestrator.service.ToscaServiceTest; @@ -122,7 +129,6 @@ public void testDoDeploy() throws IOException, URISyntaxException { csi.next(); dm.setCloudServicesOrderedIterator(csi); - Mockito .when(deploymentRepository.findOne(deployment.getId())) .thenReturn(deployment); @@ -130,12 +136,39 @@ public void testDoDeploy() throws IOException, URISyntaxException { doReturn(new AppsV1Api(Config.defaultClient())).when(kubernetesServiceImpl).connectApi(dm); -// Mockito -// .when(toscaService.parseAndValidateTemplate(deployment.getTemplate(), inputs)) -// .thenReturn(deployment); Assertions - .assertThat(kubernetesServiceImpl.doDeploy(dm)) - .isTrue(); + .assertThatExceptionOfType(DeploymentException.class); + + AbstractThrowableAssert assertion = assertThatCode( + () -> kubernetesServiceImpl.doDeploy(dm)); + + assertion.isInstanceOf(DeploymentException.class) + .hasCauseExactlyInstanceOf(ApiException.class); + } + + @Test + public void testDoUndeploy() throws IOException { + Deployment deployment = generateDeployment(); + DeploymentMessage dm = generateDeployDm(deployment); + + KubernetesService cs = buildService(); + + CloudServicesOrderedIterator csi = new CloudServicesOrderedIterator(Lists.newArrayList(cs)); + csi.next(); + dm.setCloudServicesOrderedIterator(csi); + + Mockito + .when(deploymentRepository.findOne(deployment.getId())) + .thenReturn(deployment); + ; + + doReturn(new AppsV1Api(Config.defaultClient())).when(kubernetesServiceImpl).connectApi(dm); + + AbstractThrowableAssert assertion = assertThatCode( + () -> kubernetesServiceImpl.doUndeploy(dm)); + + assertion.isInstanceOf(DeploymentException.class) + .hasCauseExactlyInstanceOf(ApiException.class); } private DeploymentMessage generateDeployDmKuber(Deployment deployment) { From c755647ef0c77bd88300f2657353184dfd4dcf13 Mon Sep 17 00:00:00 2001 From: "A.Lapshynov" Date: Wed, 26 Feb 2020 14:56:24 +0100 Subject: [PATCH 10/11] connectApi edit --- .../providers/KubernetesServiceImpl.java | 49 +++++++++++------ .../providers/KubernetesServiceTest.java | 52 ++++++++++++++++--- 2 files changed, 77 insertions(+), 24 deletions(-) diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index d9d5c71996..beca3195fa 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -25,6 +25,7 @@ import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.AppsV1Api; import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.auth.ApiKeyAuth; import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1ContainerBuilder; import io.kubernetes.client.openapi.models.V1Deployment; @@ -73,6 +74,7 @@ import javax.validation.constraints.NotNull; import lombok.AccessLevel; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; @@ -358,19 +360,23 @@ protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kuberne * @param deploymentMessage DeploymentMessage as parameter * @return */ - public AppsV1Api connectApi(DeploymentMessage deploymentMessage) throws IOException { + public AppsV1Api connectApi(DeploymentMessage deploymentMessage, + String cpEndpoint) throws IOException { final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); String accessToken = oauth2TokenService .getAccessToken(CommonUtils.checkNotNull(requestedWithToken)); ApiClient x = new ApiClient(); - x.setAccessToken(accessToken); - x.setReadTimeout(Integer.parseInt(deploymentMessage.getTimeout())); - x.setConnectTimeout(Integer.parseInt(deploymentMessage.getTimeout())); - x.setBasePath(deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint()); - ApiClient client = Config.fromToken( - deploymentMessage.getChosenCloudProviderEndpoint().getCpEndpoint(), accessToken); + ApiKeyAuth bearerToken = (ApiKeyAuth) x.getAuthentication("BearerToken"); + bearerToken.setApiKey(accessToken); + + //x.setAccessToken(accessToken); + if (cpEndpoint != null && !cpEndpoint.isEmpty()) { + x.setBasePath(cpEndpoint); + } + + ApiClient client = Config.fromToken(cpEndpoint, accessToken); if (client == null) { client = Config.defaultClient(); //client.setBasePath("https://kubernetes.docker.internal:6443"); @@ -381,12 +387,13 @@ public AppsV1Api connectApi(DeploymentMessage deploymentMessage) throws IOExcept @Override public boolean doDeploy(DeploymentMessage deploymentMessage) { - + Deployment deployment = getDeployment(deploymentMessage); V1Deployment v1Deployment = createV1Deployment(deploymentMessage); //V1Deployment v1Deployment = createV1DeploymentForTest(deploymentMessage); try { - AppsV1Api app = connectApi(deploymentMessage); + AppsV1Api app = connectApi(deploymentMessage, + deployment.getCloudProviderEndpoint().getCpEndpoint()); app.createNamespacedDeployment( "default", v1Deployment, @@ -397,10 +404,12 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { //TODO handle exception in out } catch (ApiException e) { LOG.error("Error in doDeploy:" + e.getCode() + " - " + e.getMessage()); - throw new DeploymentException("Error in doDeploy:" + e.getCode() + " - " + e.getMessage(), e); + throw new DeploymentException("Error in doDeploy:" + + e.getCode() + " - " + e.getMessage(), e); } catch (IOException e) { LOG.error("Error in doDeploy:" + e.getCause() + " - " + e.getMessage()); - throw new DeploymentException("Error in doDeploy:" + e.getCause() + " - " + e.getMessage(), e); + throw new DeploymentException("Error in doDeploy:" + + e.getCause() + " - " + e.getMessage(), e); } LOG.info("Creating Kubernetes V1Deployment for deployment {} with definition:\n{}", deploymentMessage.getDeploymentId(), v1Deployment.getMetadata().getName()); @@ -414,7 +423,7 @@ public boolean isDeployed(DeploymentMessage deploymentMessage) { AppsV1Api app; V1Deployment v1Deployment = new V1Deployment(); try { - app = connectApi(deploymentMessage); + app = connectApi(deploymentMessage, deployment.getCloudProviderEndpoint().getCpEndpoint()); v1Deployment = app.readNamespacedDeploymentStatus(deployment.getId(), "default", "true"); @@ -490,7 +499,7 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { if (cloudProviderEndpoint != null) { AppsV1Api app; try { - app = connectApi(deploymentMessage); + app = connectApi(deploymentMessage, deployment.getCloudProviderEndpoint().getCpEndpoint()); V1Status status = app.deleteNamespacedDeployment( deployment.getId(), @@ -515,10 +524,13 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { // if(e.getCode()!=404) { // throw new HttpResponseException(e.getCode(), "KubernetesApiException"); // } - throw new DeploymentException("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage(), e); + throw new DeploymentException("Error in doUndeploy:" + + e.getCode() + " - " + e.getMessage(), e); } catch (IOException e) { - LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); - throw new DeploymentException("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage(), e); + LOG.error("Error in doUndeploy:" + + e.getCause() + " - " + e.getMessage()); + throw new DeploymentException("Error in doUndeploy:" + + e.getCause() + " - " + e.getMessage(), e); } } return true; @@ -531,7 +543,7 @@ public boolean isUndeployed(DeploymentMessage deploymentMessage) { AppsV1Api app; try { - app = connectApi(deploymentMessage); + app = connectApi(deploymentMessage, deployment.getCloudProviderEndpoint().getCpEndpoint()); V1Deployment deplSimleToCheck = app.readNamespacedDeploymentStatus( deployment.getId(), @@ -704,3 +716,6 @@ private V1Deployment createV1DeploymentForTest(DeploymentMessage deploymentMessa } } + + + diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java index 0faff2f902..3fd32dd334 100644 --- a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -31,6 +31,7 @@ import org.assertj.core.api.AbstractThrowableAssert; import org.assertj.core.api.Assertions; +import org.hamcrest.core.IsEqual; import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; @@ -48,6 +49,7 @@ import com.google.common.collect.Lists; +import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.AppsV1Api; import io.kubernetes.client.util.Config; @@ -74,6 +76,7 @@ import it.reply.orchestrator.service.ToscaService; import it.reply.orchestrator.service.ToscaServiceTest; import it.reply.orchestrator.service.VaultService; +import it.reply.orchestrator.service.security.OAuth2TokenService; import it.reply.orchestrator.util.TestUtil; import junitparams.JUnitParamsRunner; @@ -103,6 +106,9 @@ public class KubernetesServiceTest extends ToscaParserAwareTest { @MockBean private VaultService vaultService; + @SpyBean + private OAuth2TokenService oauth2TokenService; + @SpyBean private Config config; @@ -134,24 +140,25 @@ public void testDoDeploy() throws IOException, URISyntaxException { .thenReturn(deployment); ; - doReturn(new AppsV1Api(Config.defaultClient())).when(kubernetesServiceImpl).connectApi(dm); + doReturn(new AppsV1Api(Config.defaultClient())).when(kubernetesServiceImpl) + .connectApi(dm, deployment.getCloudProviderEndpoint().getCpEndpoint()); Assertions .assertThatExceptionOfType(DeploymentException.class); - + AbstractThrowableAssert assertion = assertThatCode( () -> kubernetesServiceImpl.doDeploy(dm)); - + assertion.isInstanceOf(DeploymentException.class) .hasCauseExactlyInstanceOf(ApiException.class); } - + @Test public void testDoUndeploy() throws IOException { Deployment deployment = generateDeployment(); DeploymentMessage dm = generateDeployDm(deployment); - KubernetesService cs = buildService(); + KubernetesService cs = buildServiceLocal(); CloudServicesOrderedIterator csi = new CloudServicesOrderedIterator(Lists.newArrayList(cs)); csi.next(); @@ -162,15 +169,32 @@ public void testDoUndeploy() throws IOException { .thenReturn(deployment); ; - doReturn(new AppsV1Api(Config.defaultClient())).when(kubernetesServiceImpl).connectApi(dm); + doReturn(new AppsV1Api(Config.defaultClient())).when(kubernetesServiceImpl) + .connectApi(dm, deployment.getCloudProviderEndpoint().getCpEndpoint()); AbstractThrowableAssert assertion = assertThatCode( () -> kubernetesServiceImpl.doUndeploy(dm)); - + assertion.isInstanceOf(DeploymentException.class) .hasCauseExactlyInstanceOf(ApiException.class); } + // @Test + // public void testConnectApi() throws IOException { + // Deployment deployment = generateDeployment(); + // DeploymentMessage dm = generateDeployDm(deployment); + // + // AppsV1Api expected = new AppsV1Api(new ApiClient()); + // + // Mockito + // .when(oauth2TokenService.getAccessToken(Mockito.any())) + // .thenReturn("token"); + // + // Assertions + // .assertThat(kubernetesServiceImpl.connectApi(dm)) + // .isEqualTo(expected); + // } + private DeploymentMessage generateDeployDmKuber(Deployment deployment) { DeploymentMessage dm = new DeploymentMessage(); dm.setDeploymentId(deployment.getId()); @@ -280,4 +304,18 @@ private KubernetesService buildService() { .build(); return cs; } + private KubernetesService buildServiceLocal() { + KubernetesService cs = KubernetesService + .kubernetesBuilder() + .endpoint("https://kubernetes.docker.internal:6443/") + .serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) + .hostname("localhost") + .providerId("RECAS-BARI") + .id("https://kubernetes.docker.internal:6443/") + .type(CloudServiceType.COMPUTE) + .iamEnabled(false) + .publicService(true) + .build(); + return cs; + } } From 6c90d110356d3885cf65b89b1a22ea9aeb239b16 Mon Sep 17 00:00:00 2001 From: "A.Lapshynov" Date: Fri, 28 Feb 2020 12:29:03 +0100 Subject: [PATCH 11/11] Port mapping and resource usage edited --- .../dto/kubernetes/KubernetesContainer.java | 8 +- .../dto/kubernetes/KubernetesPortMapping.java | 36 +++ .../dto/kubernetes/KubernetesTask.java | 17 +- .../service/KubernetesException.java | 42 +++ .../providers/KubernetesServiceImpl.java | 251 ++++++++++-------- .../factory/KubernetesClientFactory.java | 79 +++--- .../tosca-definitions/custom_types.yaml | 4 - .../providers/KubernetesServiceTest.java | 37 +-- src/test/resources/tosca/kubernetes_app.yaml | 8 +- 9 files changed, 316 insertions(+), 166 deletions(-) create mode 100644 src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesPortMapping.java create mode 100644 src/main/java/it/reply/orchestrator/exception/service/KubernetesException.java diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java index a09c3c2969..de68acd9d6 100644 --- a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java @@ -19,6 +19,9 @@ import lombok.Data; import lombok.Getter; +import java.util.ArrayList; +import java.util.List; + import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -28,14 +31,15 @@ public class KubernetesContainer { @Nullable private String image; - private Integer port; + @NonNull + private List portMappings = new ArrayList<>(); @NonNull private Type type; @Getter public enum Type { - DOCKER("DOCKER", "tosca.artifacts.Deployment.Image.Container.Docker"); + DOCKER("docker", "tosca.artifacts.Deployment.Image.Container.Docker"); Type(String name, String toscaName) { this.name = name; diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesPortMapping.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesPortMapping.java new file mode 100644 index 0000000000..ca4f32758a --- /dev/null +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesPortMapping.java @@ -0,0 +1,36 @@ +package it.reply.orchestrator.dto.kubernetes; + +import it.reply.orchestrator.utils.Named; + +import lombok.Data; +import lombok.Getter; + +import org.checkerframework.checker.nullness.qual.NonNull; + +@Data +public class KubernetesPortMapping { + + @Getter + public enum Protocol implements Named { + + TCP("tcp"), + UDP("udp"), + IGMP("igmp"); + + private final String name; + + Protocol(String name) { + this.name = name; + } + + } + + @NonNull + private Integer containerPort; + + private Integer servicePort; + + @NonNull + private Protocol protocol = Protocol.TCP; + +} diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java index 8ef8d6b317..dc8d3ad2be 100644 --- a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java @@ -16,12 +16,11 @@ package it.reply.orchestrator.dto.kubernetes; -import io.kubernetes.client.custom.Quantity; - -import it.reply.orchestrator.utils.ToscaConstants; - +import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import it.reply.orchestrator.utils.ToscaConstants; import lombok.Data; @Data @@ -37,12 +36,18 @@ public final String getToscaNodeName() { private KubernetesContainer container; - private String cpu; + private Double cpu; - private String memory; + private Double memory; private Double replicas; private Integer instances; + + private List volumes = new ArrayList<>(); + + public Optional getContainer() { + return Optional.ofNullable(this.container); + } } diff --git a/src/main/java/it/reply/orchestrator/exception/service/KubernetesException.java b/src/main/java/it/reply/orchestrator/exception/service/KubernetesException.java new file mode 100644 index 0000000000..8457240bff --- /dev/null +++ b/src/main/java/it/reply/orchestrator/exception/service/KubernetesException.java @@ -0,0 +1,42 @@ +package it.reply.orchestrator.exception.service; + + +public class KubernetesException extends RuntimeException { + + /** + * + */ + private static final long serialVersionUID = 908837224074715831L; + + private int status; + private String message; + + public KubernetesException(String message) { + super(message); + } + + public KubernetesException(String message, Throwable cause) { + super(message, cause); + } + + protected KubernetesException(int status, String message) { + this.status = status; + this.message = message; + } + + /** + * Gets the status code of the failure, such as 404. + * + * @return status code + */ + public int getStatus() { + return status; + } + + @Override + public String getMessage() { + return message + " (status: " + status + ")"; + } + + +} diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java index beca3195fa..6382891d30 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -19,8 +19,10 @@ import alien4cloud.tosca.model.ArchiveRoot; import com.google.common.collect.MoreCollectors; +import com.google.common.primitives.Ints; import io.kubernetes.client.custom.Quantity; +import io.kubernetes.client.custom.Quantity.Format; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.AppsV1Api; @@ -28,6 +30,8 @@ import io.kubernetes.client.openapi.auth.ApiKeyAuth; import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1ContainerBuilder; +import io.kubernetes.client.openapi.models.V1ContainerPort; +import io.kubernetes.client.openapi.models.V1ContainerPortBuilder; import io.kubernetes.client.openapi.models.V1Deployment; import io.kubernetes.client.openapi.models.V1DeploymentBuilder; import io.kubernetes.client.openapi.models.V1DeploymentSpec; @@ -46,14 +50,20 @@ import it.reply.orchestrator.dto.CloudProviderEndpoint; import it.reply.orchestrator.dto.deployment.DeploymentMessage; import it.reply.orchestrator.dto.kubernetes.KubernetesContainer; +import it.reply.orchestrator.dto.kubernetes.KubernetesPortMapping; +import it.reply.orchestrator.dto.kubernetes.KubernetesPortMapping.Protocol; import it.reply.orchestrator.dto.kubernetes.KubernetesTask; -import it.reply.orchestrator.dto.mesos.MesosContainer; import it.reply.orchestrator.enums.DeploymentProvider; import it.reply.orchestrator.exception.service.BusinessWorkflowException; import it.reply.orchestrator.exception.service.DeploymentException; +import it.reply.orchestrator.exception.service.KubernetesException; +import it.reply.orchestrator.exception.service.ToscaException; +import it.reply.orchestrator.function.ThrowingConsumer; +import it.reply.orchestrator.function.ThrowingFunction; import it.reply.orchestrator.service.IndigoInputsPreProcessorService; import it.reply.orchestrator.service.IndigoInputsPreProcessorService.RuntimeProperties; import it.reply.orchestrator.service.ToscaService; +import it.reply.orchestrator.service.deployment.providers.factory.KubernetesClientFactory; import it.reply.orchestrator.service.security.OAuth2TokenService; import it.reply.orchestrator.utils.CommonUtils; import it.reply.orchestrator.utils.EnumUtils; @@ -63,6 +73,7 @@ import it.reply.orchestrator.utils.WorkflowConstants.ErrorCode; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -86,8 +97,11 @@ import org.alien4cloud.tosca.model.templates.RelationshipTemplate; import org.alien4cloud.tosca.model.templates.Topology; import org.alien4cloud.tosca.normative.types.FloatType; +import org.alien4cloud.tosca.normative.types.IntegerType; +import org.alien4cloud.tosca.normative.types.SizeType; import org.alien4cloud.tosca.normative.types.StringType; import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; import org.jgrapht.graph.DirectedMultigraph; import org.jgrapht.traverse.TopologicalOrderIterator; import org.springframework.beans.factory.annotation.Autowired; @@ -98,9 +112,6 @@ @Slf4j public class KubernetesServiceImpl extends AbstractDeploymentProviderService { - // @Autowired - // private KubernetesClientFactory kubernetesClientFactory; - @Autowired private IndigoInputsPreProcessorService indigoInputsPreProcessorService; @@ -112,11 +123,28 @@ public class KubernetesServiceImpl extends AbstractDeploymentProviderService { @Autowired private ResourceRepository resourceRepository; + + @Autowired + private KubernetesClientFactory kubernetesClientFactory; private static CoreV1Api COREV1_API; private static final String HOST_CAPABILITY_NAME = "host"; + protected AppsV1Api getClient(CloudProviderEndpoint cloudProviderEndpoint, + @Nullable OidcTokenId requestedWithToken) { + String accessToken = oauth2TokenService.getAccessToken(requestedWithToken); + AppsV1Api outClient = null; + try { + outClient = kubernetesClientFactory.build(cloudProviderEndpoint, accessToken); + } catch (IOException e) { + LOG.error("Error in doDeploy:" + e.getCause() + " - " + e.getMessage()); + throw new KubernetesException("Error in doDeploy:" + + e.getCause() + " - " + e.getMessage(), e); + } + return outClient; + } + protected ArchiveRoot prepareTemplate(Deployment deployment, DeploymentMessage deploymentMessage) { RuntimeProperties runtimeProperties = @@ -165,7 +193,7 @@ protected V1Deployment createV1Deployment(DeploymentMessage deploymentMessage) { for (NodeTemplate kuberNode : orderedKubernetesApps) { - KubernetesTask kuberTask = buildTask(graph, kuberNode, kuberNode.getName()); + KubernetesTask kuberTask = buildTask(graph, kuberNode, deployment.getId()); List resources = resourceRepository .findByToscaNodeNameAndDeployment_id(kuberNode.getName(), deployment.getId()); @@ -230,7 +258,7 @@ public KubernetesTask buildTask(DirectedMultigraph artifact type check List supportedTypes = EnumUtils - .toList(MesosContainer.Type.class, MesosContainer.Type::getToscaName); + .toList(KubernetesContainer.Type.class, KubernetesContainer.Type::getToscaName); KubernetesContainer.Type containerType = EnumUtils .fromPredicate(KubernetesContainer.Type.class, @@ -251,7 +279,10 @@ public KubernetesTask buildTask(DirectedMultigraph memSize.convert("MB")) + .ifPresent(kubernetesTask::setMemory); + + + ToscaUtils + .extractList(containerCapability.getProperties(), "volumes", String.class::cast) + .ifPresent(kubernetesTask::setVolumes); kubernetesTask.setReplicas( ToscaUtils .extractScalar(containerCapability.getProperties(), "replicas", FloatType.class) .orElse(1.0));//ifPresent(kubernetesTask::setReplicas); + + ToscaUtils + .extractList(containerCapability.getProperties(), "publish_ports", l -> + this.generatePortMapping((Map) l) + ) + .ifPresent(portMappings -> kubernetesTask + .getContainer() + .orElseThrow( + () -> new RuntimeException( + "there are ports to publish but no container is present")) + .setPortMappings(portMappings)); return kubernetesTask; } + + protected KubernetesPortMapping generatePortMapping(Map portMappingProperties) { + + int sourcePortValue = CommonUtils + .getFromOptionalMap(portMappingProperties, "source") + .map(value -> ToscaUtils.parseScalar((String) value, IntegerType.class)) + .map(Ints::checkedCast) + .orElseThrow(() -> new ToscaException( + "source port in 'publish_ports' property must be provided")); + + KubernetesPortMapping portMapping = new KubernetesPortMapping(sourcePortValue); + + CommonUtils.getFromOptionalMap(portMappingProperties, "target") + .map(value -> ToscaUtils.parseScalar((String) value, IntegerType.class)) + .map(Ints::checkedCast) + .ifPresent(portMapping::setServicePort); + + CommonUtils.getFromOptionalMap(portMappingProperties, "protocol") + .map(value -> EnumUtils.fromNameOrThrow(Protocol.class, (String) value)) + .ifPresent(portMapping::setProtocol); + + return portMapping; + } protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kubernetesTask, String deploymentId) { @@ -291,28 +362,21 @@ protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kuberne Map requestsRes = new HashMap(); - String mem = kubernetesTask.getMemory().replace("Mb", "M").replaceAll(" ", ""); + String mem = kubernetesTask.getMemory().toString();//.replace("Mb", "M").replaceAll(" ", ""); //TODO cpu and ram to string and not quantity maybe - requestsRes.put("cpu", new Quantity(kubernetesTask.getCpu())); - requestsRes.put("memory", new Quantity(mem)); - - Map limitRes = new HashMap(); - limitRes.put("cpu", new Quantity(kubernetesTask.getCpu())); - limitRes.put("memory", new Quantity(mem)); + requestsRes.put("cpu", new Quantity(new BigDecimal(kubernetesTask.getCpu()), Format.DECIMAL_SI)); + requestsRes.put("memory", new Quantity(new BigDecimal(kubernetesTask.getMemory()), Format.DECIMAL_SI)); List v1Containers = new ArrayList(); if (kubernetesTask.getContainers() == null || kubernetesTask.getContainers().isEmpty()) { V1Container contV1 = new V1ContainerBuilder() - .withName(kubernetesTask.getContainer().getType().getName()) - .withImage(kubernetesTask.getContainer().getImage()) + .withName(kubernetesTask.getToscaNodeName()+"_"+kubernetesTask.getId()) .withNewResources() .withRequests(requestsRes) .endResources() - .addNewPort() - .withContainerPort(kubernetesTask.getContainer().getPort()) - .endPort() .build(); + setContainerPorts(contV1, kubernetesTask.getContainer().get().getPortMappings()); v1Containers.add(contV1); } else { for (KubernetesContainer cont : kubernetesTask.getContainers()) { @@ -322,10 +386,12 @@ protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kuberne .withNewResources() .withRequests(requestsRes) .endResources() - .addNewPort() - .withContainerPort(cont.getPort()) - .endPort() + //.addNewPort() + // .withHostPort(1)//targhet port optional + // .withContainerPort(cont.getPort().intValue())//source port + //.endPort() .build(); + setContainerPorts(contV1, cont.getPortMappings()); v1Containers.add(contV1); } } @@ -354,36 +420,24 @@ protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kuberne return v1Deployment; } + + private void setContainerPorts(V1Container contV1, List portMappings) { + for(KubernetesPortMapping kubPort : portMappings) { + V1ContainerPort v1ContainerPort = new V1ContainerPortBuilder() + .withProtocol(kubPort.getProtocol().getName()) + .withContainerPort(kubPort.getContainerPort()) //not nullable port + .withHostPort(kubPort.getServicePort()) //nullable port + .build(); + contV1.addPortsItem(v1ContainerPort); + } + } /** * Connecting Kubernetes Api config. * @param deploymentMessage DeploymentMessage as parameter * @return */ - public AppsV1Api connectApi(DeploymentMessage deploymentMessage, - String cpEndpoint) throws IOException { - final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); - String accessToken = oauth2TokenService - .getAccessToken(CommonUtils.checkNotNull(requestedWithToken)); - - ApiClient x = new ApiClient(); - ApiKeyAuth bearerToken = (ApiKeyAuth) x.getAuthentication("BearerToken"); - bearerToken.setApiKey(accessToken); - - //x.setAccessToken(accessToken); - if (cpEndpoint != null && !cpEndpoint.isEmpty()) { - x.setBasePath(cpEndpoint); - } - - ApiClient client = Config.fromToken(cpEndpoint, accessToken); - if (client == null) { - client = Config.defaultClient(); - //client.setBasePath("https://kubernetes.docker.internal:6443"); - } - return new AppsV1Api(client); - //TODO manage ApiException or throw some exception - } @Override public boolean doDeploy(DeploymentMessage deploymentMessage) { @@ -391,26 +445,26 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { V1Deployment v1Deployment = createV1Deployment(deploymentMessage); //V1Deployment v1Deployment = createV1DeploymentForTest(deploymentMessage); - try { - AppsV1Api app = connectApi(deploymentMessage, - deployment.getCloudProviderEndpoint().getCpEndpoint()); - app.createNamespacedDeployment( - "default", - v1Deployment, - "true", - null, - null); - - //TODO handle exception in out - } catch (ApiException e) { - LOG.error("Error in doDeploy:" + e.getCode() + " - " + e.getMessage()); - throw new DeploymentException("Error in doDeploy:" - + e.getCode() + " - " + e.getMessage(), e); - } catch (IOException e) { - LOG.error("Error in doDeploy:" + e.getCause() + " - " + e.getMessage()); - throw new DeploymentException("Error in doDeploy:" - + e.getCause() + " - " + e.getMessage(), e); - } + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + +// try { + AppsV1Api apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); + +// V1Deployment depCreated = apiClient.createNamespacedDeployment( +// "default", +// v1Deployment, +// "true", +// null, +// null); +// +// LOG.debug(depCreated.getStatus().toString()); +// +// //TODO handle exception in out +// } catch (ApiException e) { +// LOG.error("Error in doDeploy:" + e.getCode() + " - " + e.getMessage() + e.getResponseBody()); +// throw new DeploymentException("Error in doDeploy:" +// + e.getCode() + " - " + e.getMessage(), e); +// } LOG.info("Creating Kubernetes V1Deployment for deployment {} with definition:\n{}", deploymentMessage.getDeploymentId(), v1Deployment.getMetadata().getName()); return true; @@ -420,20 +474,19 @@ public boolean doDeploy(DeploymentMessage deploymentMessage) { public boolean isDeployed(DeploymentMessage deploymentMessage) { Deployment deployment = getDeployment(deploymentMessage); - AppsV1Api app; + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + + AppsV1Api apiClient; V1Deployment v1Deployment = new V1Deployment(); try { - app = connectApi(deploymentMessage, deployment.getCloudProviderEndpoint().getCpEndpoint()); + apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); - v1Deployment = app.readNamespacedDeploymentStatus(deployment.getId(), "default", "true"); + v1Deployment = apiClient.readNamespacedDeploymentStatus(deployment.getId(), "default", "true"); printPodList(); } catch (ApiException e) { LOG.error("Error in isDeployed:" + e.getCode() + " - " + e.getMessage()); throw new DeploymentException(e.getMessage(), e); - } catch (IOException e) { - LOG.error("Error in isDeployed:" + e.getCause() + " - " + e.getMessage()); - throw new DeploymentException(e.getMessage(), e); } boolean isDeployed = @@ -454,19 +507,16 @@ public void cleanFailedDeploy(DeploymentMessage deploymentMessage) { @Override public boolean doUpdate(DeploymentMessage deploymentMessage, String template) { Deployment deployment = getDeployment(deploymentMessage); - + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); V1Deployment v1Deployment = new V1DeploymentBuilder() .withNewMetadata() .withName(deployment.getId()) .endMetadata() .build(); - AppsV1Api app; - CoreV1Api api; + AppsV1Api apiClient; try { - app = new AppsV1Api(Config.defaultClient()); - api = new CoreV1Api(); - - app.patchNamespacedDeployment( + apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); + apiClient.patchNamespacedDeployment( deployment.getId(), "default", v1Deployment, @@ -474,16 +524,13 @@ public boolean doUpdate(DeploymentMessage deploymentMessage, String template) { null, null, null); - // app.replaceNamespacedDeployment(name, namespace, body, pretty, dryRun, fieldManager); + // apiClient.replaceNamespacedDeployment(name, namespace, body, pretty, dryRun, fieldManager); } catch (ApiException e) { LOG.error("Error in doUpdate:" + e.getCode() + " - " + e.getMessage()); throw new DeploymentException(e.getMessage(), e); - } catch (IOException e) { - LOG.error("Error in doUpdate:" + e.getCause() + " - " + e.getMessage()); - throw new DeploymentException(e.getMessage(), e); - } - throw new UnsupportedOperationException("Marathon app deployments do not support update."); + } + return true; } @Override @@ -496,12 +543,13 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { Deployment deployment = getDeployment(deploymentMessage); CloudProviderEndpoint cloudProviderEndpoint = deployment.getCloudProviderEndpoint(); + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); if (cloudProviderEndpoint != null) { - AppsV1Api app; + AppsV1Api apiClient; try { - app = connectApi(deploymentMessage, deployment.getCloudProviderEndpoint().getCpEndpoint()); - - V1Status status = app.deleteNamespacedDeployment( + apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); + + V1Status status = apiClient.deleteNamespacedDeployment( deployment.getId(), "default", "true", @@ -526,12 +574,7 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { // } throw new DeploymentException("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage(), e); - } catch (IOException e) { - LOG.error("Error in doUndeploy:" - + e.getCause() + " - " + e.getMessage()); - throw new DeploymentException("Error in doUndeploy:" - + e.getCause() + " - " + e.getMessage(), e); - } + } } return true; } @@ -540,17 +583,18 @@ public boolean doUndeploy(DeploymentMessage deploymentMessage) { public boolean isUndeployed(DeploymentMessage deploymentMessage) { boolean isUndeployed = false; Deployment deployment = getDeployment(deploymentMessage); + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); - AppsV1Api app; + AppsV1Api apiClient; try { - app = connectApi(deploymentMessage, deployment.getCloudProviderEndpoint().getCpEndpoint()); + apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); - V1Deployment deplSimleToCheck = app.readNamespacedDeploymentStatus( + V1Deployment deplSimleToCheck = apiClient.readNamespacedDeploymentStatus( deployment.getId(), "default", "true"); - V1Deployment depl = app.readNamespacedDeployment( + V1Deployment depl = apiClient.readNamespacedDeployment( deployment.getId(), "default", "true", @@ -567,9 +611,6 @@ public boolean isUndeployed(DeploymentMessage deploymentMessage) { isUndeployed = true; } throw new DeploymentException(e.getMessage(), e); - } catch (IOException e) { - LOG.error("Error in doUndeploy:" + e.getCause() + " - " + e.getMessage()); - throw new DeploymentException(e.getMessage(), e); } return isUndeployed; diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java index 883ae36092..a7b6c61221 100644 --- a/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java @@ -18,14 +18,23 @@ import feign.Feign; import feign.Logger.Level; +import feign.auth.BasicAuthRequestInterceptor; import feign.RequestInterceptor; import feign.slf4j.Slf4jLogger; import io.kubernetes.client.openapi.ApiClient; - +import io.kubernetes.client.openapi.apis.AppsV1Api; +import io.kubernetes.client.openapi.auth.ApiKeyAuth; +import io.kubernetes.client.util.Config; import it.infn.ba.deep.qcg.client.utils.QcgException; +import it.reply.orchestrator.dal.entity.OidcTokenId; import it.reply.orchestrator.dto.CloudProviderEndpoint; +import it.reply.orchestrator.dto.cmdb.CloudService; +import it.reply.orchestrator.dto.deployment.DeploymentMessage; +import it.reply.orchestrator.dto.security.GenericServiceCredential; +import it.reply.orchestrator.utils.CommonUtils; +import java.io.IOException; import java.util.Objects; import lombok.extern.slf4j.Slf4j; @@ -35,42 +44,50 @@ @Service @Slf4j -public class KubernetesClientFactory { - - /** - * Build a Kubernetes Api client object. - * @param cloudProviderEndpoint the service endpoint. - * @param accessToken the input accesstoken. - * @return the Kubernetes Api client object. - */ - public ApiClient build(CloudProviderEndpoint cloudProviderEndpoint, String accessToken) { - final RequestInterceptor requestInterceptor; - Objects.requireNonNull(accessToken, "Access Token must not be null"); - requestInterceptor = requestTemplate -> - requestTemplate.header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken); +public class KubernetesClientFactory extends CloudService{ - return build(cloudProviderEndpoint.getCpEndpoint(), requestInterceptor); - } +// /** +// * Build a Kubernetes Api client object. +// * @param cloudProviderEndpoint the service endpoint. +// * @param accessToken the input accesstoken. +// * @return the Kubernetes Api client object. +// */ +// public void build(CloudProviderEndpoint cloudProviderEndpoint, OidcTokenId accessToken) { +// final RequestInterceptor requestInterceptor; +// Objects.requireNonNull(accessToken, "Access Token must not be null"); +// requestInterceptor = requestTemplate -> +// requestTemplate.header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken); +// +// //TODO if needed +// } /** * Build a Kubernetes client object. - * @param apiClientEndpoint the input Kubernetes service endpoint. - * @param authInterceptor the input request interceptor. - * @return the Kubernetes client object. + * @param cloudProviderEndpoint the input Kubernetes service endpoint. + * @param accessToken. + * @return AppsV1Api the Kubernetes client object. + * @throws IOException */ - public ApiClient build(String apiClientEndpoint, RequestInterceptor authInterceptor) { - LOG.info("Generating Qcg client with endpoint {}", apiClientEndpoint); + public AppsV1Api build(CloudProviderEndpoint cloudProviderEndpoint, String accessToken) throws IOException { + LOG.info("Generating Kubernetes client with endpoint {}", cloudProviderEndpoint.getCpEndpoint()); + + ApiClient x = new ApiClient(); + + ApiKeyAuth bearerToken = (ApiKeyAuth) x.getAuthentication("BearerToken"); + bearerToken.setApiKey(accessToken); + + String cpEndpoint = cloudProviderEndpoint.getCpEndpoint(); + + //x.setAccessToken(accessToken); + if (cpEndpoint != null && !cpEndpoint.isEmpty()) { + x.setBasePath(cpEndpoint); + } - //TODO * new ApiEncoder()).decoder(new QcgDecoder() - return Feign.builder().encoder(null/* * */) - .logger(new Slf4jLogger(ApiClient.class)) - .logLevel(Level.FULL) - .errorDecoder((methodKey, response) -> new QcgException(response.status(), - response.reason())) - .requestInterceptor(authInterceptor).requestInterceptor(template -> { - template.header(HttpHeaders.ACCEPT, "application/json"); - template.header(HttpHeaders.CONTENT_TYPE, "application/json"); - }).target(ApiClient.class, apiClientEndpoint); + ApiClient client = Config.fromToken(cpEndpoint, accessToken); + if (client == null) { + client = Config.defaultClient(); + } + return new AppsV1Api(client); } } diff --git a/src/main/resources/tosca-definitions/custom_types.yaml b/src/main/resources/tosca-definitions/custom_types.yaml index 72bea5955a..7813a35641 100644 --- a/src/main/resources/tosca-definitions/custom_types.yaml +++ b/src/main/resources/tosca-definitions/custom_types.yaml @@ -2367,10 +2367,6 @@ node_types: type: integer default: 1 required: no - container_port: - type: integer - description: container port - default: 80 tosca.nodes.indigo.JupyterHub: derived_from: tosca.nodes.SoftwareComponent diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java index 3fd32dd334..610c6697f5 100644 --- a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -17,21 +17,17 @@ package it.reply.orchestrator.service.deployment.providers; import static org.assertj.core.api.Assertions.assertThatCode; -import static org.mockito.Mockito.doReturn; import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.UUID; import org.assertj.core.api.AbstractThrowableAssert; import org.assertj.core.api.Assertions; -import org.hamcrest.core.IsEqual; import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; @@ -49,7 +45,6 @@ import com.google.common.collect.Lists; -import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.AppsV1Api; import io.kubernetes.client.util.Config; @@ -64,18 +59,17 @@ import it.reply.orchestrator.dto.cmdb.CloudService; import it.reply.orchestrator.dto.cmdb.CloudServiceType; import it.reply.orchestrator.dto.cmdb.KubernetesService; -import it.reply.orchestrator.dto.cmdb.MarathonService; import it.reply.orchestrator.dto.deployment.DeploymentMessage; import it.reply.orchestrator.dto.onedata.OneData; import it.reply.orchestrator.dto.onedata.OneData.OneDataProviderInfo; import it.reply.orchestrator.dto.workflow.CloudServicesOrderedIterator; import it.reply.orchestrator.enums.NodeStates; -import it.reply.orchestrator.exception.service.BusinessWorkflowException; import it.reply.orchestrator.exception.service.DeploymentException; import it.reply.orchestrator.function.ThrowingFunction; import it.reply.orchestrator.service.ToscaService; import it.reply.orchestrator.service.ToscaServiceTest; import it.reply.orchestrator.service.VaultService; +import it.reply.orchestrator.service.deployment.providers.factory.KubernetesClientFactory; import it.reply.orchestrator.service.security.OAuth2TokenService; import it.reply.orchestrator.util.TestUtil; import junitparams.JUnitParamsRunner; @@ -105,6 +99,9 @@ public class KubernetesServiceTest extends ToscaParserAwareTest { @MockBean private VaultService vaultService; + + @MockBean + private KubernetesClientFactory kubernetesClientFactory; @SpyBean private OAuth2TokenService oauth2TokenService; @@ -136,12 +133,17 @@ public void testDoDeploy() throws IOException, URISyntaxException { dm.setCloudServicesOrderedIterator(csi); Mockito - .when(deploymentRepository.findOne(deployment.getId())) - .thenReturn(deployment); + .when(deploymentRepository.findOne(deployment.getId())) + .thenReturn(deployment); ; - doReturn(new AppsV1Api(Config.defaultClient())).when(kubernetesServiceImpl) - .connectApi(dm, deployment.getCloudProviderEndpoint().getCpEndpoint()); + Mockito + .when(oauth2TokenService.getAccessToken(dm.getRequestedWithToken())) + .thenReturn("token"); + + Mockito + .when(kubernetesClientFactory.build(deployment.getCloudProviderEndpoint(), "token")) + .thenReturn(new AppsV1Api(Config.defaultClient())); Assertions .assertThatExceptionOfType(DeploymentException.class); @@ -165,12 +167,17 @@ public void testDoUndeploy() throws IOException { dm.setCloudServicesOrderedIterator(csi); Mockito - .when(deploymentRepository.findOne(deployment.getId())) - .thenReturn(deployment); + .when(deploymentRepository.findOne(deployment.getId())) + .thenReturn(deployment); ; - doReturn(new AppsV1Api(Config.defaultClient())).when(kubernetesServiceImpl) - .connectApi(dm, deployment.getCloudProviderEndpoint().getCpEndpoint()); + Mockito + .when(oauth2TokenService.getAccessToken(dm.getRequestedWithToken())) + .thenReturn("token"); + + Mockito + .when(kubernetesClientFactory.build(deployment.getCloudProviderEndpoint(), Mockito.anyString())) + .thenReturn(new AppsV1Api(Config.defaultClient())); AbstractThrowableAssert assertion = assertThatCode( () -> kubernetesServiceImpl.doUndeploy(dm)); diff --git a/src/test/resources/tosca/kubernetes_app.yaml b/src/test/resources/tosca/kubernetes_app.yaml index 809c9279cd..73e703f785 100644 --- a/src/test/resources/tosca/kubernetes_app.yaml +++ b/src/test/resources/tosca/kubernetes_app.yaml @@ -13,10 +13,12 @@ topology_template: image: file: hello-world type: tosca.artifacts.Deployment.Image.Container.Docker + properties: + replicas: 2 requirements: - - host: Docker + - host: docker - Docker: + docker: type: tosca.nodes.indigo.Container.Runtime.Docker capabilities: host: @@ -30,4 +32,4 @@ topology_template: outputs: endpoint: - value: { concat: [ { get_attribute : [ kubernetes, attr, 0 ] }, ':', { get_attribute : [ Docker, host, publish_ports, 0, target ] } ] } + value: { concat: [ { get_attribute : [ kubernetes, attr, 0 ] }, ':', { get_attribute : [ docker, host, publish_ports, 0, target ] } ] }