diff --git a/pom.xml b/pom.xml index a2669d9208..aa1a4e045c 100644 --- a/pom.xml +++ b/pom.xml @@ -95,6 +95,7 @@ 1.1.0 1.16.20 2.0.1 + 7.0.0 @@ -456,6 +457,13 @@ test + + io.kubernetes + client-java + ${kubernetes.version} + compile + + diff --git a/src/main/java/it/reply/orchestrator/dto/CloudProviderEndpoint.java b/src/main/java/it/reply/orchestrator/dto/CloudProviderEndpoint.java index 2d2e3e0004..19a9a312f2 100644 --- a/src/main/java/it/reply/orchestrator/dto/CloudProviderEndpoint.java +++ b/src/main/java/it/reply/orchestrator/dto/CloudProviderEndpoint.java @@ -51,7 +51,8 @@ public enum IaaSType { AZURE, CHRONOS, MARATHON, - QCG; + QCG, + KUBERNETES; } @Nullable diff --git a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java index e6c0c943e3..c4284474bf 100644 --- a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java +++ b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudService.java @@ -96,6 +96,7 @@ public class CloudService implements CmdbIdentifiable { private static final String COMPUTE_SERVICE_PREFIX = EGI_SERVICE_PREFIX + ".vm-management"; private static final String STORAGE_SERVICE_PREFIX = EGI_SERVICE_PREFIX + ".storage-management"; + private static final String DEEP_SERVICE_PREFIX = "eu.deep"; public static final String OPENSTACK_COMPUTE_SERVICE = "org.openstack.nova"; public static final String OPENNEBULA_COMPUTE_SERVICE = COMPUTE_SERVICE_PREFIX + ".opennebula"; @@ -112,7 +113,8 @@ public class CloudService implements CmdbIdentifiable { public static final String MARATHON_COMPUTE_SERVICE = INDIGO_SERVICE_PREFIX + ".marathon"; public static final String CHRONOS_COMPUTE_SERVICE = INDIGO_SERVICE_PREFIX + ".chronos"; - public static final String QCG_COMPUTE_SERVICE = "eu.deep.qcg"; + public static final String QCG_COMPUTE_SERVICE = DEEP_SERVICE_PREFIX + ".qcg"; + public static final String KUBERNETES_COMPUTE_SERVICE = INDIGO_SERVICE_PREFIX + ".kubernetes"; /** * Get if the the service is a OpenStack compute service. @@ -234,9 +236,14 @@ public boolean isQcgComputeProviderService() { return QCG_COMPUTE_SERVICE.equals(this.serviceType); } + /** + * Get if the the service is a Kubernetes compute service. + * + * @return true if the service is a Kubernetes compute service + */ @JsonIgnore - public boolean isCredentialsRequired() { - return isAwsComputeProviderService() || isAzureComputeProviderService() - || isOtcComputeProviderService(); + public boolean isKubernetesComputeProviderService() { + return KUBERNETES_COMPUTE_SERVICE.equals(this.serviceType); } + } diff --git a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java index 3cb6777eb3..678f097ccd 100644 --- a/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java +++ b/src/main/java/it/reply/orchestrator/dto/cmdb/CloudServiceResolver.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import static it.reply.orchestrator.dto.cmdb.CloudService.AWS_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.AZURE_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.CHRONOS_COMPUTE_SERVICE; +import static it.reply.orchestrator.dto.cmdb.CloudService.KUBERNETES_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.MARATHON_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.OCCI_COMPUTE_SERVICE; import static it.reply.orchestrator.dto.cmdb.CloudService.OPENNEBULA_COMPUTE_SERVICE; @@ -64,6 +65,9 @@ public JavaType typeFromId(DatabindContext context, String id) { case QCG_COMPUTE_SERVICE: subType = QcgService.class; break; + case KUBERNETES_COMPUTE_SERVICE: + subType = KubernetesService.class; + break; case OCCI_COMPUTE_SERVICE: case OPENNEBULA_COMPUTE_SERVICE: case OPENNEBULA_TOSCA_SERVICE: diff --git a/src/main/java/it/reply/orchestrator/dto/cmdb/KubernetesService.java b/src/main/java/it/reply/orchestrator/dto/cmdb/KubernetesService.java new file mode 100644 index 0000000000..3fda33094c --- /dev/null +++ b/src/main/java/it/reply/orchestrator/dto/cmdb/KubernetesService.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.dto.cmdb; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class KubernetesService extends CloudService { + + @Builder(builderMethodName = "kubernetesBuilder") + public KubernetesService( + @NonNull String id, + @NonNull String serviceType, + @NonNull String endpoint, + @NonNull String providerId, + @NonNull CloudServiceType type, + boolean publicService, + @Nullable String region, + @NonNull String hostname, + @Nullable String parentServiceId, + boolean iamEnabled) { + super(id, serviceType, endpoint, providerId, type, publicService, region, hostname, + parentServiceId, iamEnabled); + } +} diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java new file mode 100644 index 0000000000..de68acd9d6 --- /dev/null +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesContainer.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.dto.kubernetes; + +import lombok.Data; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; + +@Data +public class KubernetesContainer { + + @Nullable + private String image; + + @NonNull + private List portMappings = new ArrayList<>(); + + @NonNull + private Type type; + + @Getter + public enum Type { + DOCKER("docker", "tosca.artifacts.Deployment.Image.Container.Docker"); + + Type(String name, String toscaName) { + this.name = name; + this.toscaName = toscaName; + } + + private final String name; + + private final String toscaName; + } +} diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesPortMapping.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesPortMapping.java new file mode 100644 index 0000000000..ca4f32758a --- /dev/null +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesPortMapping.java @@ -0,0 +1,36 @@ +package it.reply.orchestrator.dto.kubernetes; + +import it.reply.orchestrator.utils.Named; + +import lombok.Data; +import lombok.Getter; + +import org.checkerframework.checker.nullness.qual.NonNull; + +@Data +public class KubernetesPortMapping { + + @Getter + public enum Protocol implements Named { + + TCP("tcp"), + UDP("udp"), + IGMP("igmp"); + + private final String name; + + Protocol(String name) { + this.name = name; + } + + } + + @NonNull + private Integer containerPort; + + private Integer servicePort; + + @NonNull + private Protocol protocol = Protocol.TCP; + +} diff --git a/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java new file mode 100644 index 0000000000..dc8d3ad2be --- /dev/null +++ b/src/main/java/it/reply/orchestrator/dto/kubernetes/KubernetesTask.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.dto.kubernetes; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import it.reply.orchestrator.utils.ToscaConstants; +import lombok.Data; + +@Data +public class KubernetesTask { + + public final String getToscaNodeName() { + return ToscaConstants.Nodes.Types.KUBERNETES; + } + + private String id; + + private List containers; + + private KubernetesContainer container; + + private Double cpu; + + private Double memory; + + private Double replicas; + + private Integer instances; + + private List volumes = new ArrayList<>(); + + public Optional getContainer() { + return Optional.ofNullable(this.container); + } + +} diff --git a/src/main/java/it/reply/orchestrator/enums/DeploymentProvider.java b/src/main/java/it/reply/orchestrator/enums/DeploymentProvider.java index 5dbfc7fb11..eb927b73bf 100644 --- a/src/main/java/it/reply/orchestrator/enums/DeploymentProvider.java +++ b/src/main/java/it/reply/orchestrator/enums/DeploymentProvider.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ public enum DeploymentProvider { HEAT, CHRONOS, MARATHON, - QCG; + QCG, + KUBERNETES; } diff --git a/src/main/java/it/reply/orchestrator/enums/DeploymentType.java b/src/main/java/it/reply/orchestrator/enums/DeploymentType.java index a2287ffecc..e186a76002 100644 --- a/src/main/java/it/reply/orchestrator/enums/DeploymentType.java +++ b/src/main/java/it/reply/orchestrator/enums/DeploymentType.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,8 @@ public enum DeploymentType { CHRONOS, MARATHON, TOSCA, - QCG; + QCG, + KUBERNETES; public static boolean isMesosDeployment(DeploymentType deploymentType) { return deploymentType == CHRONOS || deploymentType == DeploymentType.MARATHON; diff --git a/src/main/java/it/reply/orchestrator/exception/service/KubernetesException.java b/src/main/java/it/reply/orchestrator/exception/service/KubernetesException.java new file mode 100644 index 0000000000..8457240bff --- /dev/null +++ b/src/main/java/it/reply/orchestrator/exception/service/KubernetesException.java @@ -0,0 +1,42 @@ +package it.reply.orchestrator.exception.service; + + +public class KubernetesException extends RuntimeException { + + /** + * + */ + private static final long serialVersionUID = 908837224074715831L; + + private int status; + private String message; + + public KubernetesException(String message) { + super(message); + } + + public KubernetesException(String message, Throwable cause) { + super(message, cause); + } + + protected KubernetesException(int status, String message) { + this.status = status; + this.message = message; + } + + /** + * Gets the status code of the failure, such as 404. + * + * @return status code + */ + public int getStatus() { + return status; + } + + @Override + public String getMessage() { + return message + " (status: " + status + ")"; + } + + +} diff --git a/src/main/java/it/reply/orchestrator/service/CloudProviderEndpointServiceImpl.java b/src/main/java/it/reply/orchestrator/service/CloudProviderEndpointServiceImpl.java index 18b4396d2a..56e70da456 100644 --- a/src/main/java/it/reply/orchestrator/service/CloudProviderEndpointServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/CloudProviderEndpointServiceImpl.java @@ -123,6 +123,8 @@ public CloudProviderEndpoint getCloudProviderEndpoint(CloudService computeServic iaasType = IaaSType.MARATHON; } else if (computeService.isQcgComputeProviderService()) { iaasType = IaaSType.QCG; + } else if (computeService.isKubernetesComputeProviderService()) { + iaasType = IaaSType.KUBERNETES; } else { throw new IllegalArgumentException("Unknown Cloud Provider type: " + computeService); } @@ -164,6 +166,8 @@ public DeploymentProvider getDeploymentProvider(DeploymentType deploymentType, return DeploymentProvider.QCG; case TOSCA: return DeploymentProvider.IM; + case KUBERNETES: + return DeploymentProvider.KUBERNETES; default: throw new DeploymentException("Unknown DeploymentType: " + deploymentType.toString()); } diff --git a/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java b/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java index 84d899b948..5cff0a6ecd 100644 --- a/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/CmdbServiceImpl.java @@ -24,6 +24,7 @@ import it.reply.orchestrator.dto.cmdb.ComputeService; import it.reply.orchestrator.dto.cmdb.Flavor; import it.reply.orchestrator.dto.cmdb.Image; +import it.reply.orchestrator.dto.cmdb.KubernetesService; import it.reply.orchestrator.dto.cmdb.Tenant; import it.reply.orchestrator.dto.cmdb.wrappers.CmdbDataWrapper; import it.reply.orchestrator.dto.cmdb.wrappers.CmdbHasManyList; @@ -287,6 +288,15 @@ public CloudProvider fillCloudProviderInfo(String providerId, }) .collect(Collectors.toMap(CloudService::getId, Function.identity())); + //services.put(providerId+"_kuberServiceId123", KubernetesService.kubernetesBuilder() + //.id(providerId+"kuberServiceId123") + //.providerId(providerId) + //.serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) + //.type(CloudServiceType.COMPUTE) + //.hostname("localhost:8001") + //.endpoint("localhost:8001") + //.build()); + provider.setServices(services); return provider; } diff --git a/src/main/java/it/reply/orchestrator/service/DeploymentServiceImpl.java b/src/main/java/it/reply/orchestrator/service/DeploymentServiceImpl.java index 0422b472c9..f541f51046 100644 --- a/src/main/java/it/reply/orchestrator/service/DeploymentServiceImpl.java +++ b/src/main/java/it/reply/orchestrator/service/DeploymentServiceImpl.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -280,6 +280,8 @@ private DeploymentType inferDeploymentType(Map nodes) { return DeploymentType.MARATHON; } else if (toscaService.isOfToscaType(node, ToscaConstants.Nodes.Types.QCG)) { return DeploymentType.QCG; + } else if (toscaService.isOfToscaType(node, ToscaConstants.Nodes.Types.KUBERNETES)) { + return DeploymentType.KUBERNETES; } } return DeploymentType.TOSCA; @@ -293,6 +295,8 @@ private static DeploymentType inferDeploymentType(DeploymentProvider deploymentP return DeploymentType.MARATHON; case QCG: return DeploymentType.QCG; + case KUBERNETES: + return DeploymentType.KUBERNETES; case HEAT: case IM: default: @@ -361,7 +365,8 @@ public void updateDeployment(String id, DeploymentRequest request) { if (deployment.getDeploymentProvider() == DeploymentProvider.CHRONOS || deployment.getDeploymentProvider() == DeploymentProvider.MARATHON - || deployment.getDeploymentProvider() == DeploymentProvider.QCG) { + || deployment.getDeploymentProvider() == DeploymentProvider.QCG + || deployment.getDeploymentProvider() == DeploymentProvider.KUBERNETES) { throw new BadRequestException(String.format("%s deployments cannot be updated.", deployment.getDeploymentProvider().toString())); } diff --git a/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java b/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java index 4daace1d40..ed715fc2fd 100644 --- a/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java +++ b/src/main/java/it/reply/orchestrator/service/commands/PrefilterCloudProviders.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ import it.reply.orchestrator.dto.cmdb.CloudService; import it.reply.orchestrator.dto.cmdb.CloudServiceType; import it.reply.orchestrator.dto.cmdb.ComputeService; +import it.reply.orchestrator.dto.cmdb.KubernetesService; import it.reply.orchestrator.dto.cmdb.MarathonService; import it.reply.orchestrator.dto.cmdb.MesosFrameworkService; import it.reply.orchestrator.dto.cmdb.QcgService; @@ -143,6 +144,14 @@ public void execute(DelegateExecution execution, addServiceToDiscard(servicesToDiscard, cloudProviderService); } break; + case KUBERNETES: + if ((cloudProviderService instanceof KubernetesService)) { + KubernetesService kubernetesService = + (KubernetesService) cloudProviderService; + } else { + addServiceToDiscard(servicesToDiscard, cloudProviderService); + } + break; default: throw new DeploymentException("Unknown Deployment Type: " + type); } @@ -243,8 +252,7 @@ private void discardOnPlacementPolicies(Map placementPolici .stream() .flatMap(policy -> policy.getServicesId().stream()) .anyMatch(serviceId -> serviceId.equals(cloudService.getId())); - boolean credentialsRequired = cloudService.isCredentialsRequired(); - if (!serviceIsInSlaPolicy && (slaPlacementRequired || credentialsRequired)) { + if (!serviceIsInSlaPolicy && slaPlacementRequired) { LOG.debug( "Discarded service {} of provider {} because it doesn't match SLA policies", cloudService.getId(), cloudProvider.getId()); diff --git a/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java b/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java index 8a5358b9e9..121db4e5b0 100644 --- a/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java +++ b/src/main/java/it/reply/orchestrator/service/commands/UpdateDeployment.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -66,6 +66,7 @@ public void execute(DelegateExecution execution, DeploymentMessage deploymentMes RankCloudProvidersMessage.class); CloudServicesOrderedIterator servicesIt = deploymentMessage.getCloudServicesOrderedIterator(); + if (servicesIt == null) { servicesIt = cloudProviderEndpointService .generateCloudProvidersOrderedIterator(rankCloudProvidersMessage, diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java new file mode 100644 index 0000000000..6382891d30 --- /dev/null +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceImpl.java @@ -0,0 +1,762 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.service.deployment.providers; + +import alien4cloud.tosca.model.ArchiveRoot; + +import com.google.common.collect.MoreCollectors; +import com.google.common.primitives.Ints; + +import io.kubernetes.client.custom.Quantity; +import io.kubernetes.client.custom.Quantity.Format; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.AppsV1Api; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.auth.ApiKeyAuth; +import io.kubernetes.client.openapi.models.V1Container; +import io.kubernetes.client.openapi.models.V1ContainerBuilder; +import io.kubernetes.client.openapi.models.V1ContainerPort; +import io.kubernetes.client.openapi.models.V1ContainerPortBuilder; +import io.kubernetes.client.openapi.models.V1Deployment; +import io.kubernetes.client.openapi.models.V1DeploymentBuilder; +import io.kubernetes.client.openapi.models.V1DeploymentSpec; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodList; +import io.kubernetes.client.openapi.models.V1ResourceRequirements; +import io.kubernetes.client.openapi.models.V1ResourceRequirementsBuilder; +import io.kubernetes.client.openapi.models.V1Status; +import io.kubernetes.client.util.Config; + +import it.reply.orchestrator.annotation.DeploymentProviderQualifier; +import it.reply.orchestrator.dal.entity.Deployment; +import it.reply.orchestrator.dal.entity.OidcTokenId; +import it.reply.orchestrator.dal.entity.Resource; +import it.reply.orchestrator.dal.repository.ResourceRepository; +import it.reply.orchestrator.dto.CloudProviderEndpoint; +import it.reply.orchestrator.dto.deployment.DeploymentMessage; +import it.reply.orchestrator.dto.kubernetes.KubernetesContainer; +import it.reply.orchestrator.dto.kubernetes.KubernetesPortMapping; +import it.reply.orchestrator.dto.kubernetes.KubernetesPortMapping.Protocol; +import it.reply.orchestrator.dto.kubernetes.KubernetesTask; +import it.reply.orchestrator.enums.DeploymentProvider; +import it.reply.orchestrator.exception.service.BusinessWorkflowException; +import it.reply.orchestrator.exception.service.DeploymentException; +import it.reply.orchestrator.exception.service.KubernetesException; +import it.reply.orchestrator.exception.service.ToscaException; +import it.reply.orchestrator.function.ThrowingConsumer; +import it.reply.orchestrator.function.ThrowingFunction; +import it.reply.orchestrator.service.IndigoInputsPreProcessorService; +import it.reply.orchestrator.service.IndigoInputsPreProcessorService.RuntimeProperties; +import it.reply.orchestrator.service.ToscaService; +import it.reply.orchestrator.service.deployment.providers.factory.KubernetesClientFactory; +import it.reply.orchestrator.service.security.OAuth2TokenService; +import it.reply.orchestrator.utils.CommonUtils; +import it.reply.orchestrator.utils.EnumUtils; +import it.reply.orchestrator.utils.OneDataUtils; +import it.reply.orchestrator.utils.ToscaConstants; +import it.reply.orchestrator.utils.ToscaUtils; +import it.reply.orchestrator.utils.WorkflowConstants.ErrorCode; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import javax.validation.constraints.NotNull; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import org.alien4cloud.tosca.model.definitions.DeploymentArtifact; +import org.alien4cloud.tosca.model.templates.Capability; +import org.alien4cloud.tosca.model.templates.NodeTemplate; +import org.alien4cloud.tosca.model.templates.RelationshipTemplate; +import org.alien4cloud.tosca.model.templates.Topology; +import org.alien4cloud.tosca.normative.types.FloatType; +import org.alien4cloud.tosca.normative.types.IntegerType; +import org.alien4cloud.tosca.normative.types.SizeType; +import org.alien4cloud.tosca.normative.types.StringType; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.jgrapht.graph.DirectedMultigraph; +import org.jgrapht.traverse.TopologicalOrderIterator; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +@DeploymentProviderQualifier(DeploymentProvider.KUBERNETES) +@Slf4j +public class KubernetesServiceImpl extends AbstractDeploymentProviderService { + + @Autowired + private IndigoInputsPreProcessorService indigoInputsPreProcessorService; + + @Autowired + private ToscaService toscaService; + + @Autowired + private OAuth2TokenService oauth2TokenService; + + @Autowired + private ResourceRepository resourceRepository; + + @Autowired + private KubernetesClientFactory kubernetesClientFactory; + + private static CoreV1Api COREV1_API; + + private static final String HOST_CAPABILITY_NAME = "host"; + + protected AppsV1Api getClient(CloudProviderEndpoint cloudProviderEndpoint, + @Nullable OidcTokenId requestedWithToken) { + String accessToken = oauth2TokenService.getAccessToken(requestedWithToken); + AppsV1Api outClient = null; + try { + outClient = kubernetesClientFactory.build(cloudProviderEndpoint, accessToken); + } catch (IOException e) { + LOG.error("Error in doDeploy:" + e.getCause() + " - " + e.getMessage()); + throw new KubernetesException("Error in doDeploy:" + + e.getCause() + " - " + e.getMessage(), e); + } + return outClient; + } + + protected ArchiveRoot prepareTemplate(Deployment deployment, + DeploymentMessage deploymentMessage) { + RuntimeProperties runtimeProperties = + OneDataUtils.getOneDataRuntimeProperties(deploymentMessage); + Map inputs = deployment.getParameters(); + ArchiveRoot ar = toscaService.parseAndValidateTemplate(deployment.getTemplate(), inputs); + if (runtimeProperties.getVaules().size() > 0) { + indigoInputsPreProcessorService.processGetInputAttributes(ar, inputs, runtimeProperties); + } else { + indigoInputsPreProcessorService.processGetInput(ar, inputs); + } + return ar; + } + + protected V1Deployment createV1Deployment(DeploymentMessage deploymentMessage) { + Deployment deployment = getDeployment(deploymentMessage); + ArchiveRoot ar = prepareTemplate(deployment, deploymentMessage); + + Map nodes = Optional + .ofNullable(ar.getTopology()) + .map(Topology::getNodeTemplates) + .orElseGet(HashMap::new); + + DirectedMultigraph graph = + toscaService.buildNodeGraph(nodes, false); + + TopologicalOrderIterator orderIterator = + new TopologicalOrderIterator<>(graph); + + List orderedKubernetesApps = CommonUtils + .iteratorToStream(orderIterator) + .filter(node -> toscaService.isOfToscaType(node, ToscaConstants.Nodes.Types.KUBERNETES)) + .collect(Collectors.toList()); + + // Map resources = deployment.getResources().stream() + // .filter(resource -> toscaService.isOfToscaType(resource, + // ToscaConstants.Nodes.Types.KUBERNETES)) + // .collect(Collectors.toMap(Resource::getToscaNodeName, res -> res)); + + LinkedHashMap containersByKuberNode = + new LinkedHashMap(); + + List deploymentList = new ArrayList<>(); + + V1Deployment kubernetesDeployment = new V1Deployment(); + + for (NodeTemplate kuberNode : orderedKubernetesApps) { + + KubernetesTask kuberTask = buildTask(graph, kuberNode, deployment.getId()); + + List resources = resourceRepository + .findByToscaNodeNameAndDeployment_id(kuberNode.getName(), deployment.getId()); + + resources.forEach(resource -> resource.setIaasId(kuberTask.getId())); + kuberTask.setInstances(resources.size()); + + // Resource kuberResource = resources.get(resources.indexOf(kuberNode.getName())); + Resource kuberResource = resources.stream() + .filter(resource -> kuberNode.getName().equals(resource.getToscaNodeName())) + .findAny() + .orElse(null); + + String id = Optional.ofNullable(kuberResource.getIaasId()).orElseGet(() -> { + kuberResource.setIaasId(kuberResource.getId()); + return kuberResource.getIaasId(); + }); + + //TODO Check what id it is + containersByKuberNode.put(kuberNode.getName(), kuberTask); + + kubernetesDeployment = generateExternalTaskRepresentation(kuberTask, deployment.getId()); + + DeepKubernetesDeployment deepV1Deployment = + new DeepKubernetesDeployment(kubernetesDeployment, kuberNode.getName()); + deploymentList.add(deepV1Deployment); + + } + + return kubernetesDeployment; + } + + /** + * Build a Kubernetes task object. + * @param graph the input nodegraph. + * @param taskNode the input tasknode. + * @param taskId the input taskid. + * @return the KubernetesTask. + */ + public KubernetesTask buildTask(DirectedMultigraph graph, + NodeTemplate taskNode, String taskId) { + + KubernetesTask kubernetesTask = new KubernetesTask(); + + // orchestrator internal + kubernetesTask.setId(taskId); + + // TODO MAP ALL PROPETIES FROM TOSCA + + Capability containerCapability = getHostCapability(graph, taskNode); + + ToscaUtils + .extractList(containerCapability.getProperties(), + "container", + KubernetesContainer.class::cast) + .ifPresent(kubernetesTask::setContainers); + + DeploymentArtifact image = toscaService + .getNodeArtifactByName(taskNode, "image") + .orElseThrow(() -> new IllegalArgumentException( + String.format(" artifact not found in node <%s> of type <%s>", + taskNode.getName(), taskNode.getType()))); + // artifact type check + List supportedTypes = EnumUtils + .toList(KubernetesContainer.Type.class, KubernetesContainer.Type::getToscaName); + + KubernetesContainer.Type containerType = EnumUtils + .fromPredicate(KubernetesContainer.Type.class, + enumItem -> enumItem.getToscaName().equals(image.getArtifactType())) + .orElseThrow(() -> new IllegalArgumentException(String.format( + "Unsupported artifact type for artifact in node <%s> of type <%s>." + + " Given <%s>, supported <%s>", + taskNode, taskNode.getType(), image.getArtifactType(), supportedTypes))); + + KubernetesContainer container = new KubernetesContainer(containerType); + + String imageName = Optional + .ofNullable(image.getArtifactRef()) + .orElseThrow(() -> + new IllegalArgumentException( + " field for artifact in node <" + taskNode.getName() + + "> must be provided") + ); + + container.setImage(imageName); + + + + + kubernetesTask.setContainer(container); + + /* TODO + * cpu and memory in Kubernetes Container are rappresented as Quantity.class, + * + * The expression 0.1 is equivalent to the expression 100m, + * which can be read as “one hundred millicpu”. */ + + ToscaUtils + .extractScalar(containerCapability.getProperties(), "num_cpus", FloatType.class) + .ifPresent(kubernetesTask::setCpu); + + /* Limits and requests for memory are measured in bytes. + * You can express memory as a plain integer or as a fixed-point integer + * using one of these suffixes: E, P, T, G, M, K. + * You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki.*/ + + ToscaUtils + .extractScalar(containerCapability.getProperties(), "mem_size", SizeType.class) + .map(memSize -> memSize.convert("MB")) + .ifPresent(kubernetesTask::setMemory); + + + ToscaUtils + .extractList(containerCapability.getProperties(), "volumes", String.class::cast) + .ifPresent(kubernetesTask::setVolumes); + + kubernetesTask.setReplicas( + ToscaUtils + .extractScalar(containerCapability.getProperties(), "replicas", FloatType.class) + .orElse(1.0));//ifPresent(kubernetesTask::setReplicas); + + ToscaUtils + .extractList(containerCapability.getProperties(), "publish_ports", l -> + this.generatePortMapping((Map) l) + ) + .ifPresent(portMappings -> kubernetesTask + .getContainer() + .orElseThrow( + () -> new RuntimeException( + "there are ports to publish but no container is present")) + .setPortMappings(portMappings)); + + return kubernetesTask; + } + + protected KubernetesPortMapping generatePortMapping(Map portMappingProperties) { + + int sourcePortValue = CommonUtils + .getFromOptionalMap(portMappingProperties, "source") + .map(value -> ToscaUtils.parseScalar((String) value, IntegerType.class)) + .map(Ints::checkedCast) + .orElseThrow(() -> new ToscaException( + "source port in 'publish_ports' property must be provided")); + + KubernetesPortMapping portMapping = new KubernetesPortMapping(sourcePortValue); + + CommonUtils.getFromOptionalMap(portMappingProperties, "target") + .map(value -> ToscaUtils.parseScalar((String) value, IntegerType.class)) + .map(Ints::checkedCast) + .ifPresent(portMapping::setServicePort); + + CommonUtils.getFromOptionalMap(portMappingProperties, "protocol") + .map(value -> EnumUtils.fromNameOrThrow(Protocol.class, (String) value)) + .ifPresent(portMapping::setProtocol); + + return portMapping; + } + + protected V1Deployment generateExternalTaskRepresentation(KubernetesTask kubernetesTask, + String deploymentId) { + + // V1Deployment v1Deployment = new V1Deployment(); + + // v1Deployment.setApiVersion("apps/v1"); + // v1Deployment.setKind("Deployment"); + + Map requestsRes = new HashMap(); + + String mem = kubernetesTask.getMemory().toString();//.replace("Mb", "M").replaceAll(" ", ""); + + //TODO cpu and ram to string and not quantity maybe + requestsRes.put("cpu", new Quantity(new BigDecimal(kubernetesTask.getCpu()), Format.DECIMAL_SI)); + requestsRes.put("memory", new Quantity(new BigDecimal(kubernetesTask.getMemory()), Format.DECIMAL_SI)); + + List v1Containers = new ArrayList(); + if (kubernetesTask.getContainers() == null || kubernetesTask.getContainers().isEmpty()) { + V1Container contV1 = new V1ContainerBuilder() + .withName(kubernetesTask.getToscaNodeName()+"_"+kubernetesTask.getId()) + .withNewResources() + .withRequests(requestsRes) + .endResources() + .build(); + setContainerPorts(contV1, kubernetesTask.getContainer().get().getPortMappings()); + v1Containers.add(contV1); + } else { + for (KubernetesContainer cont : kubernetesTask.getContainers()) { + V1Container contV1 = new V1ContainerBuilder() + .withName(cont.getType().getName()) + .withImage(cont.getImage()) + .withNewResources() + .withRequests(requestsRes) + .endResources() + //.addNewPort() + // .withHostPort(1)//targhet port optional + // .withContainerPort(cont.getPort().intValue())//source port + //.endPort() + .build(); + setContainerPorts(contV1, cont.getPortMappings()); + v1Containers.add(contV1); + } + } + + V1Deployment v1Deployment = new V1DeploymentBuilder() + .withApiVersion("apps/v1") + .withKind("Deployment") + .withNewMetadata() + .withName(kubernetesTask.getId()) + .endMetadata() + .withNewSpec() + .withReplicas(kubernetesTask.getReplicas().intValue()) /*TODO check if good value*/ + .withNewSelector() + .addToMatchLabels("app", kubernetesTask.getToscaNodeName()) + .endSelector() + .withNewTemplate() + .withNewMetadata() + .addToLabels("app", kubernetesTask.getToscaNodeName()) + .endMetadata() + .withNewSpec() + .addAllToContainers(v1Containers) + .endSpec() + .endTemplate() + .endSpec() + .build(); + + return v1Deployment; + } + + private void setContainerPorts(V1Container contV1, List portMappings) { + for(KubernetesPortMapping kubPort : portMappings) { + V1ContainerPort v1ContainerPort = new V1ContainerPortBuilder() + .withProtocol(kubPort.getProtocol().getName()) + .withContainerPort(kubPort.getContainerPort()) //not nullable port + .withHostPort(kubPort.getServicePort()) //nullable port + .build(); + contV1.addPortsItem(v1ContainerPort); + } + } + + /** + * Connecting Kubernetes Api config. + * @param deploymentMessage DeploymentMessage as parameter + * @return + */ + + + @Override + public boolean doDeploy(DeploymentMessage deploymentMessage) { + Deployment deployment = getDeployment(deploymentMessage); + V1Deployment v1Deployment = createV1Deployment(deploymentMessage); + //V1Deployment v1Deployment = createV1DeploymentForTest(deploymentMessage); + + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + +// try { + AppsV1Api apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); + +// V1Deployment depCreated = apiClient.createNamespacedDeployment( +// "default", +// v1Deployment, +// "true", +// null, +// null); +// +// LOG.debug(depCreated.getStatus().toString()); +// +// //TODO handle exception in out +// } catch (ApiException e) { +// LOG.error("Error in doDeploy:" + e.getCode() + " - " + e.getMessage() + e.getResponseBody()); +// throw new DeploymentException("Error in doDeploy:" +// + e.getCode() + " - " + e.getMessage(), e); +// } + LOG.info("Creating Kubernetes V1Deployment for deployment {} with definition:\n{}", + deploymentMessage.getDeploymentId(), v1Deployment.getMetadata().getName()); + return true; + } + + @Override + public boolean isDeployed(DeploymentMessage deploymentMessage) { + Deployment deployment = getDeployment(deploymentMessage); + + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + + AppsV1Api apiClient; + V1Deployment v1Deployment = new V1Deployment(); + try { + apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); + + v1Deployment = apiClient.readNamespacedDeploymentStatus(deployment.getId(), "default", "true"); + + printPodList(); + } catch (ApiException e) { + LOG.error("Error in isDeployed:" + e.getCode() + " - " + e.getMessage()); + throw new DeploymentException(e.getMessage(), e); + } + + boolean isDeployed = + v1Deployment.getStatus().getReplicas() == v1Deployment.getStatus().getReadyReplicas(); + LOG.debug("Kubernetes App Group for deployment {} is deployed? {}", deployment.getId(), + isDeployed); + if (!isDeployed) { + LOG.warn(v1Deployment.getStatus().getConditions().get(0).getMessage()); + } + return isDeployed; + } + + @Override + public void cleanFailedDeploy(DeploymentMessage deploymentMessage) { + doUndeploy(deploymentMessage); + } + + @Override + public boolean doUpdate(DeploymentMessage deploymentMessage, String template) { + Deployment deployment = getDeployment(deploymentMessage); + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + V1Deployment v1Deployment = new V1DeploymentBuilder() + .withNewMetadata() + .withName(deployment.getId()) + .endMetadata() + .build(); + AppsV1Api apiClient; + try { + apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); + apiClient.patchNamespacedDeployment( + deployment.getId(), + "default", + v1Deployment, + "true", + null, + null, + null); + // apiClient.replaceNamespacedDeployment(name, namespace, body, pretty, dryRun, fieldManager); + + } catch (ApiException e) { + LOG.error("Error in doUpdate:" + e.getCode() + " - " + e.getMessage()); + throw new DeploymentException(e.getMessage(), e); + } + return true; + } + + @Override + public void cleanFailedUpdate(DeploymentMessage deploymentMessage) { + doUndeploy(deploymentMessage); + } + + @Override + public boolean doUndeploy(DeploymentMessage deploymentMessage) { + + Deployment deployment = getDeployment(deploymentMessage); + CloudProviderEndpoint cloudProviderEndpoint = deployment.getCloudProviderEndpoint(); + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + if (cloudProviderEndpoint != null) { + AppsV1Api apiClient; + try { + apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); + + V1Status status = apiClient.deleteNamespacedDeployment( + deployment.getId(), + "default", + "true", + null, + null, + null, + null, + null); + + LOG.debug("Deleting deployment exited with :" + + status.getCode() + + " - " + + status.getMessage() + + " - " + + status.getStatus()); + + } catch (ApiException e) { + LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + // TODO manage throwing errorCode exception + // if(e.getCode()!=404) { + // throw new HttpResponseException(e.getCode(), "KubernetesApiException"); + // } + throw new DeploymentException("Error in doUndeploy:" + + e.getCode() + " - " + e.getMessage(), e); + } + } + return true; + } + + @Override + public boolean isUndeployed(DeploymentMessage deploymentMessage) { + boolean isUndeployed = false; + Deployment deployment = getDeployment(deploymentMessage); + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + + AppsV1Api apiClient; + try { + apiClient = getClient(deployment.getCloudProviderEndpoint(), requestedWithToken); + + V1Deployment deplSimleToCheck = apiClient.readNamespacedDeploymentStatus( + deployment.getId(), + "default", + "true"); + + V1Deployment depl = apiClient.readNamespacedDeployment( + deployment.getId(), + "default", + "true", + null, + null); + + if (depl == null) { + isUndeployed = true; + } + + } catch (ApiException e) { + LOG.error("Error in doUndeploy:" + e.getCode() + " - " + e.getMessage()); + if (e.getCode() == 404) { + isUndeployed = true; + } + throw new DeploymentException(e.getMessage(), e); + } + + return isUndeployed; + } + + @Override + public void doProviderTimeout(DeploymentMessage deploymentMessage) { + throw new BusinessWorkflowException(ErrorCode.CLOUD_PROVIDER_ERROR, + "Error executing request to Kubernetes", + new DeploymentException("Kubernetes provider timeout during deployment")); + } + + @Override + protected Optional getAdditionalErrorInfoInternal(DeploymentMessage deploymentMessage) { + Deployment deployment = getDeployment(deploymentMessage); + + final OidcTokenId requestedWithToken = deploymentMessage.getRequestedWithToken(); + + List cloudProviderEndpoints = + deployment.getCloudProviderEndpoint().getAllCloudProviderEndpoint(); + //TODO + return Optional.empty(); + } + + private void printPodList() throws ApiException { + COREV1_API = new CoreV1Api(); + V1PodList list = COREV1_API.listPodForAllNamespaces( + null, + null, + null, + null, + null, + null, + null, + null, + null); + for (V1Pod item : list.getItems()) { + System.out.println(item.getMetadata().getName()); + } + } + + @Data + @NoArgsConstructor(access = AccessLevel.PROTECTED) + @RequiredArgsConstructor + public static class DeepKubernetesDeployment { + + @NonNull + @NotNull + private V1Deployment v1Deployment; + + @NonNull + @NotNull + private String toscaNodeName; + + } + + protected Capability getHostCapability( + DirectedMultigraph graph, NodeTemplate taskNode) { + + NodeTemplate hostNode = getHostNode(graph, taskNode); + + // at this point we're sure that it exists + return hostNode.getCapabilities().get(HOST_CAPABILITY_NAME); + } + + protected NodeTemplate getHostNode(DirectedMultigraph graph, + NodeTemplate taskNode) { + return graph + .incomingEdgesOf(taskNode) + .stream() + .filter( + relationship -> HOST_CAPABILITY_NAME.equals(relationship.getTargetedCapabilityName())) + .map(graph::getEdgeSource) + // if more than 1 node is present -> IllegalArgumentException + .collect(MoreCollectors.toOptional()) + .orElseThrow(() -> new IllegalArgumentException( + String.format("No hosting node provided for node <%s>", taskNode.getName()))); + } + + private V1Deployment createV1DeploymentForTest(DeploymentMessage deploymentMessage) { + /* note , cpu and ram are shared between all containers in the pod + * so it is enought difine it once*/ + + //TODO manage needed field for deployment and get them from DeploymentMessage + + Map requestsRes = new HashMap(); + /*The expression 0.1 is equivalent to the expression 100m, + *which can be read as “one hundred millicpu”.*/ + requestsRes.put("cpu", new Quantity("32Mi")); + requestsRes.put("memory", new Quantity("100m")); + + Map limitRes = new HashMap(); + limitRes.put("cpu", new Quantity("64Mi")); + limitRes.put("memory", new Quantity("200m")); + + V1ResourceRequirements resources = new V1ResourceRequirementsBuilder() + .withRequests(requestsRes) + .withLimits(limitRes) + .build(); + + V1DeploymentSpec spec = new V1DeploymentSpec(); + V1Container cont = new V1ContainerBuilder() + .withName("nginx") + .withImage("nginx:1.7.9") + .withResources(resources) + .addNewPort() + .withContainerPort(80) + .endPort() + .build(); + + Deployment deployment = getDeployment(deploymentMessage); + + V1Deployment v1Deployment = new V1DeploymentBuilder() + .withApiVersion("apps/v1") + .withKind("Deployment") + .withNewMetadata() + .withName(deployment.getId()) + .endMetadata() + .withNewSpec() + .withReplicas((Integer) deployment.getParameters().get("replicas")) + .withNewSelector() + .addToMatchLabels("app", "nginx") + .endSelector() + .withNewTemplate() + .withNewMetadata() + .addToLabels("app", "nginx") + .endMetadata() + .withNewSpec() + .addNewContainer() + .withImage("nginx:1.7.9") + .withName("nginx") + .addNewPort() + .withContainerPort(80) + .endPort() + .endContainer() + .withContainers(cont) + + .endSpec() + .endTemplate() + .endSpec() + .build(); + + return v1Deployment; + } + +} + + + diff --git a/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java b/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java new file mode 100644 index 0000000000..a7b6c61221 --- /dev/null +++ b/src/main/java/it/reply/orchestrator/service/deployment/providers/factory/KubernetesClientFactory.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.service.deployment.providers.factory; + +import feign.Feign; +import feign.Logger.Level; +import feign.auth.BasicAuthRequestInterceptor; +import feign.RequestInterceptor; +import feign.slf4j.Slf4jLogger; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.apis.AppsV1Api; +import io.kubernetes.client.openapi.auth.ApiKeyAuth; +import io.kubernetes.client.util.Config; +import it.infn.ba.deep.qcg.client.utils.QcgException; +import it.reply.orchestrator.dal.entity.OidcTokenId; +import it.reply.orchestrator.dto.CloudProviderEndpoint; +import it.reply.orchestrator.dto.cmdb.CloudService; +import it.reply.orchestrator.dto.deployment.DeploymentMessage; +import it.reply.orchestrator.dto.security.GenericServiceCredential; +import it.reply.orchestrator.utils.CommonUtils; + +import java.io.IOException; +import java.util.Objects; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.http.HttpHeaders; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class KubernetesClientFactory extends CloudService{ + +// /** +// * Build a Kubernetes Api client object. +// * @param cloudProviderEndpoint the service endpoint. +// * @param accessToken the input accesstoken. +// * @return the Kubernetes Api client object. +// */ +// public void build(CloudProviderEndpoint cloudProviderEndpoint, OidcTokenId accessToken) { +// final RequestInterceptor requestInterceptor; +// Objects.requireNonNull(accessToken, "Access Token must not be null"); +// requestInterceptor = requestTemplate -> +// requestTemplate.header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken); +// +// //TODO if needed +// } + + /** + * Build a Kubernetes client object. + * @param cloudProviderEndpoint the input Kubernetes service endpoint. + * @param accessToken. + * @return AppsV1Api the Kubernetes client object. + * @throws IOException + */ + public AppsV1Api build(CloudProviderEndpoint cloudProviderEndpoint, String accessToken) throws IOException { + LOG.info("Generating Kubernetes client with endpoint {}", cloudProviderEndpoint.getCpEndpoint()); + + ApiClient x = new ApiClient(); + + ApiKeyAuth bearerToken = (ApiKeyAuth) x.getAuthentication("BearerToken"); + bearerToken.setApiKey(accessToken); + + String cpEndpoint = cloudProviderEndpoint.getCpEndpoint(); + + //x.setAccessToken(accessToken); + if (cpEndpoint != null && !cpEndpoint.isEmpty()) { + x.setBasePath(cpEndpoint); + } + + ApiClient client = Config.fromToken(cpEndpoint, accessToken); + if (client == null) { + client = Config.defaultClient(); + } + return new AppsV1Api(client); + } + +} diff --git a/src/main/java/it/reply/orchestrator/utils/ToscaConstants.java b/src/main/java/it/reply/orchestrator/utils/ToscaConstants.java index 7c5f3bd7c0..af1f8bb133 100644 --- a/src/main/java/it/reply/orchestrator/utils/ToscaConstants.java +++ b/src/main/java/it/reply/orchestrator/utils/ToscaConstants.java @@ -32,6 +32,8 @@ public static class Types { BASE_INDIGO_NAME + "Container.Application.Docker.Chronos"; public static final String MARATHON = BASE_INDIGO_NAME + "Container.Application.Docker.Marathon"; + public static final String KUBERNETES = + BASE_INDIGO_NAME + "Container.Application.Docker.Kubernetes"; public static final String COMPUTE = BASE_INDIGO_NAME + "Compute"; public static final String QCG = BASE_INDIGO_NAME + "Qcg.Job"; public static final String ELASTIC_CLUSTER = BASE_INDIGO_NAME + "ElasticCluster"; diff --git a/src/main/resources/tosca-definitions/custom_types.yaml b/src/main/resources/tosca-definitions/custom_types.yaml index 1973e6d501..7813a35641 100644 --- a/src/main/resources/tosca-definitions/custom_types.yaml +++ b/src/main/resources/tosca-definitions/custom_types.yaml @@ -2349,6 +2349,25 @@ node_types: kube_admin_username: { get_property: [ SELF, admin_username] } kube_admin_token: { get_property: [ SELF, admin_token] } + tosca.nodes.indigo.Container.Application.Docker.Kubernetes: + derived_from: tosca.nodes.indigo.Container.Application.Docker + metadata: + icon: /images/kubernetesWN.png + attributes: + some_attrib: + type: list + entry_schema: string + properties: + secrets: + entry_schema: + type: string + required: no + type: map + replicas: + type: integer + default: 1 + required: no + tosca.nodes.indigo.JupyterHub: derived_from: tosca.nodes.SoftwareComponent metadata: diff --git a/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java b/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java index 4d3d44d5c1..842ac6f6e4 100644 --- a/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java +++ b/src/test/java/it/reply/orchestrator/service/DeploymentServiceTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2019 Santer Reply S.p.A. + * Copyright © 2015-2020 Santer Reply S.p.A. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -506,7 +506,8 @@ public void deleteDeploymentSuccesfulWithProvider(Status status) throws Exceptio @Parameters({ "CHRONOS", "MARATHON", - "QCG"}) + "QCG", + "KUBERNETES"}) public void updateDeploymentBadRequest(DeploymentProvider provider) throws Exception { String id = UUID.randomUUID().toString(); diff --git a/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java new file mode 100644 index 0000000000..610c6697f5 --- /dev/null +++ b/src/test/java/it/reply/orchestrator/service/deployment/providers/KubernetesServiceTest.java @@ -0,0 +1,328 @@ +/* + * Copyright © 2015-2020 Santer Reply S.p.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.reply.orchestrator.service.deployment.providers; + +import static org.assertj.core.api.Assertions.assertThatCode; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.assertj.core.api.AbstractThrowableAssert; +import org.assertj.core.api.Assertions; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.junit4.rules.SpringClassRule; +import org.springframework.test.context.junit4.rules.SpringMethodRule; + +import com.google.common.collect.Lists; + +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.AppsV1Api; +import io.kubernetes.client.util.Config; +import it.reply.orchestrator.config.specific.ToscaParserAwareTest; +import it.reply.orchestrator.controller.ControllerTestUtils; +import it.reply.orchestrator.dal.entity.Deployment; +import it.reply.orchestrator.dal.entity.Resource; +import it.reply.orchestrator.dal.repository.DeploymentRepository; +import it.reply.orchestrator.dal.repository.ResourceRepository; +import it.reply.orchestrator.dto.CloudProviderEndpoint; +import it.reply.orchestrator.dto.CloudProviderEndpoint.IaaSType; +import it.reply.orchestrator.dto.cmdb.CloudService; +import it.reply.orchestrator.dto.cmdb.CloudServiceType; +import it.reply.orchestrator.dto.cmdb.KubernetesService; +import it.reply.orchestrator.dto.deployment.DeploymentMessage; +import it.reply.orchestrator.dto.onedata.OneData; +import it.reply.orchestrator.dto.onedata.OneData.OneDataProviderInfo; +import it.reply.orchestrator.dto.workflow.CloudServicesOrderedIterator; +import it.reply.orchestrator.enums.NodeStates; +import it.reply.orchestrator.exception.service.DeploymentException; +import it.reply.orchestrator.function.ThrowingFunction; +import it.reply.orchestrator.service.ToscaService; +import it.reply.orchestrator.service.ToscaServiceTest; +import it.reply.orchestrator.service.VaultService; +import it.reply.orchestrator.service.deployment.providers.factory.KubernetesClientFactory; +import it.reply.orchestrator.service.security.OAuth2TokenService; +import it.reply.orchestrator.util.TestUtil; +import junitparams.JUnitParamsRunner; + +@RunWith(JUnitParamsRunner.class) +public class KubernetesServiceTest extends ToscaParserAwareTest { + + @ClassRule + public static final SpringClassRule SPRING_CLASS_RULE = new SpringClassRule(); + + @Rule + public final SpringMethodRule springMethodRule = new SpringMethodRule(); + + @Spy + @InjectMocks + private KubernetesServiceImpl kubernetesServiceImpl; + + @SpyBean + @Autowired + protected ToscaService toscaService; + + @MockBean + private ResourceRepository resourceRepository; + + @MockBean + private DeploymentRepository deploymentRepository; + + @MockBean + private VaultService vaultService; + + @MockBean + private KubernetesClientFactory kubernetesClientFactory; + + @SpyBean + private OAuth2TokenService oauth2TokenService; + + @SpyBean + private Config config; + + private static final String defaultVaultEndpoint = "https://default.vault.com:8200"; + + @Before + public void setup() throws Exception { + MockitoAnnotations.initMocks(this); + Mockito + .when(oauth2tokenService.executeWithClientForResult( + Mockito.any(), Mockito.any(), Mockito.any())) + .thenAnswer(y -> ((ThrowingFunction) y.getArguments()[1]).apply("token")); + + } + + @Test + public void testDoDeploy() throws IOException, URISyntaxException { + Deployment deployment = generateDeployment(); + DeploymentMessage dm = generateDeployDm(deployment); + + KubernetesService cs = buildService(); + + CloudServicesOrderedIterator csi = new CloudServicesOrderedIterator(Lists.newArrayList(cs)); + csi.next(); + dm.setCloudServicesOrderedIterator(csi); + + Mockito + .when(deploymentRepository.findOne(deployment.getId())) + .thenReturn(deployment); + ; + + Mockito + .when(oauth2TokenService.getAccessToken(dm.getRequestedWithToken())) + .thenReturn("token"); + + Mockito + .when(kubernetesClientFactory.build(deployment.getCloudProviderEndpoint(), "token")) + .thenReturn(new AppsV1Api(Config.defaultClient())); + + Assertions + .assertThatExceptionOfType(DeploymentException.class); + + AbstractThrowableAssert assertion = assertThatCode( + () -> kubernetesServiceImpl.doDeploy(dm)); + + assertion.isInstanceOf(DeploymentException.class) + .hasCauseExactlyInstanceOf(ApiException.class); + } + + @Test + public void testDoUndeploy() throws IOException { + Deployment deployment = generateDeployment(); + DeploymentMessage dm = generateDeployDm(deployment); + + KubernetesService cs = buildServiceLocal(); + + CloudServicesOrderedIterator csi = new CloudServicesOrderedIterator(Lists.newArrayList(cs)); + csi.next(); + dm.setCloudServicesOrderedIterator(csi); + + Mockito + .when(deploymentRepository.findOne(deployment.getId())) + .thenReturn(deployment); + ; + + Mockito + .when(oauth2TokenService.getAccessToken(dm.getRequestedWithToken())) + .thenReturn("token"); + + Mockito + .when(kubernetesClientFactory.build(deployment.getCloudProviderEndpoint(), Mockito.anyString())) + .thenReturn(new AppsV1Api(Config.defaultClient())); + + AbstractThrowableAssert assertion = assertThatCode( + () -> kubernetesServiceImpl.doUndeploy(dm)); + + assertion.isInstanceOf(DeploymentException.class) + .hasCauseExactlyInstanceOf(ApiException.class); + } + + // @Test + // public void testConnectApi() throws IOException { + // Deployment deployment = generateDeployment(); + // DeploymentMessage dm = generateDeployDm(deployment); + // + // AppsV1Api expected = new AppsV1Api(new ApiClient()); + // + // Mockito + // .when(oauth2TokenService.getAccessToken(Mockito.any())) + // .thenReturn("token"); + // + // Assertions + // .assertThat(kubernetesServiceImpl.connectApi(dm)) + // .isEqualTo(expected); + // } + + private DeploymentMessage generateDeployDmKuber(Deployment deployment) { + DeploymentMessage dm = new DeploymentMessage(); + dm.setDeploymentId(deployment.getId()); + CloudProviderEndpoint chosenCloudProviderEndpoint = CloudProviderEndpoint + .builder() + .cpComputeServiceId(UUID.randomUUID().toString()) + .cpEndpoint("http://www.example.com/api") + .iaasType(IaaSType.KUBERNETES) + .build(); + dm.setChosenCloudProviderEndpoint(chosenCloudProviderEndpoint); + deployment.setCloudProviderEndpoint(chosenCloudProviderEndpoint); + return dm; + } + + private DeploymentMessage generateDeployDm(Deployment deployment) { + DeploymentMessage dm = new DeploymentMessage(); + dm.setDeploymentId(deployment.getId()); + CloudProviderEndpoint chosenCloudProviderEndpoint = CloudProviderEndpoint + .builder() + .cpComputeServiceId(UUID.randomUUID().toString()) + .cpEndpoint("http://example.com") + .iaasType(IaaSType.KUBERNETES) + .build(); + dm.setChosenCloudProviderEndpoint(chosenCloudProviderEndpoint); + deployment.setCloudProviderEndpoint(chosenCloudProviderEndpoint); + Map oneDataParameters = new HashMap<>(); + OneDataProviderInfo providerInfo = OneDataProviderInfo + .builder() + .cloudProviderId("provider-1") + .cloudServiceId(UUID.randomUUID().toString()) + .endpoint("http://example.onedata.com") + .id("test") + .build(); + List oneproviders = new ArrayList(); + oneproviders.add(providerInfo); + OneData parameter = OneData + .builder() + .oneproviders(oneproviders) + .onezone("test") + .path("/tmp/") + .selectedOneprovider(providerInfo) + .serviceSpace(true) + .smartScheduling(false) + .space("test") + .token("0123456789-onedata-token") + .build(); + oneDataParameters.put("provider-1", parameter); + dm.setOneDataParameters(oneDataParameters); + return dm; + } + + private Deployment generateDeployment() throws IOException { + Deployment deployment = ControllerTestUtils.createDeployment(); + deployment.setCloudProviderEndpoint(CloudProviderEndpoint.builder() + .cpComputeServiceId(UUID.randomUUID().toString()) + .cpEndpoint("example.com") + .iaasType(IaaSType.KUBERNETES) + .build()); + deployment.setTemplate( + TestUtil.getFileContentAsString(ToscaServiceTest.TEMPLATES_BASE_DIR + "kubernetes_app.yaml")); + + Resource runtime = new Resource(); + runtime.setDeployment(deployment); + runtime.setId("1"); + runtime.setState(NodeStates.INITIAL); + runtime.setToscaNodeName("Docker"); + runtime.setToscaNodeType("tosca.nodes.indigo.Container.Runtime.Docker"); + deployment.getResources().add(runtime); + + Resource app = new Resource(); + app.setDeployment(deployment); + app.setId("2"); + app.setState(NodeStates.INITIAL); + app.setToscaNodeName("kubernetes"); + app.setToscaNodeType("tosca.nodes.indigo.Container.Application.Docker.Kubernetes"); + app.addRequiredResource(runtime); + deployment.getResources().add(app); + + Map paramMap = new HashMap(); + paramMap.put("replicas", 1); + + deployment.setParameters(paramMap); + + Mockito + .when(resourceRepository.findByToscaNodeNameAndDeployment_id("Docker", + deployment.getId())) + .thenReturn(Lists.newArrayList(runtime)); + + Mockito + .when(resourceRepository.findByToscaNodeNameAndDeployment_id("kubernetes", + deployment.getId())) + .thenReturn(Lists.newArrayList(app)); + return deployment; + } + + private KubernetesService buildService() { + KubernetesService cs = KubernetesService + .kubernetesBuilder() + .endpoint("http://localhost:8080") + .serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) + .hostname("localhost") + .providerId("RECAS-BARI") + .id("http://localhost:8080") + .type(CloudServiceType.COMPUTE) + .iamEnabled(false) + .publicService(true) + .build(); + return cs; + } + private KubernetesService buildServiceLocal() { + KubernetesService cs = KubernetesService + .kubernetesBuilder() + .endpoint("https://kubernetes.docker.internal:6443/") + .serviceType(CloudService.KUBERNETES_COMPUTE_SERVICE) + .hostname("localhost") + .providerId("RECAS-BARI") + .id("https://kubernetes.docker.internal:6443/") + .type(CloudServiceType.COMPUTE) + .iamEnabled(false) + .publicService(true) + .build(); + return cs; + } +} diff --git a/src/test/resources/tosca/kubernetes_app.yaml b/src/test/resources/tosca/kubernetes_app.yaml new file mode 100644 index 0000000000..73e703f785 --- /dev/null +++ b/src/test/resources/tosca/kubernetes_app.yaml @@ -0,0 +1,35 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +imports: + - indigo_custom_types: https://raw.githubusercontent.com/indigo-dc/tosca-types/master/custom_types.yaml + +topology_template: + + node_templates: + + kubernetes: + type: tosca.nodes.indigo.Container.Application.Docker.Kubernetes + artifacts: + image: + file: hello-world + type: tosca.artifacts.Deployment.Image.Container.Docker + properties: + replicas: 2 + requirements: + - host: docker + + docker: + type: tosca.nodes.indigo.Container.Runtime.Docker + capabilities: + host: + properties: + num_cpus: 1.0 + mem_size: 256 Mb + publish_ports: + - protocol: tcp + source: 8080 + volumes: [ 'kubernetes:/data:rw:dvdi:rexray', '/data:rw' ] + + outputs: + endpoint: + value: { concat: [ { get_attribute : [ kubernetes, attr, 0 ] }, ':', { get_attribute : [ docker, host, publish_ports, 0, target ] } ] }