From e5de42e407efee3482e99f22af2aa290bf2cc91e Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 20 Dec 2023 19:23:52 -0400 Subject: [PATCH] Engine traefik docker labels. --- .../docker_kurtosis_backend/consts/consts.go | 3 + .../engine_functions/create_engine.go | 2 + .../object_attributes_provider.go | 82 ++++++++++++++++++- .../engine_functions/create_engine.go | 4 +- .../objects/engine/consts.go | 5 +- 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go index 8b8e50ed08..d2ede4273b 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go @@ -15,6 +15,9 @@ const ( // be stored in the port spec label KurtosisInternalContainerGrpcPortId = "grpc" + // The ID of the REST API port. + KurtosisInternalContainerRESTAPIPortId = "rest-api" + // The engine server uses gRPC so MUST listen on TCP (no other protocols are supported) EngineTransportProtocol = port_spec.TransportProtocol_TCP diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 924bc27209..59f9222263 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -172,6 +172,8 @@ func CreateEngine( engineGuid, consts.KurtosisInternalContainerGrpcPortId, privateGrpcPortSpec, + consts.KurtosisInternalContainerRESTAPIPortId, + restAPIPortSpec, ) if err != nil { return nil, stacktrace.Propagate( diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go index 52fcbc54d6..4c949d6b99 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go @@ -1,6 +1,10 @@ package object_attributes_provider import ( + "fmt" + "strconv" + "strings" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_value" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_object_name" @@ -10,7 +14,6 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/engine" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/stacktrace" - "strings" ) const ( @@ -18,6 +21,7 @@ const ( logsAggregatorName = "kurtosis-logs-aggregator" logsStorageVolumeName = "kurtosis-logs-storage" reverseProxyName = "kurtosis-reverse-proxy" + engineRESTAPIPortStr = "engine-rest-api" ) type DockerObjectAttributesProvider interface { @@ -25,6 +29,8 @@ type DockerObjectAttributesProvider interface { guid engine.EngineGUID, grpcPortId string, grpcPortSpec *port_spec.PortSpec, + restAPIPortId string, + restAPIPortSpec *port_spec.PortSpec, ) (DockerObjectAttributes, error) ForEnclave(enclaveUuid enclave.EnclaveUUID) (DockerEnclaveObjectAttributesProvider, error) ForLogsAggregator() (DockerObjectAttributes, error) @@ -47,6 +53,8 @@ func (provider *dockerObjectAttributesProviderImpl) ForEngineServer( guid engine.EngineGUID, grpcPortId string, grpcPortSpec *port_spec.PortSpec, + restAPIPortId string, + restAPIPortSpec *port_spec.PortSpec, ) (DockerObjectAttributes, error) { nameStr := strings.Join( @@ -85,6 +93,14 @@ func (provider *dockerObjectAttributesProviderImpl) ForEngineServer( docker_label_key.GUIDDockerLabelKey: guidLabelValue, } + traefikLabels, err := provider.getTraefikLabelsForEngine(restAPIPortSpec) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting traefik labels for engine server") + } + for traefikLabelKey, traefikLabelValue := range traefikLabels { + labels[traefikLabelKey] = traefikLabelValue + } + objectAttributes, err := newDockerObjectAttributesImpl(name, labels) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred while creating the ObjectAttributesImpl with the name '%s' and labels '%+v'", name, labels) @@ -153,3 +169,67 @@ func (provider *dockerObjectAttributesProviderImpl) ForReverseProxy() (DockerObj } return objectAttributes, nil } + +// Return Traefik labels +// Including the labels required to route traffic to the engine rest api port if the Host header is set to "engine". +// +// "traefik.enable": "true", +// "traefik.http.routers.engine-rest-api.rule": "Host(`engine`)", +// "traefik.http.routers.engine-rest-api.service": "engine-rest-api", +// "traefik.http.services.engine-rest-api.loadbalancer.server.port": "" +func (provider *dockerObjectAttributesProviderImpl) getTraefikLabelsForEngine(restAPIPortSpec *port_spec.PortSpec) (map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue, error) { + labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{} + + // Header Host rule + ruleKeySuffix := fmt.Sprintf("http.routers.%s.rule", engineRESTAPIPortStr) + ruleLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey(ruleKeySuffix) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the traefik rule label key with suffix '%v'", ruleKeySuffix) + } + ruleValue := fmt.Sprintf("Host(`%s`)", engine.RESTAPIPortHostHeader) + ruleLabelValue, err := docker_label_value.CreateNewDockerLabelValue(ruleValue) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating the traefik rule label value with value '%v'", ruleValue) + } + labels[ruleLabelKey] = ruleLabelValue + + // Service name + serviceKeySuffix := fmt.Sprintf("http.routers.%s.service", engineRESTAPIPortStr) + serviceLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey(serviceKeySuffix) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the traefik service label key with suffix '%v'", serviceKeySuffix) + } + serviceValue := engineRESTAPIPortStr + serviceLabelValue, err := docker_label_value.CreateNewDockerLabelValue(serviceValue) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating the traefik service label value with value '%v'", serviceValue) + } + labels[serviceLabelKey] = serviceLabelValue + + // Service port number + portKeySuffix := fmt.Sprintf("http.services.%s.loadbalancer.server.port", engineRESTAPIPortStr) + portLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey(portKeySuffix) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the traefik port label key with suffix '%v'", portKeySuffix) + } + portValue := strconv.Itoa(int(restAPIPortSpec.GetNumber())) + portLabelValue, err := docker_label_value.CreateNewDockerLabelValue(portValue) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating the traefik port label value with value '%v'", portValue) + } + labels[portLabelKey] = portLabelValue + + // Enable Traefik + traefikEnableLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey("enable") + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the traefik enable label key") + } + traefikEnableValue := "true" + traefikEnableLabelValue, err := docker_label_value.CreateNewDockerLabelValue(traefikEnableValue) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating the traefik enable label value with value '%v'", traefikEnableValue) + } + labels[traefikEnableLabelKey] = traefikEnableLabelValue + + return labels, nil +} diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go index 937b9d2e18..ebafaf1d87 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go @@ -28,8 +28,6 @@ const ( maxWaitForEngineContainerAvailabilityRetries = 30 timeBetweenWaitForEngineContainerAvailabilityRetries = 1 * time.Second httpApplicationProtocol = "http" - - restAPIPortHost = "engine" ) var noWait *port_spec.Wait = nil @@ -584,7 +582,7 @@ func getEngineIngressRules( ) ([]netv1.IngressRule, error) { var ingressRules []netv1.IngressRule ingressRule := netv1.IngressRule{ - Host: restAPIPortHost, + Host: engine.RESTAPIPortHostHeader, IngressRuleValue: netv1.IngressRuleValue{ HTTP: &netv1.HTTPIngressRuleValue{ Paths: []netv1.HTTPIngressPath{ diff --git a/container-engine-lib/lib/backend_interface/objects/engine/consts.go b/container-engine-lib/lib/backend_interface/objects/engine/consts.go index 91c1633998..226ad72257 100644 --- a/container-engine-lib/lib/backend_interface/objects/engine/consts.go +++ b/container-engine-lib/lib/backend_interface/objects/engine/consts.go @@ -1,6 +1,7 @@ package engine const ( - RESTAPIPortAddr uint16 = 9779 - RESTAPIHostIP string = "0.0.0.0" + RESTAPIPortAddr uint16 = 9779 + RESTAPIHostIP string = "0.0.0.0" + RESTAPIPortHostHeader string = "engine" )