Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

allow to customize readiness probe path and configure additional component ports #251

Merged
merged 8 commits into from
Apr 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pkg/components/controller_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

"go.ytsaurus.tech/library/go/ptr"
corev1 "k8s.io/api/core/v1"

ytv1 "github.com/ytsaurus/yt-k8s-operator/api/v1"
"github.com/ytsaurus/yt-k8s-operator/pkg/apiproxy"
Expand Down Expand Up @@ -41,6 +42,11 @@ func NewControllerAgent(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus,
"ca",
"controller-agents",
func() ([]byte, error) { return cfgen.GetControllerAgentConfig(resource.Spec.ControllerAgents) },
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.ControllerAgentRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

return &ControllerAgent{
Expand Down
6 changes: 6 additions & 0 deletions pkg/components/data_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

"go.ytsaurus.tech/library/go/ptr"
corev1 "k8s.io/api/core/v1"

ytv1 "github.com/ytsaurus/yt-k8s-operator/api/v1"
"github.com/ytsaurus/yt-k8s-operator/pkg/apiproxy"
Expand Down Expand Up @@ -48,6 +49,11 @@ func NewDataNode(
func() ([]byte, error) {
return cfgen.GetDataNodeConfig(spec)
},
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.DataNodeRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

return &DataNode{
Expand Down
6 changes: 6 additions & 0 deletions pkg/components/discovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

"go.ytsaurus.tech/library/go/ptr"
corev1 "k8s.io/api/core/v1"

ytv1 "github.com/ytsaurus/yt-k8s-operator/api/v1"
"github.com/ytsaurus/yt-k8s-operator/pkg/apiproxy"
Expand Down Expand Up @@ -42,6 +43,11 @@ func NewDiscovery(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus) *Disco
func() ([]byte, error) {
return cfgen.GetDiscoveryConfig(&resource.Spec.Discovery)
},
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.DiscoveryRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

return &Discovery{
Expand Down
6 changes: 6 additions & 0 deletions pkg/components/exec_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

"go.ytsaurus.tech/library/go/ptr"
corev1 "k8s.io/api/core/v1"

ytv1 "github.com/ytsaurus/yt-k8s-operator/api/v1"
"github.com/ytsaurus/yt-k8s-operator/pkg/apiproxy"
Expand Down Expand Up @@ -47,6 +48,11 @@ func NewExecNode(
func() ([]byte, error) {
return cfgen.GetExecNodeConfig(spec)
},
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.ExecNodeRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

var sidecarConfig *ConfigHelper
Expand Down
6 changes: 6 additions & 0 deletions pkg/components/exec_node_remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

"go.ytsaurus.tech/library/go/ptr"
corev1 "k8s.io/api/core/v1"

ytv1 "github.com/ytsaurus/yt-k8s-operator/api/v1"
"github.com/ytsaurus/yt-k8s-operator/pkg/apiproxy"
Expand Down Expand Up @@ -47,6 +48,11 @@ func NewRemoteExecNodes(
func() ([]byte, error) {
return cfgen.GetExecNodeConfig(spec)
},
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.ExecNodeRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)
return &RemoteExecNode{
baseComponent: baseComponent{labeller: &l},
Expand Down
20 changes: 19 additions & 1 deletion pkg/components/httpproxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"

"go.ytsaurus.tech/library/go/ptr"

corev1 "k8s.io/api/core/v1"

ytv1 "github.com/ytsaurus/yt-k8s-operator/api/v1"
Expand Down Expand Up @@ -56,6 +55,25 @@ func NewHTTPProxy(
func() ([]byte, error) {
return cfgen.GetHTTPProxyConfig(spec)
},
WithContainerPorts(
corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.HTTPProxyRPCPort,
Protocol: corev1.ProtocolTCP,
},
corev1.ContainerPort{
Name: "http",
ContainerPort: consts.HTTPProxyHTTPPort,
Protocol: corev1.ProtocolTCP,
},
corev1.ContainerPort{
Name: "https",
ContainerPort: consts.HTTPProxyHTTPSPort,
Protocol: corev1.ProtocolTCP,
},
),
WithCustomReadinessProbeEndpointPort(consts.HTTPProxyHTTPPort),
WithCustomReadinessProbeEndpointPath("/ping"),
)

var httpsSecret *resources.TLSSecret
Expand Down
5 changes: 5 additions & 0 deletions pkg/components/master.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ func NewMaster(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus) *Master {
cfgen.GetMastersStatefulSetName(),
cfgen.GetMastersServiceName(),
func() ([]byte, error) { return cfgen.GetMasterConfig(&resource.Spec.PrimaryMasters) },
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.MasterRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

initJob := NewInitJob(
Expand Down
6 changes: 6 additions & 0 deletions pkg/components/master_caches.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

"go.ytsaurus.tech/library/go/ptr"
corev1 "k8s.io/api/core/v1"

ytv1 "github.com/ytsaurus/yt-k8s-operator/api/v1"
"github.com/ytsaurus/yt-k8s-operator/pkg/apiproxy"
Expand Down Expand Up @@ -41,6 +42,11 @@ func NewMasterCache(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus) *Mas
cfgen.GetMasterCachesStatefulSetName(),
cfgen.GetMasterCachesServiceName(),
func() ([]byte, error) { return cfgen.GetMasterCachesConfig(resource.Spec.MasterCaches) },
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.MasterCachesRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

return &MasterCache{
Expand Down
5 changes: 5 additions & 0 deletions pkg/components/query_tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ func NewQueryTracker(
cfgen.GetQueryTrackerStatefulSetName(),
cfgen.GetQueryTrackerServiceName(),
func() ([]byte, error) { return cfgen.GetQueryTrackerConfig(resource.Spec.QueryTrackers) },
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.QueryTrackerRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

image := ytsaurus.GetResource().Spec.CoreImage
Expand Down
5 changes: 5 additions & 0 deletions pkg/components/queue_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ func NewQueueAgent(
cfgen.GetQueueAgentStatefulSetName(),
cfgen.GetQueueAgentServiceName(),
func() ([]byte, error) { return cfgen.GetQueueAgentConfig(resource.Spec.QueueAgents) },
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.QueueAgentRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

image := ytsaurus.GetResource().Spec.CoreImage
Expand Down
5 changes: 5 additions & 0 deletions pkg/components/rpcproxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ func NewRPCProxy(
func() ([]byte, error) {
return cfgen.GetRPCProxyConfig(spec)
},
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.RPCProxyRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

var balancingService *resources.RPCService = nil
Expand Down
5 changes: 5 additions & 0 deletions pkg/components/scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ func NewScheduler(
func() ([]byte, error) {
return cfgen.GetSchedulerConfig(resource.Spec.Schedulers)
},
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.SchedulerRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

return &Scheduler{
Expand Down
44 changes: 33 additions & 11 deletions pkg/components/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
)

const (
serverAPIPortName = "http"
readinessProbeHTTPPath = "/orchid/service"
)

Expand Down Expand Up @@ -55,6 +54,11 @@ type serverImpl struct {
configHelper *ConfigHelper

builtStatefulSet *appsv1.StatefulSet

componentContainerPorts []corev1.ContainerPort

readinessProbePort intstr.IntOrString
readinessProbeHTTPPath string
}

func newServer(
Expand All @@ -63,6 +67,7 @@ func newServer(
instanceSpec *ytv1.InstanceSpec,
binaryPath, configFileName, statefulSetName, serviceName string,
generator ytconfig.YsonGeneratorFunc,
options ...Option,
) server {
proxy := ytsaurus.APIProxy()
commonSpec := ytsaurus.GetCommonSpec()
Expand All @@ -73,6 +78,7 @@ func newServer(
instanceSpec,
binaryPath, configFileName, statefulSetName, serviceName,
generator,
options...,
)
}

Expand All @@ -83,6 +89,7 @@ func newServerConfigured(
instanceSpec *ytv1.InstanceSpec,
binaryPath, configFileName, statefulSetName, serviceName string,
generator ytconfig.YsonGeneratorFunc,
optFuncs ...Option,
) server {
image := commonSpec.CoreImage
if instanceSpec.Image != nil {
Expand All @@ -107,6 +114,22 @@ func newServerConfigured(
consts.BusSecretMountPoint)
}

opts := &options{
containerPorts: []corev1.ContainerPort{
{
Name: consts.YTMonitoringContainerPortName,
ContainerPort: *instanceSpec.MonitoringPort,
Protocol: corev1.ProtocolTCP,
},
},
readinessProbeEndpointPort: intstr.FromString(consts.YTMonitoringContainerPortName),
readinessProbeEndpointPath: readinessProbeHTTPPath,
}

for _, fn := range optFuncs {
fn(opts)
}

return &serverImpl{
labeller: l,
image: image,
Expand Down Expand Up @@ -143,6 +166,11 @@ func newServerConfigured(
Fmt: ytconfig.ConfigFormatYson,
},
}),

componentContainerPorts: opts.containerPorts,

readinessProbePort: opts.readinessProbeEndpointPort,
readinessProbeHTTPPath: opts.readinessProbeEndpointPath,
}
kruftik marked this conversation as resolved.
Show resolved Hide resolved
}

Expand Down Expand Up @@ -272,19 +300,13 @@ func (s *serverImpl) rebuildStatefulSet() *appsv1.StatefulSet {
Name: consts.YTServerContainerName,
Command: command,
VolumeMounts: volumeMounts,
Ports: []corev1.ContainerPort{
{
Name: serverAPIPortName,
ContainerPort: *s.instanceSpec.MonitoringPort,
Protocol: corev1.ProtocolTCP,
},
},
Resources: s.instanceSpec.Resources,
Ports: s.componentContainerPorts,
Resources: s.instanceSpec.Resources,
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Port: intstr.FromString(serverAPIPortName),
Path: readinessProbeHTTPPath,
Port: s.readinessProbePort,
Path: s.readinessProbeHTTPPath,
},
},
InitialDelaySeconds: readinessProbeParams.InitialDelaySeconds,
Expand Down
33 changes: 33 additions & 0 deletions pkg/components/serveroptions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package components

import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)

type options struct {
containerPorts []corev1.ContainerPort

readinessProbeEndpointPort intstr.IntOrString
readinessProbeEndpointPath string
}

type Option func(opts *options)

func WithCustomReadinessProbeEndpointPort(port int32) Option {
return func(opts *options) {
opts.readinessProbeEndpointPort = intstr.FromInt32(port)
}
}

func WithCustomReadinessProbeEndpointPath(path string) Option {
return func(opts *options) {
opts.readinessProbeEndpointPath = path
}
}

func WithContainerPorts(ports ...corev1.ContainerPort) Option {
return func(opts *options) {
opts.containerPorts = append(opts.containerPorts, ports...)
}
}
6 changes: 6 additions & 0 deletions pkg/components/tablet_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"go.ytsaurus.tech/library/go/ptr"
"go.ytsaurus.tech/yt/go/ypath"
"go.ytsaurus.tech/yt/go/yt"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/log"

Expand Down Expand Up @@ -62,6 +63,11 @@ func NewTabletNode(
func() ([]byte, error) {
return cfgen.GetTabletNodeConfig(spec)
},
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.TabletNodeRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

return &TabletNode{
Expand Down
5 changes: 5 additions & 0 deletions pkg/components/yql_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ func NewYQLAgent(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus, master
func() ([]byte, error) {
return cfgen.GetYQLAgentConfig(resource.Spec.YQLAgents)
},
WithContainerPorts(corev1.ContainerPort{
Name: consts.YTRPCPortName,
ContainerPort: consts.YQLAgentRPCPort,
Protocol: corev1.ProtocolTCP,
}),
)

return &YqlAgent{
Expand Down
9 changes: 7 additions & 2 deletions pkg/consts/address.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
package consts

const YTMonitoringPortName = "ytsaurus-metrics"
const YTMonitoringPort = 10000
const (
YTRPCPortName = "rpc"

YTMonitoringContainerPortName = "metrics"
YTMonitoringServicePortName = "ytsaurus-metrics"
YTMonitoringPort = 10000
)

const (
DiscoveryRPCPort = 9020
Expand Down
Loading
Loading