Skip to content

Commit

Permalink
k8s status should return immediately if the api server is down (#154)
Browse files Browse the repository at this point in the history
* k8s status: fail fast if we cannot determine kube-apiserver endpoints for our cluster

* k8s status: tweak fail fast errors

We want status to fail fast if:
- not bootstrapped
- kube-apiserver is unreachable
- no cluster endpoints are known
  • Loading branch information
kwmonroe authored Feb 26, 2024
1 parent 26906ea commit 726c261
Show file tree
Hide file tree
Showing 5 changed files with 35 additions and 2 deletions.
3 changes: 3 additions & 0 deletions src/k8s/api/v1/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,8 @@ var ErrTimeout = errors.New("context deadline exceeded")
// ErrConnectionFailed indicates that a connection to the k8sd daemon could not be established.
var ErrConnectionFailed = errors.New("dial unix")

// ErrAPIServerFailed indicates that kube-apiserver endpoint(s) could not be determined.
var ErrAPIServerFailed = errors.New("failed to get kube-apiserver endpoints")

// ErrUnknown indicates that the server returns an unknown error.
var ErrUnknown = errors.New("unknown error")
6 changes: 4 additions & 2 deletions src/k8s/cmd/k8s/errors/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ import (
)

var genericErrorMsgs = map[error]string{
v1.ErrNotBootstrapped: fmt.Sprintln("The cluster has not been initialized yet. Please call:\n\n sudo k8s bootstrap"),
v1.ErrConnectionFailed: "Unable to connect to the cluster. Verify that that the services are running:\n\n sudo snap services k8s\n\n" +
v1.ErrNotBootstrapped: "The cluster has not been initialized yet. Please call:\n\n sudo k8s bootstrap",
v1.ErrConnectionFailed: "Unable to connect to the cluster. Verify k8s services are running:\n\n sudo snap services k8s\n\n" +
"and see logs for more details:\n\n sudo journalctl -n 300 -u snap.k8s.k8sd\n\n",
v1.ErrAPIServerFailed: "Unable to get Kubernetes API server endpoints. Verify k8s services are running:\n\n sudo snap services k8s\n\n" +
"and see logs for more details:\n\n sudo journalctl -n 300 -u snap.k8s.kube-apiserver\n\n",
v1.ErrTimeout: "Command timed out. See logs for more details:\n\n" +
" sudo journalctl -n 300 -u snap.k8s.k8sd\n\n" +
"You may increase the timeout with `--timeout 3m`.",
Expand Down
12 changes: 12 additions & 0 deletions src/k8s/cmd/k8s/k8s_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"time"

v1 "github.com/canonical/k8s/api/v1"
"github.com/canonical/k8s/cmd/k8s/errors"
"github.com/canonical/k8s/cmd/k8s/formatter"
"github.com/spf13/cobra"
Expand All @@ -27,6 +28,17 @@ func newStatusCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) (err error) {
defer errors.Transform(&err, nil)

// fail fast if we're not bootstrapped
if !k8sdClient.IsBootstrapped(cmd.Context()) {
return v1.ErrNotBootstrapped
}
// fail fast if we're not explicitly waiting and we can't get kube-apiserver endpoints
if !statusCmdOpts.waitReady {
if ready := k8sdClient.IsKubernetesAPIServerReady(cmd.Context()); !ready {
return fmt.Errorf("failed to get kube-apiserver endpoints; cluster status is unavailable")
}
}

const minTimeout = 3 * time.Second
if statusCmdOpts.timeout < minTimeout {
cmd.PrintErrf("Timeout %v is less than minimum of %v. Using the minimum %v instead.\n", statusCmdOpts.timeout, minTimeout, minTimeout)
Expand Down
14 changes: 14 additions & 0 deletions src/k8s/pkg/k8s/client/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
apiv1 "github.com/canonical/k8s/api/v1"
"github.com/canonical/k8s/pkg/config"
"github.com/canonical/k8s/pkg/utils/control"
"github.com/canonical/k8s/pkg/utils/k8s"
"github.com/canonical/lxd/lxd/util"
"github.com/canonical/lxd/shared/api"
)
Expand Down Expand Up @@ -84,3 +85,16 @@ func (c *k8sdClient) KubeConfig(ctx context.Context) (string, error) {
}
return response.KubeConfig, nil
}

// IsKubernetesAPIServerReady checks if kube-apiserver is reachable.
func (c *k8sdClient) IsKubernetesAPIServerReady(ctx context.Context) bool {
kc, err := k8s.NewClient(c.opts.Snap)
if err != nil {
return false
}
_, err = kc.GetKubeAPIServerEndpoints(ctx)
if err != nil {
return false
}
return err == nil
}
2 changes: 2 additions & 0 deletions src/k8s/pkg/k8s/client/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
type Client interface {
// Bootstrap initializes a new cluster member using the provided bootstrap configuration.
Bootstrap(ctx context.Context, bootstrapConfig apiv1.BootstrapConfig) (apiv1.ClusterMember, error)
// IsKubernetesAPIServerReady checks if kube-apiserver is reachable.
IsKubernetesAPIServerReady(ctx context.Context) bool
// IsBootstrapped checks whether the current node is already bootstrapped.
IsBootstrapped(ctx context.Context) bool
// CleanupNode performs cleanup operations for a specific node in the cluster.
Expand Down

0 comments on commit 726c261

Please sign in to comment.