diff --git a/flake.nix b/flake.nix index ba4ddd25..845e10d6 100644 --- a/flake.nix +++ b/flake.nix @@ -158,6 +158,7 @@ container_pkgs.nettools container_pkgs.gnugrep container_pkgs.coreutils + container_pkgs.cacert ]; pathsToLink = ["/bin"]; }; @@ -166,6 +167,7 @@ if !needsCrossCompilation then ["${service}/bin/${service.pname}"] else ["${service}/bin/${os}_${arch}/${service.pname}"]; + config.Env = ["SSL_CERT_FILE=${container_pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"]; }; }; }) { diff --git a/kardinal-cli/cmd/root.go b/kardinal-cli/cmd/root.go index 9c6c3e6a..0633b092 100644 --- a/kardinal-cli/cmd/root.go +++ b/kardinal-cli/cmd/root.go @@ -4,13 +4,14 @@ import ( "context" "encoding/json" "fmt" - "kardinal.cli/tenant" - "log" - "net/http" - "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" + "github.com/kurtosis-tech/stacktrace" "github.com/spf13/cobra" + "kardinal.cli/deployment" + "kardinal.cli/tenant" + "log" + "net/http" api "github.com/kurtosis-tech/kardinal/libs/cli-kontrol-api/api/golang/client" api_types "github.com/kurtosis-tech/kardinal/libs/cli-kontrol-api/api/golang/types" @@ -21,6 +22,17 @@ const ( devMode = true kontrolServiceApiUrl = "ad718d90d54d54dd084dea50a9f011af-1140086995.us-east-1.elb.amazonaws.com" kontrolServicePort = 8080 + + kontrolLocationLocalMinikube = "local-minikube" + kontrolLocationKloudKontrol = "kloud-kontrol" + + kontrolClusterResourcesEndpointTmpl = "%s://%s/tenant/%s/cluster-resources" + + localMinikubeKontrolAPIHost = "host.minikube.internal:8080" + kloudKontrolAPIHost = "app.kardinal.dev/api" + + httpSchme = "http" + httpsScheme = httpSchme + "s" ) var composeFile string @@ -35,6 +47,11 @@ var flowCmd = &cobra.Command{ Short: "Manage deployment flows", } +var managerCmd = &cobra.Command{ + Use: "manager", + Short: "Manage Kardinal manager", +} + var deployCmd = &cobra.Command{ Use: "deploy", Short: "Deploy services", @@ -94,10 +111,47 @@ var deleteCmd = &cobra.Command{ }, } +var deployManagerCmd = &cobra.Command{ + Use: fmt.Sprintf("deploy [kontrol location] accepted values: %s and %s ", kontrolLocationLocalMinikube, kontrolLocationKloudKontrol), + Short: "Deploy Kardinal manager into the cluster", + ValidArgs: []string{kontrolLocationLocalMinikube, kontrolLocationKloudKontrol}, + Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), + Run: func(cmd *cobra.Command, args []string) { + + kontroLocation := args[0] + + tenantUuid, err := tenant.GetOrCreateUserTenantUUID() + if err != nil { + log.Fatal("Error getting or creating user tenant UUID", err) + } + + if err := deployManager(tenantUuid.String(), kontroLocation); err != nil { + log.Fatal("Error deploying Kardinal manager", err) + } + + fmt.Printf("Kardinal manager deployed using '%s' Kontrol", kontroLocation) + }, +} + +var removeManagerCmd = &cobra.Command{ + Use: "remove", + Short: "Remove Kardinal manager from the cluster", + Args: cobra.ExactArgs(0), + Run: func(cmd *cobra.Command, args []string) { + if err := removeManager(); err != nil { + log.Fatal("Error removing Kardinal manager", err) + } + + fmt.Print("Kardinal manager removed from cluster") + }, +} + func init() { rootCmd.AddCommand(flowCmd) + rootCmd.AddCommand(managerCmd) rootCmd.AddCommand(deployCmd) flowCmd.AddCommand(createCmd, deleteCmd) + managerCmd.AddCommand(deployManagerCmd, removeManagerCmd) flowCmd.PersistentFlags().StringVarP(&composeFile, "docker-compose", "d", "", "Path to the Docker Compose file") flowCmd.MarkPersistentFlagRequired("docker-compose") @@ -206,6 +260,43 @@ func deleteFlow(tenantUuid api_types.Uuid, services []types.ServiceConfig) { fmt.Printf("Response: %s\n", string(resp.Body)) } +func deployManager(tenantUuid api_types.Uuid, kontrolLocation string) error { + var ( + ctx = context.Background() + scheme string + host string + ) + + switch kontrolLocation { + case kontrolLocationLocalMinikube: + scheme = httpSchme + host = localMinikubeKontrolAPIHost + case kontrolLocationKloudKontrol: + scheme = httpsScheme + host = kloudKontrolAPIHost + default: + return stacktrace.NewError("invalid kontrol location: %s", kontrolLocation) + } + + clusterResourcesURL := fmt.Sprintf(kontrolClusterResourcesEndpointTmpl, scheme, host, tenantUuid) + + if err := deployment.DeployKardinalManagerInCluster(ctx, clusterResourcesURL); err != nil { + return stacktrace.Propagate(err, "An error occurred deploying Kardinal manager into the cluster with cluster resources URL '%s'", clusterResourcesURL) + } + + return nil +} + +func removeManager() error { + ctx := context.Background() + + if err := deployment.RemoveKardinalManagerFromCluster(ctx); err != nil { + return stacktrace.Propagate(err, "An error occurred removing Kardinal manager from the cluster") + } + + return nil +} + func getKontrolServiceClient() *api.ClientWithResponses { if devMode { client, err := api.NewClientWithResponses("http://localhost:8080", api.WithHTTPClient(http.DefaultClient)) diff --git a/kardinal-cli/consts/consts.go b/kardinal-cli/consts/consts.go new file mode 100644 index 00000000..a99d242e --- /dev/null +++ b/kardinal-cli/consts/consts.go @@ -0,0 +1,6 @@ +package consts + +const ( + KardinalAppIDLabelKey = "dev.kardinal.app-id" + KardinalManagerAppIDLabelValue = "kardinal-manager" +) diff --git a/kardinal-cli/deployment/deployment.go b/kardinal-cli/deployment/deployment.go new file mode 100644 index 00000000..d8b52ddc --- /dev/null +++ b/kardinal-cli/deployment/deployment.go @@ -0,0 +1,141 @@ +package deployment + +import ( + "bytes" + "context" + "github.com/kurtosis-tech/stacktrace" + "kardinal.cli/consts" + "text/template" +) + +const ( + kardinalNamespace = "default" + kardinalManagerDeploymentTmplName = "kardinal-manager-deployment" + + kardinalManagerDeploymentTmpl = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kardinal-manager + namespace: {{.Namespace}} + labels: + {{.KardinalAppIDLabelKey}}: {{.KardinalManagerAppIDLabelValue}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kardinal-manager-role + labels: + {{.KardinalAppIDLabelKey}}: {{.KardinalManagerAppIDLabelValue}} +rules: + - apiGroups: ["*"] + resources: ["namespaces", "pods", "services", "deployments", "virtualservices", "workloadgroups", "workloadentries", "sidecars", "serviceentries", "gateways", "envoyfilters", "destinationrules"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kardinal-manager-binding + labels: + {{.KardinalAppIDLabelKey}}: {{.KardinalManagerAppIDLabelValue}} +subjects: + - kind: ServiceAccount + name: kardinal-manager + namespace: default +roleRef: + kind: ClusterRole + name: kardinal-manager-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kardinal-manager + namespace: {{.Namespace}} + labels: + {{.KardinalAppIDLabelKey}}: {{.KardinalManagerAppIDLabelValue}} +spec: + replicas: 1 + selector: + matchLabels: + {{.KardinalAppIDLabelKey}}: {{.KardinalManagerAppIDLabelValue}} + template: + metadata: + labels: + {{.KardinalAppIDLabelKey}}: {{.KardinalManagerAppIDLabelValue}} + spec: + serviceAccountName: kardinal-manager + containers: + - name: kardinal-manager + image: kurtosistech/kardinal-manager:latest + # TODO: Policy to local dev only - figure a way to remove it + imagePullPolicy: Never + env: + - name: KUBERNETES_SERVICE_HOST + value: "kubernetes.default.svc" + - name: KUBERNETES_SERVICE_PORT + value: "443" + - name: KARDINAL_MANAGER_CLUSTER_CONFIG_ENDPOINT + value: "{{.ClusterResourcesURL}}" + - name: KARDINAL_MANAGER_FETCHER_JOB_DURATION_SECONDS + value: "10" +` +) + +type templateData struct { + Namespace string + ClusterResourcesURL string + KardinalAppIDLabelKey string + KardinalManagerAppIDLabelValue string +} + +func DeployKardinalManagerInCluster(ctx context.Context, clusterResourcesURL string) error { + kubernetesClientObj, err := createKubernetesClient() + if err != nil { + return stacktrace.Propagate(err, "An error occurred while creating the Kubernetes client") + } + + kardinalManagerDeploymentTemplate, err := template.New(kardinalManagerDeploymentTmplName).Parse(kardinalManagerDeploymentTmpl) + if err != nil { + return stacktrace.Propagate(err, "An error occurred while parsing the kardinal-manager deployment template") + } + + templateDataObj := templateData{ + Namespace: kardinalNamespace, + ClusterResourcesURL: clusterResourcesURL, + KardinalAppIDLabelKey: consts.KardinalAppIDLabelKey, + KardinalManagerAppIDLabelValue: consts.KardinalManagerAppIDLabelValue, + } + + yamlFileContentsBuffer := &bytes.Buffer{} + + if err = kardinalManagerDeploymentTemplate.Execute(yamlFileContentsBuffer, templateDataObj); err != nil { + return stacktrace.Propagate(err, "An error occurred while executing the template '%s' with data objects '%+v'", kardinalManagerDeploymentTmplName, templateDataObj) + } + + if err = kubernetesClientObj.ApplyYamlFileContentInNamespace(ctx, kardinalNamespace, yamlFileContentsBuffer.Bytes()); err != nil { + return stacktrace.Propagate(err, "An error occurred while applying the kardinal-manager deployment") + } + + return nil +} + +func RemoveKardinalManagerFromCluster(ctx context.Context) error { + kubernetesClientObj, err := createKubernetesClient() + if err != nil { + return stacktrace.Propagate(err, "An error occurred while creating the Kubernetes client") + } + + labels := map[string]string{ + consts.KardinalAppIDLabelKey: consts.KardinalManagerAppIDLabelValue, + } + + if err = kubernetesClientObj.RemoveNamespaceResourcesByLabels(ctx, kardinalNamespace, labels); err != nil { + return stacktrace.Propagate(err, "An error occurred while removing the kardinal-manager from the cluster using labels '%+v'", labels) + } + + return nil +} diff --git a/kardinal-cli/deployment/kubernetes_client.go b/kardinal-cli/deployment/kubernetes_client.go new file mode 100644 index 00000000..a4022a09 --- /dev/null +++ b/kardinal-cli/deployment/kubernetes_client.go @@ -0,0 +1,150 @@ +package deployment + +import ( + "bytes" + "context" + "errors" + "github.com/kurtosis-tech/stacktrace" + "gopkg.in/yaml.v3" + "io" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" +) + +const ( + fieldManager = "kardinal-cli" + + listOptionsTimeoutSeconds int64 = 10 + deleteOptionsGracePeriodSeconds int64 = 0 +) + +type kubernetesClient struct { + config *rest.Config + clientSet *kubernetes.Clientset + dynamicClient *dynamic.DynamicClient + discoveryMapper *restmapper.DeferredDiscoveryRESTMapper +} + +func newKubernetesClient(config *rest.Config, clientSet *kubernetes.Clientset, dynamicClient *dynamic.DynamicClient, discoveryMapper *restmapper.DeferredDiscoveryRESTMapper) *kubernetesClient { + return &kubernetesClient{config: config, clientSet: clientSet, dynamicClient: dynamicClient, discoveryMapper: discoveryMapper} +} + +func (client *kubernetesClient) ApplyYamlFileContentInNamespace(ctx context.Context, namespace string, yamlFileContent []byte) error { + yamlReader := bytes.NewReader(yamlFileContent) + + dec := yaml.NewDecoder(yamlReader) + + for { + unstructuredObject := &unstructured.Unstructured{Object: map[string]interface{}{}} + err := dec.Decode(unstructuredObject.Object) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return stacktrace.Propagate(err, "An error occurred decoding the unstructured object") + } + if unstructuredObject.Object == nil { + return stacktrace.NewError("Expected to find the object value after decoding the unstructured object but it was not found") + } + + groupVersionKind := unstructuredObject.GroupVersionKind() + restMapping, err := client.discoveryMapper.RESTMapping(groupVersionKind.GroupKind(), groupVersionKind.Version) + if err != nil { + return stacktrace.Propagate(err, "An error occurred getting the rest mapping for GVK") + } + + groupVersionResource := restMapping.Resource + + if unstructuredObject.GetNamespace() != "" && namespace != unstructuredObject.GetNamespace() { + return stacktrace.NewError( + "The namespace '%s' in resource '%s' kind '%s' is different from the main namespace '%s'", + unstructuredObject.GetNamespace(), + unstructuredObject.GetName(), + unstructuredObject.GetKind(), + namespace, + ) + } + + applyOpts := metav1.ApplyOptions{FieldManager: fieldManager} + + var resource dynamic.ResourceInterface + + resource = client.dynamicClient.Resource(groupVersionResource) + if unstructuredObject.GetNamespace() != "" { + resource = client.dynamicClient.Resource(groupVersionResource).Namespace(namespace) + } + + _, err = resource.Apply(ctx, unstructuredObject.GetName(), unstructuredObject, applyOpts) + if err != nil { + return stacktrace.Propagate(err, "An error occurred applying the k8s resource with name '%s' in namespace '%s'", unstructuredObject.GetName(), unstructuredObject.GetNamespace()) + } + + } +} + +func (client *kubernetesClient) RemoveNamespaceResourcesByLabels(ctx context.Context, namespace string, labels map[string]string) error { + + opts := buildListOptionsFromLabels(labels) + + deleteOptions := metav1.NewDeleteOptions(deleteOptionsGracePeriodSeconds) + + // Delete deployments + if err := client.clientSet.AppsV1().Deployments(namespace).DeleteCollection(ctx, *deleteOptions, opts); err != nil { + return stacktrace.Propagate(err, "An error occurred removing deployments in namespace '%s'", namespace) + } + + // Delete services one by one because there is not DeleteCollection function for services + servicesToRemove, err := client.clientSet.CoreV1().Services(namespace).List(ctx, opts) + if err != nil { + return stacktrace.Propagate(err, "An error occurred listing services") + } + + for _, service := range servicesToRemove.Items { + if err := client.clientSet.CoreV1().Services(namespace).Delete(ctx, service.GetName(), *deleteOptions); err != nil { + return stacktrace.Propagate(err, "An error occurred removing service '%s' from namespace '%s'", service.GetName(), namespace) + } + } + + // Delete cluster role bindings + if err := client.clientSet.RbacV1().ClusterRoleBindings().DeleteCollection(ctx, *deleteOptions, opts); err != nil { + return stacktrace.Propagate(err, "An error occurred removing cluster role bindings") + } + + // Delete cluster roles + if err := client.clientSet.RbacV1().ClusterRoles().DeleteCollection(ctx, *deleteOptions, opts); err != nil { + return stacktrace.Propagate(err, "An error occurred removing cluster roles") + } + + // Delete service accounts + if err := client.clientSet.CoreV1().ServiceAccounts(namespace).DeleteCollection(ctx, *deleteOptions, opts); err != nil { + return stacktrace.Propagate(err, "An error occurred removing service accounts from namespace '%s'", namespace) + } + + return nil +} + +func buildListOptionsFromLabels(labelsMap map[string]string) metav1.ListOptions { + return metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + LabelSelector: labels.SelectorFromSet(labelsMap).String(), + FieldSelector: "", + Watch: false, + AllowWatchBookmarks: false, + ResourceVersion: "", + ResourceVersionMatch: "", + TimeoutSeconds: int64Ptr(listOptionsTimeoutSeconds), + Limit: 0, + Continue: "", + SendInitialEvents: nil, + } +} + +func int64Ptr(i int64) *int64 { return &i } diff --git a/kardinal-cli/deployment/kubernetes_client_factory.go b/kardinal-cli/deployment/kubernetes_client_factory.go new file mode 100644 index 00000000..1e099432 --- /dev/null +++ b/kardinal-cli/deployment/kubernetes_client_factory.go @@ -0,0 +1,46 @@ +package deployment + +import ( + "github.com/kurtosis-tech/stacktrace" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + "path/filepath" +) + +func createKubernetesClient() (*kubernetesClient, error) { + var config *rest.Config + + // Load in-cluster configuration + config, err := rest.InClusterConfig() + if err != nil { + // Fallback to out-of-cluster configuration (for local development) + home := homedir.HomeDir() + kubeConfig := filepath.Join(home, ".kube", "config") + config, err = clientcmd.BuildConfigFromFlags("", kubeConfig) + if err != nil { + return nil, stacktrace.Propagate(err, "impossible to get kubernetes client config either inside or outside the cluster") + } + } + + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while creating kubernetes client using config '%+v'", config) + } + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while creating kubernetes dynamic client using config '%+v'", config) + } + + discoveryClient := memory.NewMemCacheClient(clientSet.Discovery()) + discoveryMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) + + kubernetesClientObj := newKubernetesClient(config, clientSet, dynamicClient, discoveryMapper) + + return kubernetesClientObj, nil +} diff --git a/kardinal-cli/kardinal b/kardinal-cli/kardinal new file mode 100755 index 00000000..6f85af96 Binary files /dev/null and b/kardinal-cli/kardinal differ diff --git a/kardinal-cli/main.go b/kardinal-cli/main.go index c7978acd..3cc28c10 100644 --- a/kardinal-cli/main.go +++ b/kardinal-cli/main.go @@ -1,10 +1,9 @@ package main -import ( - "kardinal.cli/cmd" -) +import "kardinal.cli/cmd" func main() { + if err := cmd.Execute(); err != nil { println("Error:", err.Error()) } diff --git a/kardinal-manager/kardinal-manager/fetcher/fetcher.go b/kardinal-manager/kardinal-manager/fetcher/fetcher.go index 3c9aa0a9..e9fcb1e3 100644 --- a/kardinal-manager/kardinal-manager/fetcher/fetcher.go +++ b/kardinal-manager/kardinal-manager/fetcher/fetcher.go @@ -84,6 +84,7 @@ func (fetcher *fetcher) getClusterResourcesFromCloud() (*types.ClusterResources, return nil, stacktrace.Propagate(err, "Error fetching cluster resources from endpoint '%s'", fetcher.configEndpoint) } defer resp.Body.Close() + logrus.Debugf("Fetching cluster resources from endpoint '%s'", fetcher.configEndpoint) responseBodyBytes, err := io.ReadAll(resp.Body) if err != nil {