diff --git a/cmd/talosctl/cmd/mgmt/cluster/cluster.go b/cmd/talosctl/cmd/mgmt/cluster/cluster.go index 4e81ed6666..0348bf16b2 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/cluster.go +++ b/cmd/talosctl/cmd/mgmt/cluster/cluster.go @@ -6,12 +6,17 @@ package cluster import ( - "errors" "path/filepath" "github.com/spf13/cobra" clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" + "github.com/siderolabs/talos/pkg/provision/providers" +) + +const ( + // ProvisionerFlag is the flag with which the provisioner is configured. + ProvisionerFlag = "provisioner" ) // Cmd represents the cluster command. @@ -19,32 +24,33 @@ var Cmd = &cobra.Command{ Use: "cluster", Short: "A collection of commands for managing local docker-based or QEMU-based clusters", Long: ``, - PersistentPreRunE: func(*cobra.Command, []string) error { - if provisionerName == docker && !bootloaderEnabled { - return errors.New("docker provisioner requires bootloader to be enabled") - } +} - return nil - }, +// CmdOps are the options for the cluster command. +type CmdOps struct { + ProvisionerName string + StateDir string + ClusterName string } var ( - provisionerName string - stateDir string - clusterName string - defaultStateDir string - defaultCNIDir string + + // DefaultCNIDir is the default location of the cni binaries. + DefaultCNIDir string ) +// Flags are the flags of the cluster command. +var Flags CmdOps + func init() { talosDir, err := clientconfig.GetTalosDirectory() if err == nil { defaultStateDir = filepath.Join(talosDir, "clusters") - defaultCNIDir = filepath.Join(talosDir, "cni") + DefaultCNIDir = filepath.Join(talosDir, "cni") } - Cmd.PersistentFlags().StringVar(&provisionerName, "provisioner", docker, "Talos cluster provisioner to use") - Cmd.PersistentFlags().StringVar(&stateDir, "state", defaultStateDir, "directory path to store cluster state") - Cmd.PersistentFlags().StringVar(&clusterName, "name", "talos-default", "the name of the cluster") + Cmd.PersistentFlags().StringVar(&Flags.ProvisionerName, ProvisionerFlag, providers.DockerProviderName, "Talos cluster provisioner to use") + Cmd.PersistentFlags().StringVar(&Flags.StateDir, "state", defaultStateDir, "directory path to store cluster state") + Cmd.PersistentFlags().StringVar(&Flags.ClusterName, "name", "talos-default", "the name of the cluster") } diff --git a/cmd/talosctl/cmd/mgmt/cluster/create.go b/cmd/talosctl/cmd/mgmt/cluster/create.go deleted file mode 100644 index cf715a7b4b..0000000000 --- a/cmd/talosctl/cmd/mgmt/cluster/create.go +++ /dev/null @@ -1,1652 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package cluster - -import ( - "bytes" - "context" - "encoding/base64" - "errors" - "fmt" - "math/big" - "net" - "net/netip" - "net/url" - "os" - "path/filepath" - stdruntime "runtime" - "slices" - "strconv" - "strings" - "time" - - "github.com/docker/cli/opts" - "github.com/dustin/go-humanize" - "github.com/google/uuid" - "github.com/hashicorp/go-getter/v2" - "github.com/klauspost/compress/zstd" - "github.com/siderolabs/crypto/x509" - "github.com/siderolabs/gen/maps" - "github.com/siderolabs/go-blockdevice/v2/encryption" - "github.com/siderolabs/go-kubeconfig" - "github.com/siderolabs/go-pointer" - "github.com/siderolabs/go-procfs/procfs" - sideronet "github.com/siderolabs/net" - "github.com/spf13/cobra" - "k8s.io/client-go/tools/clientcmd" - - "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/internal/firewallpatch" - "github.com/siderolabs/talos/cmd/talosctl/pkg/mgmt/helpers" - "github.com/siderolabs/talos/pkg/cli" - "github.com/siderolabs/talos/pkg/cluster/check" - "github.com/siderolabs/talos/pkg/images" - clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" - "github.com/siderolabs/talos/pkg/machinery/config" - "github.com/siderolabs/talos/pkg/machinery/config/bundle" - "github.com/siderolabs/talos/pkg/machinery/config/configloader" - "github.com/siderolabs/talos/pkg/machinery/config/configpatcher" - "github.com/siderolabs/talos/pkg/machinery/config/container" - "github.com/siderolabs/talos/pkg/machinery/config/encoder" - "github.com/siderolabs/talos/pkg/machinery/config/generate" - "github.com/siderolabs/talos/pkg/machinery/config/machine" - "github.com/siderolabs/talos/pkg/machinery/config/types/security" - "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" - "github.com/siderolabs/talos/pkg/machinery/constants" - "github.com/siderolabs/talos/pkg/machinery/imager/quirks" - "github.com/siderolabs/talos/pkg/machinery/nethelpers" - "github.com/siderolabs/talos/pkg/machinery/version" - "github.com/siderolabs/talos/pkg/provision" - "github.com/siderolabs/talos/pkg/provision/access" - "github.com/siderolabs/talos/pkg/provision/providers" -) - -const ( - docker = "docker" - - // gatewayOffset is the offset from the network address of the IP address of the network gateway. - gatewayOffset = 1 - - // nodesOffset is the offset from the network address of the beginning of the IP addresses to be used for nodes. - nodesOffset = 2 - - // vipOffset is the offset from the network address of the CIDR to use for allocating the Virtual (shared) IP address, if enabled. - vipOffset = 50 - - inputDirFlag = "input-dir" - networkIPv4Flag = "ipv4" - networkIPv6Flag = "ipv6" - networkMTUFlag = "mtu" - networkCIDRFlag = "cidr" - networkNoMasqueradeCIDRsFlag = "no-masquerade-cidrs" - nameserversFlag = "nameservers" - clusterDiskPreallocateFlag = "disk-preallocate" - clusterDisksFlag = "user-disk" - clusterDiskSizeFlag = "disk" - useVIPFlag = "use-vip" - bootloaderEnabledFlag = "with-bootloader" - controlPlanePortFlag = "control-plane-port" - firewallFlag = "with-firewall" - tpm2EnabledFlag = "with-tpm2" - withDebugShellFlag = "with-debug-shell" - withIOMMUFlag = "with-iommu" - - // The following flags are the gen options - the options that are only used in machine configuration (i.e., not during the qemu/docker provisioning). - // They are not applicable when no machine configuration is generated, hence mutually exclusive with the --input-dir flag. - - nodeInstallImageFlag = "install-image" - configDebugFlag = "with-debug" - dnsDomainFlag = "dns-domain" - withClusterDiscoveryFlag = "with-cluster-discovery" - registryMirrorFlag = "registry-mirror" - registryInsecureFlag = "registry-insecure-skip-verify" - customCNIUrlFlag = "custom-cni-url" - talosVersionFlag = "talos-version" - encryptStatePartitionFlag = "encrypt-state" - encryptEphemeralPartitionFlag = "encrypt-ephemeral" - enableKubeSpanFlag = "with-kubespan" - forceEndpointFlag = "endpoint" - kubePrismFlag = "kubeprism-port" - diskEncryptionKeyTypesFlag = "disk-encryption-key-types" -) - -var ( - talosconfig string - nodeImage string - nodeInstallImage string - registryMirrors []string - registryInsecure []string - kubernetesVersion string - nodeVmlinuzPath string - nodeInitramfsPath string - nodeISOPath string - nodeUSBPath string - nodeUKIPath string - nodeDiskImagePath string - nodeIPXEBootScript string - applyConfigEnabled bool - bootloaderEnabled bool - uefiEnabled bool - tpm2Enabled bool - extraUEFISearchPaths []string - configDebug bool - networkCIDR string - networkNoMasqueradeCIDRs []string - networkMTU int - networkIPv4 bool - networkIPv6 bool - wireguardCIDR string - nameservers []string - dnsDomain string - workers int - controlplanes int - controlPlaneCpus string - workersCpus string - controlPlaneMemory int - workersMemory int - clusterDiskSize int - clusterDiskPreallocate bool - diskBlockSize uint - clusterDisks []string - extraDisks int - extraDiskSize int - extraDisksDrivers []string - targetArch string - clusterWait bool - clusterWaitTimeout time.Duration - forceInitNodeAsEndpoint bool - forceEndpoint string - inputDir string - cniBinPath []string - cniConfDir string - cniCacheDir string - cniBundleURL string - ports string - dockerHostIP string - withInitNode bool - customCNIUrl string - skipKubeconfig bool - skipInjectingConfig bool - talosVersion string - encryptStatePartition bool - encryptEphemeralPartition bool - useVIP bool - enableKubeSpan bool - enableClusterDiscovery bool - configPatch []string - configPatchControlPlane []string - configPatchWorker []string - badRTC bool - extraBootKernelArgs string - dockerDisableIPv6 bool - controlPlanePort int - kubePrismPort int - dhcpSkipHostname bool - skipK8sNodeReadinessCheck bool - networkChaos bool - jitter time.Duration - latency time.Duration - packetLoss float64 - packetReorder float64 - packetCorrupt float64 - bandwidth int - diskEncryptionKeyTypes []string - withFirewall string - withUUIDHostnames bool - withSiderolinkAgent agentFlag - withJSONLogs bool - debugShellEnabled bool - withIOMMU bool - configInjectionMethodFlag string - mountOpts opts.MountOpt -) - -// createCmd represents the cluster up command. -var createCmd = &cobra.Command{ - Use: "create", - Short: "Creates a local docker-based or QEMU-based kubernetes cluster", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return cli.WithContext(context.Background(), create) - }, -} - -//nolint:gocyclo -func downloadBootAssets(ctx context.Context) error { - // download & cache images if provides as URLs - for _, downloadableImage := range []struct { - path *string - disableArchive bool - }{ - { - path: &nodeVmlinuzPath, - }, - { - path: &nodeInitramfsPath, - disableArchive: true, - }, - { - path: &nodeISOPath, - }, - { - path: &nodeUSBPath, - }, - { - path: &nodeUKIPath, - }, - { - path: &nodeDiskImagePath, - }, - } { - if *downloadableImage.path == "" { - continue - } - - u, err := url.Parse(*downloadableImage.path) - if err != nil || !(u.Scheme == "http" || u.Scheme == "https") { - // not a URL - continue - } - - defaultStateDir, err := clientconfig.GetTalosDirectory() - if err != nil { - return err - } - - cacheDir := filepath.Join(defaultStateDir, "cache") - - if os.MkdirAll(cacheDir, 0o755) != nil { - return err - } - - destPath := strings.ReplaceAll( - strings.ReplaceAll(u.String(), "/", "-"), - ":", "-") - - _, err = os.Stat(filepath.Join(cacheDir, destPath)) - if err == nil { - *downloadableImage.path = filepath.Join(cacheDir, destPath) - - // already cached - continue - } - - fmt.Fprintf(os.Stderr, "downloading asset from %q to %q\n", u.String(), filepath.Join(cacheDir, destPath)) - - client := getter.Client{ - Getters: []getter.Getter{ - &getter.HttpGetter{ - HeadFirstTimeout: 30 * time.Minute, - ReadTimeout: 30 * time.Minute, - }, - }, - } - - if downloadableImage.disableArchive { - q := u.Query() - - q.Set("archive", "false") - - u.RawQuery = q.Encode() - } - - _, err = client.Get(ctx, &getter.Request{ - Src: u.String(), - Dst: filepath.Join(cacheDir, destPath), - GetMode: getter.ModeFile, - }) - if err != nil { - // clean up the destination on failure - os.Remove(filepath.Join(cacheDir, destPath)) //nolint:errcheck - - return err - } - - *downloadableImage.path = filepath.Join(cacheDir, destPath) - } - - return nil -} - -//nolint:gocyclo,cyclop -func create(ctx context.Context) error { - if err := downloadBootAssets(ctx); err != nil { - return err - } - - if controlplanes < 1 { - return errors.New("number of controlplanes can't be less than 1") - } - - controlPlaneNanoCPUs, err := parseCPUShare(controlPlaneCpus) - if err != nil { - return fmt.Errorf("error parsing --cpus: %s", err) - } - - workerNanoCPUs, err := parseCPUShare(workersCpus) - if err != nil { - return fmt.Errorf("error parsing --cpus-workers: %s", err) - } - - controlPlaneMemory := int64(controlPlaneMemory) * 1024 * 1024 - workerMemory := int64(workersMemory) * 1024 * 1024 - - // Validate CIDR range and allocate IPs - fmt.Fprintln(os.Stderr, "validating CIDR and reserving IPs") - - cidr4, err := netip.ParsePrefix(networkCIDR) - if err != nil { - return fmt.Errorf("error validating cidr block: %w", err) - } - - if !cidr4.Addr().Is4() { - return errors.New("--cidr is expected to be IPV4 CIDR") - } - - // use ULA IPv6 network fd00::/8, add 'TAL' in hex to build /32 network, add IPv4 CIDR to build /64 unique network - cidr6, err := netip.ParsePrefix( - fmt.Sprintf( - "fd74:616c:%02x%02x:%02x%02x::/64", - cidr4.Addr().As4()[0], cidr4.Addr().As4()[1], cidr4.Addr().As4()[2], cidr4.Addr().As4()[3], - ), - ) - if err != nil { - return fmt.Errorf("error validating cidr IPv6 block: %w", err) - } - - var cidrs []netip.Prefix - - if networkIPv4 { - cidrs = append(cidrs, cidr4) - } - - if networkIPv6 { - cidrs = append(cidrs, cidr6) - } - - if len(cidrs) == 0 { - return errors.New("neither IPv4 nor IPv6 network was enabled") - } - - // Gateway addr at 1st IP in range, ex. 192.168.0.1 - gatewayIPs := make([]netip.Addr, len(cidrs)) - - for j := range gatewayIPs { - gatewayIPs[j], err = sideronet.NthIPInNetwork(cidrs[j], gatewayOffset) - if err != nil { - return err - } - } - - // Set starting ip at 2nd ip in range, ex: 192.168.0.2 - ips := make([][]netip.Addr, len(cidrs)) - - for j := range cidrs { - ips[j] = make([]netip.Addr, controlplanes+workers) - - for i := range ips[j] { - ips[j][i], err = sideronet.NthIPInNetwork(cidrs[j], nodesOffset+i) - if err != nil { - return err - } - } - } - - noMasqueradeCIDRs := make([]netip.Prefix, 0, len(networkNoMasqueradeCIDRs)) - - for _, cidr := range networkNoMasqueradeCIDRs { - var parsedCIDR netip.Prefix - - parsedCIDR, err = netip.ParsePrefix(cidr) - if err != nil { - return fmt.Errorf("error parsing non-masquerade CIDR %q: %w", cidr, err) - } - - noMasqueradeCIDRs = append(noMasqueradeCIDRs, parsedCIDR) - } - - // Parse nameservers - nameserverIPs := make([]netip.Addr, len(nameservers)) - - for i := range nameserverIPs { - nameserverIPs[i], err = netip.ParseAddr(nameservers[i]) - if err != nil { - return fmt.Errorf("failed parsing nameserver IP %q: %w", nameservers[i], err) - } - } - - // Virtual (shared) IP at the vipOffset IP in range, ex. 192.168.0.50 - var vip netip.Addr - - if useVIP { - vip, err = sideronet.NthIPInNetwork(cidrs[0], vipOffset) - if err != nil { - return err - } - } - - // Validate network chaos flags - if !networkChaos { - if jitter != 0 || latency != 0 || packetLoss != 0 || packetReorder != 0 || packetCorrupt != 0 || bandwidth != 0 { - return errors.New("network chaos flags can only be used with --with-network-chaos") - } - } - - provisioner, err := providers.Factory(ctx, provisionerName) - if err != nil { - return err - } - - defer provisioner.Close() //nolint:errcheck - - // Craft cluster and node requests - request := provision.ClusterRequest{ - Name: clusterName, - - Network: provision.NetworkRequest{ - Name: clusterName, - CIDRs: cidrs, - NoMasqueradeCIDRs: noMasqueradeCIDRs, - GatewayAddrs: gatewayIPs, - MTU: networkMTU, - Nameservers: nameserverIPs, - LoadBalancerPorts: []int{controlPlanePort}, - CNI: provision.CNIConfig{ - BinPath: cniBinPath, - ConfDir: cniConfDir, - CacheDir: cniCacheDir, - - BundleURL: cniBundleURL, - }, - DHCPSkipHostname: dhcpSkipHostname, - DockerDisableIPv6: dockerDisableIPv6, - NetworkChaos: networkChaos, - Jitter: jitter, - Latency: latency, - PacketLoss: packetLoss, - PacketReorder: packetReorder, - PacketCorrupt: packetCorrupt, - Bandwidth: bandwidth, - }, - - Image: nodeImage, - KernelPath: nodeVmlinuzPath, - InitramfsPath: nodeInitramfsPath, - ISOPath: nodeISOPath, - USBPath: nodeUSBPath, - UKIPath: nodeUKIPath, - IPXEBootScript: nodeIPXEBootScript, - DiskImagePath: nodeDiskImagePath, - - SelfExecutable: os.Args[0], - StateDirectory: stateDir, - } - - provisionOptions := []provision.Option{ - provision.WithDockerPortsHostIP(dockerHostIP), - provision.WithBootlader(bootloaderEnabled), - provision.WithUEFI(uefiEnabled), - provision.WithTPM2(tpm2Enabled), - provision.WithDebugShell(debugShellEnabled), - provision.WithIOMMU(withIOMMU), - provision.WithExtraUEFISearchPaths(extraUEFISearchPaths), - provision.WithTargetArch(targetArch), - provision.WithSiderolinkAgent(withSiderolinkAgent.IsEnabled()), - } - - var configBundleOpts []bundle.Option - - if debugShellEnabled { - if provisionerName != "qemu" { - return errors.New("debug shell only supported with qemu provisioner") - } - } - - if ports != "" { - if provisionerName != docker { - return errors.New("exposed-ports flag only supported with docker provisioner") - } - - portList := strings.Split(ports, ",") - provisionOptions = append(provisionOptions, provision.WithDockerPorts(portList)) - } - - disks, err := getDisks() - if err != nil { - return err - } - - if inputDir != "" { - configBundleOpts = append(configBundleOpts, bundle.WithExistingConfigs(inputDir)) - } else { - genOptions := []generate.Option{ - generate.WithInstallImage(nodeInstallImage), - generate.WithDebug(configDebug), - generate.WithDNSDomain(dnsDomain), - generate.WithClusterDiscovery(enableClusterDiscovery), - } - - for _, registryMirror := range registryMirrors { - left, right, ok := strings.Cut(registryMirror, "=") - if !ok { - return fmt.Errorf("invalid registry mirror spec: %q", registryMirror) - } - - genOptions = append(genOptions, generate.WithRegistryMirror(left, right)) - } - - for _, registryHost := range registryInsecure { - genOptions = append(genOptions, generate.WithRegistryInsecureSkipVerify(registryHost)) - } - - genOptions = append(genOptions, provisioner.GenOptions(request.Network)...) - - if customCNIUrl != "" { - genOptions = append(genOptions, generate.WithClusterCNIConfig(&v1alpha1.CNIConfig{ - CNIName: constants.CustomCNI, - CNIUrls: []string{customCNIUrl}, - })) - } - - if len(disks) > 1 { - // convert provision disks to machine disks - machineDisks := make([]*v1alpha1.MachineDisk, len(disks)-1) - for i, disk := range disks[1:] { - machineDisks[i] = &v1alpha1.MachineDisk{ - DeviceName: provisioner.UserDiskName(i + 1), - DiskPartitions: disk.Partitions, - } - } - - genOptions = append(genOptions, generate.WithUserDisks(machineDisks)) - } - - if talosVersion == "" { - if provisionerName == docker { - parts := strings.Split(nodeImage, ":") - - talosVersion = parts[len(parts)-1] - } else { - parts := strings.Split(nodeInstallImage, ":") - - talosVersion = parts[len(parts)-1] - } - } - - var versionContract *config.VersionContract - - if talosVersion != "latest" { - versionContract, err = config.ParseContractFromVersion(talosVersion) - if err != nil { - return fmt.Errorf("error parsing Talos version %q: %w", talosVersion, err) - } - - genOptions = append(genOptions, generate.WithVersionContract(versionContract)) - } - - if encryptStatePartition || encryptEphemeralPartition { - diskEncryptionConfig := &v1alpha1.SystemDiskEncryptionConfig{} - - var keys []*v1alpha1.EncryptionKey - - for i, key := range diskEncryptionKeyTypes { - switch key { - case "uuid": - keys = append(keys, &v1alpha1.EncryptionKey{ - KeyNodeID: &v1alpha1.EncryptionKeyNodeID{}, - KeySlot: i, - }) - case "kms": - var ip netip.Addr - - // get bridge IP - ip, err = sideronet.NthIPInNetwork(cidr4, 1) - if err != nil { - return err - } - - const port = 4050 - - keys = append(keys, &v1alpha1.EncryptionKey{ - KeyKMS: &v1alpha1.EncryptionKeyKMS{ - KMSEndpoint: "grpc://" + nethelpers.JoinHostPort(ip.String(), port), - }, - KeySlot: i, - }) - - provisionOptions = append(provisionOptions, provision.WithKMS(nethelpers.JoinHostPort("0.0.0.0", port))) - case "tpm": - keyTPM := &v1alpha1.EncryptionKeyTPM{} - - if versionContract.SecureBootEnrollEnforcementSupported() { - keyTPM.TPMCheckSecurebootStatusOnEnroll = pointer.To(true) - } - - keys = append(keys, &v1alpha1.EncryptionKey{ - KeyTPM: keyTPM, - KeySlot: i, - }) - default: - return fmt.Errorf("unknown key type %q", key) - } - } - - if len(keys) == 0 { - return errors.New("no disk encryption key types enabled") - } - - if encryptStatePartition { - diskEncryptionConfig.StatePartition = &v1alpha1.EncryptionConfig{ - EncryptionProvider: encryption.LUKS2, - EncryptionKeys: keys, - } - } - - if encryptEphemeralPartition { - diskEncryptionConfig.EphemeralPartition = &v1alpha1.EncryptionConfig{ - EncryptionProvider: encryption.LUKS2, - EncryptionKeys: keys, - } - } - - genOptions = append(genOptions, generate.WithSystemDiskEncryption(diskEncryptionConfig)) - } - - if useVIP { - genOptions = append(genOptions, - generate.WithNetworkOptions( - v1alpha1.WithNetworkInterfaceVirtualIP(provisioner.GetFirstInterface(), vip.String()), - ), - ) - } - - if enableKubeSpan { - genOptions = append(genOptions, - generate.WithNetworkOptions( - v1alpha1.WithKubeSpan(), - ), - ) - } - - if !bootloaderEnabled { - // disable kexec, as this would effectively use the bootloader - genOptions = append(genOptions, - generate.WithSysctls(map[string]string{ - "kernel.kexec_load_disabled": "1", - }), - ) - } - - if controlPlanePort != constants.DefaultControlPlanePort { - genOptions = append(genOptions, - generate.WithLocalAPIServerPort(controlPlanePort), - ) - } - - if kubePrismPort != constants.DefaultKubePrismPort { - genOptions = append(genOptions, - generate.WithKubePrismPort(kubePrismPort), - ) - } - - externalKubernetesEndpoint := provisioner.GetExternalKubernetesControlPlaneEndpoint(request.Network, controlPlanePort) - - if useVIP { - externalKubernetesEndpoint = "https://" + nethelpers.JoinHostPort(vip.String(), controlPlanePort) - } - - provisionOptions = append(provisionOptions, provision.WithKubernetesEndpoint(externalKubernetesEndpoint)) - - endpointList := provisioner.GetTalosAPIEndpoints(request.Network) - - switch { - case forceEndpoint != "": - // using non-default endpoints, provision additional cert SANs and fix endpoint list - endpointList = []string{forceEndpoint} - genOptions = append(genOptions, generate.WithAdditionalSubjectAltNames(endpointList)) - case forceInitNodeAsEndpoint: - endpointList = []string{ips[0][0].String()} - case len(endpointList) > 0: - for _, endpointHostPort := range endpointList { - endpointHost, _, err := net.SplitHostPort(endpointHostPort) - if err != nil { - endpointHost = endpointHostPort - } - - genOptions = append(genOptions, generate.WithAdditionalSubjectAltNames([]string{endpointHost})) - } - case endpointList == nil: - // use control plane nodes as endpoints, client-side load-balancing - for i := range controlplanes { - endpointList = append(endpointList, ips[0][i].String()) - } - } - - inClusterEndpoint := provisioner.GetInClusterKubernetesControlPlaneEndpoint(request.Network, controlPlanePort) - - if useVIP { - inClusterEndpoint = "https://" + nethelpers.JoinHostPort(vip.String(), controlPlanePort) - } - - genOptions = append(genOptions, generate.WithEndpointList(endpointList)) - configBundleOpts = append(configBundleOpts, - bundle.WithInputOptions( - &bundle.InputOptions{ - ClusterName: clusterName, - Endpoint: inClusterEndpoint, - KubeVersion: strings.TrimPrefix(kubernetesVersion, "v"), - GenOptions: genOptions, - }), - ) - } - - addConfigPatch := func(configPatches []string, configOpt func([]configpatcher.Patch) bundle.Option) error { - var patches []configpatcher.Patch - - patches, err = configpatcher.LoadPatches(configPatches) - if err != nil { - return fmt.Errorf("error parsing config JSON patch: %w", err) - } - - configBundleOpts = append(configBundleOpts, configOpt(patches)) - - return nil - } - - if err = addConfigPatch(configPatch, bundle.WithPatch); err != nil { - return err - } - - if err = addConfigPatch(configPatchControlPlane, bundle.WithPatchControlPlane); err != nil { - return err - } - - if err = addConfigPatch(configPatchWorker, bundle.WithPatchWorker); err != nil { - return err - } - - if withFirewall != "" { - var defaultAction nethelpers.DefaultAction - - defaultAction, err = nethelpers.DefaultActionString(withFirewall) - if err != nil { - return err - } - - var controlplaneIPs []netip.Addr - - for i := range ips { - controlplaneIPs = append(controlplaneIPs, ips[i][:controlplanes]...) - } - - configBundleOpts = append(configBundleOpts, - bundle.WithPatchControlPlane([]configpatcher.Patch{firewallpatch.ControlPlane(defaultAction, cidrs, gatewayIPs, controlplaneIPs)}), - bundle.WithPatchWorker([]configpatcher.Patch{firewallpatch.Worker(defaultAction, cidrs, gatewayIPs)}), - ) - } - - var slb *siderolinkBuilder - - if withSiderolinkAgent.IsEnabled() { - slb, err = newSiderolinkBuilder(gatewayIPs[0].String(), withSiderolinkAgent.IsTLS()) - if err != nil { - return err - } - } - - if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil { - trustedRootsPatch, err := configloader.NewFromBytes(trustedRootsConfig) - if err != nil { - return fmt.Errorf("error loading trusted roots config: %w", err) - } - - configBundleOpts = append(configBundleOpts, bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(trustedRootsPatch)})) - } - - if withJSONLogs { - const port = 4003 - - provisionOptions = append(provisionOptions, provision.WithJSONLogs(nethelpers.JoinHostPort(gatewayIPs[0].String(), port))) - - cfg := container.NewV1Alpha1( - &v1alpha1.Config{ - ConfigVersion: "v1alpha1", - MachineConfig: &v1alpha1.MachineConfig{ - MachineLogging: &v1alpha1.LoggingConfig{ - LoggingDestinations: []v1alpha1.LoggingDestination{ - { - LoggingEndpoint: &v1alpha1.Endpoint{ - URL: &url.URL{ - Scheme: "tcp", - Host: nethelpers.JoinHostPort(gatewayIPs[0].String(), port), - }, - }, - LoggingFormat: "json_lines", - }, - }, - }, - }, - }) - configBundleOpts = append(configBundleOpts, bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(cfg)})) - } - - configBundle, err := bundle.NewBundle(configBundleOpts...) - if err != nil { - return err - } - - bundleTalosconfig := configBundle.TalosConfig() - if bundleTalosconfig == nil { - if clusterWait { - return errors.New("no talosconfig in the config bundle: cannot wait for cluster") - } - - if applyConfigEnabled { - return errors.New("no talosconfig in the config bundle: cannot apply config") - } - } - - if skipInjectingConfig { - types := []machine.Type{machine.TypeControlPlane, machine.TypeWorker} - - if withInitNode { - types = slices.Insert(types, 0, machine.TypeInit) - } - - if err = configBundle.Write(".", encoder.CommentsAll, types...); err != nil { - return err - } - } - - // Wireguard configuration. - var wireguardConfigBundle *helpers.WireguardConfigBundle - if wireguardCIDR != "" { - wireguardConfigBundle, err = helpers.NewWireguardConfigBundle(ips[0], wireguardCIDR, 51111, controlplanes) - if err != nil { - return err - } - } - - var extraKernelArgs *procfs.Cmdline - - if extraBootKernelArgs != "" || withSiderolinkAgent.IsEnabled() { - extraKernelArgs = procfs.NewCmdline(extraBootKernelArgs) - } - - err = slb.SetKernelArgs(extraKernelArgs, withSiderolinkAgent.IsTunnel()) - if err != nil { - return err - } - - // Add talosconfig to provision options, so we'll have it to parse there - provisionOptions = append(provisionOptions, provision.WithTalosConfig(configBundle.TalosConfig())) - - var configInjectionMethod provision.ConfigInjectionMethod - - switch configInjectionMethodFlag { - case "", "default", "http": - configInjectionMethod = provision.ConfigInjectionMethodHTTP - case "metal-iso": - configInjectionMethod = provision.ConfigInjectionMethodMetalISO - default: - return fmt.Errorf("unknown config injection method %q", configInjectionMethod) - } - - // Create the controlplane nodes. - for i := range controlplanes { - var cfg config.Provider - - nodeIPs := make([]netip.Addr, len(cidrs)) - for j := range nodeIPs { - nodeIPs[j] = ips[j][i] - } - - nodeUUID := uuid.New() - - err = slb.DefineIPv6ForUUID(nodeUUID) - if err != nil { - return err - } - - nodeReq := provision.NodeRequest{ - Name: nodeName(clusterName, "controlplane", i+1, nodeUUID), - Type: machine.TypeControlPlane, - Quirks: quirks.New(talosVersion), - IPs: nodeIPs, - Memory: controlPlaneMemory, - NanoCPUs: controlPlaneNanoCPUs, - Disks: disks, - Mounts: mountOpts.Value(), - SkipInjectingConfig: skipInjectingConfig, - ConfigInjectionMethod: configInjectionMethod, - BadRTC: badRTC, - ExtraKernelArgs: extraKernelArgs, - UUID: pointer.To(nodeUUID), - } - - if withInitNode && i == 0 { - cfg = configBundle.Init() - nodeReq.Type = machine.TypeInit - } else { - cfg = configBundle.ControlPlane() - } - - if wireguardConfigBundle != nil { - cfg, err = wireguardConfigBundle.PatchConfig(nodeIPs[0], cfg) - if err != nil { - return err - } - } - - nodeReq.Config = cfg - - request.Nodes = append(request.Nodes, nodeReq) - } - - // append extra disks - for i := range extraDisks { - driver := "ide" - - // ide driver is not supported on arm64 - if targetArch == "arm64" { - driver = "virtio" - } - - if i < len(extraDisksDrivers) { - driver = extraDisksDrivers[i] - } - - disks = append(disks, &provision.Disk{ - Size: uint64(extraDiskSize) * 1024 * 1024, - SkipPreallocate: !clusterDiskPreallocate, - Driver: driver, - BlockSize: diskBlockSize, - }) - } - - for i := 1; i <= workers; i++ { - cfg := configBundle.Worker() - - nodeIPs := make([]netip.Addr, len(cidrs)) - for j := range nodeIPs { - nodeIPs[j] = ips[j][controlplanes+i-1] - } - - if wireguardConfigBundle != nil { - cfg, err = wireguardConfigBundle.PatchConfig(nodeIPs[0], cfg) - if err != nil { - return err - } - } - - nodeUUID := uuid.New() - - err = slb.DefineIPv6ForUUID(nodeUUID) - if err != nil { - return err - } - - request.Nodes = append(request.Nodes, - provision.NodeRequest{ - Name: nodeName(clusterName, "worker", i, nodeUUID), - Type: machine.TypeWorker, - IPs: nodeIPs, - Quirks: quirks.New(talosVersion), - Memory: workerMemory, - NanoCPUs: workerNanoCPUs, - Disks: disks, - Mounts: mountOpts.Value(), - Config: cfg, - ConfigInjectionMethod: configInjectionMethod, - SkipInjectingConfig: skipInjectingConfig, - BadRTC: badRTC, - ExtraKernelArgs: extraKernelArgs, - UUID: pointer.To(nodeUUID), - }) - } - - request.SiderolinkRequest = slb.SiderolinkRequest() - - cluster, err := provisioner.Create(ctx, request, provisionOptions...) - if err != nil { - return err - } - - if debugShellEnabled { - fmt.Println("You can now connect to debug shell on any node using these commands:") - - for _, node := range request.Nodes { - talosDir, err := clientconfig.GetTalosDirectory() - if err != nil { - return nil - } - - fmt.Printf("socat - UNIX-CONNECT:%s\n", filepath.Join(talosDir, "clusters", clusterName, node.Name+".serial")) - } - - return nil - } - - // No talosconfig in the bundle - skip the operations below - if bundleTalosconfig == nil { - return nil - } - - // Create and save the talosctl configuration file. - if err = saveConfig(bundleTalosconfig); err != nil { - return err - } - - clusterAccess := access.NewAdapter(cluster, provisionOptions...) - defer clusterAccess.Close() //nolint:errcheck - - if applyConfigEnabled { - err = clusterAccess.ApplyConfig(ctx, request.Nodes, request.SiderolinkRequest, os.Stdout) - if err != nil { - return err - } - } - - if err = postCreate(ctx, clusterAccess); err != nil { - return err - } - - return showCluster(cluster) -} - -func nodeName(clusterName, role string, index int, uuid uuid.UUID) string { - if withUUIDHostnames { - return fmt.Sprintf("machine-%s", uuid) - } - - return fmt.Sprintf("%s-%s-%d", clusterName, role, index) -} - -func postCreate(ctx context.Context, clusterAccess *access.Adapter) error { - if !withInitNode { - if err := clusterAccess.Bootstrap(ctx, os.Stdout); err != nil { - return fmt.Errorf("bootstrap error: %w", err) - } - } - - if !clusterWait { - return nil - } - - // Run cluster readiness checks - checkCtx, checkCtxCancel := context.WithTimeout(ctx, clusterWaitTimeout) - defer checkCtxCancel() - - checks := check.DefaultClusterChecks() - - if skipK8sNodeReadinessCheck { - checks = slices.Concat(check.PreBootSequenceChecks(), check.K8sComponentsReadinessChecks()) - } - - checks = append(checks, check.ExtraClusterChecks()...) - - if err := check.Wait(checkCtx, clusterAccess, checks, check.StderrReporter()); err != nil { - return err - } - - if !skipKubeconfig { - if err := mergeKubeconfig(ctx, clusterAccess); err != nil { - return err - } - } - - return nil -} - -func saveConfig(talosConfigObj *clientconfig.Config) (err error) { - c, err := clientconfig.Open(talosconfig) - if err != nil { - return fmt.Errorf("error opening talos config: %w", err) - } - - renames := c.Merge(talosConfigObj) - for _, rename := range renames { - fmt.Fprintf(os.Stderr, "renamed talosconfig context %s\n", rename.String()) - } - - return c.Save(talosconfig) -} - -func mergeKubeconfig(ctx context.Context, clusterAccess *access.Adapter) error { - kubeconfigPath, err := kubeconfig.SinglePath() - if err != nil { - return err - } - - fmt.Fprintf(os.Stderr, "\nmerging kubeconfig into %q\n", kubeconfigPath) - - k8sconfig, err := clusterAccess.Kubeconfig(ctx) - if err != nil { - return fmt.Errorf("error fetching kubeconfig: %w", err) - } - - kubeConfig, err := clientcmd.Load(k8sconfig) - if err != nil { - return fmt.Errorf("error parsing kubeconfig: %w", err) - } - - if clusterAccess.ForceEndpoint != "" { - for name := range kubeConfig.Clusters { - kubeConfig.Clusters[name].Server = clusterAccess.ForceEndpoint - } - } - - _, err = os.Stat(kubeconfigPath) - if err != nil { - if !os.IsNotExist(err) { - return err - } - - return clientcmd.WriteToFile(*kubeConfig, kubeconfigPath) - } - - merger, err := kubeconfig.Load(kubeconfigPath) - if err != nil { - return fmt.Errorf("error loading existing kubeconfig: %w", err) - } - - err = merger.Merge(kubeConfig, kubeconfig.MergeOptions{ - ActivateContext: true, - OutputWriter: os.Stdout, - ConflictHandler: func(component kubeconfig.ConfigComponent, name string) (kubeconfig.ConflictDecision, error) { - return kubeconfig.RenameDecision, nil - }, - }) - if err != nil { - return fmt.Errorf("error merging kubeconfig: %w", err) - } - - return merger.Write(kubeconfigPath) -} - -func parseCPUShare(cpus string) (int64, error) { - cpu, ok := new(big.Rat).SetString(cpus) - if !ok { - return 0, fmt.Errorf("failed to parsing as a rational number: %s", cpus) - } - - nano := cpu.Mul(cpu, big.NewRat(1e9, 1)) - if !nano.IsInt() { - return 0, errors.New("value is too precise") - } - - return nano.Num().Int64(), nil -} - -func getDisks() ([]*provision.Disk, error) { - const GPTAlignment = 2 * 1024 * 1024 // 2 MB - - // should have at least a single primary disk - disks := []*provision.Disk{ - { - Size: uint64(clusterDiskSize) * 1024 * 1024, - SkipPreallocate: !clusterDiskPreallocate, - Driver: "virtio", - BlockSize: diskBlockSize, - }, - } - - for _, disk := range clusterDisks { - var ( - partitions = strings.Split(disk, ":") - diskPartitions = make([]*v1alpha1.DiskPartition, len(partitions)/2) - diskSize uint64 - ) - - if len(partitions)%2 != 0 { - return nil, errors.New("failed to parse malformed partition definitions") - } - - partitionIndex := 0 - - for j := 0; j < len(partitions); j += 2 { - partitionPath := partitions[j] - - if !strings.HasPrefix(partitionPath, "/var") { - return nil, errors.New("user disk partitions can only be mounted into /var folder") - } - - value, e := strconv.ParseInt(partitions[j+1], 10, 0) - partitionSize := uint64(value) - - if e != nil { - partitionSize, e = humanize.ParseBytes(partitions[j+1]) - - if e != nil { - return nil, errors.New("failed to parse partition size") - } - } - - diskPartitions[partitionIndex] = &v1alpha1.DiskPartition{ - DiskSize: v1alpha1.DiskSize(partitionSize), - DiskMountPoint: partitionPath, - } - diskSize += partitionSize - partitionIndex++ - } - - disks = append(disks, &provision.Disk{ - // add 2 MB per partition to make extra room for GPT and alignment - Size: diskSize + GPTAlignment*uint64(len(diskPartitions)+1), - Partitions: diskPartitions, - SkipPreallocate: !clusterDiskPreallocate, - Driver: "ide", - BlockSize: diskBlockSize, - }) - } - - return disks, nil -} - -func init() { - createCmd.Flags().StringVar( - &talosconfig, - "talosconfig", - "", - fmt.Sprintf("The path to the Talos configuration file. Defaults to '%s' env variable if set, otherwise '%s' and '%s' in order.", - constants.TalosConfigEnvVar, - filepath.Join("$HOME", constants.TalosDir, constants.TalosconfigFilename), - filepath.Join(constants.ServiceAccountMountPath, constants.TalosconfigFilename), - ), - ) - createCmd.Flags().StringVar(&nodeImage, "image", helpers.DefaultImage(images.DefaultTalosImageRepository), "the image to use") - createCmd.Flags().StringVar(&nodeInstallImage, nodeInstallImageFlag, helpers.DefaultImage(images.DefaultInstallerImageRepository), "the installer image to use") - createCmd.Flags().StringVar(&nodeVmlinuzPath, "vmlinuz-path", helpers.ArtifactPath(constants.KernelAssetWithArch), "the compressed kernel image to use") - createCmd.Flags().StringVar(&nodeISOPath, "iso-path", "", "the ISO path to use for the initial boot (VM only)") - createCmd.Flags().StringVar(&nodeUSBPath, "usb-path", "", "the USB stick image path to use for the initial boot (VM only)") - createCmd.Flags().StringVar(&nodeUKIPath, "uki-path", "", "the UKI image path to use for the initial boot (VM only)") - createCmd.Flags().StringVar(&nodeInitramfsPath, "initrd-path", helpers.ArtifactPath(constants.InitramfsAssetWithArch), "initramfs image to use") - createCmd.Flags().StringVar(&nodeDiskImagePath, "disk-image-path", "", "disk image to use") - createCmd.Flags().StringVar(&nodeIPXEBootScript, "ipxe-boot-script", "", "iPXE boot script (URL) to use") - createCmd.Flags().BoolVar(&applyConfigEnabled, "with-apply-config", false, "enable apply config when the VM is starting in maintenance mode") - createCmd.Flags().BoolVar(&bootloaderEnabled, bootloaderEnabledFlag, true, "enable bootloader to load kernel and initramfs from disk image after install") - createCmd.Flags().BoolVar(&uefiEnabled, "with-uefi", true, "enable UEFI on x86_64 architecture") - createCmd.Flags().BoolVar(&tpm2Enabled, tpm2EnabledFlag, false, "enable TPM2 emulation support using swtpm") - createCmd.Flags().BoolVar(&debugShellEnabled, withDebugShellFlag, false, "drop talos into a maintenance shell on boot, this is for advanced debugging for developers only") - createCmd.Flags().BoolVar(&withIOMMU, withIOMMUFlag, false, "enable IOMMU support, this also add a new PCI root port and an interface attached to it (qemu only)") - createCmd.Flags().MarkHidden("with-debug-shell") //nolint:errcheck - createCmd.Flags().StringSliceVar(&extraUEFISearchPaths, "extra-uefi-search-paths", []string{}, "additional search paths for UEFI firmware (only applies when UEFI is enabled)") - createCmd.Flags().StringSliceVar(®istryMirrors, registryMirrorFlag, []string{}, "list of registry mirrors to use in format: =") - createCmd.Flags().StringSliceVar(®istryInsecure, registryInsecureFlag, []string{}, "list of registry hostnames to skip TLS verification for") - createCmd.Flags().BoolVar(&configDebug, configDebugFlag, false, "enable debug in Talos config to send service logs to the console") - createCmd.Flags().IntVar(&networkMTU, networkMTUFlag, 1500, "MTU of the cluster network") - createCmd.Flags().StringVar(&networkCIDR, networkCIDRFlag, "10.5.0.0/24", "CIDR of the cluster network (IPv4, ULA network for IPv6 is derived in automated way)") - createCmd.Flags().StringSliceVar(&networkNoMasqueradeCIDRs, networkNoMasqueradeCIDRsFlag, []string{}, "list of CIDRs to exclude from NAT (QEMU provisioner only)") - createCmd.Flags().BoolVar(&networkIPv4, networkIPv4Flag, true, "enable IPv4 network in the cluster") - createCmd.Flags().BoolVar(&networkIPv6, networkIPv6Flag, false, "enable IPv6 network in the cluster (QEMU provisioner only)") - createCmd.Flags().StringVar(&wireguardCIDR, "wireguard-cidr", "", "CIDR of the wireguard network") - createCmd.Flags().StringSliceVar(&nameservers, nameserversFlag, []string{"8.8.8.8", "1.1.1.1", "2001:4860:4860::8888", "2606:4700:4700::1111"}, "list of nameservers to use") - createCmd.Flags().IntVar(&workers, "workers", 1, "the number of workers to create") - createCmd.Flags().IntVar(&controlplanes, "masters", 1, "the number of masters to create") - createCmd.Flags().MarkDeprecated("masters", "use --controlplanes instead") //nolint:errcheck - createCmd.Flags().IntVar(&controlplanes, "controlplanes", 1, "the number of controlplanes to create") - createCmd.Flags().StringVar(&controlPlaneCpus, "cpus", "2.0", "the share of CPUs as fraction (each control plane/VM)") - createCmd.Flags().StringVar(&workersCpus, "cpus-workers", "2.0", "the share of CPUs as fraction (each worker/VM)") - createCmd.Flags().IntVar(&controlPlaneMemory, "memory", 2048, "the limit on memory usage in MB (each control plane/VM)") - createCmd.Flags().IntVar(&workersMemory, "memory-workers", 2048, "the limit on memory usage in MB (each worker/VM)") - createCmd.Flags().IntVar(&clusterDiskSize, clusterDiskSizeFlag, 6*1024, "default limit on disk size in MB (each VM)") - createCmd.Flags().UintVar(&diskBlockSize, "disk-block-size", 512, "disk block size (VM only)") - createCmd.Flags().BoolVar(&clusterDiskPreallocate, clusterDiskPreallocateFlag, true, "whether disk space should be preallocated") - createCmd.Flags().StringSliceVar(&clusterDisks, clusterDisksFlag, []string{}, "list of disks to create for each VM in format: :::") - createCmd.Flags().IntVar(&extraDisks, "extra-disks", 0, "number of extra disks to create for each worker VM") - createCmd.Flags().StringSliceVar(&extraDisksDrivers, "extra-disks-drivers", nil, "driver for each extra disk (virtio, ide, ahci, scsi, nvme, megaraid)") - createCmd.Flags().IntVar(&extraDiskSize, "extra-disks-size", 5*1024, "default limit on disk size in MB (each VM)") - createCmd.Flags().StringVar(&targetArch, "arch", stdruntime.GOARCH, "cluster architecture") - createCmd.Flags().BoolVar(&clusterWait, "wait", true, "wait for the cluster to be ready before returning") - createCmd.Flags().DurationVar(&clusterWaitTimeout, "wait-timeout", 20*time.Minute, "timeout to wait for the cluster to be ready") - createCmd.Flags().BoolVar(&forceInitNodeAsEndpoint, "init-node-as-endpoint", false, "use init node as endpoint instead of any load balancer endpoint") - createCmd.Flags().StringVar(&forceEndpoint, forceEndpointFlag, "", "use endpoint instead of provider defaults") - createCmd.Flags().StringVar(&kubernetesVersion, "kubernetes-version", constants.DefaultKubernetesVersion, "desired kubernetes version to run") - createCmd.Flags().StringVarP(&inputDir, inputDirFlag, "i", "", "location of pre-generated config files") - createCmd.Flags().StringSliceVar(&cniBinPath, "cni-bin-path", []string{filepath.Join(defaultCNIDir, "bin")}, "search path for CNI binaries (VM only)") - createCmd.Flags().StringVar(&cniConfDir, "cni-conf-dir", filepath.Join(defaultCNIDir, "conf.d"), "CNI config directory path (VM only)") - createCmd.Flags().StringVar(&cniCacheDir, "cni-cache-dir", filepath.Join(defaultCNIDir, "cache"), "CNI cache directory path (VM only)") - createCmd.Flags().StringVar(&cniBundleURL, "cni-bundle-url", fmt.Sprintf("https://github.com/%s/talos/releases/download/%s/talosctl-cni-bundle-%s.tar.gz", - images.Username, version.Trim(version.Tag), constants.ArchVariable), "URL to download CNI bundle from (VM only)") - createCmd.Flags().StringVarP(&ports, - "exposed-ports", - "p", - "", - "Comma-separated list of ports/protocols to expose on init node. Ex -p :/ (Docker provisioner only)", - ) - createCmd.Flags().StringVar(&dockerHostIP, "docker-host-ip", "0.0.0.0", "Host IP to forward exposed ports to (Docker provisioner only)") - createCmd.Flags().BoolVar(&withInitNode, "with-init-node", false, "create the cluster with an init node") - createCmd.Flags().StringVar(&customCNIUrl, customCNIUrlFlag, "", "install custom CNI from the URL (Talos cluster)") - createCmd.Flags().StringVar(&dnsDomain, dnsDomainFlag, "cluster.local", "the dns domain to use for cluster") - createCmd.Flags().BoolVar(&skipKubeconfig, "skip-kubeconfig", false, "skip merging kubeconfig from the created cluster") - createCmd.Flags().BoolVar(&skipInjectingConfig, "skip-injecting-config", false, "skip injecting config from embedded metadata server, write config files to current directory") - createCmd.Flags().BoolVar(&encryptStatePartition, encryptStatePartitionFlag, false, "enable state partition encryption") - createCmd.Flags().BoolVar(&encryptEphemeralPartition, encryptEphemeralPartitionFlag, false, "enable ephemeral partition encryption") - createCmd.Flags().StringArrayVar(&diskEncryptionKeyTypes, diskEncryptionKeyTypesFlag, []string{"uuid"}, "encryption key types to use for disk encryption (uuid, kms)") - createCmd.Flags().StringVar(&talosVersion, talosVersionFlag, "", "the desired Talos version to generate config for (if not set, defaults to image version)") - createCmd.Flags().BoolVar(&useVIP, useVIPFlag, false, "use a virtual IP for the controlplane endpoint instead of the loadbalancer") - createCmd.Flags().BoolVar(&enableClusterDiscovery, withClusterDiscoveryFlag, true, "enable cluster discovery") - createCmd.Flags().BoolVar(&enableKubeSpan, enableKubeSpanFlag, false, "enable KubeSpan system") - createCmd.Flags().StringArrayVar(&configPatch, "config-patch", nil, "patch generated machineconfigs (applied to all node types), use @file to read a patch from file") - createCmd.Flags().StringArrayVar(&configPatchControlPlane, "config-patch-control-plane", nil, "patch generated machineconfigs (applied to 'init' and 'controlplane' types)") - createCmd.Flags().StringArrayVar(&configPatchWorker, "config-patch-worker", nil, "patch generated machineconfigs (applied to 'worker' type)") - createCmd.Flags().BoolVar(&badRTC, "bad-rtc", false, "launch VM with bad RTC state (QEMU only)") - createCmd.Flags().StringVar(&extraBootKernelArgs, "extra-boot-kernel-args", "", "add extra kernel args to the initial boot from vmlinuz and initramfs (QEMU only)") - createCmd.Flags().BoolVar(&dockerDisableIPv6, "docker-disable-ipv6", false, "skip enabling IPv6 in containers (Docker only)") - createCmd.Flags().IntVar(&controlPlanePort, controlPlanePortFlag, constants.DefaultControlPlanePort, "control plane port (load balancer and local API port, QEMU only)") - createCmd.Flags().IntVar(&kubePrismPort, kubePrismFlag, constants.DefaultKubePrismPort, "KubePrism port (set to 0 to disable)") - createCmd.Flags().BoolVar(&dhcpSkipHostname, "disable-dhcp-hostname", false, "skip announcing hostname via DHCP (QEMU only)") - createCmd.Flags().BoolVar(&skipK8sNodeReadinessCheck, "skip-k8s-node-readiness-check", false, "skip k8s node readiness checks") - createCmd.Flags().BoolVar(&networkChaos, "with-network-chaos", false, "enable to use network chaos parameters when creating a qemu cluster") - createCmd.Flags().DurationVar(&jitter, "with-network-jitter", 0, "specify jitter on the bridge interface when creating a qemu cluster") - createCmd.Flags().DurationVar(&latency, "with-network-latency", 0, "specify latency on the bridge interface when creating a qemu cluster") - createCmd.Flags().Float64Var(&packetLoss, "with-network-packet-loss", 0.0, "specify percent of packet loss on the bridge interface when creating a qemu cluster. e.g. 50% = 0.50 (default: 0.0)") - createCmd.Flags().Float64Var(&packetReorder, "with-network-packet-reorder", 0.0, - "specify percent of reordered packets on the bridge interface when creating a qemu cluster. e.g. 50% = 0.50 (default: 0.0)") - createCmd.Flags().Float64Var(&packetCorrupt, "with-network-packet-corrupt", 0.0, - "specify percent of corrupt packets on the bridge interface when creating a qemu cluster. e.g. 50% = 0.50 (default: 0.0)") - createCmd.Flags().IntVar(&bandwidth, "with-network-bandwidth", 0, "specify bandwidth restriction (in kbps) on the bridge interface when creating a qemu cluster") - createCmd.Flags().StringVar(&withFirewall, firewallFlag, "", "inject firewall rules into the cluster, value is default policy - accept/block (QEMU only)") - createCmd.Flags().BoolVar(&withUUIDHostnames, "with-uuid-hostnames", false, "use machine UUIDs as default hostnames (QEMU only)") - createCmd.Flags().Var(&withSiderolinkAgent, "with-siderolink", "enables the use of siderolink agent as configuration apply mechanism. `true` or `wireguard` enables the agent, `tunnel` enables the agent with grpc tunneling") //nolint:lll - createCmd.Flags().BoolVar(&withJSONLogs, "with-json-logs", false, "enable JSON logs receiver and configure Talos to send logs there") - createCmd.Flags().StringVar(&configInjectionMethodFlag, "config-injection-method", "", "a method to inject machine config: default is HTTP server, 'metal-iso' to mount an ISO (QEMU only)") - createCmd.Flags().Var(&mountOpts, "mount", "attach a mount to the container (Docker only)") - - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, nodeInstallImageFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, configDebugFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, dnsDomainFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, withClusterDiscoveryFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, registryMirrorFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, registryInsecureFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, customCNIUrlFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, talosVersionFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, encryptStatePartitionFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, encryptEphemeralPartitionFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, enableKubeSpanFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, forceEndpointFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, kubePrismFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, diskEncryptionKeyTypesFlag) - - Cmd.AddCommand(createCmd) -} - -func newSiderolinkBuilder(wgHost string, useTLS bool) (*siderolinkBuilder, error) { - prefix, err := networkPrefix("") - if err != nil { - return nil, err - } - - result := &siderolinkBuilder{ - wgHost: wgHost, - binds: map[uuid.UUID]netip.Addr{}, - prefix: prefix, - nodeIPv6Addr: prefix.Addr().Next().String(), - } - - if useTLS { - ca, err := x509.NewSelfSignedCertificateAuthority(x509.ECDSA(true), x509.IPAddresses([]net.IP{net.ParseIP(wgHost)})) - if err != nil { - return nil, err - } - - result.apiCert = ca.CrtPEM - result.apiKey = ca.KeyPEM - } - - var resultErr error - - for range 10 { - for _, d := range []struct { - field *int - net string - what string - }{ - {&result.wgPort, "udp", "WireGuard"}, - {&result.apiPort, "tcp", "gRPC API"}, - {&result.sinkPort, "tcp", "Event Sink"}, - {&result.logPort, "tcp", "Log Receiver"}, - } { - var err error - - *d.field, err = getDynamicPort(d.net) - if err != nil { - return nil, fmt.Errorf("failed to get dynamic port for %s: %w", d.what, err) - } - } - - resultErr = checkPortsDontOverlap(result.wgPort, result.apiPort, result.sinkPort, result.logPort) - if resultErr == nil { - break - } - } - - if resultErr != nil { - return nil, fmt.Errorf("failed to get non-overlapping dynamic ports in 10 attempts: %w", resultErr) - } - - return result, nil -} - -type siderolinkBuilder struct { - wgHost string - - binds map[uuid.UUID]netip.Addr - prefix netip.Prefix - nodeIPv6Addr string - wgPort int - apiPort int - sinkPort int - logPort int - - apiCert []byte - apiKey []byte -} - -// DefineIPv6ForUUID defines an IPv6 address for a given UUID. It is safe to call this method on a nil pointer. -func (slb *siderolinkBuilder) DefineIPv6ForUUID(id uuid.UUID) error { - if slb == nil { - return nil - } - - result, err := generateRandomNodeAddr(slb.prefix) - if err != nil { - return err - } - - slb.binds[id] = result.Addr() - - return nil -} - -// SiderolinkRequest returns a SiderolinkRequest based on the current state of the builder. -// It is safe to call this method on a nil pointer. -func (slb *siderolinkBuilder) SiderolinkRequest() provision.SiderolinkRequest { - if slb == nil { - return provision.SiderolinkRequest{} - } - - return provision.SiderolinkRequest{ - WireguardEndpoint: net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.wgPort)), - APIEndpoint: ":" + strconv.Itoa(slb.apiPort), - APICertificate: slb.apiCert, - APIKey: slb.apiKey, - SinkEndpoint: ":" + strconv.Itoa(slb.sinkPort), - LogEndpoint: ":" + strconv.Itoa(slb.logPort), - SiderolinkBind: maps.ToSlice(slb.binds, func(k uuid.UUID, v netip.Addr) provision.SiderolinkBind { - return provision.SiderolinkBind{ - UUID: k, - Addr: v, - } - }), - } -} - -// TrustedRootsConfig returns the trusted roots config for the current builder. -func (slb *siderolinkBuilder) TrustedRootsConfig() []byte { - if slb == nil || slb.apiCert == nil { - return nil - } - - trustedRootsConfig := security.NewTrustedRootsConfigV1Alpha1() - trustedRootsConfig.MetaName = "siderolink-ca" - trustedRootsConfig.Certificates = string(slb.apiCert) - - marshaled, err := encoder.NewEncoder(trustedRootsConfig, encoder.WithComments(encoder.CommentsDisabled)).Encode() - if err != nil { - panic(fmt.Sprintf("failed to marshal trusted roots config: %s", err)) - } - - return marshaled -} - -// SetKernelArgs sets the kernel arguments for the current builder. It is safe to call this method on a nil pointer. -func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tunnel bool) error { - switch { - case slb == nil: - return nil - case extraKernelArgs.Get("siderolink.api") != nil, - extraKernelArgs.Get("talos.events.sink") != nil, - extraKernelArgs.Get("talos.logging.kernel") != nil: - return errors.New("siderolink kernel arguments are already set, cannot run with --with-siderolink") - default: - scheme := "grpc://" - - if slb.apiCert != nil { - scheme = "https://" - } - - apiLink := scheme + net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.apiPort)) + "?jointoken=foo" - - if tunnel { - apiLink += "&grpc_tunnel=true" - } - - extraKernelArgs.Append("siderolink.api", apiLink) - extraKernelArgs.Append("talos.events.sink", net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.sinkPort))) - extraKernelArgs.Append("talos.logging.kernel", "tcp://"+net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.logPort))) - - if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil { - var buf bytes.Buffer - - zencoder, err := zstd.NewWriter(&buf) - if err != nil { - return fmt.Errorf("failed to create zstd encoder: %w", err) - } - - _, err = zencoder.Write(trustedRootsConfig) - if err != nil { - return fmt.Errorf("failed to write zstd data: %w", err) - } - - if err = zencoder.Close(); err != nil { - return fmt.Errorf("failed to close zstd encoder: %w", err) - } - - extraKernelArgs.Append(constants.KernelParamConfigInline, base64.StdEncoding.EncodeToString(buf.Bytes())) - } - - return nil - } -} - -func getDynamicPort(network string) (int, error) { - var ( - closeFn func() error - addrFn func() net.Addr - ) - - switch network { - case "tcp", "tcp4", "tcp6": - l, err := net.Listen(network, "127.0.0.1:0") - if err != nil { - return 0, err - } - - addrFn, closeFn = l.Addr, l.Close - case "udp", "udp4", "udp6": - l, err := net.ListenPacket(network, "127.0.0.1:0") - if err != nil { - return 0, err - } - - addrFn, closeFn = l.LocalAddr, l.Close - default: - return 0, fmt.Errorf("unsupported network: %s", network) - } - - _, portStr, err := net.SplitHostPort(addrFn().String()) - if err != nil { - return 0, handleCloseErr(err, closeFn()) - } - - port, err := strconv.Atoi(portStr) - if err != nil { - return 0, err - } - - return port, handleCloseErr(nil, closeFn()) -} - -func handleCloseErr(err error, closeErr error) error { - switch { - case err != nil && closeErr != nil: - return fmt.Errorf("error: %w, close error: %w", err, closeErr) - case err == nil && closeErr != nil: - return closeErr - case err != nil && closeErr == nil: - return err - default: - return nil - } -} - -func checkPortsDontOverlap(ports ...int) error { - slices.Sort(ports) - - if len(ports) != len(slices.Compact(ports)) { - return errors.New("generated ports overlap") - } - - return nil -} - -type agentFlag uint8 - -func (a *agentFlag) String() string { - switch *a { - case 1: - return "wireguard" - case 2: - return "grpc-tunnel" - case 3: - return "wireguard+tls" - case 4: - return "grpc-tunnel+tls" - default: - return "none" - } -} - -func (a *agentFlag) Set(s string) error { - switch s { - case "true", "wireguard": - *a = 1 - case "tunnel": - *a = 2 - case "wireguard+tls": - *a = 3 - case "grpc-tunnel+tls": - *a = 4 - default: - return fmt.Errorf("unknown type: %s, possible values: 'true', 'wireguard' for the usual WG; 'tunnel' for WG over GRPC, add '+tls' to enable TLS for API", s) - } - - return nil -} - -func (a *agentFlag) Type() string { return "agent" } -func (a *agentFlag) IsEnabled() bool { return *a != 0 } -func (a *agentFlag) IsTunnel() bool { return *a == 2 || *a == 4 } -func (a *agentFlag) IsTLS() bool { return *a == 3 || *a == 4 } diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/maker.go b/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/maker.go new file mode 100644 index 0000000000..0b12d83d42 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/maker.go @@ -0,0 +1,600 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package clustermaker + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "net/url" + "os" + "slices" + "strings" + + sideronet "github.com/siderolabs/net" + + clustercmd "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster" + "github.com/siderolabs/talos/cmd/talosctl/pkg/mgmt/helpers" + "github.com/siderolabs/talos/pkg/cluster/check" + clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/bundle" + "github.com/siderolabs/talos/pkg/machinery/config/configpatcher" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/encoder" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/access" +) + +type clusterMaker struct { + // input fields + options Options + provisioner provision.Provisioner + talosVersion string + + // fields available post init + request provision.ClusterRequest + cidr4 netip.Prefix + provisionOpts []provision.Option + cfgBundleOpts []bundle.Option + genOpts []generate.Option + ips [][]netip.Addr + versionContract *config.VersionContract + + // fields available post finalization + inClusterEndpoint string + configBundle *bundle.Bundle + bundleTalosconfig *clientconfig.Config + cluster provision.Cluster +} + +// Input is the input options for clusterMaker. +type Input struct { + Ops Options + Provisioner provision.Provisioner + TalosVersion string +} + +// New initializes a new ClusterMaker. +func New(input Input) (ClusterMaker, error) { + cm, err := newClusterMaker(input) + + return &cm, err +} + +// newClusterMaker is used in tests. +func newClusterMaker(input Input) (clusterMaker, error) { + cm := clusterMaker{} + err := cm.init(input) + + return cm, err +} + +func (cm *clusterMaker) GetPartialClusterRequest() PartialClusterRequest { + return PartialClusterRequest(cm.request) +} + +func (cm *clusterMaker) AddGenOps(opts ...generate.Option) { + cm.genOpts = append(cm.genOpts, opts...) +} + +func (cm *clusterMaker) AddProvisionOps(opts ...provision.Option) { + cm.provisionOpts = append(cm.provisionOpts, opts...) +} + +func (cm *clusterMaker) AddCfgBundleOpts(opts ...bundle.Option) { + cm.cfgBundleOpts = append(cm.cfgBundleOpts, opts...) +} + +func (cm *clusterMaker) SetInClusterEndpoint(endpoint string) { + cm.inClusterEndpoint = endpoint +} + +func (cm *clusterMaker) CreateCluster(ctx context.Context, request PartialClusterRequest) error { + cm.request = provision.ClusterRequest(request) + + err := cm.finalizeRequest() + if err != nil { + return err + } + + cluster, err := cm.provisioner.Create(ctx, cm.request, cm.provisionOpts...) + if err != nil { + return err + } + + cm.cluster = cluster + + return nil +} + +func (cm *clusterMaker) PostCreate(ctx context.Context) error { + // No talosconfig in the bundle - skip the operations below + if cm.bundleTalosconfig == nil { + return nil + } + + clusterAccess := access.NewAdapter(cm.cluster, cm.provisionOpts...) + defer clusterAccess.Close() //nolint:errcheck + + if err := cm.applyConfigs(ctx, clusterAccess); err != nil { + return err + } + + if err := cm.bootstrapCluster(ctx, clusterAccess); err != nil { + return err + } + + return clustercmd.ShowCluster(cm.cluster) +} + +func (cm *clusterMaker) GetCIDR4() netip.Prefix { + return cm.cidr4 +} + +func (cm *clusterMaker) GetVersionContract() *config.VersionContract { + return cm.versionContract +} + +func (cm *clusterMaker) bootstrapCluster(ctx context.Context, clusterAccess *access.Adapter) error { + if !cm.options.WithInitNode { + if err := clusterAccess.Bootstrap(ctx, os.Stdout); err != nil { + return fmt.Errorf("bootstrap error: %w", err) + } + } + + if !cm.options.ClusterWait { + return nil + } + + // Run cluster readiness checks + checkCtx, checkCtxCancel := context.WithTimeout(ctx, cm.options.ClusterWaitTimeout) + defer checkCtxCancel() + + checks := check.DefaultClusterChecks() + + if cm.options.SkipK8sNodeReadinessCheck { + checks = slices.Concat(check.PreBootSequenceChecks(), check.K8sComponentsReadinessChecks()) + } + + checks = append(checks, check.ExtraClusterChecks()...) + + if err := check.Wait(checkCtx, clusterAccess, checks, check.StderrReporter()); err != nil { + return err + } + + if !cm.options.SkipKubeconfig { + if err := mergeKubeconfig(ctx, clusterAccess); err != nil { + return err + } + } + + return nil +} + +func (cm *clusterMaker) applyConfigs(ctx context.Context, clusterAccess *access.Adapter) error { + // Create and save the talosctl configuration file. + if err := saveConfig(cm.bundleTalosconfig, cm.options); err != nil { + return err + } + + if cm.options.ApplyConfigEnabled { + if err := clusterAccess.ApplyConfig(ctx, cm.request.Nodes, cm.request.SiderolinkRequest, os.Stdout); err != nil { + return err + } + } + + return nil +} + +func (cm *clusterMaker) init(input Input) error { + cm.provisioner = input.Provisioner + cm.options = input.Ops + cm.talosVersion = input.TalosVersion + + if cm.options.TalosVersion != "latest" { + versionContract, err := config.ParseContractFromVersion(cm.talosVersion) + if err != nil { + return fmt.Errorf("error parsing Talos version %q: %w", cm.options.TalosVersion, err) + } + + cm.versionContract = versionContract + } + + if err := cm.createPartialClusterRequest(); err != nil { + return err + } + + if err := cm.createNodeRequests(); err != nil { + return err + } + + if err := cm.initProvisionOpts(); err != nil { + return err + } + + if err := cm.initConfigBundleOpts(); err != nil { + return err + } + + if err := cm.initGenOps(); err != nil { + return err + } + + return nil +} + +func (cm *clusterMaker) finalizeRequest() error { + if cm.options.InputDir != "" { + cm.AddCfgBundleOpts(bundle.WithExistingConfigs(cm.options.InputDir)) + } else { + if cm.inClusterEndpoint == "" { + cm.inClusterEndpoint = cm.provisioner.GetInClusterKubernetesControlPlaneEndpoint(cm.request.Network, cm.options.ControlPlanePort) + } + + cm.AddCfgBundleOpts(bundle.WithInputOptions( + &bundle.InputOptions{ + ClusterName: cm.options.RootOps.ClusterName, + Endpoint: cm.inClusterEndpoint, + KubeVersion: strings.TrimPrefix(cm.options.KubernetesVersion, "v"), + GenOptions: cm.genOpts, + })) + } + + configBundle, bundleTalosconfig, err := cm.getConfigBundle() + if err != nil { + return err + } + + cm.AddProvisionOps(provision.WithTalosConfig(configBundle.TalosConfig())) + + cm.bundleTalosconfig = bundleTalosconfig + cm.configBundle = configBundle + + return cm.applyNodeCfgs() +} + +func (cm *clusterMaker) getConfigBundle() (configBundle *bundle.Bundle, bundleTalosconfig *clientconfig.Config, err error) { + configBundle, err = bundle.NewBundle(cm.cfgBundleOpts...) + if err != nil { + return nil, nil, err + } + + bundleTalosconfig = configBundle.TalosConfig() + if bundleTalosconfig == nil { + if cm.options.ClusterWait { + return nil, nil, errors.New("no talosconfig in the config bundle: cannot wait for cluster") + } + + if cm.options.ApplyConfigEnabled { + return nil, nil, errors.New("no talosconfig in the config bundle: cannot apply config") + } + } + + if cm.options.SkipInjectingConfig { + types := []machine.Type{machine.TypeControlPlane, machine.TypeWorker} + + if cm.options.WithInitNode { + types = slices.Insert(types, 0, machine.TypeInit) + } + + if err = configBundle.Write(".", encoder.CommentsAll, types...); err != nil { + return nil, nil, err + } + } + + return +} + +func (cm *clusterMaker) initProvisionOpts() error { + if cm.options.WithJSONLogs { + cm.AddProvisionOps(provision.WithJSONLogs(nethelpers.JoinHostPort(cm.request.Network.GatewayAddrs[0].String(), jsonLogsPort))) + } + + externalKubernetesEndpoint := cm.provisioner.GetExternalKubernetesControlPlaneEndpoint(cm.request.Network, cm.options.ControlPlanePort) + cm.AddProvisionOps(provision.WithKubernetesEndpoint(externalKubernetesEndpoint)) + + return nil +} + +func (cm *clusterMaker) initConfigBundleOpts() error { + addConfigPatch := func(configPatches []string, configOpt func([]configpatcher.Patch) bundle.Option) error { + var patches []configpatcher.Patch + + patches, err := configpatcher.LoadPatches(configPatches) + if err != nil { + return fmt.Errorf("error parsing config JSON patch: %w", err) + } + + cm.AddCfgBundleOpts(configOpt(patches)) + + return nil + } + + if err := addConfigPatch(cm.options.ConfigPatch, bundle.WithPatch); err != nil { + return err + } + + if err := addConfigPatch(cm.options.ConfigPatchControlPlane, bundle.WithPatchControlPlane); err != nil { + return err + } + + if err := addConfigPatch(cm.options.ConfigPatchWorker, bundle.WithPatchWorker); err != nil { + return err + } + + if cm.options.WithJSONLogs { + cfg := container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineLogging: &v1alpha1.LoggingConfig{ + LoggingDestinations: []v1alpha1.LoggingDestination{ + { + LoggingEndpoint: &v1alpha1.Endpoint{ + URL: &url.URL{ + Scheme: "tcp", + Host: nethelpers.JoinHostPort(cm.request.Network.GatewayAddrs[0].String(), jsonLogsPort), + }, + }, + LoggingFormat: "json_lines", + }, + }, + }, + }, + }) + cm.AddCfgBundleOpts(bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(cfg)})) + } + + return nil +} + +func (cm *clusterMaker) initGenOps() error { + cm.AddGenOps( + generate.WithDebug(cm.options.ConfigDebug), + generate.WithDNSDomain(cm.options.DNSDomain), + generate.WithClusterDiscovery(cm.options.EnableClusterDiscovery), + ) + cm.AddGenOps(cm.provisioner.GenOptions(cm.request.Network)...) + + for _, registryMirror := range cm.options.RegistryMirrors { + left, right, ok := strings.Cut(registryMirror, "=") + if !ok { + return fmt.Errorf("invalid registry mirror spec: %q", registryMirror) + } + + cm.AddGenOps(generate.WithRegistryMirror(left, right)) + } + + for _, registryHost := range cm.options.RegistryInsecure { + cm.AddGenOps(generate.WithRegistryInsecureSkipVerify(registryHost)) + } + + if cm.versionContract != nil { + cm.AddGenOps(generate.WithVersionContract(cm.versionContract)) + } + + if cm.options.CustomCNIUrl != "" { + cm.AddGenOps(generate.WithClusterCNIConfig(&v1alpha1.CNIConfig{ + CNIName: constants.CustomCNI, + CNIUrls: []string{cm.options.CustomCNIUrl}, + })) + } + + if cm.options.KubePrismPort != constants.DefaultKubePrismPort { + cm.AddGenOps(generate.WithKubePrismPort(cm.options.KubePrismPort)) + } + + if cm.options.ControlPlanePort != constants.DefaultControlPlanePort { + cm.AddGenOps(generate.WithLocalAPIServerPort(cm.options.ControlPlanePort)) + } + + if cm.options.EnableKubeSpan { + cm.AddGenOps(generate.WithNetworkOptions(v1alpha1.WithKubeSpan())) + } + + return cm.addEnpointListGenOption() +} + +func (cm *clusterMaker) addEnpointListGenOption() error { + endpointList := cm.provisioner.GetTalosAPIEndpoints(cm.request.Network) + + switch { + case cm.options.ForceEndpoint != "": + // using non-default endpoints, provision additional cert SANs and fix endpoint list + endpointList = []string{cm.options.ForceEndpoint} + cm.AddGenOps(generate.WithAdditionalSubjectAltNames(endpointList)) + case cm.options.ForceInitNodeAsEndpoint: + endpointList = []string{cm.ips[0][0].String()} + case len(endpointList) > 0: + for _, endpointHostPort := range endpointList { + endpointHost, _, err := net.SplitHostPort(endpointHostPort) + if err != nil { + endpointHost = endpointHostPort + } + + cm.AddGenOps(generate.WithAdditionalSubjectAltNames([]string{endpointHost})) + } + case endpointList == nil: + // use control plane nodes as endpoints, client-side load-balancing + for i := range cm.options.Controlplanes { + endpointList = append(endpointList, cm.ips[0][i].String()) + } + } + + cm.AddGenOps(generate.WithEndpointList(endpointList)) + + return nil +} + +func (cm *clusterMaker) createPartialClusterRequest() error { + cm.request = provision.ClusterRequest{ + Name: cm.options.RootOps.ClusterName, + SelfExecutable: os.Args[0], + StateDirectory: cm.options.RootOps.StateDir, + } + + if err := cm.initNetworkParams(); err != nil { + return err + } + + return cm.initNetworkParams() +} + +func (cm *clusterMaker) createNodeRequests() error { + if cm.options.Controlplanes < 1 { + return errors.New("number of controlplanes can't be less than 1") + } + + controlPlaneNanoCPUs, err := parseCPUShare(cm.options.ControlPlaneCpus) + if err != nil { + return fmt.Errorf("error parsing --cpus: %s", err) + } + + workerNanoCPUs, err := parseCPUShare(cm.options.WorkersCpus) + if err != nil { + return fmt.Errorf("error parsing --cpus-workers: %s", err) + } + + controlPlaneMemory := int64(cm.options.ControlPlaneMemory) * 1024 * 1024 + workerMemory := int64(cm.options.WorkersMemory) * 1024 * 1024 + + for i := range cm.options.Controlplanes { + machineType := machine.TypeControlPlane + nodeIPs := getNodeIP(cm.request.Network.CIDRs, cm.ips, i) + cm.request.Nodes = append(cm.request.Nodes, provision.NodeRequest{ + Name: fmt.Sprintf("%s-%s-%d", cm.options.RootOps.ClusterName, "controlplane", i+1), + IPs: nodeIPs, + Type: machineType, + Memory: controlPlaneMemory, + NanoCPUs: controlPlaneNanoCPUs, + SkipInjectingConfig: cm.options.SkipInjectingConfig, + }) + } + + for workerIndex := range cm.options.Workers { + nodeIndex := cm.options.Controlplanes + workerIndex + nodeIPs := getNodeIP(cm.request.Network.CIDRs, cm.ips, nodeIndex) + cm.request.Nodes = append(cm.request.Nodes, provision.NodeRequest{ + Name: fmt.Sprintf("%s-%s-%d", cm.options.RootOps.ClusterName, "worker", workerIndex+1), + IPs: nodeIPs, + Type: machine.TypeWorker, + Memory: workerMemory, + NanoCPUs: workerNanoCPUs, + SkipInjectingConfig: cm.options.SkipInjectingConfig, + }) + } + + return nil +} + +func (cm *clusterMaker) applyNodeCfgs() (err error) { + var wireguardConfigBundle *helpers.WireguardConfigBundle + if cm.options.WireguardCIDR != "" { + wireguardConfigBundle, err = helpers.NewWireguardConfigBundle(cm.ips[0], cm.options.WireguardCIDR, 51111, cm.options.Controlplanes) + if err != nil { + return err + } + } + + for i, n := range cm.request.Nodes { + cfg := cm.configBundle.ControlPlane() + if n.Type == machine.TypeInit { + cfg = cm.configBundle.Init() + } else if n.Type == machine.TypeWorker { + cfg = cm.configBundle.Worker() + } + + cfg, err := patchWireguard(wireguardConfigBundle, cfg, n.IPs) + if err != nil { + return err + } + + cm.request.Nodes[i].Config = cfg + } + + return nil +} + +func (cm *clusterMaker) initNetworkParams() error { + cidr4, err := netip.ParsePrefix(cm.options.NetworkCIDR) + if err != nil { + return fmt.Errorf("error validating cidr block: %w", err) + } + + if !cidr4.Addr().Is4() { + return errors.New("--cidr is expected to be IPV4 CIDR") + } + + cm.cidr4 = cidr4 + + var cidrs []netip.Prefix + if cm.options.NetworkIPv4 { + cidrs = append(cidrs, cidr4) + } + + // use ULA IPv6 network fd00::/8, add 'TAL' in hex to build /32 network, add IPv4 CIDR to build /64 unique network + cidr6, err := netip.ParsePrefix( + fmt.Sprintf( + "fd74:616c:%02x%02x:%02x%02x::/64", + cidr4.Addr().As4()[0], cidr4.Addr().As4()[1], cidr4.Addr().As4()[2], cidr4.Addr().As4()[3], + ), + ) + if err != nil { + return fmt.Errorf("error validating cidr IPv6 block: %w", err) + } + + if cm.options.NetworkIPv6 { + cidrs = append(cidrs, cidr6) + } + + // Gateway addr at 1st IP in range, ex. 192.168.0.1 + gatewayIPs := make([]netip.Addr, len(cidrs)) + + for j := range gatewayIPs { + gatewayIPs[j], err = sideronet.NthIPInNetwork(cidrs[j], gatewayOffset) + if err != nil { + return err + } + } + + cm.request.Network = provision.NetworkRequest{ + Name: cm.options.RootOps.ClusterName, + CIDRs: cidrs, + GatewayAddrs: gatewayIPs, + MTU: cm.options.NetworkMTU, + } + + return cm.initIps() +} + +func (cm *clusterMaker) initIps() error { + cidrs := cm.request.Network.CIDRs + ips := make([][]netip.Addr, len(cidrs)) + + for j := range cidrs { + ips[j] = make([]netip.Addr, cm.options.Controlplanes+cm.options.Workers) + + for i := range ips[j] { + ip, err := sideronet.NthIPInNetwork(cidrs[j], nodesOffset+i) + if err != nil { + return err + } + + ips[j][i] = ip + } + } + + cm.ips = ips + + return nil +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/maker_test.go b/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/maker_test.go new file mode 100644 index 0000000000..61d34a57fa --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/maker_test.go @@ -0,0 +1,461 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package clustermaker //nolint:testpackage + +import ( + "fmt" + "reflect" + "slices" + "strconv" + "testing" + + "github.com/siderolabs/gen/xslices" + "github.com/stretchr/testify/assert" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster" + "github.com/siderolabs/talos/pkg/machinery/config/bundle" + "github.com/siderolabs/talos/pkg/machinery/config/encoder" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/provision" +) + +// getTestOps returns the barebones options needed to initialize the clustermaker. +func getTestOps() Options { + return Options{ + RootOps: &cluster.CmdOps{ + ProvisionerName: "test-provisioner", + StateDir: "/state-dir", + ClusterName: "test-cluster", + }, + Workers: 2, + Controlplanes: 2, + NetworkCIDR: "10.5.0.0/24", + NetworkMTU: 1500, + WorkersCpus: "2.0", + ControlPlaneCpus: "4", + ControlPlaneMemory: 4096, + WorkersMemory: 2048, + NetworkIPv4: true, + } +} + +type testProvisioner struct { + provision.Provisioner +} + +func (p testProvisioner) GenOptions(r provision.NetworkRequest) []generate.Option { + return []generate.Option{func(o *generate.Options) error { + o.CNIConfig = &v1alpha1.CNIConfig{ + CNIName: "testname", + } + + return nil + }} +} + +func (p testProvisioner) GetTalosAPIEndpoints(provision.NetworkRequest) []string { + return []string{"talos-api-endpoint.test"} +} + +func (p testProvisioner) GetInClusterKubernetesControlPlaneEndpoint(networkReq provision.NetworkRequest, controlPlanePort int) string { + return "https://" + nethelpers.JoinHostPort(networkReq.CIDRs[0].Addr().Next().Next().String(), controlPlanePort) +} + +func (p testProvisioner) GetExternalKubernetesControlPlaneEndpoint(networkReq provision.NetworkRequest, controlPlanePort int) string { + return "https://" + nethelpers.JoinHostPort(networkReq.CIDRs[0].Addr().Next().Next().String(), controlPlanePort) +} + +type testFields = Options + +// n returns the field names of s. +func n(s any, fields ...any) []string { + var names []string + + for _, f := range fields { + rv := reflect.ValueOf(s).Elem() + for i := range rv.NumField() { + fv := rv.Field(i) + fp := fv.Addr().Interface() + + if f == fp { + names = append(names, rv.Type().Field(i).Name) + } + } + } + + return names +} + +var tf = testFields{} + +func bundleApply(t *testing.T, opts ...bundle.Option) bundle.Options { + options := bundle.Options{} + for _, opt := range opts { + if err := opt(&options); err != nil { + t.Error("failed to apply option: ", err) + } + } + + return options +} + +const testTalosVersion = "1.1" + +func init() { + addFieldTest("TestCriticalOptions", + n(&tf, &tf.Workers, &tf.Controlplanes, &tf.NetworkCIDR, &tf.NetworkMTU, &tf.WorkersCpus, &tf.ControlPlaneCpus, + &tf.ControlPlaneMemory, &tf.WorkersMemory, &tf.NetworkIPv4, &tf.RootOps, &tf.Controlplanes), + func(t *testing.T) { + input := getTestOps() + cm := getFinalizedClusterMaker(t, input) + result := cm.GetPartialClusterRequest() + + // Nodes + workersResult := result.Nodes.WorkerNodes() + controlsResult := result.Nodes.ControlPlaneNodes() + assert.Equal(t, input.Controlplanes, len(controlsResult)) + assert.Equal(t, input.Workers, len(workersResult)) + + for i := range input.Controlplanes { + n := controlsResult[i] + assert.EqualValues(t, 4000000000, n.NanoCPUs) + assert.EqualValues(t, 4294967296, n.Memory) + assert.Equal(t, false, n.SkipInjectingConfig) + assert.Equal(t, machine.TypeControlPlane, n.Type) + assert.Equal(t, "test-cluster-controlplane-"+strconv.Itoa(i+1), n.Name) + assert.Equal(t, 1, len(n.IPs)) + } + + for i := range input.Workers { + n := workersResult[i] + assert.EqualValues(t, 2000000000, n.NanoCPUs) + assert.EqualValues(t, 2147483648, n.Memory) + assert.Equal(t, false, n.SkipInjectingConfig) + assert.Equal(t, machine.TypeWorker, n.Type) + assert.Equal(t, "test-cluster-worker-"+strconv.Itoa(i+1), n.Name) + assert.Equal(t, 1, len(n.IPs)) + } + + // ClusterRequest + assert.Equal(t, input.RootOps.ClusterName, cm.request.Name) + assert.Equal(t, input.RootOps.StateDir, cm.request.StateDirectory) + assert.NotZero(t, cm.request.SelfExecutable) + + // Network + networkResult := cm.request.Network + assert.Equal(t, 1, len(networkResult.CIDRs)) + assert.Equal(t, input.NetworkCIDR, networkResult.CIDRs[0].String()) + assert.Equal(t, 1, len(networkResult.GatewayAddrs)) + assert.Equal(t, "10.5.0.1", networkResult.GatewayAddrs[0].String()) + assert.Equal(t, input.NetworkMTU, networkResult.MTU) + assert.Equal(t, input.RootOps.ClusterName, networkResult.Name) + assert.Equal(t, cm.cidr4, networkResult.CIDRs[0]) + assert.Equal(t, 1, len(cm.ips)) + assert.Equal(t, 4, len(cm.ips[0])) + assert.Equal(t, "10.5.0.2", cm.ips[0][0].String()) + assert.Equal(t, "10.5.0.5", cm.ips[0][3].String()) + assert.Equal(t, "10.5.0.2", cm.request.Nodes.ControlPlaneNodes()[0].IPs[0].String()) + assert.Equal(t, "10.5.0.5", cm.request.Nodes.WorkerNodes()[1].IPs[0].String()) + }) + + // + // Generate Options + // + addFieldTest("TestRegistryMirrors", n(&tf, &tf.RegistryMirrors, &tf.RegistryInsecure), func(t *testing.T) { + input := getTestOps() + input.RegistryMirrors = []string{"test.test=https://test.mirror", "insecure.test=https://insecure.mirror"} + input.RegistryInsecure = []string{"insecure.test"} + + options := getGenOpts(t, input) + + assert.Equal(t, 2, len(options.RegistryMirrors)) + assert.Equal(t, "https://test.mirror", options.RegistryMirrors["test.test"].MirrorEndpoints[0]) + assert.Equal(t, "https://insecure.mirror", options.RegistryMirrors["insecure.test"].MirrorEndpoints[0]) + assert.Equal(t, true, options.RegistryConfig["insecure.test"].RegistryTLS.InsecureSkipVerify()) + }) + addFieldTest("", n(&tf, &tf.ConfigDebug), func(t *testing.T) { + input := getTestOps() + input.ConfigDebug = true + + options := getGenOpts(t, input) + + assert.Equal(t, true, options.Debug) + }) + addFieldTest("", n(&tf, &tf.DNSDomain), func(t *testing.T) { + input := getTestOps() + input.DNSDomain = "test.dns" + + options := getGenOpts(t, input) + + assert.Equal(t, "test.dns", options.DNSDomain) + }) + addFieldTest("", n(&tf, &tf.EnableClusterDiscovery), func(t *testing.T) { + input := getTestOps() + input.EnableClusterDiscovery = true + + options := getGenOpts(t, input) + + assert.Equal(t, true, *options.DiscoveryEnabled) + }) + addFieldTest("", n(&tf, &tf.CustomCNIUrl), func(t *testing.T) { + input := getTestOps() + input.CustomCNIUrl = "test.url" + + options := getGenOpts(t, input) + + assert.EqualValues(t, []string{"test.url"}, options.CNIConfig.CNIUrls) + }) + addFieldTest("", n(&tf, &tf.ForceInitNodeAsEndpoint), func(t *testing.T) { + input := getTestOps() + input.ForceInitNodeAsEndpoint = true + + options := getGenOpts(t, input) + + assert.EqualValues(t, 1, len(options.EndpointList)) + assert.EqualValues(t, "10.5.0.2", options.EndpointList[0]) + }) + addFieldTest("", n(&tf, &tf.ControlPlanePort), func(t *testing.T) { + input := getTestOps() + input.ControlPlanePort = 1111 + + options := getGenOpts(t, input) + + assert.EqualValues(t, 1111, options.LocalAPIServerPort) + }) + addFieldTest("", n(&tf, &tf.KubePrismPort), func(t *testing.T) { + input := getTestOps() + input.KubePrismPort = 2222 + + options := getGenOpts(t, input) + + assert.EqualValues(t, 2222, *options.KubePrismPort.Ptr()) + }) + addFieldTest("", n(&tf, &tf.ForceEndpoint), func(t *testing.T) { + input := getTestOps() + input.ForceEndpoint = "test" + + options := getGenOpts(t, input) + + assert.EqualValues(t, 1, len(options.EndpointList)) + assert.EqualValues(t, "test", options.EndpointList[0]) + assert.EqualValues(t, 1, len(options.AdditionalSubjectAltNames)) + assert.EqualValues(t, "test", options.AdditionalSubjectAltNames[0]) + }) + addFieldTest("", n(&tf, &tf.TalosVersion), func(t *testing.T) { + input := getTestOps() + + cm, err := newClusterMaker(Input{input, testProvisioner{}, "v0.1"}) + assert.NoError(t, err) + err = cm.finalizeRequest() + assert.NoError(t, err) + result, err := generate.NewInput(input.RootOps.ClusterName, "cluster.endpoint", "k8sv1", cm.genOpts...) + assert.NoError(t, err) + + assert.EqualValues(t, "v0.1", result.Options.VersionContract.String()) + }) + addFieldTest("TestInvalidTalosVersion", n(&tf, &tf.TalosVersion), func(t *testing.T) { + input := getTestOps() + + _, err := New(Input{input, testProvisioner{}, "invalid"}) + assert.ErrorContains(t, err, "error parsing Talos version") + }) + + // + // Config bundle options + // + addFieldTest("", n(&tf, &tf.KubernetesVersion), func(t *testing.T) { + input := getTestOps() + input.KubernetesVersion = "1.1.1-test" + + cm := getFinalizedClusterMaker(t, input) + + opts := bundleApply(t, cm.cfgBundleOpts...) + assert.Equal(t, "1.1.1-test", opts.InputOptions.KubeVersion) + }) + addFieldTest("", n(&tf, &tf.EnableKubeSpan), func(t *testing.T) { + input := getTestOps() + input.EnableKubeSpan = true + + cm := getFinalizedClusterMaker(t, input) + + assert.EqualValues(t, true, cm.configBundle.Init().RawV1Alpha1().MachineConfig.MachineNetwork.KubeSpan().Enabled()) + }) + addFieldTest("", n(&tf, &tf.WithJSONLogs), func(t *testing.T) { + input := getTestOps() + input.WithJSONLogs = true + + cm := getFinalizedClusterMaker(t, input) + + assert.EqualValues(t, "json_lines", cm.configBundle.Init().RawV1Alpha1().MachineConfig.MachineLogging.LoggingDestinations[0].LoggingFormat) + }) + addFieldTest("", n(&tf, &tf.WireguardCIDR), func(t *testing.T) { + input := getTestOps() + input.WireguardCIDR = "10.1.0.0/16" + + cm := getFinalizedClusterMaker(t, input) + + assert.EqualValues(t, 1, len(cm.request.Nodes.WorkerNodes()[0].Config.RawV1Alpha1().MachineConfig.MachineNetwork.NetworkInterfaces)) + assert.EqualValues(t, 1, len(cm.request.Nodes.ControlPlaneNodes()[0].Config.RawV1Alpha1().MachineConfig.MachineNetwork.NetworkInterfaces)) + assert.EqualValues(t, 1, len(cm.request.Nodes.ControlPlaneNodes()[1].Config.RawV1Alpha1().MachineConfig.MachineNetwork.NetworkInterfaces)) + assert.EqualValues(t, "wg0", cm.request.Nodes.WorkerNodes()[0].Config.RawV1Alpha1().MachineConfig.MachineNetwork.NetworkInterfaces[0].DeviceInterface) + }) + addFieldTest("TestConfigPatches", n(&tf, &tf.ConfigPatch, &tf.ConfigPatchControlPlane, &tf.ConfigPatchWorker), func(t *testing.T) { + input := getTestOps() + input.ConfigPatch = []string{`[{"op": "add", "path": "/machine/network/hostname", "value": "test-hostname"}]`} + input.ConfigPatchControlPlane = []string{`[{"op": "add", "path": "/machine/kubelet/image", "value": "test-control"}]`} + input.ConfigPatchWorker = []string{`[{"op": "add", "path": "/machine/kubelet/image", "value": "test-worker"}]`} + + cm := getFinalizedClusterMaker(t, input) + + assert.EqualValues(t, "test-hostname", cm.request.Nodes.WorkerNodes()[0].Config.RawV1Alpha1().MachineConfig.Network().Hostname()) + assert.EqualValues(t, "test-control", cm.request.Nodes.ControlPlaneNodes()[0].Config.RawV1Alpha1().MachineConfig.Kubelet().Image()) + assert.EqualValues(t, "test-worker", cm.request.Nodes.WorkerNodes()[0].Config.RawV1Alpha1().MachineConfig.Kubelet().Image()) + }) + + // Most of the logic is in the post create part so these two are just smoke tests + addFieldTest("TestPostCreateFields", n( + &tf, &tf.ApplyConfigEnabled, &tf.ClusterWait, &tf.ClusterWaitTimeout, &tf.WithInitNode, &tf.Talosconfig, + ), func(t *testing.T) { + input := getTestOps() + input.ApplyConfigEnabled = true + input.ClusterWait = true + input.ClusterWaitTimeout = 1000 + input.WithInitNode = true + input.Talosconfig = "test-conf" + + getFinalizedClusterMaker(t, input) + }) + addFieldTest("TestPostCreateSkipFields", n(&tf, &tf.SkipInjectingConfig, &tf.SkipK8sNodeReadinessCheck, &tf.SkipKubeconfig), func(t *testing.T) { + input := getTestOps() + input.SkipK8sNodeReadinessCheck = true + input.SkipKubeconfig = true + + getFinalizedClusterMaker(t, input) + }) + + addFieldTest("", n(&tf, &tf.NetworkIPv6), func(t *testing.T) { + input := getTestOps() + input.NetworkIPv6 = true + + cm := getFinalizedClusterMaker(t, input) + + nodes := slices.Concat(cm.request.Nodes.ControlPlaneNodes(), cm.request.Nodes.WorkerNodes()) + + for _, n := range nodes { + assert.Equal(t, 2, len(n.IPs)) + assert.Equal(t, true, n.IPs[1].Is6()) + } + + assert.Equal(t, "10.5.0.2", nodes[0].IPs[0].String()) + assert.Equal(t, "fd74:616c:a05::2", nodes[0].IPs[1].String()) + assert.Equal(t, "10.5.0.5", nodes[3].IPs[0].String()) + assert.Equal(t, "fd74:616c:a05::5", nodes[3].IPs[1].String()) + }) + addFieldTest("", n(&tf, &tf.InputDir), func(t *testing.T) { + dir := t.TempDir() + input := getTestOps() + input.RootOps.StateDir = dir + input.RegistryMirrors = []string{"test.test=https://test.mirror"} + cm := getFinalizedClusterMaker(t, input) + + err := cm.configBundle.Write(dir, encoder.CommentsDisabled, machine.TypeControlPlane, machine.TypeWorker) + assert.NoError(t, err) + + input = getTestOps() + input.InputDir = dir + cm = getFinalizedClusterMaker(t, input) + + assert.EqualValues(t, "https://test.mirror", cm.request.Nodes.WorkerNodes()[0].Config.RawV1Alpha1().MachineConfig.Registries().Mirrors()["test.test"].Endpoints()[0]) + assert.EqualValues(t, "https://test.mirror", cm.request.Nodes.ControlPlaneNodes()[0].Config.RawV1Alpha1().MachineConfig.Registries().Mirrors()["test.test"].Endpoints()[0]) + }) +} + +func getFinalizedClusterMaker(t *testing.T, input Options) clusterMaker { + cm, err := newClusterMaker(Input{input, testProvisioner{}, testTalosVersion}) + assert.NoError(t, err) + err = cm.finalizeRequest() + assert.NoError(t, err) + + return cm +} + +func getGenOpts(t *testing.T, input Options) generate.Options { + cm, err := newClusterMaker(Input{input, testProvisioner{}, testTalosVersion}) + assert.NoError(t, err) + err = cm.finalizeRequest() + assert.NoError(t, err) + result, err := generate.NewInput(input.RootOps.ClusterName, "cluster.endpoint", "k8sv1", cm.genOpts...) + assert.NoError(t, err) + + return result.Options +} + +func addFieldTest(testName string, fieldNames []string, test func(t *testing.T)) { + testedFields = append(testedFields, fieldNames...) + name := fmt.Sprintf("Test%sField", fieldNames[0]) + + if testName != "" { + name = testName + } + + if len(fieldNames) > 1 && testName == "" { + panic("no test name specified with multiple fields tested") + } + + fieldTests = append(fieldTests, fieldTest{fieldNames, test, name}) +} + +var fieldTests []fieldTest + +type fieldTest = struct { + fieldNames []string + test func(t *testing.T) + name string +} + +var testedFields = []string{} + +func TestAllOptionFields(t *testing.T) { + for _, fieldTest := range fieldTests { + t.Run(fieldTest.name, fieldTest.test) + } + + type testField = struct { + name string + tested bool + } + + typeof := reflect.TypeOf(Options{}) + allFields := make([]testField, typeof.NumField()) + + for i := range typeof.NumField() { + allFields[i].name = typeof.Field(i).Name + } + + for _, testedField := range testedFields { + allFields = xslices.Map(allFields, func(tf testField) testField { + if tf.name == testedField { + tf.tested = true + } + + return tf + }) + } + + untested := xslices.Filter(allFields, func(tf testField) bool { return !tf.tested }) + untestedNames := xslices.Map(untested, func(tf testField) string { return tf.name }) + assert.Equal(t, 0, len(untested), "all fields of Options struct need to be tested. Untested fields: ", untestedNames) +} + +func TestProvisionerGenOptions(t *testing.T) { + input := getTestOps() + + options := getGenOpts(t, input) + + assert.EqualValues(t, "testname", options.CNIConfig.CNIName) +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/types.go b/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/types.go new file mode 100644 index 0000000000..ef498e04fa --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/types.go @@ -0,0 +1,86 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package clustermaker + +import ( + "context" + "net/netip" + "time" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/bundle" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/provision" +) + +// PartialClusterRequest is the provision.ClusterRequest with only common provider options applied. +// PartialClusterRequest can be modified and then has to be handed to CreateCluster. +type PartialClusterRequest provision.ClusterRequest + +// ClusterMaker is an abstraction around cluster creation. +type ClusterMaker interface { + // GetPartialClusterRequest returns a partially rendered cluster request. + // This request can be medified and then has to be passed to CreateCluster, + // which adds final configs and creates the cluster. + GetPartialClusterRequest() PartialClusterRequest + GetVersionContract() *config.VersionContract + GetCIDR4() netip.Prefix + + AddGenOps(opts ...generate.Option) + AddProvisionOps(opts ...provision.Option) + AddCfgBundleOpts(opts ...bundle.Option) + + // SetInClusterEndpoint can be optionally used to override the in cluster endpoint. + SetInClusterEndpoint(endpoint string) + + // CreateCluster finalizes the clusterRequest by rendering and applying configs, + // after which it creates the cluster via the provisioner. + CreateCluster(ctx context.Context, request PartialClusterRequest) error + PostCreate(ctx context.Context) error +} + +// Options to make a cluster. +type Options struct { + // RootOps are the options from the root cluster command + RootOps *cluster.CmdOps + Talosconfig string + RegistryMirrors []string + RegistryInsecure []string + KubernetesVersion string + ApplyConfigEnabled bool + ConfigDebug bool + NetworkCIDR string + NetworkMTU int + NetworkIPv4 bool + DNSDomain string + Workers int + Controlplanes int + ControlPlaneCpus string + WorkersCpus string + ControlPlaneMemory int + WorkersMemory int + ClusterWait bool + ClusterWaitTimeout time.Duration + ForceInitNodeAsEndpoint bool + ForceEndpoint string + InputDir string + ControlPlanePort int + WithInitNode bool + CustomCNIUrl string + SkipKubeconfig bool + SkipInjectingConfig bool + TalosVersion string + EnableKubeSpan bool + EnableClusterDiscovery bool + ConfigPatch []string + ConfigPatchControlPlane []string + ConfigPatchWorker []string + KubePrismPort int + SkipK8sNodeReadinessCheck bool + WithJSONLogs bool + WireguardCIDR string + NetworkIPv6 bool +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/util.go b/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/util.go new file mode 100644 index 0000000000..76dfa4df5c --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker/util.go @@ -0,0 +1,128 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package clustermaker + +import ( + "context" + "errors" + "fmt" + "math/big" + "net/netip" + "os" + + "github.com/siderolabs/go-kubeconfig" + "k8s.io/client-go/tools/clientcmd" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/mgmt/helpers" + clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/provision/access" +) + +const ( + // gatewayOffset is the offset from the network address of the IP address of the network gateway. + gatewayOffset = 1 + + // nodesOffset is the offset from the network address of the beginning of the IP addresses to be used for nodes. + nodesOffset = 2 + jsonLogsPort = 4003 +) + +func patchWireguard(wireguardConfigBundle *helpers.WireguardConfigBundle, cfg config.Provider, nodeIPs []netip.Addr) (config.Provider, error) { + if wireguardConfigBundle != nil { + return wireguardConfigBundle.PatchConfig(nodeIPs[0], cfg) + } + + return cfg, nil +} + +func saveConfig(talosConfigObj *clientconfig.Config, commonOps Options) (err error) { + c, err := clientconfig.Open(commonOps.Talosconfig) + if err != nil { + return fmt.Errorf("error opening talos config: %w", err) + } + + renames := c.Merge(talosConfigObj) + for _, rename := range renames { + fmt.Fprintf(os.Stderr, "renamed talosconfig context %s\n", rename.String()) + } + + return c.Save(commonOps.Talosconfig) +} + +func mergeKubeconfig(ctx context.Context, clusterAccess *access.Adapter) error { + kubeconfigPath, err := kubeconfig.DefaultPath() + if err != nil { + return err + } + + fmt.Fprintf(os.Stderr, "\nmerging kubeconfig into %q\n", kubeconfigPath) + + k8sconfig, err := clusterAccess.Kubeconfig(ctx) + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeConfig, err := clientcmd.Load(k8sconfig) + if err != nil { + return fmt.Errorf("error parsing kubeconfig: %w", err) + } + + if clusterAccess.ForceEndpoint != "" { + for name := range kubeConfig.Clusters { + kubeConfig.Clusters[name].Server = clusterAccess.ForceEndpoint + } + } + + _, err = os.Stat(kubeconfigPath) + if err != nil { + if !os.IsNotExist(err) { + return err + } + + return clientcmd.WriteToFile(*kubeConfig, kubeconfigPath) + } + + merger, err := kubeconfig.Load(kubeconfigPath) + if err != nil { + return fmt.Errorf("error loading existing kubeconfig: %w", err) + } + + err = merger.Merge(kubeConfig, kubeconfig.MergeOptions{ + ActivateContext: true, + OutputWriter: os.Stdout, + ConflictHandler: func(component kubeconfig.ConfigComponent, name string) (kubeconfig.ConflictDecision, error) { + return kubeconfig.RenameDecision, nil + }, + }) + if err != nil { + return fmt.Errorf("error merging kubeconfig: %w", err) + } + + return merger.Write(kubeconfigPath) +} + +func parseCPUShare(cpus string) (int64, error) { + cpu, ok := new(big.Rat).SetString(cpus) + if !ok { + return 0, fmt.Errorf("failed to parsing as a rational number: %s", cpus) + } + + nano := cpu.Mul(cpu, big.NewRat(1e9, 1)) + if !nano.IsInt() { + return 0, errors.New("value is too precise") + } + + return nano.Num().Int64(), nil +} + +func getNodeIP(cidrs []netip.Prefix, ips [][]netip.Addr, nodeIndex int) []netip.Addr { + nodeIPs := make([]netip.Addr, len(cidrs)) + for j := range nodeIPs { + nodeIPs[j] = ips[j][nodeIndex] + } + + return nodeIPs +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/create.go b/cmd/talosctl/cmd/mgmt/cluster/create/create.go new file mode 100644 index 0000000000..cf1c43d5e1 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/create.go @@ -0,0 +1,526 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create + +import ( + "context" + "fmt" + "path/filepath" + stdruntime "runtime" + "time" + + "github.com/docker/cli/opts" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster" + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker" + "github.com/siderolabs/talos/cmd/talosctl/pkg/mgmt/helpers" + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/images" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/version" + "github.com/siderolabs/talos/pkg/provision/providers" +) + +// CommonOps are the options common across all the providers. +type commonOps = clustermaker.Options + +type qemuOps struct { + nodeInstallImage string + nodeVmlinuzPath string + nodeInitramfsPath string + nodeISOPath string + nodeUSBPath string + nodeUKIPath string + nodeDiskImagePath string + nodeIPXEBootScript string + bootloaderEnabled bool + uefiEnabled bool + tpm2Enabled bool + extraUEFISearchPaths []string + networkNoMasqueradeCIDRs []string + nameservers []string + clusterDiskSize int + diskBlockSize uint + clusterDiskPreallocate bool + clusterDisks []string + extraDisks int + extraDiskSize int + extraDisksDrivers []string + targetArch string + cniBinPath []string + cniConfDir string + cniCacheDir string + cniBundleURL string + encryptStatePartition bool + encryptEphemeralPartition bool + useVIP bool + badRTC bool + extraBootKernelArgs string + dhcpSkipHostname bool + networkChaos bool + jitter time.Duration + latency time.Duration + packetLoss float64 + packetReorder float64 + packetCorrupt float64 + bandwidth int + diskEncryptionKeyTypes []string + withFirewall string + withUUIDHostnames bool + withSiderolinkAgent agentFlag + debugShellEnabled bool + withIOMMU bool + configInjectionMethodFlagVal string +} + +type dockerOps struct { + dockerHostIP string + dockerDisableIPv6 bool + mountOpts opts.MountOpt + ports string + nodeImage string +} + +type createOps struct { + common *commonOps + docker *dockerOps + qemu *qemuOps +} + +type createFlags struct { + common *pflag.FlagSet + docker *pflag.FlagSet + qemu *pflag.FlagSet +} + +func getCreateCommand() *cobra.Command { + const ( + dockerHostIPFlag = "docker-host-ip" + nodeImageFlag = "image" + portsFlag = "exposed-ports" + dockerDisableIPv6Flag = "docker-disable-ipv6" + mountOptsFlag = "mount" + inputDirFlag = "input-dir" + networkIPv4Flag = "ipv4" + networkIPv6Flag = "ipv6" + networkMTUFlag = "mtu" + networkCIDRFlag = "cidr" + networkNoMasqueradeCIDRsFlag = "no-masquerade-cidrs" + nameserversFlag = "nameservers" + clusterDiskPreallocateFlag = "disk-preallocate" + clusterDisksFlag = "user-disk" + clusterDiskSizeFlag = "disk" + useVIPFlag = "use-vip" + bootloaderEnabledFlag = "with-bootloader" + controlPlanePortFlag = "control-plane-port" + firewallFlag = "with-firewall" + tpm2EnabledFlag = "with-tpm2" + withDebugShellFlag = "with-debug-shell" + withIOMMUFlag = "with-iommu" + talosconfigFlag = "talosconfig" + applyConfigEnabledFlag = "with-apply-config" + wireguardCIDRFlag = "wireguard-cidr" + workersFlag = "workers" + mastersFlag = "masters" + controlplanesFlag = "controlplanes" + controlPlaneCpusFlag = "cpus" + workersCpusFlag = "cpus-workers" + controlPlaneMemoryFlag = "memory" + workersMemoryFlag = "memory-workers" + clusterWaitFlag = "wait" + clusterWaitTimeoutFlag = "wait-timeout" + forceInitNodeAsEndpointFlag = "init-node-as-endpoint" + kubernetesVersionFlag = "kubernetes-version" + withInitNodeFlag = "with-init-node" + skipKubeconfigFlag = "skip-kubeconfig" + skipInjectingConfigFlag = "skip-injecting-config" + configPatchFlag = "config-patch" + configPatchControlPlaneFlag = "config-patch-control-plane" + configPatchWorkerFlag = "config-patch-worker" + skipK8sNodeReadinessCheckFlag = "skip-k8s-node-readiness-check" + withJSONLogsFlag = "with-json-logs" + nodeVmlinuzPathFlag = "vmlinuz-path" + nodeISOPathFlag = "iso-path" + nodeUSBPathFlag = "usb-path" + nodeUKIPathFlag = "uki-path" + nodeInitramfsPathFlag = "initrd-path" + nodeDiskImagePathFlag = "disk-image-path" + nodeIPXEBootScriptFlag = "ipxe-boot-script" + uefiEnabledFlag = "with-uefi" + extraUEFISearchPathsFlag = "extra-uefi-search-paths" + extraDisksFlag = "extra-disks" + extraDisksDriversFlag = "extra-disks-drivers" + extraDiskSizeFlag = "extra-disks-size" + targetArchFlag = "arch" + cniBinPathFlag = "cni-bin-path" + cniConfDirFlag = "cni-conf-dir" + cniCacheDirFlag = "cni-cache-dir" + cniBundleURLFlag = "cni-bundle-url" + badRTCFlag = "bad-rtc" + extraBootKernelArgsFlag = "extra-boot-kernel-args" + dhcpSkipHostnameFlag = "disable-dhcp-hostname" + networkChaosFlag = "with-network-chaos" + jitterFlag = "with-network-jitter" + latencyFlag = "with-network-latency" + packetLossFlag = "with-network-packet-loss" + packetReorderFlag = "with-network-packet-reorder" + packetCorruptFlag = "with-network-packet-corrupt" + bandwidthFlag = "with-network-bandwidth" + withUUIDHostnamesFlag = "with-uuid-hostnames" + withSiderolinkAgentFlag = "with-siderolink" + configInjectionMethodFlag = "config-injection-method" + + // The following flags are the gen options - the options that are only used in machine configuration (i.e., not during the qemu/docker provisioning). + // They are not applicable when no machine configuration is generated, hence mutually exclusive with the --input-dir flag. + + nodeInstallImageFlag = "install-image" + configDebugFlag = "with-debug" + dnsDomainFlag = "dns-domain" + withClusterDiscoveryFlag = "with-cluster-discovery" + registryMirrorFlag = "registry-mirror" + registryInsecureFlag = "registry-insecure-skip-verify" + customCNIUrlFlag = "custom-cni-url" + talosVersionFlag = "talos-version" + encryptStatePartitionFlag = "encrypt-state" + encryptEphemeralPartitionFlag = "encrypt-ephemeral" + enableKubeSpanFlag = "with-kubespan" + forceEndpointFlag = "endpoint" + kubePrismFlag = "kubeprism-port" + diskEncryptionKeyTypesFlag = "disk-encryption-key-types" + ) + + ops := createOps{ + common: &commonOps{}, + docker: &dockerOps{}, + qemu: &qemuOps{}, + } + ops.common.RootOps = &cluster.Flags + + getDockerFlags := func() *pflag.FlagSet { + dockerFlags := pflag.NewFlagSet("", pflag.PanicOnError) + dockerFlags.StringVar(&ops.docker.dockerHostIP, dockerHostIPFlag, "0.0.0.0", "Host IP to forward exposed ports to") + dockerFlags.StringVar(&ops.docker.nodeImage, nodeImageFlag, helpers.DefaultImage(images.DefaultTalosImageRepository), "the image to use") + dockerFlags.StringVarP(&ops.docker.ports, portsFlag, "p", "", + "Comma-separated list of ports/protocols to expose on init node. Ex -p :/") + dockerFlags.BoolVar(&ops.docker.dockerDisableIPv6, dockerDisableIPv6Flag, false, "skip enabling IPv6 in containers") + dockerFlags.Var(&ops.docker.mountOpts, mountOptsFlag, "attach a mount to the container") + + dockerFlags.VisitAll(func(f *pflag.Flag) { + f.Usage = "(docker only) " + f.Usage + }) + + return dockerFlags + } + + getQemuFlags := func() *pflag.FlagSet { + qemuFlags := pflag.NewFlagSet("", pflag.PanicOnError) + qemuFlags.StringVar(&ops.qemu.nodeInstallImage, nodeInstallImageFlag, helpers.DefaultImage(images.DefaultInstallerImageRepository), "the installer image to use") + qemuFlags.StringVar(&ops.qemu.nodeVmlinuzPath, nodeVmlinuzPathFlag, helpers.ArtifactPath(constants.KernelAssetWithArch), "the compressed kernel image to use") + qemuFlags.StringVar(&ops.qemu.nodeISOPath, nodeISOPathFlag, "", "the ISO path to use for the initial boot") + qemuFlags.StringVar(&ops.qemu.nodeUSBPath, nodeUSBPathFlag, "", "the USB stick image path to use for the initial boot") + qemuFlags.StringVar(&ops.qemu.nodeUKIPath, nodeUKIPathFlag, "", "the UKI image path to use for the initial boot") + qemuFlags.StringVar(&ops.qemu.nodeInitramfsPath, nodeInitramfsPathFlag, helpers.ArtifactPath(constants.InitramfsAssetWithArch), "initramfs image to use") + qemuFlags.StringVar(&ops.qemu.nodeDiskImagePath, nodeDiskImagePathFlag, "", "disk image to use") + qemuFlags.StringVar(&ops.qemu.nodeIPXEBootScript, nodeIPXEBootScriptFlag, "", "iPXE boot script (URL) to use") + qemuFlags.BoolVar(&ops.qemu.bootloaderEnabled, bootloaderEnabledFlag, true, "enable bootloader to load kernel and initramfs from disk image after install") + qemuFlags.BoolVar(&ops.qemu.uefiEnabled, uefiEnabledFlag, true, "enable UEFI on x86_64 architecture") + qemuFlags.BoolVar(&ops.qemu.tpm2Enabled, tpm2EnabledFlag, false, "enable TPM2 emulation support using swtpm") + qemuFlags.BoolVar(&ops.qemu.debugShellEnabled, withDebugShellFlag, false, "drop talos into a maintenance shell on boot, this is for advanced debugging for developers only") + qemuFlags.BoolVar(&ops.qemu.withIOMMU, withIOMMUFlag, false, "enable IOMMU support, this also add a new PCI root port and an interface attached to it)") + qemuFlags.MarkHidden("with-debug-shell") //nolint:errcheck + qemuFlags.StringSliceVar(&ops.qemu.extraUEFISearchPaths, extraUEFISearchPathsFlag, []string{}, "additional search paths for UEFI firmware (only applies when UEFI is enabled)") + qemuFlags.StringSliceVar(&ops.qemu.networkNoMasqueradeCIDRs, networkNoMasqueradeCIDRsFlag, []string{}, "list of CIDRs to exclude from NAT") + // This can be set to true only with qemu, but is still passed along with common ops due to the convenience of not having to separate the otherwise same logic + qemuFlags.BoolVar(&ops.common.NetworkIPv6, networkIPv6Flag, false, "enable IPv6 network in the cluster") + qemuFlags.StringSliceVar(&ops.qemu.nameservers, nameserversFlag, []string{"8.8.8.8", "1.1.1.1", "2001:4860:4860::8888", "2606:4700:4700::1111"}, "list of nameservers to use") + qemuFlags.IntVar(&ops.qemu.clusterDiskSize, clusterDiskSizeFlag, 6*1024, "default limit on disk size in MB (each VM)") + qemuFlags.UintVar(&ops.qemu.diskBlockSize, "disk-block-size", 512, "disk block size") + qemuFlags.BoolVar(&ops.qemu.clusterDiskPreallocate, clusterDiskPreallocateFlag, true, "whether disk space should be preallocated") + qemuFlags.StringSliceVar(&ops.qemu.clusterDisks, clusterDisksFlag, []string{}, "list of disks to create for each VM in format: :::") + qemuFlags.IntVar(&ops.qemu.extraDisks, extraDisksFlag, 0, "number of extra disks to create for each worker VM") + qemuFlags.StringSliceVar(&ops.qemu.extraDisksDrivers, extraDisksDriversFlag, nil, "driver for each extra disk (virtio, ide, ahci, scsi, nvme, megaraid)") + qemuFlags.IntVar(&ops.qemu.extraDiskSize, extraDiskSizeFlag, 5*1024, "default limit on disk size in MB (each VM)") + qemuFlags.StringVar(&ops.qemu.targetArch, targetArchFlag, stdruntime.GOARCH, "cluster architecture") + qemuFlags.StringSliceVar(&ops.qemu.cniBinPath, cniBinPathFlag, []string{filepath.Join(cluster.DefaultCNIDir, "bin")}, "search path for CNI binaries") + qemuFlags.StringVar(&ops.qemu.cniConfDir, cniConfDirFlag, filepath.Join(cluster.DefaultCNIDir, "conf.d"), "CNI config directory path") + qemuFlags.StringVar(&ops.qemu.cniCacheDir, cniCacheDirFlag, filepath.Join(cluster.DefaultCNIDir, "cache"), "CNI cache directory path") + qemuFlags.StringVar(&ops.qemu.cniBundleURL, cniBundleURLFlag, fmt.Sprintf("https://github.com/%s/talos/releases/download/%s/talosctl-cni-bundle-%s.tar.gz", + images.Username, version.Trim(version.Tag), constants.ArchVariable), "URL to download CNI bundle from") + qemuFlags.BoolVar(&ops.qemu.encryptStatePartition, encryptStatePartitionFlag, false, "enable state partition encryption") + qemuFlags.BoolVar(&ops.qemu.encryptEphemeralPartition, encryptEphemeralPartitionFlag, false, "enable ephemeral partition encryption") + qemuFlags.StringArrayVar(&ops.qemu.diskEncryptionKeyTypes, diskEncryptionKeyTypesFlag, []string{"uuid"}, "encryption key types to use for disk encryption (uuid, kms)") + qemuFlags.BoolVar(&ops.qemu.useVIP, useVIPFlag, false, "use a virtual IP for the controlplane endpoint instead of the loadbalancer") + qemuFlags.BoolVar(&ops.qemu.badRTC, badRTCFlag, false, "launch VM with bad RTC state") + qemuFlags.StringVar(&ops.qemu.extraBootKernelArgs, extraBootKernelArgsFlag, "", "add extra kernel args to the initial boot from vmlinuz and initramfs") + qemuFlags.BoolVar(&ops.qemu.dhcpSkipHostname, dhcpSkipHostnameFlag, false, "skip announcing hostname via DHCP") + qemuFlags.BoolVar(&ops.qemu.networkChaos, networkChaosFlag, false, "enable network chaos parameters when creating a qemu cluster") + qemuFlags.DurationVar(&ops.qemu.jitter, jitterFlag, 0, "specify jitter on the bridge interface") + qemuFlags.DurationVar(&ops.qemu.latency, latencyFlag, 0, "specify latency on the bridge interface") + qemuFlags.Float64Var(&ops.qemu.packetLoss, packetLossFlag, 0.0, "specify percent of packet loss on the bridge interface. e.g. 50% = 0.50 (default: 0.0)") + qemuFlags.Float64Var(&ops.qemu.packetReorder, packetReorderFlag, 0.0, "specify percent of reordered packets on the bridge interface. e.g. 50% = 0.50 (default: 0.0)") + qemuFlags.Float64Var(&ops.qemu.packetCorrupt, packetCorruptFlag, 0.0, "specify percent of corrupt packets on the bridge interface. e.g. 50% = 0.50 (default: 0.0)") + qemuFlags.IntVar(&ops.qemu.bandwidth, bandwidthFlag, 0, "specify bandwidth restriction (in kbps) on the bridge interface") + qemuFlags.StringVar(&ops.qemu.withFirewall, firewallFlag, "", "inject firewall rules into the cluster, value is default policy - accept/block") + qemuFlags.BoolVar(&ops.qemu.withUUIDHostnames, withUUIDHostnamesFlag, false, "use machine UUIDs as default hostnames") + qemuFlags.Var(&ops.qemu.withSiderolinkAgent, withSiderolinkAgentFlag, "enables the use of siderolink agent as configuration apply mechanism. `true` or `wireguard` enables the agent, `tunnel` enables the agent with grpc tunneling") //nolint:lll + qemuFlags.StringVar(&ops.qemu.configInjectionMethodFlagVal, configInjectionMethodFlag, "", "a method to inject machine config: default is HTTP server, 'metal-iso' to mount an ISO") + + qemuFlags.VisitAll(func(f *pflag.Flag) { + f.Usage = "(QEMU only) " + f.Usage + }) + + return qemuFlags + } + + getCommonFlags := func() *pflag.FlagSet { + commonFlags := pflag.NewFlagSet("", pflag.PanicOnError) + commonFlags.StringVar(&ops.common.Talosconfig, talosconfigFlag, "", + fmt.Sprintf("The path to the Talos configuration file. Defaults to '%s' env variable if set, otherwise '%s' and '%s' in order.", + constants.TalosConfigEnvVar, + filepath.Join("$HOME", constants.TalosDir, constants.TalosconfigFilename), + filepath.Join(constants.ServiceAccountMountPath, constants.TalosconfigFilename), + ), + ) + commonFlags.BoolVar(&ops.common.ApplyConfigEnabled, applyConfigEnabledFlag, false, "enable apply config when the VM is starting in maintenance mode") + commonFlags.StringSliceVar(&ops.common.RegistryMirrors, registryMirrorFlag, []string{}, "list of registry mirrors to use in format: =") + commonFlags.StringSliceVar(&ops.common.RegistryInsecure, registryInsecureFlag, []string{}, "list of registry hostnames to skip TLS verification for") + commonFlags.BoolVar(&ops.common.ConfigDebug, configDebugFlag, false, "enable debug in Talos config to send service logs to the console") + commonFlags.IntVar(&ops.common.NetworkMTU, networkMTUFlag, 1500, "MTU of the cluster network") + commonFlags.StringVar(&ops.common.NetworkCIDR, networkCIDRFlag, "10.5.0.0/24", "CIDR of the cluster network (IPv4, ULA network for IPv6 is derived in automated way)") + commonFlags.IntVar(&ops.common.ControlPlanePort, controlPlanePortFlag, constants.DefaultControlPlanePort, "control plane port (load balancer and local API port)") + commonFlags.BoolVar(&ops.common.NetworkIPv4, networkIPv4Flag, true, "enable IPv4 network in the cluster") + commonFlags.StringVar(&ops.common.WireguardCIDR, wireguardCIDRFlag, "", "CIDR of the wireguard network") + commonFlags.IntVar(&ops.common.Workers, workersFlag, 1, "the number of workers to create") + commonFlags.IntVar(&ops.common.Controlplanes, mastersFlag, 1, "the number of masters to create") + commonFlags.MarkDeprecated("commonOps.masters", "use --controlplanes instead") //nolint:errcheck + commonFlags.IntVar(&ops.common.Controlplanes, controlplanesFlag, 1, "the number of controlplanes to create") + commonFlags.StringVar(&ops.common.ControlPlaneCpus, controlPlaneCpusFlag, "2.0", "the share of CPUs as fraction (each control plane/VM)") + commonFlags.StringVar(&ops.common.WorkersCpus, workersCpusFlag, "2.0", "the share of CPUs as fraction (each worker/VM)") + commonFlags.IntVar(&ops.common.ControlPlaneMemory, controlPlaneMemoryFlag, 2048, "the limit on memory usage in MB (each control plane/VM)") + commonFlags.IntVar(&ops.common.WorkersMemory, workersMemoryFlag, 2048, "the limit on memory usage in MB (each worker/VM)") + commonFlags.BoolVar(&ops.common.ClusterWait, clusterWaitFlag, true, "wait for the cluster to be ready before returning") + commonFlags.DurationVar(&ops.common.ClusterWaitTimeout, clusterWaitTimeoutFlag, 20*time.Minute, "timeout to wait for the cluster to be ready") + commonFlags.BoolVar(&ops.common.ForceInitNodeAsEndpoint, forceInitNodeAsEndpointFlag, false, "use init node as endpoint instead of any load balancer endpoint") + commonFlags.StringVar(&ops.common.ForceEndpoint, forceEndpointFlag, "", "use endpoint instead of provider defaults") + commonFlags.StringVar(&ops.common.KubernetesVersion, kubernetesVersionFlag, constants.DefaultKubernetesVersion, "desired kubernetes version to run") + commonFlags.StringVarP(&ops.common.InputDir, inputDirFlag, "i", "", "location of pre-generated config files") + commonFlags.BoolVar(&ops.common.WithInitNode, withInitNodeFlag, false, "create the cluster with an init node") + commonFlags.StringVar(&ops.common.CustomCNIUrl, customCNIUrlFlag, "", "install custom CNI from the URL (Talos cluster)") + commonFlags.StringVar(&ops.common.DNSDomain, dnsDomainFlag, "cluster.local", "the dns domain to use for cluster") + commonFlags.BoolVar(&ops.common.SkipKubeconfig, skipKubeconfigFlag, false, "skip merging kubeconfig from the created cluster") + commonFlags.BoolVar(&ops.common.SkipInjectingConfig, skipInjectingConfigFlag, false, "skip injecting config from embedded metadata server, write config files to current directory") + commonFlags.StringVar(&ops.common.TalosVersion, talosVersionFlag, "", "the desired Talos version to generate config for (if not set, defaults to image version)") + commonFlags.BoolVar(&ops.common.EnableClusterDiscovery, withClusterDiscoveryFlag, true, "enable cluster discovery") + commonFlags.BoolVar(&ops.common.EnableKubeSpan, enableKubeSpanFlag, false, "enable KubeSpan system") + commonFlags.StringArrayVar(&ops.common.ConfigPatch, configPatchFlag, nil, "patch generated machineconfigs (applied to all node types), use @file to read a patch from file") + commonFlags.StringArrayVar(&ops.common.ConfigPatchControlPlane, configPatchControlPlaneFlag, nil, "patch generated machineconfigs (applied to 'init' and 'controlplane' types)") + commonFlags.StringArrayVar(&ops.common.ConfigPatchWorker, configPatchWorkerFlag, nil, "patch generated machineconfigs (applied to 'worker' type)") + commonFlags.IntVar(&ops.common.KubePrismPort, kubePrismFlag, constants.DefaultKubePrismPort, "KubePrism port (set to 0 to disable)") + commonFlags.BoolVar(&ops.common.SkipK8sNodeReadinessCheck, skipK8sNodeReadinessCheckFlag, false, "skip k8s node readiness checks") + commonFlags.BoolVar(&ops.common.WithJSONLogs, withJSONLogsFlag, false, "enable JSON logs receiver and configure Talos to send logs there") + + return commonFlags + } + + flags := createFlags{ + common: getCommonFlags(), + qemu: getQemuFlags(), + docker: getDockerFlags(), + } + + createQemuCmd := &cobra.Command{ + Use: providers.QemuProviderName, + Short: "Creates a local qemu based kubernetes cluster (linux only)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + provisionerFlag := cmd.Flag(cluster.ProvisionerFlag) + if err := validateCmdProvisioner(provisionerFlag, providers.QemuProviderName); err != nil { + return err + } + ops.common.RootOps.ProvisionerName = providers.QemuProviderName + if err := validateProviderFlags(ops, flags); err != nil { + return err + } + + return cli.WithContext(context.Background(), func(ctx context.Context) error { + return createQemuCluster(ctx, *ops.common, *ops.qemu) + }) + }, + } + + createDockerCmd := &cobra.Command{ + Use: providers.DockerProviderName, + Short: "Creates a local docker based kubernetes cluster", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + provisionerFlag := cmd.Flag(cluster.ProvisionerFlag) + if err := validateCmdProvisioner(provisionerFlag, providers.DockerProviderName); err != nil { + return err + } + if err := validateProviderFlags(ops, flags); err != nil { + return err + } + + return cli.WithContext(context.Background(), func(ctx context.Context) error { + return createDockerCluster(ctx, *ops.common, *ops.docker) + }) + }, + } + + createCmd := &cobra.Command{ + Use: "create", + Short: "Creates a local docker-based or QEMU-based kubernetes cluster", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if err := providers.IsValidProvider(ops.common.RootOps.ProvisionerName); err != nil { + return err + } + if err := validateProviderFlags(ops, flags); err != nil { + return err + } + + return cli.WithContext(context.Background(), func(ctx context.Context) error { + if ops.common.RootOps.ProvisionerName == providers.DockerProviderName { + return createDockerCluster(ctx, *ops.common, *ops.docker) + } + + return createQemuCluster(ctx, *ops.common, *ops.qemu) + }) + }, + } + + createCmd.Flags().AddFlagSet(flags.common) + createCmd.Flags().AddFlagSet(flags.docker) + createCmd.Flags().AddFlagSet(flags.qemu) + createDockerCmd.Flags().AddFlagSet(flags.common) + createDockerCmd.Flags().AddFlagSet(flags.docker) + createQemuCmd.Flags().AddFlagSet(flags.common) + createQemuCmd.Flags().AddFlagSet(flags.qemu) + + // The individual flagsets are still sorted + createCmd.Flags().SortFlags = false + createDockerCmd.Flags().SortFlags = false + createQemuCmd.Flags().SortFlags = false + + markInputDirFlagsExclusive := func(cmd *cobra.Command) { + exclusiveFlags := []string{ + nodeInstallImageFlag, + configDebugFlag, + dnsDomainFlag, + withClusterDiscoveryFlag, + registryMirrorFlag, + registryInsecureFlag, + customCNIUrlFlag, + talosVersionFlag, + encryptStatePartitionFlag, + encryptEphemeralPartitionFlag, + enableKubeSpanFlag, + forceEndpointFlag, + kubePrismFlag, + diskEncryptionKeyTypesFlag, + } + + for _, f := range exclusiveFlags { + if cmd.Flag(f) != nil { + cmd.MarkFlagsMutuallyExclusive(inputDirFlag, f) + } + } + } + markInputDirFlagsExclusive(createCmd) + markInputDirFlagsExclusive(createQemuCmd) + markInputDirFlagsExclusive(createDockerCmd) + + createCmd.AddCommand(createDockerCmd) + createCmd.AddCommand(createQemuCmd) + + return createCmd +} + +// validateProviderFlags checks if flags not applicable for the given provisioner are passed. +func validateProviderFlags(ops createOps, flags createFlags) error { + var invalidFlags *pflag.FlagSet + + switch ops.common.RootOps.ProvisionerName { + case providers.DockerProviderName: + invalidFlags = flags.qemu + case providers.QemuProviderName: + invalidFlags = flags.docker + } + + errMsg := "" + + invalidFlags.VisitAll(func(invalidFlag *pflag.Flag) { + if invalidFlag.Changed { + errMsg += fmt.Sprintf("%s flag has been set but has no effect with the %s provisioner\n", invalidFlag.Name, ops.common.RootOps.ProvisionerName) + } + }) + + if errMsg != "" { + fmt.Println() + + return fmt.Errorf(errMsg, "invalid provisioner flags found") + } + + return nil +} + +func init() { + createCmd := getCreateCommand() + + cluster.Cmd.AddCommand(createCmd) +} + +// validateCmdProvisioner checks if the passed provisionerFlag matches the command provisioner. +func validateCmdProvisioner(provisionerFlag *pflag.Flag, provisioner string) error { + if !provisionerFlag.Changed || + provisionerFlag.Value.String() == "" || + provisionerFlag.Value.String() == provisioner { + return nil + } + + return fmt.Errorf(`invalid provisioner: "%s" +--provisioner must be omitted or has to be "%s" when using cluster create %s`, provisionerFlag.Value.String(), provisioner, provisioner) +} + +type agentFlag uint8 + +func (a *agentFlag) String() string { + switch *a { + case 1: + return "wireguard" + case 2: + return "grpc-tunnel" + case 3: + return "wireguard+tls" + case 4: + return "grpc-tunnel+tls" + default: + return "none" + } +} + +func (a *agentFlag) Set(s string) error { + switch s { + case "true", "wireguard": + *a = 1 + case "tunnel": + *a = 2 + case "wireguard+tls": + *a = 3 + case "grpc-tunnel+tls": + *a = 4 + default: + return fmt.Errorf("unknown type: %s, possible values: 'true', 'wireguard' for the usual WG; 'tunnel' for WG over GRPC, add '+tls' to enable TLS for API", s) + } + + return nil +} + +func (a *agentFlag) Type() string { return "agent" } +func (a *agentFlag) IsEnabled() bool { return *a != 0 } +func (a *agentFlag) IsTunnel() bool { return *a == 2 || *a == 4 } +func (a *agentFlag) IsTLS() bool { return *a == 3 || *a == 4 } diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/create_test.go b/cmd/talosctl/cmd/mgmt/cluster/create/create_test.go new file mode 100644 index 0000000000..cefb690670 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/create_test.go @@ -0,0 +1,201 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create //nolint:testpackage + +import ( + "bytes" + "context" + "fmt" + "net/netip" + "testing" + + sideronet "github.com/siderolabs/net" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster" + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/bundle" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/provision" +) + +func runCmd(cmd *cobra.Command, args ...string) (*cobra.Command, string, error) { //nolint:unparam + outBuf := bytes.NewBufferString("") + cmd.SetOut(outBuf) + cmd.SetErr(outBuf) + cmd.SetArgs(args) + c, err := cmd.ExecuteC() + + return c, outBuf.String(), err +} + +func TestCreateCommandInvalidProvisioner(t *testing.T) { + _, _, err := runCmd(cluster.Cmd, "create", "--provisioner=asd") + assert.ErrorContains(t, err, "unsupported provisioner") +} + +func TestCreateCommandInvalidProvisionerFlagQemu(t *testing.T) { + _, _, err := runCmd(cluster.Cmd, "create", "--provisioner=qemu", "--docker-disable-ipv6=true") + assert.ErrorContains(t, err, "docker-disable-ipv6 flag has been set but has no effect with the qemu provisioner") +} + +func TestCreateCommandInvalidProvisionerFlagDocker(t *testing.T) { + _, _, err := runCmd(cluster.Cmd, "create", "--provisioner=docker", "--with-network-chaos=true") + assert.ErrorContains(t, err, "with-network-chaos flag has been set but has no effect with the docker provisioner") +} + +func TestCreateQemuCommandInvalidProvisionerFlag(t *testing.T) { + _, _, err := runCmd(cluster.Cmd, "create", "qemu", "--provisioner=docker") + assert.ErrorContains(t, err, "invalid provisioner") +} + +func TestCreateDockerCommandInvalidProvisionerFlag(t *testing.T) { + _, _, err := runCmd(cluster.Cmd, "create", "docker", "--provisioner=qemu") + assert.ErrorContains(t, err, "invalid provisioner") +} + +func TestCreateDockerCommandInvalidFlag(t *testing.T) { + _, _, err := runCmd(cluster.Cmd, "create", "docker", "--with-network-chaosr=true") + assert.ErrorContains(t, err, "unknown flag: --with-network-chaosr") +} + +func TestCreateQemuCommandInvalidFlag(t *testing.T) { + _, _, err := runCmd(cluster.Cmd, "create", "qemu", "--docker-disable-ipv6=true") + assert.ErrorContains(t, err, "unknown flag: --docker-disable-ipv6") +} + +func TestCreateDockerCommand(t *testing.T) { + command, _, _ := runCmd(cluster.Cmd, "create", "docker", "--with-network-chaosr=true") //nolint:errcheck + assert.Equal(t, "docker", command.Name()) +} + +func TestCreateQemuCommand(t *testing.T) { + command, _, _ := runCmd(cluster.Cmd, "create", "qemu", "--docker-disable-ipv6=true") //nolint:errcheck + assert.Equal(t, "qemu", command.Name()) +} + +type testClusterMaker struct { + provisionOpts []provision.Option + cfgBundleOpts []bundle.Option + genOpts []generate.Option + cidr4 netip.Prefix + versionContract *config.VersionContract + inClusterEndpoint string + + partialReq provision.ClusterRequest + finalReq provision.ClusterRequest + + postCreateCalled bool +} + +func (cm *testClusterMaker) GetPartialClusterRequest() clustermaker.PartialClusterRequest { + return clustermaker.PartialClusterRequest(cm.partialReq) +} + +func (cm *testClusterMaker) AddGenOps(opts ...generate.Option) { + cm.genOpts = append(cm.genOpts, opts...) +} + +func (cm *testClusterMaker) AddProvisionOps(opts ...provision.Option) { + cm.provisionOpts = append(cm.provisionOpts, opts...) +} + +func (cm *testClusterMaker) AddCfgBundleOpts(opts ...bundle.Option) { + cm.cfgBundleOpts = append(cm.cfgBundleOpts, opts...) +} + +func (cm *testClusterMaker) SetInClusterEndpoint(endpoint string) { + cm.inClusterEndpoint = endpoint +} + +func (cm *testClusterMaker) CreateCluster(ctx context.Context, request clustermaker.PartialClusterRequest) error { + cm.finalReq = provision.ClusterRequest(request) + + return nil +} + +func (cm *testClusterMaker) GetCIDR4() netip.Prefix { + return cm.cidr4 +} + +func (cm *testClusterMaker) GetVersionContract() *config.VersionContract { + return cm.versionContract +} + +func (cm *testClusterMaker) PostCreate(ctx context.Context) error { + cm.postCreateCalled = true + + return nil +} + +func (cm *testClusterMaker) getProvisionOpts() (*provision.Options, error) { + options := provision.Options{} + + for _, opt := range cm.provisionOpts { + if err := opt(&options); err != nil { + return nil, err + } + } + + return &options, nil +} + +func (cm *testClusterMaker) getCfgBundleOpts(t *testing.T) bundle.Options { + options := bundle.Options{} + for _, opt := range cm.cfgBundleOpts { + if err := opt(&options); err != nil { + t.Error("failed to apply option: ", err) + } + } + + return options +} + +func getTestClustermaker() testClusterMaker { + cidr4, err := netip.ParsePrefix("10.50.0.0/24") + if err != nil { + panic(err) + } + + totalNodes := 0 + + getTestNode := func(isControl bool) provision.NodeRequest { + ip, err := sideronet.NthIPInNetwork(cidr4, totalNodes+1) + if err != nil { + panic(err) + } + + nodeType := machine.TypeControlPlane + if !isControl { + nodeType = machine.TypeWorker + } + + node := provision.NodeRequest{ + Name: fmt.Sprint("test-node-", totalNodes), + IPs: []netip.Addr{ip}, + Type: nodeType, + } + totalNodes++ + + return node + } + + return testClusterMaker{ + cidr4: cidr4, + partialReq: provision.ClusterRequest{ + Name: "test-cluster", + Network: provision.NetworkRequest{ + Name: "test-cluster", + CIDRs: []netip.Prefix{cidr4}, + }, + Nodes: provision.NodeRequests{ + getTestNode(true), getTestNode(true), getTestNode(false), getTestNode(false), + }, + }, + } +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/docker.go b/cmd/talosctl/cmd/mgmt/cluster/create/docker.go new file mode 100644 index 0000000000..bc2950d897 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/docker.go @@ -0,0 +1,74 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create + +import ( + "context" + "fmt" + "strings" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker" + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/providers/docker" +) + +func createDockerCluster(ctx context.Context, cOps commonOps, dOps dockerOps) error { + provisioner, err := docker.NewProvisioner(ctx) + if err != nil { + return err + } + + defer func() { + if err := provisioner.Close(); err != nil { + fmt.Printf("failed to close docker provisioner: %v", err) + } + }() + + maker, err := getDockerClusterMaker(cOps, dOps, provisioner) + if err != nil { + return err + } + + return _createDockerCluster(ctx, dOps, maker) +} + +func getDockerClusterMaker(cOps commonOps, dOps dockerOps, provisioner provision.Provisioner) ( + clustermaker.ClusterMaker, error, +) { + talosversion := cOps.TalosVersion + if talosversion == "" { + parts := strings.Split(dOps.nodeImage, ":") + talosversion = parts[len(parts)-1] + } + + return clustermaker.New(clustermaker.Input{ + Ops: cOps, + Provisioner: provisioner, + TalosVersion: talosversion, + }) +} + +func _createDockerCluster(ctx context.Context, dOps dockerOps, cm clustermaker.ClusterMaker) error { + clusterReq := cm.GetPartialClusterRequest() + cm.AddProvisionOps(provision.WithDockerPortsHostIP(dOps.dockerHostIP)) + + if dOps.ports != "" { + portList := strings.Split(dOps.ports, ",") + cm.AddProvisionOps(provision.WithDockerPorts(portList)) + } + + clusterReq.Image = dOps.nodeImage + clusterReq.Network.DockerDisableIPv6 = dOps.dockerDisableIPv6 + + for i := range clusterReq.Nodes { + clusterReq.Nodes[i].Mounts = dOps.mountOpts.Value() + } + + if err := cm.CreateCluster(ctx, clusterReq); err != nil { + return err + } + + return cm.PostCreate(ctx) +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/docker_test.go b/cmd/talosctl/cmd/mgmt/cluster/create/docker_test.go new file mode 100644 index 0000000000..c78d26a748 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/docker_test.go @@ -0,0 +1,68 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create //nolint:testpackage + +import ( + "context" + "testing" + + "github.com/docker/cli/opts" + "github.com/stretchr/testify/assert" +) + +func TestNodeImageParam(t *testing.T) { + cm := getTestClustermaker() + + err := _createDockerCluster(context.Background(), dockerOps{nodeImage: "test-image"}, &cm) + assert.NoError(t, err) + + assert.Equal(t, "test-image", cm.finalReq.Image) +} + +func TestHostIpParam(t *testing.T) { + cm := getTestClustermaker() + + err := _createDockerCluster(context.Background(), dockerOps{dockerHostIP: "1.1.1.1"}, &cm) + assert.NoError(t, err) + result, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Equal(t, "1.1.1.1", result.DockerPortsHostIP) +} + +func TestPortsParam(t *testing.T) { + cm := getTestClustermaker() + + err := _createDockerCluster(context.Background(), dockerOps{ports: "20:20,33:30"}, &cm) + assert.NoError(t, err) + result, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Equal(t, []string{"20:20", "33:30"}, result.DockerPorts) +} + +func TestMountsParam(t *testing.T) { + cm := getTestClustermaker() + mount := opts.MountOpt{} + err := mount.Set("type=tmpfs,destination=/run") + assert.NoError(t, err) + + err = _createDockerCluster(context.Background(), dockerOps{mountOpts: mount}, &cm) + assert.NoError(t, err) + + assert.Equal(t, 1, len(cm.finalReq.Nodes[0].Mounts)) + assert.Equal(t, "/run", cm.finalReq.Nodes[0].Mounts[0].Target) + assert.Equal(t, 1, len(cm.finalReq.Nodes[3].Mounts)) + assert.Equal(t, "/run", cm.finalReq.Nodes[3].Mounts[0].Target) +} + +func TestDisableIPv6Param(t *testing.T) { + cm := getTestClustermaker() + + err := _createDockerCluster(context.Background(), dockerOps{dockerDisableIPv6: true}, &cm) + assert.NoError(t, err) + + assert.Equal(t, true, cm.finalReq.Network.DockerDisableIPv6) +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/qemu.go b/cmd/talosctl/cmd/mgmt/cluster/create/qemu.go new file mode 100644 index 0000000000..689a74ac1a --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/qemu.go @@ -0,0 +1,842 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "net" + "net/netip" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/google/uuid" + "github.com/hashicorp/go-getter/v2" + "github.com/klauspost/compress/zstd" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/go-blockdevice/v2/encryption" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + sideronet "github.com/siderolabs/net" + "github.com/siderolabs/siderolink/pkg/wireguard" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker" + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/internal/firewallpatch" + clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" + "github.com/siderolabs/talos/pkg/machinery/config/bundle" + "github.com/siderolabs/talos/pkg/machinery/config/configloader" + "github.com/siderolabs/talos/pkg/machinery/config/configpatcher" + "github.com/siderolabs/talos/pkg/machinery/config/encoder" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/types/security" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/imager/quirks" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/provision" +) + +// vipOffset is the offset from the network address of the CIDR to use for allocating the Virtual (shared) IP address, if enabled. +const vipOffset = 50 + +func getQemuClusterMaker(qOps qemuOps, cOps commonOps, provisioner provision.Provisioner) (clustermaker.ClusterMaker, error) { + talosversion := getQemuTalosVersion(cOps, qOps) + + return clustermaker.New(clustermaker.Input{ + Ops: cOps, + Provisioner: provisioner, + TalosVersion: talosversion, + }) +} + +func getQemuTalosVersion(cOps commonOps, qOps qemuOps) string { + talosversion := cOps.TalosVersion + if talosversion == "" { + parts := strings.Split(qOps.nodeInstallImage, ":") + talosversion = parts[len(parts)-1] + } + + return talosversion +} + +//nolint:gocyclo,cyclop +func _createQemuCluster(ctx context.Context, qOps qemuOps, cOps commonOps, provisioner provision.Provisioner, cm clustermaker.ClusterMaker) error { + clusterReq := cm.GetPartialClusterRequest() + + disks, err := getDisks(qOps) + if err != nil { + return err + } + + cm.AddProvisionOps( + provision.WithBootlader(qOps.bootloaderEnabled), + provision.WithUEFI(qOps.uefiEnabled), + provision.WithTPM2(qOps.tpm2Enabled), + provision.WithDebugShell(qOps.debugShellEnabled), + provision.WithExtraUEFISearchPaths(qOps.extraUEFISearchPaths), + provision.WithTargetArch(qOps.targetArch), + provision.WithSiderolinkAgent(qOps.withSiderolinkAgent.IsEnabled()), + provision.WithIOMMU(qOps.withIOMMU), + ) + + if qOps.withFirewall != "" { + var defaultAction nethelpers.DefaultAction + + defaultAction, err = nethelpers.DefaultActionString(qOps.withFirewall) + if err != nil { + return err + } + + var controlplaneIPs []netip.Addr + + for _, n := range clusterReq.Nodes.ControlPlaneNodes() { + controlplaneIPs = append(controlplaneIPs, n.IPs...) + } + + cm.AddCfgBundleOpts( + bundle.WithPatchControlPlane([]configpatcher.Patch{firewallpatch.ControlPlane(defaultAction, clusterReq.Network.CIDRs, clusterReq.Network.GatewayAddrs, controlplaneIPs)}), + bundle.WithPatchWorker([]configpatcher.Patch{firewallpatch.Worker(defaultAction, clusterReq.Network.CIDRs, clusterReq.Network.GatewayAddrs)}), + ) + } + + var slb *siderolinkBuilder + + if qOps.withSiderolinkAgent.IsEnabled() { + slb, err = newSiderolinkBuilder(clusterReq.Network.GatewayAddrs[0].String(), qOps.withSiderolinkAgent.IsTLS()) + if err != nil { + return err + } + } + + if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil { + trustedRootsPatch, err := configloader.NewFromBytes(trustedRootsConfig) + if err != nil { + return fmt.Errorf("error loading trusted roots config: %w", err) + } + + cm.AddCfgBundleOpts(bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(trustedRootsPatch)})) + } + + // If pre-existing talos config is not provided: + if cOps.InputDir == "" { + cm.AddGenOps(generate.WithInstallImage(qOps.nodeInstallImage)) + + if len(disks) > 1 { + // convert provision disks to machine disks + machineDisks := make([]*v1alpha1.MachineDisk, len(disks)-1) + for i, disk := range disks[1:] { + machineDisks[i] = &v1alpha1.MachineDisk{ + DeviceName: provisioner.UserDiskName(i + 1), + DiskPartitions: disk.Partitions, + } + } + + cm.AddGenOps(generate.WithUserDisks(machineDisks)) + } + + if qOps.encryptStatePartition || qOps.encryptEphemeralPartition { + diskEncryptionConfig := &v1alpha1.SystemDiskEncryptionConfig{} + + var keys []*v1alpha1.EncryptionKey + + for i, key := range qOps.diskEncryptionKeyTypes { + switch key { + case "uuid": + keys = append(keys, &v1alpha1.EncryptionKey{ + KeyNodeID: &v1alpha1.EncryptionKeyNodeID{}, + KeySlot: i, + }) + case "kms": + var ip netip.Addr + + // get bridge IP + ip, err = sideronet.NthIPInNetwork(cm.GetCIDR4(), 1) + if err != nil { + return err + } + + const port = 4050 + + keys = append(keys, &v1alpha1.EncryptionKey{ + KeyKMS: &v1alpha1.EncryptionKeyKMS{ + KMSEndpoint: "grpc://" + nethelpers.JoinHostPort(ip.String(), port), + }, + KeySlot: i, + }) + + cm.AddProvisionOps(provision.WithKMS(nethelpers.JoinHostPort("0.0.0.0", port))) + case "tpm": + keyTPM := &v1alpha1.EncryptionKeyTPM{} + + if cm.GetVersionContract().SecureBootEnrollEnforcementSupported() { + keyTPM.TPMCheckSecurebootStatusOnEnroll = pointer.To(true) + } + + keys = append(keys, &v1alpha1.EncryptionKey{ + KeyTPM: keyTPM, + KeySlot: i, + }) + default: + return fmt.Errorf("unknown key type %q", key) + } + } + + if len(keys) == 0 { + return errors.New("no disk encryption key types enabled") + } + + if qOps.encryptStatePartition { + diskEncryptionConfig.StatePartition = &v1alpha1.EncryptionConfig{ + EncryptionProvider: encryption.LUKS2, + EncryptionKeys: keys, + } + } + + if qOps.encryptEphemeralPartition { + diskEncryptionConfig.EphemeralPartition = &v1alpha1.EncryptionConfig{ + EncryptionProvider: encryption.LUKS2, + EncryptionKeys: keys, + } + } + + cm.AddGenOps(generate.WithSystemDiskEncryption(diskEncryptionConfig)) + } + + if qOps.useVIP { + vip, err := sideronet.NthIPInNetwork(clusterReq.Network.CIDRs[0], vipOffset) + if err != nil { + return fmt.Errorf("failed to get virtual IP: %w", err) + } + + cm.AddGenOps(generate.WithNetworkOptions( + v1alpha1.WithNetworkInterfaceVirtualIP(provisioner.GetFirstInterface(), vip.String()), + )) + + externalKubernetesEndpoint := "https://" + nethelpers.JoinHostPort(vip.String(), cOps.ControlPlanePort) + + cm.SetInClusterEndpoint(externalKubernetesEndpoint) + cm.AddProvisionOps(provision.WithKubernetesEndpoint(externalKubernetesEndpoint)) + } + + if !qOps.bootloaderEnabled { + // disable kexec, as this would effectively use the bootloader + cm.AddGenOps(generate.WithSysctls(map[string]string{"kernel.kexec_load_disabled": "1"})) + } + } + + fmt.Fprintln(os.Stderr, "validating CIDR and reserving IPs") + + if len(clusterReq.Network.CIDRs) == 0 { + return errors.New("neither IPv4 nor IPv6 network was enabled") + } + + // Validate network chaos flags + if !qOps.networkChaos { + if qOps.jitter != 0 || qOps.latency != 0 || qOps.packetLoss != 0 || qOps.packetReorder != 0 || qOps.packetCorrupt != 0 || qOps.bandwidth != 0 { + return errors.New("network chaos flags can only be used with --with-network-chaos") + } + } + + err = downloadBootAssets(ctx, qOps) + if err != nil { + return err + } + + networkRequest, err := getQemuNetworkRequest(clusterReq, qOps, cOps) + if err != nil { + return err + } + + // Craft cluster and node requests + clusterReq.Network = networkRequest + clusterReq.KernelPath = qOps.nodeVmlinuzPath + clusterReq.InitramfsPath = qOps.nodeInitramfsPath + clusterReq.ISOPath = qOps.nodeISOPath + clusterReq.IPXEBootScript = qOps.nodeIPXEBootScript + clusterReq.DiskImagePath = qOps.nodeDiskImagePath + clusterReq.USBPath = qOps.nodeUSBPath + clusterReq.UKIPath = qOps.nodeUKIPath + + var extraKernelArgs *procfs.Cmdline + + if qOps.extraBootKernelArgs != "" || qOps.withSiderolinkAgent.IsEnabled() { + extraKernelArgs = procfs.NewCmdline(qOps.extraBootKernelArgs) + } + + err = slb.SetKernelArgs(extraKernelArgs, qOps.withSiderolinkAgent.IsTunnel()) + if err != nil { + return err + } + + var configInjectionMethod provision.ConfigInjectionMethod + + switch qOps.configInjectionMethodFlagVal { + case "", "default", "http": + configInjectionMethod = provision.ConfigInjectionMethodHTTP + case "metal-iso": + configInjectionMethod = provision.ConfigInjectionMethodMetalISO + default: + return fmt.Errorf("unknown config injection method %q", configInjectionMethod) + } + + nodes := []provision.NodeRequest{} + + // Create the controlplane nodes. + for i, n := range clusterReq.Nodes.ControlPlaneNodes() { + nodeUUID := uuid.New() + + err = slb.DefineIPv6ForUUID(nodeUUID) + if err != nil { + return err + } + + n.Name = getQemuNodeName(clusterReq.Name, "controlplane", i+1, nodeUUID, qOps) + n.Disks = disks + n.ConfigInjectionMethod = configInjectionMethod + n.BadRTC = qOps.badRTC + n.ExtraKernelArgs = extraKernelArgs + n.UUID = pointer.To(nodeUUID) + n.Quirks = quirks.New(getQemuTalosVersion(cOps, qOps)) + + nodes = append(nodes, n) + } + + // append extra worker disks + for i := range qOps.extraDisks { + driver := "ide" + + // ide driver is not supported on arm64 + if qOps.targetArch == "arm64" { + driver = "virtio" + } + + if i < len(qOps.extraDisksDrivers) { + driver = qOps.extraDisksDrivers[i] + } + + disks = append(disks, &provision.Disk{ + Size: uint64(qOps.extraDiskSize) * 1024 * 1024, + SkipPreallocate: !qOps.clusterDiskPreallocate, + Driver: driver, + }) + } + + for i, n := range clusterReq.Nodes.WorkerNodes() { + nodeUUID := uuid.New() + + err = slb.DefineIPv6ForUUID(nodeUUID) + if err != nil { + return err + } + + n.Name = getQemuNodeName(clusterReq.Name, "worker", i+1, nodeUUID, qOps) + n.Disks = disks + n.ConfigInjectionMethod = configInjectionMethod + n.BadRTC = qOps.badRTC + n.ExtraKernelArgs = extraKernelArgs + n.UUID = pointer.To(nodeUUID) + n.Quirks = quirks.New(getQemuTalosVersion(cOps, qOps)) + + nodes = append(nodes, n) + } + + clusterReq.Nodes = nodes + + clusterReq.SiderolinkRequest = slb.SiderolinkRequest() + + err = cm.CreateCluster(ctx, clusterReq) + if err != nil { + return err + } + + if qOps.debugShellEnabled { + fmt.Println("You can now connect to debug shell on any node using these commands:") + + for _, node := range nodes { + talosDir, err := clientconfig.GetTalosDirectory() + if err != nil { + return err + } + + fmt.Printf("socat - UNIX-CONNECT:%s\n", filepath.Join(talosDir, "clusters", clusterReq.Name, node.Name+".serial")) + } + + return nil + } + + return cm.PostCreate(ctx) +} + +//nolint:gocyclo +func downloadBootAssets(ctx context.Context, qOps qemuOps) error { + // download & cache images if provides as URLs + for _, downloadableImage := range []struct { + path *string + disableArchive bool + }{ + { + path: &qOps.nodeVmlinuzPath, + }, + { + path: &qOps.nodeInitramfsPath, + disableArchive: true, + }, + { + path: &qOps.nodeISOPath, + }, + { + path: &qOps.nodeUKIPath, + }, + { + path: &qOps.nodeUSBPath, + }, + { + path: &qOps.nodeDiskImagePath, + }, + } { + if *downloadableImage.path == "" { + continue + } + + u, err := url.Parse(*downloadableImage.path) + if err != nil || !(u.Scheme == "http" || u.Scheme == "https") { + // not a URL + continue + } + + defaultStateDir, err := clientconfig.GetTalosDirectory() + if err != nil { + return err + } + + cacheDir := filepath.Join(defaultStateDir, "cache") + + if os.MkdirAll(cacheDir, 0o755) != nil { + return err + } + + destPath := strings.ReplaceAll( + strings.ReplaceAll(u.String(), "/", "-"), + ":", "-") + + _, err = os.Stat(filepath.Join(cacheDir, destPath)) + if err == nil { + *downloadableImage.path = filepath.Join(cacheDir, destPath) + + // already cached + continue + } + + fmt.Fprintf(os.Stderr, "downloading asset from %q to %q\n", u.String(), filepath.Join(cacheDir, destPath)) + + client := getter.Client{ + Getters: []getter.Getter{ + &getter.HttpGetter{ + HeadFirstTimeout: 30 * time.Minute, + ReadTimeout: 30 * time.Minute, + }, + }, + } + + if downloadableImage.disableArchive { + q := u.Query() + + q.Set("archive", "false") + + u.RawQuery = q.Encode() + } + + _, err = client.Get(ctx, &getter.Request{ + Src: u.String(), + Dst: filepath.Join(cacheDir, destPath), + GetMode: getter.ModeFile, + }) + if err != nil { + // clean up the destination on failure + os.Remove(filepath.Join(cacheDir, destPath)) //nolint:errcheck + + return err + } + + *downloadableImage.path = filepath.Join(cacheDir, destPath) + } + + return nil +} + +func getDisks(qemuOps qemuOps) ([]*provision.Disk, error) { + const GPTAlignment = 2 * 1024 * 1024 // 2 MB + + // should have at least a single primary disk + disks := []*provision.Disk{ + { + Size: uint64(qemuOps.clusterDiskSize) * 1024 * 1024, + SkipPreallocate: !qemuOps.clusterDiskPreallocate, + Driver: "virtio", + BlockSize: qemuOps.diskBlockSize, + }, + } + + for _, disk := range qemuOps.clusterDisks { + var ( + partitions = strings.Split(disk, ":") + diskPartitions = make([]*v1alpha1.DiskPartition, len(partitions)/2) + diskSize uint64 + ) + + if len(partitions)%2 != 0 { + return nil, errors.New("failed to parse malformed partition definitions") + } + + partitionIndex := 0 + + for j := 0; j < len(partitions); j += 2 { + partitionPath := partitions[j] + + if !strings.HasPrefix(partitionPath, "/var") { + return nil, errors.New("user disk partitions can only be mounted into /var folder") + } + + value, e := strconv.ParseInt(partitions[j+1], 10, 0) + partitionSize := uint64(value) + + if e != nil { + partitionSize, e = humanize.ParseBytes(partitions[j+1]) + + if e != nil { + return nil, errors.New("failed to parse partition size") + } + } + + diskPartitions[partitionIndex] = &v1alpha1.DiskPartition{ + DiskSize: v1alpha1.DiskSize(partitionSize), + DiskMountPoint: partitionPath, + } + diskSize += partitionSize + partitionIndex++ + } + + disks = append(disks, &provision.Disk{ + // add 2 MB per partition to make extra room for GPT and alignment + Size: diskSize + GPTAlignment*uint64(len(diskPartitions)+1), + Partitions: diskPartitions, + SkipPreallocate: !qemuOps.clusterDiskPreallocate, + Driver: "ide", + BlockSize: qemuOps.diskBlockSize, + }) + } + + return disks, nil +} + +func getQemuNodeName(clusterName, role string, index int, uuid uuid.UUID, qemuOps qemuOps) string { + if qemuOps.withUUIDHostnames { + return fmt.Sprintf("machine-%s", uuid) + } + + return fmt.Sprintf("%s-%s-%d", clusterName, role, index) +} + +func newSiderolinkBuilder(wgHost string, useTLS bool) (*siderolinkBuilder, error) { + prefix := networkPrefix("") + + result := &siderolinkBuilder{ + wgHost: wgHost, + binds: map[uuid.UUID]netip.Addr{}, + prefix: prefix, + nodeIPv6Addr: prefix.Addr().Next().String(), + } + + if useTLS { + ca, err := x509.NewSelfSignedCertificateAuthority(x509.ECDSA(true), x509.IPAddresses([]net.IP{net.ParseIP(wgHost)})) + if err != nil { + return nil, err + } + + result.apiCert = ca.CrtPEM + result.apiKey = ca.KeyPEM + } + + var resultErr error + + for range 10 { + for _, d := range []struct { + field *int + net string + what string + }{ + {&result.wgPort, "udp", "WireGuard"}, + {&result.apiPort, "tcp", "gRPC API"}, + {&result.sinkPort, "tcp", "Event Sink"}, + {&result.logPort, "tcp", "Log Receiver"}, + } { + var err error + + *d.field, err = getDynamicPort(d.net) + if err != nil { + return nil, fmt.Errorf("failed to get dynamic port for %s: %w", d.what, err) + } + } + + resultErr = checkPortsDontOverlap(result.wgPort, result.apiPort, result.sinkPort, result.logPort) + if resultErr == nil { + break + } + } + + if resultErr != nil { + return nil, fmt.Errorf("failed to get non-overlapping dynamic ports in 10 attempts: %w", resultErr) + } + + return result, nil +} + +type siderolinkBuilder struct { + wgHost string + + binds map[uuid.UUID]netip.Addr + prefix netip.Prefix + nodeIPv6Addr string + wgPort int + apiPort int + sinkPort int + logPort int + + apiCert []byte + apiKey []byte +} + +// DefineIPv6ForUUID defines an IPv6 address for a given UUID. It is safe to call this method on a nil pointer. +func (slb *siderolinkBuilder) DefineIPv6ForUUID(id uuid.UUID) error { + if slb == nil { + return nil + } + + result, err := generateRandomNodeAddr(slb.prefix) + if err != nil { + return err + } + + slb.binds[id] = result.Addr() + + return nil +} + +// SiderolinkRequest returns a SiderolinkRequest based on the current state of the builder. +// It is safe to call this method on a nil pointer. +func (slb *siderolinkBuilder) SiderolinkRequest() provision.SiderolinkRequest { + if slb == nil { + return provision.SiderolinkRequest{} + } + + return provision.SiderolinkRequest{ + WireguardEndpoint: net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.wgPort)), + APIEndpoint: ":" + strconv.Itoa(slb.apiPort), + APICertificate: slb.apiCert, + APIKey: slb.apiKey, + SinkEndpoint: ":" + strconv.Itoa(slb.sinkPort), + LogEndpoint: ":" + strconv.Itoa(slb.logPort), + SiderolinkBind: maps.ToSlice(slb.binds, func(k uuid.UUID, v netip.Addr) provision.SiderolinkBind { + return provision.SiderolinkBind{ + UUID: k, + Addr: v, + } + }), + } +} + +// TrustedRootsConfig returns the trusted roots config for the current builder. +func (slb *siderolinkBuilder) TrustedRootsConfig() []byte { + if slb == nil || slb.apiCert == nil { + return nil + } + + trustedRootsConfig := security.NewTrustedRootsConfigV1Alpha1() + trustedRootsConfig.MetaName = "siderolink-ca" + trustedRootsConfig.Certificates = string(slb.apiCert) + + marshaled, err := encoder.NewEncoder(trustedRootsConfig, encoder.WithComments(encoder.CommentsDisabled)).Encode() + if err != nil { + panic(fmt.Sprintf("failed to marshal trusted roots config: %s", err)) + } + + return marshaled +} + +// SetKernelArgs sets the kernel arguments for the current builder. It is safe to call this method on a nil pointer. +func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tunnel bool) error { + switch { + case slb == nil: + return nil + case extraKernelArgs.Get("siderolink.api") != nil, + extraKernelArgs.Get("talos.events.sink") != nil, + extraKernelArgs.Get("talos.logging.kernel") != nil: + return errors.New("siderolink kernel arguments are already set, cannot run with --with-siderolink") + default: + scheme := "grpc://" + + if slb.apiCert != nil { + scheme = "https://" + } + + apiLink := scheme + net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.apiPort)) + "?jointoken=foo" + + if tunnel { + apiLink += "&grpc_tunnel=true" + } + + extraKernelArgs.Append("siderolink.api", apiLink) + extraKernelArgs.Append("talos.events.sink", net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.sinkPort))) + extraKernelArgs.Append("talos.logging.kernel", "tcp://"+net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.logPort))) + + if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil { + var buf bytes.Buffer + + zencoder, err := zstd.NewWriter(&buf) + if err != nil { + return fmt.Errorf("failed to create zstd encoder: %w", err) + } + + _, err = zencoder.Write(trustedRootsConfig) + if err != nil { + return fmt.Errorf("failed to write zstd data: %w", err) + } + + if err = zencoder.Close(); err != nil { + return fmt.Errorf("failed to close zstd encoder: %w", err) + } + + extraKernelArgs.Append(constants.KernelParamConfigInline, base64.StdEncoding.EncodeToString(buf.Bytes())) + } + + return nil + } +} + +func getDynamicPort(network string) (int, error) { + var ( + closeFn func() error + addrFn func() net.Addr + ) + + switch network { + case "tcp", "tcp4", "tcp6": + l, err := net.Listen(network, "127.0.0.1:0") + if err != nil { + return 0, err + } + + addrFn, closeFn = l.Addr, l.Close + case "udp", "udp4", "udp6": + l, err := net.ListenPacket(network, "127.0.0.1:0") + if err != nil { + return 0, err + } + + addrFn, closeFn = l.LocalAddr, l.Close + default: + return 0, fmt.Errorf("unsupported network: %s", network) + } + + _, portStr, err := net.SplitHostPort(addrFn().String()) + if err != nil { + return 0, handleCloseErr(err, closeFn()) + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return 0, err + } + + return port, handleCloseErr(nil, closeFn()) +} + +func handleCloseErr(err error, closeErr error) error { + switch { + case err != nil && closeErr != nil: + return fmt.Errorf("error: %w, close error: %w", err, closeErr) + case err == nil && closeErr != nil: + return closeErr + case err != nil && closeErr == nil: + return err + default: + return nil + } +} + +func checkPortsDontOverlap(ports ...int) error { + slices.Sort(ports) + + if len(ports) != len(slices.Compact(ports)) { + return errors.New("generated ports overlap") + } + + return nil +} + +func generateRandomNodeAddr(prefix netip.Prefix) (netip.Prefix, error) { + return wireguard.GenerateRandomNodeAddr(prefix) +} + +func networkPrefix(prefix string) netip.Prefix { + return wireguard.NetworkPrefix(prefix) +} + +func getQemuNetworkRequest(partial clustermaker.PartialClusterRequest, qOps qemuOps, cOps commonOps) (req provision.NetworkRequest, err error) { + // Parse nameservers + req = partial.Network + nameserverIPs := make([]netip.Addr, len(qOps.nameservers)) + + for i := range nameserverIPs { + nameserverIPs[i], err = netip.ParseAddr(qOps.nameservers[i]) + if err != nil { + return req, fmt.Errorf("failed parsing nameserver IP %q: %w", qOps.nameservers[i], err) + } + } + + noMasqueradeCIDRs := make([]netip.Prefix, 0, len(qOps.networkNoMasqueradeCIDRs)) + + for _, cidr := range qOps.networkNoMasqueradeCIDRs { + var parsedCIDR netip.Prefix + + parsedCIDR, err = netip.ParsePrefix(cidr) + if err != nil { + return req, fmt.Errorf("error parsing non-masquerade CIDR %q: %w", cidr, err) + } + + noMasqueradeCIDRs = append(noMasqueradeCIDRs, parsedCIDR) + } + + req.Nameservers = nameserverIPs + req.CNI = provision.CNIConfig{ + BinPath: qOps.cniBinPath, + ConfDir: qOps.cniConfDir, + CacheDir: qOps.cniCacheDir, + + BundleURL: qOps.cniBundleURL, + } + req.LoadBalancerPorts = []int{cOps.ControlPlanePort} + req.DHCPSkipHostname = qOps.dhcpSkipHostname + req.NetworkChaos = qOps.networkChaos + req.Jitter = qOps.jitter + req.Latency = qOps.latency + req.PacketLoss = qOps.packetLoss + req.PacketReorder = qOps.packetReorder + req.PacketCorrupt = qOps.packetCorrupt + req.Bandwidth = qOps.bandwidth + req.NoMasqueradeCIDRs = noMasqueradeCIDRs + + return req, nil +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/qemu_linux.go b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_linux.go new file mode 100644 index 0000000000..380228fd10 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_linux.go @@ -0,0 +1,33 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create + +import ( + "context" + "fmt" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker" + "github.com/siderolabs/talos/pkg/provision/providers/qemu" +) + +func createQemuCluster(ctx context.Context, cOps clustermaker.Options, qOps qemuOps) error { + provisioner, err := qemu.NewProvisioner(ctx) + if err != nil { + return err + } + + defer func() { + if err := provisioner.Close(); err != nil { + fmt.Printf("failed to close qemu provisioner: %v", err) + } + }() + + cm, err := getQemuClusterMaker(qOps, cOps, provisioner) + if err != nil { + return err + } + + return _createQemuCluster(ctx, qOps, cOps, provisioner, cm) +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/qemu_other.go b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_other.go new file mode 100644 index 0000000000..b59e2b9021 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_other.go @@ -0,0 +1,18 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build !linux + +package create + +import ( + "context" + "errors" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/create/clustermaker" +) + +func createQemuCluster(_ context.Context, _ clustermaker.Options, _ qemuOps) error { + return errors.New("Qemu is only supported on linux") +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/qemu_test.go b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_test.go new file mode 100644 index 0000000000..318bbf50ef --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_test.go @@ -0,0 +1,401 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create //nolint:testpackage + +import ( + "context" + "net/netip" + "strings" + "testing" + "time" + + sideronet "github.com/siderolabs/net" + "github.com/stretchr/testify/assert" + + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/provision" +) + +type testProvisioner struct { + provision.Provisioner +} + +func prepTest() (testClusterMaker, qemuOps) { //nolint:unparam + cm := getTestClustermaker() + + return cm, qemuOps{} +} + +func getGenOpts(t *testing.T, finalizedClusterMaker testClusterMaker) generate.Options { + cm := finalizedClusterMaker + result, err := generate.NewInput(cm.finalReq.Name, "cluster.endpoint", "k8sv1", cm.genOpts...) + assert.NoError(t, err) + + return result.Options +} + +/* +// Generate options. +*/ +func TestNodeInstallImageOption(t *testing.T) { + cm, ops := prepTest() + ops.nodeInstallImage = "test-image" + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + options := getGenOpts(t, cm) + + assert.Equal(t, "test-image", options.InstallImage) +} + +func TestBootloaderEnabledOptionTrue(t *testing.T) { + cm, ops := prepTest() + ops.bootloaderEnabled = true + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + genOpts := getGenOpts(t, cm) + provisionOpts, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Equal(t, "", genOpts.Sysctls["kernel.kexec_load_disabled"]) + assert.Equal(t, true, provisionOpts.BootloaderEnabled) +} + +func TestBootloaderEnabledOptionFalse(t *testing.T) { + cm, ops := prepTest() + ops.bootloaderEnabled = false + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + genOpts := getGenOpts(t, cm) + provisionOpts, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Equal(t, "1", genOpts.Sysctls["kernel.kexec_load_disabled"]) + assert.Equal(t, false, provisionOpts.BootloaderEnabled) +} + +func TestEncriptionOptions(t *testing.T) { + cm, ops := prepTest() + ops.encryptEphemeralPartition = true + ops.encryptStatePartition = true + ops.diskEncryptionKeyTypes = []string{"kms", "uuid"} + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + genOpts := getGenOpts(t, cm) + + assert.Equal(t, 2, len(genOpts.SystemDiskEncryptionConfig.EphemeralPartition.EncryptionKeys)) + assert.Equal(t, 2, len(genOpts.SystemDiskEncryptionConfig.StatePartition.EncryptionKeys)) +} + +/* +// Provision options. +*/ +func TestTargetArchOption(t *testing.T) { + cm, ops := prepTest() + ops.targetArch = "arm64" + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + provisionOpts, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Equal(t, "arm64", provisionOpts.TargetArch) +} + +func TestWithIOMMUOption(t *testing.T) { + cm, ops := prepTest() + ops.withIOMMU = true + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + provisionOpts, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Equal(t, true, provisionOpts.IOMMUEnabled) +} + +func TestUefiEnabledOption(t *testing.T) { + cm, ops := prepTest() + ops.uefiEnabled = true + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + provisionOpts, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Equal(t, true, provisionOpts.UEFIEnabled) +} + +func TestExtraUEFISearchPathsOption(t *testing.T) { + cm, ops := prepTest() + ops.extraUEFISearchPaths = []string{"/test-1", "test-2"} + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + provisionOpts, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Equal(t, []string{"/test-1", "test-2"}, provisionOpts.ExtraUEFISearchPaths) +} + +func TestTpm2EnabledOption(t *testing.T) { + cm, ops := prepTest() + ops.tpm2Enabled = true + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + provisionOpts, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Equal(t, true, provisionOpts.TPM2Enabled) +} + +/* +// Cluster request options. +*/ +func TestSimplePathOptions(t *testing.T) { + cm, ops := prepTest() + ops.nodeVmlinuzPath = "/test-path-kernel" + ops.nodeInitramfsPath = "/test-path-initramfs" + ops.nodeISOPath = "/test-path-iso" + ops.nodeUSBPath = "/test-path-usb" + ops.nodeUKIPath = "/test-path-uki" + ops.nodeDiskImagePath = "/test-path-disk-img" + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.Equal(t, "/test-path-kernel", cm.finalReq.KernelPath) + assert.Equal(t, "/test-path-initramfs", cm.finalReq.InitramfsPath) + assert.Equal(t, "/test-path-iso", cm.finalReq.ISOPath) + assert.Equal(t, "/test-path-usb", cm.finalReq.USBPath) + assert.Equal(t, "/test-path-uki", cm.finalReq.UKIPath) + assert.Equal(t, "/test-path-disk-img", cm.finalReq.DiskImagePath) +} + +func TestNodeIPXEBootScriptOption(t *testing.T) { + cm, ops := prepTest() + ops.nodeIPXEBootScript = "test-script" + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.Equal(t, "test-script", cm.finalReq.IPXEBootScript) +} + +func TestExtraBootKernelArgsOption(t *testing.T) { + cm, ops := prepTest() + ops.extraBootKernelArgs = "test=arg" + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.Equal(t, "test=arg", cm.finalReq.Nodes[0].ExtraKernelArgs.String()) + assert.Equal(t, "test=arg", cm.finalReq.Nodes[3].ExtraKernelArgs.String()) +} + +func TestConfigInjectionMethodOption(t *testing.T) { + cm, ops := prepTest() + ops.configInjectionMethodFlagVal = "metal-iso" + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.Equal(t, provision.ConfigInjectionMethodMetalISO, cm.finalReq.Nodes[0].ConfigInjectionMethod) + assert.Equal(t, provision.ConfigInjectionMethodMetalISO, cm.finalReq.Nodes[3].ConfigInjectionMethod) +} + +func TestWithSiderolinkAgentOption(t *testing.T) { + cm, ops := prepTest() + ops.withSiderolinkAgent = 1 + gatewayIP, err := netip.ParseAddr("10.50.0.0") + assert.NoError(t, err) + + cm.partialReq.Network.GatewayAddrs = []netip.Addr{gatewayIP} + + err = _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + provisionOpts, err := cm.getProvisionOpts() + assert.NoError(t, err) + + assert.Contains(t, cm.finalReq.Nodes[0].ExtraKernelArgs.String(), "siderolink.api=grpc://10.50.0.0") + assert.Contains(t, cm.finalReq.Nodes[3].ExtraKernelArgs.String(), "siderolink.api=grpc://10.50.0.0") + assert.True(t, provisionOpts.SiderolinkEnabled) +} + +func TestWithUUIDHostnames(t *testing.T) { + cm, ops := prepTest() + ops.withUUIDHostnames = true + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.Contains(t, cm.finalReq.Nodes[0].Name, "machine-") + assert.Equal(t, 6, len(strings.Split(cm.finalReq.Nodes[0].Name, "-"))) + + assert.Contains(t, cm.finalReq.Nodes[3].Name, "machine-") + assert.Equal(t, 6, len(strings.Split(cm.finalReq.Nodes[3].Name, "-"))) +} + +func TestNoMasqueradeCIDRsOption(t *testing.T) { + cm, ops := prepTest() + ops.networkNoMasqueradeCIDRs = []string{"10.50.0.0/32"} + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + expected, err := netip.ParsePrefix("10.50.0.0/32") + if err != nil { + panic(err) + } + + assert.Equal(t, 1, len(cm.finalReq.Network.NoMasqueradeCIDRs)) + assert.Equal(t, expected, cm.finalReq.Network.NoMasqueradeCIDRs[0]) +} + +func TestDhcpSkipHostnameOption(t *testing.T) { + cm, ops := prepTest() + ops.dhcpSkipHostname = true + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.Equal(t, true, cm.finalReq.Network.DHCPSkipHostname) +} + +func TestNameserverIPsOption(t *testing.T) { + cm, ops := prepTest() + ops.nameservers = []string{"1.1.1.1", "2.2.2.2"} + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.Equal(t, 2, len(cm.finalReq.Network.Nameservers)) + assert.Equal(t, "1.1.1.1", cm.finalReq.Network.Nameservers[0].String()) + assert.Equal(t, "2.2.2.2", cm.finalReq.Network.Nameservers[1].String()) +} + +func TestBadRTCOption(t *testing.T) { + cm, ops := prepTest() + ops.badRTC = true + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.True(t, cm.finalReq.Nodes[0].BadRTC) + assert.True(t, cm.finalReq.Nodes[3].BadRTC) +} + +func TestNetworkChaosOptions(t *testing.T) { + cm, ops := prepTest() + ops.networkChaos = true + ops.jitter = time.Hour + ops.latency = time.Millisecond + ops.packetLoss = 1 + ops.packetReorder = 2 + ops.packetCorrupt = 3 + ops.bandwidth = 4 + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.Equal(t, true, cm.finalReq.Network.NetworkChaos) + assert.Equal(t, time.Hour, cm.finalReq.Network.Jitter) + assert.Equal(t, time.Millisecond, cm.finalReq.Network.Latency) + assert.EqualValues(t, 1, cm.finalReq.Network.PacketLoss) + assert.EqualValues(t, 2, cm.finalReq.Network.PacketReorder) + assert.EqualValues(t, 3, cm.finalReq.Network.PacketCorrupt) + assert.EqualValues(t, 4, cm.finalReq.Network.Bandwidth) +} + +func TestCNIOptions(t *testing.T) { + cm, ops := prepTest() + ops.cniBinPath = []string{"/test-path"} + ops.cniBundleURL = "bundle.url" + ops.cniCacheDir = "/test-cache-dir" + ops.cniConfDir = "/cni-conf" + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.Equal(t, []string{"/test-path"}, cm.finalReq.Network.CNI.BinPath) + assert.Equal(t, "bundle.url", cm.finalReq.Network.CNI.BundleURL) + assert.Equal(t, "/test-cache-dir", cm.finalReq.Network.CNI.CacheDir) + assert.Equal(t, "/cni-conf", cm.finalReq.Network.CNI.ConfDir) +} + +/* +// Other options. +*/ + +func (testProvisioner) GetFirstInterface() v1alpha1.IfaceSelector { + return v1alpha1.IfaceByName("eth0") +} + +func TestVIPOption(t *testing.T) { + cm, ops := prepTest() + ops.useVIP = true + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + provisionOpts, err := cm.getProvisionOpts() + assert.NoError(t, err) + genOpts := getGenOpts(t, cm) + assert.NoError(t, err) + + expectedIP, err := sideronet.NthIPInNetwork(cm.cidr4, vipOffset) + assert.NoError(t, err) + + assert.Equal(t, "https://"+expectedIP.String()+":0", provisionOpts.KubernetesEndpoint) + + netcfg := v1alpha1.NetworkConfig{} + for _, o := range genOpts.NetworkConfigOptions { + err := o(machine.TypeControlPlane, &netcfg) + assert.NoError(t, err) + } + + assert.NotNil(t, netcfg.NetworkInterfaces[0].DeviceVIPConfig) +} + +func TestWithFirewallOption(t *testing.T) { + cm, ops := prepTest() + gatewayIP, err := netip.ParseAddr("10.50.0.0") + assert.NoError(t, err) + + cm.partialReq.Network.GatewayAddrs = []netip.Addr{gatewayIP} + ops.withFirewall = "block" + + err = _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + cfgBudnle := cm.getCfgBundleOpts(t) + + assert.Equal(t, 1, len(cfgBudnle.PatchesControlPlane)) + assert.Equal(t, 1, len(cfgBudnle.PatchesWorker)) +} + +func TestDebugShellEnabledOptionTrue(t *testing.T) { + cm, ops := prepTest() + ops.debugShellEnabled = true + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.False(t, cm.postCreateCalled) +} + +func TestDebugShellEnabledOptionFalse(t *testing.T) { + cm, ops := prepTest() + ops.debugShellEnabled = false + + err := _createQemuCluster(context.Background(), ops, commonOps{}, testProvisioner{}, &cm) + assert.NoError(t, err) + + assert.True(t, cm.postCreateCalled) +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create_linux.go b/cmd/talosctl/cmd/mgmt/cluster/create_linux.go deleted file mode 100644 index 8d07e6048a..0000000000 --- a/cmd/talosctl/cmd/mgmt/cluster/create_linux.go +++ /dev/null @@ -1,19 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package cluster - -import ( - "net/netip" - - "github.com/siderolabs/siderolink/pkg/wireguard" -) - -func generateRandomNodeAddr(prefix netip.Prefix) (netip.Prefix, error) { - return wireguard.GenerateRandomNodeAddr(prefix) -} - -func networkPrefix(prefix string) (netip.Prefix, error) { - return wireguard.NetworkPrefix(prefix), nil -} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create_other.go b/cmd/talosctl/cmd/mgmt/cluster/create_other.go deleted file mode 100644 index ce5d868081..0000000000 --- a/cmd/talosctl/cmd/mgmt/cluster/create_other.go +++ /dev/null @@ -1,20 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -//go:build !linux - -package cluster - -import ( - "errors" - "net/netip" -) - -func generateRandomNodeAddr(prefix netip.Prefix) (netip.Prefix, error) { - return netip.Prefix{}, nil -} - -func networkPrefix(prefix string) (netip.Prefix, error) { - return netip.Prefix{}, errors.New("unsupported platform") -} diff --git a/cmd/talosctl/cmd/mgmt/cluster/destroy.go b/cmd/talosctl/cmd/mgmt/cluster/destroy.go index 1dbc35657e..ce07af6dc1 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/destroy.go +++ b/cmd/talosctl/cmd/mgmt/cluster/destroy.go @@ -32,14 +32,14 @@ var destroyCmd = &cobra.Command{ } func destroy(ctx context.Context) error { - provisioner, err := providers.Factory(ctx, provisionerName) + provisioner, err := providers.Factory(ctx, Flags.ProvisionerName) if err != nil { return err } defer provisioner.Close() //nolint:errcheck - cluster, err := provisioner.Reflect(ctx, clusterName, stateDir) + cluster, err := provisioner.Reflect(ctx, Flags.ClusterName, Flags.StateDir) if err != nil { return err } diff --git a/cmd/talosctl/cmd/mgmt/cluster/show.go b/cmd/talosctl/cmd/mgmt/cluster/show.go index cfc088af7d..1633b5d588 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/show.go +++ b/cmd/talosctl/cmd/mgmt/cluster/show.go @@ -35,22 +35,23 @@ var showCmd = &cobra.Command{ } func show(ctx context.Context) error { - provisioner, err := providers.Factory(ctx, provisionerName) + provisioner, err := providers.Factory(ctx, Flags.ProvisionerName) if err != nil { return err } defer provisioner.Close() //nolint:errcheck - cluster, err := provisioner.Reflect(ctx, clusterName, stateDir) + cluster, err := provisioner.Reflect(ctx, Flags.ClusterName, Flags.StateDir) if err != nil { return err } - return showCluster(cluster) + return ShowCluster(cluster) } -func showCluster(cluster provision.Cluster) error { +// ShowCluster prints the details about the cluster. +func ShowCluster(cluster provision.Cluster) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) fmt.Fprintf(w, "PROVISIONER\t%s\n", cluster.Provisioner()) fmt.Fprintf(w, "NAME\t%s\n", cluster.Info().ClusterName) diff --git a/cmd/talosctl/cmd/root.go b/cmd/talosctl/cmd/root.go index b105fd6934..eb3068ae62 100644 --- a/cmd/talosctl/cmd/root.go +++ b/cmd/talosctl/cmd/root.go @@ -16,6 +16,7 @@ import ( "github.com/siderolabs/talos/cmd/talosctl/cmd/common" "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt" + _ "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/create" // import to get the command registered via the init() function. "github.com/siderolabs/talos/cmd/talosctl/cmd/talos" "github.com/siderolabs/talos/pkg/cli" "github.com/siderolabs/talos/pkg/machinery/constants" diff --git a/pkg/provision/providers/factory.go b/pkg/provision/providers/factory.go index 308d15c170..084ac7b149 100644 --- a/pkg/provision/providers/factory.go +++ b/pkg/provision/providers/factory.go @@ -12,14 +12,34 @@ import ( "github.com/siderolabs/talos/pkg/provision/providers/docker" ) +const ( + // QemuProviderName is the name of the qemu provider. + QemuProviderName = "qemu" + // DockerProviderName is the name of the docker provider. + DockerProviderName = "docker" +) + // Factory instantiates provision provider by name. func Factory(ctx context.Context, name string) (provision.Provisioner, error) { + if err := IsValidProvider(name); err != nil { + return nil, err + } + switch name { - case "docker": + case DockerProviderName: return docker.NewProvisioner(ctx) - case "qemu": + case QemuProviderName: return newQemu(ctx) - default: - return nil, fmt.Errorf("unsupported provisioner %q", name) } + + return nil, nil +} + +// IsValidProvider returns an error if the passed provider doesn't exist. +func IsValidProvider(name string) error { + if name != QemuProviderName && name != DockerProviderName { + return fmt.Errorf("unsupported provisioner %q", name) + } + + return nil } diff --git a/website/content/v1.10/reference/cli.md b/website/content/v1.10/reference/cli.md index 9e32a5740b..33a7db1b07 100644 --- a/website/content/v1.10/reference/cli.md +++ b/website/content/v1.10/reference/cli.md @@ -129,6 +129,191 @@ talosctl cgroups [flags] * [talosctl](#talosctl) - A CLI for out-of-band management of Kubernetes nodes created by Talos +## talosctl cluster create docker + +Creates a local docker based kubernetes cluster + +``` +talosctl cluster create docker [flags] +``` + +### Options + +``` + --cidr string CIDR of the cluster network (IPv4, ULA network for IPv6 is derived in automated way) (default "10.5.0.0/24") + --config-patch stringArray patch generated machineconfigs (applied to all node types), use @file to read a patch from file + --config-patch-control-plane stringArray patch generated machineconfigs (applied to 'init' and 'controlplane' types) + --config-patch-worker stringArray patch generated machineconfigs (applied to 'worker' type) + --control-plane-port int control plane port (load balancer and local API port) (default 6443) + --controlplanes int the number of controlplanes to create (default 1) + --cpus string the share of CPUs as fraction (each control plane/VM) (default "2.0") + --cpus-workers string the share of CPUs as fraction (each worker/VM) (default "2.0") + --custom-cni-url string install custom CNI from the URL (Talos cluster) + --dns-domain string the dns domain to use for cluster (default "cluster.local") + --endpoint string use endpoint instead of provider defaults + --init-node-as-endpoint use init node as endpoint instead of any load balancer endpoint + -i, --input-dir string location of pre-generated config files + --ipv4 enable IPv4 network in the cluster (default true) + --kubeprism-port int KubePrism port (set to 0 to disable) (default 7445) + --kubernetes-version string desired kubernetes version to run (default "1.32.1") + --masters int the number of masters to create (default 1) + --memory int the limit on memory usage in MB (each control plane/VM) (default 2048) + --memory-workers int the limit on memory usage in MB (each worker/VM) (default 2048) + --mtu int MTU of the cluster network (default 1500) + --registry-insecure-skip-verify strings list of registry hostnames to skip TLS verification for + --registry-mirror strings list of registry mirrors to use in format: = + --skip-injecting-config skip injecting config from embedded metadata server, write config files to current directory + --skip-k8s-node-readiness-check skip k8s node readiness checks + --skip-kubeconfig skip merging kubeconfig from the created cluster + --talos-version string the desired Talos version to generate config for (if not set, defaults to image version) + --talosconfig string The path to the Talos configuration file. Defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order. + --wait wait for the cluster to be ready before returning (default true) + --wait-timeout duration timeout to wait for the cluster to be ready (default 20m0s) + --wireguard-cidr string CIDR of the wireguard network + --with-apply-config enable apply config when the VM is starting in maintenance mode + --with-cluster-discovery enable cluster discovery (default true) + --with-debug enable debug in Talos config to send service logs to the console + --with-init-node create the cluster with an init node + --with-json-logs enable JSON logs receiver and configure Talos to send logs there + --with-kubespan enable KubeSpan system + --workers int the number of workers to create (default 1) + --docker-disable-ipv6 (docker only) skip enabling IPv6 in containers + --docker-host-ip string (docker only) Host IP to forward exposed ports to (default "0.0.0.0") + -p, --exposed-ports string (docker only) Comma-separated list of ports/protocols to expose on init node. Ex -p :/ + --image string (docker only) the image to use (default "ghcr.io/siderolabs/talos:latest") + --mount mount (docker only) attach a mount to the container + -h, --help help for docker +``` + +### Options inherited from parent commands + +``` + --cluster string Cluster to connect to if a proxy endpoint is used. + --context string Context to be used in command + -e, --endpoints strings override default endpoints in Talos configuration + --name string the name of the cluster (default "talos-default") + -n, --nodes strings target the specified nodes + --provisioner string Talos cluster provisioner to use (default "docker") + --state string directory path to store cluster state (default "/home/user/.talos/clusters") +``` + +### SEE ALSO + +* [talosctl cluster create](#talosctl-cluster-create) - Creates a local docker-based or QEMU-based kubernetes cluster + +## talosctl cluster create qemu + +Creates a local qemu based kubernetes cluster (linux only) + +``` +talosctl cluster create qemu [flags] +``` + +### Options + +``` + --cidr string CIDR of the cluster network (IPv4, ULA network for IPv6 is derived in automated way) (default "10.5.0.0/24") + --config-patch stringArray patch generated machineconfigs (applied to all node types), use @file to read a patch from file + --config-patch-control-plane stringArray patch generated machineconfigs (applied to 'init' and 'controlplane' types) + --config-patch-worker stringArray patch generated machineconfigs (applied to 'worker' type) + --control-plane-port int control plane port (load balancer and local API port) (default 6443) + --controlplanes int the number of controlplanes to create (default 1) + --cpus string the share of CPUs as fraction (each control plane/VM) (default "2.0") + --cpus-workers string the share of CPUs as fraction (each worker/VM) (default "2.0") + --custom-cni-url string install custom CNI from the URL (Talos cluster) + --dns-domain string the dns domain to use for cluster (default "cluster.local") + --endpoint string use endpoint instead of provider defaults + --init-node-as-endpoint use init node as endpoint instead of any load balancer endpoint + -i, --input-dir string location of pre-generated config files + --ipv4 enable IPv4 network in the cluster (default true) + --kubeprism-port int KubePrism port (set to 0 to disable) (default 7445) + --kubernetes-version string desired kubernetes version to run (default "1.32.1") + --masters int the number of masters to create (default 1) + --memory int the limit on memory usage in MB (each control plane/VM) (default 2048) + --memory-workers int the limit on memory usage in MB (each worker/VM) (default 2048) + --mtu int MTU of the cluster network (default 1500) + --registry-insecure-skip-verify strings list of registry hostnames to skip TLS verification for + --registry-mirror strings list of registry mirrors to use in format: = + --skip-injecting-config skip injecting config from embedded metadata server, write config files to current directory + --skip-k8s-node-readiness-check skip k8s node readiness checks + --skip-kubeconfig skip merging kubeconfig from the created cluster + --talos-version string the desired Talos version to generate config for (if not set, defaults to image version) + --talosconfig string The path to the Talos configuration file. Defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order. + --wait wait for the cluster to be ready before returning (default true) + --wait-timeout duration timeout to wait for the cluster to be ready (default 20m0s) + --wireguard-cidr string CIDR of the wireguard network + --with-apply-config enable apply config when the VM is starting in maintenance mode + --with-cluster-discovery enable cluster discovery (default true) + --with-debug enable debug in Talos config to send service logs to the console + --with-init-node create the cluster with an init node + --with-json-logs enable JSON logs receiver and configure Talos to send logs there + --with-kubespan enable KubeSpan system + --workers int the number of workers to create (default 1) + --arch string (QEMU only) cluster architecture (default "amd64") + --bad-rtc (QEMU only) launch VM with bad RTC state + --cni-bin-path strings (QEMU only) search path for CNI binaries (default [/home/user/.talos/cni/bin]) + --cni-bundle-url string (QEMU only) URL to download CNI bundle from (default "https://github.com/siderolabs/talos/releases/download/v1.10.0-alpha.1/talosctl-cni-bundle-${ARCH}.tar.gz") + --cni-cache-dir string (QEMU only) CNI cache directory path (default "/home/user/.talos/cni/cache") + --cni-conf-dir string (QEMU only) CNI config directory path (default "/home/user/.talos/cni/conf.d") + --config-injection-method string (QEMU only) a method to inject machine config: default is HTTP server, 'metal-iso' to mount an ISO + --disable-dhcp-hostname (QEMU only) skip announcing hostname via DHCP + --disk int (QEMU only) default limit on disk size in MB (each VM) (default 6144) + --disk-block-size uint (QEMU only) disk block size (default 512) + --disk-encryption-key-types stringArray (QEMU only) encryption key types to use for disk encryption (uuid, kms) (default [uuid]) + --disk-image-path string (QEMU only) disk image to use + --disk-preallocate (QEMU only) whether disk space should be preallocated (default true) + --encrypt-ephemeral (QEMU only) enable ephemeral partition encryption + --encrypt-state (QEMU only) enable state partition encryption + --extra-boot-kernel-args string (QEMU only) add extra kernel args to the initial boot from vmlinuz and initramfs + --extra-disks int (QEMU only) number of extra disks to create for each worker VM + --extra-disks-drivers strings (QEMU only) driver for each extra disk (virtio, ide, ahci, scsi, nvme, megaraid) + --extra-disks-size int (QEMU only) default limit on disk size in MB (each VM) (default 5120) + --extra-uefi-search-paths strings (QEMU only) additional search paths for UEFI firmware (only applies when UEFI is enabled) + --initrd-path string (QEMU only) initramfs image to use (default "_out/initramfs-${ARCH}.xz") + --install-image string (QEMU only) the installer image to use (default "ghcr.io/siderolabs/installer:latest") + --ipv6 (QEMU only) enable IPv6 network in the cluster + --ipxe-boot-script string (QEMU only) iPXE boot script (URL) to use + --iso-path string (QEMU only) the ISO path to use for the initial boot + --nameservers strings (QEMU only) list of nameservers to use (default [8.8.8.8,1.1.1.1,2001:4860:4860::8888,2606:4700:4700::1111]) + --no-masquerade-cidrs strings (QEMU only) list of CIDRs to exclude from NAT + --uki-path string (QEMU only) the UKI image path to use for the initial boot + --usb-path string (QEMU only) the USB stick image path to use for the initial boot + --use-vip (QEMU only) use a virtual IP for the controlplane endpoint instead of the loadbalancer + --user-disk strings (QEMU only) list of disks to create for each VM in format: ::: + --vmlinuz-path string (QEMU only) the compressed kernel image to use (default "_out/vmlinuz-${ARCH}") + --with-bootloader (QEMU only) enable bootloader to load kernel and initramfs from disk image after install (default true) + --with-firewall string (QEMU only) inject firewall rules into the cluster, value is default policy - accept/block + --with-iommu (QEMU only) enable IOMMU support, this also add a new PCI root port and an interface attached to it) + --with-network-bandwidth int (QEMU only) specify bandwidth restriction (in kbps) on the bridge interface + --with-network-chaos (QEMU only) enable network chaos parameters when creating a qemu cluster + --with-network-jitter duration (QEMU only) specify jitter on the bridge interface + --with-network-latency duration (QEMU only) specify latency on the bridge interface + --with-network-packet-corrupt float (QEMU only) specify percent of corrupt packets on the bridge interface. e.g. 50% = 0.50 (default: 0.0) + --with-network-packet-loss float (QEMU only) specify percent of packet loss on the bridge interface. e.g. 50% = 0.50 (default: 0.0) + --with-network-packet-reorder float (QEMU only) specify percent of reordered packets on the bridge interface. e.g. 50% = 0.50 (default: 0.0) + --with-siderolink true (QEMU only) enables the use of siderolink agent as configuration apply mechanism. true or `wireguard` enables the agent, `tunnel` enables the agent with grpc tunneling (default none) + --with-tpm2 (QEMU only) enable TPM2 emulation support using swtpm + --with-uefi (QEMU only) enable UEFI on x86_64 architecture (default true) + --with-uuid-hostnames (QEMU only) use machine UUIDs as default hostnames + -h, --help help for qemu +``` + +### Options inherited from parent commands + +``` + --cluster string Cluster to connect to if a proxy endpoint is used. + --context string Context to be used in command + -e, --endpoints strings override default endpoints in Talos configuration + --name string the name of the cluster (default "talos-default") + -n, --nodes strings target the specified nodes + --provisioner string Talos cluster provisioner to use (default "docker") + --state string directory path to store cluster state (default "/home/user/.talos/clusters") +``` + +### SEE ALSO + +* [talosctl cluster create](#talosctl-cluster-create) - Creates a local docker-based or QEMU-based kubernetes cluster + ## talosctl cluster create Creates a local docker-based or QEMU-based kubernetes cluster @@ -140,58 +325,26 @@ talosctl cluster create [flags] ### Options ``` - --arch string cluster architecture (default "amd64") - --bad-rtc launch VM with bad RTC state (QEMU only) --cidr string CIDR of the cluster network (IPv4, ULA network for IPv6 is derived in automated way) (default "10.5.0.0/24") - --cni-bin-path strings search path for CNI binaries (VM only) (default [/home/user/.talos/cni/bin]) - --cni-bundle-url string URL to download CNI bundle from (VM only) (default "https://github.com/siderolabs/talos/releases/download/v1.10.0-alpha.1/talosctl-cni-bundle-${ARCH}.tar.gz") - --cni-cache-dir string CNI cache directory path (VM only) (default "/home/user/.talos/cni/cache") - --cni-conf-dir string CNI config directory path (VM only) (default "/home/user/.talos/cni/conf.d") - --config-injection-method string a method to inject machine config: default is HTTP server, 'metal-iso' to mount an ISO (QEMU only) --config-patch stringArray patch generated machineconfigs (applied to all node types), use @file to read a patch from file --config-patch-control-plane stringArray patch generated machineconfigs (applied to 'init' and 'controlplane' types) --config-patch-worker stringArray patch generated machineconfigs (applied to 'worker' type) - --control-plane-port int control plane port (load balancer and local API port, QEMU only) (default 6443) + --control-plane-port int control plane port (load balancer and local API port) (default 6443) --controlplanes int the number of controlplanes to create (default 1) --cpus string the share of CPUs as fraction (each control plane/VM) (default "2.0") --cpus-workers string the share of CPUs as fraction (each worker/VM) (default "2.0") --custom-cni-url string install custom CNI from the URL (Talos cluster) - --disable-dhcp-hostname skip announcing hostname via DHCP (QEMU only) - --disk int default limit on disk size in MB (each VM) (default 6144) - --disk-block-size uint disk block size (VM only) (default 512) - --disk-encryption-key-types stringArray encryption key types to use for disk encryption (uuid, kms) (default [uuid]) - --disk-image-path string disk image to use - --disk-preallocate whether disk space should be preallocated (default true) --dns-domain string the dns domain to use for cluster (default "cluster.local") - --docker-disable-ipv6 skip enabling IPv6 in containers (Docker only) - --docker-host-ip string Host IP to forward exposed ports to (Docker provisioner only) (default "0.0.0.0") - --encrypt-ephemeral enable ephemeral partition encryption - --encrypt-state enable state partition encryption --endpoint string use endpoint instead of provider defaults - -p, --exposed-ports string Comma-separated list of ports/protocols to expose on init node. Ex -p :/ (Docker provisioner only) - --extra-boot-kernel-args string add extra kernel args to the initial boot from vmlinuz and initramfs (QEMU only) - --extra-disks int number of extra disks to create for each worker VM - --extra-disks-drivers strings driver for each extra disk (virtio, ide, ahci, scsi, nvme, megaraid) - --extra-disks-size int default limit on disk size in MB (each VM) (default 5120) - --extra-uefi-search-paths strings additional search paths for UEFI firmware (only applies when UEFI is enabled) - -h, --help help for create - --image string the image to use (default "ghcr.io/siderolabs/talos:latest") --init-node-as-endpoint use init node as endpoint instead of any load balancer endpoint - --initrd-path string initramfs image to use (default "_out/initramfs-${ARCH}.xz") -i, --input-dir string location of pre-generated config files - --install-image string the installer image to use (default "ghcr.io/siderolabs/installer:latest") --ipv4 enable IPv4 network in the cluster (default true) - --ipv6 enable IPv6 network in the cluster (QEMU provisioner only) - --ipxe-boot-script string iPXE boot script (URL) to use - --iso-path string the ISO path to use for the initial boot (VM only) --kubeprism-port int KubePrism port (set to 0 to disable) (default 7445) --kubernetes-version string desired kubernetes version to run (default "1.32.1") + --masters int the number of masters to create (default 1) --memory int the limit on memory usage in MB (each control plane/VM) (default 2048) --memory-workers int the limit on memory usage in MB (each worker/VM) (default 2048) - --mount mount attach a mount to the container (Docker only) --mtu int MTU of the cluster network (default 1500) - --nameservers strings list of nameservers to use (default [8.8.8.8,1.1.1.1,2001:4860:4860::8888,2606:4700:4700::1111]) - --no-masquerade-cidrs strings list of CIDRs to exclude from NAT (QEMU provisioner only) --registry-insecure-skip-verify strings list of registry hostnames to skip TLS verification for --registry-mirror strings list of registry mirrors to use in format: = --skip-injecting-config skip injecting config from embedded metadata server, write config files to current directory @@ -199,35 +352,68 @@ talosctl cluster create [flags] --skip-kubeconfig skip merging kubeconfig from the created cluster --talos-version string the desired Talos version to generate config for (if not set, defaults to image version) --talosconfig string The path to the Talos configuration file. Defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order. - --uki-path string the UKI image path to use for the initial boot (VM only) - --usb-path string the USB stick image path to use for the initial boot (VM only) - --use-vip use a virtual IP for the controlplane endpoint instead of the loadbalancer - --user-disk strings list of disks to create for each VM in format: ::: - --vmlinuz-path string the compressed kernel image to use (default "_out/vmlinuz-${ARCH}") --wait wait for the cluster to be ready before returning (default true) --wait-timeout duration timeout to wait for the cluster to be ready (default 20m0s) --wireguard-cidr string CIDR of the wireguard network --with-apply-config enable apply config when the VM is starting in maintenance mode - --with-bootloader enable bootloader to load kernel and initramfs from disk image after install (default true) --with-cluster-discovery enable cluster discovery (default true) --with-debug enable debug in Talos config to send service logs to the console - --with-firewall string inject firewall rules into the cluster, value is default policy - accept/block (QEMU only) --with-init-node create the cluster with an init node - --with-iommu enable IOMMU support, this also add a new PCI root port and an interface attached to it (qemu only) --with-json-logs enable JSON logs receiver and configure Talos to send logs there --with-kubespan enable KubeSpan system - --with-network-bandwidth int specify bandwidth restriction (in kbps) on the bridge interface when creating a qemu cluster - --with-network-chaos enable to use network chaos parameters when creating a qemu cluster - --with-network-jitter duration specify jitter on the bridge interface when creating a qemu cluster - --with-network-latency duration specify latency on the bridge interface when creating a qemu cluster - --with-network-packet-corrupt float specify percent of corrupt packets on the bridge interface when creating a qemu cluster. e.g. 50% = 0.50 (default: 0.0) - --with-network-packet-loss float specify percent of packet loss on the bridge interface when creating a qemu cluster. e.g. 50% = 0.50 (default: 0.0) - --with-network-packet-reorder float specify percent of reordered packets on the bridge interface when creating a qemu cluster. e.g. 50% = 0.50 (default: 0.0) - --with-siderolink true enables the use of siderolink agent as configuration apply mechanism. true or `wireguard` enables the agent, `tunnel` enables the agent with grpc tunneling (default none) - --with-tpm2 enable TPM2 emulation support using swtpm - --with-uefi enable UEFI on x86_64 architecture (default true) - --with-uuid-hostnames use machine UUIDs as default hostnames (QEMU only) --workers int the number of workers to create (default 1) + --docker-disable-ipv6 (docker only) skip enabling IPv6 in containers + --docker-host-ip string (docker only) Host IP to forward exposed ports to (default "0.0.0.0") + -p, --exposed-ports string (docker only) Comma-separated list of ports/protocols to expose on init node. Ex -p :/ + --image string (docker only) the image to use (default "ghcr.io/siderolabs/talos:latest") + --mount mount (docker only) attach a mount to the container + --arch string (QEMU only) cluster architecture (default "amd64") + --bad-rtc (QEMU only) launch VM with bad RTC state + --cni-bin-path strings (QEMU only) search path for CNI binaries (default [/home/user/.talos/cni/bin]) + --cni-bundle-url string (QEMU only) URL to download CNI bundle from (default "https://github.com/siderolabs/talos/releases/download/v1.10.0-alpha.1/talosctl-cni-bundle-${ARCH}.tar.gz") + --cni-cache-dir string (QEMU only) CNI cache directory path (default "/home/user/.talos/cni/cache") + --cni-conf-dir string (QEMU only) CNI config directory path (default "/home/user/.talos/cni/conf.d") + --config-injection-method string (QEMU only) a method to inject machine config: default is HTTP server, 'metal-iso' to mount an ISO + --disable-dhcp-hostname (QEMU only) skip announcing hostname via DHCP + --disk int (QEMU only) default limit on disk size in MB (each VM) (default 6144) + --disk-block-size uint (QEMU only) disk block size (default 512) + --disk-encryption-key-types stringArray (QEMU only) encryption key types to use for disk encryption (uuid, kms) (default [uuid]) + --disk-image-path string (QEMU only) disk image to use + --disk-preallocate (QEMU only) whether disk space should be preallocated (default true) + --encrypt-ephemeral (QEMU only) enable ephemeral partition encryption + --encrypt-state (QEMU only) enable state partition encryption + --extra-boot-kernel-args string (QEMU only) add extra kernel args to the initial boot from vmlinuz and initramfs + --extra-disks int (QEMU only) number of extra disks to create for each worker VM + --extra-disks-drivers strings (QEMU only) driver for each extra disk (virtio, ide, ahci, scsi, nvme, megaraid) + --extra-disks-size int (QEMU only) default limit on disk size in MB (each VM) (default 5120) + --extra-uefi-search-paths strings (QEMU only) additional search paths for UEFI firmware (only applies when UEFI is enabled) + --initrd-path string (QEMU only) initramfs image to use (default "_out/initramfs-${ARCH}.xz") + --install-image string (QEMU only) the installer image to use (default "ghcr.io/siderolabs/installer:latest") + --ipv6 (QEMU only) enable IPv6 network in the cluster + --ipxe-boot-script string (QEMU only) iPXE boot script (URL) to use + --iso-path string (QEMU only) the ISO path to use for the initial boot + --nameservers strings (QEMU only) list of nameservers to use (default [8.8.8.8,1.1.1.1,2001:4860:4860::8888,2606:4700:4700::1111]) + --no-masquerade-cidrs strings (QEMU only) list of CIDRs to exclude from NAT + --uki-path string (QEMU only) the UKI image path to use for the initial boot + --usb-path string (QEMU only) the USB stick image path to use for the initial boot + --use-vip (QEMU only) use a virtual IP for the controlplane endpoint instead of the loadbalancer + --user-disk strings (QEMU only) list of disks to create for each VM in format: ::: + --vmlinuz-path string (QEMU only) the compressed kernel image to use (default "_out/vmlinuz-${ARCH}") + --with-bootloader (QEMU only) enable bootloader to load kernel and initramfs from disk image after install (default true) + --with-firewall string (QEMU only) inject firewall rules into the cluster, value is default policy - accept/block + --with-iommu (QEMU only) enable IOMMU support, this also add a new PCI root port and an interface attached to it) + --with-network-bandwidth int (QEMU only) specify bandwidth restriction (in kbps) on the bridge interface + --with-network-chaos (QEMU only) enable network chaos parameters when creating a qemu cluster + --with-network-jitter duration (QEMU only) specify jitter on the bridge interface + --with-network-latency duration (QEMU only) specify latency on the bridge interface + --with-network-packet-corrupt float (QEMU only) specify percent of corrupt packets on the bridge interface. e.g. 50% = 0.50 (default: 0.0) + --with-network-packet-loss float (QEMU only) specify percent of packet loss on the bridge interface. e.g. 50% = 0.50 (default: 0.0) + --with-network-packet-reorder float (QEMU only) specify percent of reordered packets on the bridge interface. e.g. 50% = 0.50 (default: 0.0) + --with-siderolink true (QEMU only) enables the use of siderolink agent as configuration apply mechanism. true or `wireguard` enables the agent, `tunnel` enables the agent with grpc tunneling (default none) + --with-tpm2 (QEMU only) enable TPM2 emulation support using swtpm + --with-uefi (QEMU only) enable UEFI on x86_64 architecture (default true) + --with-uuid-hostnames (QEMU only) use machine UUIDs as default hostnames + -h, --help help for create ``` ### Options inherited from parent commands @@ -245,6 +431,8 @@ talosctl cluster create [flags] ### SEE ALSO * [talosctl cluster](#talosctl-cluster) - A collection of commands for managing local docker-based or QEMU-based clusters +* [talosctl cluster create docker](#talosctl-cluster-create-docker) - Creates a local docker based kubernetes cluster +* [talosctl cluster create qemu](#talosctl-cluster-create-qemu) - Creates a local qemu based kubernetes cluster (linux only) ## talosctl cluster destroy diff --git a/website/content/v1.10/talos-guides/install/local-platforms/docker.md b/website/content/v1.10/talos-guides/install/local-platforms/docker.md index ea135d69b7..a5ed728db5 100644 --- a/website/content/v1.10/talos-guides/install/local-platforms/docker.md +++ b/website/content/v1.10/talos-guides/install/local-platforms/docker.md @@ -34,7 +34,7 @@ Further, when running on a Mac in docker, due to networking limitations, VIPs ar Creating a local cluster is as simple as: ```bash -talosctl cluster create +talosctl cluster create docker ``` Once the above finishes successfully, your `talosconfig` (`~/.talos/config`) and `kubeconfig` (`~/.kube/config`) will be configured to point to the new cluster. diff --git a/website/content/v1.10/talos-guides/install/local-platforms/qemu.md b/website/content/v1.10/talos-guides/install/local-platforms/qemu.md index f6287c2d22..f6a64d6a10 100644 --- a/website/content/v1.10/talos-guides/install/local-platforms/qemu.md +++ b/website/content/v1.10/talos-guides/install/local-platforms/qemu.md @@ -78,7 +78,7 @@ mkdir -p ~/.talos/clusters Create the cluster: ```bash -sudo --preserve-env=HOME talosctl cluster create --provisioner qemu +sudo --preserve-env=HOME talosctl cluster create qemu ``` Before the first cluster is created, `talosctl` will download the CNI bundle for the VM provisioning and install it to `~/.talos/cni` directory.