feat: support swap and zram configurations

cmd/ctl/options/cli_options.go

@@ -33,6 +33,7 @@ type CliTerminusInstallOptions struct {
 	WithJuiceFS     bool
 	MiniKubeProfile string
 	BaseDir         string
+	common.SwapConfig
 }
 
 func NewCliTerminusInstallOptions() *CliTerminusInstallOptions {
@@ -45,6 +46,7 @@ func (o *CliTerminusInstallOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolVar(&o.WithJuiceFS, "with-juicefs", false, "Use JuiceFS as the rootfs for Olares workloads, rather than the local disk.")
 	cmd.Flags().StringVarP(&o.MiniKubeProfile, "profile", "p", "", "Set Minikube profile name, only in MacOS platform, defaults to "+common.MinikubeDefaultProfile)
 	cmd.Flags().StringVarP(&o.BaseDir, "base-dir", "b", "", "Set Olares package base dir, defaults to $HOME/"+cc.DefaultBaseDir)
+	(&o.SwapConfig).AddFlags(cmd.Flags())
 }
 
 type CliPrepareSystemOptions struct {

go.mod

@@ -178,7 +178,6 @@ require (
 	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
 	github.com/spf13/cast v1.6.0 // indirect

go.sum

@@ -865,8 +865,6 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR
 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c h1:aqg5Vm5dwtvL+YgDpBcK1ITf3o96N/K7/wsRXQnUTEs=
 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1qZoYLZzLnBw+QkPP9WZnjlSWihhxAJC1+/M=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 h1:OfRzdxCzDhp+rsKWXuOO2I/quKMJ/+TQwVbIP/gltZg=
-github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92/go.mod h1:7/OT02F6S6I7v6WXb+IjhMuZEYfH/RJ5RwEWnEo5BMg=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=

pkg/bootstrap/os/module.go

@@ -177,12 +177,21 @@ func (c *ConfigureOSModule) Init() {
 		Parallel: true,
 	}
 
+	configureSwap := &task.RemoteTask{
+		Name:     "ConfigureSwap",
+		Hosts:    c.Runtime.GetAllHosts(),
+		Prepare:  &kubernetes.NodeInCluster{Not: true},
+		Action:   new(ConfigureSwapTask),
+		Parallel: true,
+	}
+
 	c.Tasks = []task.Interface{
 		getOSData,
 		initOS,
 		GenerateScript,
 		ExecScript,
 		ConfigureNtpServer,
+		configureSwap,
 	}
 }
 

pkg/bootstrap/os/tasks.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	osrelease "github.com/dominodatalab/os-release"
@@ -249,6 +250,54 @@ func (n *NodeConfigureOS) Execute(runtime connector.Runtime) error {
 	return nil
 }
 
+type ConfigureSwapTask struct {
+	common.KubeAction
+}
+
+func (t *ConfigureSwapTask) Execute(runtime connector.Runtime) error {
+	if !t.KubeConf.Arg.EnableZRAM && t.KubeConf.Arg.Swappiness == 0 {
+		return nil
+	}
+	if t.KubeConf.Arg.EnableZRAM {
+		if _, err := util.GetCommand(common.CommandZRAMCtl); err != nil {
+			_, err := runtime.GetRunner().SudoCmd("apt-get install -y util-linux", false, true)
+			if err != nil {
+				return errors.Wrap(err, "failed to install util-linux to configure zram and swap")
+			}
+		}
+
+		if t.KubeConf.Arg.ZRAMSize == "" {
+			t.KubeConf.Arg.ZRAMSize = strconv.Itoa(int(runtime.GetSystemInfo().GetTotalMemory() / 2))
+		}
+		if t.KubeConf.Arg.ZRAMSwapPriority == 0 {
+			t.KubeConf.Arg.ZRAMSwapPriority = 100
+		}
+	}
+	swapServiceStr, err := util.Render(templates.SwapServiceTmpl, t.KubeConf.Arg.SwapConfig)
+	if err != nil {
+		return errors.Wrap(err, "failed to generate swap configuring service")
+	}
+
+	swapServiceName := templates.SwapServiceTmpl.Name()
+	swapServicePath := path.Join("/etc/systemd/system", swapServiceName)
+
+	if err := util.WriteFile(swapServicePath, []byte(swapServiceStr), cc.FileMode0755); err != nil {
+		return errors.Wrap(err, "failed to write swap configuring service file")
+	}
+	if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload", false, true); err != nil {
+		return errors.Wrap(err, "failed to reload swap configuring service")
+	}
+	if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("systemctl enable %s", swapServiceName), false, true); err != nil {
+		return errors.Wrap(err, "failed to enable swap configuring service")
+	}
+
+	// the service type is oneshot, issue restart to make it actually execute
+	if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("systemctl restart %s", swapServiceName), false, true); err != nil {
+		return errors.Wrap(err, "failed to start swap configuring service")
+	}
+	return nil
+}
+
 func addUsers(runtime connector.Runtime, node connector.Host) error {
 	if _, err := runtime.GetRunner().SudoCmd("useradd -M -c 'Kubernetes user' -s /sbin/nologin -r kube || :", false, false); err != nil {
 		return err
@@ -353,6 +402,7 @@ var (
 		"/tmp/kubekey",
 		"/etc/kubekey",
 		"/etc/kke/version",
+		"/etc/systemd/system/olares-swap.service",
 	}
 
 	networkResetCmds = []string{

pkg/bootstrap/os/templates/init_script.go

@@ -44,9 +44,6 @@ var InitOsScriptTmpl = template.Must(template.New("initOS.sh").Parse(
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-swapoff -a
-sed -i /^[^#]*swap*/s/^/\#/g /etc/fstab
-
 # See https://github.com/kubernetes/website/issues/14457
 if [ -f /etc/selinux/config ]; then 
   sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
@@ -65,7 +62,6 @@ echo 'net.bridge.bridge-nf-call-ip6tables = 1' >> /etc/sysctl.conf
 echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf
 echo 'net.ipv4.ip_local_reserved_ports = 30000-32767' >> /etc/sysctl.conf
 echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf
-echo 'vm.swappiness = 1' >> /etc/sysctl.conf
 echo 'fs.inotify.max_user_instances = 524288' >> /etc/sysctl.conf
 echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf
 
@@ -90,7 +86,6 @@ sed -r -i  "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bri
 sed -r -i  "s@#{0,}?net.bridge.bridge-nf-call-iptables ?= ?(0|1)@net.bridge.bridge-nf-call-iptables = 1@g" /etc/sysctl.conf
 sed -r -i  "s@#{0,}?net.ipv4.ip_local_reserved_ports ?= ?([0-9]{1,}-{0,1},{0,1}){1,}@net.ipv4.ip_local_reserved_ports = 30000-32767@g" /etc/sysctl.conf
 sed -r -i  "s@#{0,}?vm.max_map_count ?= ?([0-9]{1,})@vm.max_map_count = 262144@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?vm.swappiness ?= ?([0-9]{1,})@vm.swappiness = 1@g" /etc/sysctl.conf
 sed -r -i  "s@#{0,}?fs.inotify.max_user_instances ?= ?([0-9]{1,})@fs.inotify.max_user_instances = 524288@g" /etc/sysctl.conf
 sed -r -i  "s@#{0,}?kernel.pid_max ?= ?([0-9]{1,})@kernel.pid_max = 65535@g" /etc/sysctl.conf
 

pkg/bootstrap/os/templates/swap.go

@@ -0,0 +1,42 @@
+package templates
+
+import (
+	"text/template"
+
+	"github.com/lithammer/dedent"
+)
+
+var SwapServiceTmpl = template.Must(template.New("olares-swap.service").Parse(
+	dedent.Dedent(`[Unit]
+Description=Olares Swap Configuring Service
+After=local-fs.target
+StartLimitIntervalSec=0
+
+[Service]
+Type=oneshot
+
+{{- if .EnableZRAM }}
+ExecStart=/usr/sbin/modprobe zram
+ExecStart=-/usr/sbin/swapoff /dev/zram0
+ExecStart=-/usr/sbin/zramctl -r /dev/zram0
+ExecStart=/usr/sbin/zramctl -f -s {{ .ZRAMSize }}
+ExecStart=/usr/sbin/mkswap /dev/zram0
+ExecStart=/usr/sbin/swapon -p {{ .ZRAMSwapPriority }} /dev/zram0
+{{- end }}
+{{- if .Swappiness }}
+ExecStart=/usr/sbin/sysctl vm.swappiness={{ .Swappiness }}
+{{- end }}
+
+{{ if .EnableZRAM }}
+ExecStop=-/usr/sbin/swapoff /dev/zram0
+ExecStop=-/usr/sbin/zramctl -r /dev/zram0
+{{ end }}
+
+RemainAfterExit=yes
+Delegate=yes
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=sysinit.target
+`)))

pkg/common/common.go

@@ -209,6 +209,7 @@ const (
 	CommandVelero       = "velero"
 	CommandUpdatePciids = "update-pciids"
 	CommandNmcli        = "nmcli"
+	CommandZRAMCtl      = "zramctl"
 
 	CacheCommandKubectlPath  = "kubectl_bin_path"
 	CacheCommandMinikubePath = "minikube_bin_path"

pkg/common/kube_runtime.go

@@ -35,6 +35,7 @@ import (
 	"bytetrade.io/web3os/installer/pkg/core/logger"
 	"bytetrade.io/web3os/installer/pkg/core/storage"
 	"bytetrade.io/web3os/installer/pkg/core/util"
+	kresource "k8s.io/apimachinery/pkg/api/resource"
 )
 
 type KubeRuntime struct {
@@ -81,6 +82,9 @@ type Argument struct {
 	RegistryMirrors string `json:"registry_mirrors"`
 	DownloadCdnUrl  string `json:"download_cdn_url"`
 
+	// Swap config
+	*SwapConfig
+
 	// master node ssh config
 	*MasterHostConfig
 
@@ -125,6 +129,42 @@ type Argument struct {
 	IsOlaresInContainer bool `json:"is_olares_in_container"`
 }
 
+type SwapConfig struct {
+	EnablePodSwap    bool   `json:"enable_pod_swap"`
+	Swappiness       int    `json:"swappiness"`
+	EnableZRAM       bool   `json:"enable_zram"`
+	ZRAMSize         string `json:"zram_size"`
+	ZRAMSwapPriority int    `json:"zram_swap_priority"`
+}
+
+func (cfg *SwapConfig) AddFlags(fs *pflag.FlagSet) {
+	fs.BoolVar(&cfg.EnablePodSwap, "enable-pod-swap", false, "Enable pods on Kubernetes cluster to use swap, setting --enable-zram, --zram-size or --zram-swap-priority implicitly enables this option, regardless of the command line args, note that only pods of the BestEffort QOS group can use swap due to K8s design")
+	fs.IntVar(&cfg.Swappiness, "swappiness", 0, "Configure the Linux swappiness value, if not set, the current configuration is remained")
+	fs.BoolVar(&cfg.EnableZRAM, "enable-zram", false, "Set up a ZRAM device to be used for swap, setting --zram-size or --zram-swap-priority implicitly enables this option, regardless of the command line args")
+	fs.StringVar(&cfg.ZRAMSize, "zram-size", "", "Set the size of the ZRAM device, takes a format of https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity, defaults to half of the total RAM")
+	fs.IntVar(&cfg.ZRAMSwapPriority, "zram-swap-priority", 0, "Set the swap priority of the ZRAM device, between -1 and 32767, defaults to 100")
+}
+
+func (cfg *SwapConfig) Validate() error {
+	if cfg.ZRAMSize == "" {
+		return nil
+	}
+	processedZRAMSize := cfg.ZRAMSize
+	if strings.HasSuffix(processedZRAMSize, "b") || strings.HasSuffix(processedZRAMSize, "B") {
+		processedZRAMSize = strings.TrimSuffix(cfg.ZRAMSize, "b")
+		processedZRAMSize = strings.TrimSuffix(cfg.ZRAMSize, "B")
+	}
+	processedZRAMSize = strings.ReplaceAll(processedZRAMSize, "g", "G")
+	processedZRAMSize = strings.ReplaceAll(processedZRAMSize, "k", "K")
+	processedZRAMSize = strings.ReplaceAll(processedZRAMSize, "m", "M")
+	q, err := kresource.ParseQuantity(processedZRAMSize)
+	if err != nil {
+		return fmt.Errorf("invalid zram size %s: %w", cfg.ZRAMSize, err)
+	}
+	cfg.ZRAMSize = q.String() + "B"
+	return nil
+}
+
 type MasterHostConfig struct {
 	MasterHost              string `json:"master_host"`
 	MasterNodeName          string `json:"master_node_name"`
@@ -413,6 +453,23 @@ func (a *Argument) SetConsoleLog(fileName string, truncate bool) {
 	a.ConsoleLogTruncate = truncate
 }
 
+func (a *Argument) SetSwapConfig(config SwapConfig) {
+	a.SwapConfig = &SwapConfig{}
+	if config.ZRAMSize != "" || config.ZRAMSwapPriority != 0 {
+		a.EnableZRAM = true
+	} else {
+		a.EnableZRAM = config.EnableZRAM
+	}
+	if a.EnableZRAM {
+		a.ZRAMSize = config.ZRAMSize
+		a.ZRAMSwapPriority = config.ZRAMSwapPriority
+		a.EnablePodSwap = true
+	} else {
+		a.EnablePodSwap = config.EnablePodSwap
+	}
+	a.Swappiness = config.Swappiness
+}
+
 func (a *Argument) SetMasterHostOverride(config MasterHostConfig) {
 	if config.MasterHost != "" {
 		a.MasterHost = config.MasterHost

pkg/core/util/util.go

@@ -37,7 +37,7 @@ import (
 type Data map[string]interface{}
 
 // Render text template with given `variables` Render-context
-func Render(tmpl *template.Template, variables map[string]interface{}) (string, error) {
+func Render(tmpl *template.Template, variables any) (string, error) {
 
 	var buf strings.Builder
 

pkg/k3s/tasks.go

@@ -255,7 +255,8 @@ func (g *GenerateK3sService) Execute(runtime connector.Runtime) error {
 		Template: templates.K3sKubeletConfig,
 		Dst:      filepath.Join("/etc/rancher/k3s/", templates.K3sKubeletConfig.Name()),
 		Data: util.Data{
-			"MaxPods": g.KubeConf.Cluster.Kubernetes.MaxPods,
+			"MaxPods":       g.KubeConf.Cluster.Kubernetes.MaxPods,
+			"EnablePodSwap": g.KubeConf.Arg.EnablePodSwap,
 		},
 	}
 

pkg/k3s/templates/k3sService.go

@@ -68,6 +68,10 @@ ExecStart=/usr/local/bin/k3s $K3S_ROLE $K3S_ARGS $K3S_EXTRA_ARGS $K3S_SERVER_ARG
 		dedent.Dedent(`apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
 maxPods: {{ .MaxPods }}
+{{- if .EnablePodSwap }}
+memorySwap:
+  swapBehavior: LimitedSwap
+{{- end }}
 		`)))
 
 	// * --kubelet-arg=image-gc-high-threshold=85 --kubelet-arg=image-gc-low-threshold=70

pkg/kubernetes/templates/v1beta2/kubeadm_config.go

@@ -281,6 +281,7 @@ func GetKubeletConfiguration(runtime connector.Runtime, kubeConf *common.KubeCon
 		"maxPods":            kubeConf.Cluster.Kubernetes.MaxPods,
 		"podPidsLimit":       kubeConf.Cluster.Kubernetes.PodPidsLimit,
 		"rotateCertificates": true,
+		"failSwapOn":         false,
 		"kubeReserved": map[string]string{
 			"cpu":    "200m",
 			"memory": "250Mi",
@@ -373,6 +374,12 @@ func GetKubeletConfiguration(runtime connector.Runtime, kubeConf *common.KubeCon
 		}
 	}
 
+	if kubeConf.Arg.EnablePodSwap {
+		kubeletConfiguration["memorySwap"] = map[string]string{
+			"swapBehavior": "LimitedSwap",
+		}
+	}
+
 	if kubeConf.Arg.Debug {
 		logger.Debugf("Set kubeletConfiguration: %v", kubeletConfiguration)
 	}

pkg/pipelines/install_terminus.go

@@ -30,6 +30,10 @@ func CliInstallTerminusPipeline(opts *options.CliTerminusInstallOptions) error {
 	arg.SetStorage(getStorageValueFromEnv())
 	arg.SetReverseProxy()
 	arg.SetTokenMaxAge()
+	arg.SetSwapConfig(opts.SwapConfig)
+	if err := arg.SwapConfig.Validate(); err != nil {
+		return err
+	}
 	if opts.WithJuiceFS {
 		arg.WithJuiceFS = true
 	}