chore(version): merge commits from release v1.6.8

.gitignore

@@ -17,3 +17,5 @@ ci/tf/*.tfstate
 ci/tf/*.tfstate.backup
 ci/tf/hosts.yaml
 ci/tf/values.yaml
+
+*.DS_Store

Makefile

@@ -2,6 +2,7 @@
 VERSION ?= $(shell git log -1 --date='format:%Y%m%d' --format='format:%ad').$(shell git describe --always --contains HEAD)
 BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 DATE = $(shell date +"%Y-%m-%d_%H:%M:%S")
+COMMIT = $(shell git rev-parse HEAD | head -c 8)
 
 # Information of OS and ARCH
 OS = $(shell uname -s)
@@ -21,7 +22,7 @@ IMAGE_NAME_TAG ?= $(IMAGE_NAME):$(IMAGE_TAG)
 # GO FLAGS
 GOPROXY ?=
 GO_FLAGS=-ldflags="-s -w"
-CNI_VERSION_LD_FLAG=-ldflags="-X github.com/volcengine/cello/pkg/version.Version=$(VERSION)@$(BRANCH)"
+CNI_VERSION_LD_FLAG=-ldflags="-X github.com/volcengine/cello/pkg/version.Version=$(VERSION)@$(BRANCH) -X github.com/volcengine/cello/pkg/version.GitCommit=$(COMMIT)"
 BUILD_INFO=-ldflags="-X main.BuildInfo=$(VERSION)@$(BRANCH)_$(DATE)"
 
 # BUILD FLAGS

chart/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: vpc-cni
 description: cello
 type: application
-version: v1.6.2
-appVersion: v1.6.2
+version: 1.6.6
+appVersion: v1.6.6

chart/templates/daemonset.yaml

@@ -141,8 +141,6 @@ spec:
           securityContext:
             privileged: true
           volumeMounts:
-            - mountPath: /host
-              name: host-root
             - mountPath: /lib/modules
               name: host-lib-modules
             - mountPath: /etc/cni/net.d
@@ -155,10 +153,6 @@ spec:
               name: host-cilium-run
             - mountPath: /var/run/netns
               name: host-run-netns
-            - mountPath: /host/opt/cni/bin
-              name: host-cni-bin-dir
-            - mountPath: /host/etc/cni/net.d
-              name: host-cni-conf-dir
             - mountPath: /run/xtables.lock
               name: host-xtables-lock
             - mountPath: /etc/cilium/cilium-config
@@ -210,10 +204,6 @@ spec:
         - name: host-lib-modules
           hostPath:
             path: /lib/modules
-        - name: host-root
-          hostPath:
-            path: /
-            type: Directory
         - name: host-cilium-run
           hostPath:
             path: /var/run/cilium

chart/values.yaml

@@ -1,6 +1,6 @@
 image:
   repository: "cr-cn-beijing.volces.com/cello/cello"
-  tag: v1.6.2
+  tag: v1.6.8
   pullPolicy: "Always"
 
 cello:

cmd/cello-agent/cello-agent.go

@@ -17,6 +17,8 @@ package main
 
 import (
 	"github.com/volcengine/cello/pkg/daemon"
+
+	_ "go.uber.org/automaxprocs"
 )
 
 func main() {

cmd/launcher/cilium/launcher.go

@@ -18,23 +18,22 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path"
 	"path/filepath"
-	"strings"
 	"sync"
 	"syscall"
 	"time"
 
-	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/volcengine/cello/pkg/config"
 	"github.com/volcengine/cello/pkg/utils/datatype"
-	"github.com/volcengine/cello/pkg/utils/kernel"
 	"github.com/volcengine/cello/pkg/utils/logger"
 	"github.com/volcengine/cello/pkg/utils/sysctl"
 	"github.com/volcengine/cello/types"
@@ -43,7 +42,6 @@ import (
 // cilium Launcher pre_start_check and start cilium while work with cello
 
 const (
-	bpfFsPath        = "/sys/fs/bpf"
 	ciliumConfigPath = "/etc/cilium/cilium-config"
 	celloConfigPath  = "/etc/cilium/cello-config"
 )
@@ -131,8 +129,8 @@ func main() {
 			log.Warnf("Cello agent not ready, err: %v", err)
 			continue
 		}
-		bodyText, err := ioutil.ReadAll(response.Body)
-		response.Body.Close()
+		bodyText, err := io.ReadAll(response.Body)
+		_ = response.Body.Close()
 		if err != nil {
 			time.Sleep(1 * time.Second)
 			log.Warnf("Cello agent not ready, err: %v", err)
@@ -144,31 +142,12 @@ func main() {
 	}
 	log.Infof("Cello ready, launch cilium...")
 
-	// kernel version must equal and above 4.19
-	if !kernel.CheckKernelVersion(4, 19, 0) {
-		log.Fatalf("Linux kernel version < 4.19, skipping load cilium")
-	}
-
-	// ensure bpf mount
-	err := ensureBpfFsMounted()
-	if err != nil {
-		log.Fatalf("BPF filesystem not mount, %v", err)
-	}
-
 	// disable rp_filter
-	err = sysctl.Disable("net.ipv4.conf.eth0.rp_filter")
+	err := sysctl.Disable("net.ipv4.conf.eth0.rp_filter")
 	if err != nil {
 		log.Fatalf("Disable rp_filter for eth0 failed, %v", err)
 	}
 
-	// modprobe ipvlan
-	cmd := exec.Command("modprobe", "ipvlan")
-	_, err = cmd.Output()
-	if err != nil {
-		log.Fatalf("Modprobe ipvlan failed, %v", err)
-	}
-	log.Infof("Node init success")
-
 	// check apiServer info
 	host := os.Getenv("KUBERNETES_SERVICE_HOST")
 	if host == "" {
@@ -219,7 +198,7 @@ func main() {
 			}
 		}
 		if !info.IsDir() {
-			value, err := ioutil.ReadFile(path)
+			value, err := os.ReadFile(path)
 			if err != nil {
 				return err
 			}
@@ -232,15 +211,11 @@ func main() {
 		log.Fatalf("Read custom cilium config failed, %v", err)
 	}
 
+	policyState := fmt.Sprintf("%v", ciliumArgs["enable-policy"])
 	ciliumExitChan := make(chan struct{})
 	var lock sync.Mutex
 
 	go func() {
-		defer func() {
-			if err := recover(); err != nil {
-				log.Errorf("Cilium panic, %v", err)
-			}
-		}()
 		log.Infof("Run cilium-agent with args: %v", ciliumArgs.ToArgs())
 		lock.Lock()
 		ciliumCmd = exec.Command("cilium-agent", ciliumArgs.ToArgs()...)
@@ -261,75 +236,102 @@ func main() {
 		close(ciliumExitChan)
 	}()
 
+	policyCfgPath := path.Join(ciliumConfigPath, "enable-policy")
+	policyEvent := make(chan *ValueEvent, 1)
+	watchPath(policyCfgPath, &policyState, policyEvent, 10*time.Second)
+
 	// press signal
 	sigCh := make(chan os.Signal, 1)
 	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
 
-	select {
-	case sig := <-sigCh:
-		log.Infof("%d signal: %s", os.Getpid(), sig.String())
-		lock.Lock()
-		if ciliumCmd != nil {
-			err = ciliumCmd.Process.Signal(syscall.SIGINT)
-			if err != nil {
-				log.Infof("INT cilium failed: %v", err)
-			}
-
-			t := time.NewTimer(30 * time.Second)
-			select {
-			case <-ciliumExitChan:
-				log.Infof("cilium exited, code: %d", ciliumCmd.ProcessState.ExitCode())
-				os.Exit(ciliumCmd.ProcessState.ExitCode())
-			case <-t.C:
-				t.Stop()
-				log.Infof("wait cilium finish timeout")
-				os.Exit(1)
-			}
+	exitCilium := func() {
+		log.Infof("cilium exiting")
+		err = ciliumCmd.Process.Signal(syscall.SIGINT)
+		if err != nil {
+			log.Infof("INT cilium failed: %v", err)
 		}
-	case <-ciliumExitChan:
-		log.Infof("cilium unexpect exited, code: %d", ciliumCmd.ProcessState.ExitCode())
-		os.Exit(ciliumCmd.ProcessState.ExitCode())
-	}
-}
 
-func ensureBpfFsMounted() error {
-	initNs, err := ns.GetNS("/proc/1/ns/net")
-	if err != nil {
-		return fmt.Errorf("nsenter pid 1 failed, %w", err)
+		t := time.NewTimer(30 * time.Second)
+		select {
+		case <-ciliumExitChan:
+			log.Infof("cilium exited, code: %d", ciliumCmd.ProcessState.ExitCode())
+			os.Exit(ciliumCmd.ProcessState.ExitCode())
+		case <-t.C:
+			t.Stop()
+			log.Infof("wait cilium finish timeout")
+			os.Exit(1)
+		}
 	}
 
-	err = initNs.Do(func(netNS ns.NetNS) error {
-		// not mount
-		if !isBpfMountExist() {
-			// mount
-			log.Infof("Mounting BPF filesystem...")
-			inErr := syscall.Mount("bpffs", bpfFsPath, "bpf", 0, "")
-			if inErr != nil {
-				return fmt.Errorf("mount bpf filesystem failed, %w", err)
+	for {
+		select {
+		case sig := <-sigCh:
+			log.Infof("%d signal: %s", os.Getpid(), sig.String())
+			lock.Lock()
+			if ciliumCmd != nil {
+				exitCilium()
+			}
+		case pe := <-policyEvent:
+			lock.Lock()
+			if ciliumCmd != nil {
+				if pe.err != nil {
+					log.Errorf("watch policy state failed, %v", pe.err)
+					exitCilium()
+				}
+				if pe.value != "default" && pe.value != "always" && pe.value != "never" {
+					log.Errorf("Invalid value '%s' for enable-policy", pe.value)
+				} else {
+					if err = setPolicyState(pe.value); err != nil {
+						log.Errorf("Switch enable-policy to %s failed, %v", pe.value, err)
+						exitCilium()
+					}
+				}
 			}
-			log.Infof("BPF filesystem mounted")
-		} else {
-			log.Infof("BPF filesystem has mounted")
+			lock.Unlock()
+		case <-ciliumExitChan:
+			log.Infof("cilium unexpect exited, code: %d", ciliumCmd.ProcessState.ExitCode())
+			os.Exit(ciliumCmd.ProcessState.ExitCode())
 		}
-		return nil
-	})
+	}
+}
 
+func setPolicyState(value string) error {
+	log.Infof("Switch enable-policy to %s", value)
+	cfg := fmt.Sprintf("PolicyEnforcement=%s", value)
+	policyCmd := exec.Command("cilium", "config", cfg)
+	output, err := policyCmd.CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("ensure bpf filesystem mount failed, %w", err)
+		return fmt.Errorf("cmd execute failed, output: %v, err: %v", output, err)
 	}
-
+	log.Infof("Switch enable-policy to %s success", value)
 	return nil
 }
 
-func isBpfMountExist() bool {
-	cmd := exec.Command("mount", "-t", "bpf")
-	output, err := cmd.Output()
-	if err != nil {
-		log.Errorf("exec mount command failed, %v", err)
-		return false
-	}
-	if strings.Contains(string(output), bpfFsPath) {
-		return true
-	}
-	return false
+type ValueEvent struct {
+	value string
+	err   error
+}
+
+func watchPath(path string, oldValue *string, valueEvent chan<- *ValueEvent, period time.Duration) {
+	go wait.Forever(func() {
+		value, err := os.ReadFile(path)
+		if os.IsNotExist(err) {
+			return
+		}
+		if err != nil {
+			event := &ValueEvent{
+				value: "",
+				err:   fmt.Errorf("read %s failed, %v", path, err),
+			}
+			valueEvent <- event
+			return
+		}
+		newValue := string(value)
+		if newValue != *oldValue {
+			*oldValue = newValue
+			valueEvent <- &ValueEvent{
+				value: newValue,
+			}
+		}
+	}, period)
 }

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/vishvananda/netlink v1.2.1-beta.2
 	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f
 	github.com/volcengine/volcengine-go-sdk v1.0.61
+	go.uber.org/automaxprocs v1.5.2
 	golang.org/x/net v0.9.0
 	golang.org/x/sys v0.7.0
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac

go.sum

@@ -450,6 +450,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -616,6 +617,8 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
 go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
+go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=