From 9f7e83974fcae77139b831e8a2b0cd956291eab0 Mon Sep 17 00:00:00 2001 From: cyclinder Date: Thu, 6 Jul 2023 19:25:30 +0800 Subject: [PATCH] fix mutlus auto-generated cni files && opt coordinator cni config filed --- .github/workflows/auto-nightly-ci.yaml | 2 +- api/v1/agent/models/coordinator_config.go | 34 +- api/v1/agent/openapi.yaml | 10 +- api/v1/agent/server/embedded_spec.go | 28 +- charts/spiderpool/README.md | 8 +- ...rpool.spidernet.io_spidercoordinators.yaml | 22 +- ...pool.spidernet.io_spidermultusconfigs.yaml | 23 +- .../templates/multus-daemonset.yaml | 22 +- charts/spiderpool/templates/pod.yaml | 27 +- charts/spiderpool/templates/role.yaml | 12 + charts/spiderpool/values.yaml | 14 +- cmd/coordinator/cmd/cni_types.go | 35 +- cmd/coordinator/cmd/command_add.go | 24 +- cmd/coordinator/cmd/command_del.go | 83 +-- cmd/coordinator/cmd/utils.go | 1 + cmd/spiderpool-agent/cmd/coordinator.go | 6 +- cmd/spiderpool-init/cmd/client.go | 26 + cmd/spiderpool-init/cmd/config.go | 131 +++- cmd/spiderpool-init/cmd/root.go | 39 +- .../calico_ippool_informer.go | 4 +- .../coordinator_informer.go | 6 +- .../coordinator_validate.go | 2 +- .../spiderpool.spidernet.io/v2beta1/rbac.go | 2 +- .../v2beta1/spidercoordinator_types.go | 6 +- .../v2beta1/spidermultus_types.go | 4 +- .../v2beta1/zz_generated.deepcopy.go | 12 +- pkg/multuscniconfig/multusconfig_informer.go | 85 ++- pkg/multuscniconfig/utils.go | 7 +- pkg/networking/ipchecking/ipchecking.go | 6 + pkg/networking/networking/route.go | 2 +- test/Makefile | 1 - test/scripts/install-default-cni.sh | 2 +- test/scripts/install-multus.sh | 24 +- .../containernetworking/cni/libcni/api.go | 679 ++++++++++++++++++ .../containernetworking/cni/libcni/conf.go | 270 +++++++ .../cni/pkg/invoke/args.go | 128 ++++ .../cni/pkg/invoke/delegate.go | 80 +++ .../cni/pkg/invoke/exec.go | 187 +++++ .../cni/pkg/invoke/find.go | 48 ++ .../cni/pkg/invoke/os_unix.go | 20 + .../cni/pkg/invoke/os_windows.go | 18 + .../cni/pkg/invoke/raw_exec.go | 88 +++ vendor/modules.txt | 2 + 43 files changed, 1997 insertions(+), 233 deletions(-) create mode 100644 vendor/github.com/containernetworking/cni/libcni/api.go create mode 100644 vendor/github.com/containernetworking/cni/libcni/conf.go create mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/args.go create mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go create mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/exec.go create mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/find.go create mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/os_unix.go create mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/os_windows.go create mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/raw_exec.go diff --git a/.github/workflows/auto-nightly-ci.yaml b/.github/workflows/auto-nightly-ci.yaml index 69c9737e32..a6d8e020d6 100644 --- a/.github/workflows/auto-nightly-ci.yaml +++ b/.github/workflows/auto-nightly-ci.yaml @@ -189,7 +189,7 @@ jobs: - name: create an issue uses: dacbd/create-issue-action@v1.2.1 with: - token: ${{ secrets.WELAN_PAT }} + token: ${{ secrets.GITHUB_TOKEN }} title: "Night CI ${{ ENV.TIMESTAMP }}: Failed" body: | action url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/api/v1/agent/models/coordinator_config.go b/api/v1/agent/models/coordinator_config.go index 2d44dcdda5..4cb7dc336a 100644 --- a/api/v1/agent/models/coordinator_config.go +++ b/api/v1/agent/models/coordinator_config.go @@ -28,8 +28,8 @@ type CoordinatorConfig struct { // detect IP conflict DetectIPConflict bool `json:"detectIPConflict,omitempty"` - // extra c ID r - ExtraCIDR []string `json:"extraCIDR"` + // hijack c ID r + HijackCIDR []string `json:"hijackCIDR"` // host r p filter HostRPFilter int64 `json:"hostRPFilter,omitempty"` @@ -37,9 +37,13 @@ type CoordinatorConfig struct { // host rule table HostRuleTable int64 `json:"hostRuleTable,omitempty"` - // pod c ID r + // mode // Required: true - PodCIDR []string `json:"podCIDR"` + Mode *string `json:"mode"` + + // overlay pod c ID r + // Required: true + OverlayPodCIDR []string `json:"overlayPodCIDR"` // pod default route n i c PodDefaultRouteNIC string `json:"podDefaultRouteNIC,omitempty"` @@ -51,10 +55,6 @@ type CoordinatorConfig struct { // Required: true ServiceCIDR []string `json:"serviceCIDR"` - // tune mode - // Required: true - TuneMode *string `json:"tuneMode"` - // tune pod routes // Required: true TunePodRoutes *bool `json:"tunePodRoutes"` @@ -64,15 +64,15 @@ type CoordinatorConfig struct { func (m *CoordinatorConfig) Validate(formats strfmt.Registry) error { var res []error - if err := m.validatePodCIDR(formats); err != nil { + if err := m.validateMode(formats); err != nil { res = append(res, err) } - if err := m.validateServiceCIDR(formats); err != nil { + if err := m.validateOverlayPodCIDR(formats); err != nil { res = append(res, err) } - if err := m.validateTuneMode(formats); err != nil { + if err := m.validateServiceCIDR(formats); err != nil { res = append(res, err) } @@ -86,27 +86,27 @@ func (m *CoordinatorConfig) Validate(formats strfmt.Registry) error { return nil } -func (m *CoordinatorConfig) validatePodCIDR(formats strfmt.Registry) error { +func (m *CoordinatorConfig) validateMode(formats strfmt.Registry) error { - if err := validate.Required("podCIDR", "body", m.PodCIDR); err != nil { + if err := validate.Required("mode", "body", m.Mode); err != nil { return err } return nil } -func (m *CoordinatorConfig) validateServiceCIDR(formats strfmt.Registry) error { +func (m *CoordinatorConfig) validateOverlayPodCIDR(formats strfmt.Registry) error { - if err := validate.Required("serviceCIDR", "body", m.ServiceCIDR); err != nil { + if err := validate.Required("overlayPodCIDR", "body", m.OverlayPodCIDR); err != nil { return err } return nil } -func (m *CoordinatorConfig) validateTuneMode(formats strfmt.Registry) error { +func (m *CoordinatorConfig) validateServiceCIDR(formats strfmt.Registry) error { - if err := validate.Required("tuneMode", "body", m.TuneMode); err != nil { + if err := validate.Required("serviceCIDR", "body", m.ServiceCIDR); err != nil { return err } diff --git a/api/v1/agent/openapi.yaml b/api/v1/agent/openapi.yaml index ea210fd0e7..14aada2b07 100644 --- a/api/v1/agent/openapi.yaml +++ b/api/v1/agent/openapi.yaml @@ -300,9 +300,9 @@ definitions: description: Coordinator config type: object properties: - tuneMode: + mode: type: string - podCIDR: + overlayPodCIDR: type: array items: type: string @@ -310,7 +310,7 @@ definitions: type: array items: type: string - extraCIDR: + hijackCIDR: type: array items: type: string @@ -329,8 +329,8 @@ definitions: detectGateway: type: boolean required: - - tuneMode - - podCIDR + - mode + - overlayPodCIDR - serviceCIDR - tunePodRoutes GetCoordinatorArgs: diff --git a/api/v1/agent/server/embedded_spec.go b/api/v1/agent/server/embedded_spec.go index b7d5cc9012..7dd998a6fc 100644 --- a/api/v1/agent/server/embedded_spec.go +++ b/api/v1/agent/server/embedded_spec.go @@ -263,8 +263,8 @@ func init() { "description": "Coordinator config", "type": "object", "required": [ - "tuneMode", - "podCIDR", + "mode", + "overlayPodCIDR", "serviceCIDR", "tunePodRoutes" ], @@ -275,7 +275,7 @@ func init() { "detectIPConflict": { "type": "boolean" }, - "extraCIDR": { + "hijackCIDR": { "type": "array", "items": { "type": "string" @@ -287,7 +287,10 @@ func init() { "hostRuleTable": { "type": "integer" }, - "podCIDR": { + "mode": { + "type": "string" + }, + "overlayPodCIDR": { "type": "array", "items": { "type": "string" @@ -305,9 +308,6 @@ func init() { "type": "string" } }, - "tuneMode": { - "type": "string" - }, "tunePodRoutes": { "type": "boolean" } @@ -760,8 +760,8 @@ func init() { "description": "Coordinator config", "type": "object", "required": [ - "tuneMode", - "podCIDR", + "mode", + "overlayPodCIDR", "serviceCIDR", "tunePodRoutes" ], @@ -772,7 +772,7 @@ func init() { "detectIPConflict": { "type": "boolean" }, - "extraCIDR": { + "hijackCIDR": { "type": "array", "items": { "type": "string" @@ -784,7 +784,10 @@ func init() { "hostRuleTable": { "type": "integer" }, - "podCIDR": { + "mode": { + "type": "string" + }, + "overlayPodCIDR": { "type": "array", "items": { "type": "string" @@ -802,9 +805,6 @@ func init() { "type": "string" } }, - "tuneMode": { - "type": "string" - }, "tunePodRoutes": { "type": "boolean" } diff --git a/charts/spiderpool/README.md b/charts/spiderpool/README.md index 1fc60cc4c2..f6b74b2f85 100644 --- a/charts/spiderpool/README.md +++ b/charts/spiderpool/README.md @@ -106,6 +106,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \ | `global.commonAnnotations` | Annotations to add to all deployed objects | `{}` | | `global.commonLabels` | Labels to add to all deployed objects | `{}` | | `global.ipamBinHostPath` | the host path of the IPAM plugin directory. | `/opt/cni/bin` | +| `global.cniConfHostPath` | the host path of the cni config directory | `/etc/cni/net.d` | | `global.ipamUNIXSocketHostPath` | the host path of unix domain socket for ipam plugin | `/var/run/spidernet/spiderpool.sock` | | `global.configName` | the configmap name | `spiderpool-conf` | @@ -135,11 +136,12 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \ | ------------------------------ | ------------------------------------------------------------------------- | ---------- | | `coordinator.enabled` | enable SpiderCoordinator | `true` | | `coordinator.name` | the name of the default SpiderCoordinator CR | `default` | -| `coordinator.tuneMode` | optional network mode, ["underlay", "overlay", "disabled"] | `underlay` | +| `coordinator.mode` | optional network mode, ["underlay", "overlay", "disabled"] | `underlay` | | `coordinator.podCIDRType` | Pod CIDR type that should be collected, [ "cluster", "calico", "cilium" ] | `cluster` | -| `coordinator.detectGateway` | detect the reachability of the gateway | `true` | -| `coordinator.detectIPConflict` | detect IP address conflicts | `true` | +| `coordinator.detectGateway` | detect the reachability of the gateway | `false` | +| `coordinator.detectIPConflict` | detect IP address conflicts | `false` | | `coordinator.tunePodRoutes` | tune Pod routes | `true` | +| `coordinator.hijackCIDR` | Additional subnets that need to be hijacked to the host forward | `[]` | ### multus parameters diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml index 9159073c06..7de700a285 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml @@ -45,7 +45,7 @@ spec: detectIPConflict: default: false type: boolean - extraCIDR: + hijackCIDR: items: type: string type: array @@ -55,19 +55,19 @@ spec: hostRuleTable: default: 500 type: integer - podCIDRType: - type: string - podDefaultRouteNIC: - type: string - podMACPrefix: - type: string - tuneMode: + mode: default: underlay enum: - underlay - overlay - disabled type: string + podCIDRType: + type: string + podDefaultRouteNIC: + type: string + podMACPrefix: + type: string tunePodRoutes: default: true type: boolean @@ -77,12 +77,12 @@ spec: status: description: CoordinationStatus defines the observed state of SpiderCoordinator. properties: - phase: - type: string - podCIDR: + overlayPodCIDR: items: type: string type: array + phase: + type: string serviceCIDR: items: type: string diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml index 4cb05e5e07..91c4c2a196 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml @@ -39,8 +39,13 @@ spec: description: Spec is the specification of the MultusCNIConfig properties: cniType: - description: macvlan、ipvlan、sriov、custom + description: calico、cilium-cni、flannel、weave-net、kube-ovn、macvlan、ipvlan、sriov、custom enum: + - calico + - cilium-cni + - flannel + - weave-net + - kube-ovn - macvlan - ipvlan - sriov @@ -55,7 +60,7 @@ spec: detectIPConflict: default: false type: boolean - extraCIDR: + hijackCIDR: items: type: string type: array @@ -65,19 +70,19 @@ spec: hostRuleTable: default: 500 type: integer - podCIDRType: - type: string - podDefaultRouteNIC: - type: string - podMACPrefix: - type: string - tuneMode: + mode: default: underlay enum: - underlay - overlay - disabled type: string + podCIDRType: + type: string + podDefaultRouteNIC: + type: string + podMACPrefix: + type: string tunePodRoutes: default: true type: boolean diff --git a/charts/spiderpool/templates/multus-daemonset.yaml b/charts/spiderpool/templates/multus-daemonset.yaml index cbf5fc8be6..c1672cb64a 100644 --- a/charts/spiderpool/templates/multus-daemonset.yaml +++ b/charts/spiderpool/templates/multus-daemonset.yaml @@ -217,9 +217,14 @@ spec: imagePullPolicy: {{ .Values.multus.multusCNI.image.pullPolicy}} image: {{ include "spiderpool.multus.image" . | quote }} command: - - "cp" - - "/usr/src/multus-cni/bin/multus" - - "/host/opt/cni/bin/multus" + - "/bin/sh" + - "-c" + - | + cp /usr/src/multus-cni/bin/multus /host/opt/cni/bin/multus + # TODO: We can revert the changes if https://github.com/k8snetworkplumbingwg/multus-cni/pull/1119 merged. + {{- if eq .Values.multus.multusCNI.defaultCniCRName "" }} + cp /host/etc/cni/net.d/*.conf* /host/etc/cni/multus/net.d/ + {{- end }} resources: requests: cpu: "10m" @@ -230,6 +235,12 @@ spec: - name: cnibin mountPath: /host/opt/cni/bin mountPropagation: Bidirectional + {{- if eq .Values.multus.multusCNI.defaultCniCRName "" }} + - mountPath: /host/etc/cni/net.d + name: cni + - mountPath: /host/etc/cni/multus/net.d + name: multus-cni-dir + {{- end }} terminationGracePeriodSeconds: 10 volumes: - name: cni @@ -245,6 +256,11 @@ spec: items: - key: cni-conf.json path: 00-multus.conf + {{- else }} + - hostPath: + path: /etc/cni/multus/net.d + type: DirectoryOrCreate + name: multus-cni-dir {{- end }} {{- if .Values.multus.multusCNI.extraVolumeMounts }} {{- include "tplvalues.render" ( dict "value" .Values.multus.multusCNI.extraVolumeMounts "context" $ ) | nindent 8 }} diff --git a/charts/spiderpool/templates/pod.yaml b/charts/spiderpool/templates/pod.yaml index c525f90c19..13336fb0ca 100644 --- a/charts/spiderpool/templates/pod.yaml +++ b/charts/spiderpool/templates/pod.yaml @@ -63,6 +63,10 @@ spec: resources: {{- toYaml . | trim | nindent 6 }} {{- end }} + {{- with .Values.spiderpoolInit.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} env: {{- with .Values.spiderpoolInit.extraEnv }} {{- toYaml . | nindent 4 }} @@ -74,8 +78,8 @@ spec: {{- if .Values.coordinator.enabled }} - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_NAME value: {{ .Values.coordinator.name | quote }} - - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_TUNE_MODE - value: {{ .Values.coordinator.tuneMode | quote }} + - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_MODE + value: {{ .Values.coordinator.mode | quote }} - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_POD_CIDR_TYPE value: {{ .Values.coordinator.podCIDRType | quote }} - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_DETECT_GATEWAY @@ -84,6 +88,8 @@ spec: value: {{ .Values.coordinator.detectIPConflict | quote }} - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_TUNE_POD_ROUTES value: {{ .Values.coordinator.tunePodRoutes | quote }} + - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_HIJACK_CIDR + value: {{ toJson .Values.coordinator.hijackCIDR | quote }} {{- end }} {{- if and .Values.clusterDefaultPool.installIPv4IPPool .Values.ipam.enableIPv4 }} - name: SPIDERPOOL_INIT_DEFAULT_IPV4_IPPOOL_NAME @@ -113,9 +119,20 @@ spec: value: {{ .Values.clusterDefaultPool.ipv6SubnetName | quote }} {{- end }} {{- end }} - {{- with .Values.spiderpoolInit.securityContext }} - securityContext: - {{- toYaml . | nindent 4 }} + - name: SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG + value: {{ .Values.multus.enableMultusConfig | quote }} + - name: SPIDERPOOL_INIT_DEFAULT_CNI_NAME + value: {{ .Values.multus.multusCNI.defaultCniCRName | quote }} + {{- if eq .Values.multus.multusCNI.defaultCniCRName "" }} + - name: SPIDERPOOL_INIT_DEFAULT_CNI_DIR + value: {{ .Values.global.cniConfHostPath | quote }} + volumeMounts: + - name: cni + mountPath: {{ .Values.global.cniConfHostPath }} + volumes: + - name: cni + hostPath: + path: {{ .Values.global.cniConfHostPath }} {{- end }} --- apiVersion: v1 diff --git a/charts/spiderpool/templates/role.yaml b/charts/spiderpool/templates/role.yaml index f8d9915ae2..3e27876223 100644 --- a/charts/spiderpool/templates/role.yaml +++ b/charts/spiderpool/templates/role.yaml @@ -133,6 +133,18 @@ rules: - get - patch - update +- apiGroups: + - spiderpool.spidernet.io + resources: + - spidermultusconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - spiderpool.spidernet.io resources: diff --git a/charts/spiderpool/values.yaml b/charts/spiderpool/values.yaml index 1a6b675839..3ec835c1a6 100644 --- a/charts/spiderpool/values.yaml +++ b/charts/spiderpool/values.yaml @@ -26,6 +26,9 @@ global: ## @param global.ipamBinHostPath the host path of the IPAM plugin directory. ipamBinHostPath: /opt/cni/bin + ## @param global.cniConfHostPath the host path of the cni config directory + cniConfHostPath: /etc/cni/net.d + ## @param global.ipamUNIXSocketHostPath the host path of unix domain socket for ipam plugin ipamUNIXSocketHostPath: /var/run/spidernet/spiderpool.sock @@ -87,21 +90,24 @@ coordinator: ## @param coordinator.name the name of the default SpiderCoordinator CR name: "default" - ## @param coordinator.tuneMode optional network mode, ["underlay", "overlay", "disabled"] - tuneMode: "underlay" + ## @param coordinator.mode optional network mode, ["underlay", "overlay", "disabled"] + mode: "underlay" ## @param coordinator.podCIDRType Pod CIDR type that should be collected, [ "cluster", "calico", "cilium" ] podCIDRType: "cluster" ## @param coordinator.detectGateway detect the reachability of the gateway - detectGateway: true + detectGateway: false ## @param coordinator.detectIPConflict detect IP address conflicts - detectIPConflict: true + detectIPConflict: false ## @param coordinator.tunePodRoutes tune Pod routes tunePodRoutes: true + ## @param coordinator.hijackCIDR Additional subnets that need to be hijacked to the host forward + hijackCIDR: [] + ## @section multus parameters ## multus: diff --git a/cmd/coordinator/cmd/cni_types.go b/cmd/coordinator/cmd/cni_types.go index 69146c13b0..cbac04fd41 100644 --- a/cmd/coordinator/cmd/cni_types.go +++ b/cmd/coordinator/cmd/cni_types.go @@ -42,17 +42,16 @@ const ( type Config struct { types.NetConf - OnlyHardware bool `json:"onlyHardware,omitempty"` DetectGateway *bool `json:"detectGateway,omitempty"` MacPrefix string `json:"podMACPrefix,omitempty"` - InterfacePrefix string `json:"ifacePrefix,omitempty"` - PodFirstInterface string `json:"podDefaultInterface,omitempty"` - ClusterCIDR []string `json:"podCIDR,omitempty"` + MultusNicPrefix string `json:"multusNicPrefix,omitempty"` + PodDefaultCniNic string `json:"podDefaultCniNic,omitempty"` + OverlayPodCIDR []string `json:"overlayPodCIDR,omitempty"` ServiceCIDR []string `json:"serviceCIDR,omitempty"` - ExtraCIDR []string `json:"extraCIDR,omitempty"` + HijackCIDR []string `json:"hijackCIDR,omitempty"` TunePodRoutes *bool `json:"tunePodRoutes,omitempty"` PodDefaultRouteNIC string `json:"podDefaultRouteNic,omitempty"` - TuneMode Mode `json:"tuneMode,omitempty"` + Mode Mode `json:"mode,omitempty"` HostRuleTable *int64 `json:"hostRuleTable,omitempty"` RPFilter int32 `json:"hostRPFilter,omitempty" ` IPConflict *bool `json:"detectIPConflict,omitempty"` @@ -102,12 +101,12 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co return nil, err } - if conf.PodFirstInterface == "" { - conf.PodFirstInterface = defaultOverlayVethName + if conf.PodDefaultCniNic == "" { + conf.PodDefaultCniNic = defaultOverlayVethName } - if conf.InterfacePrefix == "" { - conf.InterfacePrefix = defaultNICPrefix + if conf.MultusNicPrefix == "" { + conf.MultusNicPrefix = defaultNICPrefix } if conf.LogOptions == nil { @@ -167,8 +166,8 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co conf.TunePodRoutes = coordinatorConfig.TunePodRoutes } - if conf.TuneMode == "" { - conf.TuneMode = Mode(*coordinatorConfig.TuneMode) + if conf.Mode == "" { + conf.Mode = Mode(*coordinatorConfig.Mode) } if conf.PodDefaultRouteNIC == "" && coordinatorConfig.PodDefaultRouteNIC != "" { @@ -199,12 +198,12 @@ func ValidateRoutes(conf *Config, coordinatorConfig *models.CoordinatorConfig) e conf.ServiceCIDR = coordinatorConfig.ServiceCIDR } - if len(conf.ClusterCIDR) == 0 { - conf.ClusterCIDR = coordinatorConfig.PodCIDR + if len(conf.OverlayPodCIDR) == 0 { + conf.OverlayPodCIDR = coordinatorConfig.OverlayPodCIDR } - if len(conf.ExtraCIDR) == 0 { - conf.ExtraCIDR = coordinatorConfig.ExtraCIDR + if len(conf.HijackCIDR) == 0 { + conf.HijackCIDR = coordinatorConfig.HijackCIDR } var err error @@ -213,12 +212,12 @@ func ValidateRoutes(conf *Config, coordinatorConfig *models.CoordinatorConfig) e return err } - err = validateRoutes(conf.ClusterCIDR) + err = validateRoutes(conf.OverlayPodCIDR) if err != nil { return err } - err = validateRoutes(conf.ExtraCIDR) + err = validateRoutes(conf.HijackCIDR) if err != nil { return err } diff --git a/cmd/coordinator/cmd/command_add.go b/cmd/coordinator/cmd/command_add.go index c1d012adc1..19068bd49a 100644 --- a/cmd/coordinator/cmd/command_add.go +++ b/cmd/coordinator/cmd/command_add.go @@ -56,7 +56,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) { if err != nil { return err } - if conf.TuneMode == ModeDisable { + if conf.Mode == ModeDisable { return types.PrintResult(conf.PrevResult, conf.CNIVersion) } @@ -75,7 +75,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) { zap.String("PodName", string(k8sArgs.K8S_POD_NAME)), zap.String("PodNamespace", string(k8sArgs.K8S_POD_NAMESPACE)), ) - logger.Info(fmt.Sprintf("start to implement ADD command in %v mode", conf.TuneMode)) + logger.Info(fmt.Sprintf("start to implement ADD command in %v mode", conf.Mode)) // parse prevResult prevResult, err := current.GetResult(conf.PrevResult) @@ -91,15 +91,15 @@ func CmdAdd(args *skel.CmdArgs) (err error) { } c := &coordinator{ - HijackCIDR: conf.ClusterCIDR, + HijackCIDR: conf.OverlayPodCIDR, hostRuleTable: int(*conf.HostRuleTable), ipFamily: ipFamily, currentInterface: args.IfName, - tuneMode: conf.TuneMode, - interfacePrefix: conf.InterfacePrefix, + tuneMode: conf.Mode, + interfacePrefix: conf.MultusNicPrefix, } c.HijackCIDR = append(c.HijackCIDR, conf.ServiceCIDR...) - c.HijackCIDR = append(c.HijackCIDR, conf.ExtraCIDR...) + c.HijackCIDR = append(c.HijackCIDR, conf.HijackCIDR...) c.netns, err = ns.GetNS(args.Netns) if err != nil { @@ -109,14 +109,14 @@ func CmdAdd(args *skel.CmdArgs) (err error) { defer c.netns.Close() // check if it's first time invoke - err = c.coordinatorFirstInvoke(conf.PodFirstInterface) + err = c.coordinatorFirstInvoke(conf.PodDefaultCniNic) if err != nil { logger.Error(err.Error()) return err } // get basic info - switch conf.TuneMode { + switch conf.Mode { case ModeUnderlay: c.podVethName = defaultUnderlayVethName c.hostVethName = getHostVethName(args.ContainerID) @@ -140,8 +140,8 @@ func CmdAdd(args *skel.CmdArgs) (err error) { logger.Info("TuneMode is disable, nothing to do") return types.PrintResult(conf.PrevResult, conf.CNIVersion) default: - logger.Error("Unknown tuneMode", zap.String("invalid tuneMode", string(conf.TuneMode))) - return fmt.Errorf("unknown tuneMode: %s", conf.TuneMode) + logger.Error("Unknown tuneMode", zap.String("invalid tuneMode", string(conf.Mode))) + return fmt.Errorf("unknown tuneMode: %s", conf.Mode) } logger.Sugar().Infof("Get coordinator config: %v", c) @@ -198,10 +198,6 @@ func CmdAdd(args *skel.CmdArgs) (err error) { } logger.Info("Override hardware address successfully", zap.String("interface", args.IfName), zap.String("hardware address", hwAddr)) - if conf.OnlyHardware { - logger.Debug("Only override hardware address, exit now") - return types.PrintResult(conf.PrevResult, conf.CNIVersion) - } } // get all ip address on the node diff --git a/cmd/coordinator/cmd/command_del.go b/cmd/coordinator/cmd/command_del.go index e9b38868cb..f83d559d3f 100644 --- a/cmd/coordinator/cmd/command_del.go +++ b/cmd/coordinator/cmd/command_del.go @@ -49,6 +49,10 @@ func CmdDel(args *skel.CmdArgs) (err error) { return err } + if conf.Mode == ModeDisable { + return nil + } + logger, err := logutils.SetupFileLogging(conf.LogOptions.LogLevel, conf.LogOptions.LogFilePath, conf.LogOptions.LogFileMaxSize, conf.LogOptions.LogFileMaxAge, conf.LogOptions.LogFileMaxCount) @@ -62,7 +66,8 @@ func CmdDel(args *skel.CmdArgs) (err error) { zap.String("Netns", args.Netns), zap.String("IfName", args.IfName), ) - logger.Info(fmt.Sprintf("start to implement DELETE command in %v mode", conf.TuneMode)) + + logger.Info(fmt.Sprintf("start to implement DELETE command in %v mode", conf.Mode)) c := &coordinator{ hostRuleTable: int(*conf.HostRuleTable), @@ -70,57 +75,55 @@ func CmdDel(args *skel.CmdArgs) (err error) { c.netns, err = ns.GetNS(args.Netns) if err != nil { - _, ok := err.(ns.NSPathNotExistErr) - if ok { - logger.Debug("Pod's netns already gone. Nothing to do.") - } else { - logger.Warn("failed to GetNS, container maybe gone, ignore ", zap.Error(err)) - } - } else { - defer c.netns.Close() - - err = c.netns.Do(func(netNS ns.NetNS) error { - c.currentAddress, err = networking.GetAddersByName(args.IfName, netlink.FAMILY_ALL) - if err != nil { - logger.Error("failed to GetAddersByName", zap.String("interface", args.IfName)) - return err - } + if _, ok := err.(ns.NSPathNotExistErr); ok { + logger.Debug("Pod's netns already gone. Nothing to do.") return nil - }) - if err != nil { - // ignore err - logger.Warn("failed to GetAddersByName, ignore error", zap.Error(err)) - } else { - for idx := range c.currentAddress { - ipNet := networking.ConvertMaxMaskIPNet(c.currentAddress[idx].IP) - err = networking.DelToRuleTable(ipNet, c.hostRuleTable) - if err != nil && !os.IsNotExist(err) { - logger.Error("failed to DelToRuleTable", zap.Int("HostRuleTable", c.hostRuleTable), zap.String("Dst", ipNet.String()), zap.Error(err)) - } - } } + logger.Error("failed to GetNS,", zap.Error(err)) + return fmt.Errorf("failed to GetNS %s: %v", args.Netns, err) } + defer c.netns.Close() - if conf.TuneMode == ModeUnderlay { + if conf.Mode == ModeUnderlay { hostVeth := getHostVethName(args.ContainerID) vethLink, err := netlink.LinkByName(hostVeth) if err != nil { if _, ok := err.(netlink.LinkNotFoundError); ok { logger.Debug("Host veth has gone, nothing to do", zap.String("HostVeth", hostVeth)) - } else { - logger.Warn(fmt.Sprintf("failed to get host veth device %s: %v", hostVeth, err)) - return fmt.Errorf("failed to get host veth device %s: %v", hostVeth, err) - } - } else { - if err = netlink.LinkDel(vethLink); err != nil { - logger.Warn("failed to del hostVeth", zap.Error(err)) - return fmt.Errorf("failed to del hostVeth %s: %w", hostVeth, err) - } else { - logger.Debug("success to del hostVeth", zap.String("HostVeth", hostVeth)) + return nil } + logger.Warn(fmt.Sprintf("failed to get host veth device %s: %v", hostVeth, err)) + return fmt.Errorf("failed to get host veth device %s: %v", hostVeth, err) + } + if err = netlink.LinkDel(vethLink); err != nil { + logger.Warn("failed to del hostVeth", zap.Error(err)) + return fmt.Errorf("failed to del hostVeth %s: %w", hostVeth, err) + } + logger.Debug("success to del hostVeth", zap.String("HostVeth", hostVeth)) + } + + err = c.netns.Do(func(netNS ns.NetNS) error { + c.currentAddress, err = networking.GetAddersByName(args.IfName, netlink.FAMILY_ALL) + if err != nil { + return err + } + return nil + }) + + if err != nil { + // ignore err + logger.Warn("failed to GetAddersByName, ignore error", zap.Error(err)) + } + + for idx := range c.currentAddress { + ipNet := networking.ConvertMaxMaskIPNet(c.currentAddress[idx].IP) + err = networking.DelToRuleTable(ipNet, c.hostRuleTable) + if err != nil && !os.IsNotExist(err) { + logger.Error("failed to DelToRuleTable", zap.Int("HostRuleTable", c.hostRuleTable), zap.String("Dst", ipNet.String()), zap.Error(err)) + return fmt.Errorf("failed to DelToRuleTable: %v", err) } } logger.Info("cmdDel end") - return nil + return } diff --git a/cmd/coordinator/cmd/utils.go b/cmd/coordinator/cmd/utils.go index 2786eb6771..87b66a69fa 100644 --- a/cmd/coordinator/cmd/utils.go +++ b/cmd/coordinator/cmd/utils.go @@ -136,6 +136,7 @@ func (c *coordinator) setupNeighborhood(logger *zap.Logger) error { zap.String("PodVethName", c.podVethName), zap.String("PodVethMac", c.podVethHwAddress.String())) + // do any clean? for _, ipAddr := range c.currentAddress { if err = networking.AddStaticNeighborTable(c.hostVethName, ipAddr.IP, c.podVethHwAddress); err != nil { logger.Error(err.Error()) diff --git a/cmd/spiderpool-agent/cmd/coordinator.go b/cmd/spiderpool-agent/cmd/coordinator.go index c02c6787bf..950fbb02b5 100644 --- a/cmd/spiderpool-agent/cmd/coordinator.go +++ b/cmd/spiderpool-agent/cmd/coordinator.go @@ -56,10 +56,10 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig } config := &models.CoordinatorConfig{ - TuneMode: coord.Spec.TuneMode, - PodCIDR: coord.Status.PodCIDR, + Mode: coord.Spec.Mode, + OverlayPodCIDR: coord.Status.OverlayPodCIDR, ServiceCIDR: coord.Status.ServiceCIDR, - ExtraCIDR: coord.Spec.ExtraCIDR, + HijackCIDR: coord.Spec.HijackCIDR, PodMACPrefix: prefix, TunePodRoutes: coord.Spec.TunePodRoutes, PodDefaultRouteNIC: nic, diff --git a/cmd/spiderpool-init/cmd/client.go b/cmd/spiderpool-init/cmd/client.go index fdd3011482..436861a3ae 100644 --- a/cmd/spiderpool-init/cmd/client.go +++ b/cmd/spiderpool-init/cmd/client.go @@ -154,3 +154,29 @@ func (c *CoreClient) CheckEndpointsAvailable(ctx context.Context, namespace, nam return false } + +func (c *CoreClient) WaitMultusCNIConfigCreated(ctx context.Context, multuscniconfig *spiderpoolv2beta1.SpiderMultusConfig) error { + logger := logutils.FromContext(ctx) + + for { + err := c.Create(ctx, multuscniconfig) + if err == nil { + logger.Sugar().Infof("Succeed to create multuscniconfig %s/%s: %+v", multuscniconfig.Namespace, multuscniconfig.Name, multuscniconfig) + return nil + } + + if apierrors.IsAlreadyExists(err) { + logger.Sugar().Infof("multuscniconfig %s/%s is already exists, ignore creating", multuscniconfig.Namespace, multuscniconfig.Name) + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + interval := retryIntervalSec * time.Second + logger.Sugar().Infof("Failed to create multuscniconfig %s/%s, recreate in %s: %v", multuscniconfig.Namespace, multuscniconfig.Name, interval, err) + time.Sleep(interval) + } + } +} diff --git a/cmd/spiderpool-init/cmd/config.go b/cmd/spiderpool-init/cmd/config.go index 442be20f88..9e56d1c6c4 100644 --- a/cmd/spiderpool-init/cmd/config.go +++ b/cmd/spiderpool-init/cmd/config.go @@ -4,12 +4,20 @@ package cmd import ( + "fmt" + "k8s.io/utils/pointer" + "net/netip" "os" + "sort" "strconv" "strings" "github.com/spidernet-io/spiderpool/pkg/constant" spiderpoolip "github.com/spidernet-io/spiderpool/pkg/ip" + spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1" + + "github.com/containernetworking/cni/libcni" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -17,11 +25,12 @@ const ( ENVSpiderpoolControllerName = "SPIDERPOOL_CONTROLLER_NAME" ENVDefaultCoordinatorName = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_NAME" - ENVDefaultCoordinatorTuneMode = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_TUNE_MODE" + ENVDefaultCoordinatorTuneMode = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_MODE" ENVDefaultCoordinatorPodCIDRType = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_POD_CIDR_TYPE" ENVDefaultCoordinatorDetectGateway = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_DETECT_GATEWAY" ENVDefaultCoordinatorDetectIPConflict = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_DETECT_IP_CONFLICT" ENVDefaultCoordinatorTunePodRoutes = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_TUNE_POD_ROUTES" + ENVDefaultCoordiantorHijackCIDR = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_HIJACK_CIDR" ENVDefaultIPv4SubnetName = "SPIDERPOOL_INIT_DEFAULT_IPV4_SUBNET_NAME" ENVDefaultIPv4IPPoolName = "SPIDERPOOL_INIT_DEFAULT_IPV4_IPPOOL_NAME" @@ -34,6 +43,15 @@ const ( ENVDefaultIPv6CIDR = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_SUBNET" ENVDefaultIPv6IPRanges = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_IPRANGES" ENVDefaultIPv6Gateway = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_GATEWAY" + + ENVEnableMultusConfig = "SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG" + ENVDefaultCNIDir = "SPIDERPOOL_INIT_DEFAULT_CNI_DIR" + ENVDefaultCNIName = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME" +) + +var ( + legacyCalicoCniName = "k8s-pod-network" + calicoCniName = "calico" ) type InitDefaultConfig struct { @@ -41,13 +59,14 @@ type InitDefaultConfig struct { ControllerName string CoordinatorName string - CoordinatorTuneMode string + CoordinatorMode string CoordinatorPodCIDRType string CoordinatorPodDefaultRouteNic string CoordinatorPodMACPrefix string CoordinatorDetectGateway bool CoordinatorDetectIPConflict bool CoordinatorTunePodRoutes bool + CoordinatorHijackCIDR []string V4SubnetName string V4IPPoolName string @@ -60,6 +79,11 @@ type InitDefaultConfig struct { V6CIDR string V6IPRanges []string V6Gateway string + + // multuscniconfig + enableMultusConfig bool + DefaultCNIDir string + DefaultCNIName string } func NewInitDefaultConfig() InitDefaultConfig { @@ -80,7 +104,7 @@ func parseENVAsDefault() InitDefaultConfig { // Coordinator config.CoordinatorName = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorName), "\"", "") if len(config.CoordinatorName) != 0 { - config.CoordinatorTuneMode = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorTuneMode), "\"", "") + config.CoordinatorMode = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorTuneMode), "\"", "") config.CoordinatorPodCIDRType = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorPodCIDRType), "\"", "") edg := strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorDetectGateway), "\"", "") @@ -103,13 +127,30 @@ func parseENVAsDefault() InitDefaultConfig { logger.Sugar().Fatalf("ENV %s %s: %v", ENVDefaultCoordinatorTunePodRoutes, etpr, err) } config.CoordinatorTunePodRoutes = tpr - switch config.CoordinatorTuneMode { + switch config.CoordinatorMode { case "underlay": config.CoordinatorPodDefaultRouteNic = "eth0" case "overlay": config.CoordinatorPodDefaultRouteNic = "net1" } config.CoordinatorPodMACPrefix = "" + v := os.Getenv(ENVDefaultCoordiantorHijackCIDR) + if len(v) > 0 { + v = strings.ReplaceAll(v, "\"", "") + v = strings.ReplaceAll(v, "\\", "") + v = strings.ReplaceAll(v, "[", "") + v = strings.ReplaceAll(v, "]", "") + v = strings.ReplaceAll(v, ",", " ") + subnets := strings.Fields(v) + + for _, r := range subnets { + _, err := netip.ParsePrefix(r) + if err != nil { + logger.Sugar().Fatalf("ENV %s invalid: %v", ENVDefaultCoordiantorHijackCIDR, err) + } + } + config.CoordinatorHijackCIDR = subnets + } } else { logger.Info("Ignore creating default Coordinator") } @@ -210,7 +251,89 @@ func parseENVAsDefault() InitDefaultConfig { config.V6IPPoolName, ) } + + var err error + enableMultusConfig := strings.ReplaceAll(os.Getenv(ENVEnableMultusConfig), "\"", "") + config.enableMultusConfig, err = strconv.ParseBool(enableMultusConfig) + if err != nil { + logger.Sugar().Fatalf("ENV %s: %s invalid: %v", ENVEnableMultusConfig, enableMultusConfig, err) + } + + config.DefaultCNIDir = strings.ReplaceAll(os.Getenv(ENVDefaultCNIDir), "\"", "") + if config.DefaultCNIDir != "" { + _, err = os.ReadDir(config.DefaultCNIDir) + if err != nil { + logger.Sugar().Fatalf("ENV %s:%s invalid: %v", ENVDefaultCNIDir, config.DefaultCNIDir, err) + } + } + + config.DefaultCNIName = strings.ReplaceAll(os.Getenv(ENVDefaultCNIName), "\"", "") + logger.Sugar().Infof("Init default config: %+v", config) return config } + +func findDefaultCNIConf(cniDir string) (string, error) { + cnifiles, err := libcni.ConfFiles(cniDir, []string{".conf", ".conflist"}) + if err != nil { + return "", fmt.Errorf("failed to load cni files in %s: %v", cniDir, err) + } + + logger.Sugar().Infof("found cni config in %s: %v", cniDir, cnifiles) + + var cniPluginConfigs []string + for _, file := range cnifiles { + if strings.Contains(file, "00-multus") { + continue + } + cniPluginConfigs = append(cniPluginConfigs, file) + } + + if len(cniPluginConfigs) == 0 { + return "", nil + } + sort.Strings(cniPluginConfigs) + + logger.Sugar().Infof("the cni config file \"%s\" is the alphabetically first name in the %s directory on each node", cniPluginConfigs[0], cniDir) + return cniPluginConfigs[0], nil +} + +// parseCNIFromConfig parse cni's name and type from given cni config path +func parseCNIFromConfig(cniConfigPath string) (string, string, error) { + if strings.HasSuffix(cniConfigPath, ".conflist") { + confList, err := libcni.ConfListFromFile(cniConfigPath) + if err != nil { + return "", "", fmt.Errorf("error loading CNI conflist file %s: %v", cniConfigPath, err) + } + return confList.Name, confList.Plugins[0].Network.Type, nil + } + + conf, err := libcni.ConfFromFile(cniConfigPath) + if err != nil { + return "", "", fmt.Errorf("error loading CNI config file %s: %v", cniConfigPath, err) + } + return conf.Network.Name, conf.Network.Type, nil +} + +func getMultusCniConfig(cniName, cniType string) *spiderpoolv2beta1.SpiderMultusConfig { + annotations := make(map[string]string) + // change calico cni name from k8s-pod-network to calico + // more datails see: + // https://github.com/projectcalico/calico/issues/7837 + if cniName == legacyCalicoCniName { + cniName = calicoCniName + annotations[constant.AnnoNetAttachConfName] = legacyCalicoCniName + } + return &spiderpoolv2beta1.SpiderMultusConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: cniName, + Namespace: "kube-system", + Annotations: annotations, + }, + Spec: spiderpoolv2beta1.MultusCNIConfigSpec{ + CniType: spiderpoolv2beta1.CniType(cniType), + EnableCoordinator: pointer.Bool(false), + }, + } +} diff --git a/cmd/spiderpool-init/cmd/root.go b/cmd/spiderpool-init/cmd/root.go index c6303a5d41..25b57cd01e 100644 --- a/cmd/spiderpool-init/cmd/root.go +++ b/cmd/spiderpool-init/cmd/root.go @@ -5,6 +5,7 @@ package cmd import ( "context" + "fmt" "time" "go.uber.org/zap" @@ -42,13 +43,14 @@ func Execute() { Name: config.CoordinatorName, }, Spec: spiderpoolv2beta1.CoordinatorSpec{ - TuneMode: &config.CoordinatorTuneMode, + Mode: &config.CoordinatorMode, PodCIDRType: config.CoordinatorPodCIDRType, TunePodRoutes: &config.CoordinatorTunePodRoutes, DetectIPConflict: &config.CoordinatorDetectIPConflict, DetectGateway: &config.CoordinatorDetectGateway, PodDefaultRouteNIC: &config.CoordinatorPodDefaultRouteNic, PodMACPrefix: &config.CoordinatorPodMACPrefix, + HijackCIDR: config.CoordinatorHijackCIDR, }, } if err := client.WaitForCoordinatorCreated(ctx, coord); err != nil { @@ -146,8 +148,43 @@ func Execute() { } } + // create multuscniconfig for default network + if config.DefaultCNIName == "" { + logger.Sugar().Infof("Try to create MultusCniConfig default network in %s", config.DefaultCNIDir) + if err = InitDefaultMultusCNIConfig(ctx, client, config.DefaultCNIDir); err != nil { + logger.Fatal(err.Error()) + } + } + logger.Info("Finish init") // Wait for helm --wait. time.Sleep(300 * time.Second) } + +func InitDefaultMultusCNIConfig(ctx context.Context, client *CoreClient, cniDir string) error { + defaultCNIConfPath, err := findDefaultCNIConf(cniDir) + if err != nil { + logger.Sugar().Errorf("failed to findDefaultCNIConf: %v", err) + return fmt.Errorf("failed to findDefaultCNIConf: %v", err) + } + + if defaultCNIConfPath == "" { + // no networks in /etc/cni/net.d + logger.Sugar().Warnf("No network found in %s, Skip create multuscniconfig", cniDir) + return nil + } + + // parse default cni config + cniName, cniType, err := parseCNIFromConfig(defaultCNIConfPath) + if err != nil { + logger.Sugar().Errorf("failed to parseCNIFromConfig: %v", err) + return fmt.Errorf("failed to parseCNIFromConfig: %v", err) + } + + if err = client.WaitMultusCNIConfigCreated(ctx, getMultusCniConfig(cniName, cniType)); err != nil { + return fmt.Errorf("failed to WaitMultusCNIConfigCreated: %v", err) + } + + return nil +} diff --git a/pkg/coordinatormanager/calico_ippool_informer.go b/pkg/coordinatormanager/calico_ippool_informer.go index 3bf74af28c..4854399441 100644 --- a/pkg/coordinatormanager/calico_ippool_informer.go +++ b/pkg/coordinatormanager/calico_ippool_informer.go @@ -67,13 +67,13 @@ func (r *calicoIPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{Requeue: true}, err } - if coordinator.Status.Phase == synced && reflect.DeepEqual(coordinator.Status.PodCIDR, podCIDR) { + if coordinator.Status.Phase == synced && reflect.DeepEqual(coordinator.Status.OverlayPodCIDR, podCIDR) { return ctrl.Result{}, nil } origin := coordinator.DeepCopy() coordinator.Status.Phase = synced - coordinator.Status.PodCIDR = podCIDR + coordinator.Status.OverlayPodCIDR = podCIDR if err := r.client.Status().Patch(ctx, &coordinator, client.MergeFrom(origin)); err != nil { return ctrl.Result{Requeue: true}, err } diff --git a/pkg/coordinatormanager/coordinator_informer.go b/pkg/coordinatormanager/coordinator_informer.go index 35133e8cd5..100f8cbeeb 100644 --- a/pkg/coordinatormanager/coordinator_informer.go +++ b/pkg/coordinatormanager/coordinator_informer.go @@ -348,7 +348,7 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam cc.caliCtrlCanncel = nil } coordCopy.Status.Phase = synced - coordCopy.Status.PodCIDR = k8sPodCIDR + coordCopy.Status.OverlayPodCIDR = k8sPodCIDR case calico: if _, err := cc.ConfigmapLister.ConfigMaps(metav1.NamespaceSystem).Get(calicoConfig); err != nil { if apierrors.IsNotFound(err) { @@ -425,9 +425,9 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam break } coordCopy.Status.Phase = synced - coordCopy.Status.PodCIDR = ciliumPodCIDR + coordCopy.Status.OverlayPodCIDR = ciliumPodCIDR if ipam == "kubernetes" { - coordCopy.Status.PodCIDR = k8sPodCIDR + coordCopy.Status.OverlayPodCIDR = k8sPodCIDR } } diff --git a/pkg/coordinatormanager/coordinator_validate.go b/pkg/coordinatormanager/coordinator_validate.go index a1d4ed2a97..4bece1570e 100644 --- a/pkg/coordinatormanager/coordinator_validate.go +++ b/pkg/coordinatormanager/coordinator_validate.go @@ -51,7 +51,7 @@ func ValidateCoordinatorSpec(spec *spiderpoolv2beta1.CoordinatorSpec) *field.Err if err := validateCoordinatorPodCIDRType(spec.PodCIDRType); err != nil { return err } - if err := validateCoordinatorExtraCIDR(spec.ExtraCIDR); err != nil { + if err := validateCoordinatorExtraCIDR(spec.HijackCIDR); err != nil { return err } if err := validateCoordinatorPodMACPrefix(spec.PodMACPrefix); err != nil { diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/rbac.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/rbac.go index e0bbeeaccd..b6d7115361 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/rbac.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/rbac.go @@ -9,7 +9,7 @@ // +kubebuilder:rbac:groups=spiderpool.spidernet.io,resources=spiderreservedips,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=spiderpool.spidernet.io,resources=spidercoordinators,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=spiderpool.spidernet.io,resources=spidercoordinators/status,verbs=get;update;patch -// +kubebuilder:raac:groups=spiderpool.spidernet.io,resources=spidermultusconfigs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=spiderpool.spidernet.io,resources=spidermultusconfigs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="coordination.k8s.io",resources=leases,verbs=create;get;update // +kubebuilder:rbac:groups="apps",resources=statefulsets;deployments;replicasets;daemonsets,verbs=get;list;watch;update diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go index 636d1dd9f7..0fb7a86309 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go @@ -12,13 +12,13 @@ type CoordinatorSpec struct { // +kubebuilder:default=underlay // +kubebuilder:validation:Enum=underlay;overlay;disabled // +kubebuilder:validation:Optional - TuneMode *string `json:"tuneMode,omitempty"` + Mode *string `json:"mode,omitempty"` // +kubebuilder:validation:Required PodCIDRType string `json:"podCIDRType"` // +kubebuilder:validation:Optional - ExtraCIDR []string `json:"extraCIDR,omitempty"` + HijackCIDR []string `json:"hijackCIDR,omitempty"` // +kubebuilder:validation:Optional PodMACPrefix *string `json:"podMACPrefix,omitempty"` @@ -53,7 +53,7 @@ type CoordinatorStatus struct { Phase string `json:"phase"` // +kubebuilder:validation:Optional - PodCIDR []string `json:"podCIDR,omitempty"` + OverlayPodCIDR []string `json:"overlayPodCIDR,omitempty"` // +kubebuilder:validation:Optional ServiceCIDR []string `json:"serviceCIDR,omitempty"` diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go index f8d7bc9936..1ac905c85d 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go @@ -25,13 +25,13 @@ type SpiderMultusConfigList struct { Items []SpiderMultusConfig `json:"items"` } -// macvlan、ipvlan、sriov、custom +// calico、cilium-cni、flannel、weave-net、kube-ovn、macvlan、ipvlan、sriov、custom type CniType string // MultusCNIConfigSpec defines the desired state of SpiderMultusConfig. type MultusCNIConfigSpec struct { // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=macvlan;ipvlan;sriov;custom + // +kubebuilder:validation:Enum=calico;cilium-cni;flannel;weave-net;kube-ovn;macvlan;ipvlan;sriov;custom CniType CniType `json:"cniType"` // +kubebuilder:validation:Optional diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go index 3c4c8e7e39..0a232b2a52 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go @@ -36,13 +36,13 @@ func (in *BondConfig) DeepCopy() *BondConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CoordinatorSpec) DeepCopyInto(out *CoordinatorSpec) { *out = *in - if in.TuneMode != nil { - in, out := &in.TuneMode, &out.TuneMode + if in.Mode != nil { + in, out := &in.Mode, &out.Mode *out = new(string) **out = **in } - if in.ExtraCIDR != nil { - in, out := &in.ExtraCIDR, &out.ExtraCIDR + if in.HijackCIDR != nil { + in, out := &in.HijackCIDR, &out.HijackCIDR *out = make([]string, len(*in)) copy(*out, *in) } @@ -96,8 +96,8 @@ func (in *CoordinatorSpec) DeepCopy() *CoordinatorSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CoordinatorStatus) DeepCopyInto(out *CoordinatorStatus) { *out = *in - if in.PodCIDR != nil { - in, out := &in.PodCIDR, &out.PodCIDR + if in.OverlayPodCIDR != nil { + in, out := &in.OverlayPodCIDR, &out.OverlayPodCIDR *out = make([]string, len(*in)) copy(*out, *in) } diff --git a/pkg/multuscniconfig/multusconfig_informer.go b/pkg/multuscniconfig/multusconfig_informer.go index e54709716e..280233ce55 100644 --- a/pkg/multuscniconfig/multusconfig_informer.go +++ b/pkg/multuscniconfig/multusconfig_informer.go @@ -273,7 +273,7 @@ func (mcc *MultusConfigController) syncHandler(ctx context.Context, multusConfig Namespace: multusConfig.Namespace, Name: netAttachName, }, netAttachDef) - if nil != err { + if err != nil { if apierrors.IsNotFound(err) { isExist = false } else { @@ -356,6 +356,17 @@ func generateNetAttachDef(netAttachName string, multusConf *spiderpoolv2beta1.Sp plugins = append(plugins, coordinatorCNIConf) } + // we'll use the default CNI version 0.3.1 if the annotation doesn't have it. + cniVersion := cmd.CniVersion031 + if annCniVersion, ok := anno[constant.AnnoMultusConfigCNIVersion]; ok { + if !slices.Contains(cmd.SupportCNIVersions, annCniVersion) { + return nil, fmt.Errorf("%w: unsupported CNI version %s", constant.ErrWrongInput, annCniVersion) + } + cniVersion = annCniVersion + } + + var confStr string + var err error switch multusConfSpec.CniType { case MacVlanType: macvlanCNIConf := generateMacvlanCNIConf(*multusConfSpec) @@ -368,6 +379,11 @@ func generateNetAttachDef(netAttachName string, multusConf *spiderpoolv2beta1.Sp multusConfSpec.MacvlanConfig.Bond) plugins = append([]interface{}{subVlanCNIConf}, plugins...) } + confStr, err = marshalCniConfig2String(netAttachName, cniVersion, plugins) + if err != nil { + return nil, fmt.Errorf("failed to marshalCniConfig2String: %w", err) + } + case IpVlanType: ipvlanCNIConf := generateIPvlanCNIConf(*multusConfSpec) // head insertion @@ -379,6 +395,11 @@ func generateNetAttachDef(netAttachName string, multusConf *spiderpoolv2beta1.Sp multusConfSpec.IPVlanConfig.Bond) plugins = append([]interface{}{subVlanCNIConf}, plugins...) } + + confStr, err = marshalCniConfig2String(netAttachName, cniVersion, plugins) + if err != nil { + return nil, fmt.Errorf("failed to marshalCniConfig2String: %w", err) + } case SriovType: // SRIOV special annotation anno[constant.ResourceNameAnnot] = multusConfSpec.SriovConfig.ResourceName @@ -386,43 +407,22 @@ func generateNetAttachDef(netAttachName string, multusConf *spiderpoolv2beta1.Sp sriovCNIConf := generateSriovCNIConf(*multusConfSpec) // head insertion plugins = append([]interface{}{sriovCNIConf}, plugins...) - case CustomType: + confStr, err = marshalCniConfig2String(netAttachName, cniVersion, plugins) + if err != nil { + return nil, fmt.Errorf("failed to marshalCniConfig2String: %w", err) + } + case CalicoType, CiliumType, FlannelType, WeaveType, KubeOvnType, CustomType: + if multusConfSpec.CustomCNIConfig != nil && len(*multusConfSpec.CustomCNIConfig) > 0 { + confStr = *multusConfSpec.CustomCNIConfig + } default: // It's impossible get into the default branch return nil, fmt.Errorf("%w: unrecognized CNI type %s", constant.ErrWrongInput, multusConfSpec.CniType) } - cniVersion, ok := anno[constant.AnnoMultusConfigCNIVersion] - if !ok { - // we'll use the default CNI version 0.3.1 if the annotation doesn't have it. - cniVersion = cmd.CniVersion031 - } else { - if !slices.Contains(cmd.SupportCNIVersions, cniVersion) { - return nil, fmt.Errorf("%w: unsupported CNI version %s", constant.ErrWrongInput, cniVersion) - } - } - fmt.Printf("Length: %d, Cap: %d\n", len(plugins), cap(plugins)) - var confStr string - if multusConfSpec.CniType != CustomType { - rawList := map[string]interface{}{ - "name": netAttachName, - "cniVersion": cniVersion, - "plugins": plugins, - } - bytes, err := json.Marshal(rawList) - if nil != err { - return nil, err - } - confStr = string(bytes) - } else { - if multusConfSpec.CustomCNIConfig != nil && len(*multusConfSpec.CustomCNIConfig) > 0 { - confStr = *multusConfSpec.CustomCNIConfig - } - } - netAttachDef := &netv1.NetworkAttachmentDefinition{ ObjectMeta: metav1.ObjectMeta{ Name: netAttachName, @@ -435,7 +435,6 @@ func generateNetAttachDef(netAttachName string, multusConf *spiderpoolv2beta1.Sp Config: confStr, } } - return netAttachDef, nil } @@ -517,7 +516,7 @@ func generateSriovCNIConf(multusConfSpec spiderpoolv2beta1.MultusCNIConfigSpec) } if multusConfSpec.SriovConfig.VlanID != nil { - netConf.Vlan = pointer.Int(int(*multusConfSpec.SriovConfig.VlanID)) + netConf.Vlan = multusConfSpec.SriovConfig.VlanID } // set default IPPools for spiderpool cni configuration @@ -559,11 +558,11 @@ func generateCoordinatorCNIConf(coordinatorSpec *spiderpoolv2beta1.CoordinatorSp // coordinatorSpec could be nil, and we just need the coorinator CNI specified and use the default configuration if coordinatorSpec != nil { - if coordinatorSpec.TuneMode != nil { - coordinatorNetConf.TuneMode = coordinatorcmd.Mode(*coordinatorSpec.TuneMode) + if coordinatorSpec.Mode != nil { + coordinatorNetConf.Mode = coordinatorcmd.Mode(*coordinatorSpec.Mode) } - if len(coordinatorSpec.ExtraCIDR) != 0 { - coordinatorNetConf.ExtraCIDR = coordinatorSpec.ExtraCIDR + if len(coordinatorSpec.HijackCIDR) != 0 { + coordinatorNetConf.HijackCIDR = coordinatorSpec.HijackCIDR } if coordinatorSpec.PodMACPrefix != nil { coordinatorNetConf.MacPrefix = *coordinatorSpec.PodMACPrefix @@ -590,3 +589,17 @@ func generateCoordinatorCNIConf(coordinatorSpec *spiderpoolv2beta1.CoordinatorSp return coordinatorNetConf } + +func marshalCniConfig2String(netAttachName, cniVersion string, plugins interface{}) (string, error) { + rawList := map[string]interface{}{ + "name": netAttachName, + "cniVersion": cniVersion, + "plugins": plugins, + } + bytes, err := json.Marshal(rawList) + if err != nil { + return "", err + } + + return string(bytes), nil +} diff --git a/pkg/multuscniconfig/utils.go b/pkg/multuscniconfig/utils.go index d64c2886c2..0183073fdf 100644 --- a/pkg/multuscniconfig/utils.go +++ b/pkg/multuscniconfig/utils.go @@ -34,6 +34,11 @@ import ( ) const ( + CalicoType spiderpoolv2beta1.CniType = "calico" + CiliumType spiderpoolv2beta1.CniType = "cilium-cni" + FlannelType spiderpoolv2beta1.CniType = "flannel" + WeaveType spiderpoolv2beta1.CniType = "weave-net" + KubeOvnType spiderpoolv2beta1.CniType = "kube-ovn" MacVlanType spiderpoolv2beta1.CniType = "macvlan" IpVlanType spiderpoolv2beta1.CniType = "ipvlan" SriovType spiderpoolv2beta1.CniType = "sriov" @@ -57,7 +62,7 @@ type SRIOVNetConf struct { Type string `json:"type"` ResourceName string `json:"resourceName"` // required IPAM spiderpoolcmd.IPAMConfig `json:"ipam"` - Vlan *int `json:"vlan,omitempty"` + Vlan *int32 `json:"vlan,omitempty"` DeviceID string `json:"deviceID,omitempty"` } diff --git a/pkg/networking/ipchecking/ipchecking.go b/pkg/networking/ipchecking/ipchecking.go index 31d44dc0cc..a4da962cca 100644 --- a/pkg/networking/ipchecking/ipchecking.go +++ b/pkg/networking/ipchecking/ipchecking.go @@ -211,6 +211,8 @@ func (ipc *IPChecker) ipCheckingByNDP() error { return nil } +// sendReceiveLoop send ndp message and waiting for receive. +// Copyright Authors of mdlayher/ndp: https://github.com/mdlayher/ndp/ func (ipc *IPChecker) sendReceiveLoop(msg ndp.Message) (string, error) { var hwAddr string var err error @@ -234,6 +236,10 @@ func (ipc *IPChecker) sendReceiveLoop(msg ndp.Message) (string, error) { return "", nil } +// sendReceive send and receive ndp message,return error if error occurred. +// if the returned string isn't empty, it indicates that there are an +// IPv6 address conflict. +// Copyright Authors of mdlayher/ndp: https://github.com/mdlayher/ndp/ func (ipc *IPChecker) sendReceive(m ndp.Message) (string, error) { // Always multicast the message to the target's solicited-node multicast // group as if we have no knowledge of its MAC address. diff --git a/pkg/networking/networking/route.go b/pkg/networking/networking/route.go index a9f763120b..7c73e4d5a4 100644 --- a/pkg/networking/networking/route.go +++ b/pkg/networking/networking/route.go @@ -87,7 +87,7 @@ func DelFromRuleTable(src *net.IPNet, ruleTable int) error { rule := netlink.NewRule() rule.Table = ruleTable rule.Src = src - return netlink.RuleAdd(rule) + return netlink.RuleDel(rule) } // AddRoute add static route to specify rule table diff --git a/test/Makefile b/test/Makefile index 553099b37c..51bc77c20b 100644 --- a/test/Makefile +++ b/test/Makefile @@ -98,7 +98,6 @@ endif @echo "" @ KUBECONFIG=$(E2E_KUBECONFIG) kubectl get pods -o wide -A - .PHONY: setup_kind setup_kind: -@ kind delete cluster --name $(E2E_CLUSTER_NAME) &>/dev/null diff --git a/test/scripts/install-default-cni.sh b/test/scripts/install-default-cni.sh index d595ea1ef3..7d7855a70a 100755 --- a/test/scripts/install-default-cni.sh +++ b/test/scripts/install-default-cni.sh @@ -142,7 +142,7 @@ function install_cilium() { --set ipv6.enabled=true " ;; *) - echo "the value of IP_FAMILY: ipv4 or ipv6 or dual" + echo "the value of E2E_IP_FAMILY: ipv4 or ipv6 or dual" exit 1 esac diff --git a/test/scripts/install-multus.sh b/test/scripts/install-multus.sh index 50ee5a348c..0e382c5701 100755 --- a/test/scripts/install-multus.sh +++ b/test/scripts/install-multus.sh @@ -26,12 +26,6 @@ echo "$CURRENT_FILENAME : CLUSTER_PATH $CLUSTER_PATH " [ -z "$E2E_IP_FAMILY" ] && echo "error, miss E2E_IP_FAMILY" && exit 1 echo "$CURRENT_FILENAME : E2E_IP_FAMILY $E2E_IP_FAMILY " -[ -z "$MULTUS_DEFAULT_CNI_CALICO" ] && echo "error, miss MULTUS_DEFAULT_CNI_CALICO" && exit 1 -echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_CALICO $MULTUS_DEFAULT_CNI_CALICO " - -[ -z "$MULTUS_DEFAULT_CNI_CILIUM" ] && echo "error, miss MULTUS_DEFAULT_CNI_CILIUM" && exit 1 -echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_CILIUM $MULTUS_DEFAULT_CNI_CILIUM " - [ -z "$MULTUS_DEFAULT_CNI_NAME" ] && echo "error, miss MULTUS_DEFAULT_CNI_NAME" && exit 1 echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_NAME $MULTUS_DEFAULT_CNI_NAME " @@ -54,22 +48,6 @@ if [ ${OS} == "darwin" ]; then SED_COMMAND=gsed ; fi Install::MultusCR(){ - cat << EOF | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - - apiVersion: k8s.cni.cncf.io/v1 - kind: NetworkAttachmentDefinition - metadata: - name: ${MULTUS_DEFAULT_CNI_CALICO} - namespace: ${RELEASE_NAMESPACE} -EOF - - cat << EOF | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - - apiVersion: k8s.cni.cncf.io/v1 - kind: NetworkAttachmentDefinition - metadata: - name: ${MULTUS_DEFAULT_CNI_CILIUM} - namespace: ${RELEASE_NAMESPACE} -EOF - MACVLAN_CR_TEMPLATE=' apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition @@ -91,7 +69,7 @@ spec: } },{ "type": "coordinator", - "tuneMode": "<>" + "mode": "<>" } ] } diff --git a/vendor/github.com/containernetworking/cni/libcni/api.go b/vendor/github.com/containernetworking/cni/libcni/api.go new file mode 100644 index 0000000000..0d82a2dd3c --- /dev/null +++ b/vendor/github.com/containernetworking/cni/libcni/api.go @@ -0,0 +1,679 @@ +// Copyright 2015 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package libcni + +// Note this is the actual implementation of the CNI specification, which +// is reflected in the https://github.com/containernetworking/cni/blob/master/SPEC.md file +// it is typically bundled into runtime providers (i.e. containerd or cri-o would use this +// before calling runc or hcsshim). It is also bundled into CNI providers as well, for example, +// to add an IP to a container, to parse the configuration of the CNI and so on. + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/containernetworking/cni/pkg/invoke" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/create" + "github.com/containernetworking/cni/pkg/utils" + "github.com/containernetworking/cni/pkg/version" +) + +var ( + CacheDir = "/var/lib/cni" +) + +const ( + CNICacheV1 = "cniCacheV1" +) + +// A RuntimeConf holds the arguments to one invocation of a CNI plugin +// excepting the network configuration, with the nested exception that +// the `runtimeConfig` from the network configuration is included +// here. +type RuntimeConf struct { + ContainerID string + NetNS string + IfName string + Args [][2]string + // A dictionary of capability-specific data passed by the runtime + // to plugins as top-level keys in the 'runtimeConfig' dictionary + // of the plugin's stdin data. libcni will ensure that only keys + // in this map which match the capabilities of the plugin are passed + // to the plugin + CapabilityArgs map[string]interface{} + + // DEPRECATED. Will be removed in a future release. + CacheDir string +} + +type NetworkConfig struct { + Network *types.NetConf + Bytes []byte +} + +type NetworkConfigList struct { + Name string + CNIVersion string + DisableCheck bool + Plugins []*NetworkConfig + Bytes []byte +} + +type CNI interface { + AddNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) (types.Result, error) + CheckNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error + DelNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error + GetNetworkListCachedResult(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error) + GetNetworkListCachedConfig(net *NetworkConfigList, rt *RuntimeConf) ([]byte, *RuntimeConf, error) + + AddNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) (types.Result, error) + CheckNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error + DelNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error + GetNetworkCachedResult(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) + GetNetworkCachedConfig(net *NetworkConfig, rt *RuntimeConf) ([]byte, *RuntimeConf, error) + + ValidateNetworkList(ctx context.Context, net *NetworkConfigList) ([]string, error) + ValidateNetwork(ctx context.Context, net *NetworkConfig) ([]string, error) +} + +type CNIConfig struct { + Path []string + exec invoke.Exec + cacheDir string +} + +// CNIConfig implements the CNI interface +var _ CNI = &CNIConfig{} + +// NewCNIConfig returns a new CNIConfig object that will search for plugins +// in the given paths and use the given exec interface to run those plugins, +// or if the exec interface is not given, will use a default exec handler. +func NewCNIConfig(path []string, exec invoke.Exec) *CNIConfig { + return NewCNIConfigWithCacheDir(path, "", exec) +} + +// NewCNIConfigWithCacheDir returns a new CNIConfig object that will search for plugins +// in the given paths use the given exec interface to run those plugins, +// or if the exec interface is not given, will use a default exec handler. +// The given cache directory will be used for temporary data storage when needed. +func NewCNIConfigWithCacheDir(path []string, cacheDir string, exec invoke.Exec) *CNIConfig { + return &CNIConfig{ + Path: path, + cacheDir: cacheDir, + exec: exec, + } +} + +func buildOneConfig(name, cniVersion string, orig *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (*NetworkConfig, error) { + var err error + + inject := map[string]interface{}{ + "name": name, + "cniVersion": cniVersion, + } + // Add previous plugin result + if prevResult != nil { + inject["prevResult"] = prevResult + } + + // Ensure every config uses the same name and version + orig, err = InjectConf(orig, inject) + if err != nil { + return nil, err + } + + return injectRuntimeConfig(orig, rt) +} + +// This function takes a libcni RuntimeConf structure and injects values into +// a "runtimeConfig" dictionary in the CNI network configuration JSON that +// will be passed to the plugin on stdin. +// +// Only "capabilities arguments" passed by the runtime are currently injected. +// These capabilities arguments are filtered through the plugin's advertised +// capabilities from its config JSON, and any keys in the CapabilityArgs +// matching plugin capabilities are added to the "runtimeConfig" dictionary +// sent to the plugin via JSON on stdin. For example, if the plugin's +// capabilities include "portMappings", and the CapabilityArgs map includes a +// "portMappings" key, that key and its value are added to the "runtimeConfig" +// dictionary to be passed to the plugin's stdin. +func injectRuntimeConfig(orig *NetworkConfig, rt *RuntimeConf) (*NetworkConfig, error) { + var err error + + rc := make(map[string]interface{}) + for capability, supported := range orig.Network.Capabilities { + if !supported { + continue + } + if data, ok := rt.CapabilityArgs[capability]; ok { + rc[capability] = data + } + } + + if len(rc) > 0 { + orig, err = InjectConf(orig, map[string]interface{}{"runtimeConfig": rc}) + if err != nil { + return nil, err + } + } + + return orig, nil +} + +// ensure we have a usable exec if the CNIConfig was not given one +func (c *CNIConfig) ensureExec() invoke.Exec { + if c.exec == nil { + c.exec = &invoke.DefaultExec{ + RawExec: &invoke.RawExec{Stderr: os.Stderr}, + PluginDecoder: version.PluginDecoder{}, + } + } + return c.exec +} + +type cachedInfo struct { + Kind string `json:"kind"` + ContainerID string `json:"containerId"` + Config []byte `json:"config"` + IfName string `json:"ifName"` + NetworkName string `json:"networkName"` + CniArgs [][2]string `json:"cniArgs,omitempty"` + CapabilityArgs map[string]interface{} `json:"capabilityArgs,omitempty"` + RawResult map[string]interface{} `json:"result,omitempty"` + Result types.Result `json:"-"` +} + +// getCacheDir returns the cache directory in this order: +// 1) global cacheDir from CNIConfig object +// 2) deprecated cacheDir from RuntimeConf object +// 3) fall back to default cache directory +func (c *CNIConfig) getCacheDir(rt *RuntimeConf) string { + if c.cacheDir != "" { + return c.cacheDir + } + if rt.CacheDir != "" { + return rt.CacheDir + } + return CacheDir +} + +func (c *CNIConfig) getCacheFilePath(netName string, rt *RuntimeConf) (string, error) { + if netName == "" || rt.ContainerID == "" || rt.IfName == "" { + return "", fmt.Errorf("cache file path requires network name (%q), container ID (%q), and interface name (%q)", netName, rt.ContainerID, rt.IfName) + } + return filepath.Join(c.getCacheDir(rt), "results", fmt.Sprintf("%s-%s-%s", netName, rt.ContainerID, rt.IfName)), nil +} + +func (c *CNIConfig) cacheAdd(result types.Result, config []byte, netName string, rt *RuntimeConf) error { + cached := cachedInfo{ + Kind: CNICacheV1, + ContainerID: rt.ContainerID, + Config: config, + IfName: rt.IfName, + NetworkName: netName, + CniArgs: rt.Args, + CapabilityArgs: rt.CapabilityArgs, + } + + // We need to get type.Result into cachedInfo as JSON map + // Marshal to []byte, then Unmarshal into cached.RawResult + data, err := json.Marshal(result) + if err != nil { + return err + } + + err = json.Unmarshal(data, &cached.RawResult) + if err != nil { + return err + } + + newBytes, err := json.Marshal(&cached) + if err != nil { + return err + } + + fname, err := c.getCacheFilePath(netName, rt) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(fname), 0700); err != nil { + return err + } + + return ioutil.WriteFile(fname, newBytes, 0600) +} + +func (c *CNIConfig) cacheDel(netName string, rt *RuntimeConf) error { + fname, err := c.getCacheFilePath(netName, rt) + if err != nil { + // Ignore error + return nil + } + return os.Remove(fname) +} + +func (c *CNIConfig) getCachedConfig(netName string, rt *RuntimeConf) ([]byte, *RuntimeConf, error) { + var bytes []byte + + fname, err := c.getCacheFilePath(netName, rt) + if err != nil { + return nil, nil, err + } + bytes, err = ioutil.ReadFile(fname) + if err != nil { + // Ignore read errors; the cached result may not exist on-disk + return nil, nil, nil + } + + unmarshaled := cachedInfo{} + if err := json.Unmarshal(bytes, &unmarshaled); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal cached network %q config: %w", netName, err) + } + if unmarshaled.Kind != CNICacheV1 { + return nil, nil, fmt.Errorf("read cached network %q config has wrong kind: %v", netName, unmarshaled.Kind) + } + + newRt := *rt + if unmarshaled.CniArgs != nil { + newRt.Args = unmarshaled.CniArgs + } + newRt.CapabilityArgs = unmarshaled.CapabilityArgs + + return unmarshaled.Config, &newRt, nil +} + +func (c *CNIConfig) getLegacyCachedResult(netName, cniVersion string, rt *RuntimeConf) (types.Result, error) { + fname, err := c.getCacheFilePath(netName, rt) + if err != nil { + return nil, err + } + data, err := ioutil.ReadFile(fname) + if err != nil { + // Ignore read errors; the cached result may not exist on-disk + return nil, nil + } + + // Load the cached result + result, err := create.CreateFromBytes(data) + if err != nil { + return nil, err + } + + // Convert to the config version to ensure plugins get prevResult + // in the same version as the config. The cached result version + // should match the config version unless the config was changed + // while the container was running. + result, err = result.GetAsVersion(cniVersion) + if err != nil { + return nil, fmt.Errorf("failed to convert cached result to config version %q: %w", cniVersion, err) + } + return result, nil +} + +func (c *CNIConfig) getCachedResult(netName, cniVersion string, rt *RuntimeConf) (types.Result, error) { + fname, err := c.getCacheFilePath(netName, rt) + if err != nil { + return nil, err + } + fdata, err := ioutil.ReadFile(fname) + if err != nil { + // Ignore read errors; the cached result may not exist on-disk + return nil, nil + } + + cachedInfo := cachedInfo{} + if err := json.Unmarshal(fdata, &cachedInfo); err != nil || cachedInfo.Kind != CNICacheV1 { + return c.getLegacyCachedResult(netName, cniVersion, rt) + } + + newBytes, err := json.Marshal(&cachedInfo.RawResult) + if err != nil { + return nil, fmt.Errorf("failed to marshal cached network %q config: %w", netName, err) + } + + // Load the cached result + result, err := create.CreateFromBytes(newBytes) + if err != nil { + return nil, err + } + + // Convert to the config version to ensure plugins get prevResult + // in the same version as the config. The cached result version + // should match the config version unless the config was changed + // while the container was running. + result, err = result.GetAsVersion(cniVersion) + if err != nil { + return nil, fmt.Errorf("failed to convert cached result to config version %q: %w", cniVersion, err) + } + return result, nil +} + +// GetNetworkListCachedResult returns the cached Result of the previous +// AddNetworkList() operation for a network list, or an error. +func (c *CNIConfig) GetNetworkListCachedResult(list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) { + return c.getCachedResult(list.Name, list.CNIVersion, rt) +} + +// GetNetworkCachedResult returns the cached Result of the previous +// AddNetwork() operation for a network, or an error. +func (c *CNIConfig) GetNetworkCachedResult(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) { + return c.getCachedResult(net.Network.Name, net.Network.CNIVersion, rt) +} + +// GetNetworkListCachedConfig copies the input RuntimeConf to output +// RuntimeConf with fields updated with info from the cached Config. +func (c *CNIConfig) GetNetworkListCachedConfig(list *NetworkConfigList, rt *RuntimeConf) ([]byte, *RuntimeConf, error) { + return c.getCachedConfig(list.Name, rt) +} + +// GetNetworkCachedConfig copies the input RuntimeConf to output +// RuntimeConf with fields updated with info from the cached Config. +func (c *CNIConfig) GetNetworkCachedConfig(net *NetworkConfig, rt *RuntimeConf) ([]byte, *RuntimeConf, error) { + return c.getCachedConfig(net.Network.Name, rt) +} + +func (c *CNIConfig) addNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) { + c.ensureExec() + pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path) + if err != nil { + return nil, err + } + if err := utils.ValidateContainerID(rt.ContainerID); err != nil { + return nil, err + } + if err := utils.ValidateNetworkName(name); err != nil { + return nil, err + } + if err := utils.ValidateInterfaceName(rt.IfName); err != nil { + return nil, err + } + + newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt) + if err != nil { + return nil, err + } + + return invoke.ExecPluginWithResult(ctx, pluginPath, newConf.Bytes, c.args("ADD", rt), c.exec) +} + +// AddNetworkList executes a sequence of plugins with the ADD command +func (c *CNIConfig) AddNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) { + var err error + var result types.Result + for _, net := range list.Plugins { + result, err = c.addNetwork(ctx, list.Name, list.CNIVersion, net, result, rt) + if err != nil { + return nil, fmt.Errorf("plugin %s failed (add): %w", pluginDescription(net.Network), err) + } + } + + if err = c.cacheAdd(result, list.Bytes, list.Name, rt); err != nil { + return nil, fmt.Errorf("failed to set network %q cached result: %w", list.Name, err) + } + + return result, nil +} + +func (c *CNIConfig) checkNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) error { + c.ensureExec() + pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path) + if err != nil { + return err + } + + newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt) + if err != nil { + return err + } + + return invoke.ExecPluginWithoutResult(ctx, pluginPath, newConf.Bytes, c.args("CHECK", rt), c.exec) +} + +// CheckNetworkList executes a sequence of plugins with the CHECK command +func (c *CNIConfig) CheckNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) error { + // CHECK was added in CNI spec version 0.4.0 and higher + if gtet, err := version.GreaterThanOrEqualTo(list.CNIVersion, "0.4.0"); err != nil { + return err + } else if !gtet { + return fmt.Errorf("configuration version %q does not support the CHECK command", list.CNIVersion) + } + + if list.DisableCheck { + return nil + } + + cachedResult, err := c.getCachedResult(list.Name, list.CNIVersion, rt) + if err != nil { + return fmt.Errorf("failed to get network %q cached result: %w", list.Name, err) + } + + for _, net := range list.Plugins { + if err := c.checkNetwork(ctx, list.Name, list.CNIVersion, net, cachedResult, rt); err != nil { + return err + } + } + + return nil +} + +func (c *CNIConfig) delNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) error { + c.ensureExec() + pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path) + if err != nil { + return err + } + + newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt) + if err != nil { + return err + } + + return invoke.ExecPluginWithoutResult(ctx, pluginPath, newConf.Bytes, c.args("DEL", rt), c.exec) +} + +// DelNetworkList executes a sequence of plugins with the DEL command +func (c *CNIConfig) DelNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) error { + var cachedResult types.Result + + // Cached result on DEL was added in CNI spec version 0.4.0 and higher + if gtet, err := version.GreaterThanOrEqualTo(list.CNIVersion, "0.4.0"); err != nil { + return err + } else if gtet { + cachedResult, err = c.getCachedResult(list.Name, list.CNIVersion, rt) + if err != nil { + return fmt.Errorf("failed to get network %q cached result: %w", list.Name, err) + } + } + + for i := len(list.Plugins) - 1; i >= 0; i-- { + net := list.Plugins[i] + if err := c.delNetwork(ctx, list.Name, list.CNIVersion, net, cachedResult, rt); err != nil { + return fmt.Errorf("plugin %s failed (delete): %w", pluginDescription(net.Network), err) + } + } + _ = c.cacheDel(list.Name, rt) + + return nil +} + +func pluginDescription(net *types.NetConf) string { + if net == nil { + return "" + } + pluginType := net.Type + out := fmt.Sprintf("type=%q", pluginType) + name := net.Name + if name != "" { + out += fmt.Sprintf(" name=%q", name) + } + return out +} + +// AddNetwork executes the plugin with the ADD command +func (c *CNIConfig) AddNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) (types.Result, error) { + result, err := c.addNetwork(ctx, net.Network.Name, net.Network.CNIVersion, net, nil, rt) + if err != nil { + return nil, err + } + + if err = c.cacheAdd(result, net.Bytes, net.Network.Name, rt); err != nil { + return nil, fmt.Errorf("failed to set network %q cached result: %w", net.Network.Name, err) + } + + return result, nil +} + +// CheckNetwork executes the plugin with the CHECK command +func (c *CNIConfig) CheckNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error { + // CHECK was added in CNI spec version 0.4.0 and higher + if gtet, err := version.GreaterThanOrEqualTo(net.Network.CNIVersion, "0.4.0"); err != nil { + return err + } else if !gtet { + return fmt.Errorf("configuration version %q does not support the CHECK command", net.Network.CNIVersion) + } + + cachedResult, err := c.getCachedResult(net.Network.Name, net.Network.CNIVersion, rt) + if err != nil { + return fmt.Errorf("failed to get network %q cached result: %w", net.Network.Name, err) + } + return c.checkNetwork(ctx, net.Network.Name, net.Network.CNIVersion, net, cachedResult, rt) +} + +// DelNetwork executes the plugin with the DEL command +func (c *CNIConfig) DelNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error { + var cachedResult types.Result + + // Cached result on DEL was added in CNI spec version 0.4.0 and higher + if gtet, err := version.GreaterThanOrEqualTo(net.Network.CNIVersion, "0.4.0"); err != nil { + return err + } else if gtet { + cachedResult, err = c.getCachedResult(net.Network.Name, net.Network.CNIVersion, rt) + if err != nil { + return fmt.Errorf("failed to get network %q cached result: %w", net.Network.Name, err) + } + } + + if err := c.delNetwork(ctx, net.Network.Name, net.Network.CNIVersion, net, cachedResult, rt); err != nil { + return err + } + _ = c.cacheDel(net.Network.Name, rt) + return nil +} + +// ValidateNetworkList checks that a configuration is reasonably valid. +// - all the specified plugins exist on disk +// - every plugin supports the desired version. +// +// Returns a list of all capabilities supported by the configuration, or error +func (c *CNIConfig) ValidateNetworkList(ctx context.Context, list *NetworkConfigList) ([]string, error) { + version := list.CNIVersion + + // holding map for seen caps (in case of duplicates) + caps := map[string]interface{}{} + + errs := []error{} + for _, net := range list.Plugins { + if err := c.validatePlugin(ctx, net.Network.Type, version); err != nil { + errs = append(errs, err) + } + for c, enabled := range net.Network.Capabilities { + if !enabled { + continue + } + caps[c] = struct{}{} + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("%v", errs) + } + + // make caps list + cc := make([]string, 0, len(caps)) + for c := range caps { + cc = append(cc, c) + } + + return cc, nil +} + +// ValidateNetwork checks that a configuration is reasonably valid. +// It uses the same logic as ValidateNetworkList) +// Returns a list of capabilities +func (c *CNIConfig) ValidateNetwork(ctx context.Context, net *NetworkConfig) ([]string, error) { + caps := []string{} + for c, ok := range net.Network.Capabilities { + if ok { + caps = append(caps, c) + } + } + if err := c.validatePlugin(ctx, net.Network.Type, net.Network.CNIVersion); err != nil { + return nil, err + } + return caps, nil +} + +// validatePlugin checks that an individual plugin's configuration is sane +func (c *CNIConfig) validatePlugin(ctx context.Context, pluginName, expectedVersion string) error { + c.ensureExec() + pluginPath, err := c.exec.FindInPath(pluginName, c.Path) + if err != nil { + return err + } + if expectedVersion == "" { + expectedVersion = "0.1.0" + } + + vi, err := invoke.GetVersionInfo(ctx, pluginPath, c.exec) + if err != nil { + return err + } + for _, vers := range vi.SupportedVersions() { + if vers == expectedVersion { + return nil + } + } + return fmt.Errorf("plugin %s does not support config version %q", pluginName, expectedVersion) +} + +// GetVersionInfo reports which versions of the CNI spec are supported by +// the given plugin. +func (c *CNIConfig) GetVersionInfo(ctx context.Context, pluginType string) (version.PluginInfo, error) { + c.ensureExec() + pluginPath, err := c.exec.FindInPath(pluginType, c.Path) + if err != nil { + return nil, err + } + + return invoke.GetVersionInfo(ctx, pluginPath, c.exec) +} + +// ===== +func (c *CNIConfig) args(action string, rt *RuntimeConf) *invoke.Args { + return &invoke.Args{ + Command: action, + ContainerID: rt.ContainerID, + NetNS: rt.NetNS, + PluginArgs: rt.Args, + IfName: rt.IfName, + Path: strings.Join(c.Path, string(os.PathListSeparator)), + } +} diff --git a/vendor/github.com/containernetworking/cni/libcni/conf.go b/vendor/github.com/containernetworking/cni/libcni/conf.go new file mode 100644 index 0000000000..3cd6a59d1c --- /dev/null +++ b/vendor/github.com/containernetworking/cni/libcni/conf.go @@ -0,0 +1,270 @@ +// Copyright 2015 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package libcni + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + + "github.com/containernetworking/cni/pkg/types" +) + +type NotFoundError struct { + Dir string + Name string +} + +func (e NotFoundError) Error() string { + return fmt.Sprintf(`no net configuration with name "%s" in %s`, e.Name, e.Dir) +} + +type NoConfigsFoundError struct { + Dir string +} + +func (e NoConfigsFoundError) Error() string { + return fmt.Sprintf(`no net configurations found in %s`, e.Dir) +} + +func ConfFromBytes(bytes []byte) (*NetworkConfig, error) { + conf := &NetworkConfig{Bytes: bytes, Network: &types.NetConf{}} + if err := json.Unmarshal(bytes, conf.Network); err != nil { + return nil, fmt.Errorf("error parsing configuration: %w", err) + } + if conf.Network.Type == "" { + return nil, fmt.Errorf("error parsing configuration: missing 'type'") + } + return conf, nil +} + +func ConfFromFile(filename string) (*NetworkConfig, error) { + bytes, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("error reading %s: %w", filename, err) + } + return ConfFromBytes(bytes) +} + +func ConfListFromBytes(bytes []byte) (*NetworkConfigList, error) { + rawList := make(map[string]interface{}) + if err := json.Unmarshal(bytes, &rawList); err != nil { + return nil, fmt.Errorf("error parsing configuration list: %w", err) + } + + rawName, ok := rawList["name"] + if !ok { + return nil, fmt.Errorf("error parsing configuration list: no name") + } + name, ok := rawName.(string) + if !ok { + return nil, fmt.Errorf("error parsing configuration list: invalid name type %T", rawName) + } + + var cniVersion string + rawVersion, ok := rawList["cniVersion"] + if ok { + cniVersion, ok = rawVersion.(string) + if !ok { + return nil, fmt.Errorf("error parsing configuration list: invalid cniVersion type %T", rawVersion) + } + } + + disableCheck := false + if rawDisableCheck, ok := rawList["disableCheck"]; ok { + disableCheck, ok = rawDisableCheck.(bool) + if !ok { + return nil, fmt.Errorf("error parsing configuration list: invalid disableCheck type %T", rawDisableCheck) + } + } + + list := &NetworkConfigList{ + Name: name, + DisableCheck: disableCheck, + CNIVersion: cniVersion, + Bytes: bytes, + } + + var plugins []interface{} + plug, ok := rawList["plugins"] + if !ok { + return nil, fmt.Errorf("error parsing configuration list: no 'plugins' key") + } + plugins, ok = plug.([]interface{}) + if !ok { + return nil, fmt.Errorf("error parsing configuration list: invalid 'plugins' type %T", plug) + } + if len(plugins) == 0 { + return nil, fmt.Errorf("error parsing configuration list: no plugins in list") + } + + for i, conf := range plugins { + newBytes, err := json.Marshal(conf) + if err != nil { + return nil, fmt.Errorf("failed to marshal plugin config %d: %w", i, err) + } + netConf, err := ConfFromBytes(newBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse plugin config %d: %w", i, err) + } + list.Plugins = append(list.Plugins, netConf) + } + + return list, nil +} + +func ConfListFromFile(filename string) (*NetworkConfigList, error) { + bytes, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("error reading %s: %w", filename, err) + } + return ConfListFromBytes(bytes) +} + +func ConfFiles(dir string, extensions []string) ([]string, error) { + // In part, adapted from rkt/networking/podenv.go#listFiles + files, err := ioutil.ReadDir(dir) + switch { + case err == nil: // break + case os.IsNotExist(err): + return nil, nil + default: + return nil, err + } + + confFiles := []string{} + for _, f := range files { + if f.IsDir() { + continue + } + fileExt := filepath.Ext(f.Name()) + for _, ext := range extensions { + if fileExt == ext { + confFiles = append(confFiles, filepath.Join(dir, f.Name())) + } + } + } + return confFiles, nil +} + +func LoadConf(dir, name string) (*NetworkConfig, error) { + files, err := ConfFiles(dir, []string{".conf", ".json"}) + switch { + case err != nil: + return nil, err + case len(files) == 0: + return nil, NoConfigsFoundError{Dir: dir} + } + sort.Strings(files) + + for _, confFile := range files { + conf, err := ConfFromFile(confFile) + if err != nil { + return nil, err + } + if conf.Network.Name == name { + return conf, nil + } + } + return nil, NotFoundError{dir, name} +} + +func LoadConfList(dir, name string) (*NetworkConfigList, error) { + files, err := ConfFiles(dir, []string{".conflist"}) + if err != nil { + return nil, err + } + sort.Strings(files) + + for _, confFile := range files { + conf, err := ConfListFromFile(confFile) + if err != nil { + return nil, err + } + if conf.Name == name { + return conf, nil + } + } + + // Try and load a network configuration file (instead of list) + // from the same name, then upconvert. + singleConf, err := LoadConf(dir, name) + if err != nil { + // A little extra logic so the error makes sense + if _, ok := err.(NoConfigsFoundError); len(files) != 0 && ok { + // Config lists found but no config files found + return nil, NotFoundError{dir, name} + } + + return nil, err + } + return ConfListFromConf(singleConf) +} + +func InjectConf(original *NetworkConfig, newValues map[string]interface{}) (*NetworkConfig, error) { + config := make(map[string]interface{}) + err := json.Unmarshal(original.Bytes, &config) + if err != nil { + return nil, fmt.Errorf("unmarshal existing network bytes: %w", err) + } + + for key, value := range newValues { + if key == "" { + return nil, fmt.Errorf("keys cannot be empty") + } + + if value == nil { + return nil, fmt.Errorf("key '%s' value must not be nil", key) + } + + config[key] = value + } + + newBytes, err := json.Marshal(config) + if err != nil { + return nil, err + } + + return ConfFromBytes(newBytes) +} + +// ConfListFromConf "upconverts" a network config in to a NetworkConfigList, +// with the single network as the only entry in the list. +func ConfListFromConf(original *NetworkConfig) (*NetworkConfigList, error) { + // Re-deserialize the config's json, then make a raw map configlist. + // This may seem a bit strange, but it's to make the Bytes fields + // actually make sense. Otherwise, the generated json is littered with + // golang default values. + + rawConfig := make(map[string]interface{}) + if err := json.Unmarshal(original.Bytes, &rawConfig); err != nil { + return nil, err + } + + rawConfigList := map[string]interface{}{ + "name": original.Network.Name, + "cniVersion": original.Network.CNIVersion, + "plugins": []interface{}{rawConfig}, + } + + b, err := json.Marshal(rawConfigList) + if err != nil { + return nil, err + } + return ConfListFromBytes(b) +} diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/args.go b/vendor/github.com/containernetworking/cni/pkg/invoke/args.go new file mode 100644 index 0000000000..3cdb4bc8da --- /dev/null +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/args.go @@ -0,0 +1,128 @@ +// Copyright 2015 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package invoke + +import ( + "fmt" + "os" + "strings" +) + +type CNIArgs interface { + // For use with os/exec; i.e., return nil to inherit the + // environment from this process + // For use in delegation; inherit the environment from this + // process and allow overrides + AsEnv() []string +} + +type inherited struct{} + +var inheritArgsFromEnv inherited + +func (*inherited) AsEnv() []string { + return nil +} + +func ArgsFromEnv() CNIArgs { + return &inheritArgsFromEnv +} + +type Args struct { + Command string + ContainerID string + NetNS string + PluginArgs [][2]string + PluginArgsStr string + IfName string + Path string +} + +// Args implements the CNIArgs interface +var _ CNIArgs = &Args{} + +func (args *Args) AsEnv() []string { + env := os.Environ() + pluginArgsStr := args.PluginArgsStr + if pluginArgsStr == "" { + pluginArgsStr = stringify(args.PluginArgs) + } + + // Duplicated values which come first will be overridden, so we must put the + // custom values in the end to avoid being overridden by the process environments. + env = append(env, + "CNI_COMMAND="+args.Command, + "CNI_CONTAINERID="+args.ContainerID, + "CNI_NETNS="+args.NetNS, + "CNI_ARGS="+pluginArgsStr, + "CNI_IFNAME="+args.IfName, + "CNI_PATH="+args.Path, + ) + return dedupEnv(env) +} + +// taken from rkt/networking/net_plugin.go +func stringify(pluginArgs [][2]string) string { + entries := make([]string, len(pluginArgs)) + + for i, kv := range pluginArgs { + entries[i] = strings.Join(kv[:], "=") + } + + return strings.Join(entries, ";") +} + +// DelegateArgs implements the CNIArgs interface +// used for delegation to inherit from environments +// and allow some overrides like CNI_COMMAND +var _ CNIArgs = &DelegateArgs{} + +type DelegateArgs struct { + Command string +} + +func (d *DelegateArgs) AsEnv() []string { + env := os.Environ() + + // The custom values should come in the end to override the existing + // process environment of the same key. + env = append(env, + "CNI_COMMAND="+d.Command, + ) + return dedupEnv(env) +} + +// dedupEnv returns a copy of env with any duplicates removed, in favor of later values. +// Items not of the normal environment "key=value" form are preserved unchanged. +func dedupEnv(env []string) []string { + out := make([]string, 0, len(env)) + envMap := map[string]string{} + + for _, kv := range env { + // find the first "=" in environment, if not, just keep it + eq := strings.Index(kv, "=") + if eq < 0 { + out = append(out, kv) + continue + } + envMap[kv[:eq]] = kv[eq+1:] + } + + for k, v := range envMap { + out = append(out, fmt.Sprintf("%s=%s", k, v)) + } + + return out +} diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go b/vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go new file mode 100644 index 0000000000..8defe4dd39 --- /dev/null +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go @@ -0,0 +1,80 @@ +// Copyright 2016 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package invoke + +import ( + "context" + "os" + "path/filepath" + + "github.com/containernetworking/cni/pkg/types" +) + +func delegateCommon(delegatePlugin string, exec Exec) (string, Exec, error) { + if exec == nil { + exec = defaultExec + } + + paths := filepath.SplitList(os.Getenv("CNI_PATH")) + pluginPath, err := exec.FindInPath(delegatePlugin, paths) + if err != nil { + return "", nil, err + } + + return pluginPath, exec, nil +} + +// DelegateAdd calls the given delegate plugin with the CNI ADD action and +// JSON configuration +func DelegateAdd(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) (types.Result, error) { + pluginPath, realExec, err := delegateCommon(delegatePlugin, exec) + if err != nil { + return nil, err + } + + // DelegateAdd will override the original "CNI_COMMAND" env from process with ADD + return ExecPluginWithResult(ctx, pluginPath, netconf, delegateArgs("ADD"), realExec) +} + +// DelegateCheck calls the given delegate plugin with the CNI CHECK action and +// JSON configuration +func DelegateCheck(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) error { + pluginPath, realExec, err := delegateCommon(delegatePlugin, exec) + if err != nil { + return err + } + + // DelegateCheck will override the original CNI_COMMAND env from process with CHECK + return ExecPluginWithoutResult(ctx, pluginPath, netconf, delegateArgs("CHECK"), realExec) +} + +// DelegateDel calls the given delegate plugin with the CNI DEL action and +// JSON configuration +func DelegateDel(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) error { + pluginPath, realExec, err := delegateCommon(delegatePlugin, exec) + if err != nil { + return err + } + + // DelegateDel will override the original CNI_COMMAND env from process with DEL + return ExecPluginWithoutResult(ctx, pluginPath, netconf, delegateArgs("DEL"), realExec) +} + +// return CNIArgs used by delegation +func delegateArgs(action string) *DelegateArgs { + return &DelegateArgs{ + Command: action, + } +} diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go b/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go new file mode 100644 index 0000000000..3ad07aa8f2 --- /dev/null +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go @@ -0,0 +1,187 @@ +// Copyright 2015 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package invoke + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/create" + "github.com/containernetworking/cni/pkg/version" +) + +// Exec is an interface encapsulates all operations that deal with finding +// and executing a CNI plugin. Tests may provide a fake implementation +// to avoid writing fake plugins to temporary directories during the test. +type Exec interface { + ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) + FindInPath(plugin string, paths []string) (string, error) + Decode(jsonBytes []byte) (version.PluginInfo, error) +} + +// Plugin must return result in same version as specified in netconf; but +// for backwards compatibility reasons if the result version is empty use +// config version (rather than technically correct 0.1.0). +// https://github.com/containernetworking/cni/issues/895 +func fixupResultVersion(netconf, result []byte) (string, []byte, error) { + versionDecoder := &version.ConfigDecoder{} + confVersion, err := versionDecoder.Decode(netconf) + if err != nil { + return "", nil, err + } + + var rawResult map[string]interface{} + if err := json.Unmarshal(result, &rawResult); err != nil { + return "", nil, fmt.Errorf("failed to unmarshal raw result: %w", err) + } + + // plugin output of "null" is successfully unmarshalled, but results in a nil + // map which causes a panic when the confVersion is assigned below. + if rawResult == nil { + rawResult = make(map[string]interface{}) + } + + // Manually decode Result version; we need to know whether its cniVersion + // is empty, while built-in decoders (correctly) substitute 0.1.0 for an + // empty version per the CNI spec. + if resultVerRaw, ok := rawResult["cniVersion"]; ok { + resultVer, ok := resultVerRaw.(string) + if ok && resultVer != "" { + return resultVer, result, nil + } + } + + // If the cniVersion is not present or empty, assume the result is + // the same CNI spec version as the config + rawResult["cniVersion"] = confVersion + newBytes, err := json.Marshal(rawResult) + if err != nil { + return "", nil, fmt.Errorf("failed to remarshal fixed result: %w", err) + } + + return confVersion, newBytes, nil +} + +// For example, a testcase could pass an instance of the following fakeExec +// object to ExecPluginWithResult() to verify the incoming stdin and environment +// and provide a tailored response: +// +//import ( +// "encoding/json" +// "path" +// "strings" +//) +// +//type fakeExec struct { +// version.PluginDecoder +//} +// +//func (f *fakeExec) ExecPlugin(pluginPath string, stdinData []byte, environ []string) ([]byte, error) { +// net := &types.NetConf{} +// err := json.Unmarshal(stdinData, net) +// if err != nil { +// return nil, fmt.Errorf("failed to unmarshal configuration: %v", err) +// } +// pluginName := path.Base(pluginPath) +// if pluginName != net.Type { +// return nil, fmt.Errorf("plugin name %q did not match config type %q", pluginName, net.Type) +// } +// for _, e := range environ { +// // Check environment for forced failure request +// parts := strings.Split(e, "=") +// if len(parts) > 0 && parts[0] == "FAIL" { +// return nil, fmt.Errorf("failed to execute plugin %s", pluginName) +// } +// } +// return []byte("{\"CNIVersion\":\"0.4.0\"}"), nil +//} +// +//func (f *fakeExec) FindInPath(plugin string, paths []string) (string, error) { +// if len(paths) > 0 { +// return path.Join(paths[0], plugin), nil +// } +// return "", fmt.Errorf("failed to find plugin %s in paths %v", plugin, paths) +//} + +func ExecPluginWithResult(ctx context.Context, pluginPath string, netconf []byte, args CNIArgs, exec Exec) (types.Result, error) { + if exec == nil { + exec = defaultExec + } + + stdoutBytes, err := exec.ExecPlugin(ctx, pluginPath, netconf, args.AsEnv()) + if err != nil { + return nil, err + } + + resultVersion, fixedBytes, err := fixupResultVersion(netconf, stdoutBytes) + if err != nil { + return nil, err + } + + return create.Create(resultVersion, fixedBytes) +} + +func ExecPluginWithoutResult(ctx context.Context, pluginPath string, netconf []byte, args CNIArgs, exec Exec) error { + if exec == nil { + exec = defaultExec + } + _, err := exec.ExecPlugin(ctx, pluginPath, netconf, args.AsEnv()) + return err +} + +// GetVersionInfo returns the version information available about the plugin. +// For recent-enough plugins, it uses the information returned by the VERSION +// command. For older plugins which do not recognize that command, it reports +// version 0.1.0 +func GetVersionInfo(ctx context.Context, pluginPath string, exec Exec) (version.PluginInfo, error) { + if exec == nil { + exec = defaultExec + } + args := &Args{ + Command: "VERSION", + + // set fake values required by plugins built against an older version of skel + NetNS: "dummy", + IfName: "dummy", + Path: "dummy", + } + stdin := []byte(fmt.Sprintf(`{"cniVersion":%q}`, version.Current())) + stdoutBytes, err := exec.ExecPlugin(ctx, pluginPath, stdin, args.AsEnv()) + if err != nil { + if err.Error() == "unknown CNI_COMMAND: VERSION" { + return version.PluginSupports("0.1.0"), nil + } + return nil, err + } + + return exec.Decode(stdoutBytes) +} + +// DefaultExec is an object that implements the Exec interface which looks +// for and executes plugins from disk. +type DefaultExec struct { + *RawExec + version.PluginDecoder +} + +// DefaultExec implements the Exec interface +var _ Exec = &DefaultExec{} + +var defaultExec = &DefaultExec{ + RawExec: &RawExec{Stderr: os.Stderr}, +} diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/find.go b/vendor/github.com/containernetworking/cni/pkg/invoke/find.go new file mode 100644 index 0000000000..e62029eb78 --- /dev/null +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/find.go @@ -0,0 +1,48 @@ +// Copyright 2015 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package invoke + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// FindInPath returns the full path of the plugin by searching in the provided path +func FindInPath(plugin string, paths []string) (string, error) { + if plugin == "" { + return "", fmt.Errorf("no plugin name provided") + } + + if strings.ContainsRune(plugin, os.PathSeparator) { + return "", fmt.Errorf("invalid plugin name: %s", plugin) + } + + if len(paths) == 0 { + return "", fmt.Errorf("no paths provided") + } + + for _, path := range paths { + for _, fe := range ExecutableFileExtensions { + fullpath := filepath.Join(path, plugin) + fe + if fi, err := os.Stat(fullpath); err == nil && fi.Mode().IsRegular() { + return fullpath, nil + } + } + } + + return "", fmt.Errorf("failed to find plugin %q in path %s", plugin, paths) +} diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/os_unix.go b/vendor/github.com/containernetworking/cni/pkg/invoke/os_unix.go new file mode 100644 index 0000000000..9bcfb45536 --- /dev/null +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/os_unix.go @@ -0,0 +1,20 @@ +// Copyright 2016 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package invoke + +// Valid file extensions for plugin executables. +var ExecutableFileExtensions = []string{""} diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/os_windows.go b/vendor/github.com/containernetworking/cni/pkg/invoke/os_windows.go new file mode 100644 index 0000000000..7665125b13 --- /dev/null +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/os_windows.go @@ -0,0 +1,18 @@ +// Copyright 2016 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package invoke + +// Valid file extensions for plugin executables. +var ExecutableFileExtensions = []string{".exe", ""} diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/raw_exec.go b/vendor/github.com/containernetworking/cni/pkg/invoke/raw_exec.go new file mode 100644 index 0000000000..5ab5cc8857 --- /dev/null +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/raw_exec.go @@ -0,0 +1,88 @@ +// Copyright 2016 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package invoke + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "github.com/containernetworking/cni/pkg/types" +) + +type RawExec struct { + Stderr io.Writer +} + +func (e *RawExec) ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) { + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + c := exec.CommandContext(ctx, pluginPath) + c.Env = environ + c.Stdin = bytes.NewBuffer(stdinData) + c.Stdout = stdout + c.Stderr = stderr + + // Retry the command on "text file busy" errors + for i := 0; i <= 5; i++ { + err := c.Run() + + // Command succeeded + if err == nil { + break + } + + // If the plugin is currently about to be written, then we wait a + // second and try it again + if strings.Contains(err.Error(), "text file busy") { + time.Sleep(time.Second) + continue + } + + // All other errors except than the busy text file + return nil, e.pluginErr(err, stdout.Bytes(), stderr.Bytes()) + } + + // Copy stderr to caller's buffer in case plugin printed to both + // stdout and stderr for some reason. Ignore failures as stderr is + // only informational. + if e.Stderr != nil && stderr.Len() > 0 { + _, _ = stderr.WriteTo(e.Stderr) + } + return stdout.Bytes(), nil +} + +func (e *RawExec) pluginErr(err error, stdout, stderr []byte) error { + emsg := types.Error{} + if len(stdout) == 0 { + if len(stderr) == 0 { + emsg.Msg = fmt.Sprintf("netplugin failed with no error message: %v", err) + } else { + emsg.Msg = fmt.Sprintf("netplugin failed: %q", string(stderr)) + } + } else if perr := json.Unmarshal(stdout, &emsg); perr != nil { + emsg.Msg = fmt.Sprintf("netplugin failed but error parsing its diagnostic message %q: %v", string(stdout), perr) + } + return &emsg +} + +func (e *RawExec) FindInPath(plugin string, paths []string) (string, error) { + return FindInPath(plugin, paths) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 82811b3dfb..5c4799abd8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -22,6 +22,8 @@ github.com/beorn7/perks/quantile github.com/cespare/xxhash/v2 # github.com/containernetworking/cni v1.1.2 ## explicit; go 1.14 +github.com/containernetworking/cni/libcni +github.com/containernetworking/cni/pkg/invoke github.com/containernetworking/cni/pkg/skel github.com/containernetworking/cni/pkg/types github.com/containernetworking/cni/pkg/types/020