diff --git a/pkg/app/pipedv1/plugin/kubernetes/provider/kubernetes.go b/pkg/app/pipedv1/plugin/kubernetes/provider/kubernetes.go new file mode 100644 index 0000000000..04d14d2cd1 --- /dev/null +++ b/pkg/app/pipedv1/plugin/kubernetes/provider/kubernetes.go @@ -0,0 +1,19 @@ +// Copyright 2024 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +const ( + AnnotationOrder = "pipecd.dev/order" // The order number of resource used to sort them before using. +) diff --git a/pkg/app/pipedv1/plugin/kubernetes/provider/loader.go b/pkg/app/pipedv1/plugin/kubernetes/provider/loader.go index aeb59eb1e0..87511789e8 100644 --- a/pkg/app/pipedv1/plugin/kubernetes/provider/loader.go +++ b/pkg/app/pipedv1/plugin/kubernetes/provider/loader.go @@ -14,6 +14,172 @@ package provider +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" + + "github.com/pipe-cd/pipecd/pkg/model" +) + +type TemplatingMethod string + +const ( + TemplatingMethodHelm TemplatingMethod = "helm" + TemplatingMethodKustomize TemplatingMethod = "kustomize" + TemplatingMethodNone TemplatingMethod = "none" +) + type LoaderInput struct { + AppDir string + ConfigFilename string + Manifests []string + + Namespace string + TemplatingMethod TemplatingMethod + // TODO: define fields for LoaderInput. } + +type Loader struct { +} + +func (l *Loader) LoadManifests(input LoaderInput) (manifests []Manifest, err error) { + defer func() { + // Override namespace if set because ParseManifests does not parse it + // if namespace is not explicitly specified in the manifests. + setNamespace(manifests, input.Namespace) + sortManifests(manifests) + }() + + switch input.TemplatingMethod { + case TemplatingMethodHelm: + return nil, errors.New("not implemented yet") + case TemplatingMethodKustomize: + return nil, errors.New("not implemented yet") + case TemplatingMethodNone: + return LoadPlainYAMLManifests(input.AppDir, input.Manifests, input.ConfigFilename) + default: + return nil, fmt.Errorf("unsupported templating method %s", input.TemplatingMethod) + } +} + +func setNamespace(manifests []Manifest, namespace string) { + if namespace == "" { + return + } + for i := range manifests { + manifests[i].Key.Namespace = namespace + } +} + +func sortManifests(manifests []Manifest) { + if len(manifests) < 2 { + return + } + + slices.SortFunc(manifests, func(a, b Manifest) int { + iAns := a.Body.GetAnnotations() + // Ignore the converting error since it is not so much important. + iIndex, _ := strconv.Atoi(iAns[AnnotationOrder]) + + jAns := b.Body.GetAnnotations() + // Ignore the converting error since it is not so much important. + jIndex, _ := strconv.Atoi(jAns[AnnotationOrder]) + + return iIndex - jIndex + }) +} + +func LoadPlainYAMLManifests(dir string, names []string, configFilename string) ([]Manifest, error) { + // If no name was specified we have to walk the app directory to collect the manifest list. + if len(names) == 0 { + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if path == dir { + return nil + } + if d.IsDir() { + return fs.SkipDir + } + if ext := filepath.Ext(d.Name()); ext != ".yaml" && ext != ".yml" && ext != ".json" { + return nil + } + if model.IsApplicationConfigFile(d.Name()) { + // MEMO: can we remove this check because we have configFilename? + return nil + } + if d.Name() == configFilename { + return nil + } + names = append(names, d.Name()) + return nil + }) + if err != nil { + return nil, err + } + } + + manifests := make([]Manifest, 0, len(names)) + for _, name := range names { + path := filepath.Join(dir, name) + ms, err := LoadManifestsFromYAMLFile(path) + if err != nil { + return nil, fmt.Errorf("failed to load manifest at %s (%w)", path, err) + } + manifests = append(manifests, ms...) + } + + return manifests, nil +} + +// LoadManifestsFromYAMLFile loads the manifests from the given file. +func LoadManifestsFromYAMLFile(path string) ([]Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return ParseManifests(string(data)) +} + +// ParseManifests parses the given data and returns a list of Manifest. +func ParseManifests(data string) ([]Manifest, error) { + const separator = "\n---" + var ( + parts = strings.Split(data, separator) + manifests = make([]Manifest, 0, len(parts)) + ) + + for i, part := range parts { + // Ignore all the cases where no content between separator. + if len(strings.TrimSpace(part)) == 0 { + continue + } + // Append new line which trim by document separator. + if i != len(parts)-1 { + part += "\n" + } + var obj unstructured.Unstructured + if err := yaml.Unmarshal([]byte(part), &obj); err != nil { + return nil, err + } + if len(obj.Object) == 0 { + continue + } + manifests = append(manifests, Manifest{ + Key: MakeResourceKey(&obj), + Body: &obj, + }) + } + return manifests, nil +} diff --git a/pkg/app/pipedv1/plugin/kubernetes/provider/loader_test.go b/pkg/app/pipedv1/plugin/kubernetes/provider/loader_test.go new file mode 100644 index 0000000000..2f17402902 --- /dev/null +++ b/pkg/app/pipedv1/plugin/kubernetes/provider/loader_test.go @@ -0,0 +1,339 @@ +// Copyright 2024 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestParseManifests(t *testing.T) { + tests := []struct { + name string + data string + want []Manifest + wantErr bool + }{ + { + name: "single manifest", + data: ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-config +`, + want: []Manifest{ + { + Key: ResourceKey{ + APIVersion: "v1", + Kind: "ConfigMap", + Name: "test-config", + }, + Body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test-config", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "multiple manifests", + data: ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-config +--- +apiVersion: v1 +kind: Service +metadata: + name: test-service +`, + want: []Manifest{ + { + Key: ResourceKey{ + APIVersion: "v1", + Kind: "ConfigMap", + Name: "test-config", + }, + Body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test-config", + }, + }, + }, + }, + { + Key: ResourceKey{ + APIVersion: "v1", + Kind: "Service", + Name: "test-service", + }, + Body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Service", + "metadata": map[string]interface{}{ + "name": "test-service", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "invalid manifest", + data: ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-config +--- +invalid yaml +`, + want: nil, + wantErr: true, + }, + { + name: "empty manifest", + data: ` +--- +`, + want: []Manifest{}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseManifests(tt.data) + if (err != nil) != tt.wantErr { + t.Errorf("ParseManifests() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseManifests() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLoadPlainYAMLManifests(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + dir string + names []string + configFilename string + setup func(dir string) error + want []Manifest + wantErr bool + }{ + { + name: "load single manifest", + dir: "testdata/single", + names: []string{"configmap.yaml"}, + configFilename: "pipecd-config.yaml", + setup: func(dir string) error { + return os.WriteFile(filepath.Join(dir, "configmap.yaml"), []byte(` +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-config +`), 0644) + }, + want: []Manifest{ + { + Key: ResourceKey{ + APIVersion: "v1", + Kind: "ConfigMap", + Name: "test-config", + }, + Body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test-config", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "ignore config file", + dir: "testdata/ignore-config", + names: []string{}, + configFilename: "pipecd-config.yaml", + setup: func(dir string) error { + // Place dummy files to ensure the loader ignores them. + if err := os.WriteFile(filepath.Join(dir, "pipecd-config.yaml"), []byte(` +apiVersion: v1 +kind: ConfigMap +metadata: + name: pipecd-config +`), 0644); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "service.yaml"), []byte(` +apiVersion: v1 +kind: Service +metadata: + name: test-service +`), 0644) + }, + want: []Manifest{ + { + Key: ResourceKey{ + APIVersion: "v1", + Kind: "Service", + Name: "test-service", + }, + Body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Service", + "metadata": map[string]interface{}{ + "name": "test-service", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "load multiple manifests", + dir: "testdata/multiple", + names: []string{"configmap.yaml", "service.yaml"}, + configFilename: "pipecd-config.yaml", + setup: func(dir string) error { + if err := os.WriteFile(filepath.Join(dir, "configmap.yaml"), []byte(` +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-config +`), 0644); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "service.yaml"), []byte(` +apiVersion: v1 +kind: Service +metadata: + name: test-service +`), 0644) + }, + want: []Manifest{ + { + Key: ResourceKey{ + APIVersion: "v1", + Kind: "ConfigMap", + Name: "test-config", + }, + Body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test-config", + }, + }, + }, + }, + { + Key: ResourceKey{ + APIVersion: "v1", + Kind: "Service", + Name: "test-service", + }, + Body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Service", + "metadata": map[string]interface{}{ + "name": "test-service", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "invalid manifest", + dir: "testdata/invalid", + names: []string{"invalid.yaml"}, + configFilename: "pipecd-config.yaml", + setup: func(dir string) error { + return os.WriteFile(filepath.Join(dir, "invalid.yaml"), []byte(` +invalid yaml content +`), 0644) + }, + want: nil, + wantErr: true, + }, + { + name: "no manifests", + dir: "testdata/empty", + names: []string{}, + configFilename: "pipecd-config.yaml", + setup: func(dir string) error { + return nil + }, + want: []Manifest{}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + dir := filepath.Join(t.TempDir(), tt.dir) + require.NoError(t, os.MkdirAll(dir, 0755)) + + if tt.setup != nil { + require.NoError(t, tt.setup(dir)) + } + + got, err := LoadPlainYAMLManifests(dir, tt.names, tt.configFilename) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.ElementsMatch(t, tt.want, got) + }) + } +} diff --git a/pkg/app/pipedv1/plugin/kubernetes/provider/manifest.go b/pkg/app/pipedv1/plugin/kubernetes/provider/manifest.go index 9856d60558..93b3cc7b9e 100644 --- a/pkg/app/pipedv1/plugin/kubernetes/provider/manifest.go +++ b/pkg/app/pipedv1/plugin/kubernetes/provider/manifest.go @@ -18,7 +18,7 @@ import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" // Manifest represents a Kubernetes resource manifest. type Manifest struct { - // TODO: define ResourceKey and add as a field here. + Key ResourceKey Body *unstructured.Unstructured } diff --git a/pkg/app/pipedv1/plugin/kubernetes/provider/resource.go b/pkg/app/pipedv1/plugin/kubernetes/provider/resource.go index 35d95aff8f..3337e1d4d6 100644 --- a/pkg/app/pipedv1/plugin/kubernetes/provider/resource.go +++ b/pkg/app/pipedv1/plugin/kubernetes/provider/resource.go @@ -14,4 +14,37 @@ package provider +import ( + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + + + const KindDeployment = "Deployment" + +type ResourceKey struct { + APIVersion string + Kind string + Namespace string + Name string +} + +func (k ResourceKey) String() string { + return fmt.Sprintf("%s:%s:%s:%s", k.APIVersion, k.Kind, k.Namespace, k.Name) +} + +func (k ResourceKey) ReadableString() string { + return fmt.Sprintf("name=%q, kind=%q, namespace=%q, apiVersion=%q", k.Name, k.Kind, k.Namespace, k.APIVersion) +} + +func MakeResourceKey(obj *unstructured.Unstructured) ResourceKey { + k := ResourceKey{ + APIVersion: obj.GetAPIVersion(), + Kind: obj.GetKind(), + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + return k +}