From 6b40867122d20e31ce35a54fd7926f7adf4a340c Mon Sep 17 00:00:00 2001 From: Dat Dao Date: Thu, 5 Dec 2024 13:11:45 +0700 Subject: [PATCH] internal/controller: use hcl pkg to generate atlas.hcl for declarative flow --- api/v1alpha1/atlasschema_types.go | 66 +++++++- charts/atlas-operator/templates/crds/crd.yaml | 6 +- .../crd/bases/db.atlasgo.io_atlasschemas.yaml | 6 +- go.mod | 4 +- internal/controller/atlasschema_controller.go | 142 ++++++++++++++---- .../controller/atlasschema_controller_test.go | 39 ++--- internal/controller/common.go | 10 ++ .../controller/templates/atlas_schema.tmpl | 88 ----------- 8 files changed, 208 insertions(+), 153 deletions(-) delete mode 100644 internal/controller/templates/atlas_schema.tmpl diff --git a/api/v1alpha1/atlasschema_types.go b/api/v1alpha1/atlasschema_types.go index 806f69e..2ef3ab0 100644 --- a/api/v1alpha1/atlasschema_types.go +++ b/api/v1alpha1/atlasschema_types.go @@ -22,6 +22,8 @@ import ( "path/filepath" "strings" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/zclconf/go-cty/cty" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -107,8 +109,8 @@ type ( // Lint defines the linting policies to apply before applying the schema. Lint struct { Destructive *CheckConfig `json:"destructive,omitempty"` - // Review defines the review policy to apply after linting the schema changes. - // +kubebuilder:default=ERROR + // Review defines the review policy to apply after linting the schema changes (default: "ERROR"). + // Atlas Cloud login is required. Review LintReview `json:"review,omitempty"` } // CheckConfig defines the configuration of a linting check. @@ -266,3 +268,63 @@ func (s Schema) DesiredState(ctx context.Context, r client.Reader, ns string) (* } return nil, nil, fmt.Errorf("no desired state specified") } + +// AsBlock returns the HCL block representation of the diff. +func (d Diff) AsBlock() *hclwrite.Block { + blk := hclwrite.NewBlock("diff", nil) + body := blk.Body() + if v := d.ConcurrentIndex; v != nil { + b := body.AppendNewBlock("concurrent_index", nil).Body() + b.SetAttributeValue("create", cty.BoolVal(v.Create)) + b.SetAttributeValue("drop", cty.BoolVal(v.Drop)) + } + if v := d.Skip; v != nil { + b := body.AppendNewBlock("skip", nil).Body() + if v.AddSchema { + b.SetAttributeValue("add_schema", cty.BoolVal(v.AddSchema)) + } + if v.DropSchema { + b.SetAttributeValue("drop_schema", cty.BoolVal(v.DropSchema)) + } + if v.ModifySchema { + b.SetAttributeValue("modify_schema", cty.BoolVal(v.ModifySchema)) + } + if v.AddTable { + b.SetAttributeValue("add_table", cty.BoolVal(v.AddTable)) + } + if v.DropTable { + b.SetAttributeValue("drop_table", cty.BoolVal(v.DropTable)) + } + if v.ModifyTable { + b.SetAttributeValue("modify_table", cty.BoolVal(v.ModifyTable)) + } + if v.AddColumn { + b.SetAttributeValue("add_column", cty.BoolVal(v.AddColumn)) + } + if v.DropColumn { + b.SetAttributeValue("drop_column", cty.BoolVal(v.DropColumn)) + } + if v.ModifyColumn { + b.SetAttributeValue("modify_column", cty.BoolVal(v.ModifyColumn)) + } + if v.AddIndex { + b.SetAttributeValue("add_index", cty.BoolVal(v.AddIndex)) + } + if v.DropIndex { + b.SetAttributeValue("drop_index", cty.BoolVal(v.DropIndex)) + } + if v.ModifyIndex { + b.SetAttributeValue("modify_index", cty.BoolVal(v.ModifyIndex)) + } + if v.AddForeignKey { + b.SetAttributeValue("add_foreign_key", cty.BoolVal(v.AddForeignKey)) + } + if v.DropForeignKey { + b.SetAttributeValue("drop_foreign_key", cty.BoolVal(v.DropForeignKey)) + } + if v.ModifyForeignKey { + b.SetAttributeValue("modify_foreign_key", cty.BoolVal(v.ModifyForeignKey)) + } + } + return blk +} diff --git a/charts/atlas-operator/templates/crds/crd.yaml b/charts/atlas-operator/templates/crds/crd.yaml index 7ce239f..62ca8cb 100644 --- a/charts/atlas-operator/templates/crds/crd.yaml +++ b/charts/atlas-operator/templates/crds/crd.yaml @@ -719,9 +719,9 @@ spec: type: boolean type: object review: - default: ERROR - description: Review defines the review policy to apply after - linting the schema changes. + description: |- + Review defines the review policy to apply after linting the schema changes (default: "ERROR"). + Atlas Cloud login is required. enum: - ALWAYS - WARNING diff --git a/config/crd/bases/db.atlasgo.io_atlasschemas.yaml b/config/crd/bases/db.atlasgo.io_atlasschemas.yaml index db800a4..218c7ab 100644 --- a/config/crd/bases/db.atlasgo.io_atlasschemas.yaml +++ b/config/crd/bases/db.atlasgo.io_atlasschemas.yaml @@ -308,9 +308,9 @@ spec: type: boolean type: object review: - default: ERROR - description: Review defines the review policy to apply after - linting the schema changes. + description: |- + Review defines the review policy to apply after linting the schema changes (default: "ERROR"). + Atlas Cloud login is required. enum: - ALWAYS - WARNING diff --git a/go.mod b/go.mod index fb483b7..cca5017 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,10 @@ toolchain go1.23.2 require ( ariga.io/atlas v0.28.1 ariga.io/atlas-go-sdk v0.6.4 + github.com/hashicorp/hcl/v2 v2.18.1 github.com/rogpeppe/go-internal v1.13.1 github.com/stretchr/testify v1.9.0 + github.com/zclconf/go-cty v1.14.4 golang.org/x/mod v0.21.0 k8s.io/api v0.31.0 k8s.io/apimachinery v0.31.1 @@ -42,7 +44,6 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/hashicorp/hcl/v2 v2.18.1 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -59,7 +60,6 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect - github.com/zclconf/go-cty v1.14.4 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect diff --git a/internal/controller/atlasschema_controller.go b/internal/controller/atlasschema_controller.go index 0d0a693..e199a35 100644 --- a/internal/controller/atlasschema_controller.go +++ b/internal/controller/atlasschema_controller.go @@ -15,6 +15,7 @@ package controller import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -24,7 +25,6 @@ import ( "net/url" "path" "path/filepath" - "strconv" "strings" "time" @@ -42,6 +42,8 @@ import ( "github.com/ariga/atlas-operator/api/v1alpha1" dbv1alpha1 "github.com/ariga/atlas-operator/api/v1alpha1" "github.com/ariga/atlas-operator/internal/controller/watch" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/zclconf/go-cty/cty" ) //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;update;delete;get;list;watch;create;update;patch;delete @@ -167,6 +169,16 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Create a working directory for the Atlas CLI // The working directory contains the atlas.hcl config. wd, err := atlasexec.NewWorkingDir(opts...) + // This function will be used to edit and re-render the atlas.hcl file in the working directory. + editAtlasHCL := func(fn func(m *managedData)) error { + fn(data) + var buf bytes.Buffer + if err := data.render(&buf); err != nil { + return err + } + _, err = wd.WriteFile("atlas.hcl", buf.Bytes()) + return err + } if err != nil { res.SetNotReady("CreatingWorkingDir", err.Error()) r.recordErrEvent(res, err) @@ -194,21 +206,15 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) switch desiredURL := data.Desired.String(); { // The resource is connected to Atlas Cloud. case whoami != nil: - vars := atlasexec.Vars2{ - "lint_destructive": "true", - "lint_review": dbv1alpha1.LintReviewError, - } - if p := data.Policy; p != nil && p.Lint != nil { - if d := p.Lint.Destructive; d != nil { - vars["lint_destructive"] = strconv.FormatBool(d.Error) - } - if r := p.Lint.Review; r != "" { - vars["lint_review"] = r - } + err = editAtlasHCL(func(m *managedData) { + m.enableDestructive(false) + m.setLintReview(dbv1alpha1.LintReviewError, false) + }) + if err != nil { + return result(err) } params := &atlasexec.SchemaApplyParams{ Env: data.EnvName, - Vars: vars, To: desiredURL, TxMode: string(data.TxMode), } @@ -229,7 +235,6 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) // This ensures push is idempotent. tag, err := cli.SchemaInspect(ctx, &atlasexec.SchemaInspectParams{ Env: data.EnvName, - Vars: vars, URL: desiredURL, Format: `{{ .Hash | base64url }}`, }) @@ -245,7 +250,6 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) } state, err := cli.SchemaPush(ctx, &atlasexec.SchemaPushParams{ Env: data.EnvName, - Vars: vars, Name: path.Join(repo.Host, repo.Path), Tag: fmt.Sprintf("operator-plan-%.8s", strings.ToLower(tag)), URL: []string{desiredURL}, @@ -266,7 +270,6 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Create a new plan for the pending changes. plan, err := cli.SchemaPlan(ctx, &atlasexec.SchemaPlanParams{ Env: data.EnvName, - Vars: vars, Repo: repo.String(), From: []string{"env://url"}, To: []string{desiredURL}, @@ -302,7 +305,6 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) // List the schema plans to check if there are any plans. switch plans, err := cli.SchemaPlanList(ctx, &atlasexec.SchemaPlanListParams{ Env: data.EnvName, - Vars: vars, Repo: repo.String(), From: []string{"env://url"}, To: []string{desiredURL}, @@ -330,7 +332,7 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) r.recordErrEvent(res, err) return result(err) // There are no pending plans, but Atlas has been asked to review the changes ALWAYS. - case len(plans) == 0 && vars["lint_review"] == dbv1alpha1.LintReviewAlways: + case len(plans) == 0 && data.Policy.Lint.Review == dbv1alpha1.LintReviewAlways: // Create a plan for the pending changes. return createPlan() // The plan is pending approval, show the plan to the user. @@ -358,9 +360,14 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // Verify the first run doesn't contain destructive changes. case res.Status.LastApplied == 0: - err = r.lint(ctx, wd, data, atlasexec.Vars2{ - "lint_destructive": "true", - }) + // For the first run, force the destructive linting policy to true. + // Then revert it back to the original value after the linting is done. + if err = editAtlasHCL(func(m *managedData) { + m.enableDestructive(true) + }); err != nil { + return result(err) + } + err = r.lint(ctx, wd, data, nil) switch d := (*destructiveErr)(nil); { case errors.As(err, &d): reason, msg := d.FirstRun() @@ -378,6 +385,12 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) r.recordErrEvent(res, err) return result(err) } + // Revert the destructive linting policy back to the original value. + if err = editAtlasHCL(func(m *managedData) { + m.Policy.Lint.Destructive.Error = false + }); err != nil { + return result(err) + } report, err = cli.SchemaApply(ctx, &atlasexec.SchemaApplyParams{ Env: data.EnvName, To: desiredURL, @@ -386,13 +399,7 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) }) // Run the linting policy. case data.shouldLint(): - vars := atlasexec.Vars2{} - if p := data.Policy; p != nil && p.Lint != nil { - if d := p.Lint.Destructive; d != nil { - vars["lint_destructive"] = strconv.FormatBool(d.Error) - } - } - if err = r.lint(ctx, wd, data, vars); err != nil { + if err = r.lint(ctx, wd, data, nil); err != nil { reason, msg := "LintPolicyError", err.Error() res.SetNotReady(reason, msg) r.recorder.Event(res, corev1.EventTypeWarning, reason, msg) @@ -404,7 +411,6 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) } report, err = cli.SchemaApply(ctx, &atlasexec.SchemaApplyParams{ Env: data.EnvName, - Vars: vars, To: desiredURL, TxMode: string(data.TxMode), AutoApprove: true, @@ -598,7 +604,83 @@ func (d *managedData) render(w io.Writer) error { if d.Desired == nil { return errors.New("the desired state is not set") } - return tmpl.ExecuteTemplate(w, "atlas_schema.tmpl", d) + f := hclwrite.NewFile() + fBody := f.Body() + for _, b := range d.asBlocks() { + fBody.AppendBlock(b) + } + if _, err := f.WriteTo(w); err != nil { + return err + } + return nil +} + +// enableDestructive enables the linting policy for destructive changes. +// If the force is set to true, it will override the existing value. +func (d *managedData) enableDestructive(force bool) { + check := &dbv1alpha1.CheckConfig{Error: true} + destructive := &dbv1alpha1.Lint{Destructive: check} + switch { + case d.Policy == nil: + d.Policy = &dbv1alpha1.Policy{Lint: destructive} + case d.Policy.Lint == nil: + d.Policy.Lint = destructive + case d.Policy.Lint.Destructive == nil, force: + d.Policy.Lint.Destructive = check + } +} + +// setLintReview sets the lint review policy. +// If the force is set to true, it will override the existing value. +func (d *managedData) setLintReview(v dbv1alpha1.LintReview, force bool) { + lint := &dbv1alpha1.Lint{Review: v} + switch { + case d.Policy == nil: + d.Policy = &dbv1alpha1.Policy{Lint: lint} + case d.Policy.Lint == nil: + d.Policy.Lint = lint + case d.Policy.Lint.Review == "", force: + d.Policy.Lint.Review = v + } +} + +// asBlocks returns the HCL block for the environment configuration. +func (d *managedData) asBlocks() []*hclwrite.Block { + var blocks []*hclwrite.Block + env := hclwrite.NewBlock("env", []string{d.EnvName}) + blocks = append(blocks, env) + envBody := env.Body() + if d.URL != nil { + envBody.SetAttributeValue("url", cty.StringVal(d.URL.String())) + } + if d.DevURL != "" { + envBody.SetAttributeValue("dev", cty.StringVal(d.DevURL)) + } + if l := d.Schemas; len(l) > 0 { + envBody.SetAttributeValue("schemas", listStringVal(l)) + } + if l := d.Exclude; len(l) > 0 { + envBody.SetAttributeValue("exclude", listStringVal(l)) + } + if p := d.Policy; p != nil { + if d := p.Diff; d != nil { + envBody.AppendBlock(d.AsBlock()) + } + if l := p.Lint; l != nil { + lint := envBody.AppendNewBlock("lint", nil).Body() + if v := l.Destructive; v != nil { + b := lint.AppendNewBlock("destructive", nil).Body() + b.SetAttributeValue("error", cty.BoolVal(v.Error)) + } + if v := l.Review; v != "" { + lint.SetAttributeValue("review", cty.StringVal(string(v))) + } + } + } + if v := d.TxMode; v != "" { + envBody.SetAttributeValue("tx_mode", cty.StringVal(string(v))) + } + return blocks } func truncateSQL(s []string, size int) []string { diff --git a/internal/controller/atlasschema_controller_test.go b/internal/controller/atlasschema_controller_test.go index 88b5f12..44611a9 100644 --- a/internal/controller/atlasschema_controller_test.go +++ b/internal/controller/atlasschema_controller_test.go @@ -429,35 +429,24 @@ func TestConfigTemplate(t *testing.T) { } err := data.render(&buf) require.NoError(t, err) - expected := `variable "lint_destructive" { - type = bool - default = false -} -variable "lint_review" { - type = string - default = "" -} -diff { - concurrent_index { - create = true - drop = true - } - skip { - drop_schema = true - drop_table = true + expected := `env "kubernetes" { + url = "mysql://root:password@localhost:3306/test" + dev = "mysql://root:password@localhost:3306/dev" + schemas = ["foo", "bar"] + diff { + concurrent_index { + create = true + drop = true + } + skip { + drop_schema = true + drop_table = true + } } -} -env { - name = atlas.env - url = "mysql://root:password@localhost:3306/test" - dev = "mysql://root:password@localhost:3306/dev" - schemas = ["foo","bar"] - exclude = [] lint { destructive { - error = var.lint_destructive + error = true } - review = var.lint_review != "" ? var.lint_review : null } } ` diff --git a/internal/controller/common.go b/internal/controller/common.go index 5bca0a7..bbd39a3 100644 --- a/internal/controller/common.go +++ b/internal/controller/common.go @@ -26,6 +26,7 @@ import ( "ariga.io/atlas-go-sdk/atlasexec" "ariga.io/atlas/sql/migrate" + "github.com/zclconf/go-cty/cty" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -223,3 +224,12 @@ func result(err error) (r ctrl.Result, _ error) { } return r, nil } + +// listStringVal returns a cty.ListVal from the given slice of strings. +func listStringVal(s []string) cty.Value { + v := make([]cty.Value, len(s)) + for i, s := range s { + v[i] = cty.StringVal(s) + } + return cty.ListVal(v) +} diff --git a/internal/controller/templates/atlas_schema.tmpl b/internal/controller/templates/atlas_schema.tmpl deleted file mode 100644 index 7f0f8a8..0000000 --- a/internal/controller/templates/atlas_schema.tmpl +++ /dev/null @@ -1,88 +0,0 @@ -variable "lint_destructive" { - type = bool - default = false -} -variable "lint_review" { - type = string - default = "" -} -{{- with .Policy }} - {{- with .Diff }} - {{- if or .ConcurrentIndex .Skip }} -diff { - {{- with .ConcurrentIndex }} - concurrent_index { - {{- if .Create }} - create = true - {{- end }} - {{- if .Drop }} - drop = true - {{- end }} - } - {{- end }} - {{- with .Skip }} - skip { - {{- if .AddSchema }} - add_schema = true - {{- end }} - {{- if .DropSchema }} - drop_schema = true - {{- end }} - {{- if .ModifySchema }} - modify_schema = true - {{- end }} - {{- if .AddTable }} - add_table = true - {{- end }} - {{- if .DropTable }} - drop_table = true - {{- end }} - {{- if .ModifyTable }} - modify_table = true - {{- end }} - {{- if .AddColumn }} - add_column = true - {{- end }} - {{- if .DropColumn }} - drop_column = true - {{- end }} - {{- if .ModifyColumn }} - modify_column = true - {{- end }} - {{- if .AddIndex }} - add_index = true - {{- end }} - {{- if .DropIndex }} - drop_index = true - {{- end }} - {{- if .ModifyIndex }} - modify_index = true - {{- end }} - {{- if .AddForeignKey }} - add_foreign_key = true - {{- end }} - {{- if .DropForeignKey }} - drop_foreign_key = true - {{- end }} - {{- if .ModifyForeignKey }} - modify_foreign_key = true - {{- end }} - } - {{- end }} -} - {{- end }} -{{- end }} -{{- end }} -env { - name = atlas.env - url = "{{ removeSpecialChars .URL }}" - dev = "{{ removeSpecialChars .DevURL }}" - schemas = {{ slides .Schemas }} - exclude = {{ slides .Exclude }} - lint { - destructive { - error = var.lint_destructive - } - review = var.lint_review != "" ? var.lint_review : null - } -}