Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: extra_templates allows users to control templates #1046

Merged
merged 19 commits into from
Apr 29, 2024
Merged

feat: extra_templates allows users to control templates #1046

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
079487b
feat: allow variables to specify values in main.tf
jakeyheath Apr 25, 2024
2dfe715
filter out nonexistent variables
jakeyheath Apr 25, 2024
13e2c7a
loop
jakeyheath Apr 25, 2024
c4a468b
feat: generate provider configs for kubectl & helm
jakeyheath Apr 25, 2024
a3e8b56
remove custom provider
jakeyheath Apr 25, 2024
304b06f
tidy
jakeyheath Apr 25, 2024
2918ee7
conditionals
jakeyheath Apr 25, 2024
2bcc406
Merge branch 'main' into heathj/providers
jakeyheath Apr 26, 2024
8cc8963
index
jakeyheath Apr 26, 2024
9bde2b5
Merge branch 'heathj/providers' of github.com:chanzuckerberg/fogg int…
jakeyheath Apr 26, 2024
201ebe7
feat: extra_templates allows users to control templates
jakeyheath Apr 26, 2024
3812c7a
commit from ci -- updated golden files
jakeyheath Apr 26, 2024
16ae90a
no template header
jakeyheath Apr 26, 2024
10b55c9
Merge branch 'heathj/extra_files' of github.com:chanzuckerberg/fogg i…
jakeyheath Apr 26, 2024
e18f831
Merge branch 'main' into heathj/extra_files
jakeyheath Apr 26, 2024
91855af
Merge branch 'main' into heathj/extra_files
jakeyheath Apr 27, 2024
2ee374a
Update config/v2/config.go
jakeyheath Apr 29, 2024
0928565
maps
jakeyheath Apr 29, 2024
655c77e
Merge branch 'main' into heathj/extra_files
jakeyheath Apr 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 56 additions & 15 deletions apply/apply.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,41 +298,78 @@ func applyRepo(fs afero.Fs, p *plan.Plan, repoTemplates, commonTemplates fs.FS)
return applyTree(fs, repoTemplates, commonTemplates, "", p)
}

func applyExtraTemplates(fs afero.Fs, p plan.ComponentCommon, commonBox fs.FS, path string) error {
for _, templateCfg := range p.ExtraTemplates {
for filename, template := range templateCfg.Files {
target := getTargetPath(path, filename)
_, err := fs.Stat(target)
if err == nil && !templateCfg.Overwrite {
// file exists and we don't want to overwrite
continue
}

err = applyTemplate(strings.NewReader(template), commonBox, fs, target, p)
if err != nil {
return errs.WrapUser(err, "applying extra templates")
}

if filepath.Ext(filename) == ".tf" {
err = fmtHcl(fs, target, true)
if err != nil {
return errs.WrapUser(err, "formating HCL of extra templates")
}
}
}
}

return nil
}

func applyGlobal(fs afero.Fs, p plan.Component, repoBox, commonBox fs.FS) error {
logrus.Debug("applying global")
path := fmt.Sprintf("%s/global", rootPath)
e := fs.MkdirAll(path, 0755)
if e != nil {
return errs.WrapUserf(e, "unable to make directory %s", path)
}
return applyTree(fs, repoBox, commonBox, path, p)
err := applyTree(fs, repoBox, commonBox, path, p)
if err != nil {
return err
}

return applyExtraTemplates(fs, p.ComponentCommon, commonBox, path)
}

func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox, commonBox fs.FS) (e error) {
func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox, commonBox fs.FS) error {
for account, accountPlan := range p.Accounts {
path := fmt.Sprintf("%s/accounts/%s", rootPath, account)
e = fs.MkdirAll(path, 0755)
if e != nil {
return errs.WrapUser(e, "unable to make directories for accounts")
err := fs.MkdirAll(path, 0755)
if err != nil {
return errs.WrapUser(err, "unable to make directories for accounts")
}
e = applyTree(fs, accountBox, commonBox, path, accountPlan)
if e != nil {
return errs.WrapUser(e, "unable to apply templates to account")
err = applyTree(fs, accountBox, commonBox, path, accountPlan)
if err != nil {
return errs.WrapUser(err, "unable to apply templates to account")
}

err = applyExtraTemplates(fs, accountPlan.ComponentCommon, commonBox, path)
if err != nil {
return errs.WrapUser(err, "apply extra templates")
}
}
return nil
}

func applyModules(fs afero.Fs, p map[string]plan.Module, moduleBox, commonBox fs.FS) (e error) {
func applyModules(fs afero.Fs, p map[string]plan.Module, moduleBox, commonBox fs.FS) error {
for module, modulePlan := range p {
path := fmt.Sprintf("%s/modules/%s", rootPath, module)
e = fs.MkdirAll(path, 0755)
if e != nil {
return errs.WrapUserf(e, "unable to make path %s", path)
err := fs.MkdirAll(path, 0755)
if err != nil {
return errs.WrapUserf(err, "unable to make path %s", path)
}
e = applyTree(fs, moduleBox, commonBox, path, modulePlan)
if e != nil {
return errs.WrapUser(e, "unable to apply tree")
err = applyTree(fs, moduleBox, commonBox, path, modulePlan)
if err != nil {
return errs.WrapUser(err, "unable to apply tree")
}
}
return nil
Expand Down Expand Up @@ -375,6 +412,10 @@ func applyEnvs(
if err != nil {
return errs.WrapUser(err, "unable to apply templates for component")
}
err = applyExtraTemplates(fs, componentPlan.ComponentCommon, commonBox, path)
if err != nil {
return errs.WrapUser(err, "apply extra templates")
}

if componentPlan.ModuleSource != nil {
downloader, err := util.MakeDownloader(*componentPlan.ModuleSource)
Expand Down
6 changes: 6 additions & 0 deletions config/v2/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,11 @@ type TFE struct {
AdditionalGithubRequiredChecks *[]string `yaml:"additional_gh_required_checks,omitempty"`
}

type ExtraTemplate struct {
Overwrite bool
Files map[string]string
}

type Common struct {
Backend *Backend `yaml:"backend,omitempty"`
ExtraVars map[string]string `yaml:"extra_vars,omitempty"`
Expand All @@ -84,6 +89,7 @@ type Common struct {
TerraformVersion *string `yaml:"terraform_version,omitempty"`
Tools *Tools `yaml:"tools,omitempty"`
NeedsAWSAccountsVariable *bool `yaml:"needs_aws_accounts_variable,omitempty"`
ExtraTemplates *[]ExtraTemplate `yaml:"extra_template,omitempty"`
jakeyheath marked this conversation as resolved.
Show resolved Hide resolved
}

type Defaults struct {
Expand Down
99 changes: 44 additions & 55 deletions config/v2/resolvers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,45 +6,12 @@ import (
"github.com/chanzuckerberg/fogg/util"
)

// lastNonNilBool, despite its name can return nil if all results are nil
func lastNonNilBool(getter func(Common) *bool, commons ...Common) *bool {
var s *bool
for _, c := range commons {
t := getter(c)
if t != nil {
s = t
}
}
return s
}

// lastNonNil, despite its name can return nil if all results are nil
func lastNonNil(getter func(Common) *string, commons ...Common) *string {
var s *string
for _, c := range commons {
t := getter(c)
if t != nil {
s = t
}
}
return s
type Nillable interface {
*bool | *float64 | *int64 | *string | []string | *[]ExtraTemplate
}

// lastNonNilInt64, despite its name can return nil if all results are nil
func lastNonNilInt64(getter func(Common) *int64, commons ...Common) *int64 {
var s *int64
for _, c := range commons {
t := getter(c)
if t != nil {
s = t
}
}
return s
}

// lastNonNilStringSlice, despite its name can return nil if all results are nil
func lastNonNilStringSlice(getter func(Common) []string, commons ...Common) []string {
var s []string
func lastNonNil[T Nillable](getter func(Common) T, commons ...Common) T {
var s T
for _, c := range commons {
t := getter(c)
if t != nil {
Expand All @@ -61,15 +28,15 @@ func ResolveRequiredString(getter func(Common) *string, commons ...Common) strin

// ResolveRequiredInt64 will resolve the value and panic if it is nil. Only to be used after validations are run.
func ResolveRequiredInt64(getter func(Common) *int64, commons ...Common) int64 {
return *lastNonNilInt64(getter, commons...)
return *lastNonNil(getter, commons...)
}

func ResolveOptionalString(getter func(Common) *string, commons ...Common) *string {
return lastNonNil(getter, commons...)
}

func ResolveOptionalStringSlice(getter func(Common) []string, commons ...Common) []string {
return lastNonNilStringSlice(getter, commons...)
return lastNonNil(getter, commons...)
}

func ResolveStringArray(def []string, override []string) []string {
Expand Down Expand Up @@ -302,7 +269,7 @@ func ResolveGithubProvider(commons ...Common) *GithubProvider {
BaseURL: lastNonNil(GithubProviderBaseURLGetter, commons...),
CommonProvider: CommonProvider{
Enabled: enabled,
CustomProvider: lastNonNilBool(GithubProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(GithubProviderCustomProviderGetter, commons...),
Version: lastNonNil(GithubProviderVersionGetter, commons...),
},
}
Expand All @@ -316,13 +283,35 @@ func AWSAccountsNeededGetter(comm Common) *bool {
}

func ResolveAWSAccountsNeeded(commons ...Common) bool {
accountsNeeded := lastNonNilBool(AWSAccountsNeededGetter, commons...)
accountsNeeded := lastNonNil(AWSAccountsNeededGetter, commons...)
if accountsNeeded == nil {
return true
}
return *accountsNeeded
}

func ExtraTemplatesGetter(comm Common) *[]ExtraTemplate {
if comm.ExtraTemplates != nil {
templates := []ExtraTemplate{}
for _, v := range *comm.ExtraTemplates {
templates = append(templates, ExtraTemplate{
Overwrite: v.Overwrite,
Files: v.Files,
})
}
return &templates
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Potential opportunity to make a helper to consolidate same mapping functionality (1)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you're right, but unfortunately, the way this codebase's packages are laid out makes this difficult without doing a bit of refactoring. The config stage structs are all pointers because the configuration can or cannot have values and then they duplicate these same structs in the plan phase once they are "resolved" (going down the tree of overwrites from global -> default -> account -> env -> component) through the tree overwrites. Resolved structs don't have pointers. I would love to simplify this code since its so copy/pasted, but maybe I will try that in a different PR

}
return nil
}

func ResolveExtraTemplates(commons ...Common) []ExtraTemplate {
templates := lastNonNil(ExtraTemplatesGetter, commons...)
if templates == nil {
return []ExtraTemplate{}
}
return *templates
}

func ResolveSnowflakeProvider(commons ...Common) *SnowflakeProvider {
account := lastNonNil(SnowflakeProviderAccountGetter, commons...)
role := lastNonNil(SnowflakeProviderRoleGetter, commons...)
Expand All @@ -335,7 +324,7 @@ func ResolveSnowflakeProvider(commons ...Common) *SnowflakeProvider {
Role: role,
Region: region,
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(SnowflakeProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(SnowflakeProviderCustomProviderGetter, commons...),
Enabled: defaultEnabled(true),
Version: version,
},
Expand All @@ -359,7 +348,7 @@ func ResolveOktaProvider(commons ...Common) *OktaProvider {
BaseURL: baseURL,
RegistryNamespace: registryNamespace,
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(OktaProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(OktaProviderCustomProviderGetter, commons...),
Enabled: defaultEnabled(true),
Version: lastNonNil(OktaProviderVersionGetter, commons...),
},
Expand All @@ -381,7 +370,7 @@ func ResolveBlessProvider(commons ...Common) *BlessProvider {
AWSRegion: region,
RoleArn: roleArn,
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(BlessProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(BlessProviderCustomProviderGetter, commons...),
Enabled: defaultEnabled(true),
Version: lastNonNil(BlessProviderVersionGetter, commons...),
},
Expand All @@ -406,7 +395,7 @@ func ResolveHerokuProvider(commons ...Common) *HerokuProvider {
if version != nil {
return &HerokuProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(HerokuProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(HerokuProviderCustomProviderGetter, commons...),
Enabled: defaultEnabled(true),
Version: version,
},
Expand All @@ -432,7 +421,7 @@ func ResolveDatadogProvider(commons ...Common) *DatadogProvider {
if version != nil {
return &DatadogProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(DatadogProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(DatadogProviderCustomProviderGetter, commons...),
Enabled: defaultEnabled(true),
Version: version,
},
Expand All @@ -458,7 +447,7 @@ func ResolvePagerdutyProvider(commons ...Common) *PagerdutyProvider {
if version != nil {
return &PagerdutyProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(PagerDutyProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(PagerDutyProviderCustomProviderGetter, commons...),
Enabled: defaultEnabled(true),
Version: version,
},
Expand All @@ -484,7 +473,7 @@ func ResolveOpsGenieProvider(commons ...Common) *OpsGenieProvider {
if version != nil {
return &OpsGenieProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(OpsGenieProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(OpsGenieProviderCustomProviderGetter, commons...),
Enabled: defaultEnabled(true),
Version: version,
},
Expand All @@ -507,7 +496,7 @@ func ResolveDatabricksProvider(commons ...Common) *DatabricksProvider {
if version != nil {
return &DatabricksProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(DatabricksProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(DatabricksProviderCustomProviderGetter, commons...),
Enabled: defaultEnabled(true),
Version: version,
},
Expand All @@ -534,7 +523,7 @@ func ResolveSentryProvider(commons ...Common) *SentryProvider {
if version != nil {
return &SentryProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(SentryProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(SentryProviderCustomProviderGetter, commons...),
Enabled: defaultEnabled(true),
Version: version,
},
Expand Down Expand Up @@ -580,7 +569,7 @@ func ResolveTfeProvider(commons ...Common) *TfeProvider {
if version != nil {
return &TfeProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(TFEProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(TFEProviderCustomProviderGetter, commons...),
Enabled: enabled,
Version: version,
},
Expand Down Expand Up @@ -623,7 +612,7 @@ func ResolveKubernetesProvider(commons ...Common) *KubernetesProvider {
return &KubernetesProvider{
ClusterComponentName: clusterComponentName,
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(KubernetesProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(KubernetesProviderCustomProviderGetter, commons...),
Enabled: enabled,
Version: version,
},
Expand Down Expand Up @@ -662,7 +651,7 @@ func ResolveHelmProvider(commons ...Common) *HelmProvider {
if version != nil {
return &HelmProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(HelmProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(HelmProviderCustomProviderGetter, commons...),
Enabled: enabled,
Version: version,
},
Expand Down Expand Up @@ -702,7 +691,7 @@ func ResolveKubectlProvider(commons ...Common) *KubectlProvider {
if version != nil {
return &KubectlProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(KubectlProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(KubectlProviderCustomProviderGetter, commons...),
Enabled: enabled,
Version: version,
},
Expand Down Expand Up @@ -742,7 +731,7 @@ func ResolveGrafanaProvider(commons ...Common) *GrafanaProvider {
if version != nil {
return &GrafanaProvider{
CommonProvider: CommonProvider{
CustomProvider: lastNonNilBool(GrafanaProviderCustomProviderGetter, commons...),
CustomProvider: lastNonNil(GrafanaProviderCustomProviderGetter, commons...),
Enabled: enabled,
Version: version,
},
Expand Down
Loading
Loading