Skip to content

Commit

Permalink
refactor(download): Handle template errors in dependency resolution
Browse files Browse the repository at this point in the history
Method signatures are refactored to return now possible teplate read/write
errors.
Additionally unused coordinate returns are removed.
  • Loading branch information
UnseenWizzard committed Sep 29, 2023
1 parent d54675e commit c4970ef
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 35 deletions.
5 changes: 4 additions & 1 deletion cmd/monaco/download/download_configs.go
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,10 @@ func doDownloadConfigs(fs afero.Fs, downloaders downloaders, opts downloadConfig
}

log.Info("Resolving dependencies between configurations")
downloadedConfigs = dependency_resolution.ResolveDependencies(downloadedConfigs)
downloadedConfigs, err = dependency_resolution.ResolveDependencies(downloadedConfigs)
if err != nil {
return err
}

log.Info("Extracting additional identifiers into YAML parameters")
// must happen after dep-resolution, as it removes IDs from the JSONs in which the dep-resolution searches as well
Expand Down
29 changes: 21 additions & 8 deletions pkg/download/dependency_resolution/dependency_resolution.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,46 +24,59 @@ import (
"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log/field"
"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/download/dependency_resolution/resolver"
"sync"
"sync/atomic"

"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config"
project "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/project/v2"
)

type dependencyResolver interface {
ResolveDependencyReferences(configToBeUpdated *config.Config)
ResolveDependencyReferences(configToBeUpdated *config.Config) error
}

// ResolveDependencies resolves all id-dependencies between downloaded configs.
//
// We do this by collecting all ids of all configs, and then simply by searching for them in templates.
// If we find an occurrence, we replace it with a generic variable and reference the config.
func ResolveDependencies(configs project.ConfigsPerType) project.ConfigsPerType {
func ResolveDependencies(configs project.ConfigsPerType) (project.ConfigsPerType, error) {
log.Debug("Resolving dependencies between configs")
resolve(configs)
err := resolve(configs)
if err != nil {
return nil, err
}
log.Debug("Finished resolving dependencies")
return configs
return configs, nil
}

func resolve(configs project.ConfigsPerType) {
func resolve(configs project.ConfigsPerType) error {
r := getResolver(configs)

errOccurred := atomic.Bool{}
wg := sync.WaitGroup{}
// currently a simple brute force attach
// currently a simple brute force approach
for _, configs := range configs {
configs := configs
for i := range configs {
wg.Add(1)

configToBeUpdated := &configs[i]
go func() {
r.ResolveDependencyReferences(configToBeUpdated)
err := r.ResolveDependencyReferences(configToBeUpdated)
if err != nil {
log.WithFields(field.Coordinate(configToBeUpdated.Coordinate), field.Error(err)).Error("Failed to resolve dependencies: %v", err)
errOccurred.Store(true)
}

wg.Done()
}()
}
}

wg.Wait()

if errOccurred.Load() {
return fmt.Errorf("failed to resolve dependencies")
}
return nil
}

func getResolver(configs project.ConfigsPerType) dependencyResolver {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -680,14 +680,14 @@ func TestDependencyResolution(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name+"_BasicResolver", func(t *testing.T) {
result := ResolveDependencies(test.setup)

result, err := ResolveDependencies(test.setup)
assert.NilError(t, err)
assert.DeepEqual(t, result, test.expected, cmp.AllowUnexported(template.InMemoryTemplate{}))
})
t.Run(test.name+"_FastResolver", func(t *testing.T) {
t.Setenv(featureflags.FastDependencyResolver().EnvName(), "true")
result := ResolveDependencies(test.setup)

result, err := ResolveDependencies(test.setup)
assert.NilError(t, err)
assert.DeepEqual(t, result, test.expected, cmp.AllowUnexported(template.InMemoryTemplate{}))
})
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
goaho "github.com/anknown/ahocorasick"
"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log"
"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config"
"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/coordinate"
"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/parameter/reference"
"golang.org/x/exp/maps"
"strings"
Expand Down Expand Up @@ -72,22 +71,27 @@ func toRuneSlices(ids []string) [][]rune {
return dict
}

func (r ahocorasickResolver) ResolveDependencyReferences(configToBeUpdated *config.Config) {
func (r ahocorasickResolver) ResolveDependencyReferences(configToBeUpdated *config.Config) error {
resolveScope(configToBeUpdated, r.ctx.configsById)
resolveTemplate(configToBeUpdated, r.ctx)
return resolveTemplate(configToBeUpdated, r.ctx)
}

func resolveTemplate(configToBeUpdated *config.Config, c dependencyResolutionContext) {
newContent, parameters, _ := findAndReplaceIDs(configToBeUpdated.Coordinate.Type, *configToBeUpdated, c)
func resolveTemplate(configToBeUpdated *config.Config, c dependencyResolutionContext) error {
newContent, parameters, err := findAndReplaceIDs(configToBeUpdated.Coordinate.Type, *configToBeUpdated, c)
if err != nil {
return err
}

maps.Copy(configToBeUpdated.Parameters, parameters)
configToBeUpdated.Template.UpdateContent(newContent)
return configToBeUpdated.Template.UpdateContent(newContent)
}

func findAndReplaceIDs(apiName string, configToBeUpdated config.Config, c dependencyResolutionContext) (string, config.Parameters, []coordinate.Coordinate) {
func findAndReplaceIDs(apiName string, configToBeUpdated config.Config, c dependencyResolutionContext) (string, config.Parameters, error) {
parameters := make(config.Parameters, 0)
content, _ := configToBeUpdated.Template.Content() //TODO - err handling
coordinates := make([]coordinate.Coordinate, 0)
content, err := configToBeUpdated.Template.Content()
if err != nil {
return "", nil, err
}

matches := c.matcher.MultiPatternSearch([]rune(content), false)
for _, m := range matches {
Expand Down Expand Up @@ -116,9 +120,8 @@ func findAndReplaceIDs(apiName string, configToBeUpdated config.Config, c depend
content = strings.ReplaceAll(content, key, "{{."+parameterName+"}}")
ref := reference.NewWithCoordinate(coord, "id")
parameters[parameterName] = ref
coordinates = append(coordinates, coord)

}

return content, parameters, coordinates
return content, parameters, nil
}
25 changes: 14 additions & 11 deletions pkg/download/dependency_resolution/resolver/basic_dep_resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package resolver
import (
"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log"
"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config"
"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/coordinate"
"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/parameter/reference"
"golang.org/x/exp/maps"
"strings"
Expand All @@ -35,22 +34,27 @@ func BasicResolver(configsById map[string]config.Config) basicResolver {
}
}

func (r basicResolver) ResolveDependencyReferences(configToBeUpdated *config.Config) {
func (r basicResolver) ResolveDependencyReferences(configToBeUpdated *config.Config) error {
resolveScope(configToBeUpdated, r.configsById)
basicResolveTemplate(configToBeUpdated, r.configsById)
return basicResolveTemplate(configToBeUpdated, r.configsById)
}

func basicResolveTemplate(configToBeUpdated *config.Config, configsById map[string]config.Config) {
newContent, parameters, _ := basicFindAndReplaceIDs(configToBeUpdated.Coordinate.Type, *configToBeUpdated, configsById)
func basicResolveTemplate(configToBeUpdated *config.Config, configsById map[string]config.Config) error {
newContent, parameters, err := basicFindAndReplaceIDs(configToBeUpdated.Coordinate.Type, *configToBeUpdated, configsById)
if err != nil {
return err
}

maps.Copy(configToBeUpdated.Parameters, parameters)
configToBeUpdated.Template.UpdateContent(newContent)
return configToBeUpdated.Template.UpdateContent(newContent)
}

func basicFindAndReplaceIDs(apiName string, configToBeUpdated config.Config, configs map[string]config.Config) (string, config.Parameters, []coordinate.Coordinate) {
func basicFindAndReplaceIDs(apiName string, configToBeUpdated config.Config, configs map[string]config.Config) (string, config.Parameters, error) {
parameters := make(config.Parameters, 0)
content, _ := configToBeUpdated.Template.Content() // TODO err handling
coordinates := make([]coordinate.Coordinate, 0)
content, err := configToBeUpdated.Template.Content()
if err != nil {
return "", nil, err
}

for key, conf := range configs {
if shouldReplaceReference(configToBeUpdated, conf, content, key) {
Expand All @@ -62,11 +66,10 @@ func basicFindAndReplaceIDs(apiName string, configToBeUpdated config.Config, con
content = strings.ReplaceAll(content, key, "{{."+parameterName+"}}")
ref := reference.NewWithCoordinate(coord, "id")
parameters[parameterName] = ref
coordinates = append(coordinates, coord)
}
}

return content, parameters, coordinates
return content, parameters, nil
}

// shouldReplaceReference checks if a given key is found in the content of another config and should be replaced
Expand Down

0 comments on commit c4970ef

Please sign in to comment.