Use polly's topsort implementation (#253)

* Use polly's topsort implementation

don't use the legacy toposort which is better designed and more modern

* fix broken test

* use FilterMap where appropriate

Co-authored-by: test <[email protected]>

cmd/plural/api.go

@@ -3,7 +3,7 @@ package main
 import (
 	"github.com/pluralsh/plural/pkg/api"
 	"github.com/pluralsh/plural/pkg/utils"
-	"github.com/pluralsh/plural/pkg/utils/containers"
+	"github.com/pluralsh/polly/algorithms"
 	"github.com/urfave/cli"
 )
 
@@ -81,7 +81,7 @@ func (p *Plural) handleInstallations(c *cli.Context) error {
 		return err
 	}
 
-	installations = containers.Filter(installations, func(v *api.Installation) bool {
+	installations = algorithms.Filter(installations, func(v *api.Installation) bool {
 		return v.Repository != nil
 	})
 
@@ -143,7 +143,7 @@ func (p *Plural) handleChartInstallations(c *cli.Context) error {
 		return err
 	}
 
-	cis := containers.Filter(chartInstallations, func(ci *api.ChartInstallation) bool {
+	cis := algorithms.Filter(chartInstallations, func(ci *api.ChartInstallation) bool {
 		return ci.Chart != nil && ci.Version != nil
 	})
 
@@ -161,7 +161,7 @@ func (p *Plural) handleTerraformInstallations(c *cli.Context) error {
 		return err
 	}
 
-	tis := containers.Filter(terraformInstallations, func(ti *api.TerraformInstallation) bool {
+	tis := algorithms.Filter(terraformInstallations, func(ti *api.TerraformInstallation) bool {
 		return ti != nil
 	})
 

cmd/plural/dependencies_test.go

@@ -24,13 +24,14 @@ func TestTopSort(t *testing.T) {
 		{
 			name: `test "topsort"`,
 			args: []string{plural.ApplicationName, "topsort"},
-			installations: []*api.Installation{{
-				Id: "abc",
-				Repository: &api.Repository{
-					Id:   "abc",
-					Name: "abc",
+			installations: []*api.Installation{
+				{
+					Id: "abc",
+					Repository: &api.Repository{
+						Id:   "abc",
+						Name: "abc",
+					},
 				},
-			},
 				{
 					Id: "cde",
 					Repository: &api.Repository{
@@ -54,7 +55,6 @@ func TestTopSort(t *testing.T) {
 			os.Args = test.args
 			res, err := captureStdout(app, os.Args)
 			assert.NoError(t, err)
-
 			assert.Equal(t, test.expectedResponse, res)
 		})
 	}

cmd/plural/deploy.go

@@ -19,6 +19,8 @@ import (
 	"github.com/pluralsh/plural/pkg/utils/git"
 	"github.com/pluralsh/plural/pkg/utils/pathing"
 	"github.com/pluralsh/plural/pkg/wkspace"
+	"github.com/pluralsh/polly/algorithms"
+	"github.com/pluralsh/polly/containers"
 	"github.com/urfave/cli"
 )
 
@@ -63,19 +65,8 @@ func getSortedNames(filter bool) ([]string, error) {
 	}
 
 	if filter {
-		result := make([]string, 0)
-		isRepo := map[string]bool{}
-		for _, repo := range diffed {
-			isRepo[repo] = true
-		}
-
-		for _, repo := range sorted {
-			if isRepo[repo] {
-				result = append(result, repo)
-			}
-		}
-
-		return result, nil
+		repos := containers.ToSet(diffed)
+		return algorithms.Filter(sorted, repos.Has), nil
 	}
 
 	return sorted, nil

cmd/plural/deploy_test.go

@@ -152,7 +152,7 @@ func TestValidate(t *testing.T) {
 			os.Args = test.args
 			resp, err := captureStdout(app, os.Args)
 			assert.NoError(t, err)
-			assert.Equal(t, resp, test.expectedResponse)
+			assert.Equal(t, test.expectedResponse, resp)
 		})
 	}
 }

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
 	github.com/pluralsh/gqlclient v1.1.11
 	github.com/pluralsh/plural-operator v0.5.3
+	github.com/pluralsh/polly v0.0.3
 	github.com/rodaine/hclencoder v0.0.1
 	github.com/thoas/go-funk v0.9.2
 	github.com/urfave/cli v1.22.10
@@ -89,8 +90,10 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
 	github.com/pluralsh/controller-reconcile-helper v0.0.4 // indirect
+	github.com/samber/lo v1.33.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/vektah/gqlparser/v2 v2.5.0 // indirect
+	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

go.sum

@@ -935,6 +935,10 @@ github.com/pluralsh/oauth v0.9.1-0.20220520000222-d76c0e7a0db9 h1:bMkXXUksi9ym+e
 github.com/pluralsh/oauth v0.9.1-0.20220520000222-d76c0e7a0db9/go.mod h1:aTUw/75rzcsbvW+/TLvWtHVDXFIdtFrDtUncOq9vHyM=
 github.com/pluralsh/plural-operator v0.5.3 h1:GaPL3LgimfzKZNHt7zXzqYZpb0hgyW9noHYnkA+rqNs=
 github.com/pluralsh/plural-operator v0.5.3/go.mod h1:WIXiz26/WDcUn0FA7Q1jPxmfsm98U1/JL8YpIdKVLX0=
+github.com/pluralsh/polly v0.0.2 h1:KBfpxlo6ssI2Ha1Iu6hWVY8Pt3v1Po/ixwwE788QUnY=
+github.com/pluralsh/polly v0.0.2/go.mod h1:GX6PeRDTRBLXNq3AgXfgJUEtfDssB7bm/JUjxDnjQ1U=
+github.com/pluralsh/polly v0.0.3 h1:0wN7MXGyDKCoZepO2+ryIb5T4a01IQd0r9IGDf6bo90=
+github.com/pluralsh/polly v0.0.3/go.mod h1:GX6PeRDTRBLXNq3AgXfgJUEtfDssB7bm/JUjxDnjQ1U=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -994,6 +998,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/samber/lo v1.33.0 h1:2aKucr+rQV6gHpY3bpeZu69uYoQOzVhGT3J22Op6Cjk=
+github.com/samber/lo v1.33.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8=
 github.com/schollz/progressbar/v3 v3.8.6 h1:QruMUdzZ1TbEP++S1m73OqRJk20ON11m6Wqv4EoGg8c=
 github.com/schollz/progressbar/v3 v3.8.6/go.mod h1:W5IEwbJecncFGBvuEh4A7HT1nZZ6WNIL2i3qbnI0WKY=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@@ -1198,6 +1204,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

pkg/diff/builder.go

@@ -11,6 +11,8 @@ import (
 	"github.com/pluralsh/plural/pkg/utils"
 	"github.com/pluralsh/plural/pkg/utils/git"
 	"github.com/pluralsh/plural/pkg/utils/pathing"
+	"github.com/pluralsh/polly/algorithms"
+	"github.com/pluralsh/polly/containers"
 	"github.com/rodaine/hclencoder"
 )
 
@@ -93,41 +95,8 @@ func (e *Diff) IgnoreFile(root string) ([]string, error) {
 }
 
 func DefaultDiff(path string, prev *Diff) (e *Diff) {
-	byName := make(map[string]*executor.Step)
-	steps := []*executor.Step{
-		{
-			Name:    "terraform-init",
-			Wkdir:   pathing.SanitizeFilepath(filepath.Join(path, "terraform")),
-			Target:  pathing.SanitizeFilepath(filepath.Join(path, "terraform")),
-			Command: "terraform",
-			Args:    []string{"init"},
-			Sha:     "",
-		},
-		{
-			Name:    "terraform",
-			Wkdir:   pathing.SanitizeFilepath(filepath.Join(path, "terraform")),
-			Target:  pathing.SanitizeFilepath(filepath.Join(path, "terraform")),
-			Command: "plural",
-			Args:    []string{"wkspace", "terraform-diff", path},
-			Sha:     "",
-		},
-		{
-			Name:    "kube-init",
-			Wkdir:   path,
-			Target:  pluralfile(path, "NONCE"),
-			Command: "plural",
-			Args:    []string{"wkspace", "kube-init", path},
-			Sha:     "",
-		},
-		{
-			Name:    "helm",
-			Wkdir:   pathing.SanitizeFilepath(filepath.Join(path, "helm")),
-			Target:  pathing.SanitizeFilepath(filepath.Join(path, "helm")),
-			Command: "plural",
-			Args:    []string{"wkspace", "helm-diff", path},
-			Sha:     "",
-		},
-	}
+	byName := map[string]*executor.Step{}
+	steps := defaultDiff(path)
 
 	for _, step := range prev.Steps {
 		byName[step.Name] = step
@@ -142,11 +111,7 @@ func DefaultDiff(path string, prev *Diff) (e *Diff) {
 	}
 
 	// set up a topsort between the two orders of operations
-	graph := utils.Graph(len(byName))
-	for k := range byName {
-		graph.AddNode(k)
-	}
-
+	graph := containers.NewGraph[string]()
 	for i := 0; i < len(steps)-1; i++ {
 		graph.AddEdge(steps[i].Name, steps[i+1].Name)
 	}
@@ -155,17 +120,8 @@ func DefaultDiff(path string, prev *Diff) (e *Diff) {
 		graph.AddEdge(steps[i].Name, steps[i+1].Name)
 	}
 
-	finalizedSteps := []*executor.Step{}
-	sorted, ok := graph.Topsort()
-	if !ok {
-		panic("deployfile cycle detected")
-	}
-
-	// dump the topsort to a list and use that from now on
-	for _, name := range sorted {
-		finalizedSteps = append(finalizedSteps, byName[name])
-	}
-
+	sorted, _ := algorithms.TopsortGraph(graph)
+	finalizedSteps := algorithms.Map(sorted, func(s string) *executor.Step { return byName[s] })
 	return &Diff{
 		Metadata: Metadata{Path: path, Name: "diff"},
 		Steps:    finalizedSteps,

pkg/diff/default.go

@@ -0,0 +1,45 @@
+package diff
+
+import (
+	"path/filepath"
+
+	"github.com/pluralsh/plural/pkg/executor"
+	"github.com/pluralsh/plural/pkg/utils/pathing"
+)
+
+func defaultDiff(path string) []*executor.Step {
+	return []*executor.Step{
+		{
+			Name:    "terraform-init",
+			Wkdir:   pathing.SanitizeFilepath(filepath.Join(path, "terraform")),
+			Target:  pathing.SanitizeFilepath(filepath.Join(path, "terraform")),
+			Command: "terraform",
+			Args:    []string{"init"},
+			Sha:     "",
+		},
+		{
+			Name:    "terraform",
+			Wkdir:   pathing.SanitizeFilepath(filepath.Join(path, "terraform")),
+			Target:  pathing.SanitizeFilepath(filepath.Join(path, "terraform")),
+			Command: "plural",
+			Args:    []string{"wkspace", "terraform-diff", path},
+			Sha:     "",
+		},
+		{
+			Name:    "kube-init",
+			Wkdir:   path,
+			Target:  pluralfile(path, "NONCE"),
+			Command: "plural",
+			Args:    []string{"wkspace", "kube-init", path},
+			Sha:     "",
+		},
+		{
+			Name:    "helm",
+			Wkdir:   pathing.SanitizeFilepath(filepath.Join(path, "helm")),
+			Target:  pathing.SanitizeFilepath(filepath.Join(path, "helm")),
+			Command: "plural",
+			Args:    []string{"wkspace", "helm-diff", path},
+			Sha:     "",
+		},
+	}
+}

pkg/executor/execution.go

@@ -7,9 +7,10 @@ import (
 	"strings"
 
 	"github.com/hashicorp/hcl"
-	"github.com/pluralsh/plural/pkg/utils"
 	"github.com/pluralsh/plural/pkg/utils/git"
 	"github.com/pluralsh/plural/pkg/utils/pathing"
+	"github.com/pluralsh/polly/algorithms"
+	"github.com/pluralsh/polly/containers"
 	"github.com/rodaine/hclencoder"
 )
 
@@ -118,11 +119,7 @@ func DefaultExecution(path string, prev *Execution) (e *Execution) {
 	}
 
 	// set up a topsort between the two orders of operations
-	graph := utils.Graph(len(byName))
-	for k := range byName {
-		graph.AddNode(k)
-	}
-
+	graph := containers.NewGraph[string]()
 	for i := 0; i < len(steps)-1; i++ {
 		graph.AddEdge(steps[i].Name, steps[i+1].Name)
 	}
@@ -131,17 +128,8 @@ func DefaultExecution(path string, prev *Execution) (e *Execution) {
 		graph.AddEdge(prev.Steps[i].Name, prev.Steps[i+1].Name)
 	}
 
-	finalizedSteps := []*Step{}
-	sorted, ok := graph.Topsort()
-	if !ok {
-		panic("deployfile cycle detected")
-	}
-
-	// dump the topsort to a list and use that from now on
-	for _, name := range sorted {
-		finalizedSteps = append(finalizedSteps, byName[name])
-	}
-
+	sorted, _ := algorithms.TopsortGraph(graph)
+	finalizedSteps := algorithms.Map(sorted, func(s string) *Step { return byName[s] })
 	return &Execution{
 		Metadata: Metadata{Path: path, Name: "deploy"},
 		Steps:    finalizedSteps,

pkg/scaffold/scaffold.go

@@ -5,10 +5,11 @@ import (
 	"path/filepath"
 
 	"github.com/pluralsh/plural/pkg/executor"
-	"github.com/pluralsh/plural/pkg/utils"
 	"github.com/pluralsh/plural/pkg/utils/git"
 	"github.com/pluralsh/plural/pkg/utils/pathing"
 	"github.com/pluralsh/plural/pkg/wkspace"
+	"github.com/pluralsh/polly/algorithms"
+	"github.com/pluralsh/polly/containers"
 	"github.com/rodaine/hclencoder"
 )
 
@@ -56,7 +57,6 @@ func Scaffolds(wk *wkspace.Workspace) (*Build, error) {
 
 func merge(build *Build, base *Build) *Build {
 	byName := make(map[string]*Scaffold)
-
 	for _, scaffold := range build.Scaffolds {
 		byName[scaffold.Name] = scaffold
 	}
@@ -70,12 +70,7 @@ func merge(build *Build, base *Build) *Build {
 	// to handle helm v3 transition
 	delete(byName, "add-repo")
 
-	graph := utils.Graph(len(byName))
-
-	for key := range byName {
-		graph.AddNode(key)
-	}
-
+	graph := containers.NewGraph[string]()
 	for i := 0; i < len(build.Scaffolds)-1; i++ {
 		graph.AddEdge(build.Scaffolds[i].Name, build.Scaffolds[i+1].Name)
 	}
@@ -84,17 +79,8 @@ func merge(build *Build, base *Build) *Build {
 		graph.AddEdge(base.Scaffolds[i].Name, base.Scaffolds[i+1].Name)
 	}
 
-	sorted, ok := graph.Topsort()
-	if !ok {
-		panic("scaffold cycle created")
-	}
-
-	scaffolds := []*Scaffold{}
-	for _, name := range sorted {
-		scaffolds = append(scaffolds, byName[name])
-	}
-	build.Scaffolds = scaffolds
-
+	sorted, _ := algorithms.TopsortGraph(graph)
+	build.Scaffolds = algorithms.Map(sorted, func(n string) *Scaffold { return byName[n] })
 	return build
 }