From a29596b3bf14f669da01166f3e9d4a83ff0379f7 Mon Sep 17 00:00:00 2001 From: Antoine Gelloz Date: Mon, 22 Apr 2024 11:44:26 +0200 Subject: [PATCH] feat: add graph command --- cmd/graph.go | 60 +++++++++++++++ go.mod | 2 +- pkg/dag/dag.go | 22 ++++++ pkg/dag/node.go | 8 +- pkg/dag/printer.go | 142 +++++++++++++++++++++++++++++++++++ pkg/dib/generate_dag.go | 106 ++++++++++++-------------- pkg/dib/generate_dag_test.go | 2 + 7 files changed, 277 insertions(+), 65 deletions(-) create mode 100644 cmd/graph.go create mode 100644 pkg/dag/printer.go diff --git a/cmd/graph.go b/cmd/graph.go new file mode 100644 index 000000000..f1b9003a7 --- /dev/null +++ b/cmd/graph.go @@ -0,0 +1,60 @@ +package main + +import ( + "fmt" + "path" + + "github.com/radiofrance/dib/internal/logger" + "github.com/radiofrance/dib/pkg/dib" + "github.com/spf13/cobra" +) + +type GraphOpts struct { + BuildPath string `mapstructure:"build_path"` + RegistryURL string `mapstructure:"registry_url"` + PlaceholderTag string `mapstructure:"placeholder_tag"` + HashListFilePath string `mapstructure:"hash_list_file_path"` +} + +// buildCmd represents the build command. +var graphCmd = &cobra.Command{ + Use: "graph", + Short: "Compute the graph of images, and print it.", + Long: "Compute the graph of images, and print it.", + Run: func(cmd *cobra.Command, _ []string) { + bindPFlagsSnakeCase(cmd.Flags()) + + opts := GraphOpts{} + hydrateOptsFromViper(&opts) + + if err := doGraph(opts); err != nil { + logger.Fatalf("Graph failed: %v", err) + } + }, +} + +func init() { + rootCmd.AddCommand(graphCmd) +} + +func doGraph(opts GraphOpts) error { + workingDir, err := getWorkingDir() + if err != nil { + logger.Fatalf("failed to get current working directory: %v", err) + } + + buildPath := path.Join(workingDir, opts.BuildPath) + logger.Infof("Building images in directory \"%s\"", buildPath) + + logger.Debugf("Generate DAG") + graph, err := dib.GenerateDAG(buildPath, opts.RegistryURL, opts.HashListFilePath, map[string]string{}) + if err != nil { + return fmt.Errorf("cannot generate DAG: %w", err) + } + logger.Debugf("Generate DAG -- Done") + + logger.Debugf("Print DAG") + graph.Print(opts.BuildPath) + logger.Debugf("Print DAG -- Done") + return nil +} diff --git a/go.mod b/go.mod index 12aaa2894..95a7252bb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.26.1 github.com/aws/aws-sdk-go-v2/config v1.27.11 github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/docker/cli v26.0.2+incompatible github.com/google/uuid v1.6.0 github.com/mholt/archiver/v3 v3.5.1 @@ -56,7 +57,6 @@ require ( github.com/containerd/containerd v1.7.12 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker v24.0.0+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect diff --git a/pkg/dag/dag.go b/pkg/dag/dag.go index 99ea40c7f..1af9e3843 100644 --- a/pkg/dag/dag.go +++ b/pkg/dag/dag.go @@ -1,8 +1,12 @@ package dag import ( + "cmp" + "slices" + "strings" "sync" + "github.com/radiofrance/dib/internal/logger" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" ) @@ -203,3 +207,21 @@ func createUniqueVisitorErr(visitor NodeVisitorFuncErr) NodeVisitorFuncErr { return uniqueVisitor } + +func sort(a, b *Node) int { + return cmp.Compare(strings.ToLower(a.Image.ShortName), strings.ToLower(b.Image.ShortName)) +} + +func (d *DAG) Print(name string) { + d.WalkInDepth(func(node *Node) { + slices.SortFunc(node.Children(), sort) + }) + slices.SortFunc(d.nodes, sort) + rootNode := &Node{ + Image: &Image{Name: name}, + children: d.nodes, + } + if err := DefaultTree.WithRoot(rootNode).Render(); err != nil { + logger.Fatalf(err.Error()) + } +} diff --git a/pkg/dag/node.go b/pkg/dag/node.go index 99e104580..a48888402 100644 --- a/pkg/dag/node.go +++ b/pkg/dag/node.go @@ -51,7 +51,7 @@ func (n *Node) Parents() []*Node { // walk applies the visitor func to the current node, then to every children nodes, recursively. func (n *Node) walk(visitor NodeVisitorFunc) { visitor(n) - for _, childNode := range n.children { + for _, childNode := range n.Children() { childNode.walk(visitor) } } @@ -63,7 +63,7 @@ func (n *Node) walkErr(visitor NodeVisitorFuncErr) error { if err != nil { return err } - for _, childNode := range n.children { + for _, childNode := range n.Children() { err = childNode.walkErr(visitor) if err != nil { return err @@ -79,7 +79,7 @@ func (n *Node) walkAsyncErr(visitor NodeVisitorFuncErr) error { errG.Go(func() error { return visitor(n) }) - for _, childNode := range n.children { + for _, childNode := range n.Children() { errG.Go(func() error { return childNode.walkAsyncErr(visitor) }) @@ -90,7 +90,7 @@ func (n *Node) walkAsyncErr(visitor NodeVisitorFuncErr) error { // walkInDepth makes a depth-first recursive walk through the graph. // It applies the visitor func to every children node, then to the current node itself. func (n *Node) walkInDepth(visitor NodeVisitorFunc) { - for _, childNode := range n.children { + for _, childNode := range n.Children() { childNode.walkInDepth(visitor) } visitor(n) diff --git a/pkg/dag/printer.go b/pkg/dag/printer.go new file mode 100644 index 000000000..b7211112e --- /dev/null +++ b/pkg/dag/printer.go @@ -0,0 +1,142 @@ +package dag + +import ( + "io" + "strings" + + "github.com/pterm/pterm" +) + +// DefaultTree contains standards, which can be used to render a TreePrinter. +var DefaultTree = TreePrinter{ + TreeStyle: &pterm.ThemeDefault.TreeStyle, + TextStyle: &pterm.ThemeDefault.TreeTextStyle, + TopRightCornerString: "└", + HorizontalString: "─", + TopRightDownString: "├", + VerticalString: "│", + RightDownLeftString: "┬", + Indent: 2, +} + +// TreePrinter is able to render a list. +type TreePrinter struct { + Root *Node + TreeStyle *pterm.Style + TextStyle *pterm.Style + TopRightCornerString string + TopRightDownString string + HorizontalString string + VerticalString string + RightDownLeftString string + Indent int + Writer io.Writer +} + +// WithTreeStyle returns a new list with a specific tree style. +func (p TreePrinter) WithTreeStyle(style *pterm.Style) *TreePrinter { + p.TreeStyle = style + return &p +} + +// WithTopRightCornerString returns a new list with a specific TopRightCornerString. +func (p TreePrinter) WithTopRightCornerString(s string) *TreePrinter { + p.TopRightCornerString = s + return &p +} + +// WithTopRightDownStringOngoing returns a new list with a specific TopRightDownString. +func (p TreePrinter) WithTopRightDownStringOngoing(s string) *TreePrinter { + p.TopRightDownString = s + return &p +} + +// WithHorizontalString returns a new list with a specific HorizontalString. +func (p TreePrinter) WithHorizontalString(s string) *TreePrinter { + p.HorizontalString = s + return &p +} + +// WithVerticalString returns a new list with a specific VerticalString. +func (p TreePrinter) WithVerticalString(s string) *TreePrinter { + p.VerticalString = s + return &p +} + +// WithRoot returns a new list with a specific Root. +func (p TreePrinter) WithRoot(root *Node) *TreePrinter { + p.Root = root + return &p +} + +// WithIndent returns a new list with a specific amount of spacing between the levels. +// Indent must be at least 1. +func (p TreePrinter) WithIndent(indent int) *TreePrinter { + if indent < 1 { + indent = 1 + } + p.Indent = indent + return &p +} + +// Render prints the list to the terminal. +func (p TreePrinter) Render() error { + s, _ := p.Srender() + pterm.Fprintln(p.Writer, s) + + return nil +} + +// Srender renders the list as a string. +func (p TreePrinter) Srender() (string, error) { + if p.TreeStyle == nil { + p.TreeStyle = pterm.NewStyle() + } + if p.TextStyle == nil { + p.TextStyle = pterm.NewStyle() + } + + var result string + if p.Root.Image.Name != "" { + result += p.TextStyle.Sprint(p.Root.Image.Name) + "\n" + } + result += walkOverTree(p.Root.Children(), p, "") + return result, nil +} + +// walkOverTree is a recursive function, +// which analyzes a TreePrinter and connects the items with specific characters. +// Returns TreePrinter as string. +func walkOverTree(nodes []*Node, printer TreePrinter, prefix string) string { + var ret string + for nodeIndex, node := range nodes { + if len(nodes) > nodeIndex+1 { // if not last in nodes + if len(node.Children()) == 0 { // if there are no children + ret += prefix + printer.TreeStyle.Sprint(printer.TopRightDownString) + + strings.Repeat(printer.TreeStyle.Sprint(printer.HorizontalString), printer.Indent) + + printer.TextStyle.Sprint(node.Image.ShortName) + "\n" + } else { // if there are children + ret += prefix + printer.TreeStyle.Sprint(printer.TopRightDownString) + + strings.Repeat(printer.TreeStyle.Sprint(printer.HorizontalString), printer.Indent-1) + + printer.TreeStyle.Sprint(printer.RightDownLeftString) + + printer.TextStyle.Sprint(node.Image.ShortName) + "\n" + ret += walkOverTree(node.Children(), printer, + prefix+printer.TreeStyle.Sprint(printer.VerticalString)+strings.Repeat(" ", printer.Indent-1)) + } + } else if len(nodes) == nodeIndex+1 { // if last in nodes + if len(node.Children()) == 0 { // if there are no children + ret += prefix + printer.TreeStyle.Sprint(printer.TopRightCornerString) + + strings.Repeat(printer.TreeStyle.Sprint(printer.HorizontalString), printer.Indent) + + printer.TextStyle.Sprint(node.Image.ShortName) + "\n" + } else { // if there are children + ret += prefix + printer.TreeStyle.Sprint(printer.TopRightCornerString) + + strings.Repeat(printer.TreeStyle.Sprint(printer.HorizontalString), printer.Indent-1) + + printer.TreeStyle.Sprint(printer.RightDownLeftString) + + printer.TextStyle.Sprint(node.Image.ShortName) + "\n" + ret += walkOverTree(node.Children(), printer, + prefix+strings.Repeat(" ", printer.Indent)) + } + } + } + return ret +} diff --git a/pkg/dib/generate_dag.go b/pkg/dib/generate_dag.go index d801ca880..91185e89d 100644 --- a/pkg/dib/generate_dag.go +++ b/pkg/dib/generate_dag.go @@ -29,90 +29,76 @@ const ( // and generates the DAG representing the relationships between images. func GenerateDAG(buildPath, registryPrefix, customHashListPath string, buildArgs map[string]string) (*dag.DAG, error) { var allFiles []string - cache := make(map[string]*dag.Node) - allParents := make(map[string][]dockerfile.ImageRef) - err := filepath.Walk(buildPath, func(filePath string, info os.FileInfo, err error) error { + nodes := make(map[string]*dag.Node) + if err := filepath.Walk(buildPath, func(filePath string, info os.FileInfo, err error) error { if err != nil { return err } + if !info.IsDir() { allFiles = append(allFiles, filePath) } - if dockerfile.IsDockerfile(filePath) { - dckfile, err := dockerfile.ParseDockerfile(filePath) - if err != nil { - return err - } + if !dockerfile.IsDockerfile(filePath) { + return nil + } - skipBuild, hasSkipLabel := dckfile.Labels["skipbuild"] - if hasSkipLabel && skipBuild == "true" { - return nil - } - imageShortName, hasNameLabel := dckfile.Labels["name"] - if !hasNameLabel { - return fmt.Errorf("missing label \"name\" in Dockerfile at path \"%s\"", filePath) - } - img := &dag.Image{ - Name: fmt.Sprintf("%s/%s", registryPrefix, imageShortName), - ShortName: imageShortName, - Dockerfile: dckfile, - } + dckfile, err := dockerfile.ParseDockerfile(filePath) + if err != nil { + return err + } - extraTagsLabel, hasLabel := img.Dockerfile.Labels["dib.extra-tags"] - if hasLabel { - img.ExtraTags = append(img.ExtraTags, strings.Split(extraTagsLabel, ",")...) - } + skipBuild, hasSkipLabel := dckfile.Labels["skipbuild"] + if hasSkipLabel && skipBuild == "true" { + return nil + } + imageShortName, hasNameLabel := dckfile.Labels["name"] + if !hasNameLabel { + return fmt.Errorf("missing label \"name\" in Dockerfile at path \"%s\"", filePath) + } + img := &dag.Image{ + Name: fmt.Sprintf("%s/%s", registryPrefix, imageShortName), + ShortName: imageShortName, + Dockerfile: dckfile, + } - useCustomHashList, hasLabel := img.Dockerfile.Labels["dib.use-custom-hash-list"] - if hasLabel && useCustomHashList == "true" { - img.UseCustomHashList = true - } + extraTagsLabel, hasLabel := img.Dockerfile.Labels["dib.extra-tags"] + if hasLabel { + img.ExtraTags = append(img.ExtraTags, strings.Split(extraTagsLabel, ",")...) + } - ignorePatterns, err := build.ReadDockerignore(path.Dir(filePath)) - if err != nil { - return fmt.Errorf("could not read ignore patterns: %w", err) - } - img.IgnorePatterns = ignorePatterns + useCustomHashList, hasLabel := img.Dockerfile.Labels["dib.use-custom-hash-list"] + if hasLabel && useCustomHashList == "true" { + img.UseCustomHashList = true + } - allParents[img.Name] = dckfile.From - cache[img.Name] = dag.NewNode(img) + ignorePatterns, err := build.ReadDockerignore(path.Dir(filePath)) + if err != nil { + return fmt.Errorf("could not read ignore patterns: %w", err) } + img.IgnorePatterns = ignorePatterns + + nodes[img.Name] = dag.NewNode(img) + return nil - }) - if err != nil { + }); err != nil { return nil, err } - // Fill parents for each image, for simplicity of use in other functions - for name, parents := range allParents { - for _, parent := range parents { - node, ok := cache[parent.Name] - if !ok { - continue - } - - // Check that children does not already exist to avoid duplicates. - childAlreadyExists := false - for _, child := range node.Children() { - if child.Image.Name == name { - childAlreadyExists = true - } + for _, node := range nodes { + for _, parent := range node.Image.Dockerfile.From { + parentNode, ok := nodes[parent.Name] + if ok { + parentNode.AddChild(node) } - - if childAlreadyExists { - continue - } - - node.AddChild(cache[name]) } } graph := &dag.DAG{} // If an image has no parents in the DAG, we consider it a root image - for name, img := range cache { + for name, img := range nodes { if len(img.Parents()) == 0 { - graph.AddNode(cache[name]) + graph.AddNode(nodes[name]) } } diff --git a/pkg/dib/generate_dag_test.go b/pkg/dib/generate_dag_test.go index 47a6dee2c..40a1983c5 100644 --- a/pkg/dib/generate_dag_test.go +++ b/pkg/dib/generate_dag_test.go @@ -21,6 +21,8 @@ func TestGenerateDAG(t *testing.T) { "eu.gcr.io/my-test-repository", "", map[string]string{}) require.NoError(t, err) + graph.Print(fixtureDir) + nodes := flattenNodes(graph) rootNode := nodes["bullseye"] subNode := nodes["sub-image"]