diff --git a/apps/distgo/app.go b/apps/distgo/app.go index 9448ffb3..f822c307 100644 --- a/apps/distgo/app.go +++ b/apps/distgo/app.go @@ -21,6 +21,7 @@ import ( "github.com/palantir/godel/apps/distgo/cmd/artifacts" "github.com/palantir/godel/apps/distgo/cmd/build" + "github.com/palantir/godel/apps/distgo/cmd/clean" "github.com/palantir/godel/apps/distgo/cmd/dist" "github.com/palantir/godel/apps/distgo/cmd/docker" "github.com/palantir/godel/apps/distgo/cmd/products" @@ -38,6 +39,7 @@ func App() *cli.App { products.Command(), artifacts.Command(), build.Command(), + clean.Command(), run.Command(), dist.Command(), docker.Command(), diff --git a/apps/distgo/cmd/clean/clean.go b/apps/distgo/cmd/clean/clean.go new file mode 100644 index 00000000..59a66e05 --- /dev/null +++ b/apps/distgo/cmd/clean/clean.go @@ -0,0 +1,285 @@ +// Copyright 2016 Palantir Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clean + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path" + "regexp" + "sort" + "strings" + + "github.com/pkg/errors" + + "github.com/palantir/godel/apps/distgo/cmd/build" + "github.com/palantir/godel/apps/distgo/cmd/dist" + "github.com/palantir/godel/apps/distgo/params" + "github.com/palantir/godel/apps/distgo/pkg/osarch" +) + +func Products(products []string, cfg params.Project, dryRun bool, wd string, stdout io.Writer) error { + buildSpecsWithDeps, err := build.SpecsWithDepsForArgs(cfg, products, wd) + if err != nil { + return err + } + for _, specWithDeps := range buildSpecsWithDeps { + if err := Run(specWithDeps, dryRun, stdout); err != nil { + return errors.Wrapf(err, "failed to clean %s", specWithDeps.Spec.ProductName) + } + } + return nil +} + +type pathInfo struct { + // path to the "root" output directory (output directory of "bin" or "dist" tasks). + rootDir string + // true if this path is a directory, false if it is a file + isDir bool +} + +// Run cleans the outputs generated by the specified product. +func Run(buildSpecWithDeps params.ProductBuildSpecWithDeps, dryRun bool, stdout io.Writer) error { + // outputDir -> product -> osArchs + outputDirToMap := make(map[string]map[string][]osarch.OSArch) + + // add primary product + outputDirToMap[path.Join(buildSpecWithDeps.Spec.ProjectDir, buildSpecWithDeps.Spec.Build.OutputDir)] = map[string][]osarch.OSArch{ + buildSpecWithDeps.Spec.ProductName: buildSpecWithDeps.Spec.Build.OSArchs, + } + + // add dependent products + for _, currDepSpec := range buildSpecWithDeps.Deps { + currMap, ok := outputDirToMap[path.Join(buildSpecWithDeps.Spec.ProjectDir, currDepSpec.Build.OutputDir)] + if !ok { + currMap = make(map[string][]osarch.OSArch) + outputDirToMap[path.Join(buildSpecWithDeps.Spec.ProjectDir, currDepSpec.Build.OutputDir)] = currMap + } + currMap[currDepSpec.ProductName] = currDepSpec.Build.OSArchs + } + + // map of paths to remove. Value is true if currPath is a directory, false otherwise. + removePaths := make(map[string]pathInfo) + + // remove binaries for specified products + for outputDir, products := range outputDirToMap { + removed, err := cleanBinOutput(outputDir, products) + if err != nil { + return errors.Wrapf(err, "failed to remove") + } + for k := range removed { + removePaths[k] = pathInfo{ + rootDir: outputDir, + isDir: false, + } + } + } + + // remove dists for product + buildSpecWithDeps.Spec.ProductVersion = ".+" + for _, currDist := range buildSpecWithDeps.Spec.Dist { + var distDirRegexps []*regexp.Regexp + distDirRegexps = append(distDirRegexps, regexp.MustCompile(fmt.Sprintf("%s-.+", buildSpecWithDeps.Spec.ProductName))) + for _, currDistPath := range dist.FullArtifactsPaths(dist.ToDister(currDist.Info), buildSpecWithDeps.Spec, currDist) { + distDirRegexps = append(distDirRegexps, regexp.MustCompile(path.Base(currDistPath))) + } + distDirFiles, err := ioutil.ReadDir(path.Join(buildSpecWithDeps.Spec.ProjectDir, currDist.OutputDir)) + if err != nil { + continue + } + for _, currDistFile := range distDirFiles { + for _, currRegexp := range distDirRegexps { + if currRegexp.MatchString(currDistFile.Name()) { + removePaths[path.Join(buildSpecWithDeps.Spec.ProjectDir, currDist.OutputDir, currDistFile.Name())] = pathInfo{ + rootDir: path.Join(buildSpecWithDeps.Spec.ProjectDir, currDist.OutputDir), + isDir: currDistFile.IsDir(), + } + break + } + } + } + } + + var sortedPaths []string + for k := range removePaths { + sortedPaths = append(sortedPaths, k) + } + sort.Strings(sortedPaths) + + prefix := "[DRY RUN]" + if dryRun { + fmt.Fprintf(stdout, "%s Clean %s will remove paths:\n", prefix, buildSpecWithDeps.Spec.ProductName) + } + + // stores all of the paths that were removed/marked for removal + removedPaths := make(map[string]struct{}) + for _, currPath := range sortedPaths { + pathInfo := removePaths[currPath] + if dryRun { + fmt.Fprintf(stdout, "%s %s\n", prefix, currPath) + } else { + // if target path exists, attempt to remove it + if _, err := os.Stat(currPath); err == nil { + if pathInfo.isDir { + if err := os.RemoveAll(currPath); err != nil { + return errors.Wrapf(err, "failed to remove directory %s", currPath) + } + } else { + if err := os.Remove(currPath); err != nil { + return errors.Wrapf(err, "failed to remove file %s", currPath) + } + } + } + } + removedPaths[currPath] = struct{}{} + + // verify that current path is direct descendant of root directory + if !strings.Contains(currPath, pathInfo.rootDir) { + return errors.Errorf("root dir path %s does not occur in %s", pathInfo.rootDir, currPath) + } + + // for each parent directory between the removed path and the root, check if removal caused it to become empty. + // If so, remove it and continue the process. + currParentDir := currPath + for { + currParentDir = path.Dir(currParentDir) + if currParentDir == pathInfo.rootDir { + break + } + if _, err := os.Stat(currParentDir); os.IsNotExist(err) { + // nothing to do if parent directory does not exist + break + } + removed, err := removeDirIfEmpty(currParentDir, removedPaths, dryRun) + if err != nil { + return err + } + if !removed { + // if there was no error and directory was not removed, nothing more to do + break + } + if dryRun { + fmt.Fprintf(stdout, "%s %s\n", prefix, currParentDir) + } + } + // remove root directory if it is now empty + rootDirRemoved, err := removeDirIfEmpty(pathInfo.rootDir, removedPaths, dryRun) + if err != nil { + return err + } + if rootDirRemoved && dryRun { + fmt.Fprintf(stdout, "%s %s\n", prefix, pathInfo.rootDir) + } + } + return nil +} + +// Removes the given path (which must be a directory) if it exists and is empty. Returns true if the directory was +// removed, false otherwise. Returns an error if the provided path was not a directory of if there was an error reading +// or removing it. +func removeDirIfEmpty(dirPath string, removedPaths map[string]struct{}, dryRun bool) (bool, error) { + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + return false, nil + } + dirFiles, err := ioutil.ReadDir(dirPath) + if err != nil { + return false, errors.Wrapf(err, "failed to read directory: %s", dirPath) + } + + // if this is a dry run, determine the files that "should" exist + if dryRun { + var dryRunDirFiles []os.FileInfo + for _, currDirFile := range dirFiles { + if _, ok := removedPaths[path.Join(dirPath, currDirFile.Name())]; ok { + // path is a path marked for removal: do not consider it + continue + } + dryRunDirFiles = append(dryRunDirFiles, currDirFile) + } + dirFiles = dryRunDirFiles + } + + if len(dirFiles) != 0 { + // if directory contains files, nothing to do: do not remove + return false, nil + } + + if !dryRun { + // if this is not a dry run, actually perform the removal + if err := os.RemoveAll(dirPath); err != nil { + return false, errors.Wrapf(err, "failed to remove directory %s", dirPath) + } + } + + removedPaths[dirPath] = struct{}{} + return true, nil +} + +func cleanBinOutput(outputDir string, products map[string][]osarch.OSArch) (map[string]struct{}, error) { + removedPaths := make(map[string]struct{}) + + osArchToProducts := make(map[string]map[string]struct{}) + for currProduct, prodOSArchs := range products { + for _, currOSArch := range prodOSArchs { + currProductsMap, ok := osArchToProducts[currOSArch.String()] + if !ok { + currProductsMap = make(map[string]struct{}) + osArchToProducts[currOSArch.String()] = currProductsMap + } + currProductsMap[currProduct] = struct{}{} + } + } + + outputDirFiles, err := ioutil.ReadDir(outputDir) + if err != nil { + return nil, errors.Wrapf(err, "failed to read directory %s", outputDir) + } + for _, outputDirFile := range outputDirFiles { + if !outputDirFile.IsDir() { + continue + } + // directory in top-level output directory: could be a tag directory. Examine all os-arch directories within it. + currTagDirPath := path.Join(outputDir, outputDirFile.Name()) + tagDirFiles, err := ioutil.ReadDir(currTagDirPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read directory %s", currTagDirPath) + } + for _, tagDirFile := range tagDirFiles { + if !tagDirFile.IsDir() { + continue + } + products, ok := osArchToProducts[tagDirFile.Name()] + if !ok { + continue + } + // at least one product of this OS/architecture exists: examine contents + currOSArchDirPath := path.Join(currTagDirPath, tagDirFile.Name()) + osArchDirFiles, err := ioutil.ReadDir(currOSArchDirPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read directory %s", currOSArchDirPath) + } + for _, osArchDirFile := range osArchDirFiles { + if _, ok := products[osArchDirFile.Name()]; !ok { + continue + } + binToRemovePath := path.Join(currOSArchDirPath, osArchDirFile.Name()) + removedPaths[binToRemovePath] = struct{}{} + } + } + } + return removedPaths, nil +} diff --git a/apps/distgo/cmd/clean/clean_test.go b/apps/distgo/cmd/clean/clean_test.go new file mode 100644 index 00000000..79dcf08c --- /dev/null +++ b/apps/distgo/cmd/clean/clean_test.go @@ -0,0 +1,136 @@ +// Copyright 2016 Palantir Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clean_test + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/nmiyake/pkg/dirs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/palantir/godel/apps/distgo/cmd/build" + "github.com/palantir/godel/apps/distgo/cmd/clean" + "github.com/palantir/godel/apps/distgo/cmd/dist" + "github.com/palantir/godel/apps/distgo/params" + "github.com/palantir/godel/apps/distgo/pkg/git" + "github.com/palantir/godel/apps/distgo/pkg/git/gittest" + "github.com/palantir/godel/apps/distgo/pkg/osarch" +) + +const ( + testMain = `package main + +import "fmt" + +var testVersionVar = "defaultVersion" + +func main() { + fmt.Println(testVersionVar) +} +` +) + +func TestDist(t *testing.T) { + tmp, cleanup, err := dirs.TempDir("", "") + defer cleanup() + require.NoError(t, err) + + for i, currCase := range []struct { + name string + spec func(projectDir string) params.ProductBuildSpecWithDeps + preDistAction func(projectDir string, buildSpec params.ProductBuildSpec) + preValidate func(caseNum int, name string, projectDir string) + postValidate func(caseNum int, name string, projectDir string) + }{ + { + name: "cleans default distribution", + spec: func(projectDir string) params.ProductBuildSpecWithDeps { + specWithDeps, err := params.NewProductBuildSpecWithDeps(params.NewProductBuildSpec( + projectDir, + "foo", + git.ProjectInfo{ + Version: "0.1.0", + }, + params.Product{ + Build: params.Build{ + MainPkg: "./.", + }, + }, + params.Project{ + GroupID: "com.test.group", + }, + ), nil) + require.NoError(t, err) + return specWithDeps + }, + preDistAction: func(projectDir string, buildSpec params.ProductBuildSpec) { + gittest.CreateGitTag(t, projectDir, "0.1.0") + }, + preValidate: func(caseNum int, name string, projectDir string) { + info, err := os.Stat(path.Join(projectDir, "build", "0.1.0", osarch.Current().String(), "foo")) + require.NoError(t, err) + assert.False(t, info.IsDir(), "Case %d: %s", caseNum, name) + + info, err = os.Stat(path.Join(projectDir, "dist", fmt.Sprintf("foo-0.1.0-%s.tgz", osarch.Current().String()))) + require.NoError(t, err) + assert.False(t, info.IsDir(), "Case %d: %s", caseNum, name) + }, + postValidate: func(caseNum int, name string, projectDir string) { + _, err := os.Stat(path.Join(projectDir, "build", "0.1.0", osarch.Current().String(), "foo")) + assert.True(t, os.IsNotExist(err), "Case %d: %s", caseNum, name) + _, err = os.Stat(path.Join(projectDir, "build")) + assert.True(t, os.IsNotExist(err), "Case %d: %s", caseNum, name) + + _, err = os.Stat(path.Join(projectDir, "dist", fmt.Sprintf("foo-0.1.0-%s.tgz", osarch.Current().String()))) + assert.True(t, os.IsNotExist(err), "Case %d: %s", caseNum, name) + _, err = os.Stat(path.Join(projectDir, "dist")) + assert.True(t, os.IsNotExist(err), "Case %d: %s", caseNum, name) + }, + }, + } { + currTmpDir, err := ioutil.TempDir(tmp, "") + require.NoError(t, err, "Case %d: %s", i, currCase.name) + + gittest.InitGitDir(t, currTmpDir) + err = ioutil.WriteFile(path.Join(currTmpDir, "main.go"), []byte(testMain), 0644) + require.NoError(t, err, "Case %d: %s", i, currCase.name) + gittest.CommitAllFiles(t, currTmpDir, "Commit") + + if currCase.preDistAction != nil { + currCase.preDistAction(currTmpDir, currCase.spec(currTmpDir).Spec) + } + + currSpecWithDeps := currCase.spec(currTmpDir) + err = build.Run(build.RequiresBuild(currSpecWithDeps, nil).Specs(), nil, build.Context{}, ioutil.Discard) + require.NoError(t, err, "Case %d: %s", i, currCase.name) + + err = dist.Run(currSpecWithDeps, ioutil.Discard) + require.NoError(t, err, "Case %d: %s", i, currCase.name) + if currCase.preValidate != nil { + currCase.preValidate(i, currCase.name, currTmpDir) + } + + err = clean.Run(currSpecWithDeps, false, ioutil.Discard) + require.NoError(t, err, "Case %d: %s", i, currCase.name) + if currCase.postValidate != nil { + currCase.postValidate(i, currCase.name, currTmpDir) + } + } +} diff --git a/apps/distgo/cmd/clean/cmd.go b/apps/distgo/cmd/clean/cmd.go new file mode 100644 index 00000000..4f12c304 --- /dev/null +++ b/apps/distgo/cmd/clean/cmd.go @@ -0,0 +1,54 @@ +// Copyright 2016 Palantir Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clean + +import ( + "github.com/nmiyake/pkg/dirs" + "github.com/palantir/pkg/cli" + "github.com/palantir/pkg/cli/cfgcli" + "github.com/palantir/pkg/cli/flag" + + "github.com/palantir/godel/apps/distgo/cmd" + "github.com/palantir/godel/apps/distgo/config" +) + +const ( + dryRunFlagName = "dry-run" +) + +func Command() cli.Command { + return cli.Command{ + Name: "clean", + Usage: "Remove the build and dist outputs for products", + Flags: []flag.Flag{ + cmd.ProductsParam, + flag.BoolFlag{ + Name: dryRunFlagName, + Usage: "Print the paths that would be removed by the operation without actually removing them", + }, + }, + Action: func(ctx cli.Context) error { + cfg, err := config.Load(cfgcli.ConfigPath, cfgcli.ConfigJSON) + if err != nil { + return err + } + wd, err := dirs.GetwdEvalSymLinks() + if err != nil { + return err + } + return Products(ctx.Slice(cmd.ProductsParamName), cfg, ctx.Bool(dryRunFlagName), wd, ctx.App.Stdout) + }, + } +} diff --git a/cmd/clicmds/cfgcli.go b/cmd/clicmds/cfgcli.go index b7d941e6..9806026a 100644 --- a/cmd/clicmds/cfgcli.go +++ b/cmd/clicmds/cfgcli.go @@ -155,6 +155,13 @@ var ( subcommandPath: []string{"dist"}, pathToCfg: []string{"dist.yml"}, }, + { + name: "clean", + app: distgoCreator, + decorator: distgoDecorator, + subcommandPath: []string{"clean"}, + pathToCfg: []string{"dist.yml"}, + }, { name: "artifacts", app: distgoCreator,