diff --git a/dev/scripts/src/alluxio.org/build/assembly.yml b/dev/scripts/src/alluxio.org/build/assembly.yml new file mode 100644 index 000000000000..aafa7a5327d5 --- /dev/null +++ b/dev/scripts/src/alluxio.org/build/assembly.yml @@ -0,0 +1,35 @@ +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +client: + generatedJarPath: assembly/client/target/alluxio-assembly-client-${VERSION}-jar-with-dependencies.jar + tarballJarPath: assembly/alluxio-client-${VERSION}.jar + fileReplacements: + libexec/alluxio-config.sh: + assembly/client/target/alluxio-assembly-client-${VERSION}-jar-with-dependencies.jar: assembly/alluxio-client-${VERSION}.jar +server: + generatedJarPath: assembly/server/target/alluxio-assembly-server-${VERSION}-jar-with-dependencies.jar + tarballJarPath: assembly/alluxio-server-${VERSION}.jar + fileReplacements: + libexec/alluxio-config.sh: + assembly/server/target/alluxio-assembly-server-${VERSION}-jar-with-dependencies.jar: assembly/alluxio-server-${VERSION}.jar +fuseBundled: + generatedJarPath: dora/integration/fuse/target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar + tarballJarPath: dora/integration/fuse/alluxio-fuse-${VERSION}.jar + fileReplacements: + dora/integration/fuse/bin/alluxio-fuse: + target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar: alluxio-fuse-${VERSION}.jar +fuseStandalone: + generatedJarPath: dora/integration/fuse/target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar + tarballJarPath: lib/alluxio-fuse-${VERSION}.jar + fileReplacements: + dora/integration/fuse/bin/alluxio-fuse: + target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar: ../../../lib/alluxio-fuse-${VERSION}.jar diff --git a/dev/scripts/src/alluxio.org/build/cmd/assembly.go b/dev/scripts/src/alluxio.org/build/cmd/assembly.go new file mode 100644 index 000000000000..ae85d9ff7a6f --- /dev/null +++ b/dev/scripts/src/alluxio.org/build/cmd/assembly.go @@ -0,0 +1,48 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package cmd + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + + "github.com/palantir/stacktrace" + "gopkg.in/yaml.v3" +) + +type AssemblyJar struct { + GeneratedJarPath string `yaml:"generatedJarPath"` // relative path of generated jar, before formatting with alluxio version string + TarballJarPath string `yaml:"tarballJarPath"` // relative path of copied jar into the tarball, before formatting with alluxio version string + FileReplacements map[string]map[string]string `yaml:"fileReplacements"` // location of file to execute replacement -> find -> replace +} + +type AssemblyJars map[string]*AssemblyJar + +func loadAssemblyJars(assemblyYml string) (AssemblyJars, error) { + wd, err := os.Getwd() + if err != nil { + return nil, stacktrace.Propagate(err, "error getting current working directory") + } + assemblyYmlPath := filepath.Join(wd, assemblyYml) + log.Printf("Reading assembly jars from %v", assemblyYmlPath) + content, err := ioutil.ReadFile(assemblyYmlPath) + if err != nil { + return nil, stacktrace.Propagate(err, "error reading file at %v", assemblyYmlPath) + } + var assemblyJars AssemblyJars + if err := yaml.Unmarshal(content, &assemblyJars); err != nil { + return nil, stacktrace.Propagate(err, "error unmarshalling assembly jars from:\n%v", string(content)) + } + return assemblyJars, nil +} diff --git a/dev/scripts/src/alluxio.org/build/cmd/build.go b/dev/scripts/src/alluxio.org/build/cmd/build.go index 0a945258e192..517fcbe27c3d 100644 --- a/dev/scripts/src/alluxio.org/build/cmd/build.go +++ b/dev/scripts/src/alluxio.org/build/cmd/build.go @@ -13,7 +13,6 @@ package cmd import ( "flag" - "fmt" "log" "os" "path/filepath" @@ -103,8 +102,8 @@ func buildTarball(opts *buildOpts) error { if opts.tarball.ClientJarName != "" { mockFiles = append(mockFiles, opts.tarball.clientJarPath(alluxioVersion)) } - for _, info := range assembledJars { - mockFiles = append(mockFiles, fmt.Sprintf(info.generatedJarPath, alluxioVersion)) + for _, info := range opts.assemblyJars { + mockFiles = append(mockFiles, strings.ReplaceAll(info.GeneratedJarPath, versionPlaceholder, alluxioVersion)) } for _, l := range opts.libModules { mockFiles = append(mockFiles, strings.ReplaceAll(l.GeneratedJarPath, versionPlaceholder, alluxioVersion)) diff --git a/dev/scripts/src/alluxio.org/build/cmd/flags.go b/dev/scripts/src/alluxio.org/build/cmd/flags.go index 88922fb90f47..e41f3377442c 100644 --- a/dev/scripts/src/alluxio.org/build/cmd/flags.go +++ b/dev/scripts/src/alluxio.org/build/cmd/flags.go @@ -41,6 +41,7 @@ var SubCmdNames = []string{ } const ( + defaultAssemblyFilePath = "src/alluxio.org/build/assembly.yml" defaultModulesFilePath = "src/alluxio.org/build/modules.yml" defaultProfilesFilePath = "src/alluxio.org/build/profiles.yml" @@ -49,6 +50,7 @@ const ( type buildOpts struct { artifactOutput string + assemblyJarFile string dryRun bool modulesFile string outputDir string @@ -56,6 +58,7 @@ type buildOpts struct { skipRepoCopy bool suppressMavenOutput bool + assemblyJars AssemblyJars mavenArgs []string libModules map[string]*LibModule pluginModules map[string]*PluginModule @@ -68,6 +71,7 @@ func parseTarballFlags(cmd *flag.FlagSet, args []string) (*buildOpts, error) { // common flags cmd.StringVar(&opts.artifactOutput, "artifact", "", "If set, writes object representing the tarball to YAML output file") + cmd.StringVar(&opts.assemblyJarFile, "assemblyFile", defaultAssemblyFilePath, "Path to assembly.yml file") cmd.StringVar(&opts.outputDir, "outputDir", repo.FindRepoRoot(), "Set output dir for generated tarball") cmd.BoolVar(&opts.dryRun, "dryRun", false, "If set, writes placeholder files instead of running maven commands to mock the final state of the build directory to be packaged as a tarball") cmd.StringVar(&opts.modulesFile, "modulesFile", defaultModulesFilePath, "Path to modules.yml file") @@ -113,6 +117,12 @@ func parseTarballFlags(cmd *flag.FlagSet, args []string) (*buildOpts, error) { if err := opts.processProfileValues(prof); err != nil { return nil, stacktrace.Propagate(err, "error processing profile values to update opts") } + // parse assembly jars + assemblyJars, err := loadAssemblyJars(opts.assemblyJarFile) + if err != nil { + return nil, stacktrace.Propagate(err, "error parsing assembly jars") + } + opts.assemblyJars = assemblyJars return opts, nil } diff --git a/dev/scripts/src/alluxio.org/build/cmd/tarball.go b/dev/scripts/src/alluxio.org/build/cmd/tarball.go index 495d52994f46..4f865298fc3c 100644 --- a/dev/scripts/src/alluxio.org/build/cmd/tarball.go +++ b/dev/scripts/src/alluxio.org/build/cmd/tarball.go @@ -13,64 +13,17 @@ package cmd import ( "bytes" - "fmt" "io/ioutil" "log" "os" "path/filepath" + "strings" "github.com/palantir/stacktrace" "alluxio.org/common/command" ) -type assembledJarsInfo struct { - generatedJarPath string // relative path of generated jar, before formatting with alluxio version string - tarballJarPath string // relative path of copied jar into the tarball, before formatting with alluxio version string - fileReplacements map[string]map[string]string // location of file to execute replacement -> find -> replace -} - -var ( - assembledJars = map[string]assembledJarsInfo{ - "client": { - generatedJarPath: "assembly/client/target/alluxio-assembly-client-%v-jar-with-dependencies.jar", - tarballJarPath: "assembly/alluxio-client-%v.jar", - fileReplacements: map[string]map[string]string{ - "libexec/alluxio-config.sh": { - fmt.Sprintf("assembly/client/target/alluxio-assembly-client-%v-jar-with-dependencies.jar", versionPlaceholder): fmt.Sprintf("assembly/alluxio-client-%v.jar", versionPlaceholder), - }, - }, - }, - "server": { - generatedJarPath: "assembly/server/target/alluxio-assembly-server-%v-jar-with-dependencies.jar", - tarballJarPath: "assembly/alluxio-server-%v.jar", - fileReplacements: map[string]map[string]string{ - "libexec/alluxio-config.sh": { - fmt.Sprintf("assembly/server/target/alluxio-assembly-server-%v-jar-with-dependencies.jar", versionPlaceholder): fmt.Sprintf("assembly/alluxio-server-%v.jar", versionPlaceholder), - }, - }, - }, - "fuseBundled": { - generatedJarPath: "dora/integration/fuse/target/alluxio-integration-fuse-%v-jar-with-dependencies.jar", - tarballJarPath: "dora/integration/fuse/alluxio-fuse-%v.jar", - fileReplacements: map[string]map[string]string{ - "dora/integration/fuse/bin/alluxio-fuse": { - fmt.Sprintf("target/alluxio-integration-fuse-%v-jar-with-dependencies.jar", versionPlaceholder): fmt.Sprintf("alluxio-fuse-%v.jar", versionPlaceholder), - }, - }, - }, - "fuseStandalone": { - generatedJarPath: "dora/integration/fuse/target/alluxio-integration-fuse-%v-jar-with-dependencies.jar", - tarballJarPath: "lib/alluxio-fuse-%v.jar", - fileReplacements: map[string]map[string]string{ - "dora/integration/fuse/bin/alluxio-fuse": { - fmt.Sprintf("target/alluxio-integration-fuse-%v-jar-with-dependencies.jar", versionPlaceholder): fmt.Sprintf("../../../lib/alluxio-fuse-%v.jar", versionPlaceholder), - }, - }, - }, - } -) - func collectTarballContents(opts *buildOpts, repoBuildDir, dstDir, alluxioVersion string) error { for _, f := range opts.tarball.FileList { if err := copyFileForTarball(filepath.Join(repoBuildDir, f), filepath.Join(dstDir, f)); err != nil { @@ -87,18 +40,18 @@ func collectTarballContents(opts *buildOpts, repoBuildDir, dstDir, alluxioVersio // add assembly jars, rename jars, and update name used in scripts for _, n := range opts.tarball.AssemblyJars { - a, ok := assembledJars[n] + a, ok := opts.assemblyJars[n] if !ok { return stacktrace.NewError("no assembly jar named %v", n) } - src := filepath.Join(repoBuildDir, fmt.Sprintf(a.generatedJarPath, alluxioVersion)) - dst := filepath.Join(dstDir, fmt.Sprintf(a.tarballJarPath, alluxioVersion)) + src := filepath.Join(repoBuildDir, strings.ReplaceAll(a.GeneratedJarPath, versionPlaceholder, alluxioVersion)) + dst := filepath.Join(dstDir, strings.ReplaceAll(a.TarballJarPath, versionPlaceholder, alluxioVersion)) if err := copyFileForTarball(src, dst); err != nil { return stacktrace.Propagate(err, "error copying file from %v to %v", src, dst) } // replace corresponding reference in scripts - for filePath, replacements := range a.fileReplacements { + for filePath, replacements := range a.FileReplacements { replacementFile := filepath.Join(dstDir, filePath) stat, err := os.Stat(replacementFile) if err != nil { @@ -109,7 +62,6 @@ func collectTarballContents(opts *buildOpts, repoBuildDir, dstDir, alluxioVersio return stacktrace.Propagate(err, "error reading file at %v", replacementFile) } for find, replace := range replacements { - // ex. "alluxio-assembly-client-${VERSION}-jar-with-dependencies.jar" -> "alluxio-assembly-client-${VERSION}.jar contents = bytes.ReplaceAll(contents, []byte(find), []byte(replace)) } if err := ioutil.WriteFile(replacementFile, contents, stat.Mode()); err != nil {