Refactor assembly jar defs in tarball gen

Alluxio · Jul 20, 2023 · 967fb9a · 967fb9a
1 parent a9e8172
commit 967fb9a
Show file tree

Hide file tree

Showing 5 changed files with 100 additions and 56 deletions.
diff --git a/dev/scripts/src/alluxio.org/build/assembly.yml b/dev/scripts/src/alluxio.org/build/assembly.yml
@@ -0,0 +1,35 @@
+#
+# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+# (the "License"). You may not use this work except in compliance with the License, which is
+# available at www.apache.org/licenses/LICENSE-2.0
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied, as more fully set forth in the License.
+#
+# See the NOTICE file distributed with this work for information regarding copyright ownership.
+#
+
+client:
+  generatedJarPath: assembly/client/target/alluxio-assembly-client-${VERSION}-jar-with-dependencies.jar
+  tarballJarPath: assembly/alluxio-client-${VERSION}.jar
+  fileReplacements:
+    libexec/alluxio-config.sh:
+      assembly/client/target/alluxio-assembly-client-${VERSION}-jar-with-dependencies.jar: assembly/alluxio-client-${VERSION}.jar
+server:
+  generatedJarPath: assembly/server/target/alluxio-assembly-server-${VERSION}-jar-with-dependencies.jar
+  tarballJarPath: assembly/alluxio-server-${VERSION}.jar
+  fileReplacements:
+    libexec/alluxio-config.sh:
+      assembly/server/target/alluxio-assembly-server-${VERSION}-jar-with-dependencies.jar: assembly/alluxio-server-${VERSION}.jar
+fuseBundled:
+  generatedJarPath: dora/integration/fuse/target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar
+  tarballJarPath: dora/integration/fuse/alluxio-fuse-${VERSION}.jar
+  fileReplacements:
+    dora/integration/fuse/bin/alluxio-fuse:
+      target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar: alluxio-fuse-${VERSION}.jar
+fuseStandalone:
+  generatedJarPath: dora/integration/fuse/target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar
+  tarballJarPath: lib/alluxio-fuse-${VERSION}.jar
+  fileReplacements:
+    dora/integration/fuse/bin/alluxio-fuse:
+      target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar: ../../../lib/alluxio-fuse-${VERSION}.jar
diff --git a/dev/scripts/src/alluxio.org/build/cmd/assembly.go b/dev/scripts/src/alluxio.org/build/cmd/assembly.go
@@ -0,0 +1,48 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package cmd
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/palantir/stacktrace"
+	"gopkg.in/yaml.v3"
+)
+
+type AssemblyJar struct {
+	GeneratedJarPath string                       `yaml:"generatedJarPath"` // relative path of generated jar, before formatting with alluxio version string
+	TarballJarPath   string                       `yaml:"tarballJarPath"`   // relative path of copied jar into the tarball, before formatting with alluxio version string
+	FileReplacements map[string]map[string]string `yaml:"fileReplacements"` // location of file to execute replacement -> find -> replace
+}
+
+type AssemblyJars map[string]*AssemblyJar
+
+func loadAssemblyJars(assemblyYml string) (AssemblyJars, error) {
+	wd, err := os.Getwd()
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "error getting current working directory")
+	}
+	assemblyYmlPath := filepath.Join(wd, assemblyYml)
+	log.Printf("Reading assembly jars from %v", assemblyYmlPath)
+	content, err := ioutil.ReadFile(assemblyYmlPath)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "error reading file at %v", assemblyYmlPath)
+	}
+	var assemblyJars AssemblyJars
+	if err := yaml.Unmarshal(content, &assemblyJars); err != nil {
+		return nil, stacktrace.Propagate(err, "error unmarshalling assembly jars from:\n%v", string(content))
+	}
+	return assemblyJars, nil
+}
diff --git a/dev/scripts/src/alluxio.org/build/cmd/build.go b/dev/scripts/src/alluxio.org/build/cmd/build.go
@@ -13,7 +13,6 @@ package cmd
 
 import (
 	"flag"
-	"fmt"
 	"log"
 	"os"
 	"path/filepath"
@@ -103,8 +102,8 @@ func buildTarball(opts *buildOpts) error {
 		if opts.tarball.ClientJarName != "" {
 			mockFiles = append(mockFiles, opts.tarball.clientJarPath(alluxioVersion))
 		}
-		for _, info := range assembledJars {
-			mockFiles = append(mockFiles, fmt.Sprintf(info.generatedJarPath, alluxioVersion))
+		for _, info := range opts.assemblyJars {
+			mockFiles = append(mockFiles, strings.ReplaceAll(info.GeneratedJarPath, versionPlaceholder, alluxioVersion))
 		}
 		for _, l := range opts.libModules {
 			mockFiles = append(mockFiles, strings.ReplaceAll(l.GeneratedJarPath, versionPlaceholder, alluxioVersion))

diff --git a/dev/scripts/src/alluxio.org/build/cmd/flags.go b/dev/scripts/src/alluxio.org/build/cmd/flags.go
@@ -41,6 +41,7 @@ var SubCmdNames = []string{
 }
 
 const (
+	defaultAssemblyFilePath = "src/alluxio.org/build/assembly.yml"
 	defaultModulesFilePath  = "src/alluxio.org/build/modules.yml"
 	defaultProfilesFilePath = "src/alluxio.org/build/profiles.yml"
 
@@ -49,13 +50,15 @@ const (
 
 type buildOpts struct {
 	artifactOutput      string
+	assemblyJarFile     string
 	dryRun              bool
 	modulesFile         string
 	outputDir           string
 	profilesFile        string
 	skipRepoCopy        bool
 	suppressMavenOutput bool
 
+	assemblyJars  AssemblyJars
 	mavenArgs     []string
 	libModules    map[string]*LibModule
 	pluginModules map[string]*PluginModule
@@ -68,6 +71,7 @@ func parseTarballFlags(cmd *flag.FlagSet, args []string) (*buildOpts, error) {
 
 	// common flags
 	cmd.StringVar(&opts.artifactOutput, "artifact", "", "If set, writes object representing the tarball to YAML output file")
+	cmd.StringVar(&opts.assemblyJarFile, "assemblyFile", defaultAssemblyFilePath, "Path to assembly.yml file")
 	cmd.StringVar(&opts.outputDir, "outputDir", repo.FindRepoRoot(), "Set output dir for generated tarball")
 	cmd.BoolVar(&opts.dryRun, "dryRun", false, "If set, writes placeholder files instead of running maven commands to mock the final state of the build directory to be packaged as a tarball")
 	cmd.StringVar(&opts.modulesFile, "modulesFile", defaultModulesFilePath, "Path to modules.yml file")
@@ -113,6 +117,12 @@ func parseTarballFlags(cmd *flag.FlagSet, args []string) (*buildOpts, error) {
 	if err := opts.processProfileValues(prof); err != nil {
 		return nil, stacktrace.Propagate(err, "error processing profile values to update opts")
 	}
+	// parse assembly jars
+	assemblyJars, err := loadAssemblyJars(opts.assemblyJarFile)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "error parsing assembly jars")
+	}
+	opts.assemblyJars = assemblyJars
 
 	return opts, nil
 }

diff --git a/dev/scripts/src/alluxio.org/build/cmd/tarball.go b/dev/scripts/src/alluxio.org/build/cmd/tarball.go
@@ -13,64 +13,17 @@ package cmd
 
 import (
 	"bytes"
-	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/palantir/stacktrace"
 
 	"alluxio.org/common/command"
 )
 
-type assembledJarsInfo struct {
-	generatedJarPath string                       // relative path of generated jar, before formatting with alluxio version string
-	tarballJarPath   string                       // relative path of copied jar into the tarball, before formatting with alluxio version string
-	fileReplacements map[string]map[string]string // location of file to execute replacement -> find -> replace
-}
-
-var (
-	assembledJars = map[string]assembledJarsInfo{
-		"client": {
-			generatedJarPath: "assembly/client/target/alluxio-assembly-client-%v-jar-with-dependencies.jar",
-			tarballJarPath:   "assembly/alluxio-client-%v.jar",
-			fileReplacements: map[string]map[string]string{
-				"libexec/alluxio-config.sh": {
-					fmt.Sprintf("assembly/client/target/alluxio-assembly-client-%v-jar-with-dependencies.jar", versionPlaceholder): fmt.Sprintf("assembly/alluxio-client-%v.jar", versionPlaceholder),
-				},
-			},
-		},
-		"server": {
-			generatedJarPath: "assembly/server/target/alluxio-assembly-server-%v-jar-with-dependencies.jar",
-			tarballJarPath:   "assembly/alluxio-server-%v.jar",
-			fileReplacements: map[string]map[string]string{
-				"libexec/alluxio-config.sh": {
-					fmt.Sprintf("assembly/server/target/alluxio-assembly-server-%v-jar-with-dependencies.jar", versionPlaceholder): fmt.Sprintf("assembly/alluxio-server-%v.jar", versionPlaceholder),
-				},
-			},
-		},
-		"fuseBundled": {
-			generatedJarPath: "dora/integration/fuse/target/alluxio-integration-fuse-%v-jar-with-dependencies.jar",
-			tarballJarPath:   "dora/integration/fuse/alluxio-fuse-%v.jar",
-			fileReplacements: map[string]map[string]string{
-				"dora/integration/fuse/bin/alluxio-fuse": {
-					fmt.Sprintf("target/alluxio-integration-fuse-%v-jar-with-dependencies.jar", versionPlaceholder): fmt.Sprintf("alluxio-fuse-%v.jar", versionPlaceholder),
-				},
-			},
-		},
-		"fuseStandalone": {
-			generatedJarPath: "dora/integration/fuse/target/alluxio-integration-fuse-%v-jar-with-dependencies.jar",
-			tarballJarPath:   "lib/alluxio-fuse-%v.jar",
-			fileReplacements: map[string]map[string]string{
-				"dora/integration/fuse/bin/alluxio-fuse": {
-					fmt.Sprintf("target/alluxio-integration-fuse-%v-jar-with-dependencies.jar", versionPlaceholder): fmt.Sprintf("../../../lib/alluxio-fuse-%v.jar", versionPlaceholder),
-				},
-			},
-		},
-	}
-)
-
 func collectTarballContents(opts *buildOpts, repoBuildDir, dstDir, alluxioVersion string) error {
 	for _, f := range opts.tarball.FileList {
 		if err := copyFileForTarball(filepath.Join(repoBuildDir, f), filepath.Join(dstDir, f)); err != nil {
@@ -87,18 +40,18 @@ func collectTarballContents(opts *buildOpts, repoBuildDir, dstDir, alluxioVersio
 
 	// add assembly jars, rename jars, and update name used in scripts
 	for _, n := range opts.tarball.AssemblyJars {
-		a, ok := assembledJars[n]
+		a, ok := opts.assemblyJars[n]
 		if !ok {
 			return stacktrace.NewError("no assembly jar named %v", n)
 		}
-		src := filepath.Join(repoBuildDir, fmt.Sprintf(a.generatedJarPath, alluxioVersion))
-		dst := filepath.Join(dstDir, fmt.Sprintf(a.tarballJarPath, alluxioVersion))
+		src := filepath.Join(repoBuildDir, strings.ReplaceAll(a.GeneratedJarPath, versionPlaceholder, alluxioVersion))
+		dst := filepath.Join(dstDir, strings.ReplaceAll(a.TarballJarPath, versionPlaceholder, alluxioVersion))
 		if err := copyFileForTarball(src, dst); err != nil {
 			return stacktrace.Propagate(err, "error copying file from %v to %v", src, dst)
 		}
 
 		// replace corresponding reference in scripts
-		for filePath, replacements := range a.fileReplacements {
+		for filePath, replacements := range a.FileReplacements {
 			replacementFile := filepath.Join(dstDir, filePath)
 			stat, err := os.Stat(replacementFile)
 			if err != nil {
@@ -109,7 +62,6 @@ func collectTarballContents(opts *buildOpts, repoBuildDir, dstDir, alluxioVersio
 				return stacktrace.Propagate(err, "error reading file at %v", replacementFile)
 			}
 			for find, replace := range replacements {
-				// ex. "alluxio-assembly-client-${VERSION}-jar-with-dependencies.jar" -> "alluxio-assembly-client-${VERSION}.jar
 				contents = bytes.ReplaceAll(contents, []byte(find), []byte(replace))
 			}
 			if err := ioutil.WriteFile(replacementFile, contents, stat.Mode()); err != nil {