diff --git a/bin/cli.sh b/bin/cli.sh new file mode 100755 index 000000000000..7791d87b303b --- /dev/null +++ b/bin/cli.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -eu + +BIN_DIR=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) +ROOT_PATH="${BIN_DIR}/.." +CLI_PATH="${BIN_DIR}/../cli/src/alluxio.org/cli/bin/alluxioCli-$(uname)-$(uname -m)" +"${ROOT_PATH}/build/cli/build-cli.sh" +"${CLI_PATH}" --rootPath="${ROOT_PATH}" "$@" diff --git a/build/cli/build-cli.sh b/build/cli/build-cli.sh new file mode 100755 index 000000000000..b0ba852c6622 --- /dev/null +++ b/build/cli/build-cli.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -eu + +CLI_DIR=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) + +# mapping of GOOS/GOARCH value to uname value +declare -A ALL_OS +ALL_OS["linux"]="Linux" +ALL_OS["darwin"]="Darwin" +declare -A ALL_ARCH +ALL_ARCH["amd64"]="x86_64" +ALL_ARCH["arm64"]="aarch64" + +MAIN_PATH="cli/main.go" +USAGE="Usage: build-cli.sh [-a] +-a Build executables for all OS and architecture combinations +" + +main() { + build_all="false" + while getopts "a" opt; do + case "${opt}" in + a) + build_all="true" + ;; + *) + echo -e "${USAGE}" >&2 + exit 1 + ;; + esac + done + + cliBinDir="${CLI_DIR}/../../cli/src/alluxio.org/cli/bin" + mkdir -p "${cliBinDir}" + + cd "${CLI_DIR}/../../cli/src/alluxio.org/" + + if [[ ${build_all} == "false" ]]; then + GO111MODULE=on go build -o "${cliBinDir}/alluxioCli-$(uname)-$(uname -m)" "${MAIN_PATH}" + else + for osKey in "${!ALL_OS[@]}"; do + for archKey in "${!ALL_ARCH[@]}"; do + echo "Building executable for ${osKey} ${archKey}" + GO111MODULE=on GOOS="${osKey}" GOARCH="${archKey}" go build -o "${cliBinDir}/alluxioCli-${ALL_OS[$osKey]}-${ALL_ARCH[$archKey]}" "${MAIN_PATH}" + done + done + fi +} + +main "$@" diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 000000000000..78c2354be1e6 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,58 @@ +# Alluxio CLI + +The Alluxio command line interface is the single entrypoint for users to: +- Initialize and configure the Alluxio cluster +- Start and stop processes +- Expose information about the running cluster +- Interact with the filesystem, running commands such as `ls` and `cp` +- Perform administrator level actions such as format or backup + +The CLI is invoked through the shell script at `bin/cli.sh`. +Commands follow the format of: +```console +bin/cli.sh [--[=]] [] +``` + +Add the `-h` flag to view more details regarding a service or operation. + +## Layout and naming conventions + +The choice of names for services, operations, and flags should be succinct: short and unambiguous. +Use of a single word is strongly preferred, but otherwise the name parts should be delimited by a dash such as `foo-bar`. + +For example, let's assume there is a `mark` operation as part of a `item` service that can be `set` or `unset` on an item `name`. +The recommended format for a command is +```console +bin/cli.sh item mark --set name +bin/cli.sh item mark --unset name +``` +where it is expected that either `--set` or `--unset` are specified. +This is preferred over the alternative of two separate commands with `setMark` and `unsetMark` as the operations. + +## User input + +### Flags and arguments +After selecting the desired command, additional user input can be parsed, as a mix of arguments and/or flags: +- Arguments: `bin/cli.sh command arg1 arg2 ...` +- Flags: `bin/cli.sh command --flag1 --flag2 val2 ...` + +Flags are strictly preferred over arguments. +- The flag name conveys context; an argument does not +- Arguments must be ordered; flags can be declared arbitrarily +- Flags can be designated as required to ensure user input. +- Repeated flags can be defined to capture an ordered list of inputs, ex. `--target target1 --target target2` + +### Input validation + +User inputs should be validated by the CLI command as much as possible as opposed to the resulting invocation. + +## Output conventions and java invocation + +A majority of commands result in invoking a java class with arguments to execute the expected operation and possibly return some output. +The output returned from the java invocation should tend towards being plain or machine parseable, such as a JSON formatted string, +rather than terminal friendly or human readable format. +When appropriate, the CLI command will default to formatting this output to be terminal friendly, with an option to output in a machine parseable format. + +## References + +Github CLI guidelines: https://primer.style/design/native/cli/foundations diff --git a/cli/src/alluxio.org/cli/bin/.gitignore b/cli/src/alluxio.org/cli/bin/.gitignore new file mode 100644 index 000000000000..33a525764e24 --- /dev/null +++ b/cli/src/alluxio.org/cli/bin/.gitignore @@ -0,0 +1 @@ +alluxioCli* diff --git a/cli/src/alluxio.org/cli/cmd/conf/conf.go b/cli/src/alluxio.org/cli/cmd/conf/conf.go new file mode 100644 index 000000000000..1c42c088d4e4 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/conf/conf.go @@ -0,0 +1,25 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package conf + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "conf", + Description: "Get, set, and validate configuration settings, primarily those defined in conf/alluxio-site.properties", + Commands: []env.Command{ + Get, + Log, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/conf/get.go b/cli/src/alluxio.org/cli/cmd/conf/get.go new file mode 100644 index 000000000000..674d092ac888 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/conf/get.go @@ -0,0 +1,90 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package conf + +import ( + "bytes" + "fmt" + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Get = &GetCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "get", + JavaClassName: "alluxio.cli.GetConf", + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioConfValidationEnabled, false), + }, +} + +type GetCommand struct { + *env.BaseJavaCommand + + ShowMaster bool + ShowSource bool + Unit string +} + +func (c *GetCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *GetCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [key]", Get.CommandName), + Short: "Look up a configuration value by its property key or print all configuration if no key is provided", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVar(&c.ShowMaster, "master", false, "Show configuration properties used by the master") + cmd.Flags().BoolVar(&c.ShowSource, "source", false, "Show source of the configuration property instead of the value") + cmd.Flags().StringVar(&c.Unit, "unit", "", + `Unit of the value to return, converted to correspond to the given unit. +E.g., with "--unit KB", a configuration value of "4096B" will return 4 +Possible options include B, KB, MB, GB, TP, PB, MS, S, M, H, D`) + return cmd +} + +func (c *GetCommand) Run(args []string) error { + var javaArgs []string + if c.ShowMaster { + javaArgs = append(javaArgs, "--master") + } + if c.ShowSource { + javaArgs = append(javaArgs, "--source") + } + if c.Unit != "" { + javaArgs = append(javaArgs, "--unit", c.Unit) + } + javaArgs = append(javaArgs, args...) + + return c.Base().Run(javaArgs) +} + +func (c *GetCommand) FetchValue(key string) (string, error) { + cmd := c.RunJavaClassCmd([]string{key}) + + errBuf := &bytes.Buffer{} + cmd.Stderr = errBuf + + log.Logger.Debugln(cmd.String()) + out, err := cmd.Output() + if err != nil { + return "", stacktrace.Propagate(err, "error getting conf for %v\nstderr: %v", key, errBuf.String()) + } + return string(out), nil +} diff --git a/cli/src/alluxio.org/cli/cmd/conf/log.go b/cli/src/alluxio.org/cli/cmd/conf/log.go new file mode 100644 index 000000000000..60ee82c52a2e --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/conf/log.go @@ -0,0 +1,69 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package conf + +import ( + "github.com/spf13/cobra" + "strings" + + "alluxio.org/cli/env" +) + +var Log = &LogCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "log", + JavaClassName: "alluxio.cli.LogLevel", + }, +} + +type LogCommand struct { + *env.BaseJavaCommand + + LogName string + Level string + Targets []string +} + +func (c *LogCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LogCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Log.CommandName, + Short: "Get or set the log level for the specified logger", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const name = "name" + cmd.Flags().StringVar(&c.LogName, name, "", "Logger name (ex. alluxio.master.file.DefaultFileSystemMaster)") + if err := cmd.MarkFlagRequired(name); err != nil { + panic(err) + } + cmd.Flags().StringVar(&c.Level, "level", "", "If specified, sets the specified logger at the given level") + cmd.Flags().StringSliceVar(&c.Targets, "target", nil, "A target name among . Defaults to master,workers,job_master,job_workers") + return cmd +} + +func (c *LogCommand) Run(_ []string) error { + javaArgs := []string{"--logName", c.LogName} + if c.Level != "" { + javaArgs = append(javaArgs, "--level", c.Level) + } + if len(c.Targets) > 0 { + javaArgs = append(javaArgs, "--target", strings.Join(c.Targets, ",")) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/cat.go b/cli/src/alluxio.org/cli/cmd/fs/cat.go new file mode 100644 index 000000000000..6b90c2917dbb --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/cat.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Cat = &CatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cat", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"cat"}, + }, +} + +type CatCommand struct { + *env.BaseJavaCommand +} + +func (c *CatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Cat.CommandName), + Short: "Print specified file's content", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *CatCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/checksum.go b/cli/src/alluxio.org/cli/cmd/fs/checksum.go new file mode 100644 index 000000000000..86b6e41c1f97 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/checksum.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Checksum = &ChecksumCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "checksum", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"checksum"}, + }, +} + +type ChecksumCommand struct { + *env.BaseJavaCommand +} + +func (c *ChecksumCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ChecksumCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Checksum.CommandName), + Short: "Calculates the md5 checksum of a specified file", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *ChecksumCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/count.go b/cli/src/alluxio.org/cli/cmd/fs/count.go new file mode 100644 index 000000000000..e61f43a53ed5 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/count.go @@ -0,0 +1,63 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Count = &CountCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "count", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"count"}, + }, +} + +type CountCommand struct { + *env.BaseJavaCommand + + isHumanReadable bool +} + +func (c *CountCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CountCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Count.CommandName), + Short: "Displays the number of files and directories matching the specified path", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + // special case to overwrite the -h shorthand for --help flag for --human-readable + cmd.PersistentFlags().BoolP("help", "", false, "help for this command") + + cmd.Flags().BoolVarP(&c.isHumanReadable, "human-readable", "h", false, "Print sizes in human readable format") + return cmd +} + +func (c *CountCommand) Run(args []string) error { + var javaArgs []string + if c.isHumanReadable { + javaArgs = append(javaArgs, "-h") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/cp.go b/cli/src/alluxio.org/cli/cmd/fs/cp.go new file mode 100644 index 000000000000..c7398a2e8b20 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/cp.go @@ -0,0 +1,118 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + "strconv" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Cp = &CopyCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cp", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + }, +} + +type CopyCommand struct { + *env.BaseJavaCommand + + bufferSize string + isRecursive bool + preserve bool + threads int +} + +func (c *CopyCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CopyCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [srcPath] [dstPath]", Cp.CommandName), + Short: "Copy a file or directory", + Long: `Copies a file or directory in the Alluxio filesystem or between local and Alluxio filesystems +Use the file:// schema to indicate a local filesystem path (ex. file:///absolute/path/to/file) and +use the recursive flag to copy directories`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.bufferSize, "buffer-size", "", "Read buffer size when coping to or from local, with defaults of 64MB and 8MB respectively") + cmd.Flags().BoolVarP(&c.isRecursive, "recursive", "R", false, "True to copy the directory subtree to the destination directory") + cmd.Flags().BoolVarP(&c.preserve, "preserve", "p", false, "Preserve file permission attributes when copying files; all ownership, permissions, and ACLs will be preserved") + cmd.Flags().IntVar(&c.threads, "thread", 0, "Number of threads used to copy files in parallel, defaults to 2 * CPU cores") + return cmd +} + +const ( + localScheme = "file://" +) + +func (c *CopyCommand) Run(args []string) error { + src, dst := args[0], args[1] + if strings.HasPrefix(src, localScheme) && strings.HasPrefix(dst, localScheme) { + return stacktrace.NewError("both src and dst paths are local file paths") + } + var cmd string + if strings.HasPrefix(src, localScheme) { + cmd = "copyFromLocal" + if c.preserve { + return stacktrace.NewError("cannot set preserve flag when copying from local file path") + } + if c.isRecursive { + return stacktrace.NewError("cannot set recursive flag when copying from local file path") + } + } else if strings.HasPrefix(dst, localScheme) { + cmd = "copyToLocal" + if c.preserve { + return stacktrace.NewError("cannot set preserve flag when copying to local file path") + } + if c.isRecursive { + return stacktrace.NewError("cannot set recursive flag when copying to local file path") + } + if c.threads != 0 { + return stacktrace.NewError("cannot set thread flag when copying to local file path") + } + } else { + cmd = "cp" + if c.bufferSize != "" { + return stacktrace.NewError("can only set buffer-size flag when copying to or from a local file path") + } + } + if c.threads < 0 { + return stacktrace.NewError("thread value must be positive but was %v", c.threads) + } + + javaArgs := []string{cmd} + if c.bufferSize != "" { + javaArgs = append(javaArgs, "--buffersize", c.bufferSize) + } + if c.isRecursive { + javaArgs = append(javaArgs, "--recursive") + } + if c.preserve { + javaArgs = append(javaArgs, "--preserve") + } + if c.threads != 0 { + javaArgs = append(javaArgs, "--thread", strconv.Itoa(c.threads)) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/du.go b/cli/src/alluxio.org/cli/cmd/fs/du.go new file mode 100644 index 000000000000..e39baac08245 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/du.go @@ -0,0 +1,78 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Du = &DuCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "du", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"du"}, + }, +} + +type DuCommand struct { + *env.BaseJavaCommand + + groupByWorker bool + isHumanReadable bool + showMemoryInfo bool + summarize bool +} + +func (c *DuCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *DuCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Du.CommandName), + Short: "Print specified file's content", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + // special case to overwrite the -h shorthand for --help flag for --human-readable + cmd.PersistentFlags().BoolP("help", "", false, "help for this command") + + cmd.Flags().BoolVarP(&c.groupByWorker, "group-worker", "g", false, "Display distribution of data store in Alluxio grouped by worker") + cmd.Flags().BoolVarP(&c.isHumanReadable, "human-readable", "h", false, "Print sizes in human readable format") + cmd.Flags().BoolVarP(&c.showMemoryInfo, "memory", "m", false, "Show in memory size and percentage") + cmd.Flags().BoolVarP(&c.summarize, "summary", "s", false, "Summarize listed files by aggregating their sizes") + return cmd +} + +func (c *DuCommand) Run(args []string) error { + var javaArgs []string + if c.groupByWorker { + javaArgs = append(javaArgs, "-g") + } + if c.isHumanReadable { + javaArgs = append(javaArgs, "-h") + } + if c.showMemoryInfo { + javaArgs = append(javaArgs, "-m") + } + if c.summarize { + javaArgs = append(javaArgs, "-s") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/fs.go b/cli/src/alluxio.org/cli/cmd/fs/fs.go new file mode 100644 index 000000000000..98f5a47b44aa --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/fs.go @@ -0,0 +1,38 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "fs", + Description: "Operations to interface with the Alluxio filesystem", + Commands: []env.Command{ + Cat, + Checksum, + Count, + Cp, + Du, + Head, + Location, + Ls, + Mkdir, + Mv, + Rm, + Stat, + Tail, + Test, + Touch, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/head.go b/cli/src/alluxio.org/cli/cmd/fs/head.go new file mode 100644 index 000000000000..09cebb173a27 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/head.go @@ -0,0 +1,60 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Head = &HeadCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "head", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"head"}, + }, +} + +type HeadCommand struct { + *env.BaseJavaCommand + + bytes string +} + +func (c *HeadCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *HeadCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Head.CommandName), + Short: "Print the leading bytes from the specified file", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.bytes, "bytes", "", "Byte size to print") + return cmd +} + +func (c *HeadCommand) Run(args []string) error { + var javaArgs []string + if c.bytes != "" { + javaArgs = append(javaArgs, "-c", c.bytes) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/location.go b/cli/src/alluxio.org/cli/cmd/fs/location.go new file mode 100644 index 000000000000..fffa0cd8353a --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/location.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Location = &LocationCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "location", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"location"}, + }, +} + +type LocationCommand struct { + *env.BaseJavaCommand +} + +func (c *LocationCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LocationCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Location.CommandName), + Short: "Displays the list of hosts storing the specified file", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *LocationCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/ls.go b/cli/src/alluxio.org/cli/cmd/fs/ls.go new file mode 100644 index 000000000000..ead9b25c138c --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/ls.go @@ -0,0 +1,138 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Ls = &LsCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "ls", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"ls"}, + }, +} + +type LsCommand struct { + *env.BaseJavaCommand + + listDirAsFile bool + forceLoadMetadata bool + isHumanReadable bool + omitMountInfo bool + pinnedFileOnly bool + isRecursive bool + isReverse bool + sortBy string + timestamp string +} + +func (c *LsCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +var ( + sortOptions = []string{ + "creationTime", + "inMemoryPercentage", + "lastAccessTime", + "lastModificationTime", + "name", + "path", + "size", + } + timestampOptions = []string{ + "createdTime", + "lastAccessTime", + "lastModifiedTime", + } +) + +func (c *LsCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Ls.CommandName), + Short: "Prints information for files and directories at the given path", + Long: `Displays information for all files and directories directly under the specified paths, including permission, owner, group, size (bytes for files or the number of children for directories), persistence state, last modified time, the percentage of content already in Alluxio, and the path`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + // special case to overwrite the -h shorthand for --help flag for --human-readable + cmd.PersistentFlags().BoolP("help", "", false, "help for this command") + + cmd.Flags().BoolVarP(&c.listDirAsFile, "list-dir-as-file", "d", false, "List directories as files") + cmd.Flags().BoolVarP(&c.forceLoadMetadata, "load-metadata", "f", false, "Force load metadata for immediate children in a directory") + cmd.Flags().BoolVarP(&c.isHumanReadable, "human-readable", "h", false, "Print sizes in human readable format") + cmd.Flags().BoolVarP(&c.omitMountInfo, "omit-mount-info", "m", false, "Omit mount point related information such as the UFS path") + cmd.Flags().BoolVarP(&c.pinnedFileOnly, "pinned-files", "p", false, "Only show pinned files") + cmd.Flags().BoolVarP(&c.isRecursive, "recursive", "R", false, "List subdirectories recursively") + cmd.Flags().BoolVarP(&c.isReverse, "reverse", "r", false, "Reverse sorted order") + cmd.Flags().StringVar(&c.sortBy, "sort", "", fmt.Sprintf("Sort entries by column, one of {%v}", strings.Join(sortOptions, "|"))) + cmd.Flags().StringVar(&c.timestamp, "timestamp", "", fmt.Sprintf("Display specified timestamp of entry, one of {%v}", strings.Join(timestampOptions, "|"))) + return cmd +} + +func (c *LsCommand) Run(args []string) error { + var javaArgs []string + if c.listDirAsFile { + javaArgs = append(javaArgs, "-d") + } + if c.forceLoadMetadata { + javaArgs = append(javaArgs, "-f") + } + if c.isHumanReadable { + javaArgs = append(javaArgs, "-h") + } + if c.omitMountInfo { + javaArgs = append(javaArgs, "-m") + } + if c.pinnedFileOnly { + javaArgs = append(javaArgs, "-p") + } + if c.isRecursive { + javaArgs = append(javaArgs, "-R") + } + if c.isReverse { + javaArgs = append(javaArgs, "-r") + } + if c.sortBy != "" { + if err := checkAllowed(c.sortBy, sortOptions...); err != nil { + return stacktrace.Propagate(err, "error validating sort options") + } + javaArgs = append(javaArgs, "--sort", c.sortBy) + } + if c.timestamp != "" { + if err := checkAllowed(c.timestamp, timestampOptions...); err != nil { + return stacktrace.Propagate(err, "error validating timestamp options") + } + javaArgs = append(javaArgs, "--timestamp", c.timestamp) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} + +func checkAllowed(val string, allowed ...string) error { + for _, i := range allowed { + if val == i { + return nil + } + } + return stacktrace.NewError("value %v must be one of %v", val, strings.Join(allowed, ",")) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/mkdir.go b/cli/src/alluxio.org/cli/cmd/fs/mkdir.go new file mode 100644 index 000000000000..194a1f3faee2 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/mkdir.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Mkdir = &MkdirCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "mkdir", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"mkdir"}, + }, +} + +type MkdirCommand struct { + *env.BaseJavaCommand +} + +func (c *MkdirCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *MkdirCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path1 path2 ...]", Mkdir.CommandName), + Short: "Create directories at the specified paths, creating the parent directory if not exists", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *MkdirCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/mv.go b/cli/src/alluxio.org/cli/cmd/fs/mv.go new file mode 100644 index 000000000000..1d32bb4c21f9 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/mv.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Mv = &MoveCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "mv", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"mv"}, + }, +} + +type MoveCommand struct { + *env.BaseJavaCommand +} + +func (c *MoveCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *MoveCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [srcPath] [dstPath]", Mv.CommandName), + Short: "Rename a file or directory", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *MoveCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/rm.go b/cli/src/alluxio.org/cli/cmd/fs/rm.go new file mode 100644 index 000000000000..87a7f783e7ee --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/rm.go @@ -0,0 +1,75 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Rm = &RmCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "rm", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"rm"}, + }, +} + +type RmCommand struct { + *env.BaseJavaCommand + + alluxioOnly bool + deleteMount bool + isRecursive bool + skipUfsCheck bool +} + +func (c *RmCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *RmCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Rm.CommandName), + Short: "Remove the specified file", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVar(&c.alluxioOnly, "alluxio-only", false, "True to only remove data and metadata from Alluxio cache") + cmd.Flags().BoolVarP(&c.deleteMount, "delete-mount", "m", false, "True to remove mount points within the specified directory subtree, which would otherwise cause failures") + cmd.Flags().BoolVarP(&c.isRecursive, "recursive", "R", false, "True to recursively remove files within the specified directory subtree") + cmd.Flags().BoolVarP(&c.skipUfsCheck, "skip-ufs-check", "U", false, "True to skip checking if corresponding UFS contents are in sync") + return cmd +} + +func (c *RmCommand) Run(args []string) error { + var javaArgs []string + if c.alluxioOnly { + javaArgs = append(javaArgs, "--alluxioOnly") + } + if c.deleteMount { + javaArgs = append(javaArgs, "--deleteMountPoint") + } + if c.isRecursive { + javaArgs = append(javaArgs, "-R") + } + if c.skipUfsCheck { + javaArgs = append(javaArgs, "-U") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/stat.go b/cli/src/alluxio.org/cli/cmd/fs/stat.go new file mode 100644 index 000000000000..f0173413556b --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/stat.go @@ -0,0 +1,82 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Stat = &StatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "stat", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"stat"}, + }, +} + +type StatCommand struct { + *env.BaseJavaCommand + + Path string + FileId string + Format string +} + +func (c *StatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *StatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Stat.CommandName, + Short: "Displays info for the specified file or directory", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + const path, fileId = "path", "file-id" + cmd.Flags().StringVar(&c.Path, path, "", "Path to file or directory") + cmd.Flags().StringVar(&c.FileId, fileId, "", "File id of file") + cmd.Flags().StringVarP(&c.Format, "format", "f", "", `Display info in the given format: + "%N": name of the file + "%z": size of file in bytes + "%u": owner + "%g": group name of owner + "%i": file id of the file + "%y": modification time in UTC in 'yyyy-MM-dd HH:mm:ss' format + "%Y": modification time as Unix timestamp in milliseconds + "%b": Number of blocks allocated for file +`) + cmd.MarkFlagsMutuallyExclusive(path, fileId) + return cmd +} + +func (c *StatCommand) Run(_ []string) error { + if c.Path == "" && c.FileId == "" { + return stacktrace.NewError("must set one of --path or --file-id flags") + } + + var javaArgs []string + if c.Format != "" { + javaArgs = append(javaArgs, "-f", c.Format) + } + if c.Path != "" { + javaArgs = append(javaArgs, c.Path) + } else if c.FileId != "" { + javaArgs = append(javaArgs, "--file-id", c.FileId) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/tail.go b/cli/src/alluxio.org/cli/cmd/fs/tail.go new file mode 100644 index 000000000000..3ae50ca42b71 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/tail.go @@ -0,0 +1,60 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Tail = &TailCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "tail", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"tail"}, + }, +} + +type TailCommand struct { + *env.BaseJavaCommand + + bytes string +} + +func (c *TailCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TailCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Tail.CommandName), + Short: "Print the trailing bytes from the specified file", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.bytes, "bytes", "", "Byte size to print") + return cmd +} + +func (c *TailCommand) Run(args []string) error { + var javaArgs []string + if c.bytes != "" { + javaArgs = append(javaArgs, "-c", c.bytes) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/test.go b/cli/src/alluxio.org/cli/cmd/fs/test.go new file mode 100644 index 000000000000..7d1a27de57a9 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/test.go @@ -0,0 +1,80 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Test = &TestCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "test", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"test"}, + }, +} + +type TestCommand struct { + *env.BaseJavaCommand + + isDirectory bool + isExists bool + isFile bool + isNotEmpty bool + isZeroLength bool +} + +func (c *TestCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Test.CommandName), + Short: "Test a property of a path, returning 0 if the property is true, or 1 otherwise", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVarP(&c.isDirectory, "dir", "d", false, "Test if path is a directory") + cmd.Flags().BoolVarP(&c.isExists, "exists", "e", false, "Test if path exists") + cmd.Flags().BoolVarP(&c.isFile, "file", "f", false, "Test if path is a file") + cmd.Flags().BoolVarP(&c.isFile, "not-empty", "s", false, "Test if path is not empty") + cmd.Flags().BoolVarP(&c.isFile, "zero", "z", false, "Test if path is zero length") + return cmd +} + +func (c *TestCommand) Run(args []string) error { + var javaArgs []string + if c.isDirectory { + javaArgs = append(javaArgs, "-d") + } + if c.isExists { + javaArgs = append(javaArgs, "-e") + } + if c.isFile { + javaArgs = append(javaArgs, "-f") + } + if c.isNotEmpty { + javaArgs = append(javaArgs, "-s") + } + if c.isZeroLength { + javaArgs = append(javaArgs, "-z") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/touch.go b/cli/src/alluxio.org/cli/cmd/fs/touch.go new file mode 100644 index 000000000000..fb9c5229abe0 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/touch.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Touch = &TouchCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "touch", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"touch"}, + }, +} + +type TouchCommand struct { + *env.BaseJavaCommand +} + +func (c *TouchCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TouchCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [path]", Touch.CommandName), + Short: "Create a 0 byte file at the specified path, which will also be created in the under file system", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *TouchCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/cache.go b/cli/src/alluxio.org/cli/cmd/info/cache.go new file mode 100644 index 000000000000..ddf36010b83f --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/cache.go @@ -0,0 +1,70 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "strings" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Cache = &CacheCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cache", + JavaClassName: "alluxio.cli.fsadmin.FileSystemAdminShell", + Parameters: []string{"report", "capacity"}, + }, +} + +type CacheCommand struct { + *env.BaseJavaCommand + + liveWorkers bool + lostWorkers bool + workersList []string // list of worker hostnames or ip addresses +} + +func (c *CacheCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CacheCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Cache.CommandName, + Short: "Reports worker capacity information", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + const live, lost, worker = "live", "lost", "worker" + cmd.Flags().BoolVar(&c.liveWorkers, live, false, "Only show live workers for capacity report") + cmd.Flags().BoolVar(&c.lostWorkers, lost, false, "Only show lost workers for capacity report") + cmd.Flags().StringSliceVar(&c.workersList, worker, nil, "Only show specified workers for capacity report, labeled by hostname or IP address") + cmd.MarkFlagsMutuallyExclusive(live, lost, worker) + return cmd +} + +func (c *CacheCommand) Run(_ []string) error { + // TODO: output all in a serializable format and filter/trim as specified by flags + var args []string + if c.liveWorkers { + args = append(args, "-live") + } else if c.lostWorkers { + args = append(args, "-lost") + } else if len(c.workersList) > 0 { + args = append(args, "-workers", strings.Join(c.workersList, ",")) + } + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/collect.go b/cli/src/alluxio.org/cli/cmd/info/collect.go new file mode 100644 index 000000000000..b339bab336a6 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/collect.go @@ -0,0 +1,153 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Collect = &CollectCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "collect", + JavaClassName: "alluxio.cli.bundler.CollectInfo", + }, +} + +type CollectCommand struct { + *env.BaseJavaCommand + + additionalLogs []string + endTime string + excludeLogs []string + excludeWorkerMetrics bool + includeLogs []string + local bool + maxThreads int + outputPath string + startTime string +} + +func (c *CollectCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +const dateFormat = "2006-01-02T15:04:05" + +func (c *CollectCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [command]", Collect.CommandName), + Short: "Collects information such as logs, config, metrics, and more from the running Alluxio cluster and bundle into a single tarball", + Long: `Collects information such as logs, config, metrics, and more from the running Alluxio cluster and bundle into a single tarball +[command] must be one of the following values: + all runs all the commands below + cluster: runs a set of Alluxio commands to collect information about the Alluxio cluster + conf: collects the configuration files under ${ALLUXIO_HOME}/config/ + env: runs a set of linux commands to collect information about the cluster + jvm: collects jstack from the JVMs + log: collects the log files under ${ALLUXIO_HOME}/logs/ + metrics: collects Alluxio system metrics + +WARNING: This command MAY bundle credentials. To understand the risks refer to the docs here. +https://docs.alluxio.io/os/user/edge/en/operation/Troubleshooting.html#collect-alluxio-cluster-information +`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringSliceVar(&c.additionalLogs, "additional-logs", nil, "Additional file name prefixes from ${ALLUXIO_HOME}/logs to include in the tarball, inclusive of the default log files") + cmd.Flags().StringVar(&c.endTime, "end-time", "", "Logs that do not contain entries before this time will be ignored, format must be like "+dateFormat) + cmd.Flags().StringSliceVar(&c.excludeLogs, "exclude-logs", nil, "File name prefixes from ${ALLUXIO_HOME}/logs to exclude; this is evaluated after adding files from --additional-logs") + cmd.Flags().BoolVar(&c.excludeWorkerMetrics, "exclude-worker-metrics", false, "True to skip worker metrics collection") + cmd.Flags().StringSliceVar(&c.includeLogs, "include-logs", nil, "File name prefixes from ${ALLUXIO_HOME}/logs to include in the tarball, ignoring the default log files; cannot be used with --exclude-logs or --additional-logs") + cmd.Flags().BoolVar(&c.local, "local", false, "True to only collect information from the local machine") + cmd.Flags().IntVar(&c.maxThreads, "max-threads", 1, "Parallelism of the command; use a smaller value to limit network I/O when transferring tarballs") + const outputPath = "output-path" + cmd.Flags().StringVar(&c.outputPath, outputPath, "", "Output directory to write collect info tarball to") + if err := cmd.MarkFlagRequired(outputPath); err != nil { + panic(err) + } + cmd.Flags().StringVar(&c.startTime, "start-time", "", "Logs that do not contain entries after this time will be ignored, format must be like "+dateFormat) + return cmd +} + +func (c *CollectCommand) Run(args []string) error { + // TODO: use flags instead of arguments to parse user input + commands := map[string]string{ + "all": "all", + "cluster": "collectAlluxioInfo", + "conf": "collectConfig", + "env": "collectEnv", + "jvm": "collectJvmInfo", + "log": "collectLog", + "metrics": "collectMetrics", + } + commandArg, ok := commands[args[0]] + if !ok { + var cmds []string + for c := range commands { + cmds = append(cmds, c) + } + return stacktrace.NewError("first argument must be one of %v", strings.Join(cmds, ", ")) + } + + var javaArgs []string + if c.additionalLogs != nil { + if c.includeLogs != nil { + return stacktrace.NewError("cannot set both --include-logs and --additional-logs") + } + javaArgs = append(javaArgs, "--additional-logs", strings.Join(c.additionalLogs, ",")) + } + if c.endTime != "" { + if _, err := time.Parse(dateFormat, c.endTime); err != nil { + return stacktrace.Propagate(err, "could not parse end time %v", c.endTime) + } + javaArgs = append(javaArgs, "--end-time", c.endTime) + } + if c.excludeLogs != nil { + if c.includeLogs != nil { + return stacktrace.NewError("cannot set both --include-logs and --exclude-logs") + } + javaArgs = append(javaArgs, "--exclude-logs", strings.Join(c.excludeLogs, ",")) + } + if c.excludeWorkerMetrics { + javaArgs = append(javaArgs, "--exclude-worker-metrics") + } + if c.includeLogs != nil { + // already checked exclusivity with --additional-logs and --exclude-logs + javaArgs = append(javaArgs, "--include-logs", strings.Join(c.includeLogs, ",")) + } + if c.local { + javaArgs = append(javaArgs, "--local") + } + if c.maxThreads > 1 { + javaArgs = append(javaArgs, "--max-threads", strconv.Itoa(c.maxThreads)) + } + if c.startTime != "" { + if _, err := time.Parse(dateFormat, c.startTime); err != nil { + return stacktrace.Propagate(err, "could not parse start time %v", c.startTime) + } + javaArgs = append(javaArgs, "--start-time", c.startTime) + } + + javaArgs = append(javaArgs, commandArg, c.outputPath) + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/info.go b/cli/src/alluxio.org/cli/cmd/info/info.go new file mode 100644 index 000000000000..39ea06d913ad --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/info.go @@ -0,0 +1,27 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "info", + Description: "Retrieve and/or display info about the running Alluxio cluster", + Commands: []env.Command{ + Cache, + Collect, + Master, + Report, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/info/master.go b/cli/src/alluxio.org/cli/cmd/info/master.go new file mode 100644 index 000000000000..2c371f066723 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/master.go @@ -0,0 +1,50 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Master = &MasterCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "master", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"masterInfo"}, + }, +} + +type MasterCommand struct { + *env.BaseJavaCommand +} + +func (c *MasterCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *MasterCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Master.CommandName, + Short: "Prints information regarding master fault tolerance such as leader address and list of master addresses", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + return cmd +} + +func (c *MasterCommand) Run(_ []string) error { + // TODO: output in a serializable format + return c.Base().Run(nil) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/report.go b/cli/src/alluxio.org/cli/cmd/info/report.go new file mode 100644 index 000000000000..8de4c163f716 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/report.go @@ -0,0 +1,81 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Report = &ReportCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "report", + JavaClassName: "alluxio.cli.fsadmin.FileSystemAdminShell", + Parameters: []string{"report"}, + }, +} + +type ReportCommand struct { + *env.BaseJavaCommand +} + +func (c *ReportCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ReportCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [arg]", Report.CommandName), + Short: "Reports Alluxio running cluster information", + Long: `Reports Alluxio running cluster information +[arg] can be one of the following values: + jobservice: job service metrics information + metrics: metrics information + summary: cluster summary + ufs: under storage system information + +Defaults to summary if no arg is provided +`, + Args: cobra.RangeArgs(0, 1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *ReportCommand) Run(args []string) error { + reportArg := "summary" + if len(args) == 1 { + options := map[string]struct{}{ + "jobservice": {}, + "metrics": {}, + "summary": {}, + "ufs": {}, + } + if _, ok := options[args[0]]; !ok { + var cmds []string + for c := range options { + cmds = append(cmds, c) + } + return stacktrace.NewError("first argument must be one of %v", strings.Join(cmds, ", ")) + } + reportArg = args[0] + } + // TODO: output all in a serializable format and filter/trim as specified by flags + return c.Base().Run([]string{reportArg}) +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/backup.go b/cli/src/alluxio.org/cli/cmd/journal/backup.go new file mode 100644 index 000000000000..fd69357651c9 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/backup.go @@ -0,0 +1,70 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Backup = &BackupCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "backup", + JavaClassName: "alluxio.cli.fsadmin.FileSystemAdminShell", + Parameters: []string{"backup"}, + }, +} + +type BackupCommand struct { + *env.BaseJavaCommand + + backupPath string + local bool + allowLeader bool + bypassDelegation bool +} + +func (c *BackupCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *BackupCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Backup.CommandName, + Short: "Backup the local Alluxio master journal", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + cmd.Flags().StringVar(&c.backupPath, "path", "", "Directory to write backup file to") + cmd.MarkFlagRequired("path") + cmd.Flags().BoolVar(&c.local, "local", false, "If true, writes the backup file to the local filesystem instead of root UFS") + cmd.Flags().BoolVar(&c.allowLeader, "allow-leader", false, "If true, allows the leader to take the backup when there are no standby masters") + cmd.Flags().BoolVar(&c.bypassDelegation, "bypass-delegation", false, "If true, takes the backup on the leader even if standby masters are available") + return cmd +} + +func (c *BackupCommand) Run(_ []string) error { + javaArgs := []string{c.backupPath} + if c.local { + javaArgs = append(javaArgs, "--local") + } + if c.allowLeader { + javaArgs = append(javaArgs, "--allow-leader") + } + if c.bypassDelegation { + javaArgs = append(javaArgs, "--bypass-delegation") + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/checkpoint.go b/cli/src/alluxio.org/cli/cmd/journal/checkpoint.go new file mode 100644 index 000000000000..c0037bac6c31 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/checkpoint.go @@ -0,0 +1,46 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Checkpoint = &CheckpointCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "checkpoint", + JavaClassName: "alluxio.cli.fsadmin.FileSystemAdminShell", + Parameters: []string{"journal", "checkpoint"}, + }, +} + +type CheckpointCommand struct { + *env.BaseJavaCommand +} + +func (c *CheckpointCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CheckpointCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Checkpoint.CommandName, + Short: "Create a checkpoint in the primary master journal system", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + return cmd +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/format.go b/cli/src/alluxio.org/cli/cmd/journal/format.go new file mode 100644 index 000000000000..47b508e22053 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/format.go @@ -0,0 +1,69 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "bytes" + "fmt" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Format = &FormatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "format", + JavaClassName: "alluxio.cli.Format", + Parameters: []string{"master"}, + UseServerClasspath: true, + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type FormatCommand struct { + *env.BaseJavaCommand +} + +func (c *FormatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *FormatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Format.CommandName, + Short: "Format the local Alluxio master journal", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *FormatCommand) Run(_ []string) error { + return c.Base().Run(nil) +} + +func (c *FormatCommand) Format() error { + cmd := c.RunJavaClassCmd(nil) + errBuf := &bytes.Buffer{} + cmd.Stderr = errBuf + + log.Logger.Debugln(cmd.String()) + if err := cmd.Run(); err != nil { + return stacktrace.Propagate(err, "error running %v\nstderr: %v", c.CommandName, errBuf.String()) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/journal.go b/cli/src/alluxio.org/cli/cmd/journal/journal.go new file mode 100644 index 000000000000..789bcf5a38aa --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/journal.go @@ -0,0 +1,27 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "journal", + Description: "Format, backup, and other journal related operations", + Commands: []env.Command{ + Backup, + Checkpoint, + Format, + Read, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/read.go b/cli/src/alluxio.org/cli/cmd/journal/read.go new file mode 100644 index 000000000000..69c29fd02c0a --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/read.go @@ -0,0 +1,88 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "fmt" + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Read = &ReadCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "read", + JavaClassName: "alluxio.master.journal.tool.JournalTool", + UseServerClasspath: true, + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type ReadCommand struct { + *env.BaseJavaCommand + + end int + inputDir string + master string + outputDir string + start int +} + +func (c *ReadCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ReadCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Read.CommandName, + Short: "Read an Alluxio journal file to a human-readable version", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + cmd.Flags().IntVar(&c.end, "end", -1, "end log sequence number (exclusive)") + cmd.Flags().StringVar(&c.inputDir, "input-dir", "", "input directory on-disk to read the journal content from") + cmd.Flags().StringVar(&c.master, "master", "FileSystemMaster", "name of the master class") + cmd.Flags().StringVar(&c.outputDir, "output-dir", "", "output directory to write journal content to") + cmd.Flags().IntVar(&c.start, "start", 0, "start log sequence number (inclusive)") + return cmd +} + +func (c *ReadCommand) Run(_ []string) error { + var javaArgs []string + if c.start < 0 { + return stacktrace.NewError("start log sequence number must be non-negative but was %v", c.start) + } + if c.end < -1 { + return stacktrace.NewError("end log sequence number must be non-negative but was %v", c.end) + } + if c.end != -1 { + javaArgs = append(javaArgs, "-end", strconv.Itoa(c.end)) + } + if c.inputDir != "" { + javaArgs = append(javaArgs, "-inputDir", c.inputDir) + } + if c.master != "FileSystemMaster" { + javaArgs = append(javaArgs, "-master", c.master) + } + if c.outputDir != "" { + javaArgs = append(javaArgs, "-outputDir", c.outputDir) + } + if c.start > 0 { + javaArgs = append(javaArgs, "-start", strconv.Itoa(c.start)) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/process/process.go b/cli/src/alluxio.org/cli/cmd/process/process.go new file mode 100644 index 000000000000..65983a2714e6 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/process/process.go @@ -0,0 +1,25 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package process + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "process", + Description: "Start, stop, and other operations related to the cluster processes", + Commands: []env.Command{ + &env.StartProcessCommand{}, + &env.StopProcessCommand{}, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/quorum/elect.go b/cli/src/alluxio.org/cli/cmd/quorum/elect.go new file mode 100644 index 000000000000..001e2b87d3f4 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/quorum/elect.go @@ -0,0 +1,62 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package quorum + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Elect = &ElectCommand{ + QuorumCommand: &QuorumCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "elect", + JavaClassName: "alluxio.cli.fsadmin.FileSystemAdminShell", + Parameters: []string{"journal", "quorum", "elect"}, + }, + allowedDomains: []string{domainMaster}, + }, +} + +type ElectCommand struct { + *QuorumCommand + + serverAddress string +} + +func (c *ElectCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Elect.CommandName, + Short: "Transfers leadership of the quorum to the specified server", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + + const address = "address" + cmd.Flags().StringVar(&c.serverAddress, address, "", "Address of new leader server in the format :") + if err := cmd.MarkFlagRequired(address); err != nil { + panic(err) + } + return cmd +} + +func (c *ElectCommand) Run(_ []string) error { + if err := checkDomain(c.QuorumCommand.domain, c.QuorumCommand.allowedDomains...); err != nil { + return stacktrace.Propagate(err, "error checking domain %v", c.QuorumCommand.domain) + } + // TODO: ignore the domain flag for now since elect command can only operate on MASTER + // java command needs to be updated such that it can accept the domain flag + return c.Base().Run([]string{"-address", c.serverAddress}) +} diff --git a/cli/src/alluxio.org/cli/cmd/quorum/info.go b/cli/src/alluxio.org/cli/cmd/quorum/info.go new file mode 100644 index 000000000000..7c0c71091f01 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/quorum/info.go @@ -0,0 +1,48 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package quorum + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Info = &InfoCommand{ + QuorumCommand: &QuorumCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "info", + JavaClassName: "alluxio.cli.fsadmin.FileSystemAdminShell", + Parameters: []string{"journal", "quorum", "info"}, + }, + allowedDomains: []string{domainJobMaster, domainMaster}, + }, +} + +type InfoCommand struct { + *QuorumCommand +} + +func (c *InfoCommand) ToCommand() *cobra.Command { + cmd := c.QuorumCommand.InitQuorumCmd(c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Info.CommandName, + Short: "Shows quorum information", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + })) + return cmd +} + +func (c *InfoCommand) Run(_ []string) error { + return c.QuorumCommand.Run(nil) +} diff --git a/cli/src/alluxio.org/cli/cmd/quorum/quorum.go b/cli/src/alluxio.org/cli/cmd/quorum/quorum.go new file mode 100644 index 000000000000..9593642a5eb4 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/quorum/quorum.go @@ -0,0 +1,78 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package quorum + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var ( + Service = &env.Service{ + Name: "quorum", + Description: "Manage the high availability server quorum, such as electing a new leader", + Commands: []env.Command{ + Elect, + Info, + Remove, + }, + } +) + +const ( + domainJobMaster = "JOB_MASTER" + domainMaster = "MASTER" +) + +type QuorumCommand struct { + *env.BaseJavaCommand + + allowedDomains []string + domain string +} + +func (c *QuorumCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *QuorumCommand) InitQuorumCmd(cmd *cobra.Command) *cobra.Command { + const domain = "domain" + cmd.Flags().StringVar(&c.domain, domain, "", fmt.Sprintf("Quorum domain to operate on, must be one of %v", strings.Join(c.allowedDomains, ", "))) + if err := cmd.MarkFlagRequired(domain); err != nil { + panic(err) + } + return cmd +} + +func (c *QuorumCommand) Run(args []string) error { + if err := checkDomain(c.domain, c.allowedDomains...); err != nil { + return stacktrace.Propagate(err, "error checking domain %v", c.domain) + } + var javaArgs []string + javaArgs = append(javaArgs, args...) + javaArgs = append(javaArgs, "-domain", c.domain) + return c.Base().Run(javaArgs) +} + +func checkDomain(domain string, allowedDomains ...string) error { + for _, d := range allowedDomains { + if domain == d { + return nil + } + } + return stacktrace.NewError("domain is %v but must be one of %v", domain, strings.Join(allowedDomains, ",")) +} diff --git a/cli/src/alluxio.org/cli/cmd/quorum/remove.go b/cli/src/alluxio.org/cli/cmd/quorum/remove.go new file mode 100644 index 000000000000..76b84f6addf8 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/quorum/remove.go @@ -0,0 +1,55 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package quorum + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Remove = &RemoveCommand{ + QuorumCommand: &QuorumCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "remove", + JavaClassName: "alluxio.cli.fsadmin.FileSystemAdminShell", + Parameters: []string{"journal", "quorum", "remove"}, + }, + allowedDomains: []string{domainJobMaster, domainMaster}, + }, +} + +type RemoveCommand struct { + *QuorumCommand + + serverAddress string +} + +func (c *RemoveCommand) ToCommand() *cobra.Command { + cmd := c.QuorumCommand.InitQuorumCmd(c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Remove.CommandName, + Short: "Removes the specified server from the quorum", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + })) + const address = "address" + cmd.Flags().StringVar(&c.serverAddress, address, "", "Address of server in the format :") + if err := cmd.MarkFlagRequired(address); err != nil { + panic(err) + } + return cmd +} + +func (c *RemoveCommand) Run(_ []string) error { + return c.QuorumCommand.Run([]string{"-address", c.serverAddress}) +} diff --git a/cli/src/alluxio.org/cli/env/command.go b/cli/src/alluxio.org/cli/env/command.go new file mode 100644 index 000000000000..f69be504eb14 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/command.go @@ -0,0 +1,103 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/log" +) + +type Command interface { + ToCommand() *cobra.Command +} + +type BaseJavaCommand struct { + CommandName string + JavaClassName string + Parameters []string + + DebugMode bool + + UseServerClasspath bool // defaults to ALLUXIO_CLIENT_CLASSPATH, use ALLUXIO_SERVER_CLASSPATH if true + InlineJavaOpts []string // java opts provided by the user as part of the inline command + ShellJavaOpts string // default java opts encoded as part of the specific command +} + +func (c *BaseJavaCommand) InitRunJavaClassCmd(cmd *cobra.Command) *cobra.Command { + cmd.Flags().BoolVar(&c.DebugMode, "attach-debug", false, fmt.Sprintf("True to attach debug opts specified by $%v", ConfAlluxioUserAttachOpts.EnvVar)) + cmd.Flags().StringSliceVarP(&c.InlineJavaOpts, "java-opts", "D", nil, `Alluxio properties to apply, ex. -Dkey=value`) + return cmd +} + +// RunJavaClassCmd constructs a java command with a predetermined order of variable opts +// ${JAVA} ${ALLUXIO_USER_ATTACH_OPTS} -cp ${ALLUXIO_CLIENT_CLASSPATH} ${ALLUXIO_USER_JAVA_OPTS} \ +// {command default opts} {user inline opts} {command java class} {command parameter} {user inline args} +// where: +// - ${ALLUXIO_USER_*} are environment variables set by the user in alluxio-env.sh +// - {command *} are encoded as part of the command's definition +// - {user inline *} are specified by the user when entering the command +func (c *BaseJavaCommand) RunJavaClassCmd(args []string) *exec.Cmd { + var cmdArgs []string + if c.DebugMode { + if opts := Env.EnvVar.GetString(ConfAlluxioUserAttachOpts.EnvVar); opts != "" { + cmdArgs = append(cmdArgs, strings.Split(opts, " ")...) + } + } + classpath := EnvAlluxioClientClasspath + if c.UseServerClasspath { + classpath = EnvAlluxioServerClasspath + } + cmdArgs = append(cmdArgs, "-cp", Env.EnvVar.GetString(classpath)) + + if opts := Env.EnvVar.GetString(ConfAlluxioUserJavaOpts.EnvVar); opts != "" { + cmdArgs = append(cmdArgs, strings.Split(opts, " ")...) + } + if opts := strings.TrimSpace(c.ShellJavaOpts); opts != "" { + cmdArgs = append(cmdArgs, strings.Split(opts, " ")...) + } + for _, o := range c.InlineJavaOpts { + if opts := strings.TrimSpace(o); opts != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("-D%v", opts)) + } + } + cmdArgs = append(cmdArgs, c.JavaClassName) + if len(c.Parameters) > 0 { + cmdArgs = append(cmdArgs, c.Parameters...) + } + cmdArgs = append(cmdArgs, args...) + + ret := exec.Command(Env.EnvVar.GetString(ConfJava.EnvVar), cmdArgs...) + for _, k := range Env.EnvVar.AllKeys() { + ret.Env = append(ret.Env, fmt.Sprintf("%s=%v", k, Env.EnvVar.Get(k))) + } + return ret +} + +func (c *BaseJavaCommand) Run(args []string) error { + cmd := c.RunJavaClassCmd(args) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + log.Logger.Debugln(cmd.String()) + if err := cmd.Run(); err != nil { + return stacktrace.Propagate(err, "error running %v", c.CommandName) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/env/env.go b/cli/src/alluxio.org/cli/env/env.go new file mode 100644 index 000000000000..26693c0d73cb --- /dev/null +++ b/cli/src/alluxio.org/cli/env/env.go @@ -0,0 +1,258 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/palantir/stacktrace" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + + "alluxio.org/log" +) + +const ( + requiredJavaVersion = 11 + + userLoggerType = "USER_LOGGER" +) + +// Env is a global instance of the Alluxio environment +// It is assumed that InitAlluxioEnv() is run before access +// The package level variable exists to allow access by cobra.Command.RunE functions +var Env *AlluxioEnv + +type AlluxioEnv struct { + RootPath string + Version string + EnvVar *viper.Viper +} + +func InitAlluxioEnv(rootPath string) error { + /* + Note the precedence order of viper, where earlier is retrieved first: + - Set, env, config, default (https://pkg.go.dev/github.com/dvln/viper#section-readme) + Since env > config, explicitly set any values read from config to take precedence over env + - Use single instance of viper to set values and retrieve values within go logic + - Use local viper instances to read configuration files to avoid confusion in precedence order + This effectively negates the possibility of retrieving config values to simplify the precedence order to be + - Set, env, default + */ + envVar := viper.New() + envVar.AutomaticEnv() // override defaults with existing environment variable values + + // read VERSION from version.sh + var ver string + { + v := viper.New() + v.AddConfigPath(filepath.Join(rootPath, "libexec")) + v.SetConfigName("version.sh") + v.SetConfigType("env") + if err := v.ReadInConfig(); err != nil { + return stacktrace.Propagate(err, "error reading version config") + } + ver = v.GetString(envVersion) + envVar.Set(envVersion, ver) + } + + // set default values + for k, v := range map[string]string{ + ConfAlluxioHome.EnvVar: rootPath, + ConfAlluxioConfDir.EnvVar: filepath.Join(rootPath, "conf"), + ConfAlluxioLogsDir.EnvVar: filepath.Join(rootPath, "logs"), + confAlluxioUserLogsDir.EnvVar: filepath.Join(rootPath, "logs", "user"), + // TODO: add a go build flag to switch this to the finalized tarball path when building the tarball, ex. filepath.Join(rootPath, "assembly", fmt.Sprintf("alluxio-assembly-client-%v.jar", ver)) + envAlluxioAssemblyClientJar: filepath.Join(rootPath, "assembly", "client", "target", fmt.Sprintf("alluxio-assembly-client-%v-jar-with-dependencies.jar", ver)), + envAlluxioAssemblyServerJar: filepath.Join(rootPath, "assembly", "server", "target", fmt.Sprintf("alluxio-assembly-server-%v-jar-with-dependencies.jar", ver)), + } { + envVar.SetDefault(k, v) + } + + // set user-specified environment variable values from alluxio-env.sh + { + // use ALLUXIO_CONF_DIR if set externally, otherwise default to above value + confPath := envVar.GetString(ConfAlluxioConfDir.EnvVar) + const alluxioEnv = "alluxio-env.sh" + log.Logger.Debugf("Loading %v configuration from directory %v", alluxioEnv, confPath) + v := viper.New() + if _, err := os.Stat(filepath.Join(confPath, alluxioEnv)); err == nil { + v.AddConfigPath(confPath) + v.SetConfigName(alluxioEnv) + v.SetConfigType("env") + v.AllowEmptyEnv(true) + if err := v.ReadInConfig(); err != nil { + return stacktrace.Propagate(err, "error reading alluxio-env config") + } + // for each entry found in alluxio-env.sh, explicitly set to global viper + for _, k := range v.AllKeys() { + key, val := strings.ToUpper(k), v.Get(k) + log.Logger.Debugf("Setting user provided %v=%v", key, val) + envVar.Set(key, val) + } + } + } + + // set classpath variables which are dependent on user configurable values + envVar.Set(EnvAlluxioClientClasspath, strings.Join([]string{ + envVar.GetString(ConfAlluxioConfDir.EnvVar) + "/", + envVar.GetString(confAlluxioClasspath.EnvVar), + envVar.GetString(envAlluxioAssemblyClientJar), + filepath.Join(envVar.GetString(ConfAlluxioHome.EnvVar), "lib", fmt.Sprintf("alluxio-integration-tools-validation-%v.jar", ver)), + }, ":")) + envVar.Set(EnvAlluxioServerClasspath, strings.Join([]string{ + envVar.GetString(ConfAlluxioConfDir.EnvVar) + "/", + envVar.GetString(confAlluxioClasspath.EnvVar), + envVar.GetString(envAlluxioAssemblyServerJar), + }, ":")) + + // check java executable and version + if err := checkAndSetJava(envVar); err != nil { + return stacktrace.Propagate(err, "error finding installed java") + } + if err := checkJavaVersion(envVar.GetString(ConfJava.EnvVar), requiredJavaVersion); err != nil { + return stacktrace.Propagate(err, "error checking java version compatibility") + } + + // append default opts to ALLUXIO_JAVA_OPTS + alluxioJavaOpts := envVar.GetString(ConfAlluxioJavaOpts.EnvVar) + if alluxioJavaOpts != "" { + // warn about setting configuration through java opts that should be set through environment variables instead + for _, c := range []*AlluxioConfigEnvVar{ + ConfAlluxioConfDir, + ConfAlluxioLogsDir, + confAlluxioUserLogsDir, + } { + if strings.Contains(alluxioJavaOpts, c.configKey) { + log.Logger.Warnf("Setting %v through %v will be ignored. Use environment variable %v instead.", c.configKey, ConfAlluxioJavaOpts.EnvVar, c.EnvVar) + } + } + } + + for _, c := range []*AlluxioConfigEnvVar{ + ConfAlluxioHome, + ConfAlluxioConfDir, + ConfAlluxioLogsDir, + confAlluxioUserLogsDir, + } { + alluxioJavaOpts += c.ToJavaOpt(envVar, true) // mandatory java opts + } + + for _, c := range []*AlluxioConfigEnvVar{ + confAlluxioRamFolder, + confAlluxioMasterHostname, + confAlluxioMasterMountTableRootUfs, + confAlluxioWorkerRamdiskSize, + } { + alluxioJavaOpts += c.ToJavaOpt(envVar, false) // optional user provided java opts + } + + alluxioJavaOpts += fmt.Sprintf(JavaOptFormat, "log4j.configuration", "file:"+filepath.Join(envVar.GetString(ConfAlluxioConfDir.EnvVar), "log4j.properties")) + alluxioJavaOpts += fmt.Sprintf(JavaOptFormat, "org.apache.jasper.compiler.disablejsr199", true) + alluxioJavaOpts += fmt.Sprintf(JavaOptFormat, "java.net.preferIPv4Stack", true) + alluxioJavaOpts += fmt.Sprintf(JavaOptFormat, "org.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads", false) + + envVar.Set(ConfAlluxioJavaOpts.EnvVar, alluxioJavaOpts) + + for _, p := range ProcessRegistry { + p.SetEnvVars(envVar) + } + + // also set user environment variables, as they are not associated with a particular process + // ALLUXIO_USER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} {user provided opts} + userJavaOpts := fmt.Sprintf(JavaOptFormat, ConfAlluxioLoggerType, userLoggerType) + userJavaOpts += envVar.GetString(ConfAlluxioJavaOpts.EnvVar) + userJavaOpts += envVar.GetString(ConfAlluxioUserJavaOpts.EnvVar) + envVar.Set(ConfAlluxioUserJavaOpts.EnvVar, strings.TrimSpace(userJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument + + if log.Logger.IsLevelEnabled(logrus.DebugLevel) { + keys := envVar.AllKeys() + sort.Strings(keys) + log.Logger.Debugln("Environment variables:") + for _, k := range keys { + log.Logger.Debugf("%v %v", strings.ToUpper(k), envVar.Get(k)) + } + log.Logger.Debugln() + } + + Env = &AlluxioEnv{ + RootPath: rootPath, + Version: ver, + EnvVar: envVar, + } + return nil +} + +func checkAndSetJava(envVar *viper.Viper) error { + if envVar.Get(ConfJava.EnvVar) != nil { + return nil + } + // check JAVA_HOME + if javaHome := envVar.Get(confJavaHome.EnvVar); javaHome != nil { + javaHomeStr, ok := javaHome.(string) + if !ok { + return stacktrace.NewError("error casting %v to string", javaHome) + } + javaHomeBinJava := filepath.Join(javaHomeStr, "bin", "java") + // check if JAVA_HOME exists with valid $JAVA_HOME/bin/java binary + if _, err := os.Stat(javaHomeBinJava); err == nil { + envVar.Set(ConfJava.EnvVar, javaHomeBinJava) + return nil + } + } + // check if java is available via `PATH` using `which` + whichJavaPath, err := exec.Command("which", "java").Output() + if err == nil { + envVar.Set(ConfJava.EnvVar, strings.Trim(string(whichJavaPath), "\n")) + return nil + } + // cannot find java + // - ${JAVA} is not set + // - ${JAVA_HOME}/bin/java is not a valid path + // - java is not found as part of ${PATH} + return stacktrace.NewError(`Error: Cannot find 'java' on path or under $JAVA_HOME/bin/. Please set %v in alluxio-env.sh or user bash profile.`, confJavaHome.EnvVar) +} + +// matching the version string encapsulated by double quotes, ex. "11.0.19" where 11 is majorVer and 0 is minorVer +var javaVersionRe = regexp.MustCompile(`.*"(?P\d+)\.(?P\d+)[\w.-]*".*`) + +func checkJavaVersion(javaPath string, requiredJavaVersion int) error { + cmd := exec.Command("bash", "-c", fmt.Sprintf("%v -version", javaPath)) + javaVer, err := cmd.CombinedOutput() + if err != nil { + return stacktrace.Propagate(err, "error finding java version from `%v -version`", javaPath) + } + matches := javaVersionRe.FindStringSubmatch(string(javaVer)) + if len(matches) != 3 { + return stacktrace.NewError("java version output does not match expected regex pattern %v\n%v", javaVersionRe.String(), string(javaVer)) + } + majorVer, err := strconv.Atoi(matches[1]) + if err != nil { + return stacktrace.NewError("could not parse major version as an integer: %v", matches[1]) + } + if majorVer != requiredJavaVersion { + // java 8 is displayed as 1.8, so also print minor version in error message + ver := matches[1] + if _, err := strconv.Atoi(matches[2]); err == nil { + ver += "." + matches[2] + } + return stacktrace.NewError("Error: Alluxio requires Java %v, currently Java %v found.", requiredJavaVersion, ver) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/env/env_vars.go b/cli/src/alluxio.org/cli/env/env_vars.go new file mode 100644 index 000000000000..bda0c0d105e4 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/env_vars.go @@ -0,0 +1,130 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + + "github.com/spf13/viper" +) + +const ( + JavaOptFormat = " -D%v=%v" +) + +// alluxioEnvVarsTemplate lists the environment variables to include in conf/alluxio-env.sh.template +var alluxioEnvVarsTemplate = map[string]*AlluxioConfigEnvVar{} + +type AlluxioConfigEnvVar struct { + EnvVar string // environment variable read by startup script + description string // describes the environment variable in conf/alluxio-env.sh.template file + + // optional + configKey string // corresponding property key as defined in java + additionalAlluxioJavaOpts map[string]string // additional java opts to append if non-empty java opt is added +} + +func RegisterTemplateEnvVar(v *AlluxioConfigEnvVar) *AlluxioConfigEnvVar { + if _, ok := alluxioEnvVarsTemplate[v.EnvVar]; ok { + panic("Environment variable already registered: " + v.EnvVar) + } + alluxioEnvVarsTemplate[v.EnvVar] = v + return v +} + +const ( + envVersion = "VERSION" // VERSION is specified by libexec/version.sh and should not be overridden by the user + envAlluxioAssemblyClientJar = "ALLUXIO_ASSEMBLY_CLIENT_JAR" + envAlluxioAssemblyServerJar = "ALLUXIO_ASSEMBLY_SERVER_JAR" + EnvAlluxioClientClasspath = "ALLUXIO_CLIENT_CLASSPATH" + EnvAlluxioServerClasspath = "ALLUXIO_SERVER_CLASSPATH" +) + +var ( + ConfAlluxioConfDir = &AlluxioConfigEnvVar{ + configKey: "alluxio.conf.dir", + EnvVar: "ALLUXIO_CONF_DIR", + } + ConfAlluxioHome = &AlluxioConfigEnvVar{ + configKey: "alluxio.home", + EnvVar: "ALLUXIO_HOME", + } +) + +// environment variables that belong to the templatized alluxio-env.sh +// those with configKeys set will be appended to ALLUXIO_JAVA_OPTS +var ( + confJavaHome = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "JAVA_HOME", + }) + ConfJava = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "JAVA", + }) + ConfAlluxioLogsDir = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + configKey: "alluxio.logs.dir", + EnvVar: "ALLUXIO_LOGS_DIR", + }) + confAlluxioUserLogsDir = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + configKey: "alluxio.user.logs.dir", + EnvVar: "ALLUXIO_USER_LOGS_DIR", + }) + ConfAlluxioJavaOpts = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JAVA_OPTS", + }) + confAlluxioClasspath = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_CLASSPATH", + }) + ConfAlluxioUserJavaOpts = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_USER_JAVA_OPTS", + }) + ConfAlluxioUserAttachOpts = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_USER_ATTACH_OPTS", + }) +) + +// environment variables with corresponding configKeys to append to ALLUXIO_JAVA_OPTS +var ( + confAlluxioRamFolder = &AlluxioConfigEnvVar{ + configKey: "alluxio.worker.tieredstore.level0.dirs.path", + EnvVar: "ALLUXIO_RAM_FOLDER", + additionalAlluxioJavaOpts: map[string]string{ + "alluxio.worker.tieredstore.level0.alias": "MEM", + }, + } + confAlluxioMasterHostname = &AlluxioConfigEnvVar{ + configKey: "alluxio.master.hostname", + EnvVar: "ALLUXIO_MASTER_HOSTNAME", + } + confAlluxioMasterMountTableRootUfs = &AlluxioConfigEnvVar{ + configKey: "alluxio.master.mount.table.root.ufs", + EnvVar: "ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS", + } + confAlluxioWorkerRamdiskSize = &AlluxioConfigEnvVar{ + configKey: "alluxio.worker.ramdisk.size", + EnvVar: "ALLUXIO_WORKER_RAMDISK_SIZE", + } +) + +func (a *AlluxioConfigEnvVar) ToJavaOpt(env *viper.Viper, required bool) string { + v := env.Get(a.EnvVar) + if v == nil { + if required { + panic("No value set for required environment variable: " + a.EnvVar) + } + return "" + } + ret := fmt.Sprintf(JavaOptFormat, a.configKey, v) + for k2, v2 := range a.additionalAlluxioJavaOpts { + ret += fmt.Sprintf(JavaOptFormat, k2, v2) + } + return ret +} diff --git a/cli/src/alluxio.org/cli/env/process.go b/cli/src/alluxio.org/cli/env/process.go new file mode 100644 index 000000000000..b9af1c582943 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/process.go @@ -0,0 +1,225 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "syscall" + "time" + + "github.com/palantir/stacktrace" + "github.com/shirou/gopsutil/process" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/log" +) + +var ProcessRegistry = map[string]Process{} + +func RegisterProcess(p Process) Process { + name := p.Base().Name + if _, ok := ProcessRegistry[name]; ok { + panic(fmt.Sprintf("Process %v is already registered", name)) + } + ProcessRegistry[name] = p + return p +} + +type Process interface { + Base() *BaseProcess + SetEnvVars(*viper.Viper) + + Start(*StartProcessCommand) error + StartCmd(*cobra.Command) *cobra.Command + + Stop(*StopProcessCommand) error + StopCmd(*cobra.Command) *cobra.Command +} + +type BaseProcess struct { + Name string + JavaClassName string + JavaOptsEnvVarKey string + + // start + ProcessOutFile string + + // monitor + MonitorJavaClassName string +} + +func (p *BaseProcess) Launch(start *StartProcessCommand, args []string) error { + logsDir := Env.EnvVar.GetString(ConfAlluxioLogsDir.EnvVar) + if err := os.MkdirAll(logsDir, 0755); err != nil { + return stacktrace.Propagate(err, "error creating log directory at %v", logsDir) + } + + startCmd := exec.Command("nohup", args...) + for _, k := range Env.EnvVar.AllKeys() { + startCmd.Env = append(startCmd.Env, fmt.Sprintf("%s=%v", k, Env.EnvVar.Get(k))) + } + + outFile := filepath.Join(Env.EnvVar.GetString(ConfAlluxioLogsDir.EnvVar), p.ProcessOutFile) + f, err := os.OpenFile(outFile, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return stacktrace.Propagate(err, "error opening file at %v", outFile) + } + startCmd.Stdout = f + startCmd.Stderr = f + + log.Logger.Infof("Starting %v", p.Name) + log.Logger.Debugf("%v > %v 2>&1 &", startCmd.String(), outFile) + if err := startCmd.Start(); err != nil { + return stacktrace.Propagate(err, "error starting %v", p.Name) + } + + if start.AsyncStart { + log.Logger.Warnf("Skipping monitor checks for %v", p.Name) + } else { + if err := p.Monitor(); err != nil { + return stacktrace.Propagate(err, "error monitoring process after launching") + } + } + return nil +} + +var debugOptsToRemove = []*regexp.Regexp{ + regexp.MustCompile(`-agentlib:jdwp=transport=dt_socket.*address=[0-9]*`), + regexp.MustCompile(`-Dcom.sun.management.jmxremote.port=[0-9]*`), +} + +// Monitor runs the corresponding monitoring java class for the process, running the command +// ${JAVA} -cp ${ALLUXIO_CLIENT_CLASSPATH} ${ALLUXIO_MONITOR_JAVA_OPTS} {monitor java class} +// where ${ALLUXIO_MONITOR_JAVA_OPTS} is equivalent to the process specific JAVA_OPTS +// but with several debugging related options removed +func (p *BaseProcess) Monitor() error { + cmdArgs := []string{"-cp", Env.EnvVar.GetString(EnvAlluxioClientClasspath)} + + javaOpts := strings.TrimSpace(Env.EnvVar.GetString(p.JavaOptsEnvVarKey)) + // remove debugging options from java opts for monitor process + for _, re := range debugOptsToRemove { + javaOpts = re.ReplaceAllString(javaOpts, "") + } + cmdArgs = append(cmdArgs, strings.Split(javaOpts, " ")...) + + cmdArgs = append(cmdArgs, p.MonitorJavaClassName) + + cmd := exec.Command(Env.EnvVar.GetString(ConfJava.EnvVar), cmdArgs...) + for _, k := range Env.EnvVar.AllKeys() { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%v", k, Env.EnvVar.Get(k))) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + log.Logger.Debugln(cmd.String()) + if err := cmd.Run(); err != nil { + log.Logger.Errorf("[ FAILED ] The %v process is not serving requests", p.Name) + return stacktrace.Propagate(err, "error running monitor command for %v", p.Name) + } + log.Logger.Infof("[ OK ] The %v process is in a healthy state", p.Name) + return nil +} + +func (p *BaseProcess) Stop(cmd *StopProcessCommand) error { + processes, err := process.Processes() + if err != nil { + return stacktrace.Propagate(err, "error listing processes") + } + var matchingProcesses []*process.Process + for _, ps := range processes { + cmdline, err := ps.Cmdline() + if err != nil { + // process may have been terminated since listing + continue + } + if strings.Contains(cmdline, p.JavaClassName) { + matchingProcesses = append(matchingProcesses, ps) + } + } + if len(matchingProcesses) == 0 { + log.Logger.Warnf("No process to stop because could not find running process matching %v", p.JavaClassName) + return nil + } + log.Logger.Infof("Found %v running process(es) matching %v", len(matchingProcesses), p.JavaClassName) + wg, errs := &sync.WaitGroup{}, make([]error, len(matchingProcesses)) + for i, ps := range matchingProcesses { + i, ps := i, ps + wg.Add(1) + go func() { + defer wg.Done() + pid := int(ps.Pid) + proc, err := os.FindProcess(pid) + if err != nil { + errs[i] = stacktrace.Propagate(err, "error finding process with pid %v", pid) + return + } + if err := proc.Signal(syscall.SIGTERM); err != nil { // syscall.SIGTERM = kill -15 + errs[i] = stacktrace.Propagate(err, "error sending TERM signal to process for %v with pid %v", p.JavaClassName, pid) + return + } + // wait for process to exit + const killTimeoutSec = 120 + if ok := waitForProcessExit(proc, killTimeoutSec); !ok { + if cmd.SoftKill { + errs[i] = fmt.Errorf("Process for %v with pid %v did not terminate after %v seconds", p.JavaClassName, pid, killTimeoutSec) + return + } else { + log.Logger.Warnf("Force killing process for %v with pid %v", p.JavaClassName, pid) + if err := proc.Signal(syscall.SIGKILL); err != nil { // syscall.SIGKILL = kill -9 + errs[i] = stacktrace.Propagate(err, "error sending KILL signal to process for %v with pid %v", p.JavaClassName, pid) + return + } + } + } + }() + } + wg.Wait() + + var errMsg []string + for _, err := range errs { + if err != nil { + errMsg = append(errMsg, fmt.Sprintf("Failed to kill process: %v", err.Error())) + } + } + log.Logger.Infof("Successfully killed %v process(es)", len(matchingProcesses)-len(errMsg)) + if len(errMsg) != 0 { + log.Logger.Errorf("Failed to kill %v process(es):", len(errMsg)) + for _, msg := range errMsg { + log.Logger.Errorln(msg) + } + } + + return nil +} + +func waitForProcessExit(proc *os.Process, killTimeoutSec int) bool { + timeout := time.After(time.Duration(killTimeoutSec) * time.Second) + tick := time.Tick(time.Second * 10) + for { + select { + case <-timeout: + return false + case <-tick: + if err := proc.Signal(syscall.Signal(0)); err != nil { + // process is not found, done + return true + } + } + } +} diff --git a/cli/src/alluxio.org/cli/env/process_start.go b/cli/src/alluxio.org/cli/env/process_start.go new file mode 100644 index 000000000000..5d58d291adf6 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/process_start.go @@ -0,0 +1,44 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "github.com/spf13/cobra" +) + +type StartProcessCommand struct { + AsyncStart bool + SkipKillOnStart bool +} + +func (c *StartProcessCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "start", + Short: "Starts a process", + } + cmd.PersistentFlags().BoolVarP(&c.SkipKillOnStart, "skip-kill-prev", "N", false, "Avoid killing previous running processes when starting") + cmd.PersistentFlags().BoolVarP(&c.AsyncStart, "async", "a", false, "Asynchronously start processes without monitoring for start completion") + + for _, p := range ProcessRegistry { + p := p + cmd.AddCommand(p.StartCmd(&cobra.Command{ + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if !c.SkipKillOnStart { + _ = p.Stop(&StopProcessCommand{}) + } + return p.Start(c) + }, + })) + } + return cmd +} diff --git a/cli/src/alluxio.org/cli/env/process_stop.go b/cli/src/alluxio.org/cli/env/process_stop.go new file mode 100644 index 000000000000..3b4c1554d319 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/process_stop.go @@ -0,0 +1,39 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import "github.com/spf13/cobra" + +type StopProcessCommand struct { + SoftKill bool +} + +func (c *StopProcessCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "stop", + Short: "Stops a process", + } + cmd.PersistentFlags().BoolVarP(&c.SoftKill, "soft", "s", false, "Soft kill only, don't forcibly kill the process") + + for _, p := range ProcessRegistry { + p := p + cmd.AddCommand(p.StopCmd(&cobra.Command{ + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return p.Stop(&StopProcessCommand{ + SoftKill: c.SoftKill, + }) + }, + })) + } + return cmd +} diff --git a/cli/src/alluxio.org/cli/env/props.go b/cli/src/alluxio.org/cli/env/props.go new file mode 100644 index 000000000000..875310de5074 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/props.go @@ -0,0 +1,17 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +const ( + ConfAlluxioConfValidationEnabled = "alluxio.conf.validation.enabled" + ConfAlluxioLoggerType = "alluxio.logger.type" +) diff --git a/cli/src/alluxio.org/cli/env/service.go b/cli/src/alluxio.org/cli/env/service.go new file mode 100644 index 000000000000..274e2222856f --- /dev/null +++ b/cli/src/alluxio.org/cli/env/service.go @@ -0,0 +1,57 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var serviceRegistry = map[string]*Service{} + +func RegisterService(p *Service) *Service { + if _, ok := serviceRegistry[p.Name]; ok { + panic(fmt.Sprintf("Service %v is already registered", p.Name)) + } + serviceRegistry[p.Name] = p + return p +} + +func InitServiceCommandTree(rootCmd *cobra.Command) { + for _, p := range serviceRegistry { + p.InitCommandTree(rootCmd) + } +} + +type Service struct { + Name string + Description string + Commands []Command +} + +func (s *Service) InitCommandTree(rootCmd *cobra.Command) { + cmd := &cobra.Command{ + Use: s.Name, + Short: s.Description, + } + rootCmd.AddCommand(cmd) + + for _, c := range s.Commands { + cmd.AddCommand(c.ToCommand()) + } +} + +func (s *Service) AddCommands(c ...Command) *Service { + s.Commands = append(s.Commands, c...) + return s +} diff --git a/cli/src/alluxio.org/cli/launch/launch.go b/cli/src/alluxio.org/cli/launch/launch.go new file mode 100644 index 000000000000..337ff0620724 --- /dev/null +++ b/cli/src/alluxio.org/cli/launch/launch.go @@ -0,0 +1,51 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package launch + +import ( + "path/filepath" + + "github.com/palantir/stacktrace" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +func Run() error { + rootCmd := &cobra.Command{} + const rootPathName = "rootPath" + var flagRootPath string + rootCmd.PersistentFlags().StringVar(&flagRootPath, rootPathName, "", "Path to root of Alluxio installation") + if err := rootCmd.MarkPersistentFlagRequired(rootPathName); err != nil { + return stacktrace.Propagate(err, "error marking %v flag required", rootPathName) + } + if err := rootCmd.PersistentFlags().MarkHidden(rootPathName); err != nil { + return stacktrace.Propagate(err, "error marking %v flag hidden", rootPathName) + } + var flagDebugLog bool + rootCmd.PersistentFlags().BoolVar(&flagDebugLog, "debug-log", false, "True to enable debug logging") + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + if flagDebugLog { + log.Logger.SetLevel(logrus.DebugLevel) + } + if err := env.InitAlluxioEnv(filepath.Clean(flagRootPath)); err != nil { + return stacktrace.Propagate(err, "error defining alluxio environment") + } + return nil + } + + env.InitServiceCommandTree(rootCmd) + + return rootCmd.Execute() +} diff --git a/cli/src/alluxio.org/cli/main.go b/cli/src/alluxio.org/cli/main.go new file mode 100644 index 000000000000..c4132b323728 --- /dev/null +++ b/cli/src/alluxio.org/cli/main.go @@ -0,0 +1,60 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package main + +import ( + "os" + + "alluxio.org/cli/cmd/conf" + "alluxio.org/cli/cmd/fs" + "alluxio.org/cli/cmd/info" + "alluxio.org/cli/cmd/journal" + "alluxio.org/cli/cmd/process" + "alluxio.org/cli/cmd/quorum" + "alluxio.org/cli/env" + "alluxio.org/cli/launch" + "alluxio.org/cli/processes" +) + +func main() { + for _, p := range []env.Process{ + processes.All, + processes.JobMaster, + processes.JobMasters, + processes.JobWorker, + processes.JobWorkers, + processes.Local, + processes.Master, + processes.Masters, + processes.Proxy, + processes.Proxies, + processes.Worker, + processes.Workers, + } { + env.RegisterProcess(p) + } + + for _, c := range []*env.Service{ + conf.Service, + fs.Service, + info.Service, + journal.Service, + process.Service, + quorum.Service, + } { + env.RegisterService(c) + } + + if err := launch.Run(); err != nil { + os.Exit(1) + } +} diff --git a/cli/src/alluxio.org/cli/processes/all.go b/cli/src/alluxio.org/cli/processes/all.go new file mode 100644 index 000000000000..bea8146b801c --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/all.go @@ -0,0 +1,90 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "os/exec" + "path" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var All = &AllProcess{ + BaseProcess: &env.BaseProcess{ + Name: "all", + }, +} + +type AllProcess struct { + *env.BaseProcess +} + +func (p *AllProcess) SetEnvVars(envVar *viper.Viper) { + return +} + +func (p *AllProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *AllProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *AllProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *AllProcess) Start(cmd *env.StartProcessCommand) error { + // generate commands + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + components := []string{"masters", "job_masters", "workers", "job_workers", "proxies"} + var commands []string + for _, component := range components { + arguments := "process start" + " " + component + if cmd.AsyncStart { + arguments = arguments + " -a" + } + if cmd.SkipKillOnStart { + arguments = arguments + " -N" + } + commands = append(commands, cliPath+" "+arguments) + } + + for _, command := range commands { + exec.Command(command) + } + + return nil +} + +func (p *AllProcess) Stop(cmd *env.StopProcessCommand) error { + // generate commands + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + components := []string{"masters", "job_masters", "workers", "job_workers", "proxies"} + var commands []string + for _, component := range components { + arguments := "process stop" + " " + component + commands = append(commands, cliPath+" "+arguments) + } + + for _, command := range commands { + exec.Command(command) + } + + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/common.go b/cli/src/alluxio.org/cli/processes/common.go new file mode 100644 index 000000000000..cfc3b60373ab --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/common.go @@ -0,0 +1,171 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "bufio" + "fmt" + "io" + "os" + "path" + "strings" + "time" + + "alluxio.org/cli/env" + "alluxio.org/log" + "golang.org/x/crypto/ssh" +) + +func getMasters() ([]string, error) { + mastersDir := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioConfDir.EnvVar), "masters") + mastersFile, err := os.Open(mastersDir) + if err != nil { + log.Logger.Errorf("Error reading worker hostnames at %s", mastersDir) + return nil, err + } + + mastersReader := bufio.NewReader(mastersFile) + var mastersList []string + lastLine := false + for !lastLine { + // read lines of the workers file + line, err := mastersReader.ReadString('\n') + if err != nil { + if err == io.EOF { + lastLine = true + } else { + log.Logger.Errorf("Error parsing worker file at this line: %s", line) + return nil, err + } + } + // remove notes + if strings.Index(line, "#") != -1 { + line = line[:strings.Index(line, "#")] + } + line = strings.TrimSpace(line) + if line != "" { + mastersList = append(mastersList, line) + } + } + return mastersList, nil +} + +func getWorkers() ([]string, error) { + workersDir := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioConfDir.EnvVar), "workers") + workersFile, err := os.Open(workersDir) + if err != nil { + log.Logger.Errorf("Error reading worker hostnames at %s", workersDir) + return nil, err + } + + workersReader := bufio.NewReader(workersFile) + var workersList []string + lastLine := false + for !lastLine { + // read lines of the workers file + line, err := workersReader.ReadString('\n') + if err != nil { + if err == io.EOF { + lastLine = true + } else { + log.Logger.Errorf("Error parsing worker file at this line: %s", line) + return nil, err + } + } + // remove notes + if strings.Index(line, "#") != -1 { + line = line[:strings.Index(line, "#")] + } + line = strings.TrimSpace(line) + if line != "" { + workersList = append(workersList, line) + } + } + return workersList, nil +} + +func getPrivateKey() (ssh.Signer, error) { + homePath, err := os.UserHomeDir() + if err != nil { + log.Logger.Errorf("User home directory not found at %s", homePath) + return nil, err + } + privateKey, err := os.ReadFile(path.Join(homePath, ".ssh", "id_rsa")) + if err != nil { + log.Logger.Errorf("Private key file not found at %s", path.Join(homePath, ".ssh", "id_rsa")) + return nil, err + } + parsedPrivateKey, err := ssh.ParsePrivateKey(privateKey) + if err != nil { + log.Logger.Errorf("Cannot parse public key at %s", path.Join(homePath, ".ssh", "id_rsa")) + return nil, err + } + return parsedPrivateKey, nil +} + +func dialConnection(remoteAddress string, key ssh.Signer) (*ssh.Client, error) { + clientConfig := &ssh.ClientConfig{ + // TODO: how to get user name? Like ${USER} in alluxio-common.sh + User: "root", + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(key), + }, + Timeout: 5 * time.Second, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + // TODO: Some machines might have changed default SSH port. Get ssh port or remind users when get started. + dialAddr := fmt.Sprintf("%s:%d", remoteAddress, 22) + conn, err := ssh.Dial("tcp", dialAddr, clientConfig) + if err != nil { + log.Logger.Errorf("Dial failed to %s, error: %s", remoteAddress, err) + } + return conn, err +} + +func closeConnection(conn *ssh.Client) error { + err := conn.Close() + if err != nil { + log.Logger.Infof("Connection to %s closed. Error: %s", conn.RemoteAddr(), err) + } else { + log.Logger.Infof("Connection to %s closed.", conn.RemoteAddr()) + } + return err +} + +func runCommand(conn *ssh.Client, command string) error { + // create a session for each worker + session, err := conn.NewSession() + if err != nil { + log.Logger.Errorf("Cannot create session at %s", conn.RemoteAddr()) + return err + } + + session.Stdout = os.Stdout + session.Stderr = os.Stderr + + // run session + err = session.Run(command) + if err != nil { + log.Logger.Errorf("Run command %s failed at %s", command, conn.RemoteAddr()) + return err + } + + // close session + err = session.Close() + if err != nil && err != io.EOF { + log.Logger.Infof("Session at %s closed. Error: %s", conn.RemoteAddr(), err) + return err + } else { + log.Logger.Infof("Session at %s closed.", conn.RemoteAddr()) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/job_master.go b/cli/src/alluxio.org/cli/processes/job_master.go new file mode 100644 index 000000000000..a4bfcf417785 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/job_master.go @@ -0,0 +1,99 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var JobMaster = &JobMasterProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_master", + JavaClassName: "alluxio.master.AlluxioJobMaster", + JavaOptsEnvVarKey: confAlluxioJobMasterJavaOpts.EnvVar, + ProcessOutFile: "job_master.out", + MonitorJavaClassName: "alluxio.master.job.AlluxioJobMasterMonitor", + }, +} + +const ( + confAlluxioJobMasterAuditLoggerType = "alluxio.job.master.audit.logger.type" + envAlluxioAuditJobMasterLogger = "ALLUXIO_AUDIT_JOB_MASTER_LOGGER" + envAlluxioJobMasterLogger = "ALLUXIO_JOB_MASTER_LOGGER" + jobMasterAuditLoggerType = "JOB_MASTER_AUDIT_LOGGER" + jobMasterLoggerType = "JOB_MASTER_LOGGER" +) + +var ( + confAlluxioJobMasterJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_MASTER_JAVA_OPTS", + }) + confAlluxioJobMasterAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_MASTER_ATTACH_OPTS", + }) +) + +type JobMasterProcess struct { + *env.BaseProcess +} + +func (p *JobMasterProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *JobMasterProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_JOB_MASTER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_JOB_MASTER_JAVA_OPTS} + envVar.SetDefault(envAlluxioJobMasterLogger, jobMasterLoggerType) + jobMasterJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioJobMasterLogger)) + envVar.SetDefault(envAlluxioAuditJobMasterLogger, jobMasterAuditLoggerType) + jobMasterJavaOpts += fmt.Sprintf(env.JavaOptFormat, confAlluxioJobMasterAuditLoggerType, envVar.Get(envAlluxioAuditJobMasterLogger)) + + jobMasterJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + jobMasterJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(jobMasterJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *JobMasterProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobMasterProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioJobMasterAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + jobMasterJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(jobMasterJavaOpts, " ")...) + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *JobMasterProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/job_masters.go b/cli/src/alluxio.org/cli/processes/job_masters.go new file mode 100644 index 000000000000..15c1970da69a --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/job_masters.go @@ -0,0 +1,146 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "path" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var JobMasters = &JobMastersProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_masters", + }, +} + +type JobMastersProcess struct { + *env.BaseProcess +} + +func (p *JobMastersProcess) SetEnvVars(envVar *viper.Viper) { + return +} + +func (p *JobMastersProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *JobMastersProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobMastersProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobMastersProcess) Start(cmd *env.StartProcessCommand) error { + // get list of all masters, stored at mastersList + masters, err := getMasters() + if err != nil { + log.Logger.Fatalf("Cannot get masters, error: %s", err) + } + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process start job_master" + if cmd.AsyncStart { + arguments = arguments + " -a" + } + if cmd.SkipKillOnStart { + arguments = arguments + " -N" + } + command := cliPath + " " + arguments + + // for each master, create a client and run + // TODO: now start master one by one, need to do them in parallel + var errors []error + for _, master := range masters { + conn, err := dialConnection(master, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on masters: %s", command, masters) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} + +func (p *JobMastersProcess) Stop(cmd *env.StopProcessCommand) error { + // get list of all masters, stored at mastersList + masters, err := getMasters() + if err != nil { + log.Logger.Fatalf("Cannot get masters, error: %s", err) + } + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process stop job_master" + command := cliPath + " " + arguments + + // for each master, create a client and run + // TODO: now stop master one by one, need to do them in parallel + var errors []error + for _, master := range masters { + conn, err := dialConnection(master, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on masters: %s", command, masters) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/job_worker.go b/cli/src/alluxio.org/cli/processes/job_worker.go new file mode 100644 index 000000000000..a53db695af6e --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/job_worker.go @@ -0,0 +1,94 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var JobWorker = &WorkerProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_worker", + JavaClassName: "alluxio.worker.AlluxioJobWorker", + JavaOptsEnvVarKey: confAlluxioJobWorkerJavaOpts.EnvVar, + ProcessOutFile: "job_worker.out", + MonitorJavaClassName: "alluxio.worker.job.AlluxioJobWorkerMonitor", + }, +} + +const ( + envAlluxioJobWorkerLogger = "ALLUXIO_JOB_WORKER_LOGGER" + jobWorkerLoggerType = "JOB_WORKER_LOGGER" +) + +var ( + confAlluxioJobWorkerJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_WORKER_JAVA_OPTS", + }) + confAlluxioJobWorkerAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_WORKER_ATTACH_OPTS", + }) +) + +type JobWorkerProcess struct { + *env.BaseProcess +} + +func (p *JobWorkerProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *JobWorkerProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_JOB_WORKER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_WORKER_JAVA_OPTS} + envVar.SetDefault(envAlluxioJobWorkerLogger, jobWorkerLoggerType) + jobWorkerJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioJobWorkerLogger)) + + jobWorkerJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + jobWorkerJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(jobWorkerJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *JobWorkerProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobWorkerProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioJobWorkerAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + jobWorkerJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(jobWorkerJavaOpts, " ")...) + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *JobWorkerProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/job_workers.go b/cli/src/alluxio.org/cli/processes/job_workers.go new file mode 100644 index 000000000000..0520342d1b6b --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/job_workers.go @@ -0,0 +1,146 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "path" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var JobWorkers = &JobWorkersProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_workers", + }, +} + +type JobWorkersProcess struct { + *env.BaseProcess +} + +func (p *JobWorkersProcess) SetEnvVars(envVar *viper.Viper) { + return +} + +func (p *JobWorkersProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *JobWorkersProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobWorkersProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobWorkersProcess) Start(cmd *env.StartProcessCommand) error { + // get list of all workers, stored at workersList + workers, err := getWorkers() + if err != nil { + log.Logger.Fatalf("Cannot get workers, error: %s", err) + } + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process start job_worker" + if cmd.AsyncStart { + arguments = arguments + " -a" + } + if cmd.SkipKillOnStart { + arguments = arguments + " -N" + } + command := cliPath + " " + arguments + + // for each worker, create a client and run + // TODO: now start worker one by one, need to do them in parallel + var errors []error + for _, worker := range workers { + conn, err := dialConnection(worker, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on workers: %s", command, workers) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} + +func (p *JobWorkersProcess) Stop(cmd *env.StopProcessCommand) error { + // get list of all workers, stored at workersList + workers, err := getWorkers() + if err != nil { + log.Logger.Fatalf("Cannot get workers, error: %s", err) + } + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process stop job_worker" + command := cliPath + " " + arguments + + // for each worker, create a client and run + // TODO: now stop worker one by one, need to do them in parallel + var errors []error + for _, worker := range workers { + conn, err := dialConnection(worker, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on workers: %s", command, workers) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/local.go b/cli/src/alluxio.org/cli/processes/local.go new file mode 100644 index 000000000000..b4a2bcae69a3 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/local.go @@ -0,0 +1,90 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "os/exec" + "path" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var Local = &LocalProcess{ + BaseProcess: &env.BaseProcess{ + Name: "local", + }, +} + +type LocalProcess struct { + *env.BaseProcess +} + +func (p *LocalProcess) SetEnvVars(envVar *viper.Viper) { + return +} + +func (p *LocalProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *LocalProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *LocalProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *LocalProcess) Start(cmd *env.StartProcessCommand) error { + // generate commands + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + components := []string{"master", "job_master", "worker", "job_worker", "proxy"} + var commands []string + for _, component := range components { + arguments := "process start" + " " + component + if cmd.AsyncStart { + arguments = arguments + " -a" + } + if cmd.SkipKillOnStart { + arguments = arguments + " -N" + } + commands = append(commands, cliPath+" "+arguments) + } + + for _, command := range commands { + exec.Command(command) + } + + return nil +} + +func (p *LocalProcess) Stop(cmd *env.StopProcessCommand) error { + // generate commands + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + components := []string{"master", "job_master", "worker", "job_worker", "proxy"} + var commands []string + for _, component := range components { + arguments := "process stop" + " " + component + commands = append(commands, cliPath+" "+arguments) + } + + for _, command := range commands { + exec.Command(command) + } + + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/master.go b/cli/src/alluxio.org/cli/processes/master.go new file mode 100644 index 000000000000..198e016442ca --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/master.go @@ -0,0 +1,150 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "os" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/cmd/conf" + "alluxio.org/cli/cmd/journal" + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Master = &MasterProcess{ + BaseProcess: &env.BaseProcess{ + Name: "master", + JavaClassName: "alluxio.master.AlluxioMaster", + JavaOptsEnvVarKey: confAlluxioMasterJavaOpts.EnvVar, + ProcessOutFile: "master.out", + MonitorJavaClassName: "alluxio.master.AlluxioMasterMonitor", + }, +} + +const ( + confAlluxioMasterAuditLoggerType = "alluxio.master.audit.logger.type" + confAlluxioMasterJournalFolder = "alluxio.master.journal.folder" + confAlluxioMasterJournalInitFromBackup = "alluxio.master.journal.init.from.backup" + + envAlluxioAuditMasterLogger = "ALLUXIO_AUDIT_MASTER_LOGGER" + envAlluxioMasterLogger = "ALLUXIO_MASTER_LOGGER" + masterAuditLoggerType = "MASTER_AUDIT_LOGGER" + masterLoggerType = "MASTER_LOGGER" +) + +var ( + confAlluxioMasterJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_MASTER_JAVA_OPTS", + }) + confAlluxioMasterAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_MASTER_ATTACH_OPTS", + }) +) + +type MasterProcess struct { + *env.BaseProcess + + JournalBackupFile string +} + +func (p *MasterProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *MasterProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_MASTER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_MASTER_JAVA_OPTS} + envVar.SetDefault(envAlluxioMasterLogger, masterLoggerType) + masterJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioMasterLogger)) + envVar.SetDefault(envAlluxioAuditMasterLogger, masterAuditLoggerType) + masterJavaOpts += fmt.Sprintf(env.JavaOptFormat, confAlluxioMasterAuditLoggerType, envVar.Get(envAlluxioAuditMasterLogger)) + + masterJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + masterJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(masterJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *MasterProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + cmd.Flags().StringVar(&p.JournalBackupFile, "journal-backup-file", "", "Path to journal backup file to restore the master from") + return cmd +} + +func (p *MasterProcess) Start(cmd *env.StartProcessCommand) error { + if err := p.checkJournal(); err != nil { + return stacktrace.Propagate(err, "error validating journal") + } + + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioMasterAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + masterJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(masterJavaOpts, " ")...) + + if p.JournalBackupFile != "" { + cmdArgs = append(cmdArgs, strings.TrimSpace(fmt.Sprintf(env.JavaOptFormat, confAlluxioMasterJournalInitFromBackup, p.JournalBackupFile))) + } + // specify a default of -Xmx8g if no memory setting is specified + const xmxOpt = "-Xmx" + if !strings.Contains(masterJavaOpts, xmxOpt) && !strings.Contains(masterJavaOpts, "MaxRAMPercentage") { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v8g", xmxOpt)) + } + // specify a default of -XX:MetaspaceSize=256M if not set + const metaspaceSizeOpt = "-XX:MetaspaceSize" + if !strings.Contains(masterJavaOpts, metaspaceSizeOpt) { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v=256M", metaspaceSizeOpt)) + } + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *MasterProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MasterProcess) checkJournal() error { + journalDir, err := conf.Get.FetchValue(confAlluxioMasterJournalFolder) + if err != nil { + return stacktrace.Propagate(err, "error fetching value for %v", confAlluxioMasterJournalFolder) + } + stat, err := os.Stat(journalDir) + if os.IsNotExist(err) { + log.Logger.Info("Journal directory does not exist, formatting") + if err := journal.Format.Format(); err != nil { + return stacktrace.Propagate(err, "error formatting journal") + } + return nil + } + if err != nil { + return stacktrace.Propagate(err, "error listing path at %v", journalDir) + } + if !stat.IsDir() { + return stacktrace.NewError("Journal location %v is not a directory. Please remove the file and retry.", journalDir) + } + // journal folder path exists and is a directory + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/masters.go b/cli/src/alluxio.org/cli/processes/masters.go new file mode 100644 index 000000000000..221af3aa176c --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/masters.go @@ -0,0 +1,146 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "path" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Masters = &MastersProcess{ + BaseProcess: &env.BaseProcess{ + Name: "masters", + }, +} + +type MastersProcess struct { + *env.BaseProcess +} + +func (p *MastersProcess) SetEnvVars(envVar *viper.Viper) { + return +} + +func (p *MastersProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *MastersProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MastersProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MastersProcess) Start(cmd *env.StartProcessCommand) error { + // get list of all masters, stored at mastersList + masters, err := getMasters() + if err != nil { + log.Logger.Fatalf("Cannot get masters, error: %s", err) + } + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process start master" + if cmd.AsyncStart { + arguments = arguments + " -a" + } + if cmd.SkipKillOnStart { + arguments = arguments + " -N" + } + command := cliPath + " " + arguments + + // for each master, create a client and run + // TODO: now start master one by one, need to do them in parallel + var errors []error + for _, master := range masters { + conn, err := dialConnection(master, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on masters: %s", command, masters) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} + +func (p *MastersProcess) Stop(cmd *env.StopProcessCommand) error { + // get list of all masters, stored at mastersList + masters, err := getMasters() + if err != nil { + log.Logger.Fatalf("Cannot get masters, error: %s", err) + } + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process stop master" + command := cliPath + " " + arguments + + // for each master, create a client and run + // TODO: now stop master one by one, need to do them in parallel + var errors []error + for _, master := range masters { + conn, err := dialConnection(master, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on masters: %s", command, masters) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/proxies.go b/cli/src/alluxio.org/cli/processes/proxies.go new file mode 100644 index 000000000000..8e0829cbc555 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/proxies.go @@ -0,0 +1,156 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "path" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Proxies = &ProxiesProcess{ + BaseProcess: &env.BaseProcess{ + Name: "proxies", + }, +} + +type ProxiesProcess struct { + *env.BaseProcess +} + +func (p *ProxiesProcess) SetEnvVars(envVar *viper.Viper) { + return +} + +func (p *ProxiesProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *ProxiesProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *ProxiesProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *ProxiesProcess) Start(cmd *env.StartProcessCommand) error { + // get list of all masters and workers, stored at allList + masters, err := getMasters() + if err != nil { + log.Logger.Fatalf("Cannot get masters, error: %s", err) + } + workers, err := getWorkers() + if err != nil { + log.Logger.Fatalf("Cannot get workers, error: %s", err) + } + allNodes := append(masters, workers...) + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process start proxy" + if cmd.AsyncStart { + arguments = arguments + " -a" + } + if cmd.SkipKillOnStart { + arguments = arguments + " -N" + } + command := cliPath + " " + arguments + + // for each node, create a client + // TODO: now start nodes one by one, need to do them in parallel + var errors []error + for _, node := range allNodes { + conn, err := dialConnection(node, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on nodes: %s", command, allNodes) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} + +func (p *ProxiesProcess) Stop(cmd *env.StopProcessCommand) error { + // get list of all masters and workers, stored at allList + masters, err := getMasters() + if err != nil { + log.Logger.Fatalf("Cannot get masters, error: %s", err) + } + workers, err := getWorkers() + if err != nil { + log.Logger.Fatalf("Cannot get workers, error: %s", err) + } + allNodes := append(masters, workers...) + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process stop proxy" + command := cliPath + " " + arguments + + // for each node, create a client + // TODO: now stop nodes one by one, need to do them in parallel + var errors []error + for _, node := range allNodes { + conn, err := dialConnection(node, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on nodes: %s", command, allNodes) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/proxy.go b/cli/src/alluxio.org/cli/processes/proxy.go new file mode 100644 index 000000000000..250ba7fca79f --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/proxy.go @@ -0,0 +1,99 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var Proxy = &ProxyProcess{ + BaseProcess: &env.BaseProcess{ + Name: "proxy", + JavaClassName: "alluxio.proxy.AlluxioProxy", + JavaOptsEnvVarKey: confAlluxioProxyJavaOpts.EnvVar, + ProcessOutFile: "proxy.out", + MonitorJavaClassName: "alluxio.proxy.AlluxioProxyMonitor", + }, +} + +const ( + confAlluxioProxyAuditLoggerType = "alluxio.proxy.audit.logger.type" + envAlluxioAuditProxyLogger = "ALLUXIO_AUDIT_PROXY_LOGGER" + envAlluxioProxyLogger = "ALLUXIO_PROXY_LOGGER" + proxyAuditLoggerType = "PROXY_AUDIT_LOGGER" + proxyLoggerType = "PROXY_LOGGER" +) + +var ( + confAlluxioProxyJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_PROXY_JAVA_OPTS", + }) + confAlluxioProxyAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_PROXY_ATTACH_OPTS", + }) +) + +type ProxyProcess struct { + *env.BaseProcess +} + +func (p *ProxyProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *ProxyProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_PROXY_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_PROXY_JAVA_OPTS} + envVar.SetDefault(envAlluxioProxyLogger, proxyLoggerType) + proxyJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioProxyLogger)) + envVar.SetDefault(envAlluxioAuditProxyLogger, proxyAuditLoggerType) + proxyJavaOpts += fmt.Sprintf(env.JavaOptFormat, confAlluxioProxyAuditLoggerType, envVar.Get(envAlluxioAuditProxyLogger)) + + proxyJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + proxyJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(proxyJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *ProxyProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *ProxyProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioProxyAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + proxyJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(proxyJavaOpts, " ")...) + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *ProxyProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/worker.go b/cli/src/alluxio.org/cli/processes/worker.go new file mode 100644 index 000000000000..726361c00aa0 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/worker.go @@ -0,0 +1,105 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var Worker = &WorkerProcess{ + BaseProcess: &env.BaseProcess{ + Name: "worker", + JavaClassName: "alluxio.worker.AlluxioWorker", + JavaOptsEnvVarKey: confAlluxioWorkerJavaOpts.EnvVar, + ProcessOutFile: "worker.out", + MonitorJavaClassName: "alluxio.worker.AlluxioWorkerMonitor", + }, +} + +const ( + envAlluxioWorkerLogger = "ALLUXIO_WORKER_LOGGER" + workerLoggerType = "WORKER_LOGGER" +) + +var ( + confAlluxioWorkerJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_WORKER_JAVA_OPTS", + }) + confAlluxioWorkerAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_WORKER_ATTACH_OPTS", + }) +) + +type WorkerProcess struct { + *env.BaseProcess +} + +func (p *WorkerProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *WorkerProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_WORKER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_WORKER_JAVA_OPTS} + envVar.SetDefault(envAlluxioWorkerLogger, workerLoggerType) + workerJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioWorkerLogger)) + + workerJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + workerJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(workerJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *WorkerProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *WorkerProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioWorkerAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + workerJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(workerJavaOpts, " ")...) + + // specify a default of -Xmx4g if no memory setting is specified + const xmxOpt = "-Xmx" + if !strings.Contains(workerJavaOpts, xmxOpt) && !strings.Contains(workerJavaOpts, "MaxRAMPercentage") { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v4g", xmxOpt)) + } + // specify a default of -XX:MaxDirectMemorySize=4g if not set + const maxDirectMemorySize = "-XX:MaxDirectMemorySize" + if !strings.Contains(workerJavaOpts, maxDirectMemorySize) { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v=4g", maxDirectMemorySize)) + } + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *WorkerProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/workers.go b/cli/src/alluxio.org/cli/processes/workers.go new file mode 100644 index 000000000000..1e15933893e8 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/workers.go @@ -0,0 +1,146 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "path" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Workers = &WorkersProcess{ + BaseProcess: &env.BaseProcess{ + Name: "workers", + }, +} + +type WorkersProcess struct { + *env.BaseProcess +} + +func (p *WorkersProcess) SetEnvVars(envVar *viper.Viper) { + return +} + +func (p *WorkersProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *WorkersProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *WorkersProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *WorkersProcess) Start(cmd *env.StartProcessCommand) error { + // get list of all workers, stored at workersList + workers, err := getWorkers() + if err != nil { + log.Logger.Fatalf("Cannot get workers, error: %s", err) + } + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process start worker" + if cmd.AsyncStart { + arguments = arguments + " -a" + } + if cmd.SkipKillOnStart { + arguments = arguments + " -N" + } + command := cliPath + " " + arguments + + // for each worker, create a client and run + // TODO: now start worker one by one, need to do them in parallel + var errors []error + for _, worker := range workers { + conn, err := dialConnection(worker, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on workers: %s", command, workers) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} + +func (p *WorkersProcess) Stop(cmd *env.StopProcessCommand) error { + // get list of all workers, stored at workersList + workers, err := getWorkers() + if err != nil { + log.Logger.Fatalf("Cannot get workers, error: %s", err) + } + + // get public key for passwordless ssh + key, err := getPrivateKey() + if err != nil { + log.Logger.Fatalf("Cannot get private key, error: %s", err) + } + + // generate command + cliPath := path.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), "bin", "cli.sh") + arguments := "process stop worker" + command := cliPath + " " + arguments + + // for each worker, create a client and run + // TODO: now stop worker one by one, need to do them in parallel + var errors []error + for _, worker := range workers { + conn, err := dialConnection(worker, key) + if err != nil { + errors = append(errors, err) + continue + } + err = runCommand(conn, command) + if err != nil { + errors = append(errors, err) + } + err = closeConnection(conn) + if err != nil { + errors = append(errors, err) + } + } + + if len(errors) == 0 { + log.Logger.Infof("Run command %s successful on workers: %s", command, workers) + } else { + log.Logger.Fatalf("Run command %s failed, number of failures: %v", command, len(errors)) + } + return nil +} diff --git a/cli/src/alluxio.org/go.mod b/cli/src/alluxio.org/go.mod new file mode 100644 index 000000000000..958b996eb25a --- /dev/null +++ b/cli/src/alluxio.org/go.mod @@ -0,0 +1,35 @@ +module alluxio.org + +go 1.18 + +require ( + github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 + github.com/shirou/gopsutil v3.21.11+incompatible + github.com/sirupsen/logrus v1.9.0 + github.com/spf13/cobra v1.7.0 + github.com/spf13/viper v1.15.0 + golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e +) + +require ( + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/spf13/afero v1.9.3 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.8.2 // indirect + github.com/subosito/gotenv v1.4.2 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/cli/src/alluxio.org/go.sum b/cli/src/alluxio.org/go.sum new file mode 100644 index 000000000000..c364d69e05c6 --- /dev/null +++ b/cli/src/alluxio.org/go.sum @@ -0,0 +1,504 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 h1:nRlQD0u1871kaznCnn1EvYiMbum36v7hw1DLPEjds4o= +github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177/go.mod h1:ao5zGxj8Z4x60IOVYZUbDSmt3R8Ddo080vEgPosHpak= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/cli/src/alluxio.org/log/log.go b/cli/src/alluxio.org/log/log.go new file mode 100644 index 000000000000..86e50b31187b --- /dev/null +++ b/cli/src/alluxio.org/log/log.go @@ -0,0 +1,30 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package log + +import ( + "os" + + "github.com/sirupsen/logrus" +) + +var Logger *logrus.Logger + +func init() { + l := logrus.New() + l.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + }) + l.SetOutput(os.Stdout) + + Logger = l +} diff --git a/pom.xml b/pom.xml index b7447a13ed15..ab92d8f2d241 100644 --- a/pom.xml +++ b/pom.xml @@ -1569,5 +1569,29 @@ true + + goCli + + + + + exec-maven-plugin + org.codehaus.mojo + + + Build Golang CLI + compile + + exec + + + ${build.path}/cli/build-cli.sh + + + + + + +