From 7505ea45468b4cf0b48dd28a40e00a1f96df70b2 Mon Sep 17 00:00:00 2001 From: Rico Chiu Date: Tue, 22 Aug 2023 12:06:22 -0700 Subject: [PATCH] Fix load cli command - remove job subcommands that were referring to the old job service - update `load` subcommand and add necessary flags - remove checks in the java class that referred to the old load command pr-link: Alluxio/alluxio#18045 change-id: cid-31fce5258be608e7d34648e07940857d30340c60 --- cli/src/alluxio.org/cli/cmd/job/cancel.go | 61 ----------------- cli/src/alluxio.org/cli/cmd/job/cmd_status.go | 61 ----------------- cli/src/alluxio.org/cli/cmd/job/job.go | 61 +++++++++++++++-- cli/src/alluxio.org/cli/cmd/job/job_status.go | 67 ------------------- cli/src/alluxio.org/cli/cmd/job/leader.go | 51 -------------- cli/src/alluxio.org/cli/cmd/job/list.go | 52 -------------- cli/src/alluxio.org/cli/cmd/job/load.go | 44 +++++++++--- cli/src/alluxio.org/cli/cmd/names/names.go | 1 - .../alluxio/cli/fs/command/LoadCommand.java | 23 +------ 9 files changed, 92 insertions(+), 329 deletions(-) delete mode 100644 cli/src/alluxio.org/cli/cmd/job/cancel.go delete mode 100644 cli/src/alluxio.org/cli/cmd/job/cmd_status.go delete mode 100644 cli/src/alluxio.org/cli/cmd/job/job_status.go delete mode 100644 cli/src/alluxio.org/cli/cmd/job/leader.go delete mode 100644 cli/src/alluxio.org/cli/cmd/job/list.go diff --git a/cli/src/alluxio.org/cli/cmd/job/cancel.go b/cli/src/alluxio.org/cli/cmd/job/cancel.go deleted file mode 100644 index 7fb5ab5ef955..000000000000 --- a/cli/src/alluxio.org/cli/cmd/job/cancel.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package job - -import ( - "strconv" - - "github.com/palantir/stacktrace" - "github.com/spf13/cobra" - - "alluxio.org/cli/cmd/names" - "alluxio.org/cli/env" -) - -var Cancel = &CancelCommand{ - BaseJavaCommand: &env.BaseJavaCommand{ - CommandName: "cancel", - JavaClassName: names.JobShellJavaClass, - }, -} - -type CancelCommand struct { - *env.BaseJavaCommand - jobId int -} - -func (c *CancelCommand) Base() *env.BaseJavaCommand { - return c.BaseJavaCommand -} - -func (c *CancelCommand) ToCommand() *cobra.Command { - cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ - Use: Cancel.CommandName, - Short: "Cancels a job asynchronously.", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return c.Run(args) - }, - }) - const id = "id" - cmd.Flags().IntVar(&c.jobId, id, 0, "Determine a job ID to cancel") - cmd.MarkFlagRequired(id) - return cmd -} - -func (c *CancelCommand) Run(args []string) error { - if c.jobId <= 0 { - return stacktrace.NewError("Flag --id should be a positive integer") - } - javaArgs := []string{"cancel", strconv.Itoa(c.jobId)} - return c.Base().Run(javaArgs) -} diff --git a/cli/src/alluxio.org/cli/cmd/job/cmd_status.go b/cli/src/alluxio.org/cli/cmd/job/cmd_status.go deleted file mode 100644 index 5a7001752e25..000000000000 --- a/cli/src/alluxio.org/cli/cmd/job/cmd_status.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package job - -import ( - "strconv" - - "github.com/palantir/stacktrace" - "github.com/spf13/cobra" - - "alluxio.org/cli/cmd/names" - "alluxio.org/cli/env" -) - -var CmdStatus = &CmdStatusCommand{ - BaseJavaCommand: &env.BaseJavaCommand{ - CommandName: "cmdStatus", - JavaClassName: names.JobShellJavaClass, - }, -} - -type CmdStatusCommand struct { - *env.BaseJavaCommand - jobControlId int -} - -func (c *CmdStatusCommand) Base() *env.BaseJavaCommand { - return c.BaseJavaCommand -} - -func (c *CmdStatusCommand) ToCommand() *cobra.Command { - cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ - Use: CmdStatus.CommandName, - Short: "Get the status information for a distributed command.", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return c.Run(args) - }, - }) - const id = "id" - cmd.Flags().IntVar(&c.jobControlId, id, 0, "Determine the job control ID to get the status information") - cmd.MarkFlagRequired(id) - return cmd -} - -func (c *CmdStatusCommand) Run(args []string) error { - if c.jobControlId <= 0 { - return stacktrace.NewError("Flag --id should be a positive integer") - } - javaArgs := []string{"getCmdStatus", strconv.Itoa(c.jobControlId)} - return c.Base().Run(javaArgs) -} diff --git a/cli/src/alluxio.org/cli/cmd/job/job.go b/cli/src/alluxio.org/cli/cmd/job/job.go index 97327b36858d..3b168f6118bd 100644 --- a/cli/src/alluxio.org/cli/cmd/job/job.go +++ b/cli/src/alluxio.org/cli/cmd/job/job.go @@ -11,17 +11,66 @@ package job -import "alluxio.org/cli/env" +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) var Service = &env.Service{ Name: "job", Description: "Command line tool for interacting with the job service.", Commands: []env.Command{ - Cancel, - CmdStatus, - JobStatus, - Leader, - List, Load, }, } + +const ( + progress = "progress" + stop = "stop" + submit = "submit" +) + +var operations = []string{ + progress, + stop, + submit, +} + +type BaseJobCommand struct { + *env.BaseJavaCommand + + isProgress bool + isStop bool + isSubmit bool + + progressFormat string + progressVerbose bool +} + +func (c *BaseJobCommand) AttachOperationFlags(cmd *cobra.Command) { + cmd.Flags().BoolVar(&c.isProgress, progress, false, "View progress of submitted job") + cmd.Flags().BoolVar(&c.isStop, stop, false, "Stop running job") + cmd.Flags().BoolVar(&c.isSubmit, submit, false, "Submit job") + cmd.MarkFlagsMutuallyExclusive(operations...) + + cmd.Flags().StringVar(&c.progressFormat, "format", "TEXT", "[progress] Format of output, either TEXT or JSON") + cmd.Flags().BoolVar(&c.progressVerbose, "verbose", false, "[progress] Verbose output") +} + +func (c *BaseJobCommand) OperationWithArgs() ([]string, error) { + // rely on MarkFlagsMutuallyExclusive to ensure there is at most one boolean switched to true + if c.isProgress { + ret := []string{"--" + progress, "--format", c.progressFormat} + if c.progressVerbose { + ret = append(ret, "--verbose") + } + return ret, nil + } else if c.isStop { + return []string{"--" + stop}, nil + } else if c.isSubmit { + return []string{"--" + submit}, nil + } + return nil, stacktrace.NewError("Did not specify an operation flag: %v", operations) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/job_status.go b/cli/src/alluxio.org/cli/cmd/job/job_status.go deleted file mode 100644 index d9d2b94d967e..000000000000 --- a/cli/src/alluxio.org/cli/cmd/job/job_status.go +++ /dev/null @@ -1,67 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package job - -import ( - "strconv" - - "github.com/palantir/stacktrace" - "github.com/spf13/cobra" - - "alluxio.org/cli/cmd/names" - "alluxio.org/cli/env" -) - -var JobStatus = &JobStatusCommand{ - BaseJavaCommand: &env.BaseJavaCommand{ - CommandName: "jobStatus", - JavaClassName: names.JobShellJavaClass, - }, -} - -type JobStatusCommand struct { - *env.BaseJavaCommand - jobId int - everyTask bool -} - -func (c *JobStatusCommand) Base() *env.BaseJavaCommand { - return c.BaseJavaCommand -} - -func (c *JobStatusCommand) ToCommand() *cobra.Command { - cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ - Use: JobStatus.CommandName, - Short: "Displays the status info for the specific job.", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return c.Run(args) - }, - }) - const id = "id" - cmd.Flags().IntVar(&c.jobId, id, 0, "Determine the job ID to get status info") - cmd.MarkFlagRequired(id) - cmd.Flags().BoolVarP(&c.everyTask, "every-task", "v", false, "Determine display the status of every task") - return cmd -} - -func (c *JobStatusCommand) Run(args []string) error { - if c.jobId <= 0 { - return stacktrace.NewError("Flag --id should be a positive integer") - } - javaArgs := []string{"stat"} - if c.everyTask { - javaArgs = append(javaArgs, "-v") - } - javaArgs = append(javaArgs, strconv.Itoa(c.jobId)) - return c.Base().Run(javaArgs) -} diff --git a/cli/src/alluxio.org/cli/cmd/job/leader.go b/cli/src/alluxio.org/cli/cmd/job/leader.go deleted file mode 100644 index e095bbb7292f..000000000000 --- a/cli/src/alluxio.org/cli/cmd/job/leader.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package job - -import ( - "github.com/spf13/cobra" - - "alluxio.org/cli/cmd/names" - "alluxio.org/cli/env" -) - -var Leader = &LeaderCommand{ - BaseJavaCommand: &env.BaseJavaCommand{ - CommandName: "leader", - JavaClassName: names.JobShellJavaClass, - }, -} - -type LeaderCommand struct { - *env.BaseJavaCommand -} - -func (c *LeaderCommand) Base() *env.BaseJavaCommand { - return c.BaseJavaCommand -} - -func (c *LeaderCommand) ToCommand() *cobra.Command { - cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ - Use: Leader.CommandName, - Short: "Prints the hostname of the job master service leader.", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return c.Run(args) - }, - }) - return cmd -} - -func (c *LeaderCommand) Run(args []string) error { - javaArgs := []string{"leader"} - return c.Base().Run(javaArgs) -} diff --git a/cli/src/alluxio.org/cli/cmd/job/list.go b/cli/src/alluxio.org/cli/cmd/job/list.go deleted file mode 100644 index 57dc1f070180..000000000000 --- a/cli/src/alluxio.org/cli/cmd/job/list.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package job - -import ( - "github.com/spf13/cobra" - - "alluxio.org/cli/cmd/names" - "alluxio.org/cli/env" -) - -var List = &ListCommand{ - BaseJavaCommand: &env.BaseJavaCommand{ - CommandName: "list", - JavaClassName: names.JobShellJavaClass, - }, -} - -type ListCommand struct { - *env.BaseJavaCommand -} - -func (c *ListCommand) Base() *env.BaseJavaCommand { - return c.BaseJavaCommand -} - -func (c *ListCommand) ToCommand() *cobra.Command { - cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ - Use: List.CommandName, - Short: "Prints the IDs of the most recent jobs, running and finished, " + - "in the history up to the capacity set in alluxio.job.master.job.capacity", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return c.Run(args) - }, - }) - return cmd -} - -func (c *ListCommand) Run(args []string) error { - javaArgs := []string{"ls"} - return c.Base().Run(javaArgs) -} diff --git a/cli/src/alluxio.org/cli/cmd/job/load.go b/cli/src/alluxio.org/cli/cmd/job/load.go index 88734c91cb97..d13a430cf04e 100644 --- a/cli/src/alluxio.org/cli/cmd/job/load.go +++ b/cli/src/alluxio.org/cli/cmd/job/load.go @@ -12,6 +12,7 @@ package job import ( + "github.com/palantir/stacktrace" "github.com/spf13/cobra" "alluxio.org/cli/cmd/names" @@ -19,15 +20,22 @@ import ( ) var Load = &LoadCommand{ - BaseJavaCommand: &env.BaseJavaCommand{ - CommandName: "load", - JavaClassName: names.FileSystemShellJavaClass, + BaseJobCommand: &BaseJobCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "load", + JavaClassName: names.FileSystemShellJavaClass, + }, }, } type LoadCommand struct { - *env.BaseJavaCommand + *BaseJobCommand path string + + bandwidth string + verify bool + partialListing bool + metadataOnly bool } func (c *LoadCommand) Base() *env.BaseJavaCommand { @@ -37,19 +45,39 @@ func (c *LoadCommand) Base() *env.BaseJavaCommand { func (c *LoadCommand) ToCommand() *cobra.Command { cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ Use: Load.CommandName, - Short: "Submit load job to Alluxio master, update job options if already exists", + Short: "Submit or manage load jobs", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return c.Run(args) }, }) const path = "path" - cmd.Flags().StringVar(&c.path, path, "", "Determine the path of the load job to submit") + cmd.Flags().StringVar(&c.path, path, "", "[all] Source path of load operation") cmd.MarkFlagRequired(path) + c.AttachOperationFlags(cmd) + + cmd.Flags().StringVar(&c.bandwidth, "bandwidth", "", "[submit] Single worker read bandwidth limit") + cmd.Flags().BoolVar(&c.verify, "verify", false, "[submit] Run verification when load finishes and load new files if any") + cmd.Flags().BoolVar(&c.partialListing, "partial-listing", false, "[submit] Use partial directory listing, initializing load before reading the entire directory but cannot report on certain progress details") + cmd.Flags().BoolVar(&c.metadataOnly, "metadata-only", false, "[submit] Only load file metadata") return cmd } -func (c *LoadCommand) Run(args []string) error { - javaArgs := []string{"load", c.path, "--submit"} +func (c *LoadCommand) Run(_ []string) error { + opWithArgs, err := c.OperationWithArgs() + if err != nil { + return stacktrace.Propagate(err, "error parsing operation") + } + javaArgs := []string{"load", c.path} + javaArgs = append(javaArgs, opWithArgs...) + if c.bandwidth != "" { + javaArgs = append(javaArgs, "--bandwidth", c.bandwidth) + } + if c.partialListing { + javaArgs = append(javaArgs, "--partial-listing") + } + if c.metadataOnly { + javaArgs = append(javaArgs, "--metadata-only") + } return c.Base().Run(javaArgs) } diff --git a/cli/src/alluxio.org/cli/cmd/names/names.go b/cli/src/alluxio.org/cli/cmd/names/names.go index 51fff704dade..4fbd59de6d38 100644 --- a/cli/src/alluxio.org/cli/cmd/names/names.go +++ b/cli/src/alluxio.org/cli/cmd/names/names.go @@ -15,5 +15,4 @@ const ( BinAlluxio = "bin/alluxio" FileSystemAdminShellJavaClass = "alluxio.cli.fsadmin.FileSystemAdminShell" FileSystemShellJavaClass = "alluxio.cli.fs.FileSystemShell" - JobShellJavaClass = "alluxio.cli.job.JobShell" ) diff --git a/dora/shell/src/main/java/alluxio/cli/fs/command/LoadCommand.java b/dora/shell/src/main/java/alluxio/cli/fs/command/LoadCommand.java index 544fc3a09194..0422af1ffc80 100644 --- a/dora/shell/src/main/java/alluxio/cli/fs/command/LoadCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fs/command/LoadCommand.java @@ -15,7 +15,6 @@ import alluxio.annotation.PublicApi; import alluxio.cli.CommandUtils; import alluxio.client.file.FileSystemContext; -import alluxio.conf.PropertyKey; import alluxio.exception.AlluxioException; import alluxio.exception.status.InvalidArgumentException; import alluxio.grpc.JobProgressReportFormat; @@ -43,13 +42,6 @@ public final class LoadCommand extends AbstractFileSystemCommand { private static final JobProgressReportFormat DEFAULT_FORMAT = JobProgressReportFormat.TEXT; private static final String JOB_TYPE = "load"; - private static final Option LOCAL_OPTION = - Option.builder() - .longOpt("local") - .required(false) - .hasArg(false) - .desc("load the file to local worker.") - .build(); private static final Option SUBMIT_OPTION = Option.builder() .longOpt("submit") .required(false) @@ -142,21 +134,17 @@ public Options getOptions() { .addOption(PROGRESS_OPTION) .addOption(PROGRESS_FORMAT) .addOption(PROGRESS_VERBOSE) - .addOption(LOCAL_OPTION) .addOption(LOAD_METADATA_ONLY); } @Override public int run(CommandLine cl) throws AlluxioException, IOException { - System.out.println("The load command is deprecated under the new DORA architecture. " - + "Please only use it when the cluster has " + PropertyKey.DORA_ENABLED + "=false"); String[] args = cl.getArgs(); AlluxioURI path = new AlluxioURI(args[0]); if (path.containsWildcard()) { throw new UnsupportedOperationException("Load does not support wildcard path"); } - throwIfOldFormat(cl); if (cl.hasOption(SUBMIT_OPTION.getLongOpt())) { OptionalLong bandwidth = OptionalLong.empty(); @@ -188,8 +176,7 @@ public int run(CommandLine cl) throws AlluxioException, IOException { @Override public String getUsage() { - return "For backward compatibility: load [--local] \n" - + "For distributed load:\n" + return "For distributed load:\n" + "\tload --submit " + "[--bandwidth N] [--verify] [--partial-listing] [--metadata-only]\n" + "\tload --stop\n" @@ -204,7 +191,6 @@ public String getDescription() { @Override public void validateArgs(CommandLine cl) throws InvalidArgumentException { CommandUtils.checkNumOfArgsNoLessThan(this, cl, 1); - throwIfOldFormat(cl); int commands = 0; if (cl.hasOption(SUBMIT_OPTION.getLongOpt())) { commands++; @@ -283,13 +269,6 @@ private int getProgress(AlluxioURI path, JobProgressReportFormat format, } } - private void throwIfOldFormat(CommandLine cl) { - if (cl.getOptions().length == 0 - || (cl.getOptions().length == 1 && cl.hasOption(LOCAL_OPTION.getLongOpt()))) { - throw new IllegalArgumentException("load command no longer supports the old format"); - } - } - @Override protected void runPlainPath(AlluxioURI plainPath, CommandLine cl) throws AlluxioException, IOException {