From 467043dff26eea227bbec7e5e08d2923051bedda Mon Sep 17 00:00:00 2001 From: chunxiaozheng <55471457+chunxiaozheng@users.noreply.github.com> Date: Mon, 21 Aug 2023 12:36:17 +0800 Subject: [PATCH] RATIS-1860. Add ratis-shell cmd to generate a new raft-meta.conf. (#901) * Add ratis-shell cmd to generate a new raft-meta.conf. * Remove index option. * Style fix. * Use GroupCommand as the parent of RaftMetaConfCommand. * Modified by review comments. * Update cli.md * Use LocalCommand as the parent of RaftMetaConfCommand. * ReConstruct some abstract command. * Modified by review comments. * Use println instead of printf. * Checkstyle error fix(delete some unuse import). --------- Co-authored-by: idellzheng --- ratis-docs/src/site/markdown/cli.md | 12 ++ .../shell/cli/sh/command/AbstractCommand.java | 59 ++++++++ .../cli/sh/command/AbstractParentCommand.java | 5 +- .../cli/sh/command/AbstractRatisCommand.java | 33 +---- .../shell/cli/sh/command/ElectionCommand.java | 3 +- .../shell/cli/sh/command/GroupCommand.java | 3 +- .../shell/cli/sh/command/LocalCommand.java | 60 ++++++++ .../shell/cli/sh/command/PeerCommand.java | 3 +- .../shell/cli/sh/command/SnapshotCommand.java | 3 +- .../cli/sh/local/RaftMetaConfCommand.java | 132 ++++++++++++++++++ 10 files changed, 276 insertions(+), 37 deletions(-) create mode 100644 ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractCommand.java create mode 100644 ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/LocalCommand.java create mode 100644 ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/local/RaftMetaConfCommand.java diff --git a/ratis-docs/src/site/markdown/cli.md b/ratis-docs/src/site/markdown/cli.md index 75be20c12b..60958fc7ed 100644 --- a/ratis-docs/src/site/markdown/cli.md +++ b/ratis-docs/src/site/markdown/cli.md @@ -80,6 +80,7 @@ Usage: ratis sh [generic options] [group [info] [list]] [peer [add] [remove] [setPriority]] [snapshot [create]] + [local [raftMetaConf]] ``` ## generic options @@ -172,3 +173,14 @@ Trigger the specified server take snapshot. ``` $ ratis sh snapshot create -peers -peerId [-groupid ] ``` + +## local +The `local` command is used to process local operation, which no need to connect to ratis server. +It has the following subcommands: +`raftMetaConf` + +### local raftMetaConf +Generate a new raft-meta.conf file based on original raft-meta.conf and new peers, which is used to move a raft node to a new node. +``` +$ ratis sh local raftMetaConf -peers -path +``` diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractCommand.java new file mode 100644 index 0000000000..20a52a80f8 --- /dev/null +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractCommand.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.shell.cli.sh.command; + +import org.apache.ratis.shell.cli.Command; + +import java.io.PrintStream; +import java.net.InetSocketAddress; + +/** + * The base class for all the ratis shell {@link Command} classes. + */ +public abstract class AbstractCommand implements Command { + + private final PrintStream printStream; + + protected AbstractCommand(Context context) { + printStream = context.getPrintStream(); + } + + public static InetSocketAddress parseInetSocketAddress(String address) { + try { + final String[] hostPortPair = address.split(":"); + if (hostPortPair.length < 2) { + throw new IllegalArgumentException("Unexpected address format ."); + } + return new InetSocketAddress(hostPortPair[0], Integer.parseInt(hostPortPair[1])); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse the server address parameter \"" + address + "\".", e); + } + } + + protected PrintStream getPrintStream() { + return printStream; + } + + protected void printf(String format, Object... args) { + printStream.printf(format, args); + } + + protected void println(Object message) { + printStream.println(message); + } +} diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractParentCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractParentCommand.java index 01428dfedf..ec8401f4f7 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractParentCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractParentCommand.java @@ -26,11 +26,10 @@ import java.util.function.Function; import java.util.stream.Collectors; -public abstract class AbstractParentCommand extends AbstractRatisCommand{ +public abstract class AbstractParentCommand implements Command { private final Map subs; - public AbstractParentCommand(Context context, List> subCommandConstructors) { - super(context); + public AbstractParentCommand(Context context, List> subCommandConstructors) { this.subs = Collections.unmodifiableMap(subCommandConstructors.stream() .map(constructor -> constructor.apply(context)) .collect(Collectors.toMap(Command::getCommandName, Function.identity(), diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractRatisCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractRatisCommand.java index 74fcbae3dd..1888c0e0ea 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractRatisCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractRatisCommand.java @@ -20,7 +20,6 @@ import org.apache.commons.cli.Option; import org.apache.ratis.protocol.*; import org.apache.ratis.protocol.exceptions.RaftException; -import org.apache.ratis.shell.cli.Command; import org.apache.ratis.shell.cli.RaftUtils; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; @@ -34,7 +33,6 @@ import org.apache.ratis.util.function.CheckedFunction; import java.io.IOException; -import java.io.PrintStream; import java.net.InetSocketAddress; import java.util.*; import java.util.function.BiConsumer; @@ -43,25 +41,13 @@ import java.util.stream.Stream; /** - * The base class for all the ratis shell {@link Command} classes. + * The base class for the ratis shell which need to connect to server. */ -public abstract class AbstractRatisCommand implements Command { +public abstract class AbstractRatisCommand extends AbstractCommand { public static final String PEER_OPTION_NAME = "peers"; public static final String GROUPID_OPTION_NAME = "groupid"; public static final RaftGroupId DEFAULT_RAFT_GROUP_ID = RaftGroupId.randomId(); - public static InetSocketAddress parseInetSocketAddress(String address) { - try { - final String[] hostPortPair = address.split(":"); - if (hostPortPair.length < 2) { - throw new IllegalArgumentException("Unexpected address format ."); - } - return new InetSocketAddress(hostPortPair[0], Integer.parseInt(hostPortPair[1])); - } catch (Exception e) { - throw new IllegalArgumentException("Failed to parse the server address parameter \"" + address + "\".", e); - } - } - /** * Execute a given function with input parameter from the members of a list. * @@ -86,12 +72,11 @@ public static K run(Collection list, CheckedFunct return null; } - private final PrintStream printStream; private RaftGroup raftGroup; private GroupInfoReply groupInfoReply; protected AbstractRatisCommand(Context context) { - printStream = context.getPrintStream(); + super(context); } @Override @@ -154,18 +139,6 @@ public Options getOptions() { .addOption(GROUPID_OPTION_NAME, true, "Raft group id"); } - protected PrintStream getPrintStream() { - return printStream; - } - - protected void printf(String format, Object... args) { - printStream.printf(format, args); - } - - protected void println(Object message) { - printStream.println(message); - } - protected RaftGroup getRaftGroup() { return raftGroup; } diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/ElectionCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/ElectionCommand.java index 43d68467f0..054f8c6bb3 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/ElectionCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/ElectionCommand.java @@ -17,6 +17,7 @@ */ package org.apache.ratis.shell.cli.sh.command; +import org.apache.ratis.shell.cli.Command; import org.apache.ratis.shell.cli.sh.election.PauseCommand; import org.apache.ratis.shell.cli.sh.election.ResumeCommand; import org.apache.ratis.shell.cli.sh.election.StepDownCommand; @@ -28,7 +29,7 @@ import java.util.function.Function; public class ElectionCommand extends AbstractParentCommand { - private static final List> SUB_COMMAND_CONSTRUCTORS + private static final List> SUB_COMMAND_CONSTRUCTORS = Collections.unmodifiableList(Arrays.asList( TransferCommand::new, StepDownCommand::new, PauseCommand::new, ResumeCommand::new)); diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/GroupCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/GroupCommand.java index e1d7ec058f..69953a9824 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/GroupCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/GroupCommand.java @@ -17,6 +17,7 @@ */ package org.apache.ratis.shell.cli.sh.command; +import org.apache.ratis.shell.cli.Command; import org.apache.ratis.shell.cli.sh.group.GroupInfoCommand; import org.apache.ratis.shell.cli.sh.group.GroupListCommand; @@ -30,7 +31,7 @@ */ public class GroupCommand extends AbstractParentCommand { - private static final List> SUB_COMMAND_CONSTRUCTORS + private static final List> SUB_COMMAND_CONSTRUCTORS = Collections.unmodifiableList(Arrays.asList( GroupInfoCommand::new, GroupListCommand::new)); /** diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/LocalCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/LocalCommand.java new file mode 100644 index 0000000000..4a22b27631 --- /dev/null +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/LocalCommand.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.shell.cli.sh.command; + +import org.apache.ratis.shell.cli.Command; +import org.apache.ratis.shell.cli.sh.local.RaftMetaConfCommand; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +/** + * Command for local operation, which no need to connect to ratis server + */ +public class LocalCommand extends AbstractParentCommand { + + private static final List> SUB_COMMAND_CONSTRUCTORS + = Collections.unmodifiableList(Arrays.asList(RaftMetaConfCommand::new)); + + /** + * @param context command context + */ + public LocalCommand(Context context) { + super(context, SUB_COMMAND_CONSTRUCTORS); + } + + @Override + public String getCommandName() { + return "local"; + } + + @Override + public String getDescription() { + return description(); + } + + /** + * @return command's description + */ + public static String description() { + return "Command for local operation, which no need to connect to ratis server; " + + "see the sub-commands for the details."; + } +} diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/PeerCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/PeerCommand.java index 6cb2796547..2394a568fe 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/PeerCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/PeerCommand.java @@ -17,6 +17,7 @@ */ package org.apache.ratis.shell.cli.sh.command; +import org.apache.ratis.shell.cli.Command; import org.apache.ratis.shell.cli.sh.peer.AddCommand; import org.apache.ratis.shell.cli.sh.peer.RemoveCommand; import org.apache.ratis.shell.cli.sh.peer.SetPriorityCommand; @@ -31,7 +32,7 @@ */ public class PeerCommand extends AbstractParentCommand{ - private static final List> SUB_COMMAND_CONSTRUCTORS + private static final List> SUB_COMMAND_CONSTRUCTORS = Collections.unmodifiableList(Arrays.asList(AddCommand::new, RemoveCommand::new, SetPriorityCommand::new)); diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/SnapshotCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/SnapshotCommand.java index 4dd5842591..34f8786ad6 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/SnapshotCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/SnapshotCommand.java @@ -17,6 +17,7 @@ */ package org.apache.ratis.shell.cli.sh.command; +import org.apache.ratis.shell.cli.Command; import org.apache.ratis.shell.cli.sh.snapshot.TakeSnapshotCommand; import java.util.Arrays; @@ -28,7 +29,7 @@ * Command for the ratis snapshot */ public class SnapshotCommand extends AbstractParentCommand { - private static final List> SUB_COMMAND_CONSTRUCTORS + private static final List> SUB_COMMAND_CONSTRUCTORS = Collections.unmodifiableList(Arrays.asList(TakeSnapshotCommand::new)); /** diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/local/RaftMetaConfCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/local/RaftMetaConfCommand.java new file mode 100644 index 0000000000..231c643ac3 --- /dev/null +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/local/RaftMetaConfCommand.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.shell.cli.sh.local; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; +import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto; +import org.apache.ratis.proto.RaftProtos.RaftPeerProto; +import org.apache.ratis.proto.RaftProtos.RaftPeerRole; +import org.apache.ratis.shell.cli.RaftUtils; +import org.apache.ratis.shell.cli.sh.command.AbstractCommand; +import org.apache.ratis.shell.cli.sh.command.Context; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +/** + * Command for generate a new raft-meta.conf file based on original raft-meta.conf and new peers, + * which is used to move a raft node to a new node. + */ +public class RaftMetaConfCommand extends AbstractCommand { + public static final String PEER_OPTION_NAME = "peers"; + public static final String PATH_OPTION_NAME = "path"; + + private static final String RAFT_META_CONF = "raft-meta.conf"; + private static final String NEW_RAFT_META_CONF = "new-raft-meta.conf"; + + /** + * @param context command context + */ + public RaftMetaConfCommand(Context context) { + super(context); + } + + @Override + public String getCommandName() { + return "raftMetaConf"; + } + + @Override + public int run(CommandLine cl) throws IOException { + String peersStr = cl.getOptionValue(PEER_OPTION_NAME); + String path = cl.getOptionValue(PATH_OPTION_NAME); + if (peersStr == null || path == null || peersStr.isEmpty() || path.isEmpty()) { + printf("peers or path can't be empty."); + return -1; + } + List raftPeerProtos = new ArrayList<>(); + for (String address : peersStr.split(",")) { + String peerId = RaftUtils.getPeerId(parseInetSocketAddress(address)).toString(); + raftPeerProtos.add(RaftPeerProto.newBuilder() + .setId(ByteString.copyFrom(peerId.getBytes(StandardCharsets.UTF_8))).setAddress(address) + .setStartupRole(RaftPeerRole.FOLLOWER).build()); + } + try (InputStream in = Files.newInputStream(Paths.get(path, RAFT_META_CONF)); + OutputStream out = Files.newOutputStream(Paths.get(path, NEW_RAFT_META_CONF))) { + long index = LogEntryProto.newBuilder().mergeFrom(in).build().getIndex(); + println("Index in the original file is: " + index); + LogEntryProto generateLogEntryProto = LogEntryProto.newBuilder() + .setConfigurationEntry(RaftConfigurationProto.newBuilder() + .addAllPeers(raftPeerProtos).build()) + .setIndex(index + 1).build(); + printf("Generate new LogEntryProto info is:\n" + generateLogEntryProto); + generateLogEntryProto.writeTo(out); + } + return 0; + } + + @Override + public String getUsage() { + return String.format("%s" + + " -%s " + + " -%s ", + getCommandName(), PEER_OPTION_NAME, PATH_OPTION_NAME); + } + + @Override + public String getDescription() { + return description(); + } + + @Override + public Options getOptions() { + return new Options() + .addOption( + Option.builder() + .option(PEER_OPTION_NAME) + .hasArg() + .required() + .desc("Peer addresses seperated by comma") + .build()) + .addOption( + Option.builder() + .option(PATH_OPTION_NAME) + .hasArg() + .required() + .desc("The parent path of raft-meta.conf") + .build()); + } + + /** + * @return command's description + */ + public static String description() { + return "Generate a new raft-meta.conf file based on original raft-meta.conf and new peers."; + } +} +