diff --git a/dora/core/common/src/main/java/alluxio/uri/UfsUrl.java b/dora/core/common/src/main/java/alluxio/uri/UfsUrl.java index 280e235ac8d3..aa9a18de4931 100644 --- a/dora/core/common/src/main/java/alluxio/uri/UfsUrl.java +++ b/dora/core/common/src/main/java/alluxio/uri/UfsUrl.java @@ -12,8 +12,6 @@ package alluxio.uri; import alluxio.AlluxioURI; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; import alluxio.exception.InvalidPathException; import alluxio.grpc.UfsUrlMessage; import alluxio.util.io.PathUtils; @@ -21,10 +19,9 @@ import com.google.common.base.Preconditions; import org.apache.commons.io.FilenameUtils; +import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -36,140 +33,81 @@ public class UfsUrl { public static final String SCHEME_SEPARATOR = "://"; public static final String DOUBLE_SLASH_SEPARATOR = "//"; - public static final String PATH_SEPARATOR = "/"; + public static final String SLASH_SEPARATOR = "/"; public static final String COLON_SEPARATOR = ":"; - final UfsUrlMessage mProto; + private final UfsUrlMessage mProto; - /** - * Constructs a map containing scheme, authority, and path. - * @param inputUrl the input url string - * @return a map containing scheme, authority, and path - */ - public static Map extractElements(String inputUrl) { - Preconditions.checkArgument(inputUrl != null && !inputUrl.isEmpty(), - "The input url is null or empty, please input a valid url."); - Map elements = new HashMap<>(); - String scheme = null; - String authority = null; - String path = null; - - int start = 0; - int schemeSplitIndex = inputUrl.indexOf(SCHEME_SEPARATOR, start); - if (schemeSplitIndex == -1) { - scheme = ""; - } else { - scheme = inputUrl.substring(start, schemeSplitIndex); - start += scheme.length(); - } - Preconditions.checkArgument(!scheme.equalsIgnoreCase("alluxio"), - "Alluxio 3.x no longer supports alluxio:// scheme," - + " please input the UFS path directly like hdfs://host:port/path"); + private static class Parser { + private final String mScheme; + private final String mAuthority; + private final List mPathComponents; - // unified address "://" and "//" - while (start < inputUrl.length() && inputUrl.charAt(start) == COLON_SEPARATOR.charAt(0)) { - start++; - } + private Parser(String inputUrl) { + Preconditions.checkArgument(inputUrl != null && !inputUrl.isEmpty(), + "The input url is null or empty, please input a valid url."); + String scheme = null; + String authority = null; + String path = null; - if (inputUrl.startsWith(DOUBLE_SLASH_SEPARATOR, start) - && start + DOUBLE_SLASH_SEPARATOR.length() < inputUrl.length()) { - start += DOUBLE_SLASH_SEPARATOR.length(); - int authoritySplitIndex = inputUrl.indexOf(PATH_SEPARATOR, start); - if (authoritySplitIndex == -1) { - authority = inputUrl.substring(start); + int start = 0; + int schemeSplitIndex = inputUrl.indexOf(SCHEME_SEPARATOR, start); + if (schemeSplitIndex == -1) { + scheme = ""; } else { - authority = inputUrl.substring(start, authoritySplitIndex); + scheme = inputUrl.substring(start, schemeSplitIndex); + start += scheme.length(); } - } else { - authority = ""; - } - - start += authority.length(); - path = FilenameUtils.normalize(inputUrl.substring(start)); + Preconditions.checkArgument(!scheme.equalsIgnoreCase("alluxio"), + "Alluxio 3.x no longer supports alluxio:// scheme," + + " please input the UFS path directly like hdfs://host:port/path"); - elements.put("scheme", scheme); - elements.put("authority", authority); - elements.put("path", path); - return elements; - } + // unified address "://" and "//" + while (start < inputUrl.length() && inputUrl.charAt(start) == COLON_SEPARATOR.charAt(0)) { + start++; + } - /** - * Determines the scheme of this absolute path object. The returns String is not empty. - * @param rootUrl the elems map of root url, including scheme, authority and path - * @param inputUrl the elems map of input url, including scheme, authority and path - * @return the scheme of this UfsUrl object - */ - public static String generateScheme(Map rootUrl, Map inputUrl) { - if (rootUrl.get("scheme").isEmpty()) { - if (inputUrl.get("scheme").isEmpty()) { - return "file"; + if (inputUrl.startsWith(DOUBLE_SLASH_SEPARATOR, start) + && start + DOUBLE_SLASH_SEPARATOR.length() < inputUrl.length()) { + start += DOUBLE_SLASH_SEPARATOR.length(); + int authoritySplitIndex = inputUrl.indexOf(SLASH_SEPARATOR, start); + if (authoritySplitIndex == -1) { + authority = inputUrl.substring(start); + } else { + authority = inputUrl.substring(start, authoritySplitIndex); + } } else { - return inputUrl.get("scheme"); + authority = ""; } - } else { // rootUrl.scheme is not empty - if (inputUrl.get("scheme").isEmpty()) { - // means input is a relative path. - return rootUrl.get("scheme"); - } else { - return inputUrl.get("scheme"); + + start += authority.length(); + // remove the fronting slash, if any. + while (start < inputUrl.length() && inputUrl.charAt(start) == SLASH_SEPARATOR.charAt(0)) { + start++; } - } - } + path = FilenameUtils.normalizeNoEndSeparator(inputUrl.substring(start)); - /** - * Determines the authority of this absolute path object. - * @param rootUrl the elems map of root url, including scheme, authority and path - * @param inputUrl the elems map of input url, including scheme, authority and path - * @return the authority of this UfsUrl object - */ - public static String generateAuthority(Map rootUrl, - Map inputUrl) { - if (rootUrl.get("authority").isEmpty()) { - return inputUrl.get("authority"); - } else { - if (rootUrl.get("scheme").equals(inputUrl.get("scheme"))) { - if (!inputUrl.get("authority").isEmpty()) { - return inputUrl.get("authority"); - } else { - return rootUrl.get("authority"); - } + // scheme, authority, pathComponents are always not null. + mScheme = scheme; + mAuthority = authority; + + if (path.isEmpty()) { + mPathComponents = new ArrayList<>(); } else { - return inputUrl.get("authority"); + mPathComponents = Arrays.asList(path.split(SLASH_SEPARATOR)); } } - } - /** - * Determines the path components list of this absolute path. - * @param rootUrl the elems map of root url, including scheme, authority and path - * @param inputUrl the elems map of input url, including scheme, authority and path - * @return the path components list of this UfsUrl Object - */ - public static List generatePathComponents(Map rootUrl, - Map inputUrl) { - List inputPathComponents = Arrays.asList(inputUrl.get("path").split(PATH_SEPARATOR)); + public String getScheme() { + return mScheme; + } - if (rootUrl.equals(inputUrl)) { - return inputPathComponents; + public String getAuthority() { + return mAuthority; } - // inputUrl.scheme is "file" -> the inputUrl is probably a relative path. - if (inputUrl.get("scheme").equals("file")) { - // rootUrl.scheme is "file" or empty -> inputUrl is a relative path. - if (inputUrl.get("scheme").equals(rootUrl.get("scheme")) || rootUrl.get("scheme").isEmpty()) { - String concatPath = rootUrl.get("path") + inputUrl.get("path"); - return Arrays.asList(FilenameUtils - .normalizeNoEndSeparator(concatPath).split(PATH_SEPARATOR)); - } else { // inputUrl is an absolute path. - return inputPathComponents; - } - } else { // inputUrl is an absolute path. - // if inputUrl.path is "/" or "//", i.e., root directory, set a "" to represent root. - if (inputUrl.get("path").equals(PATH_SEPARATOR) - || inputUrl.get("path").equals(DOUBLE_SLASH_SEPARATOR)) { - inputPathComponents = Arrays.asList(""); - } - return inputPathComponents; + public List getPathComponents() { + return mPathComponents; } } @@ -192,40 +130,12 @@ public static UfsUrl createInstance(String ufsPath) { Preconditions.checkArgument(ufsPath != null && !ufsPath.isEmpty(), "input path is null or empty"); - String ufsRootDir = Configuration.getString(PropertyKey.DORA_CLIENT_UFS_ROOT); - - Preconditions.checkArgument(ufsRootDir != null && !ufsRootDir.isEmpty(), - "root dir is null or empty."); - - Map rootDirElems = extractElements(ufsRootDir); - Map ufsPathElems = extractElements(ufsPath); - - Preconditions.checkArgument( - rootDirElems.containsKey("scheme") && rootDirElems.get("scheme") != null - && rootDirElems.containsKey("authority") && rootDirElems.get("authority") != null - && rootDirElems.containsKey("path") && rootDirElems.get("path") != null); - - Preconditions.checkArgument( - ufsPathElems.containsKey("scheme") && ufsPathElems.get("scheme") != null - && ufsPathElems.containsKey("authority") && ufsPathElems.get("authority") != null - && ufsPathElems.containsKey("path") && ufsPathElems.get("path") != null); - - String scheme = generateScheme(rootDirElems, ufsPathElems); - Preconditions.checkArgument(!scheme.isEmpty(), - "scheme is empty, please check input Url again."); - ufsPathElems.put("scheme", scheme); - if (scheme.equals("file")) { - rootDirElems.put("scheme", scheme); - } - - String authority = generateAuthority(rootDirElems, ufsPathElems); - ufsPathElems.put("authority", authority); - List pathComponents = generatePathComponents(rootDirElems, ufsPathElems); - + Parser parser = new Parser(ufsPath); + // if not present, the builder will throw exception. return new UfsUrl(UfsUrlMessage.newBuilder() - .setScheme(scheme) - .setAuthority(authority) - .addAllPathComponents(pathComponents) + .setScheme(parser.getScheme()) + .setAuthority(parser.getAuthority()) + .addAllPathComponents(parser.getPathComponents()) .build()); } @@ -243,8 +153,6 @@ public static UfsUrl fromProto(UfsUrlMessage proto) { * @param proto the proto of the UfsUrl */ public UfsUrl(UfsUrlMessage proto) { - Preconditions.checkArgument(proto.getPathComponentsList().size() != 0, - "The proto.path is empty, please check the proto first."); mProto = proto; } @@ -255,14 +163,8 @@ public UfsUrl(UfsUrlMessage proto) { * @param path the path component of the UfsUrl */ public UfsUrl(String scheme, String authority, String path) { - String[] arrayOfPath = path.split(PATH_SEPARATOR); + String[] arrayOfPath = path.split(SLASH_SEPARATOR); List pathComponentsList = Arrays.asList(arrayOfPath); - // handle root dir "/". If equal, pathComponentsList is empty, add "" to represent "/". - if (path.equals(PATH_SEPARATOR)) { - pathComponentsList.add(""); - } - Preconditions.checkArgument(pathComponentsList.size() != 0, - "The path is empty, please input a valid path"); mProto = UfsUrlMessage.newBuilder() .setScheme(scheme) .setAuthority(authority) @@ -384,15 +286,17 @@ public UfsUrl getChildURL(String childName) { */ public String getFullPath() { StringBuilder sb = new StringBuilder(); - sb.append(PATH_SEPARATOR); + sb.append(SLASH_SEPARATOR); + int n = mProto.getPathComponentsList().size(); // Then sb is not empty. - for (int i = 1; i < mProto.getPathComponentsList().size(); i++) { - sb.append(mProto.getPathComponentsList().get(i)); - if (!mProto.getPathComponentsList().get(i).isEmpty()) { - sb.append(PATH_SEPARATOR); - } + for (int i = 0; i < n - 1; i++) { + sb.append(mProto.getPathComponents(i)); + sb.append(SLASH_SEPARATOR); + } + if (n - 1 >= 0) { + sb.append(mProto.getPathComponents(n - 1)); } - return FilenameUtils.normalizeNoEndSeparator(sb.toString()); + return sb.toString(); } /** @@ -411,10 +315,7 @@ public AlluxioURI toAlluxioURI() { * @return the depth */ public int getDepth() { - int pathComponentsSize = getPathComponents().size(); - Preconditions.checkArgument(pathComponentsSize > 0); - // "/" is represented as an empty String "". - return getPathComponents().get(0).isEmpty() ? pathComponentsSize - 1 : pathComponentsSize; + return getPathComponents().size(); } /** @@ -443,8 +344,8 @@ public boolean isAncestorOf(UfsUrl ufsUrl) throws InvalidPathException { } // TODO(Tony Sun): optimize the performance later // Both of the ufsUrls has the same scheme and authority, so just need to compare their paths. - return PathUtils.hasPrefix(PathUtils.normalizePath(ufsUrl.getFullPath(), PATH_SEPARATOR), - PathUtils.normalizePath(getFullPath(), PATH_SEPARATOR)); + return PathUtils.hasPrefix(PathUtils.normalizePath(ufsUrl.getFullPath(), SLASH_SEPARATOR), + PathUtils.normalizePath(getFullPath(), SLASH_SEPARATOR)); } /** @@ -457,7 +358,7 @@ public UfsUrl join(String suffix) { if (suffix == null || suffix.isEmpty()) { return new UfsUrl(mProto); } - String[] suffixArray = suffix.split(PATH_SEPARATOR); + String[] suffixArray = suffix.split(SLASH_SEPARATOR); int nonEmptyIndex = 0; while (nonEmptyIndex < suffixArray.length && suffixArray[nonEmptyIndex].isEmpty()) { nonEmptyIndex++; diff --git a/dora/core/common/src/main/java/alluxio/util/io/PathUtils.java b/dora/core/common/src/main/java/alluxio/util/io/PathUtils.java index ef844b3bc5ba..667a4cffd271 100644 --- a/dora/core/common/src/main/java/alluxio/util/io/PathUtils.java +++ b/dora/core/common/src/main/java/alluxio/util/io/PathUtils.java @@ -11,7 +11,7 @@ package alluxio.util.io; -import static alluxio.uri.UfsUrl.PATH_SEPARATOR; +import static alluxio.uri.UfsUrl.SLASH_SEPARATOR; import alluxio.AlluxioURI; import alluxio.conf.AlluxioConfiguration; @@ -483,10 +483,10 @@ public static List getPossibleMountPoints(String path) throws InvalidPat public static String concatStringPath(String pathA, String pathB) { Preconditions.checkArgument(pathA != null && !pathA.isEmpty()); Preconditions.checkArgument(pathB != null && !pathB.isEmpty()); - if (pathA.endsWith(PATH_SEPARATOR) && pathB.startsWith(PATH_SEPARATOR)) { + if (pathA.endsWith(SLASH_SEPARATOR) && pathB.startsWith(SLASH_SEPARATOR)) { return pathA.substring(0, pathA.length() - 1) + pathB; - } else if (!pathA.endsWith(PATH_SEPARATOR) && !pathB.startsWith(PATH_SEPARATOR)) { - return pathA + PATH_SEPARATOR + pathB; + } else if (!pathA.endsWith(SLASH_SEPARATOR) && !pathB.startsWith(SLASH_SEPARATOR)) { + return pathA + SLASH_SEPARATOR + pathB; } else { return pathA + pathB; } diff --git a/dora/core/common/src/test/java/alluxio/UfsUrlTest.java b/dora/core/common/src/test/java/alluxio/UfsUrlTest.java index ac030d57f870..0674babc11ee 100644 --- a/dora/core/common/src/test/java/alluxio/UfsUrlTest.java +++ b/dora/core/common/src/test/java/alluxio/UfsUrlTest.java @@ -18,8 +18,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; import alluxio.grpc.UfsUrlMessage; import alluxio.uri.SingleMasterAuthority; import alluxio.uri.UfsUrl; @@ -58,6 +56,17 @@ public void basicUfsUrl() { assertEquals("abc://localhost:19998/xy z/a b c", ufsUrl.toString()); } + @Test + public void UfsUrlCtorTest() { + String scheme1 = "abc"; + String authority1 = "127.0.0.1:4567"; + String path1 = "/"; + UfsUrl ufsUrl1 = new UfsUrl(scheme1, authority1, path1); + assertEquals(scheme1, ufsUrl1.getScheme().get()); + assertEquals(authority1, ufsUrl1.getAuthority().get().toString()); + assertEquals(path1, ufsUrl1.getFullPath()); + } + /** * Tests the {@link UfsUrl#createInstance(String)} constructor for basic HDFS paths. */ @@ -87,9 +96,9 @@ public void basicHdfsUri() { public void hashCodeTest() { String scheme = "xyz"; String authority = "192.168.0.1:9527"; - String path = "/a/b/c/d"; - String ufsUrlPath = scheme + "://" + authority + path; - List pathComponents = Arrays.asList(path.split(UfsUrl.PATH_SEPARATOR)); + String path = "a/b/c/d"; + String ufsUrlPath = scheme + "://" + authority + "/" + path; + List pathComponents = Arrays.asList(path.split(UfsUrl.SLASH_SEPARATOR)); UfsUrl ufsUrl1 = UfsUrl.createInstance(ufsUrlPath); UfsUrl ufsUrl2 = UfsUrl.createInstance(UfsUrlMessage.newBuilder() @@ -97,9 +106,9 @@ public void hashCodeTest() { assertEquals(ufsUrl1.toString(), ufsUrl2.toString()); assertEquals(ufsUrl1.hashCode(), ufsUrl2.hashCode()); -// UfsUrl ufsUrl3 = UfsUrl.createInstance(UfsUrl.toProto(ufsUrlPath)); -// assertEquals(ufsUrl2.toString(), ufsUrl3.toString()); -// assertEquals(ufsUrl2.hashCode(), ufsUrl3.hashCode()); + UfsUrl ufsUrl3 = new UfsUrl(scheme, authority, path); + assertEquals(ufsUrl2.toString(), ufsUrl3.toString()); + assertEquals(ufsUrl2.hashCode(), ufsUrl3.hashCode()); } /** @@ -107,9 +116,7 @@ public void hashCodeTest() { */ @Test public void getDepthTests() { - int rootDepth = UfsUrl.createInstance( - Configuration.getString(PropertyKey.DORA_CLIENT_UFS_ROOT)).getDepth(); - assertEquals(rootDepth, UfsUrl.createInstance(".").getDepth()); + assertEquals(0, UfsUrl.createInstance(".").getDepth()); assertEquals(0, UfsUrl.createInstance("abc://localhost:19998/").getDepth()); assertEquals(1, UfsUrl.createInstance("abc://localhost:19998/a").getDepth()); assertEquals(2, UfsUrl.createInstance("abc://localhost:19998/a/b.txt").getDepth()); diff --git a/dora/shell/src/main/java/alluxio/cli/fs/command/LsCommand.java b/dora/shell/src/main/java/alluxio/cli/fs/command/LsCommand.java index 91ab8ecbf61b..69309b0a2a0f 100644 --- a/dora/shell/src/main/java/alluxio/cli/fs/command/LsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fs/command/LsCommand.java @@ -25,6 +25,7 @@ import alluxio.util.CommonUtils; import alluxio.util.FormatUtils; import alluxio.util.SecurityUtils; +import alluxio.util.io.PathUtils; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; @@ -343,6 +344,18 @@ private List sortByFieldAndOrder( return statuses.stream().sorted(sortBy).collect(Collectors.toList()); } + private String handlePath(String rootDir, String inputDir) { + if (inputDir.contains(UfsUrl.SCHEME_SEPARATOR)) { + return inputDir; + } else { + if (rootDir.contains(UfsUrl.SCHEME_SEPARATOR)) { + return PathUtils.concatStringPath(rootDir, inputDir); + } else { + return "file://" + PathUtils.concatStringPath(rootDir, inputDir); + } + } + } + @Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { @@ -366,8 +379,10 @@ protected void runPlainPath(UfsUrl ufsPath, CommandLine cl) @Override public int run(CommandLine cl) throws AlluxioException, IOException { String[] args = cl.getArgs(); + String rootDir = mFsContext.getClusterConf().getString(PropertyKey.DORA_CLIENT_UFS_ROOT); for (String dirArg : args) { - UfsUrl ufsPath = UfsUrl.createInstance(dirArg); + String processedDirArg = handlePath(rootDir, dirArg); + UfsUrl ufsPath = UfsUrl.createInstance(processedDirArg); runWildCardCmd(ufsPath, cl); } return 0;