From bda3407de976f378b94b89dcc668d7ae263cf71b Mon Sep 17 00:00:00 2001 From: Bowen Ding <6999708+dbw9580@users.noreply.github.com> Date: Thu, 17 Aug 2023 22:39:04 +0800 Subject: [PATCH] squash --- .../client/file/DoraCacheFileSystem.java | 37 ++++++++++++++++++- .../file/options/FileSystemOptions.java | 17 +++++++++ .../file/options/UfsFileSystemOptions.java | 8 ++++ .../client/file/ListStatusPartialResult.java | 13 +++++-- libexec/alluxio-config.sh | 2 +- 5 files changed, 71 insertions(+), 6 deletions(-) diff --git a/dora/core/client/fs/src/main/java/alluxio/client/file/DoraCacheFileSystem.java b/dora/core/client/fs/src/main/java/alluxio/client/file/DoraCacheFileSystem.java index 2712310022c3..014c31970933 100644 --- a/dora/core/client/fs/src/main/java/alluxio/client/file/DoraCacheFileSystem.java +++ b/dora/core/client/fs/src/main/java/alluxio/client/file/DoraCacheFileSystem.java @@ -35,6 +35,7 @@ import alluxio.grpc.DeletePOptions; import alluxio.grpc.ExistsPOptions; import alluxio.grpc.GetStatusPOptions; +import alluxio.grpc.GrpcUtils; import alluxio.grpc.ListStatusPOptions; import alluxio.grpc.OpenFilePOptions; import alluxio.grpc.RenamePOptions; @@ -48,6 +49,7 @@ import alluxio.wire.BlockLocation; import alluxio.wire.BlockLocationInfo; import alluxio.wire.FileBlockInfo; +import alluxio.wire.FileInfo; import alluxio.wire.WorkerNetAddress; import com.codahale.metrics.Counter; @@ -61,6 +63,7 @@ import java.util.Collections; import java.util.List; import java.util.function.Consumer; +import java.util.stream.Collectors; /** * Dora Cache file system implementation. @@ -139,7 +142,11 @@ public URIStatus getStatus(AlluxioURI path, GetStatusPOptions options) GetStatusPOptions mergedOptions = FileSystemOptionsUtils.getStatusDefaults( mFsContext.getClusterConf()).toBuilder().mergeFrom(options).build(); - return mDoraClient.getStatus(ufsFullPath.toString(), mergedOptions); + final URIStatus status = mDoraClient.getStatus(ufsFullPath.toString(), mergedOptions); + FileInfo info = GrpcUtils.fromProto(GrpcUtils.toProto(status.getFileInfo())); + info.setPath(convertUfsPathToAlluxioPath(new AlluxioURI(info.getUfsPath())).getPath()); + URIStatus statusWithRelativeAlluxioPath = new URIStatus(info, status.getCacheContext()); + return statusWithRelativeAlluxioPath; } catch (RuntimeException ex) { if (ex instanceof StatusRuntimeException) { if (((StatusRuntimeException) ex).getStatus().getCode() == Status.NOT_FOUND.getCode()) { @@ -241,7 +248,15 @@ public List listStatus(AlluxioURI path, ListStatusPOptions options) ListStatusPOptions mergedOptions = FileSystemOptionsUtils.listStatusDefaults( mFsContext.getClusterConf()).toBuilder().mergeFrom(options).build(); - return mDoraClient.listStatus(ufsFullPath.toString(), mergedOptions); + final List uriStatuses = mDoraClient.listStatus(ufsFullPath.toString(), + mergedOptions); + List statusesWithRelativePath = uriStatuses.stream() + .map(status -> new URIStatus( + GrpcUtils.fromProto(GrpcUtils.toProto(status.getFileInfo())) + .setPath(convertUfsPathToAlluxioPath(new AlluxioURI(status.getUfsPath())) + .getPath()))) + .collect(Collectors.toList()); + return statusesWithRelativePath; } catch (RuntimeException ex) { if (ex instanceof StatusRuntimeException) { if (((StatusRuntimeException) ex).getStatus().getCode() == Status.NOT_FOUND.getCode()) { @@ -453,6 +468,24 @@ public AlluxioURI convertAlluxioPathToUFSPath(AlluxioURI alluxioPath) { } } + private AlluxioURI convertUfsPathToAlluxioPath(AlluxioURI ufsPath) { + if (mDelegatedFileSystem instanceof UfsBaseFileSystem) { + AlluxioURI rootUfs = ((UfsBaseFileSystem) mDelegatedFileSystem).getRootUFS(); + try { + if (rootUfs.isAncestorOf(ufsPath)) { + return new AlluxioURI(PathUtils.concatPath(AlluxioURI.SEPARATOR, + PathUtils.subtractPaths(ufsPath.getPath(), rootUfs.getPath()))); + } + } catch (InvalidPathException e) { + throw new RuntimeException(e); + } + + return ufsPath; + } else { + return ufsPath; + } + } + @Override public List getBlockLocations(AlluxioURI path) throws IOException, AlluxioException { diff --git a/dora/core/client/fs/src/main/java/alluxio/client/file/options/FileSystemOptions.java b/dora/core/client/fs/src/main/java/alluxio/client/file/options/FileSystemOptions.java index ba85b0e6c1c6..8a286d36f5da 100644 --- a/dora/core/client/fs/src/main/java/alluxio/client/file/options/FileSystemOptions.java +++ b/dora/core/client/fs/src/main/java/alluxio/client/file/options/FileSystemOptions.java @@ -59,6 +59,23 @@ public static Builder fromConf(AlluxioConfiguration conf) { return builder; } + /** + * Copies options from an existing {@link FileSystemOptions} object. + * + * @param source source options to copy from + * @return builder + */ + public static Builder copyFrom(FileSystemOptions source) { + Builder builder = new Builder(); + builder.setDataCacheEnabled(source.isDataCacheEnabled()) + .setDoraCacheEnabled(source.isDoraCacheEnabled()) + .setMetadataCacheEnabled(source.isMetadataCacheEnabled()) + .setUfsFallbackEnabled(source.isUfsFallbackEnabled()); + source.getUfsFileSystemOptions() + .ifPresent(builder::setUfsFileSystemOptions); + return builder; + } + /** * @return whether client metadata cache is enabled */ diff --git a/dora/core/client/fs/src/main/java/alluxio/client/file/options/UfsFileSystemOptions.java b/dora/core/client/fs/src/main/java/alluxio/client/file/options/UfsFileSystemOptions.java index 7584c544c1ea..18610419c4b2 100644 --- a/dora/core/client/fs/src/main/java/alluxio/client/file/options/UfsFileSystemOptions.java +++ b/dora/core/client/fs/src/main/java/alluxio/client/file/options/UfsFileSystemOptions.java @@ -11,6 +11,7 @@ package alluxio.client.file.options; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; /** @@ -36,4 +37,11 @@ public UfsFileSystemOptions(String ufsAddress) { public String getUfsAddress() { return mUfsAddress; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("mUfsAddress", mUfsAddress) + .toString(); + } } diff --git a/dora/core/common/src/main/java/alluxio/client/file/ListStatusPartialResult.java b/dora/core/common/src/main/java/alluxio/client/file/ListStatusPartialResult.java index 1d23a9225099..db77ad117186 100644 --- a/dora/core/common/src/main/java/alluxio/client/file/ListStatusPartialResult.java +++ b/dora/core/common/src/main/java/alluxio/client/file/ListStatusPartialResult.java @@ -42,7 +42,14 @@ public static ListStatusPartialResult fromProto(ListStatusPartialPResponse respo private final boolean mTruncated; private final long mFileCount; - private ListStatusPartialResult(List listings, boolean isTruncated, long fileCount) { + /** + * Constructor. + * + * @param listings + * @param isTruncated + * @param fileCount + */ + public ListStatusPartialResult(List listings, boolean isTruncated, long fileCount) { mListings = listings; mTruncated = isTruncated; mFileCount = fileCount; @@ -64,8 +71,8 @@ public boolean isTruncated() { /** * @return the total number of files in the listed directory, - * (i.e. the size of the result if partial listing was not used - * or -1 if the listing was recursive) + * (i.e. the size of the result if partial listing was not used + * or -1 if the listing was recursive) */ public long getFileCount() { return mFileCount; diff --git a/libexec/alluxio-config.sh b/libexec/alluxio-config.sh index f1f4282ae82d..530fc933ad23 100755 --- a/libexec/alluxio-config.sh +++ b/libexec/alluxio-config.sh @@ -15,7 +15,7 @@ # resolve links - $0 may be a softlink this="${BASH_SOURCE-$0}" -common_bin=$(cd -P -- "$(dirname -- "${this}")" && pwd -P) +common_bin=$(cd -- "$(dirname -- "${this}")" && pwd ) script="$(basename -- "${this}")" this="${common_bin}/${script}"